main.py 656 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352113531135411355113561135711358113591136011361113621136311364113651136611367113681136911370113711137211373113741137511376113771137811379113801138111382113831138411385113861138711388113891139011391113921139311394113951139611397113981139911400114011140211403114041140511406114071140811409114101141111412114131141411415114161141711418114191142011421114221142311424114251142611427114281142911430114311143211433114341143511436114371143811439114401144111442114431144411445114461144711448114491145011451114521145311454114551145611457114581145911460114611146211463114641146511466114671146811469114701147111472114731147411475114761147711478114791148011481114821148311484114851148611487114881148911490114911149211493114941149511496114971149811499115001150111502115031150411505115061150711508115091151011511115121151311514115151151611517115181151911520115211152211523115241152511526115271152811529115301153111532115331153411535115361153711538115391154011541115421154311544115451154611547115481154911550115511155211553115541155511556115571155811559115601156111562115631156411565115661156711568115691157011571115721157311574115751157611577115781157911580115811158211583115841158511586115871158811589115901159111592115931159411595115961159711598115991160011601116021160311604116051160611607116081160911610116111161211613116141161511616116171161811619116201162111622116231162411625116261162711628116291163011631116321163311634116351163611637116381163911640116411164211643116441164511646116471164811649116501165111652116531165411655116561165711658116591166011661116621166311664116651166611667116681166911670116711167211673116741167511676116771167811679116801168111682116831168411685116861168711688116891169011691116921169311694116951169611697116981169911700117011170211703117041170511706117071170811709117101171111712117131171411715117161171711718117191172011721117221172311724117251172611727117281172911730117311173211733117341173511736117371173811739117401174111742117431174411745117461174711748117491175011751117521175311754117551175611757117581175911760117611176211763117641176511766117671176811769117701177111772117731177411775117761177711778117791178011781117821178311784117851178611787117881178911790117911179211793117941179511796117971179811799118001180111802118031180411805118061180711808118091181011811118121181311814118151181611817118181181911820118211182211823118241182511826118271182811829118301183111832118331183411835118361183711838118391184011841118421184311844118451184611847118481184911850118511185211853118541185511856118571185811859118601186111862118631186411865118661186711868118691187011871118721187311874118751187611877118781187911880118811188211883118841188511886118871188811889118901189111892118931189411895118961189711898118991190011901119021190311904119051190611907119081190911910119111191211913119141191511916119171191811919119201192111922119231192411925119261192711928119291193011931119321193311934119351193611937119381193911940119411194211943119441194511946119471194811949119501195111952119531195411955119561195711958119591196011961119621196311964119651196611967119681196911970119711197211973119741197511976119771197811979119801198111982119831198411985119861198711988119891199011991119921199311994119951199611997119981199912000120011200212003120041200512006120071200812009120101201112012120131201412015120161201712018120191202012021120221202312024120251202612027120281202912030120311203212033120341203512036120371203812039120401204112042120431204412045120461204712048120491205012051120521205312054120551205612057120581205912060120611206212063120641206512066120671206812069120701207112072120731207412075120761207712078120791208012081120821208312084120851208612087120881208912090120911209212093120941209512096120971209812099121001210112102121031210412105121061210712108121091211012111121121211312114121151211612117121181211912120121211212212123121241212512126121271212812129121301213112132121331213412135121361213712138121391214012141121421214312144121451214612147121481214912150121511215212153121541215512156121571215812159121601216112162121631216412165121661216712168121691217012171121721217312174121751217612177121781217912180121811218212183121841218512186121871218812189121901219112192121931219412195121961219712198121991220012201122021220312204122051220612207122081220912210122111221212213122141221512216122171221812219122201222112222122231222412225122261222712228122291223012231122321223312234122351223612237122381223912240122411224212243122441224512246122471224812249122501225112252122531225412255122561225712258122591226012261122621226312264122651226612267122681226912270122711227212273122741227512276122771227812279122801228112282122831228412285122861228712288122891229012291122921229312294122951229612297122981229912300123011230212303123041230512306123071230812309123101231112312123131231412315123161231712318123191232012321123221232312324123251232612327123281232912330123311233212333123341233512336123371233812339123401234112342123431234412345123461234712348123491235012351123521235312354123551235612357123581235912360123611236212363123641236512366123671236812369123701237112372123731237412375123761237712378123791238012381123821238312384123851238612387123881238912390123911239212393123941239512396123971239812399124001240112402124031240412405124061240712408124091241012411124121241312414124151241612417124181241912420124211242212423124241242512426124271242812429124301243112432124331243412435124361243712438124391244012441124421244312444124451244612447124481244912450124511245212453124541245512456124571245812459124601246112462124631246412465124661246712468124691247012471124721247312474124751247612477124781247912480124811248212483124841248512486124871248812489124901249112492124931249412495124961249712498124991250012501125021250312504125051250612507125081250912510125111251212513125141251512516125171251812519125201252112522125231252412525125261252712528125291253012531125321253312534125351253612537125381253912540125411254212543125441254512546125471254812549125501255112552125531255412555125561255712558125591256012561125621256312564125651256612567125681256912570125711257212573125741257512576125771257812579125801258112582125831258412585125861258712588125891259012591125921259312594125951259612597125981259912600126011260212603126041260512606126071260812609126101261112612126131261412615126161261712618126191262012621126221262312624126251262612627126281262912630126311263212633126341263512636126371263812639126401264112642126431264412645126461264712648126491265012651126521265312654126551265612657126581265912660126611266212663126641266512666126671266812669126701267112672126731267412675126761267712678126791268012681126821268312684126851268612687126881268912690126911269212693126941269512696126971269812699127001270112702127031270412705127061270712708127091271012711127121271312714127151271612717127181271912720127211272212723127241272512726127271272812729127301273112732127331273412735127361273712738127391274012741127421274312744127451274612747127481274912750127511275212753127541275512756127571275812759127601276112762127631276412765127661276712768127691277012771127721277312774127751277612777127781277912780127811278212783127841278512786127871278812789127901279112792127931279412795127961279712798127991280012801128021280312804128051280612807128081280912810128111281212813128141281512816128171281812819128201282112822128231282412825128261282712828128291283012831128321283312834128351283612837128381283912840128411284212843128441284512846128471284812849128501285112852128531285412855128561285712858128591286012861128621286312864128651286612867128681286912870128711287212873128741287512876128771287812879128801288112882128831288412885128861288712888128891289012891128921289312894128951289612897128981289912900129011290212903129041290512906129071290812909129101291112912129131291412915129161291712918129191292012921129221292312924129251292612927129281292912930129311293212933129341293512936129371293812939129401294112942129431294412945129461294712948129491295012951129521295312954129551295612957129581295912960129611296212963129641296512966129671296812969129701297112972129731297412975129761297712978129791298012981129821298312984129851298612987129881298912990129911299212993129941299512996129971299812999130001300113002130031300413005130061300713008130091301013011130121301313014130151301613017130181301913020130211302213023130241302513026130271302813029130301303113032130331303413035130361303713038130391304013041130421304313044130451304613047130481304913050130511305213053130541305513056130571305813059130601306113062130631306413065130661306713068130691307013071130721307313074130751307613077130781307913080130811308213083130841308513086130871308813089130901309113092130931309413095130961309713098130991310013101131021310313104131051310613107131081310913110131111311213113131141311513116131171311813119131201312113122131231312413125131261312713128131291313013131131321313313134131351313613137131381313913140131411314213143131441314513146131471314813149131501315113152131531315413155131561315713158131591316013161131621316313164131651316613167131681316913170131711317213173131741317513176131771317813179131801318113182131831318413185131861318713188131891319013191131921319313194131951319613197131981319913200132011320213203132041320513206132071320813209132101321113212132131321413215132161321713218132191322013221132221322313224132251322613227132281322913230132311323213233132341323513236132371323813239132401324113242132431324413245132461324713248132491325013251132521325313254132551325613257132581325913260132611326213263132641326513266132671326813269132701327113272132731327413275132761327713278132791328013281132821328313284132851328613287132881328913290132911329213293132941329513296132971329813299133001330113302133031330413305133061330713308133091331013311133121331313314133151331613317133181331913320133211332213323133241332513326133271332813329133301333113332133331333413335133361333713338133391334013341133421334313344133451334613347133481334913350133511335213353133541335513356133571335813359133601336113362133631336413365133661336713368133691337013371133721337313374133751337613377133781337913380133811338213383133841338513386133871338813389133901339113392133931339413395133961339713398133991340013401134021340313404134051340613407134081340913410134111341213413134141341513416134171341813419134201342113422134231342413425134261342713428134291343013431134321343313434134351343613437134381343913440134411344213443134441344513446134471344813449134501345113452134531345413455134561345713458134591346013461134621346313464134651346613467134681346913470134711347213473134741347513476134771347813479134801348113482134831348413485134861348713488134891349013491134921349313494134951349613497134981349913500135011350213503135041350513506135071350813509135101351113512135131351413515135161351713518135191352013521135221352313524135251352613527135281352913530135311353213533135341353513536135371353813539135401354113542135431354413545135461354713548135491355013551135521355313554135551355613557135581355913560135611356213563135641356513566135671356813569135701357113572135731357413575135761357713578135791358013581135821358313584135851358613587135881358913590
  1. import asyncio
  2. import json
  3. import logging
  4. import math
  5. import os
  6. import posixpath
  7. import re
  8. import secrets
  9. import shutil
  10. import time
  11. from contextlib import asynccontextmanager
  12. from datetime import datetime, timedelta, timezone
  13. from logging.handlers import RotatingFileHandler
  14. from pathlib import Path, PurePosixPath
  15. from typing import NamedTuple, cast
  16. from urllib.parse import urlparse
  17. from fastapi import FastAPI
  18. from fastapi.encoders import jsonable_encoder
  19. from fastapi.exceptions import RequestValidationError
  20. from fastapi.responses import FileResponse, JSONResponse
  21. from fastapi.staticfiles import StaticFiles
  22. from sqlalchemy import delete, func, inspect as sa_inspect, or_, select, text, update
  23. from sqlalchemy.engine import CursorResult
  24. from sqlalchemy.exc import IntegrityError
  25. from sqlalchemy.ext.asyncio import AsyncSession
  26. from sqlalchemy.orm.exc import StaleDataError
  27. from backend.app.api.routes import (
  28. ams_history,
  29. announcements,
  30. api_keys,
  31. archive_purge,
  32. archives,
  33. auth,
  34. bug_report,
  35. bulk,
  36. camera,
  37. camwall,
  38. cloud,
  39. connected_apps,
  40. discovery,
  41. external_links,
  42. filaments,
  43. finance,
  44. firmware,
  45. github_backup,
  46. groups,
  47. ha_sensors,
  48. inventory,
  49. kprofiles,
  50. label_templates,
  51. labels,
  52. library,
  53. library_tags,
  54. library_trash,
  55. library_variants,
  56. local_backup,
  57. local_presets,
  58. location_ha_sensors,
  59. maintenance,
  60. makerworld,
  61. manyfold,
  62. metrics,
  63. mfa,
  64. notification_templates,
  65. notifications,
  66. obico,
  67. octoeverywhere,
  68. orca_cloud,
  69. overlay_branding,
  70. pending_uploads,
  71. pipeline_runs,
  72. print_log,
  73. print_queue,
  74. printer_locations,
  75. printer_sensor_history,
  76. printers,
  77. projects,
  78. scheduled_dryings,
  79. settings as settings_routes,
  80. slice_jobs,
  81. slicer_pipelines,
  82. slicer_presets,
  83. smart_plugs,
  84. sponsor_prompt,
  85. spoolbuddy,
  86. spoolman,
  87. spoolman_inventory,
  88. support,
  89. system,
  90. updates,
  91. user_notifications,
  92. users,
  93. virtual_printers,
  94. webhook,
  95. websocket,
  96. )
  97. from backend.app.api.routes.maintenance import (
  98. _get_printer_maintenance_internal,
  99. ensure_default_types,
  100. maintenance_alert_values,
  101. )
  102. from backend.app.api.routes.support import init_debug_logging
  103. from backend.app.core.config import APP_VERSION, settings as app_settings
  104. from backend.app.core.database import async_session, engine, init_db
  105. from backend.app.core.static_assets import AssetStaticFiles
  106. from backend.app.core.tasks import spawn_background_task
  107. from backend.app.core.websocket import ws_manager
  108. from backend.app.services import kprofile_drift, pause_eta, print_dispatch_context, slot_unlink_grace
  109. from backend.app.services.ams_assignment_lock import get_ams_assignment_lock
  110. from backend.app.services.archive import ArchiveService, peek_plate_index_in_3mf, swap_plate_suffix
  111. from backend.app.services.archive_cost_estimate import (
  112. schedule_archive_cost_estimate as _schedule_archive_cost_estimate,
  113. settle_archive_cost_estimate as _settle_archive_cost_estimate,
  114. )
  115. from backend.app.services.archive_purge import archive_purge_service
  116. from backend.app.services.bambu_ftp import (
  117. RECOVERY_DOWNLOAD_MAX_TIMEOUT,
  118. FileNotOnPrinterError,
  119. cache_3mf_download,
  120. clear_3mf_cache,
  121. download_file_async,
  122. download_file_try_paths_async,
  123. ftps_handshake_blocked,
  124. get_cached_3mf,
  125. get_ftp_retry_settings,
  126. normalize_3mf_name,
  127. with_ftp_retry,
  128. )
  129. from backend.app.services.bambu_mqtt import PrinterState
  130. from backend.app.services.camera_light import camera_light
  131. from backend.app.services.energy_plug import energy_plug_candidates, select_energy_reading
  132. from backend.app.services.github_backup import github_backup_service
  133. from backend.app.services.ha_sensor_manager import ha_sensor_manager
  134. from backend.app.services.hms_errors import hms_fault_counts as _hms_fault_counts
  135. from backend.app.services.library_trash import library_trash_service
  136. from backend.app.services.local_backup import local_backup_service
  137. from backend.app.services.location_ha_sensor_manager import location_ha_sensor_manager
  138. from backend.app.services.mqtt_relay import mqtt_relay
  139. from backend.app.services.mqtt_smart_plug import mqtt_smart_plug_service
  140. from backend.app.services.notification_service import notification_service
  141. from backend.app.services.notify_live_activities import notify_live_activities
  142. from backend.app.services.notify_widgets import notify_widgets
  143. from backend.app.services.obico_detection import obico_detection_service
  144. from backend.app.services.octoeverywhere_detection import octoeverywhere_detection_service
  145. from backend.app.services.plug_energy import get_plug_energy as _get_plug_energy
  146. from backend.app.services.print_cost_estimate import plate_scoped_run_estimate as _plate_scoped_run_estimate
  147. from backend.app.services.print_scheduler import scheduler as print_scheduler
  148. from backend.app.services.print_storage import (
  149. REASON_FTP_TRANSFER_FAILED,
  150. REASON_FTPS_COOLOFF,
  151. external_storage_present,
  152. ftp_probe_paths,
  153. print_file_reachable_over_ftp,
  154. )
  155. from backend.app.services.printer_manager import (
  156. init_printer_connections,
  157. parse_plate_id,
  158. printer_manager,
  159. printer_state_to_dict,
  160. printer_tray_layout,
  161. resolve_plate_id,
  162. )
  163. from backend.app.services.slot_kprofile import find_slot_kprofile_for_extruder
  164. from backend.app.services.slot_nozzle import (
  165. nozzle_diameter_for_extruder,
  166. nozzle_flow_for_extruder,
  167. )
  168. from backend.app.services.smart_plug_manager import smart_plug_manager
  169. from backend.app.services.spool_assignment_notifications import (
  170. notify_missing_spool_assignments_on_print_start,
  171. )
  172. from backend.app.services.spool_filament_preset import printer_safe_filament_id
  173. from backend.app.services.spoolman import close_spoolman_client, get_spoolman_client, init_spoolman_client
  174. from backend.app.services.spoolman_tracking import (
  175. cleanup_tracking as _cleanup_spoolman_tracking,
  176. release_archived_slot_rows as _release_archived_spoolman_slots,
  177. report_usage as _report_spoolman_usage,
  178. store_print_data as _store_spoolman_print_data,
  179. )
  180. from backend.app.services.telegram_reactions import telegram_reaction_poller
  181. from backend.app.services.wled import wled_manager
  182. from backend.app.utils.ams_drying import is_drying_active, temperature_alarm_suppressed
  183. from backend.app.utils.ams_humidity import ams_humidity_percent
  184. from backend.app.utils.filament_types import printer_filament_type, support_filament_type
  185. from backend.app.utils.fts_routing import extruder_for_inlet
  186. from backend.app.utils.local_time import to_naive_utc, utcnow_naive
  187. from backend.app.utils.print_jobs import is_internal_printer_job
  188. from backend.app.utils.tray_ids import AmsType, TrayLayout, external_tray_id, unit_ams_type, unit_label
  189. # =============================================================================
  190. # Dependency Check - runs before other imports to give helpful error messages
  191. # =============================================================================
  192. def _start_error_server(missing_packages: list):
  193. """Start a minimal HTTP server to display dependency errors in browser."""
  194. import os
  195. import signal
  196. from http.server import BaseHTTPRequestHandler, HTTPServer
  197. packages_html = "".join(f"<li><code>{p}</code></li>" for p in missing_packages)
  198. html = f"""<!DOCTYPE html>
  199. <html>
  200. <head>
  201. <title>Bambuddy - Setup Required</title>
  202. <style>
  203. body {{
  204. font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
  205. background: #0f172a; color: #e2e8f0;
  206. display: flex; justify-content: center; align-items: center;
  207. min-height: 100vh; margin: 0; padding: 20px; box-sizing: border-box;
  208. }}
  209. .container {{
  210. background: #1e293b; border-radius: 12px; padding: 40px;
  211. max-width: 600px; text-align: center; box-shadow: 0 4px 20px rgba(0,0,0,0.3);
  212. }}
  213. h1 {{ color: #f87171; margin-bottom: 10px; }}
  214. h2 {{ color: #94a3b8; font-weight: normal; margin-top: 0; }}
  215. .packages {{
  216. background: #0f172a; border-radius: 8px; padding: 20px;
  217. margin: 20px 0; text-align: left;
  218. }}
  219. .packages ul {{ margin: 0; padding-left: 20px; }}
  220. .packages li {{ color: #fbbf24; margin: 8px 0; }}
  221. .command {{
  222. background: #0f172a; border-radius: 8px; padding: 15px 20px;
  223. margin: 15px 0; font-family: monospace; color: #4ade80;
  224. text-align: left; overflow-x: auto;
  225. }}
  226. .note {{ color: #94a3b8; font-size: 14px; margin-top: 20px; }}
  227. </style>
  228. </head>
  229. <body>
  230. <div class="container">
  231. <h1>Setup Required</h1>
  232. <h2>Missing Python packages</h2>
  233. <div class="packages"><ul>{packages_html}</ul></div>
  234. <p>To fix, run this command on your server:</p>
  235. <div class="command">pip install -r requirements.lock</div>
  236. <p>Or if using a virtual environment:</p>
  237. <div class="command">./venv/bin/pip install -r requirements.lock</div>
  238. <p class="note">After installing, restart Bambuddy:<br>
  239. <code>sudo systemctl restart bambuddy</code></p>
  240. </div>
  241. </body>
  242. </html>"""
  243. class ErrorHandler(BaseHTTPRequestHandler):
  244. def do_GET(self):
  245. self.send_response(503)
  246. self.send_header("Content-type", "text/html")
  247. self.end_headers()
  248. self.wfile.write(html.encode())
  249. def log_message(self, format, *args):
  250. print(f"[Error Server] {args[0]}")
  251. port = int(os.environ.get("PORT", 8000))
  252. print(f"\nStarting error server on http://0.0.0.0:{port}")
  253. print("Visit this URL in your browser to see the error details.\n")
  254. server = HTTPServer(("0.0.0.0", port), ErrorHandler) # nosec B104
  255. def shutdown(signum, frame):
  256. print("\nShutting down error server...")
  257. raise SystemExit(0)
  258. signal.signal(signal.SIGTERM, shutdown)
  259. signal.signal(signal.SIGINT, shutdown)
  260. server.serve_forever()
  261. def check_dependencies():
  262. """Check that all required packages are installed."""
  263. missing = []
  264. # Map of import name -> package name (for pip install)
  265. required = {
  266. "jwt": "PyJWT",
  267. "fastapi": "fastapi",
  268. "uvicorn": "uvicorn",
  269. "sqlalchemy": "sqlalchemy",
  270. "aiosqlite": "aiosqlite",
  271. "pydantic": "pydantic",
  272. "paho.mqtt": "paho-mqtt",
  273. }
  274. for module, package in required.items():
  275. try:
  276. __import__(module)
  277. except ImportError:
  278. missing.append(package)
  279. if missing:
  280. print("\n" + "=" * 60)
  281. print("ERROR: Missing required Python packages!")
  282. print("=" * 60)
  283. print(f"\nMissing packages: {', '.join(missing)}")
  284. print("\nTo fix, run:")
  285. print(" pip install -r requirements.lock")
  286. print("\nOr if using a virtual environment:")
  287. print(" ./venv/bin/pip install -r requirements.lock")
  288. print("=" * 60 + "\n")
  289. _start_error_server(missing)
  290. check_dependencies()
  291. # =============================================================================
  292. # Import settings first for logging configuration
  293. # Configure logging based on settings
  294. # DEBUG=true -> DEBUG level, else use LOG_LEVEL setting
  295. log_level_str = "DEBUG" if app_settings.debug else app_settings.log_level.upper()
  296. log_level = getattr(logging, log_level_str, logging.INFO)
  297. # Trace ID column ([-] when no request scope is active — startup, MQTT
  298. # callbacks, scheduled tasks not chained from a request — so the column
  299. # stays visually aligned and missing values are obvious in grep). See
  300. # backend/app/core/trace.py for the ContextVar that feeds this slot.
  301. log_format = "%(asctime)s %(levelname)s [%(name)s] [%(trace_id)s] %(message)s"
  302. # Create root logger
  303. root_logger = logging.getLogger()
  304. root_logger.setLevel(log_level)
  305. # Trace-ID injection: this filter populates record.trace_id from the
  306. # per-request ContextVar so the format string above can reference it.
  307. # Attached to each HANDLER (not the root logger) because Python's
  308. # logging semantics only invoke a logger's filters on records that
  309. # *originated* at that logger — records propagated up from child
  310. # loggers (every named logger in the app) never trigger root's filter.
  311. # Putting it on the handlers means every record any handler emits gets
  312. # trace_id injected just before the formatter runs, regardless of which
  313. # logger created the record. Without this, the formatter raises
  314. # KeyError on every child-logger record and the record is silently
  315. # dropped — which is exactly the "logs/bambuddy.log only shows logs
  316. # partially" bug we hit. See backend/app/core/trace.py for the
  317. # ContextVar the filter reads.
  318. from backend.app.core.logging_filters import RefusedHeaderRedactFilter
  319. from backend.app.core.response_redaction import RefusedHeaderResponseMask
  320. from backend.app.core.trace import TraceIDFilter
  321. _trace_id_filter = TraceIDFilter()
  322. # Console handler - always enabled
  323. console_handler = logging.StreamHandler()
  324. console_handler.setLevel(log_level)
  325. console_handler.setFormatter(logging.Formatter(log_format))
  326. console_handler.addFilter(_trace_id_filter)
  327. # A refused header's value (often a token) masked before the line is written.
  328. _refused_header_filter = RefusedHeaderRedactFilter()
  329. console_handler.addFilter(_refused_header_filter)
  330. root_logger.addHandler(console_handler)
  331. def _attach_file_handler_to_uvicorn(handler: logging.Handler) -> None:
  332. """Send uvicorn's own loggers to bambuddy.log as well.
  333. Neither reaches the root handler: uvicorn.access has propagate=False, and
  334. uvicorn.error propagates only to "uvicorn", which has propagate=False.
  335. uvicorn.access is the record of which endpoint changed server state;
  336. uvicorn.error is where an unhandled route error is logged with its
  337. traceback ("Exception in ASGI application").
  338. """
  339. for name in ("uvicorn.access", "uvicorn.error"):
  340. logging.getLogger(name).addHandler(handler)
  341. # File handler - only in production or if explicitly enabled
  342. if app_settings.log_to_file:
  343. log_file = app_settings.log_dir / "bambuddy.log"
  344. file_handler = RotatingFileHandler(
  345. log_file,
  346. maxBytes=app_settings.log_max_bytes,
  347. backupCount=app_settings.log_backup_count,
  348. encoding="utf-8",
  349. )
  350. file_handler.setLevel(log_level)
  351. file_handler.setFormatter(logging.Formatter(log_format))
  352. file_handler.addFilter(_trace_id_filter)
  353. file_handler.addFilter(_refused_header_filter)
  354. root_logger.addHandler(file_handler)
  355. logging.info("Logging to file: %s", log_file)
  356. # Pipe uvicorn's HTTP access log to bambuddy.log too. Uvicorn ships its
  357. # access logger with propagate=False by default, so without this attach
  358. # there is no on-disk record of which endpoint triggered a server-state
  359. # change — the rogue stop_print mystery on 2026-04-26 was untraceable
  360. # for exactly this reason. Filtered to write methods only
  361. # (POST/PUT/PATCH/DELETE) so the high-volume status-poll GETs from the
  362. # frontend don't churn the rotation window faster than it's useful.
  363. from backend.app.core.logging_filters import (
  364. CancelledPoolNoiseFilter,
  365. WriteRequestsOnlyFilter,
  366. )
  367. _attach_file_handler_to_uvicorn(file_handler)
  368. uvicorn_access_logger = logging.getLogger("uvicorn.access")
  369. uvicorn_access_logger.addFilter(WriteRequestsOnlyFilter())
  370. # Uvicorn's access logger has propagate=False (its own default), so the
  371. # root-attached TraceIDFilter never sees these records. Attach a
  372. # second instance directly so HTTP access lines carry the same trace
  373. # ID column as the application logs they correlate with.
  374. uvicorn_access_logger.addFilter(TraceIDFilter())
  375. # Drop SQLAlchemy connection-pool log noise that's caused by Starlette's
  376. # BaseHTTPMiddleware cancelling the inner task scope on client
  377. # disconnect (#1112). The cancel-safe `get_db` already prevents the
  378. # underlying transaction leak; this filter only suppresses the residual
  379. # log records that pre-existing pools still emit during their cleanup.
  380. logging.getLogger("sqlalchemy.pool").addFilter(CancelledPoolNoiseFilter())
  381. # Query-string tokens out of uvicorn's request and WebSocket lines, on every
  382. # handler (console included), whether or not file logging is on.
  383. from backend.app.core.logging_filters import ( # noqa: E402
  384. QueryTokenRedactFilter,
  385. ShutdownCancelledRequestFilter,
  386. )
  387. # First, so the traceback of a stream cancelled at shutdown is dropped before
  388. # the filters below render it.
  389. logging.getLogger("uvicorn.error").addFilter(ShutdownCancelledRequestFilter())
  390. for _uvicorn_logger in ("uvicorn.access", "uvicorn.error"):
  391. logging.getLogger(_uvicorn_logger).addFilter(QueryTokenRedactFilter())
  392. logging.getLogger(_uvicorn_logger).addFilter(_refused_header_filter)
  393. # These stay at WARNING even in debug mode: at INFO/DEBUG httpx/httpcore log
  394. # full request URLs, and some carry credentials (bot tokens, webhook URLs, API
  395. # keys in the query). Same as the runtime log-level toggle in support.py.
  396. logging.getLogger("httpcore").setLevel(logging.WARNING)
  397. logging.getLogger("httpx").setLevel(logging.WARNING)
  398. # aiosqlite logs every statement with its parameters at DEBUG.
  399. logging.getLogger("aiosqlite").setLevel(logging.WARNING)
  400. # Reduce noise from third-party libraries in production
  401. if not app_settings.debug:
  402. logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
  403. logging.getLogger("paho.mqtt").setLevel(logging.WARNING)
  404. logging.info("Bambuddy starting - debug=%s, log_level=%s", app_settings.debug, log_level_str)
  405. # Track active prints: {(printer_id, filename): archive_id}
  406. _active_prints: dict[tuple[int, str], int] = {}
  407. # #1721: stage-22 pre-captured finish photo bytes per printer. on_finish_photo_moment
  408. # fires when stg_cur enters 22 ("Filament unloading") at end-of-print — toolhead
  409. # parked, bed not yet dropped — and grabs a single camera frame into this cache.
  410. # `_background_finish_photo` (inside on_print_complete) consumes the cached bytes
  411. # instead of running its own grab-now chain when present, so the finish photo
  412. # captures the better-framed pre-bed-drop moment without us having to force
  413. # timelapse on at dispatch (the #1397 mechanism that caused #1721's per-layer
  414. # nozzle parking on slicer profiles with Timelapse Type = Smooth).
  415. #
  416. # #2708: the bytes in here are ALWAYS already rotated by the printer's
  417. # camera_rotation. `on_finish_photo_moment` owns that, because one of its
  418. # sources (the #1867 in-print bank) is rotated before it ever reaches the
  419. # bank and the others are not — so the consumer can't tell them apart and
  420. # must not rotate again.
  421. _stage22_finish_frames: dict[int, bytes] = {}
  422. # #1790: per-printer producer-done event. Set by `on_finish_photo_moment` in its
  423. # `finally` block (whether it captured a frame or not). The consumer in
  424. # `_background_finish_photo` waits on it before reading `_stage22_finish_frames`
  425. # so the FINISH-state fallback path — where moment and completion are dispatched
  426. # back-to-back — doesn't race past the producer with an empty pop, and the
  427. # consumer's RTSP fallback can't collide with the producer's still-in-flight RTSP
  428. # grab (Bambu printers allow only one RTSP client at a time).
  429. _stage22_finish_in_flight: dict[int, asyncio.Event] = {}
  430. # #1867: rolling "last in-print camera frame" per printer. Refreshed on
  431. # layer-change and on print-progress advances (#2547) while the model is still
  432. # printing, then consumed by the FINISH-state finish-photo path when the
  433. # dispatcher recorded that it injected End G-code into this print. Bambu
  434. # reports gcode_state=FINISH AFTER the user End G-code (e.g. SwapMod
  435. # plate-swap) has run, so a live grab there would capture the swapped/empty
  436. # plate.
  437. #
  438. # The load-bearing property: both drivers are print telemetry that stops before
  439. # the End G-code executes — no further layer_num increases, and mc_percent
  440. # freezes — so the last banked frame is always the finished print before the
  441. # swap. Anything added as a third driver must hold that same property.
  442. _inprint_frame_bank: dict[int, bytes] = {}
  443. # Monotonic timestamp of the last banked frame per printer — throttles banking
  444. # so tall prints don't add a camera grab on every layer.
  445. _inprint_frame_bank_ts: dict[int, float] = {}
  446. # Minimum seconds between banked frames, except the final object layer which
  447. # always refreshes for the best framing.
  448. _INPRINT_BANK_MIN_INTERVAL = 25.0
  449. # Per-printer "connected" edge tracker. Used by `on_printer_status_change`
  450. # to fire `reconcile_stale_active_prints` exactly once per (re)connection
  451. # (#1542 follow-up — power-cycle ghost prints). The value is True after
  452. # the first connected status update for that connection; transitions back
  453. # to False whenever we observe `state.connected = False` so the next
  454. # reconnect re-arms reconciliation. Keyed by printer_id.
  455. _printer_reconciled_since_connect: dict[int, bool] = {}
  456. # Same edge, same keying, for priming the printer's calibration table exactly
  457. # once per (re)connection. Nothing else asks for it on connect: state.kprofiles
  458. # is otherwise filled only when someone opens the Profiles page or Configure
  459. # Slot, when a GitHub backup runs, or when the printer happens to answer
  460. # somebody else's query on the report topic. Until then the AMS slot card has
  461. # no K value to show on the printers whose trays carry none of their own
  462. # (#2854 — H2-series report cali_idx and nothing more).
  463. _printer_kprofiles_primed_since_connect: dict[int, bool] = {}
  464. # Track expected prints from reprint/scheduled (skip auto-archiving for these)
  465. # {(printer_id, filename): archive_id}
  466. _expected_prints: dict[tuple[int, str], int] = {}
  467. # Archives of the running prints Bambuddy dispatched itself: adopted from an
  468. # expected print, or found with Bambuddy's dispatch marker. _active_prints holds
  469. # every print on_print_start archived, those started from Bambu Studio, Handy or
  470. # the printer's screen included, so it is no proof of ownership on its own.
  471. _owned_print_archive_ids: set[int] = set()
  472. # What a dispatch registered for its print, keyed (printer_id, archive_id):
  473. # plates of one archive print on two printers at once, each with its own.
  474. # Track AMS mapping for prints: {(printer_id, archive_id): [global_tray_id_per_slot]}
  475. # Used by usage tracker to map 3MF slots to physical AMS trays
  476. _print_ams_mappings: dict[tuple[int, int], list[int]] = {}
  477. # Track cost center selection for the current print run: {(printer_id, archive_id): cost_center_id}
  478. _print_cost_center_ids: dict[tuple[int, int], int] = {}
  479. # Track plate_id for prints from multi-plate 3MFs: {(printer_id, archive_id): plate_id}
  480. # Used by usage tracker to scope 3MF parsing to the dispatched plate (#1697).
  481. # Populated by direct-Print and queue dispatch paths; queue prints also have a
  482. # redundant queue-item lookup in on_print_start so this dict isn't load-bearing
  483. # for the queue path. Cleared on print completion or TTL eviction.
  484. _print_plate_ids: dict[tuple[int, int], int] = {}
  485. # Track progress milestones for notifications: {printer_id: last_milestone_notified}
  486. # Milestones are 25, 50, 75. Value of 0 means no milestone notified yet for current print.
  487. _last_progress_milestone: dict[int, int] = {}
  488. # Track whether first layer complete notification has been sent for current print
  489. _first_layer_notified: dict[int, bool] = {}
  490. # Track whether we already sent a kill-switch stop for the current unauthorized print
  491. _unauthorized_print_kill_sent: set[int] = set()
  492. # The MQTT status callback is a hot path. Cache the two-setting kill-switch
  493. # lookup briefly so an unknown active print does not query the database on
  494. # every status frame. A short TTL keeps settings changes responsive.
  495. _KILL_SWITCH_SETTING_CACHE_TTL_SECONDS = 5.0
  496. _kill_switch_setting_cache: tuple[bool, float] | None = None
  497. # Provider notification started when the kill switch stops a print. The later
  498. # MQTT print-complete callback awaits this task and only sends its regular
  499. # provider notification when the immediate attempt failed.
  500. _kill_switch_notification_tasks: dict[int, asyncio.Task[bool]] = {}
  501. # HMS faults already notified, with when each was last seen:
  502. # {printer_id: {fault key: timestamp}}. Tracked per fault, not per printer: a
  503. # fault that flickers beside a held notice was forgotten on every gap and
  504. # notified on every return (#3226). The window is long because the measured
  505. # gaps reach 64 s.
  506. _notified_hms_errors: dict[int, dict[str, float]] = {}
  507. _HMS_CLEAR_GRACE_SECONDS = 600.0
  508. # The print state each printer was last seen in, and the faults recorded before
  509. # it last changed, which are forgotten as soon as they are gone (see
  510. # _take_new_hms_faults).
  511. _hms_print_state: dict[int, str] = {}
  512. _hms_forget_when_gone: dict[int, set[str]] = {}
  513. # Track timelapse file baselines at print start: {printer_id: set of video filenames}
  514. # Used for snapshot-diff detection at print completion
  515. _timelapse_baselines: dict[int, set[str]] = {}
  516. # Printers whose baseline above was taken off a card that could not be read,
  517. # so it is empty without the card being empty (#3412).
  518. _timelapse_baselines_unread: set[int] = set()
  519. # Track printers waiting for bed to cool after print completion.
  520. # Event-driven: fires when bed_temper arrives via MQTT below threshold.
  521. # {printer_id: {"threshold": float, "filename": str, "registered_at": float}}
  522. _bed_cool_waiters: dict[int, dict] = {}
  523. # Track printers where the user explicitly stopped the print from the queue UI.
  524. # When on_print_complete fires with status "failed" for these printers we treat it
  525. # as "cancelled" (stopped by user) so the correct notification email is sent.
  526. _user_stopped_printers: set[int] = set()
  527. # Printers whose running print the kill switch stopped. Their completion is a
  528. # cancel with the reason "killSwitch", not the "failed" the printer reports.
  529. _kill_switch_stopped: set[int] = set()
  530. # Offline-notification edge state (#1752): fire `on_printer_offline` exactly
  531. # once when a printer transitions connected → disconnected. `_printer_last_connected`
  532. # holds the previous observation so we only fire on the True → False edge (a
  533. # False → False repeat doesn't notify; an initial False at startup doesn't
  534. # notify either, since there's no prior True). `_printer_offline_notify_tasks`
  535. # holds the per-printer pending asyncio task that fires the notification
  536. # after a debounce window — cancelled if the printer reconnects before the
  537. # window elapses, so transient MQTT blips don't flood the user.
  538. _printer_last_connected: dict[int, bool] = {}
  539. # Printers whose relay offline event went out, so the reconnect publishes the
  540. # matching online event and a blip shorter than the debounce publishes neither.
  541. _printer_offline_published: set[int] = set()
  542. _printer_offline_notify_tasks: dict[int, asyncio.Task] = {}
  543. # Debounce: a printer must stay offline this long before we notify. Sized
  544. # against the staleness path (`bambu_mqtt.py::STALE_RECONNECT_COOLDOWN = 30s`)
  545. # so a single stale-trigger cooldown isn't enough to fire — only a real
  546. # offline that survives one reconnect attempt notifies.
  547. _PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS = 60.0
  548. # HMS short-code → failure_reason key. Used by _dispatch_archive_update
  549. # when status="failed" to label the print's failure_reason in archives.
  550. #
  551. # Earlier code matched on `module` alone (e.g. "any module 0x0C HMS → Layer shift"),
  552. # which is wrong on two counts:
  553. # 1. Real layer-shift codes live in module 0x03 (see Bambu wiki), not 0x0C.
  554. # 2. Module 0x0C is "Motion Controller" — broad category that also covers cameras
  555. # and visual markers, AND the H2D firmware emits a 0x0C HMS (0C00_001B, not in
  556. # the public wiki) as part of its user-cancel sequence. Matching on the module
  557. # alone caused user-cancellations to be archived as "Layer shift" failures.
  558. # We now match by full short code only — anything not in this map leaves
  559. # failure_reason=None rather than guessing.
  560. # Values are the canonical camelCase failure-reason keys, NOT display labels
  561. # (issue #2974). The vocabulary is enforced on writes by
  562. # ``_FAILURE_REASON_KEYS`` in ``api/routes/print_log.py`` and rendered through
  563. # ``t('editArchive.failureReasons.<key>')`` on both the archive editor and the
  564. # Statistics breakdown. Storing a label here instead put a second spelling of
  565. # the same cause into one column: the Failure Analysis widget groups on the raw
  566. # value, so a print the backend classified and an identical one a user
  567. # classified counted as two different reasons, and the label form could never
  568. # be translated because there was no key for ``t()`` to resolve.
  569. _HMS_FAILURE_REASONS: dict[str, str] = {
  570. # Layer shift / step loss
  571. "0300_4057": "layerShift",
  572. "0300_4068": "layerShift",
  573. "0300_800C": "layerShift",
  574. # Filament runout (printer-side & per-AMS-slot)
  575. "0300_8004": "filamentRunout",
  576. "0700_8011": "filamentRunout",
  577. "0701_8011": "filamentRunout",
  578. "0702_8011": "filamentRunout",
  579. "0703_8011": "filamentRunout",
  580. "0704_8011": "filamentRunout",
  581. "0705_8011": "filamentRunout",
  582. "0706_8011": "filamentRunout",
  583. "0707_8011": "filamentRunout",
  584. "07FF_8011": "filamentRunout",
  585. # Clogged nozzle / extruder
  586. "0300_4006": "cloggedNozzle",
  587. "0300_8016": "cloggedNozzle",
  588. "0300_801C": "cloggedNozzle",
  589. "0700_8003": "cloggedNozzle",
  590. "0700_8007": "cloggedNozzle",
  591. "0700_8013": "cloggedNozzle",
  592. "0701_8003": "cloggedNozzle",
  593. "0701_8007": "cloggedNozzle",
  594. "0701_8013": "cloggedNozzle",
  595. "0702_8003": "cloggedNozzle",
  596. # AI print monitoring — spaghetti / the model coming off the plate.
  597. # `spaghettiDetached` is a key the archive editor already offers for this
  598. # failure mode, not a new one, so a derived reason opens the dropdown on
  599. # that option rather than blank — and survives the next save, which clears
  600. # any value the editor does not recognise.
  601. #
  602. # A module-0x0C row is safe here despite the warning above: that warning is
  603. # about matching on the module alone, and 0C00_8042 is a full short code
  604. # with a documented meaning ("The AI print monitor has detected a spaghetti
  605. # defect", hms_errors.py). The H2D cancel echo is 0C00_001B, so the two
  606. # cannot collide.
  607. #
  608. # 0300_8003's own text ends "before continuing your print", but on an X2D
  609. # it arrives with the print already paused, offering only
  610. # RESUME_PRINTING_DEFECTS / STOP_PRINTING — a halt waiting on the user.
  611. #
  612. # Two neighbours are left out on purpose, so this does not get re-derived:
  613. # * 0C00_C004 "Possible spaghetti failure was detected." — "possible"
  614. # reads as a warning about a print that is still running, not a halt.
  615. # * 0300_800A is AI monitoring too, but it reports a filament pile-up in
  616. # the waste chute. That is not the print failing.
  617. #
  618. # That line is drawn from the text, not from `severity`, because severity
  619. # cannot draw it: 0300_8003 reaches us through `print_error`, a bare
  620. # module/error word with no level in it, and bambu_mqtt.py gives every
  621. # print_error entry a flat severity=3. Matching on the short code alone is
  622. # the right shape for derive_failure_reason, not an omission.
  623. "0300_8003": "spaghettiDetached",
  624. "0C00_8042": "spaghettiDetached",
  625. }
  626. def _hms_short_code(attr: int, code: int | str) -> str:
  627. """Build the canonical "MMMM_CCCC" HMS short code from raw attr/code values."""
  628. if isinstance(code, str):
  629. code_int = int(code.replace("0x", ""), 16) if code else 0
  630. else:
  631. code_int = int(code or 0)
  632. attr_int = int(attr or 0)
  633. return f"{(attr_int >> 16) & 0xFFFF:04X}_{code_int & 0xFFFF:04X}"
  634. def derive_failure_reason(status: str, hms_errors: list[dict] | None) -> str | None:
  635. """Derive the failure_reason key for an archived print.
  636. Returns "userCancelled" for cancelled/aborted prints; for failed prints,
  637. returns the first matching key from _HMS_FAILURE_REASONS, or None when
  638. no HMS code matches (don't guess — null is honest). The keys are the
  639. archive editor's vocabulary (_FAILURE_REASON_KEYS in print_log.py) and are
  640. translated at render time.
  641. """
  642. if status in ("aborted", "cancelled"):
  643. return "userCancelled"
  644. if status != "failed":
  645. return None
  646. for err in hms_errors or []:
  647. short_code = _hms_short_code(err.get("attr", 0), err.get("code", 0))
  648. if short_code in _HMS_FAILURE_REASONS:
  649. return _HMS_FAILURE_REASONS[short_code]
  650. return None
  651. # Track created_by_id for expected prints so the user email can be sent even when
  652. # the archive itself doesn't have created_by_id set (e.g. library-file-based prints).
  653. # {(printer_id, filename): created_by_id}
  654. _expected_print_creators: dict[tuple[int, str], int] = {}
  655. # Per-printer dedup for unknown_tag WS broadcasts. Keyed by
  656. # (ams_id, tray_id) -> (tag_uid, tray_uuid); we only re-broadcast when the
  657. # tag tuple changes for the slot. Cleared when the slot is reported empty
  658. # so remove + reinsert reliably re-prompts the UI.
  659. _unknown_tag_last_broadcast: dict[int, dict[tuple[int, int], tuple[str, str]]] = {}
  660. async def _broadcast_unknown_tag(
  661. *,
  662. printer_id: int,
  663. ams_id: int,
  664. tray_id: int,
  665. tag_uid: str,
  666. tray_uuid: str,
  667. tray_type: str | None = None,
  668. tray_color: str | None = None,
  669. tray_sub_brands: str | None = None,
  670. tray_count: int | None = None,
  671. ) -> None:
  672. """Broadcast unknown_tag, deduped so repeated MQTT pushes for the same slot+tag don't spam the UI."""
  673. _logger = logging.getLogger(__name__)
  674. slot_key = (ams_id, tray_id)
  675. tag_key = (tag_uid or "", tray_uuid or "")
  676. per_printer = _unknown_tag_last_broadcast.setdefault(printer_id, {})
  677. if per_printer.get(slot_key) == tag_key:
  678. _logger.debug(
  679. "unknown_tag deduped for printer=%d AMS=%d slot=%d tag=%s",
  680. printer_id,
  681. ams_id,
  682. tray_id,
  683. tag_key[0][:8] or tag_key[1][:8] or "(none)",
  684. )
  685. return
  686. _logger.info(
  687. "unknown_tag broadcast: printer=%d AMS=%d slot=%d type=%r color=%r tag=%s",
  688. printer_id,
  689. ams_id,
  690. tray_id,
  691. tray_type,
  692. tray_color,
  693. tag_key[0][:8] or tag_key[1][:8] or "(none)",
  694. )
  695. # Broadcast first; only commit the dedup if the WS write succeeds.
  696. # If broadcast raises, the next MQTT push retries instead of being
  697. # permanently silenced by a poisoned dedup entry.
  698. await ws_manager.broadcast(
  699. {
  700. "type": "unknown_tag",
  701. "printer_id": printer_id,
  702. "ams_id": ams_id,
  703. "tray_id": tray_id,
  704. "tag_uid": tag_uid,
  705. "tray_uuid": tray_uuid,
  706. "tray_type": tray_type,
  707. "tray_color": tray_color,
  708. "tray_sub_brands": tray_sub_brands,
  709. "tray_count": tray_count,
  710. }
  711. )
  712. per_printer[slot_key] = tag_key
  713. def _clear_unknown_tag_dedup(printer_id: int, ams_id: int, tray_id: int) -> None:
  714. """Drop the cached last-broadcast tag for a slot (called when slot reports empty or gets matched)."""
  715. per_printer = _unknown_tag_last_broadcast.get(printer_id)
  716. if per_printer is None:
  717. return
  718. per_printer.pop((ams_id, tray_id), None)
  719. # TTL for expected-print entries: evict registrations older than this to prevent
  720. # unbounded growth when a print is registered but never starts (e.g. printer
  721. # disconnect, app restart, print started from the printer panel).
  722. _EXPECTED_PRINT_TTL_SECONDS: int = 2 * 60 * 60 # 2 hours
  723. # Registration timestamps used for TTL eviction: {(printer_id, filename): monotonic_time}
  724. _expected_print_registered_at: dict[tuple[int, str], float] = {}
  725. # Cleanup loop interval
  726. _EXPECTED_PRINT_CLEANUP_INTERVAL: int = 15 * 60 # 15 minutes
  727. _expected_prints_cleanup_task: asyncio.Task | None = None
  728. _ACTIVE_PRINT_STATES: set[str] = {"RUNNING", "PRINTING", "PAUSE"}
  729. def _build_status_print_keys(printer_id: int, state: PrinterState) -> list[tuple[int, str]]:
  730. """Build filename keys for matching a printer status update to Bambuddy-owned jobs."""
  731. possible_keys: list[tuple[int, str]] = []
  732. filename = (state.gcode_file or state.current_print or "").strip()
  733. subtask_name = (state.subtask_name or "").strip()
  734. if subtask_name:
  735. possible_keys.append((printer_id, subtask_name))
  736. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  737. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  738. if filename:
  739. base_name = filename.rsplit("/", 1)[-1]
  740. if base_name.endswith(".gcode.3mf"):
  741. root_name = base_name[: -len(".gcode.3mf")]
  742. possible_keys.append((printer_id, root_name))
  743. possible_keys.append((printer_id, base_name))
  744. possible_keys.append((printer_id, f"{root_name}.gcode"))
  745. possible_keys.append((printer_id, f"{root_name}.3mf"))
  746. elif base_name.endswith(".3mf"):
  747. root_name = base_name[: -len(".3mf")]
  748. possible_keys.append((printer_id, root_name))
  749. possible_keys.append((printer_id, base_name))
  750. elif base_name.endswith(".gcode"):
  751. root_name = base_name[: -len(".gcode")]
  752. possible_keys.append((printer_id, root_name))
  753. possible_keys.append((printer_id, f"{root_name}.3mf"))
  754. possible_keys.append((printer_id, base_name))
  755. else:
  756. possible_keys.append((printer_id, base_name))
  757. possible_keys.append((printer_id, f"{base_name}.3mf"))
  758. return possible_keys
  759. async def _print_user_id_if_still_there(db, print_user: dict | None) -> int | None:
  760. """The id of who started the print, unless that user has been deleted since.
  761. Deleting a user forgets them in printer_manager, but a print that ended a
  762. moment before still holds them.
  763. """
  764. if not print_user or print_user.get("user_id") is None:
  765. return None
  766. from backend.app.models.user import User
  767. return await db.scalar(select(User.id).where(User.id == print_user["user_id"]))
  768. def _is_bambuddy_authorized_print_in_memory(printer_id: int, state: PrinterState) -> bool:
  769. """Check the cheap, process-local print ownership signals."""
  770. if printer_manager.get_current_print_user(printer_id):
  771. return True
  772. return any(
  773. key in _expected_prints or _active_prints.get(key) in _owned_print_archive_ids
  774. for key in _build_status_print_keys(printer_id, state)
  775. )
  776. def _reported_print_names(printer_id: int, state: PrinterState) -> set[str]:
  777. """The names the printer reports for its print, as it echoes them (spaces become underscores)."""
  778. return {_normalise_subtask_name(name) for _pid, name in _build_status_print_keys(printer_id, state)} - {""}
  779. async def _printing_archive_without_job_id(
  780. db, printer_id: int, reported: set[str], *, dispatched_only: bool, live_only: bool = False
  781. ):
  782. """The newest printing archive on ``printer_id`` of a print that reported no job id, under a reported name.
  783. An archive with a real id of its own is a print that reported one: a "0"
  784. print is not it, even under the same name (one whose completion was missed
  785. would otherwise cover it). ``dispatched_only`` keeps to archives carrying
  786. Bambuddy's dispatch marker; ``live_only`` leaves out deleted ones.
  787. """
  788. from backend.app.models.archive import PrintArchive
  789. query = select(PrintArchive).where(
  790. PrintArchive.printer_id == printer_id,
  791. PrintArchive.status == "printing",
  792. or_(PrintArchive.subtask_id.is_(None), PrintArchive.subtask_id.in_(("", "0"))),
  793. )
  794. if dispatched_only:
  795. query = query.where(or_(PrintArchive.billing_run_id.is_not(None), PrintArchive.created_by_id.is_not(None)))
  796. if live_only:
  797. query = query.where(PrintArchive.deleted_at.is_(None))
  798. archives = await db.execute(query.order_by(PrintArchive.created_at.desc()))
  799. return next((a for a in archives.scalars() if _archive_name_forms(a) & reported), None)
  800. async def running_print_archive(db, printer_id: int, state: PrinterState, *, printing_only: bool, by_name: bool):
  801. """The archive of the print running on ``printer_id``, or None (#3327).
  802. By the print's subtask_id first, which the firmware mints per print; with
  803. ``printing_only`` only an archive that is still printing (a leftover row of
  804. a completion never seen must not stand in for another job). A print
  805. started outside the cloud (Bambu Studio over LAN, Handy, the printer's
  806. screen) reports subtask_id "0", which links it to nothing: the archive
  807. Bambuddy bound to the running print comes next, the one completion charges
  808. and photographs. A restart forgets that binding and rebinds only Bambuddy's
  809. own prints, so with ``by_name`` a "0" print last falls back to its printing
  810. archive under the name the printer reports. That can be an earlier run of
  811. the same name whose completion was missed while Bambuddy was down: good
  812. enough for a plate label or a cover, not for which objects skip-objects
  813. offers, so those callers leave it off. None until the archive exists, and
  814. never a deleted one.
  815. """
  816. from backend.app.models.archive import PrintArchive
  817. from backend.app.services.print_log import open_run_on_printer
  818. subtask_id = str(getattr(state, "subtask_id", "") or "").strip()
  819. if subtask_id not in ("", "0"):
  820. # The run open on the printer for this job: the archive row is shared
  821. # by every plate of it, its printer and job id the latest start's.
  822. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  823. if run is not None and run.archive_id is not None:
  824. archive = await db.scalar(
  825. select(PrintArchive).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  826. )
  827. if archive is not None:
  828. return archive
  829. query = select(PrintArchive).where(
  830. PrintArchive.printer_id == printer_id,
  831. PrintArchive.subtask_id == subtask_id,
  832. PrintArchive.deleted_at.is_(None),
  833. )
  834. if printing_only:
  835. query = query.where(PrintArchive.status == "printing")
  836. archive = await db.scalar(query.order_by(PrintArchive.created_at.desc()).limit(1))
  837. if archive is not None:
  838. return archive
  839. bound = next(
  840. (_active_prints[key] for key in _build_status_print_keys(printer_id, state) if key in _active_prints),
  841. None,
  842. )
  843. if bound is not None:
  844. return await db.scalar(select(PrintArchive).where(PrintArchive.id == bound, PrintArchive.deleted_at.is_(None)))
  845. if not by_name or subtask_id != "0":
  846. return None
  847. reported = _reported_print_names(printer_id, state)
  848. if not reported:
  849. return None
  850. return await _printing_archive_without_job_id(db, printer_id, reported, dispatched_only=False, live_only=True)
  851. def _bind_active_print(
  852. printer_id: int,
  853. archive_id: int,
  854. archive_filename: str | None,
  855. filename: str | None = None,
  856. subtask_name: str | None = None,
  857. ) -> None:
  858. """Track a running print under every name its completion may report.
  859. One set for every path that binds a print (new archive, expected print,
  860. restart resume, fallback): the archive's file name, the name the printer
  861. reported, and the subtask with and without ``.3mf``. Paths that registered
  862. fewer names lost the bound archive at completion, and with it the plate
  863. restore's height (#3240), whenever MQTT reported a name they had left out.
  864. """
  865. for name in {archive_filename, filename, f"{subtask_name}.3mf" if subtask_name else None, subtask_name}:
  866. if name:
  867. _active_prints[(printer_id, name)] = archive_id
  868. async def _archive_of_open_run(printer_id: int, subtask_id: str | None = None) -> int | None:
  869. """The archive of the run open on the printer, if it is still there.
  870. With ``subtask_id``, only a run of that job."""
  871. try:
  872. from backend.app.models.archive import PrintArchive
  873. from backend.app.services.print_log import open_run_on_printer
  874. async with async_session() as db:
  875. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  876. if run is None or run.archive_id is None:
  877. return None
  878. return await db.scalar(
  879. select(PrintArchive.id).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  880. )
  881. except Exception as e:
  882. logging.getLogger(__name__).warning("[RUN] Could not read the open run of printer %s: %s", printer_id, e)
  883. return None
  884. async def _announce_discarded_runs(discarded, printer_id: int) -> None:
  885. """Publish the runs ``discard_open_runs`` dropped (archive redesign):
  886. each announced by run/started, now gone without an end."""
  887. if not discarded:
  888. return
  889. try:
  890. from backend.app.models.printer import Printer
  891. async with async_session() as db:
  892. printer = (
  893. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  894. ).first()
  895. if printer is not None:
  896. for run in discarded:
  897. await mqtt_relay.on_run_discarded(run, printer.serial_number, printer.name)
  898. except Exception as e:
  899. logging.getLogger(__name__).debug("[RUN] Could not announce discarded runs on printer %s: %s", printer_id, e)
  900. async def _publish_run_completed(run_id: int | None, printer_id: int) -> None:
  901. """Publish the run a print's end just wrote on the MQTT relay (archive
  902. redesign); nothing without one. Never in the way of the completion."""
  903. if run_id is None:
  904. return
  905. try:
  906. from backend.app.models.archive import PrintArchive
  907. from backend.app.models.print_log import PrintLogEntry
  908. from backend.app.models.printer import Printer
  909. async with async_session() as db:
  910. run = await db.get(PrintLogEntry, run_id)
  911. printer = (
  912. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  913. ).first()
  914. archive = await db.get(PrintArchive, run.archive_id) if run is not None and run.archive_id else None
  915. if run is not None and printer is not None:
  916. await mqtt_relay.on_run_completed(run, archive, printer.serial_number, printer.name)
  917. except Exception as e:
  918. logging.getLogger(__name__).debug("[RUN] Could not publish the end of run %s: %s", run_id, e)
  919. async def _open_run_for(archive_id: int, printer_id: int, subtask_id: str | None = None) -> None:
  920. """Write the run of the print that just started on the archive (archive
  921. redesign). In a session of its own and never in the way of the print
  922. start: until its readers move over, the archive still carries the print.
  923. The archive's start must be committed before. ``subtask_id`` is the
  924. printer's job, which finds the run again at a resume."""
  925. try:
  926. from backend.app.models.archive import PrintArchive
  927. from backend.app.models.printer import Printer
  928. from backend.app.services.print_log import open_run, open_run_on_printer, run_snapshot
  929. async with async_session() as db:
  930. archive = await db.get(PrintArchive, archive_id)
  931. if archive is None:
  932. return
  933. printer = (
  934. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  935. ).first()
  936. printer_name = printer.name if printer is not None else None
  937. already = await open_run_on_printer(db, printer_id)
  938. before = run_snapshot(already) if already is not None else None
  939. run = await open_run(
  940. db,
  941. archive,
  942. printer_id=printer_id,
  943. printer_name=printer_name,
  944. plate_id=_print_plate_ids.get((printer_id, archive_id)),
  945. subtask_id=subtask_id,
  946. )
  947. await db.commit()
  948. # Once per print: a resume or a repeated start finds the run it opened.
  949. # A new one replaced the run still open there (open_run): that one's
  950. # print ended unseen.
  951. # Compared by start too: SQLite can give the new run the id of the one
  952. # it replaced.
  953. replaced = (
  954. run is not None and before is not None and (before.id != run.id or before.started_at != run.started_at)
  955. )
  956. if run is not None and printer is not None and (before is None or replaced):
  957. if replaced:
  958. await _announce_discarded_runs([before], printer_id)
  959. try:
  960. await mqtt_relay.on_run_started(run, archive, printer.serial_number, printer_name)
  961. except Exception:
  962. pass # Don't fail the print start if MQTT fails
  963. except Exception as e:
  964. logging.getLogger(__name__).warning("[RUN] Could not open the run of archive %s: %s", archive_id, e)
  965. async def _drop_replaced_plate_videos(
  966. db, archive_id: int, plate_id: int | None, stale_relpath: str | None, logger: logging.Logger
  967. ) -> None:
  968. """Delete the videos a reprint of a plate replaces: ``stale_relpath``, the
  969. one the archive showed (when print start judged it this plate's), and
  970. those of the archive's earlier runs of the same plate. A plate keeps one
  971. video, as when the archive held the only one; another plate's stays. The
  972. runs that held them lose them. Commits."""
  973. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, PrintLogEntry
  974. from backend.app.utils.safe_path import PathTraversalError, safe_join_under
  975. runs = select(PrintLogEntry).where(
  976. PrintLogEntry.archive_id == archive_id, PrintLogEntry.timelapse_path.is_not(None)
  977. )
  978. with_video = list(await db.scalars(runs.execution_options(**{INCLUDE_OPEN_RUNS: True})))
  979. paths = {stale_relpath} if stale_relpath else set()
  980. if plate_id is not None:
  981. paths |= {r.timelapse_path for r in with_video if r.plate_id == plate_id}
  982. # A video another plate's run holds is that plate's.
  983. paths -= {r.timelapse_path for r in with_video if r.plate_id is not None and r.plate_id != plate_id}
  984. if not paths:
  985. return
  986. for run in with_video:
  987. if run.timelapse_path in paths:
  988. run.timelapse_path = None
  989. await db.commit()
  990. for relpath in sorted(p for p in paths if p):
  991. try:
  992. stale_path = safe_join_under(app_settings.base_dir, relpath, http=False)
  993. if stale_path.is_file():
  994. stale_path.unlink()
  995. logger.info("Deleted stale timelapse %s on reprint of archive %s", relpath, archive_id)
  996. except (OSError, PathTraversalError) as e:
  997. logger.warning("Failed to delete stale timelapse %s on reprint: %s", relpath, e)
  998. def _mark_owned_if_dispatched(archive) -> None:
  999. """Remember a running print's archive as Bambuddy's when it carries a
  1000. dispatch marker Bambuddy writes itself: ``billing_run_id`` (minted per
  1001. dispatch in the scheduler) or ``created_by_id`` (from the queue item)."""
  1002. if archive.billing_run_id is not None or archive.created_by_id is not None:
  1003. _owned_print_archive_ids.add(archive.id)
  1004. async def _is_printer_kill_switch_enabled_cached() -> bool:
  1005. """Return the kill-switch setting without querying on every MQTT frame."""
  1006. global _kill_switch_setting_cache
  1007. now = time.monotonic()
  1008. if _kill_switch_setting_cache is not None:
  1009. enabled, expires_at = _kill_switch_setting_cache
  1010. if now < expires_at:
  1011. return enabled
  1012. async with async_session() as db:
  1013. from backend.app.services.finance_budget import is_printer_kill_switch_enabled
  1014. enabled = await is_printer_kill_switch_enabled(db)
  1015. _kill_switch_setting_cache = (enabled, now + _KILL_SWITCH_SETTING_CACHE_TTL_SECONDS)
  1016. return enabled
  1017. async def _is_bambuddy_authorized_print(printer_id: int, state: PrinterState, db) -> bool | None:
  1018. """Resolve whether the current print was started by Bambuddy.
  1019. ``None`` means identity is not yet safe to decide. The kill switch must
  1020. defer in that case: stopping a print is irreversible, and the first status
  1021. frames after a restart may arrive before all subtask fields are populated.
  1022. """
  1023. if _is_bambuddy_authorized_print_in_memory(printer_id, state):
  1024. return True
  1025. # The printer's built-in calibration runs (Bambuddy's Calibrate, the
  1026. # printer's screen) play gcode from its read-only system partition, where no
  1027. # user file can be. Only that path counts: a calibration *name* proves
  1028. # nothing, since Bambu Studio's pressure-advance print reports a user-style
  1029. # path, and any print can be given that name.
  1030. if str(getattr(state, "gcode_file", "") or "").startswith("/usr/"):
  1031. return True
  1032. # A1, A1 Mini, P1 and A2L report the same calibration as a bare file name,
  1033. # which nothing distinguishes from a user's file given that name, so it is
  1034. # not exempt: with the kill switch on, calibrate on those printers with the
  1035. # switch off.
  1036. possible_keys = _build_status_print_keys(printer_id, state)
  1037. # In-memory ownership is lost on every Bambuddy restart, so fall back to what
  1038. # is on disk. subtask_id is minted per print and pins the answer to the job
  1039. # actually running, rather than to an unrelated one that reuses a filename.
  1040. raw_subtask_id = getattr(state, "subtask_id", None)
  1041. subtask_id = str(raw_subtask_id).strip() if raw_subtask_id is not None else ""
  1042. if subtask_id == "":
  1043. # Not reported yet: the first frames after a restart can lack it.
  1044. return None
  1045. from backend.app.models.archive import PrintArchive
  1046. if subtask_id == "0":
  1047. # A print started outside the cloud from Bambu Studio, Handy or the
  1048. # printer's screen reports "0", so there is no id to match: one of
  1049. # Bambuddy's printing archives on this printer under the name the
  1050. # printer reports, compared as the printer echoes names (spaces become
  1051. # underscores). Nothing reported to compare yet: wait.
  1052. reported = _reported_print_names(printer_id, state)
  1053. if not reported:
  1054. return None
  1055. archive = await _printing_archive_without_job_id(db, printer_id, reported, dispatched_only=True)
  1056. else:
  1057. from backend.app.services.print_log import open_run_on_printer
  1058. # This job's run open on the printer: the archive is shared by every
  1059. # plate of it, and its status, printer and job id are the latest
  1060. # start's, which may be another printer's.
  1061. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  1062. archive = await db.get(PrintArchive, run.archive_id) if run is not None and run.archive_id else None
  1063. if archive is None:
  1064. result = await db.execute(
  1065. select(PrintArchive)
  1066. .where(
  1067. PrintArchive.printer_id == printer_id,
  1068. PrintArchive.status == "printing",
  1069. PrintArchive.subtask_id == subtask_id,
  1070. )
  1071. .order_by(PrintArchive.created_at.desc())
  1072. .limit(1)
  1073. )
  1074. archive = result.scalar_one_or_none()
  1075. # An archive row on its own proves nothing: `on_print_start` archives every
  1076. # print it observes, including ones started from Bambu Studio or Handy, and
  1077. # stamps them with the same status and subtask_id. Authorizing on its mere
  1078. # existence would disable the kill switch the moment the 3MF finishes
  1079. # downloading. Only a dispatch marker Bambuddy writes itself counts —
  1080. # `billing_run_id` (minted per dispatch in the scheduler) or `created_by_id`
  1081. # (carried over from the queue item that started it).
  1082. if archive is not None and (archive.billing_run_id is not None or archive.created_by_id is not None):
  1083. # Rehydrate the fast in-memory path for subsequent status frames. Include
  1084. # both the archive filename and every normalized key reported by MQTT.
  1085. _owned_print_archive_ids.add(archive.id)
  1086. _active_prints[(printer_id, archive.filename)] = archive.id
  1087. for key in possible_keys:
  1088. _active_prints[key] = archive.id
  1089. return True
  1090. # No dispatch marker. Before calling this someone else's print, check whether
  1091. # Bambuddy has a job of its own running on this printer: a library-file
  1092. # dispatch has no archive at send time, and an archive created seconds later
  1093. # by `on_print_start` carries neither marker. The queue row, which the
  1094. # scheduler commits to status="printing" before the MQTT send, is the one
  1095. # durable record every Bambuddy print has. It cannot be tied to this
  1096. # subtask_id, so it is grounds to defer, never to authorize — stopping a
  1097. # print is irreversible, and refusing to act costs nothing but a log line.
  1098. from backend.app.models.print_queue import PrintQueueItem
  1099. dispatched_here = await db.scalar(
  1100. select(PrintQueueItem.id)
  1101. .where(
  1102. PrintQueueItem.printer_id == printer_id,
  1103. PrintQueueItem.status == "printing",
  1104. )
  1105. .limit(1)
  1106. )
  1107. if dispatched_here is not None:
  1108. return None
  1109. return False
  1110. async def _send_kill_switch_provider_notification(
  1111. printer_id: int,
  1112. printer_name: str,
  1113. data: dict,
  1114. ) -> bool:
  1115. """Send the immediate print-stopped provider notification.
  1116. Returning a success flag lets the normal MQTT completion path retry when
  1117. this early notification could not be delivered.
  1118. """
  1119. logger = logging.getLogger(__name__)
  1120. try:
  1121. async with async_session() as db:
  1122. from backend.app.services.print_log import open_run_on_printer
  1123. # The print it stopped, for {archive_id} / {run_id}: the completion
  1124. # that would name them skips its notification after this one.
  1125. run = await open_run_on_printer(db, printer_id)
  1126. await notification_service.on_print_complete(
  1127. printer_id,
  1128. printer_name,
  1129. "stopped",
  1130. data,
  1131. db,
  1132. archive_data={"archive_id": run.archive_id, "run_id": run.id} if run is not None else None,
  1133. )
  1134. return True
  1135. except Exception as e:
  1136. logger.warning(
  1137. "[KILL SWITCH] Immediate provider notification failed for printer %s: %s",
  1138. printer_id,
  1139. e,
  1140. )
  1141. return False
  1142. async def _kill_switch_notification_already_sent(task: asyncio.Task[bool] | None) -> bool:
  1143. """Wait for an immediate kill-switch notification, if one was scheduled."""
  1144. if task is None:
  1145. return False
  1146. try:
  1147. return await task
  1148. except Exception as e:
  1149. logging.getLogger(__name__).warning("[KILL SWITCH] Notification task failed: %s", e)
  1150. return False
  1151. async def _run_needs_energy_start(archive, printer_id: int, db) -> bool:
  1152. """At a resume: whether the print's run still lacks its energy start
  1153. reading. The archive's may be an earlier print's. Commits a reading
  1154. found on the archive for this run onto it."""
  1155. try:
  1156. from backend.app.services.print_log import run_needs_energy_start
  1157. needs = await run_needs_energy_start(db, archive, printer_id=printer_id)
  1158. await db.commit()
  1159. return needs
  1160. except Exception as e:
  1161. logging.getLogger(__name__).warning("[RUN] Energy start of archive %s's run not checked: %s", archive.id, e)
  1162. return archive.energy_start_kwh is None
  1163. async def _record_energy_start(archive, printer_id: int, db, *, context: str = "") -> bool:
  1164. """Capture the smart plug lifetime counter on the archive at print start.
  1165. Persists `energy_start_kwh` on the archive row (#941) so per-print energy
  1166. tracking survives a backend restart mid-print. The print-end handler reads
  1167. this value back from the DB and computes the delta against the current
  1168. plug counter.
  1169. """
  1170. _logger = logging.getLogger(__name__)
  1171. try:
  1172. candidates = await energy_plug_candidates(db, printer_id)
  1173. if not candidates:
  1174. _logger.info("[ENERGY] No smart plug for printer %s (archive %s)", printer_id, archive.id)
  1175. return False
  1176. selected = await select_energy_reading(candidates, _get_plug_energy, db)
  1177. if selected is None:
  1178. # Naming the plugs matters here: with several linked to one printer
  1179. # this is the difference between "the meter is offline" and "you
  1180. # linked only accessories" (#2859).
  1181. _logger.warning(
  1182. "[ENERGY] No plug on printer %s reports a lifetime energy counter for archive %s (tried: %s)",
  1183. printer_id,
  1184. archive.id,
  1185. ", ".join(plug.name for plug in candidates),
  1186. )
  1187. return False
  1188. plug, energy = selected
  1189. archive.energy_start_kwh = float(energy["total"])
  1190. # With the plug's snapshots, what lets the cost follow the price over
  1191. # the print rather than take the price at its end (#1251). A price that
  1192. # can't be had must not cost the print its energy reading.
  1193. archive.energy_start_at = utcnow_naive()
  1194. archive.energy_start_plug_id = plug.id
  1195. try:
  1196. from backend.app.services.energy_price import current_price
  1197. archive.energy_start_price = await current_price(db)
  1198. except Exception as e:
  1199. _logger.warning("[ENERGY] Could not read the electricity price for archive %s: %s", archive.id, e)
  1200. archive.energy_start_price = None
  1201. try:
  1202. from backend.app.services.print_log import mirror_energy_start
  1203. await mirror_energy_start(db, archive, printer_id=printer_id)
  1204. except Exception as e:
  1205. _logger.warning("[RUN] Energy start not copied onto the run of archive %s: %s", archive.id, e)
  1206. await db.commit()
  1207. _logger.info(
  1208. "[ENERGY] Recorded starting energy%s for archive %s from plug '%s': %s kWh",
  1209. f" ({context})" if context else "",
  1210. archive.id,
  1211. plug.name,
  1212. energy["total"],
  1213. )
  1214. return True
  1215. except Exception as e:
  1216. _logger.warning("[ENERGY] Failed to record starting energy for archive %s: %s", archive.id, e)
  1217. return False
  1218. def forget_print_creator(user_id: int) -> None:
  1219. """The print-start emails of a deleted user's jobs go to nobody, not to
  1220. whoever is given the id next (SQLite reuses it)."""
  1221. for key in [key for key, creator in _expected_print_creators.items() if creator == user_id]:
  1222. del _expected_print_creators[key]
  1223. async def forget_printer(printer_id: int) -> None:
  1224. """Drop what is kept for a deleted printer's id, here and in the services.
  1225. SQLite used to give the id to the next printer created, which inherited it
  1226. all: the running print's owner (its prints passed the kill switch and were
  1227. billed to that user), the plate-clear gate, pending and suppressed
  1228. notifications, the camera feed, the plate references. Ids are no longer
  1229. reused (AUTOINCREMENT), but nothing kept for a printer that is gone should
  1230. stay. Called after the delete is committed, and again by work that was in
  1231. flight and wrote state back.
  1232. Every printer-keyed map of this module is listed here or, with the reason,
  1233. in tests/unit/test_forget_printer.py.
  1234. """
  1235. from backend.app.api.routes import camera
  1236. from backend.app.services import (
  1237. bambu_ftp,
  1238. kprofile_drift,
  1239. layer_timelapse,
  1240. plate_clear,
  1241. plate_detection,
  1242. print_dispatch_context,
  1243. usage_tracker,
  1244. )
  1245. from backend.app.services.notify_live_activities import notify_live_activities
  1246. from backend.app.services.obico_detection import obico_detection_service
  1247. from backend.app.services.printer_file_cache import printer_file_cache
  1248. # Keyed (printer_id, filename). The archive-keyed maps of those prints go too.
  1249. archive_ids = set()
  1250. for tracked in (_active_prints, _expected_prints):
  1251. for key in [key for key in tracked if key[0] == printer_id]:
  1252. archive_ids.add(tracked.pop(key))
  1253. for archive_id in archive_ids:
  1254. _owned_print_archive_ids.discard(archive_id)
  1255. for registered in (_print_ams_mappings, _print_cost_center_ids, _print_plate_ids):
  1256. for key in [key for key in registered if key[0] == printer_id]:
  1257. del registered[key]
  1258. for tracked in (_expected_print_creators, _expected_print_registered_at):
  1259. for key in [key for key in tracked if key[0] == printer_id]:
  1260. del tracked[key]
  1261. for key in [key for key in _ams_index_only_logged if key[0] == printer_id]:
  1262. _ams_index_only_logged.discard(key)
  1263. for key in [key for key in _ams_alarm_cooldown if key.split(":", 1)[0] == str(printer_id)]:
  1264. del _ams_alarm_cooldown[key]
  1265. # Pending work that would act on the id: an offline notice, a 3MF retry.
  1266. # A kill-switch notice is about a stop that happened and is left to finish.
  1267. for tasks in (_printer_offline_notify_tasks, _fallback_3mf_retry_tasks):
  1268. task = tasks.pop(printer_id, None)
  1269. if task is not None:
  1270. task.cancel()
  1271. for keyed in (
  1272. _stage22_finish_frames,
  1273. _stage22_finish_in_flight,
  1274. _inprint_frame_bank,
  1275. _inprint_frame_bank_ts,
  1276. _printer_reconciled_since_connect,
  1277. _printer_kprofiles_primed_since_connect,
  1278. _last_progress_milestone,
  1279. _first_layer_notified,
  1280. _kill_switch_notification_tasks,
  1281. _notified_hms_errors,
  1282. _hms_print_state,
  1283. _hms_forget_when_gone,
  1284. _timelapse_baselines,
  1285. _bed_cool_waiters,
  1286. _printer_last_connected,
  1287. _unknown_tag_last_broadcast,
  1288. _last_status_broadcast,
  1289. _connection_watchdog_last_attempt,
  1290. ):
  1291. keyed.pop(printer_id, None)
  1292. for marked in (
  1293. _unauthorized_print_kill_sent,
  1294. _user_stopped_printers,
  1295. _kill_switch_stopped,
  1296. _printer_offline_published,
  1297. _nozzle_count_updated,
  1298. _runtime_seen_connected,
  1299. _timelapse_baselines_unread,
  1300. ):
  1301. marked.discard(printer_id)
  1302. printer_manager.forget_printer(printer_id)
  1303. printer_file_cache.forget_printer(printer_id)
  1304. print_dispatch_context.clear(printer_id)
  1305. kprofile_drift.forget_printer(printer_id, deleted=True)
  1306. layer_timelapse.cancel_session(printer_id)
  1307. usage_tracker.forget_printer(printer_id)
  1308. obico_detection_service.forget_printer(printer_id)
  1309. octoeverywhere_detection_service.forget_printer(printer_id)
  1310. bambu_ftp.clear_3mf_cache(printer_id)
  1311. await camera.forget_printer(printer_id)
  1312. printers.clear_cover_cache(printer_id)
  1313. print_scheduler.forget_printer(printer_id)
  1314. notify_live_activities.forget_printer(printer_id)
  1315. plate_detection.remove_printer_references(printer_id)
  1316. plate_clear.set_override(printer_id, None)
  1317. def register_expected_print(
  1318. printer_id: int,
  1319. filename: str,
  1320. archive_id: int,
  1321. ams_mapping: list[int] | None = None,
  1322. created_by_id: int | None = None,
  1323. cost_center_id: int | None = None,
  1324. plate_id: int | None = None,
  1325. ):
  1326. """Register an expected print from reprint/scheduled so we don't create duplicate archives."""
  1327. # Store with multiple filename variations to catch different naming patterns
  1328. _expected_prints[(printer_id, filename)] = archive_id
  1329. # Also store without .3mf extension if present
  1330. if filename.endswith(".3mf"):
  1331. base = filename[:-4]
  1332. _expected_prints[(printer_id, base)] = archive_id
  1333. _expected_prints[(printer_id, f"{base}.gcode")] = archive_id
  1334. # Store AMS mapping for usage tracking at print completion, and plate_id for
  1335. # a single-plate dispatch from a multi-plate 3MF — without it, the direct-Print
  1336. # path attributes the whole file's filament total to the spool instead of just
  1337. # the printed plate (#1697). They describe this dispatch: one that has none
  1338. # drops what an earlier dispatch of the archive left (sent, never started),
  1339. # unless a plate of the archive is printing now and still needs them.
  1340. dispatch = (printer_id, archive_id)
  1341. archive_printing = _printing_here(printer_id, archive_id)
  1342. if ams_mapping is not None:
  1343. _print_ams_mappings[dispatch] = ams_mapping
  1344. elif not archive_printing:
  1345. _print_ams_mappings.pop(dispatch, None)
  1346. if cost_center_id is not None:
  1347. _print_cost_center_ids[dispatch] = cost_center_id
  1348. elif not archive_printing:
  1349. _print_cost_center_ids.pop(dispatch, None)
  1350. if plate_id is not None:
  1351. _print_plate_ids[dispatch] = plate_id
  1352. elif not archive_printing:
  1353. _print_plate_ids.pop(dispatch, None)
  1354. # Store created_by_id so the user start email can be sent even when the archive
  1355. # itself has no created_by_id (e.g. library-file-based queue prints)
  1356. if created_by_id is not None:
  1357. _expected_print_creators[(printer_id, filename)] = created_by_id
  1358. if filename.endswith(".3mf"):
  1359. base = filename[:-4]
  1360. _expected_print_creators[(printer_id, base)] = created_by_id
  1361. _expected_print_creators[(printer_id, f"{base}.gcode")] = created_by_id
  1362. # Record registration time for TTL-based eviction
  1363. _registered_at = time.monotonic()
  1364. _expected_print_registered_at[(printer_id, filename)] = _registered_at
  1365. if filename.endswith(".3mf"):
  1366. base = filename[:-4]
  1367. _expected_print_registered_at[(printer_id, base)] = _registered_at
  1368. _expected_print_registered_at[(printer_id, f"{base}.gcode")] = _registered_at
  1369. logging.getLogger(__name__).info(
  1370. f"Registered expected print: printer={printer_id}, file={filename}, archive={archive_id}, ams_mapping={ams_mapping}, plate_id={plate_id}"
  1371. )
  1372. def unregister_expected_print(printer_id: int, filename: str, archive_id: int) -> None:
  1373. """Undo :func:`register_expected_print` when the print never went out.
  1374. Registration has to happen *before* the MQTT print command, because the
  1375. printer can report the print before the line after the send executes. So
  1376. every path that registers and then fails to send — a cancel winning the
  1377. #1853 CAS race, a ``start_print()`` that returns False, or any exception in
  1378. between — leaves an expectation for a print that will never arrive.
  1379. The TTL sweep evicts those after two hours, which is far longer than it
  1380. takes a user to react to a failed dispatch by pressing print again: that
  1381. reprint would be folded into the *old* archive and take the stale
  1382. ``ams_mapping`` / ``plate_id`` with it. Hence the explicit inverse.
  1383. Mirrors the sweep's rules, including the one that is easy to get wrong:
  1384. ``_print_ams_mappings`` / ``_print_plate_ids`` / ``_print_cost_center_ids``
  1385. are keyed by printer and archive, not by file, so they may only be dropped
  1386. once no live key of the printer still points at that archive.
  1387. """
  1388. keys = [(printer_id, filename)]
  1389. if filename.endswith(".3mf"):
  1390. base = filename[:-4]
  1391. keys.append((printer_id, base))
  1392. keys.append((printer_id, f"{base}.gcode"))
  1393. removed = False
  1394. for key in keys:
  1395. if _expected_prints.pop(key, None) is not None:
  1396. removed = True
  1397. _expected_print_creators.pop(key, None)
  1398. _expected_print_registered_at.pop(key, None)
  1399. # Still needed by another expected print of the archive on the printer,
  1400. # or a plate of it printing there, as in the sweep.
  1401. if not _expected_or_printing_here(printer_id, archive_id):
  1402. _drop_dispatch(printer_id, archive_id)
  1403. if removed:
  1404. logging.getLogger(__name__).info(
  1405. "Unregistered expected print: printer=%s, file=%s, archive=%s (print was never sent)",
  1406. printer_id,
  1407. filename,
  1408. archive_id,
  1409. )
  1410. def _compute_run_filament_grams(
  1411. status: str,
  1412. archive_filament_used_grams: float | None,
  1413. progress: float | int | None,
  1414. usage_results: list[dict] | None,
  1415. ) -> float | None:
  1416. """Per-run filament for PrintLogEntry, partial- and tracker-aware (#1378, #1390).
  1417. Priority for every status:
  1418. 1. Sum of tracked spool deltas in ``usage_results`` (AMS-measured
  1419. weight delta — same source that drives "Total Consumed" on the
  1420. Inventory page, so Stats and Inventory totals stay aligned).
  1421. 2. For ``completed``: the slicer estimate (no tracker available, fall
  1422. back to the canonical "this print used X" value).
  1423. 3. For partial statuses: ``estimate * progress%``.
  1424. 4. ``None`` if nothing is known.
  1425. """
  1426. tracked_grams = sum(r.get("weight_used") or 0 for r in (usage_results or []))
  1427. if tracked_grams > 0:
  1428. return round(tracked_grams, 1)
  1429. if status == "completed":
  1430. return archive_filament_used_grams
  1431. if archive_filament_used_grams:
  1432. scale = max(0.0, min(((progress or 0) / 100.0), 1.0))
  1433. if scale > 0:
  1434. return round(archive_filament_used_grams * scale, 1)
  1435. return None
  1436. def _printing_here(printer_id: int, archive_id: int) -> bool:
  1437. """Whether a plate of the archive is printing on the printer."""
  1438. return any(key[0] == printer_id and bound == archive_id for key, bound in _active_prints.items())
  1439. def _expected_or_printing_here(printer_id: int, archive_id: int) -> bool:
  1440. """Whether a print of the archive is expected or printing on the printer."""
  1441. return _printing_here(printer_id, archive_id) or any(
  1442. key[0] == printer_id and expected == archive_id for key, expected in _expected_prints.items()
  1443. )
  1444. def _drop_dispatch(printer_id: int, archive_id: int) -> None:
  1445. """Forget what the dispatch of the archive to the printer registered."""
  1446. dispatch = (printer_id, archive_id)
  1447. _print_ams_mappings.pop(dispatch, None)
  1448. _print_cost_center_ids.pop(dispatch, None)
  1449. _print_plate_ids.pop(dispatch, None)
  1450. def _get_start_ams_mapping(data: dict, printer_id: int, archive_id: int | None) -> list[int] | None:
  1451. """Resolve AMS mapping for print start without consuming stored queue/reprint state."""
  1452. stored_ams_mapping = data.get("ams_mapping")
  1453. if not stored_ams_mapping and archive_id:
  1454. stored_ams_mapping = _print_ams_mappings.get((printer_id, archive_id))
  1455. return stored_ams_mapping
  1456. def _get_start_plate_id(printer_id: int, archive_id: int | None) -> int | None:
  1457. """Resolve plate_id for print start without consuming stored direct-Print state.
  1458. Direct-Print of a single plate from a multi-plate 3MF registers plate_id in
  1459. ``_print_plate_ids`` at dispatch time; this lets the spoolman / usage tracker
  1460. read it back at print-start without popping (the entry is popped on print
  1461. completion or TTL eviction, mirroring ``_print_ams_mappings``).
  1462. """
  1463. if archive_id is None:
  1464. return None
  1465. return _print_plate_ids.get((printer_id, archive_id))
  1466. def _run_print_time_seconds(archive, printer_id: int) -> int | None:
  1467. """The slicer's time for the plate this print prints, for its start
  1468. notification (#3375): ``archive.print_time_seconds`` adds every plate of
  1469. the file up (#1593). The plate is the dispatch's, else the archive's. The
  1470. whole file's time when its plates are not known; None for a plate not
  1471. known of several, and the notification takes the printer's remaining
  1472. time."""
  1473. from backend.app.services.print_cost_estimate import run_estimate
  1474. plate_id = _print_plate_ids.get((printer_id, archive.id), archive.plate_id)
  1475. estimate = run_estimate(archive, plate_id)
  1476. return estimate.seconds if estimate is not None else None
  1477. _plate_estimates_task: asyncio.Task | None = None
  1478. async def _fill_plate_estimates() -> None:
  1479. from backend.app.services.print_cost_estimate import fill_plate_estimates
  1480. try:
  1481. filled = await fill_plate_estimates(async_session)
  1482. except asyncio.CancelledError:
  1483. raise
  1484. except Exception as exc:
  1485. logging.getLogger(__name__).warning("[ARCHIVE] Reading plate estimates failed: %s", exc)
  1486. return
  1487. if filled:
  1488. logging.getLogger(__name__).info("[ARCHIVE] Read the plate estimates of %d archive(s)", filled)
  1489. _gcode_hashes_task: asyncio.Task | None = None
  1490. async def _fill_gcode_hashes() -> None:
  1491. from backend.app.services.archive_gcode_hash import fill_gcode_hashes
  1492. try:
  1493. filled = await fill_gcode_hashes(async_session)
  1494. except asyncio.CancelledError:
  1495. raise
  1496. except Exception as exc:
  1497. logging.getLogger(__name__).warning("[ARCHIVE] Reading G-code fingerprints failed: %s", exc)
  1498. return
  1499. if filled:
  1500. logging.getLogger(__name__).info("[ARCHIVE] Read the G-code fingerprints of %d archive(s)", filled)
  1501. def _partial_progress_scale(progress: int | float | None) -> float:
  1502. """Clamp ``progress / 100`` into [0.0, 1.0] for partial-print scaling.
  1503. Used by every site that multiplies a "would-have-used" slicer estimate
  1504. down to "actually-used" for failed / cancelled / stopped prints. Centralised
  1505. so the three sites in ``_background_notifications`` (and the per-plate
  1506. override helper) can't drift apart on the coercion shape.
  1507. """
  1508. return max(0.0, min((progress or 0) / 100.0, 1.0))
  1509. def _scope_notification_archive_data_to_plate(
  1510. archive_data: dict,
  1511. archive_file_path: str | None,
  1512. plate_id: int | None,
  1513. print_status: str,
  1514. progress: int | float | None,
  1515. base_dir: Path,
  1516. ) -> dict:
  1517. """Override summed-across-plates totals in ``archive_data`` with the values
  1518. for ``plate_id`` so the completion notification reports what was actually
  1519. printed, not the whole project (#1785).
  1520. The 3MF parser at services/archive.py:200-264 sums ``prediction`` and
  1521. ``weight`` across every plate of a multi-plate file (#1593) — correct for
  1522. the archive card's "whole project" headline, wrong for the completion
  1523. notification of a single-plate print. The queue UI already re-reads the
  1524. 3MF per-plate at print_queue.py:272-285; this helper mirrors that for the
  1525. notification payload (filament grams, time estimate, per-slot breakdown).
  1526. No-ops when ``plate_id`` is None, the file is missing, or the 3MF carries
  1527. no per-plate values — in every fail case the original ``archive_data`` is
  1528. returned unchanged so the notification still sends.
  1529. """
  1530. if plate_id is None or not archive_file_path:
  1531. return archive_data
  1532. from backend.app.utils.threemf_tools import (
  1533. extract_filament_usage_from_3mf,
  1534. extract_print_time_from_3mf,
  1535. )
  1536. archive_path = base_dir / archive_file_path
  1537. if not archive_path.exists():
  1538. return archive_data
  1539. plate_slots = extract_filament_usage_from_3mf(archive_path, plate_id)
  1540. plate_grams = sum(f.get("used_g", 0) for f in plate_slots)
  1541. plate_time = extract_print_time_from_3mf(archive_path, plate_id)
  1542. scale = 1.0 if print_status == "completed" else _partial_progress_scale(progress)
  1543. if plate_time:
  1544. archive_data["print_time_seconds"] = plate_time
  1545. # Gate both the grams headline AND the per-slot breakdown on the same
  1546. # `plate_grams > 0` signal: if the 3MF carries per-plate filament rows but
  1547. # they all sum to zero (slicer bug / re-slice without estimate), drop back
  1548. # to the project-level grams the archive columns already provide rather
  1549. # than ship a project-level headline next to an all-zero per-plate
  1550. # breakdown.
  1551. if plate_grams > 0:
  1552. archive_data["actual_filament_grams"] = round(plate_grams * scale, 1)
  1553. archive_data["filament_slots"] = [
  1554. {
  1555. "slot_id": s.get("slot_id"),
  1556. "used_g": round((s.get("used_g") or 0) * scale, 1),
  1557. "type": s.get("type", ""),
  1558. "color": s.get("color", ""),
  1559. }
  1560. for s in plate_slots
  1561. ]
  1562. return archive_data
  1563. def _extract_filament_data_from_mqtt(data: dict, ams_mapping: list[int] | None = None) -> dict[str, str]:
  1564. """Best-effort filament metadata from the MQTT print-start snapshot.
  1565. Used when the 3MF can't be downloaded (P1S/A1/P2S firmwares lock the
  1566. file during print, see #1533) so the fallback PrintArchive still has
  1567. enough filament info to support the inventory views and AMS-expansion
  1568. planning the operator opens it for. Returns a dict with optional
  1569. ``filament_type`` and ``filament_color`` keys in the same
  1570. comma-separated format the 3MF extractor produces, so the rest of the
  1571. codebase treats the fallback archive identically to a normal one.
  1572. ``ams_mapping`` is the slicer's slot-per-print-filament list captured
  1573. from the MQTT print payload (global tray IDs, possibly -1 for VT-tray
  1574. entries). When supplied, only the slots actually consumed by this
  1575. print contribute. Without it the function falls back to every loaded
  1576. AMS slot — less accurate but still useful.
  1577. Accepts both the raw inner payload (``{"ams": {"ams": [...]}, ...}``)
  1578. that the unit tests pass directly, AND the on_print_start callback
  1579. shape (``{"raw_data": {"ams": {"ams": [...]}, ...}, ...}``) the
  1580. bambu_mqtt service hands to main.py at runtime. The original
  1581. ``_extract_filament_data_from_mqtt(data)`` shipped in #1533 only
  1582. handled the inner shape and silently returned ``{}`` for every real
  1583. print start, leaving fallback archives' filament fields NULL — the
  1584. exact regression the fix was meant to close. Reported with a log
  1585. proving the AMS state was right there at
  1586. ``data["raw_data"]["ams"]["ams"][0]["tray"][0]`` (#1533 follow-up).
  1587. """
  1588. result: dict[str, str] = {}
  1589. # Look at the on_print_start wrapper first, then the inner shape.
  1590. raw_data = (data or {}).get("raw_data")
  1591. ams_root = (raw_data or {}).get("ams") if isinstance(raw_data, dict) else None
  1592. if not isinstance(ams_root, dict):
  1593. ams_root = (data or {}).get("ams") or {}
  1594. ams_units = ams_root.get("ams") if isinstance(ams_root, dict) else None
  1595. if not isinstance(ams_units, list) or not ams_units:
  1596. return result
  1597. from backend.app.services.print_scheduler import _global_tray_id
  1598. # Map global tray id → (type, color), addressed as the print's ams_mapping
  1599. # is (utils.tray_ids).
  1600. layout = TrayLayout.from_units(ams_units)
  1601. loaded: dict[int, tuple[str, str]] = {}
  1602. for unit in ams_units:
  1603. if not isinstance(unit, dict):
  1604. continue
  1605. try:
  1606. unit_id = int(unit.get("id", 0))
  1607. except (TypeError, ValueError):
  1608. continue
  1609. for tray in unit.get("tray") or []:
  1610. if not isinstance(tray, dict):
  1611. continue
  1612. try:
  1613. tray_id = int(tray.get("id", 0))
  1614. except (TypeError, ValueError):
  1615. continue
  1616. ttype = (tray.get("tray_type") or "").strip()
  1617. # RRGGBBAA from the printer, stored as the 3MF path stores it:
  1618. # "#RRGGBB" (the Archives page's swatches and colour filter).
  1619. hex_color = (tray.get("tray_color") or "").strip().upper()[:6]
  1620. tcolor = f"#{hex_color}" if re.fullmatch(r"[0-9A-F]{6}", hex_color) else ""
  1621. if not ttype:
  1622. continue # Empty / unloaded slot.
  1623. loaded[_global_tray_id(unit_id, tray_id, layout)] = (ttype, tcolor)
  1624. if not loaded:
  1625. return result
  1626. if ams_mapping:
  1627. used_ids = [int(x) for x in ams_mapping if isinstance(x, (int, float)) and int(x) >= 0]
  1628. filaments = [loaded[g] for g in used_ids if g in loaded]
  1629. if not filaments:
  1630. return result # Mapping points entirely at slots we have no data for.
  1631. else:
  1632. filaments = [loaded[g] for g in sorted(loaded.keys())]
  1633. types_joined = ",".join(f[0] for f in filaments)
  1634. colors_joined = ",".join(f[1] for f in filaments if f[1])
  1635. # Column limits per backend/app/models/archive.py: filament_type=50,
  1636. # filament_color=200.
  1637. if types_joined:
  1638. result["filament_type"] = types_joined[:50]
  1639. if colors_joined:
  1640. result["filament_color"] = colors_joined[:200]
  1641. return result
  1642. def _maybe_start_layer_timelapse(printer, printer_id: int, archive_id: int) -> bool:
  1643. """Start a layer-timelapse session for *archive_id* when the printer has
  1644. an external camera configured. Returns True if a session was started.
  1645. Three call sites in on_print_start (expected-archive promotion, fallback
  1646. archive creation, fresh-archive creation) used to inline this same
  1647. if-block; the inline copies kept drifting (#1353 fixed only one of them
  1648. on the first pass). Centralising the conditional + call here makes the
  1649. contract testable in isolation and keeps the three sites locked in step.
  1650. """
  1651. if not (printer.external_camera_enabled and printer.external_camera_url):
  1652. return False
  1653. from backend.app.services.layer_timelapse import start_session
  1654. start_session(
  1655. printer_id,
  1656. archive_id,
  1657. printer.external_camera_url,
  1658. printer.external_camera_type or "mjpeg",
  1659. snapshot_url=printer.external_camera_snapshot_url,
  1660. rotation=getattr(printer, "camera_rotation", 0),
  1661. )
  1662. logging.getLogger(__name__).info("Started layer timelapse for printer %s, archive %s", printer_id, archive_id)
  1663. return True
  1664. def _format_hms_error_summary(hms_errors: list[dict]) -> str | None:
  1665. """Build a human-readable failure reason from MQTT hms_errors for PrintQueueItem.error_message.
  1666. Each entry has keys: code ('0x4038'), attr (32-bit int), module, severity, and
  1667. — since #2926 — the description the parser already resolved, which is preferred
  1668. when present so the queue's failure reason reads the same as the status
  1669. response. The short code still produces the bracketed label, and still
  1670. resolves the sentence for a caller whose entries predate the field. An entry
  1671. with a 16-char ``full_code`` is labelled with it instead, since the short code
  1672. of an ``hms[]`` fault is not a code anyone can look up. Falls back to the bare
  1673. label when no description is on file. Returns None for an empty list so
  1674. callers can leave error_message unset.
  1675. """
  1676. if not hms_errors:
  1677. return None
  1678. from backend.app.services.hms_errors import get_error_description
  1679. parts: list[str] = []
  1680. for err in hms_errors:
  1681. try:
  1682. # `_hms_short_code` rather than a local derivation: this one used to
  1683. # format the error without masking it to 16 bits, so an `hms[]` entry
  1684. # whose code carries an alert-level group produced a five-digit label
  1685. # like "0500_3000A" — not a code the user can look up, and never a
  1686. # catalogue key, so the sentence was lost with it.
  1687. short_code = _hms_short_code(err.get("attr", 0), err.get("code", 0))
  1688. except (TypeError, ValueError):
  1689. continue
  1690. description = err.get("description") or get_error_description(short_code)
  1691. # An `hms[]` fault's short code ("0500_000E") drops the part and level
  1692. # groups and is not a code anyone can look up; its full code in the
  1693. # printer screen's four groups is (#2728).
  1694. full_code = str(err.get("full_code") or "")
  1695. label = "-".join(full_code[i : i + 4] for i in range(0, 16, 4)) if len(full_code) == 16 else short_code
  1696. parts.append(f"[{label}] {description}" if description else f"[{label}]")
  1697. return "; ".join(parts) if parts else None
  1698. async def _bump_library_file_usage_if_completed(db, item, queue_status: str) -> None:
  1699. """Increment LibraryFile.print_count and stamp last_printed_at when a queued
  1700. print completes successfully. Gated to status=='completed': failed, cancelled
  1701. and aborted prints do not count as usage. Caller is responsible for committing
  1702. the session. No-op when the queue item has no linked library file (e.g. reprints
  1703. from an archive). See #1008."""
  1704. if queue_status != "completed" or item.library_file_id is None:
  1705. return
  1706. from backend.app.models.library import LibraryFile
  1707. lib_file = await db.scalar(select(LibraryFile).where(LibraryFile.id == item.library_file_id))
  1708. if lib_file is None:
  1709. return
  1710. lib_file.print_count = (lib_file.print_count or 0) + 1
  1711. lib_file.last_printed_at = datetime.now(timezone.utc)
  1712. def mark_printer_stopped_by_user(printer_id: int) -> None:
  1713. """Mark that the active print on this printer was stopped by the user from the queue UI.
  1714. When on_print_complete fires with status 'failed' for a printer in this set we
  1715. reclassify it as 'cancelled' so the correct 'print stopped' notification is sent
  1716. rather than a 'print failed' notification.
  1717. """
  1718. _user_stopped_printers.add(printer_id)
  1719. logging.getLogger(__name__).info("Marked printer %s as user-stopped from queue", printer_id)
  1720. _last_status_broadcast: dict[int, str] = {}
  1721. # Track printers where we've updated nozzle_count
  1722. _nozzle_count_updated: set[int] = set()
  1723. async def _maybe_notify_printer_offline(printer_id: int) -> None:
  1724. """Wait the debounce window then fire `on_printer_offline` if the printer
  1725. is still offline.
  1726. Scheduled by `on_printer_status_change` on the connected → disconnected
  1727. edge (#1752). Cancelled by the same handler if the printer reconnects
  1728. before the window elapses, so a single MQTT blip + recovery doesn't
  1729. notify. Both the staleness-detector path (`bambu_mqtt.py::check_staleness`)
  1730. and the smart-plug power-off path (`printer_manager.mark_printer_offline`)
  1731. route through the same status-change callback, so this covers both.
  1732. """
  1733. logger = logging.getLogger(__name__)
  1734. try:
  1735. await asyncio.sleep(_PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS)
  1736. still_offline = not printer_manager.is_connected(printer_id)
  1737. logger.info(
  1738. "[#1752] Printer %s offline debounce elapsed: still_offline=%s",
  1739. printer_id,
  1740. still_offline,
  1741. )
  1742. if not still_offline:
  1743. return
  1744. async with async_session() as db:
  1745. from backend.app.models.printer import Printer
  1746. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1747. printer = result.scalar_one_or_none()
  1748. if not printer:
  1749. logger.warning(
  1750. "[#1752] Printer %s missing from DB at offline-notify time; skipping",
  1751. printer_id,
  1752. )
  1753. return
  1754. # Recorded only when the relay can publish it, so the reconnect's
  1755. # online event always has an offline before it.
  1756. if mqtt_relay.enabled and mqtt_relay.connected:
  1757. _printer_offline_published.add(printer_id)
  1758. try:
  1759. await mqtt_relay.on_printer_offline(printer_id, printer.name, printer.serial_number)
  1760. except Exception as e:
  1761. logger.warning("MQTT relay offline event failed for printer %s: %s", printer_id, e)
  1762. logger.info(
  1763. "[#1752] Dispatching on_printer_offline for printer %s (%s)",
  1764. printer_id,
  1765. printer.name,
  1766. )
  1767. await notification_service.on_printer_offline(printer_id, printer.name, db)
  1768. except asyncio.CancelledError:
  1769. raise
  1770. except Exception as e:
  1771. logger.warning("Printer offline notification failed for printer %s: %s", printer_id, e)
  1772. finally:
  1773. _printer_offline_notify_tasks.pop(printer_id, None)
  1774. def _hms_notify_key(error) -> str:
  1775. """What identifies a fault for notification de-duplication.
  1776. The full code, which is unique per fault. ``attr`` alone used to be the key:
  1777. unique for a ``print_error``, but for an ``hms[]`` fault it is only the
  1778. module and part, so two faults on one part (#1840's H2C held 0500-0600-0002-0005
  1779. and -0006 together) shared a key and only the first was ever notified. An
  1780. entry without a full code falls back to attr and code together.
  1781. """
  1782. full_code = getattr(error, "full_code", "") or ""
  1783. if full_code:
  1784. return full_code.upper()
  1785. return f"{error.attr:08X}:{error.code}"
  1786. def _hms_errors_to_notify(errors: list, new_error_codes: set[str]) -> list:
  1787. """The new faults that count (see ``_hms_fault_counts``).
  1788. These go to the MQTT relay; the caller also needs a description before it
  1789. sends a notification. Level 0, Bambu's "invalid" level, never counts. This
  1790. used to read ``severity >= 2`` when severity held the part byte; on the real
  1791. level that would drop the task-stopping errors (#2728).
  1792. """
  1793. return [e for e in errors if _hms_notify_key(e) in new_error_codes and _hms_fault_counts(e)]
  1794. # What each HMS module byte is about, read off Bambu's own texts for its codes
  1795. # (hms_catalog.json). The table used to call 0x05 "Nozzle/Extruder", 0x0C
  1796. # "Motion Controller" and 0x12 "Chamber", which titled USB and MQTT errors as
  1797. # nozzle faults and an A1 mini's AMS Lite runouts as chamber errors (#3486).
  1798. _HMS_MODULE_NAMES = {
  1799. 0x03: "Print/Task", # bed, nozzle temperatures, motors, the print itself
  1800. 0x05: "System", # USB/SD, network, firmware, files, AMS communication
  1801. 0x07: "AMS/Filament",
  1802. 0x08: "Toolhead",
  1803. 0x0C: "Camera", # cameras, lidar, first-layer and AI inspection
  1804. 0x12: "AMS Lite",
  1805. 0x18: "AMS-HT",
  1806. 0x1A: "Tool Changer",
  1807. 0x29: "Air Filter",
  1808. 0x2A: "AMS 3H",
  1809. }
  1810. def _hms_error_title(module: int) -> str:
  1811. """The notification title for a fault of *module*."""
  1812. name = _HMS_MODULE_NAMES.get(module)
  1813. return f"{name} Error" if name else "Printer Error"
  1814. def _take_new_hms_faults(printer_id: int, errors: list, print_state: str | None = None) -> list:
  1815. """The faults on this printer not notified yet, and record them as notified.
  1816. Tracking is updated before anything is sent, so concurrent status callbacks
  1817. cannot notify the same fault twice. A notified fault is forgotten once it
  1818. has been gone for ``_HMS_CLEAR_GRACE_SECONDS``, so one that flickers off
  1819. and on is notified once (#3226), while one that clears and comes back much
  1820. later is notified again.
  1821. When ``print_state`` changes (a print resumed, started or finished), every
  1822. fault recorded so far is forgotten the first time it is gone, without the
  1823. wait: a runout fixed before resuming must be notified if it happens again a
  1824. few minutes later, while the printer sits paused. Not only the faults gone
  1825. at the change itself, because the update that flips the state can still
  1826. carry the old fault (a ``print_error`` entry stays until a payload brings
  1827. ``hms``).
  1828. """
  1829. now = time.time()
  1830. current = {_hms_notify_key(e) for e in errors}
  1831. seen = _notified_hms_errors.setdefault(printer_id, {})
  1832. forget_when_gone = _hms_forget_when_gone.setdefault(printer_id, set())
  1833. if print_state is not None:
  1834. print_state = print_state.upper()
  1835. if _hms_print_state.get(printer_id, print_state) != print_state:
  1836. forget_when_gone.update(seen)
  1837. _hms_print_state[printer_id] = print_state
  1838. for key, last_seen in list(seen.items()):
  1839. if key not in current and (key in forget_when_gone or now - last_seen >= _HMS_CLEAR_GRACE_SECONDS):
  1840. del seen[key]
  1841. forget_when_gone.discard(key)
  1842. new = current - seen.keys()
  1843. for key in current:
  1844. seen[key] = now
  1845. if not seen:
  1846. _notified_hms_errors.pop(printer_id, None)
  1847. if not forget_when_gone:
  1848. _hms_forget_when_gone.pop(printer_id, None)
  1849. return _hms_errors_to_notify(errors, new)
  1850. async def _take_new_hms_faults_and_save(printer_id: int, errors: list, print_state: str | None = None) -> list:
  1851. """``_take_new_hms_faults``, and store the notified set when it changed (#3486).
  1852. The set lives in memory, so without this every restart sent each fault the
  1853. printer still lists once more. A failed write only costs that protection
  1854. for the next restart, so it is logged and the notification goes ahead.
  1855. """
  1856. before = set(_notified_hms_errors.get(printer_id, ()))
  1857. new = _take_new_hms_faults(printer_id, errors, print_state)
  1858. if set(_notified_hms_errors.get(printer_id, ())) != before:
  1859. from backend.app.services.hms_notified import save_notified_hms_faults
  1860. try:
  1861. await save_notified_hms_faults(printer_id, lambda: _notified_hms_errors.get(printer_id, {}).keys())
  1862. except Exception as e:
  1863. logging.getLogger(__name__).warning(
  1864. "Could not store the notified HMS faults of printer %s: %s", printer_id, e
  1865. )
  1866. return new
  1867. async def _load_notified_hms_faults() -> None:
  1868. """Restore the faults notified before the restart (#3486).
  1869. Each counts as seen now: one still listed is not sent again, and one that
  1870. is gone is forgotten after the usual grace period.
  1871. """
  1872. from backend.app.services.hms_notified import load_notified_hms_faults
  1873. try:
  1874. loaded = await load_notified_hms_faults()
  1875. except Exception as e:
  1876. logging.getLogger(__name__).warning("Could not load the notified HMS faults: %s", e)
  1877. return
  1878. now = time.time()
  1879. for printer_id, keys in loaded.items():
  1880. seen = _notified_hms_errors.setdefault(printer_id, {})
  1881. for key in keys:
  1882. seen.setdefault(key, now)
  1883. # `stg_cur` values that mean the printer is not in a preparation stage: 0 is
  1884. # "Printing", -1 and 255 are "no stage" (#3211).
  1885. _NO_PREPARATION_STAGES = frozenset({0, -1, 255})
  1886. def _progress_milestone_to_notify(printer_id: int, state: PrinterState) -> int | None:
  1887. """The milestone (25, 50 or 75) this status update reaches for the first time, if any.
  1888. Records it as notified. Called only for a printing state with progress above 0.
  1889. Progress before the first layer is not progress through the print (#3211).
  1890. An A1 mini reports ``mc_percent`` 85 in the first frame of a print, while
  1891. still preheating the bed at layer 0, then 3, 7, 40 and 44 through its
  1892. calibration, and starts layer 1 at 45. Read as progress, that 85 sent the
  1893. 75% notification together with Print Started, and since 75 was then on
  1894. record, 25 and 50 could never fire. So nothing counts while the printer
  1895. reports a layer count but has not started layer 1.
  1896. Without a layer count -- a print whose first frame carried no
  1897. ``total_layer_num``, until the pushall that asks for it is answered
  1898. (#2702) -- the preparation stage says the same thing: before layer 1 the
  1899. A1 mini reported ``stg_cur`` 2, 4, 14 and 1 (bed preheating, calibration,
  1900. homing), and 0 from layer 1 on. -1 and 255 are "no stage". A printer that
  1901. reports neither keeps the old behaviour rather than never notifying.
  1902. """
  1903. if (state.layer_num or 0) < 1:
  1904. if (state.total_layers or 0) > 0:
  1905. return None
  1906. if state.stg_cur not in _NO_PREPARATION_STAGES:
  1907. return None
  1908. progress = state.progress or 0
  1909. current_milestone = 0
  1910. if progress >= 75:
  1911. current_milestone = 75
  1912. elif progress >= 50:
  1913. current_milestone = 50
  1914. elif progress >= 25:
  1915. current_milestone = 25
  1916. if current_milestone > _last_progress_milestone.get(printer_id, 0):
  1917. _last_progress_milestone[printer_id] = current_milestone
  1918. return current_milestone
  1919. return None
  1920. async def on_printer_status_change(printer_id: int, state: PrinterState):
  1921. """Handle printer status changes - broadcast via WebSocket."""
  1922. try:
  1923. notify_live_activities.observe(printer_id, state)
  1924. except Exception:
  1925. logging.getLogger(__name__).exception("Notify Live Activity status hook failed for printer %s", printer_id)
  1926. try:
  1927. pause_eta.observe(printer_id, state)
  1928. except Exception:
  1929. logging.getLogger(__name__).exception("Pause heads-up check failed for printer %s", printer_id)
  1930. wled_manager.handle_status(
  1931. printer_id,
  1932. state,
  1933. awaiting_plate_clear=printer_manager.is_awaiting_plate_clear(printer_id),
  1934. )
  1935. # Connected-edge reconciliation (#1542 follow-up). When the printer
  1936. # transitions disconnected → connected — which covers both Bambuddy
  1937. # startup (no prior connection) and a mid-session MQTT reconnect — fire
  1938. # `reconcile_stale_active_prints` exactly once for this connection so
  1939. # any archive still in `status="printing"` that can't actually be
  1940. # running anymore (printer IDLE / different subtask / empty subtask)
  1941. # gets a synthesised PRINT COMPLETE. Without this, a print that
  1942. # finished during a disconnect window + a smart-plug power cycle
  1943. # leaves the .3mf on the SD card and the firmware ghost-replays it on
  1944. # next boot. Reconciliation runs concurrently — it must not block the
  1945. # WebSocket dedup / broadcast logic below, and the connected edge is
  1946. # marked True BEFORE the await so concurrent status updates inside
  1947. # the same connection don't re-trigger reconciliation.
  1948. #
  1949. # Wait for a real push_status before reconciling (#1679): MQTT
  1950. # `_on_connect` broadcasts `state` IMMEDIATELY after the broker accepts
  1951. # the connection, BEFORE `_request_push_all` round-trips. At that
  1952. # instant the `PrinterState` is still on construction defaults — most
  1953. # importantly `state.state == "unknown"` and `state.subtask_name == ""`.
  1954. # If reconcile spawns here, every in-flight archive falls through to
  1955. # the empty-subtask_name trigger and gets synthesised `aborted`, which
  1956. # creates a duplicate archive on the real PRINT COMPLETE and
  1957. # double-counts filament. Gating on `state.state ∉ ("", "unknown")`
  1958. # keeps the #1542 mechanism intact: once the first real push_status
  1959. # updates `state.state` (RUNNING / IDLE / FINISH / …), this handler
  1960. # fires again with the flag still False — reconcile then runs against
  1961. # actual evidence.
  1962. state_known = bool(state.state) and state.state.upper() not in ("", "UNKNOWN")
  1963. if state.connected and state_known and not _printer_reconciled_since_connect.get(printer_id, False):
  1964. _printer_reconciled_since_connect[printer_id] = True
  1965. spawn_background_task(
  1966. reconcile_stale_active_prints(printer_id),
  1967. name=f"reconcile-stale-prints-{printer_id}",
  1968. )
  1969. elif not state.connected and _printer_reconciled_since_connect.get(printer_id, False):
  1970. # Re-arm so the next reconnect triggers reconciliation again.
  1971. _printer_reconciled_since_connect[printer_id] = False
  1972. # Same edge, for the calibration table the AMS card reads its K values from.
  1973. #
  1974. # Also gated on knowing a nozzle diameter, which is what decides *which*
  1975. # tables to ask for. A `state_known` gate alone is not enough: the first
  1976. # real push_status is what makes the state known, and the nozzle fields do
  1977. # not always arrive in it. Latching there would spend this connection's one
  1978. # attempt on a printer that could not yet say what was fitted.
  1979. nozzle_known = any(n.nozzle_diameter for n in (state.nozzles or []))
  1980. if (
  1981. state.connected
  1982. and state_known
  1983. and nozzle_known
  1984. and not _printer_kprofiles_primed_since_connect.get(printer_id, False)
  1985. ):
  1986. _printer_kprofiles_primed_since_connect[printer_id] = True
  1987. spawn_background_task(
  1988. prime_kprofile_table(printer_id),
  1989. name=f"prime-kprofiles-{printer_id}",
  1990. )
  1991. elif not state.connected and _printer_kprofiles_primed_since_connect.get(printer_id, False):
  1992. _printer_kprofiles_primed_since_connect[printer_id] = False
  1993. # A slot can lose its K-profile selection (cali_idx back to -1) with nothing
  1994. # else in the AMS report changing -- a power cycle does it to every slot --
  1995. # and on_ams_change never hears of it (#3219). Checked here, on every push
  1996. # while the printer is idle; needs_check is cheap and throttled per slot.
  1997. # Guarded: nothing here may stop the status broadcast below.
  1998. try:
  1999. if not state.connected:
  2000. kprofile_drift.forget_printer(printer_id)
  2001. elif kprofile_drift.needs_check(printer_id, state):
  2002. spawn_background_task(
  2003. kprofile_drift.reapply_lost_kprofiles(printer_id),
  2004. name=f"reapply-kprofiles-{printer_id}",
  2005. )
  2006. except Exception:
  2007. logging.getLogger(__name__).exception("[Printer %s] K-profile check failed", printer_id)
  2008. # Offline-notification edge (#1752): schedule `on_printer_offline` on
  2009. # connected → disconnected. The "back online" channel is already covered
  2010. # by the print-failure notification (firmware reports gcode_state=FAILED
  2011. # on reconnect of an interrupted print), so there is no online
  2012. # notification. The MQTT relay gets both: offline after the debounce, and
  2013. # online on the reconnect that follows a published offline.
  2014. prev_connected = _printer_last_connected.get(printer_id)
  2015. _printer_last_connected[printer_id] = state.connected
  2016. if prev_connected is True and not state.connected:
  2017. existing = _printer_offline_notify_tasks.get(printer_id)
  2018. if existing is None or existing.done():
  2019. logging.getLogger(__name__).info(
  2020. "[#1752] Printer %s connected→disconnected edge; scheduling offline notification in %.0fs",
  2021. printer_id,
  2022. _PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS,
  2023. )
  2024. _printer_offline_notify_tasks[printer_id] = asyncio.create_task(
  2025. _maybe_notify_printer_offline(printer_id),
  2026. name=f"printer-offline-notify-{printer_id}",
  2027. )
  2028. elif state.connected:
  2029. if prev_connected is False:
  2030. # A printer back from a dropped connection, often a restart, may
  2031. # take files again: don't hold it to an upload backoff from
  2032. # before (#3563). Not on Bambuddy's own start (no previous state).
  2033. client = printer_manager.get_client(printer_id)
  2034. print_scheduler.printer_reconnected(printer_id, client.ip_address if client else None)
  2035. # Kept until the relay can actually send online: with its broker down
  2036. # now, the next status update after the reconnect sends it.
  2037. if printer_id in _printer_offline_published and mqtt_relay.enabled and mqtt_relay.connected:
  2038. _printer_offline_published.discard(printer_id)
  2039. printer_info = printer_manager.get_printer(printer_id)
  2040. if printer_info:
  2041. try:
  2042. await mqtt_relay.on_printer_online(printer_id, printer_info.name, printer_info.serial_number)
  2043. except Exception as e:
  2044. logging.getLogger(__name__).warning(
  2045. "MQTT relay online event failed for printer %s: %s", printer_id, e
  2046. )
  2047. pending = _printer_offline_notify_tasks.pop(printer_id, None)
  2048. if pending is not None and not pending.done():
  2049. logging.getLogger(__name__).info(
  2050. "[#1752] Printer %s reconnected before debounce; cancelling pending offline notification",
  2051. printer_id,
  2052. )
  2053. pending.cancel()
  2054. # Only broadcast if something meaningful changed (reduce WebSocket spam)
  2055. # Include rounded temperatures to detect meaningful temp changes (within 1 degree)
  2056. temps = state.temperatures or {}
  2057. nozzle_temp = round(temps.get("nozzle", 0))
  2058. bed_temp = round(temps.get("bed", 0))
  2059. nozzle_2_temp = round(temps.get("nozzle_2", 0)) if "nozzle_2" in temps else ""
  2060. chamber_temp = round(temps.get("chamber", 0)) if "chamber" in temps else ""
  2061. # Auto-detect dual-nozzle printers from MQTT temperature data
  2062. if "nozzle_2" in temps and printer_id not in _nozzle_count_updated:
  2063. _nozzle_count_updated.add(printer_id)
  2064. # Update nozzle_count in database
  2065. async with async_session() as db:
  2066. from backend.app.models.printer import Printer
  2067. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2068. printer = result.scalar_one_or_none()
  2069. if printer and printer.nozzle_count != 2:
  2070. printer.nozzle_count = 2
  2071. await db.commit()
  2072. logging.getLogger(__name__).info(
  2073. f"Auto-detected dual-nozzle printer {printer_id}, updated nozzle_count=2"
  2074. )
  2075. # Include target temps for heating phase detection
  2076. bed_target = round(temps.get("bed_target", 0))
  2077. nozzle_target = round(temps.get("nozzle_target", 0))
  2078. # Include tray_now and vt_tray hash so external spool changes trigger broadcasts
  2079. vt_tray_key = hash(str(state.raw_data.get("vt_tray", []))) if state.raw_data else 0
  2080. # Include AMS dry_time and tray state values so drying/slot changes trigger broadcasts.
  2081. #
  2082. # dry_countdown_stalled rides along because it is the one drying signal the
  2083. # countdown itself cannot carry: the MQTT layer raises it precisely BECAUSE
  2084. # dry_time stopped moving, so on the frame that flips it every other member
  2085. # of this key is identical and the push would be deduplicated away. Mid-print
  2086. # a temperature would eventually break the tie, but a parked command on an
  2087. # idle machine changes nothing else at all — AMS temp and humidity are not in
  2088. # the key — so the badge could sit unreachable indefinitely. The flag flips at
  2089. # most once per drying cycle, so it costs no mid-print broadcast traffic.
  2090. ams_dry_key = (
  2091. tuple((a.get("dry_time", 0), bool(a.get("dry_countdown_stalled"))) for a in (state.raw_data.get("ams") or []))
  2092. if state.raw_data
  2093. else ()
  2094. )
  2095. # Include tray states so load/unload transitions (state 11→10) trigger broadcasts (#784)
  2096. #
  2097. # The filament identity fields are here because Configure Slot writes
  2098. # exactly those and nothing else. Re-configuring a slot from PLA to another
  2099. # brand or colour of PLA leaves id/tray_type/state identical, so the key
  2100. # matched, this function returned before broadcasting, and the card kept
  2101. # showing the old filament until the 30s fallback poll or a page reload —
  2102. # even though the configure route asks the printer for a fresh pushall and
  2103. # that push does carry the new values. Reset always worked, because it
  2104. # clears tray_type.
  2105. #
  2106. # These fields only change when someone configures a slot or swaps a spool,
  2107. # so unlike temperature or progress they add no broadcast traffic mid-print.
  2108. ams_tray_key = (
  2109. tuple(
  2110. (
  2111. t.get("id"),
  2112. t.get("tray_type", ""),
  2113. t.get("state"),
  2114. t.get("tray_color", ""),
  2115. t.get("tray_info_idx", ""),
  2116. t.get("tray_sub_brands", ""),
  2117. t.get("cali_idx"),
  2118. )
  2119. for a in (state.raw_data.get("ams") or [])
  2120. for t in a.get("tray", [])
  2121. )
  2122. if state.raw_data
  2123. else ()
  2124. )
  2125. # Filament Track Switch: which inlet each AMS is bound to, and whether the
  2126. # accessory is fitted at all. Neither is in ams_tray_key (it is per-tray) nor
  2127. # in the AMS change-hash (tray fields only, and widening that would fire
  2128. # spurious Spoolman syncs), so without them a "Join IN-B" on the printer
  2129. # screen changed no key at all and the card's inlet badges sat stale until a
  2130. # reload. Like the filament-backup flag, these only move when someone
  2131. # reconfigures the machine, so they add no mid-print broadcast traffic.
  2132. fts_key = (
  2133. state.fila_switch.installed if state.fila_switch else False,
  2134. tuple(sorted(state.ams_switch_inlet.items())),
  2135. # Which hotend holds which slot. Unlike the two above this does move
  2136. # mid-print, on every filament change — but only between discrete slots,
  2137. # so it adds a push per toolchange, not a stream. The AMS slot menu needs
  2138. # it live: it decides which hotend the Load dialog may offer and whether
  2139. # Unload has anything to act on.
  2140. tuple(
  2141. sorted(
  2142. ((ext, slot.ams_id, slot.slot_id, slot.has_filament) for ext, slot in state.extruder_slots.items()),
  2143. # Sort on the extruder id alone: the other members are nullable
  2144. # and comparing None with an int raises.
  2145. key=lambda entry: entry[0],
  2146. )
  2147. ),
  2148. )
  2149. status_key = (
  2150. f"{state.connected}:{state.state}:{state.progress}:{state.layer_num}:"
  2151. f"{nozzle_temp}:{bed_temp}:{nozzle_2_temp}:{chamber_temp}:"
  2152. f"{state.stg_cur}:{bed_target}:{nozzle_target}:"
  2153. f"{state.cooling_fan_speed}:{state.big_fan1_speed}:{state.big_fan2_speed}:"
  2154. f"{state.chamber_light}:{state.active_extruder}:{state.tray_now}:{vt_tray_key}:"
  2155. f"{ams_dry_key}:{ams_tray_key}:{state.door_open}:{state.ams_filament_backup}:{fts_key}"
  2156. )
  2157. # Fields that change only when the machine does (a nozzle swap, a firmware
  2158. # update, developer mode) or once per print (its AMS mapping): no traffic,
  2159. # but on an idle printer nothing else moves the key when they change.
  2160. machine_key = (
  2161. tuple((n.nozzle_type, n.nozzle_diameter) for n in (state.nozzles or [])),
  2162. state.firmware_version,
  2163. state.developer_mode,
  2164. tuple(state.print_ams_mapping) if isinstance(state.print_ams_mapping, list) else None,
  2165. # Cooling/heating, switched on the printer's screen too: on an idle
  2166. # printer nothing else moves the key (PSF-4).
  2167. state.airduct_mode,
  2168. )
  2169. status_key = f"{status_key}:{machine_key}"
  2170. is_active_print = state.state in _ACTIVE_PRINT_STATES
  2171. if not is_active_print:
  2172. _unauthorized_print_kill_sent.discard(printer_id)
  2173. elif printer_id in _unauthorized_print_kill_sent:
  2174. # stop_print() was already sent for this print; avoid all further
  2175. # ownership and settings work until the printer leaves an active state.
  2176. pass
  2177. elif _is_bambuddy_authorized_print_in_memory(printer_id, state):
  2178. # Normal Bambuddy-started prints stay entirely on the in-memory path.
  2179. _unauthorized_print_kill_sent.discard(printer_id)
  2180. else:
  2181. kill_switch_enabled = False
  2182. authorization: bool | None = None
  2183. status_logger = logging.getLogger(__name__)
  2184. try:
  2185. kill_switch_enabled = await _is_printer_kill_switch_enabled_cached()
  2186. if kill_switch_enabled:
  2187. async with async_session() as db:
  2188. authorization = await _is_bambuddy_authorized_print(printer_id, state, db)
  2189. except Exception as e:
  2190. # Fail safe: a database/reconciliation error must never turn into an
  2191. # irreversible stop of a print whose ownership is still unknown.
  2192. authorization = None
  2193. status_logger.warning(
  2194. "[KILL SWITCH] Failed to reconcile print authorization for printer %s: %s", printer_id, e
  2195. )
  2196. if not kill_switch_enabled or authorization is True:
  2197. _unauthorized_print_kill_sent.discard(printer_id)
  2198. elif authorization is None:
  2199. _unauthorized_print_kill_sent.discard(printer_id)
  2200. status_logger.debug(
  2201. "[KILL SWITCH] Deferring authorization for printer %s until archive state is reconciled",
  2202. printer_id,
  2203. )
  2204. else:
  2205. try:
  2206. stopped = printer_manager.stop_print(printer_id)
  2207. if stopped:
  2208. _unauthorized_print_kill_sent.add(printer_id)
  2209. _kill_switch_stopped.add(printer_id)
  2210. printer_info = printer_manager.get_printer(printer_id)
  2211. printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  2212. filename = state.subtask_name or state.gcode_file or state.current_print or "Unknown"
  2213. notification_data = {
  2214. "status": "stopped",
  2215. "filename": state.gcode_file or state.current_print or "",
  2216. "subtask_name": state.subtask_name or "",
  2217. "progress": state.progress,
  2218. "reason": "unauthorized_print",
  2219. }
  2220. status_logger.warning(
  2221. "[KILL SWITCH] Stopped unauthorized print on printer %s (state=%s)",
  2222. printer_id,
  2223. state.state,
  2224. )
  2225. try:
  2226. await ws_manager.broadcast(
  2227. {
  2228. "type": "kill_switch_triggered",
  2229. "printer_id": printer_id,
  2230. "printer_name": printer_name,
  2231. "filename": filename,
  2232. "reason": "unauthorized_print",
  2233. }
  2234. )
  2235. except Exception as e:
  2236. status_logger.warning(
  2237. "[KILL SWITCH] WebSocket notification failed for printer %s: %s", printer_id, e
  2238. )
  2239. previous_task = _kill_switch_notification_tasks.pop(printer_id, None)
  2240. if previous_task is not None and not previous_task.done():
  2241. previous_task.cancel()
  2242. _kill_switch_notification_tasks[printer_id] = spawn_background_task(
  2243. _send_kill_switch_provider_notification(printer_id, printer_name, notification_data),
  2244. name=f"kill-switch-notification-{printer_id}",
  2245. )
  2246. else:
  2247. status_logger.warning(
  2248. "[KILL SWITCH] Could not stop unauthorized print on printer %s (state=%s)",
  2249. printer_id,
  2250. state.state,
  2251. )
  2252. except Exception as e:
  2253. status_logger.warning(
  2254. "[KILL SWITCH] Failed to stop unauthorized print on printer %s: %s", printer_id, e
  2255. )
  2256. # MQTT relay - publish status (before dedup check - always publish to MQTT)
  2257. try:
  2258. printer_info = printer_manager.get_printer(printer_id)
  2259. if printer_info:
  2260. await mqtt_relay.on_printer_status(
  2261. printer_id,
  2262. state,
  2263. printer_info.name,
  2264. printer_info.serial_number,
  2265. printer_manager.is_awaiting_plate_clear(printer_id),
  2266. )
  2267. except Exception:
  2268. pass # Don't fail status callback if MQTT fails
  2269. if _last_status_broadcast.get(printer_id) == status_key:
  2270. return # No change, skip WebSocket broadcast
  2271. _last_status_broadcast[printer_id] = status_key
  2272. # Check for progress milestone notifications (25%, 50%, 75%)
  2273. progress = state.progress or 0
  2274. is_printing = state.state in ("RUNNING", "PRINTING")
  2275. if is_printing and progress > 0:
  2276. current_milestone = _progress_milestone_to_notify(printer_id, state)
  2277. if current_milestone is not None:
  2278. try:
  2279. from backend.app.models.printer import Printer
  2280. # Read the printer in a short session and release the connection
  2281. # BEFORE the ~15s camera snapshot below — holding it across the grab
  2282. # pinned a pooled connection per milestone, per printer (issue #2572).
  2283. async with async_session() as db:
  2284. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2285. printer = result.scalar_one_or_none()
  2286. printer_name = printer.name if printer else f"Printer {printer_id}"
  2287. filename = state.subtask_name or state.gcode_file or "Unknown"
  2288. # remaining_time is in minutes, convert to seconds for notification
  2289. remaining_time_seconds = state.remaining_time * 60 if state.remaining_time else None
  2290. # Capture camera snapshot for notification image attachment (no DB held).
  2291. image_data = await _capture_snapshot_for_notification(printer_id, printer, logging.getLogger(__name__))
  2292. # Notification send needs a session (provider/template lookups).
  2293. async with async_session() as db:
  2294. await notification_service.on_print_progress(
  2295. printer_id,
  2296. printer_name,
  2297. filename,
  2298. current_milestone,
  2299. db,
  2300. remaining_time_seconds,
  2301. image_data=image_data,
  2302. )
  2303. except Exception as e:
  2304. logging.getLogger(__name__).warning(f"Progress milestone notification failed: {e}")
  2305. elif progress < 5:
  2306. # Reset milestone tracking when print restarts or new print begins
  2307. _last_progress_milestone[printer_id] = 0
  2308. _first_layer_notified[printer_id] = False
  2309. # HMS error codes that should not trigger notifications even though they
  2310. # have known descriptions (e.g. user-initiated actions, not real errors).
  2311. _HMS_NOTIFICATION_SUPPRESS = {
  2312. "0500_400E", # Printing was cancelled (user action, not an error)
  2313. }
  2314. # Check for new HMS errors and send notifications
  2315. current_hms_errors = getattr(state, "hms_errors", []) or []
  2316. if current_hms_errors:
  2317. new_errors = await _take_new_hms_faults_and_save(printer_id, current_hms_errors, state.state)
  2318. if new_errors:
  2319. try:
  2320. from backend.app.models.printer import Printer
  2321. # Read the printer in a short session and release the connection
  2322. # BEFORE the ~15s camera snapshot below (issue #2572).
  2323. async with async_session() as db:
  2324. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2325. printer = result.scalar_one_or_none()
  2326. printer_name = printer.name if printer else f"Printer {printer_id}"
  2327. # Capture camera snapshot once for all error notifications (no DB held).
  2328. error_image_data = await _capture_snapshot_for_notification(
  2329. printer_id, printer, logging.getLogger(__name__)
  2330. )
  2331. # Notification sends need a session (provider/template lookups).
  2332. async with async_session() as db:
  2333. sent_count = 0
  2334. for error in new_errors:
  2335. # Build short code like "0700_8010"
  2336. # Mask to 16 bits to handle printers that send larger values
  2337. error_code_int = int(error.code.replace("0x", ""), 16) if error.code else 0
  2338. error_code_masked = error_code_int & 0xFFFF
  2339. short_code = f"{(error.attr >> 16) & 0xFFFF:04X}_{error_code_masked:04X}"
  2340. # Only notify for errors Bambu publishes text for — printers
  2341. # send undocumented codes that aren't real errors, and Bambu
  2342. # lists some with empty text. Resolved at parse time (#2926);
  2343. # short_code is still needed for the suppression set below.
  2344. description = error.description
  2345. if not description or short_code in _HMS_NOTIFICATION_SUPPRESS:
  2346. continue
  2347. error_type = _hms_error_title(error.module)
  2348. error_detail = description
  2349. await notification_service.on_printer_error(
  2350. printer_id, printer_name, error_type, db, error_detail, image_data=error_image_data
  2351. )
  2352. sent_count += 1
  2353. if sent_count:
  2354. logging.getLogger(__name__).info(
  2355. f"[HMS] Sent notification for {sent_count} error(s) on printer {printer_id}"
  2356. )
  2357. # Also publish to MQTT relay (no DB): every new fault that
  2358. # counts, with or without text, the same set the UI counts.
  2359. printer_info = printer_manager.get_printer(printer_id)
  2360. if printer_info:
  2361. errors_data = [
  2362. {
  2363. "code": e.code,
  2364. "attr": e.attr,
  2365. "module": e.module,
  2366. "severity": e.severity,
  2367. }
  2368. for e in new_errors
  2369. ]
  2370. await mqtt_relay.on_printer_error(
  2371. printer_id, printer_info.name, printer_info.serial_number, errors_data
  2372. )
  2373. except Exception as e:
  2374. logging.getLogger(__name__).warning(f"HMS error notification failed: {e}")
  2375. else:
  2376. # No HMS errors: nothing to send, but faults gone long enough (or gone
  2377. # across a print state change) are forgotten. Some codes, e.g. chamber
  2378. # temperature regulation during PETG prints, toggle every few seconds.
  2379. await _take_new_hms_faults_and_save(printer_id, [], state.state)
  2380. await ws_manager.send_printer_status(
  2381. printer_id,
  2382. printer_state_to_dict(
  2383. state,
  2384. printer_id,
  2385. printer_manager.get_model(printer_id),
  2386. printer_manager.get_drying_targets(printer_id),
  2387. ),
  2388. )
  2389. def _is_bambu_uuid(tray_uuid: str) -> bool:
  2390. """Check if a tray UUID looks like a valid Bambu Lab RFID UUID (non-empty, non-zero)."""
  2391. return bool(tray_uuid) and tray_uuid not in ("", "0" * len(tray_uuid))
  2392. async def on_fts_inlet_change(printer_id: int, ams_id: int, inlet: str):
  2393. """Re-point a moved AMS's K-profiles at the nozzle it now feeds.
  2394. K-profiles are per-nozzle and the printer's calibration table is numbered
  2395. per-nozzle, but a tray holds exactly one ``cali_idx``. Moving an AMS to the
  2396. switch's other inlet therefore silently invalidates every configured slot in
  2397. it: the index stays put and now resolves against the other nozzle's table.
  2398. Measured on the maintainer's H2C — one spool calibrated 0.018 on the left
  2399. and 0.020 on the right kept the left profile after the move, and a manual
  2400. RFID re-read only re-asserted the same wrong one.
  2401. Configuring a slot is a deliberate preparation step, so this re-selects
  2402. rather than re-configures: only the calibration binding moves, and only for
  2403. slots whose spool already has a stored profile for the new nozzle. A slot
  2404. Bambuddy knows nothing about is left exactly as the operator left it.
  2405. """
  2406. logger = logging.getLogger(__name__)
  2407. target_extruder = extruder_for_inlet(inlet)
  2408. if target_extruder is None:
  2409. return
  2410. client = printer_manager.get_client(printer_id)
  2411. state = printer_manager.get_status(printer_id)
  2412. if not client or not state or not state.raw_data:
  2413. return
  2414. # The nozzle the AMS now feeds -- the diameter of the TARGET extruder, not
  2415. # of nozzle 0. On a machine with two sizes fitted, moving the inlet changes
  2416. # the nozzle width, which changes both the K profile to select and the
  2417. # preset the slot should carry.
  2418. nozzle_diameter = nozzle_diameter_for_extruder(state, target_extruder, printer_manager.get_model(printer_id))
  2419. ams_raw = state.raw_data.get("ams")
  2420. ams_list = ams_raw.get("ams", []) if isinstance(ams_raw, dict) else ams_raw if isinstance(ams_raw, list) else []
  2421. unit = next((u for u in ams_list if str(u.get("id")) == str(ams_id)), None)
  2422. if not unit:
  2423. return
  2424. try:
  2425. async with async_session() as db:
  2426. for tray in unit.get("tray", []):
  2427. tray_id = int(tray.get("id", -1))
  2428. if tray_id < 0 or not tray.get("tray_type"):
  2429. continue
  2430. current_idx = tray.get("cali_idx")
  2431. profile = await find_slot_kprofile_for_extruder(
  2432. db,
  2433. printer_id,
  2434. ams_id,
  2435. tray_id,
  2436. target_extruder,
  2437. nozzle_diameter,
  2438. printer_manager.get_model(printer_id),
  2439. nozzle_flow_for_extruder(state, target_extruder, printer_manager.get_model(printer_id)),
  2440. )
  2441. if profile is None or profile.cali_idx is None:
  2442. continue
  2443. if current_idx == profile.cali_idx:
  2444. continue # Already on the right one.
  2445. logger.info(
  2446. "[Printer %s] AMS %s slot %s moved to inlet %s (nozzle %s): "
  2447. "re-selecting K-profile %s (cali_idx %s -> %s, K=%s)",
  2448. printer_id,
  2449. ams_id,
  2450. tray_id,
  2451. inlet,
  2452. target_extruder,
  2453. profile.name,
  2454. current_idx,
  2455. profile.cali_idx,
  2456. profile.k_value,
  2457. )
  2458. client.extrusion_cali_sel(
  2459. ams_id=ams_id,
  2460. tray_id=tray_id,
  2461. cali_idx=profile.cali_idx,
  2462. filament_id=printer_safe_filament_id(profile.filament_id, tray.get("tray_info_idx", "")),
  2463. nozzle_diameter=nozzle_diameter,
  2464. )
  2465. except Exception as e:
  2466. logger.warning("[Printer %s] Could not re-apply K-profiles after inlet move: %s", printer_id, e)
  2467. def _slot_feeds_print(printer_id: int, ams_id: int, tray_id: int) -> bool:
  2468. """Is this slot the one feeding the printer right now?
  2469. ``tray_now`` names one slot for the whole printer, so on a dual-nozzle
  2470. machine each extruder's own fed slot is asked as well. Only meaningful
  2471. while a print runs; the caller checks that.
  2472. """
  2473. status = printer_manager.get_status(printer_id)
  2474. if status is None:
  2475. return False
  2476. global_id = printer_tray_layout(printer_id, printer_manager).tray_from_slot_key(ams_id, tray_id)
  2477. if getattr(status, "tray_now", None) == global_id:
  2478. return True
  2479. slots = getattr(status, "extruder_slots", None) or {}
  2480. return any(slot.has_filament and slot.holds(ams_id, tray_id) for slot in slots.values())
  2481. def _tray_reports_no_filament(color: str, tray_type: str) -> bool:
  2482. """Does this tray report carry no filament data at all?
  2483. No type, and a colour that is either empty or ``00000000``: firmware sends
  2484. both for a blank slot, and ``00000000`` is what Bambuddy's own slot reset
  2485. writes. A reconnect mid-print brought it in that form and got past every
  2486. guard that only knew the empty string (#3526). A clear spool also reports
  2487. ``00000000``, but always with a type, so it is not blank.
  2488. """
  2489. return not tray_type.strip() and color.strip().upper() in ("", "00000000")
  2490. async def _unlink_stale_assignments(printer_id: int, ams_data: list, printing_now: bool) -> None:
  2491. """Unlink built-in inventory assignments whose slot no longer holds their spool.
  2492. Runs from ``on_ams_change`` and, for removals held by ``slot_unlink_grace``,
  2493. from the delayed re-check -- which is why it takes the AMS data and print
  2494. state as arguments instead of reading them from the push (#3186).
  2495. """
  2496. logger = logging.getLogger(__name__)
  2497. from backend.app.utils.color_utils import colors_similar as _colors_similar
  2498. # Auto-unlink spool assignments with stale fingerprints. Under the
  2499. # per-printer assignment lock since #3186: the held-removal re-check runs
  2500. # this outside any MQTT push, so it can now overlap one.
  2501. try:
  2502. async with get_ams_assignment_lock(printer_id), async_session() as db:
  2503. from sqlalchemy.orm import selectinload
  2504. from backend.app.api.routes.inventory import _find_tray_in_ams_data
  2505. from backend.app.models.spool import Spool as _Spool
  2506. from backend.app.models.spool_assignment import SpoolAssignment as SA
  2507. from backend.app.services.ams_slot_presence import spool_present
  2508. from backend.app.services.inventory_mode import spoolman_owns_assignments
  2509. # Built-in assignments only. Since #2812 they survive a switch to
  2510. # Spoolman mode rather than being deleted by it, and this pass ends
  2511. # in ``db.delete`` — left ungated it would unlink them one slot at a
  2512. # time as the AMS contents changed under the other mode, undoing the
  2513. # preservation more slowly but just as completely.
  2514. assignments = []
  2515. if not await spoolman_owns_assignments(db):
  2516. result = await db.execute(
  2517. select(SA)
  2518. .where(SA.printer_id == printer_id)
  2519. .options(selectinload(SA.spool).selectinload(_Spool.k_profiles))
  2520. )
  2521. assignments = result.scalars().all()
  2522. # ``printing_now`` (top of this function) keeps a runout from
  2523. # unlinking the spool that fed the print — the next idle-time pass
  2524. # unlinks it if the user really did take it out.
  2525. stale = []
  2526. # Removals this pass is holding rather than unlinking (#3186).
  2527. held: set[tuple] = set()
  2528. for assignment in assignments:
  2529. # External spool assignments (ams_id=255) live in vt_tray, not AMS data
  2530. if assignment.ams_id == 255:
  2531. ps = printer_manager.get_status(printer_id)
  2532. vt_tray_raw = ps.raw_data.get("vt_tray", []) if ps else []
  2533. ext_id = external_tray_id(assignment.tray_id) # 0→254, 1→255, rack 2-8→247-253
  2534. current_tray = None
  2535. for vt in vt_tray_raw:
  2536. if isinstance(vt, dict) and int(vt.get("id", 254)) == ext_id:
  2537. current_tray = vt
  2538. break
  2539. if not current_tray:
  2540. # vt_tray data may not have arrived yet — keep assignment
  2541. continue
  2542. else:
  2543. current_tray = _find_tray_in_ams_data(ams_data, assignment.ams_id, assignment.tray_id)
  2544. if not current_tray:
  2545. if printing_now:
  2546. logger.info(
  2547. "Auto-unlink skipped: spool %d AMS%d-T%d — slot empty during a running print (runout?)",
  2548. assignment.spool_id,
  2549. assignment.ams_id,
  2550. assignment.tray_id,
  2551. )
  2552. continue
  2553. # A whole AMS unit can drop out of one push and come back in
  2554. # the next; only a slot that stays gone is a removal (#3186).
  2555. hold_key = ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id)
  2556. if not slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2557. held.add(hold_key)
  2558. logger.info(
  2559. "Auto-unlink held: spool %d AMS%d-T%d — tray not found in AMS data; "
  2560. "unlinking if it is still gone in %ds",
  2561. assignment.spool_id,
  2562. assignment.ams_id,
  2563. assignment.tray_id,
  2564. int(slot_unlink_grace.GRACE_SECONDS),
  2565. )
  2566. continue
  2567. logger.info(
  2568. "Auto-unlink: spool %d AMS%d-T%d — tray not found in AMS data (slot empty?)",
  2569. assignment.spool_id,
  2570. assignment.ams_id,
  2571. assignment.tray_id,
  2572. )
  2573. stale.append(assignment) # Slot empty
  2574. elif _is_bambu_uuid(current_tray.get("tray_uuid", "")):
  2575. # A Bambu Lab spool is in this slot — check if it's the same spool
  2576. # that's currently assigned. If yes, keep the assignment (avoids
  2577. # unnecessary unlink/re-assign/ams_filament_setting cycle that clears
  2578. # the printer's filament preset on every startup).
  2579. tray_uuid = current_tray.get("tray_uuid", "")
  2580. tag_uid = current_tray.get("tag_uid", "")
  2581. spool = assignment.spool
  2582. spool_matches = False
  2583. if spool:
  2584. if (spool.tray_uuid and spool.tray_uuid.upper() == tray_uuid.upper()) or (
  2585. spool.tag_uid
  2586. and tag_uid
  2587. and tag_uid != "0000000000000000"
  2588. and spool.tag_uid.upper() == tag_uid.upper()
  2589. ):
  2590. spool_matches = True
  2591. if spool_matches:
  2592. # Same BL spool still in slot — keep assignment, update fingerprint if needed
  2593. cur_color = current_tray.get("tray_color", "")
  2594. cur_type = current_tray.get("tray_type", "")
  2595. fp_color = assignment.fingerprint_color or ""
  2596. fp_type = assignment.fingerprint_type or ""
  2597. if cur_color.upper() != fp_color.upper() or cur_type.upper() != fp_type.upper():
  2598. assignment.fingerprint_color = cur_color
  2599. assignment.fingerprint_type = cur_type
  2600. logger.debug(
  2601. "Auto-unlink: spool %d AMS%d-T%d — same BL spool, updated fingerprint",
  2602. assignment.spool_id,
  2603. assignment.ams_id,
  2604. assignment.tray_id,
  2605. )
  2606. continue
  2607. # Different BL spool or unrecognized — unlink so auto-assign can match
  2608. logger.info(
  2609. "Auto-unlink: spool %d AMS%d-T%d — different Bambu Lab spool detected (uuid=%s)",
  2610. assignment.spool_id,
  2611. assignment.ams_id,
  2612. assignment.tray_id,
  2613. tray_uuid,
  2614. )
  2615. stale.append(assignment)
  2616. else:
  2617. cur_color = current_tray.get("tray_color", "")
  2618. cur_type = current_tray.get("tray_type", "")
  2619. cur_state = current_tray.get("state")
  2620. fp_color = assignment.fingerprint_color or ""
  2621. fp_type = assignment.fingerprint_type or ""
  2622. # SpoolBuddy pre-config replay: fingerprint_type empty means
  2623. # the slot was empty when the user pre-assigned via SpoolBuddy
  2624. # (the firmware drops ams_filament_setting on empty slots, so
  2625. # MQTT was deferred). The moment any filament gets inserted
  2626. # — Bambu RFID, 3rd-party, or even an existing-but-now-
  2627. # reconfigured spool — fire the deferred configuration.
  2628. # The "loaded" signal is state == 11 (Bambu's "filament fed to
  2629. # extruder" code) OR, on firmwares that don't use the state
  2630. # enum meaningfully, a non-empty tray_type when state is
  2631. # NOT one of the firmware's explicit empty signals (9, 10).
  2632. # state-only was wrong for firmwares that never set 11 — A1
  2633. # Mini BMCU 01.07.02.00 and P1S Standard AMS 00.00.06.75 both
  2634. # always report state=3 — so the replay never fired for them
  2635. # (#1322). The state ∉ {9,10} guard keeps the firmware's
  2636. # explicit "empty" signals authoritative over any stale
  2637. # tray_type that might survive the relay's auto-clearing.
  2638. #
  2639. # tray_exist_bits comes first because that guard cannot tell
  2640. # a firmware "empty" from Bambuddy's own: apply_tray_exist_bits
  2641. # writes state=9 when the bit is 0 and leaves it there when the
  2642. # bit returns. A non-RFID spool inserted into a pre-assigned
  2643. # slot brings no tray_type with it, so the stale 9 made this
  2644. # expression false forever and the deferred config never fired
  2645. # — the deadlock #1322 removed from the assign path, still in
  2646. # place here (#3084, #3100).
  2647. loaded = spool_present(current_tray) is True or (
  2648. cur_state == 11 or (cur_state not in (9, 10) and cur_type.strip())
  2649. )
  2650. # Not into the slot feeding a running print: the printer refuses
  2651. # the setting for it (``ams_filament_setting refused``), and a
  2652. # spool swapped in after a runout sat through one refused send
  2653. # per AMS change. Held, it fires on the first change once the
  2654. # print is over (#3434).
  2655. if (
  2656. not fp_type.strip()
  2657. and loaded
  2658. and assignment.spool
  2659. and printing_now
  2660. and _slot_feeds_print(printer_id, assignment.ams_id, assignment.tray_id)
  2661. ):
  2662. logger.debug(
  2663. "Pre-config held for spool %d AMS%d-T%d: the slot is feeding the print",
  2664. assignment.spool_id,
  2665. assignment.ams_id,
  2666. assignment.tray_id,
  2667. )
  2668. continue
  2669. if not fp_type.strip() and loaded and assignment.spool:
  2670. # The fingerprint ends this replay. Not recorded when nothing
  2671. # could be sent (no MQTT client while the printer reconnects),
  2672. # so the next AMS change tries again. An attempt that raised is
  2673. # recorded as before: it may have sent part of the commands and
  2674. # would raise again on every change, remain% steps included.
  2675. configured: bool | None = False
  2676. try:
  2677. from backend.app.api.routes.inventory import (
  2678. apply_spool_to_slot_via_mqtt,
  2679. load_preset_lookup_user,
  2680. )
  2681. from backend.app.core.auth import is_auth_enabled
  2682. # Whoever made the assignment: their cloud sign-in
  2683. # resolves a custom preset, which nobody's did here,
  2684. # so the slot got Generic PLA on insert (#3519). With
  2685. # login off the shared sign-in applies, as before.
  2686. replay_user = None
  2687. if assignment.assigned_by_user_id is not None and await is_auth_enabled(db):
  2688. replay_user = await load_preset_lookup_user(db, assignment.assigned_by_user_id)
  2689. configured = await apply_spool_to_slot_via_mqtt(
  2690. db=db,
  2691. current_user=replay_user,
  2692. spool=assignment.spool,
  2693. printer_id=printer_id,
  2694. ams_id=assignment.ams_id,
  2695. tray_id=assignment.tray_id,
  2696. current_tray_info_idx=current_tray.get("tray_info_idx", ""),
  2697. current_tray_type=cur_type,
  2698. current_tray_sub_brands=current_tray.get("tray_sub_brands", ""),
  2699. )
  2700. except Exception:
  2701. logger.exception(
  2702. "Pre-config apply failed for spool %d on printer %d AMS%d-T%d",
  2703. assignment.spool_id,
  2704. printer_id,
  2705. assignment.ams_id,
  2706. assignment.tray_id,
  2707. )
  2708. configured = None
  2709. if configured is False:
  2710. logger.warning(
  2711. "Pre-config not sent for spool %d on printer %d AMS%d-T%d (no MQTT client); "
  2712. "retrying on the next AMS change",
  2713. assignment.spool_id,
  2714. printer_id,
  2715. assignment.ams_id,
  2716. assignment.tray_id,
  2717. )
  2718. else:
  2719. if configured:
  2720. logger.info(
  2721. "SpoolBuddy pre-config applied on insert: spool %d → printer %d AMS%d-T%d",
  2722. assignment.spool_id,
  2723. printer_id,
  2724. assignment.ams_id,
  2725. assignment.tray_id,
  2726. )
  2727. assignment.fingerprint_color = cur_color
  2728. assignment.fingerprint_type = cur_type
  2729. continue
  2730. if not _colors_similar(cur_color, fp_color) or cur_type.upper() != fp_type.upper():
  2731. # Blank tray data mid-print is a runout, not a swap: the
  2732. # firmware clears colour and type when it unloads a spool
  2733. # it just emptied. Unlinking here would erase the record
  2734. # of which spool fed the print so far.
  2735. if printing_now and _tray_reports_no_filament(cur_color, cur_type):
  2736. logger.info(
  2737. "Auto-unlink skipped: spool %d AMS%d-T%d — tray data cleared during a running print "
  2738. "(runout?)",
  2739. assignment.spool_id,
  2740. assignment.ams_id,
  2741. assignment.tray_id,
  2742. )
  2743. continue
  2744. # Same reasoning off the print, on firmware's own say-so:
  2745. # a blank tray report from a slot whose tray_exist_bits
  2746. # bit is set describes a spool the AMS cannot identify —
  2747. # a non-RFID one, or one whose slot was reset — not a
  2748. # spool that was taken out. Deleting the assignment there
  2749. # threw away the identity the user had supplied, which is
  2750. # the only place it existed (#3100). A slot the bit calls
  2751. # empty, or one that carries no bit at all, still unlinks.
  2752. #
  2753. # Unless the slot was reported empty first: a removal
  2754. # already held means a spool came out and another went
  2755. # in, so the blank report keeps the hold running below
  2756. # rather than cancelling it (#3186).
  2757. if (
  2758. spool_present(current_tray) is True
  2759. and _tray_reports_no_filament(cur_color, cur_type)
  2760. and not slot_unlink_grace.is_held(
  2761. printer_id,
  2762. ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id),
  2763. )
  2764. ):
  2765. logger.info(
  2766. "Auto-unlink skipped: spool %d AMS%d-T%d — slot still occupied, "
  2767. "tray reports no filament data yet",
  2768. assignment.spool_id,
  2769. assignment.ams_id,
  2770. assignment.tray_id,
  2771. )
  2772. continue
  2773. # A blank report the presence bit does not vouch for is
  2774. # still only one push. An idle X1C cleared a whole AMS
  2775. # unit's bits, colour and type for a moment and lost
  2776. # four saved assignments that way (#3186); unlink only
  2777. # if the slot is still blank after the grace period.
  2778. if _tray_reports_no_filament(cur_color, cur_type):
  2779. hold_key = ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id)
  2780. if not slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2781. held.add(hold_key)
  2782. logger.info(
  2783. "Auto-unlink held: spool %d AMS%d-T%d — tray reports no filament data; "
  2784. "unlinking if it is still blank in %ds",
  2785. assignment.spool_id,
  2786. assignment.ams_id,
  2787. assignment.tray_id,
  2788. int(slot_unlink_grace.GRACE_SECONDS),
  2789. )
  2790. continue
  2791. # Fingerprint mismatch — but check if tray now matches the
  2792. # assigned spool (e.g. auto-configure changed the tray).
  2793. # Both sides are reduced to the type the slot can carry
  2794. # before comparing: the assign path writes that rather
  2795. # than the spool's raw material (#2902), so a spool whose
  2796. # material is a product line — "PLA+", "HTPLA" — reports
  2797. # back as "PLA" and would otherwise fail this check and
  2798. # be auto-unlinked from the slot it was just assigned to.
  2799. # Reducing the printer's side too keeps slots configured
  2800. # by an older Bambuddy, still reporting "PLA+", matching.
  2801. spool = assignment.spool
  2802. if spool:
  2803. spool_color = (spool.rgba or "FFFFFFFF").upper()
  2804. # Two ways the assign path can have arrived at the
  2805. # slot's type, so both count as "we wrote this".
  2806. # The material column is one; the spool's preset is
  2807. # the other, and it outranks the material when the
  2808. # spool has one -- a spool whose material says PLA
  2809. # and whose preset is "Bambu PLA Aero" puts
  2810. # PLA-AERO in the slot (#2902). Read from the stored
  2811. # preset name rather than resolving the preset,
  2812. # because this runs on every AMS push and a cloud
  2813. # lookup here would be both slow and unavailable on
  2814. # the unauthenticated replay path.
  2815. spool_types = {printer_filament_type(spool.material).upper()}
  2816. if spool.slicer_filament_name:
  2817. spool_types.add(printer_filament_type(spool.slicer_filament_name).upper())
  2818. # An imported local preset stores its type outright,
  2819. # which is what the assign path used -- and the name
  2820. # above may be unset. One keyed read, and only on a
  2821. # mismatch, which is rare.
  2822. #
  2823. # slicer_filament is free text up to fifty characters,
  2824. # so the digits have to be checked against the range
  2825. # of the integer primary key they are about to be
  2826. # compared with. Postgres raises on an out-of-range
  2827. # integer rather than simply not matching, and that
  2828. # would poison this session and abandon the rest of
  2829. # the cleanup pass.
  2830. lp_ref = (spool.slicer_filament or "").strip()
  2831. if lp_ref.isdigit() and int(lp_ref) <= 2147483647:
  2832. from backend.app.models.local_preset import LocalPreset as _LP
  2833. lp_type = await db.scalar(select(_LP.filament_type).where(_LP.id == int(lp_ref)))
  2834. if lp_type:
  2835. spool_types.add(printer_filament_type(lp_type).upper())
  2836. # A support spool went in with the slicer's suffix
  2837. # ("PLA-S") that none of the above carries (#3559).
  2838. # A spool made from a support RFID tray has its
  2839. # material as "Support For PLA", so that counts too.
  2840. support_name = (
  2841. f"{spool.slicer_filament_name or ''} {spool.material or ''} {spool.subtype or ''}"
  2842. )
  2843. spool_types |= {
  2844. support_filament_type(t, spool.slicer_filament, support_name).upper()
  2845. for t in spool_types
  2846. }
  2847. if (
  2848. _colors_similar(cur_color, spool_color)
  2849. and printer_filament_type(cur_type).upper() in spool_types
  2850. ):
  2851. logger.info(
  2852. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch but tray matches spool, updating fp",
  2853. assignment.spool_id,
  2854. assignment.ams_id,
  2855. assignment.tray_id,
  2856. )
  2857. assignment.fingerprint_color = cur_color
  2858. assignment.fingerprint_type = cur_type
  2859. continue
  2860. logger.info(
  2861. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch (cur=%s/%s fp=%s/%s spool=%s/%s)",
  2862. assignment.spool_id,
  2863. assignment.ams_id,
  2864. assignment.tray_id,
  2865. cur_color,
  2866. cur_type,
  2867. fp_color,
  2868. fp_type,
  2869. spool.rgba if spool else "?",
  2870. spool.material if spool else "?",
  2871. )
  2872. stale.append(assignment) # Spool changed
  2873. slot_unlink_grace.settle(printer_id, "inventory", held)
  2874. # Snapshot slots before delete — ORM attribute access after the
  2875. # commit would refresh against a deleted row.
  2876. unlinked_slots = [(a.ams_id, a.tray_id) for a in stale]
  2877. for a in stale:
  2878. await db.delete(a)
  2879. if stale:
  2880. logger.info("Auto-unlinked %d stale spool assignments for printer %d", len(stale), printer_id)
  2881. # Commit any changes (stale deletions and/or fingerprint updates)
  2882. await db.commit()
  2883. # Tell open browsers the assignment is gone (#2575). Only the manual
  2884. # REST assign/unassign endpoints broadcast this event; without it the
  2885. # frontend's spool-assignments cache keeps rendering the unlinked
  2886. # spool on the slot until an unrelated refetch — which reads exactly
  2887. # like "the fix didn't work" (reporter verified: a browser refresh
  2888. # after the swap showed the correct state all along).
  2889. for ams_id, tray_id in unlinked_slots:
  2890. await ws_manager.broadcast(
  2891. {
  2892. "type": "spool_assignment_changed",
  2893. "printer_id": printer_id,
  2894. "ams_id": ams_id,
  2895. "tray_id": tray_id,
  2896. }
  2897. )
  2898. except Exception as e:
  2899. logger.warning("Spool assignment cleanup failed: %s", e, exc_info=True)
  2900. async def _expire_spoolman_empty_slots(printer_id: int, ams_data: list, printing_now: bool) -> None:
  2901. """Delete Spoolman slot rows whose held removal has run its grace period.
  2902. The Spoolman half of the #3186 re-check. The full sync in ``on_ams_change``
  2903. talks to Spoolman for every tray; this only needs the local rows, so it
  2904. repeats that pass's empty-slot decision -- a tray with no type or no colour
  2905. (``parse_ams_tray`` returns None for exactly those), not during a print, and
  2906. not in a slot the presence bit calls occupied -- and nothing else.
  2907. """
  2908. logger = logging.getLogger(__name__)
  2909. try:
  2910. async with async_session() as db:
  2911. from backend.app.api.routes.settings import get_setting
  2912. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  2913. from backend.app.services.ams_slot_presence import spool_present
  2914. enabled = await get_setting(db, "spoolman_enabled")
  2915. if not enabled or enabled.lower() != "true":
  2916. return
  2917. sync_mode = await get_setting(db, "spoolman_sync_mode")
  2918. if sync_mode and sync_mode != "auto":
  2919. return
  2920. trays: dict[tuple[int, int], dict] = {}
  2921. for ams_unit in ams_data or []:
  2922. if not isinstance(ams_unit, dict):
  2923. continue
  2924. for tray in ams_unit.get("tray", []):
  2925. if isinstance(tray, dict):
  2926. trays[(int(ams_unit.get("id", 0)), int(tray.get("id", 0)))] = tray
  2927. rows = (
  2928. (
  2929. await db.execute(
  2930. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  2931. )
  2932. )
  2933. .scalars()
  2934. .all()
  2935. )
  2936. held: set[tuple] = set()
  2937. expired: list[tuple[int, int]] = []
  2938. for row in rows:
  2939. tray = trays.get((row.ams_id, row.tray_id))
  2940. if tray is None or printing_now:
  2941. continue
  2942. if (tray.get("tray_type") or "").strip() and (tray.get("tray_color") or "").strip():
  2943. continue
  2944. hold_key = ("spoolman", row.ams_id, row.tray_id, row.spoolman_spool_id)
  2945. if spool_present(tray) is True and not slot_unlink_grace.is_held(printer_id, hold_key):
  2946. continue
  2947. if slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2948. expired.append((row.ams_id, row.tray_id))
  2949. else:
  2950. held.add(hold_key)
  2951. slot_unlink_grace.settle(printer_id, "spoolman", held)
  2952. if not expired:
  2953. return
  2954. # A statement rather than ORM deletes, like the sync pass: the two
  2955. # can overlap, and a row the other already removed must not fail
  2956. # this commit.
  2957. for ams_id, tray_id in expired:
  2958. await db.execute(
  2959. delete(SpoolmanSlotAssignment).where(
  2960. SpoolmanSlotAssignment.printer_id == printer_id,
  2961. SpoolmanSlotAssignment.ams_id == ams_id,
  2962. SpoolmanSlotAssignment.tray_id == tray_id,
  2963. )
  2964. )
  2965. await db.commit()
  2966. logger.info("Unlinked %d Spoolman slot(s) that stayed empty for printer %d", len(expired), printer_id)
  2967. for ams_id, tray_id in expired:
  2968. await ws_manager.broadcast(
  2969. {
  2970. "type": "spool_assignment_changed",
  2971. "printer_id": printer_id,
  2972. "ams_id": ams_id,
  2973. "tray_id": tray_id,
  2974. }
  2975. )
  2976. except Exception as e:
  2977. logger.warning("Spoolman slot re-check failed for printer %s: %s", printer_id, e, exc_info=True)
  2978. async def _recheck_held_unlinks(printer_id: int) -> None:
  2979. """Re-run just the slot cleanup against the printer's current AMS state.
  2980. Scheduled by ``slot_unlink_grace`` when it holds a removal. The unlink
  2981. passes otherwise run only when the AMS hash changes, and a slot that went
  2982. empty and stayed empty may never change it again.
  2983. """
  2984. status = printer_manager.get_status(printer_id)
  2985. # A disconnected printer's state is its last report, not a new one: acting
  2986. # on it would "confirm" a removal nobody has seen for the whole grace
  2987. # period. Leave the holds; the first push after reconnecting decides.
  2988. if status is None or not status.connected:
  2989. return
  2990. ams_raw = status.raw_data.get("ams")
  2991. ams_data = ams_raw.get("ams", []) if isinstance(ams_raw, dict) else ams_raw if isinstance(ams_raw, list) else []
  2992. printing_now = (getattr(status, "state", "") or "").upper() in ("RUNNING", "PAUSE")
  2993. await _unlink_stale_assignments(printer_id, ams_data, printing_now)
  2994. await _expire_spoolman_empty_slots(printer_id, ams_data, printing_now)
  2995. slot_unlink_grace.set_recheck(_recheck_held_unlinks)
  2996. async def on_ams_change(printer_id: int, ams_data: list):
  2997. """Handle AMS data changes - sync to Spoolman if enabled and auto mode."""
  2998. logger = logging.getLogger(__name__)
  2999. # Snapshot BEFORE any await: if a print is active, or a finished one is
  3000. # still being charged, skip weight sync later. on_print_complete may pop
  3001. # _active_sessions during our awaits (#880).
  3002. from backend.app.services.usage_tracker import printer_is_charging
  3003. _print_active = printer_is_charging(printer_id)
  3004. # A slot that reports empty while a print is running is a filament runout,
  3005. # not a spool swap: the spool is still physically in the AMS, just
  3006. # consumed. Dropping either inventory backend's slot link there loses the
  3007. # only record of which spool fed the print, so the completion path can't
  3008. # charge the runout segment to anything. Both cleanup passes below consult
  3009. # this; computed once, up front, so neither depends on the other having run.
  3010. _unlink_state = printer_manager.get_status(printer_id)
  3011. printing_now = (getattr(_unlink_state, "state", "") or "").upper() in ("RUNNING", "PAUSE")
  3012. # The session alone is not enough: a restart that finds a stale persisted
  3013. # session registers none for the print that is running.
  3014. _print_active = _print_active or printing_now
  3015. # MQTT relay - publish AMS change
  3016. try:
  3017. printer_info = printer_manager.get_printer(printer_id)
  3018. if printer_info:
  3019. await mqtt_relay.on_ams_change(printer_id, printer_info.name, printer_info.serial_number, ams_data)
  3020. except Exception:
  3021. pass # Don't fail AMS callback if MQTT fails
  3022. # Broadcast AMS change via WebSocket (bypasses status_key deduplication)
  3023. # This ensures frontend gets immediate updates when AMS slots are configured
  3024. try:
  3025. state = printer_manager.get_status(printer_id)
  3026. if state:
  3027. logger.info("[Printer %s] Broadcasting AMS change via WebSocket", printer_id)
  3028. await ws_manager.send_printer_status(
  3029. printer_id,
  3030. printer_state_to_dict(
  3031. state,
  3032. printer_id,
  3033. printer_manager.get_model(printer_id),
  3034. printer_manager.get_drying_targets(printer_id),
  3035. ),
  3036. )
  3037. except Exception as e:
  3038. logger.warning("Failed to broadcast AMS change for printer %s: %s", printer_id, e)
  3039. await _unlink_stale_assignments(printer_id, ams_data, printing_now)
  3040. # Spools whose tag this pass assigned to a slot; their other slots are
  3041. # released below, once this printer's lock is let go (#3427).
  3042. rfid_assigned: list[tuple[int, tuple[int, int, int]]] = []
  3043. # Auto-manage inventory spools from AMS tray data (skip if Spoolman manages AMS).
  3044. # Serialised per-printer via services/ams_assignment_lock: MQTT bursts can deliver
  3045. # two AMS pushes ~30 ms apart, and without the lock both callbacks read
  3046. # "no existing assignment" for the same (printer, ams, tray) and race to
  3047. # INSERT, hitting the spool_assignment_printer_id_ams_id_tray_id_key
  3048. # unique constraint on Postgres. SQLite's WAL serialises writes so the
  3049. # bug stayed latent there. See services/ams_assignment_lock for details.
  3050. try:
  3051. async with get_ams_assignment_lock(printer_id), async_session() as db:
  3052. from sqlalchemy.orm import selectinload
  3053. from backend.app.api.routes.settings import get_setting
  3054. from backend.app.models.spool import Spool
  3055. from backend.app.models.spool_assignment import SpoolAssignment as SA
  3056. from backend.app.services.spool_tag_matcher import (
  3057. auto_assign_spool,
  3058. create_spool_from_tray,
  3059. find_matching_untagged_spool,
  3060. get_spool_by_tag,
  3061. is_bambu_tag,
  3062. is_valid_tag,
  3063. link_tag_to_inventory_spool,
  3064. tag_belongs_to_archived_spool,
  3065. )
  3066. _spoolman_on = await get_setting(db, "spoolman_enabled")
  3067. _auto_add_raw = await get_setting(db, "auto_add_unknown_rfid")
  3068. _auto_add_unknown = _auto_add_raw is None or _auto_add_raw.lower() == "true"
  3069. if not _spoolman_on or _spoolman_on.lower() != "true":
  3070. for ams_unit in ams_data:
  3071. if not isinstance(ams_unit, dict):
  3072. continue
  3073. ams_id = int(ams_unit.get("id", 0))
  3074. for tray in ams_unit.get("tray", []):
  3075. if not isinstance(tray, dict):
  3076. continue
  3077. tray_id = int(tray.get("id", 0))
  3078. tag_uid = tray.get("tag_uid", "")
  3079. tray_uuid = tray.get("tray_uuid", "")
  3080. tray_info_idx = tray.get("tray_info_idx", "")
  3081. if not tray.get("tray_type"):
  3082. # Slot reported empty — drop any cached unknown-tag
  3083. # broadcast so reinserting the same spool re-prompts.
  3084. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id)
  3085. continue # Empty slot
  3086. # Check if assignment already exists for this slot
  3087. existing = await db.execute(
  3088. select(SA)
  3089. .options(selectinload(SA.spool).selectinload(Spool.k_profiles))
  3090. .where(SA.printer_id == printer_id, SA.ams_id == ams_id, SA.tray_id == tray_id)
  3091. )
  3092. existing_assignment = existing.scalar_one_or_none()
  3093. if existing_assignment:
  3094. # Sync spool weight_used from AMS remain — only INCREASE, never decrease.
  3095. # The AMS remain% is low-resolution (integer %, i.e. 10g steps for 1kg spool)
  3096. # and must not overwrite precise values from the usage tracker (3MF/G-code).
  3097. # Skip during active prints: the usage tracker handles deduction
  3098. # precisely via 3MF data on print completion. Without this guard the
  3099. # AMS remain% SET and the usage tracker ADD both fire from the same
  3100. # MQTT message, doubling the deduction (#880).
  3101. if _print_active:
  3102. continue
  3103. remain_raw = tray.get("remain")
  3104. if (
  3105. remain_raw is not None
  3106. and existing_assignment.spool
  3107. and not existing_assignment.spool.weight_locked
  3108. ):
  3109. try:
  3110. remain_val = int(remain_raw)
  3111. except (TypeError, ValueError):
  3112. remain_val = -1
  3113. if 1 <= remain_val <= 100:
  3114. lw = existing_assignment.spool.label_weight or 1000
  3115. new_used = round(lw * (100 - remain_val) / 100.0, 1)
  3116. current_used = existing_assignment.spool.weight_used or 0
  3117. if new_used > current_used + 1:
  3118. # Rechecked in the statement: the spool was loaded
  3119. # before this, and a weight set by hand (or weighed)
  3120. # and locked since then must stay (WL-6).
  3121. result = cast(
  3122. CursorResult,
  3123. await db.execute(
  3124. update(Spool)
  3125. .where(
  3126. Spool.id == existing_assignment.spool_id,
  3127. Spool.weight_locked.is_not(True),
  3128. func.coalesce(Spool.weight_used, 0) < new_used - 1,
  3129. )
  3130. .values(weight_used=new_used)
  3131. ),
  3132. )
  3133. await db.commit()
  3134. if result.rowcount:
  3135. logger.info(
  3136. "Weight sync: spool %d weight_used %s -> %s (remain=%d)",
  3137. existing_assignment.spool_id,
  3138. current_used,
  3139. new_used,
  3140. remain_val,
  3141. )
  3142. await ws_manager.broadcast({"type": "inventory_changed"})
  3143. # A K-profile the slot lost (the default K after a reset or
  3144. # power cycle) is put back by kprofile_drift on the next idle
  3145. # push; a different real profile was picked on purpose and
  3146. # stays (#3219, item 44).
  3147. continue
  3148. if is_bambu_tag(tag_uid, tray_uuid, tray_info_idx):
  3149. # BL spool with RFID tag: auto-match → inventory match → auto-create
  3150. spool = await get_spool_by_tag(db, tag_uid, tray_uuid)
  3151. # Only a spool already known by this tag is proven to be
  3152. # here; a material/colour match or a new spool is not
  3153. # grounds to take it out of other slots (#3427).
  3154. known_by_tag = spool is not None
  3155. if not spool and await tag_belongs_to_archived_spool(db, tag_uid, tray_uuid):
  3156. # An archived spool still in the AMS: the slot
  3157. # stays free rather than adopting its tag.
  3158. continue
  3159. if not spool:
  3160. # Try matching an untagged inventory spool (same material/color)
  3161. spool = await find_matching_untagged_spool(db, tray)
  3162. if spool:
  3163. await link_tag_to_inventory_spool(db, spool, tray)
  3164. elif _auto_add_unknown:
  3165. spool = await create_spool_from_tray(db, tray)
  3166. else:
  3167. # Auto-add disabled: surface the slot so the
  3168. # user can add it manually via the UI.
  3169. await _broadcast_unknown_tag(
  3170. printer_id=printer_id,
  3171. ams_id=ams_id,
  3172. tray_id=tray_id,
  3173. tag_uid=tag_uid,
  3174. tray_uuid=tray_uuid,
  3175. tray_type=tray.get("tray_type"),
  3176. tray_color=tray.get("tray_color"),
  3177. tray_sub_brands=tray.get("tray_sub_brands"),
  3178. tray_count=len(ams_unit.get("tray", [])),
  3179. )
  3180. continue
  3181. # Slot matched (existing tag, untagged inventory
  3182. # match, or freshly auto-created spool) — drop any
  3183. # stale dedup so a future tag swap re-prompts.
  3184. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id)
  3185. await auto_assign_spool(
  3186. printer_id,
  3187. ams_id,
  3188. tray_id,
  3189. spool,
  3190. printer_manager,
  3191. db,
  3192. tray_info_idx=tray_info_idx,
  3193. )
  3194. await db.commit()
  3195. if known_by_tag:
  3196. rfid_assigned.append((spool.id, (printer_id, ams_id, tray_id)))
  3197. await ws_manager.broadcast(
  3198. {
  3199. "type": "spool_auto_assigned",
  3200. "printer_id": printer_id,
  3201. "ams_id": ams_id,
  3202. "tray_id": tray_id,
  3203. "spool_id": spool.id,
  3204. }
  3205. )
  3206. logger.info(
  3207. "RFID auto-assigned spool %d to printer %d AMS%d-T%d",
  3208. spool.id,
  3209. printer_id,
  3210. ams_id,
  3211. tray_id,
  3212. )
  3213. elif is_valid_tag(tag_uid, tray_uuid) and not await tag_belongs_to_archived_spool(
  3214. db, tag_uid, tray_uuid
  3215. ):
  3216. # Non-BL spool with some tag — let user choose (not
  3217. # an archived one's, whose slot stays free)
  3218. await _broadcast_unknown_tag(
  3219. printer_id=printer_id,
  3220. ams_id=ams_id,
  3221. tray_id=tray_id,
  3222. tag_uid=tag_uid,
  3223. tray_uuid=tray_uuid,
  3224. tray_type=tray.get("tray_type"),
  3225. tray_color=tray.get("tray_color"),
  3226. tray_sub_brands=tray.get("tray_sub_brands"),
  3227. tray_count=len(ams_unit.get("tray", [])),
  3228. )
  3229. # No-tag slots (generic non-RFID filament) are left alone:
  3230. # nothing to identify, prompting "+ Add" would just create
  3231. # ghost spools with empty tags on every confirm.
  3232. except Exception as e:
  3233. logger.warning("RFID spool auto-assign failed: %s", e, exc_info=True)
  3234. if rfid_assigned:
  3235. from backend.app.services.spool_relocation import INTERNAL, release_other_slots
  3236. for spool_id, slot in rfid_assigned:
  3237. try:
  3238. await release_other_slots(INTERNAL, spool_id, slot)
  3239. except Exception as e:
  3240. logger.warning("Releasing spool %s from its other slots failed: %s", spool_id, e, exc_info=True)
  3241. try:
  3242. async with async_session() as db:
  3243. from backend.app.api.routes.settings import get_setting
  3244. from backend.app.models.printer import Printer
  3245. # Check if Spoolman is enabled
  3246. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  3247. if not spoolman_enabled or spoolman_enabled.lower() != "true":
  3248. return
  3249. # Check sync mode
  3250. sync_mode = await get_setting(db, "spoolman_sync_mode")
  3251. if sync_mode and sync_mode != "auto":
  3252. return # Only sync on auto mode
  3253. _auto_add_raw_sm = await get_setting(db, "auto_add_unknown_rfid")
  3254. auto_add_unknown_rfid = _auto_add_raw_sm is None or _auto_add_raw_sm.lower() == "true"
  3255. # `spoolman_disable_weight_sync` is deprecated (#1119) — weight is now
  3256. # always owned by per-print tracking, never by AMS auto-sync. The
  3257. # setting is still read by the settings UI for backwards compat but
  3258. # has no effect on the sync path here.
  3259. # Get Spoolman URL
  3260. spoolman_url = await get_setting(db, "spoolman_url")
  3261. if not spoolman_url:
  3262. return
  3263. # Get or create Spoolman client
  3264. client = await get_spoolman_client()
  3265. if not client:
  3266. try:
  3267. client = await init_spoolman_client(spoolman_url)
  3268. except ValueError as exc:
  3269. logger.warning("Spoolman URL %r rejected by SSRF guard: %s", spoolman_url, exc)
  3270. return
  3271. # Check if Spoolman is reachable
  3272. if not await client.health_check():
  3273. logger.warning("Spoolman not reachable at %s", spoolman_url)
  3274. return
  3275. # Get printer name for location
  3276. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3277. printer = result.scalar_one_or_none()
  3278. printer_name = printer.name if printer else f"Printer {printer_id}"
  3279. # OPTIMIZATION: Fetch all spools once before processing trays
  3280. # This eliminates redundant API calls (one per tray) when syncing multiple trays
  3281. logger.debug("[Printer %s] Fetching spools cache for AMS sync...", printer_id)
  3282. try:
  3283. cached_spools = await client.get_spools()
  3284. logger.debug("[Printer %s] Cached %d spools for batch sync", printer_id, len(cached_spools))
  3285. except Exception as e:
  3286. logger.error(
  3287. "[Printer %s] Failed to fetch spools cache after retries, aborting AMS sync: %s",
  3288. printer_id,
  3289. e,
  3290. )
  3291. return
  3292. # Load inventory weights as fallback (when AMS MQTT data lacks remain values)
  3293. from sqlalchemy.orm import selectinload
  3294. from backend.app.models.spool_assignment import SpoolAssignment
  3295. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  3296. from backend.app.services.ams_slot_presence import spool_present
  3297. from backend.app.services.inventory_mode import spoolman_owns_assignments
  3298. # Built-in remaining weight, used by sync_ams_tray only when the
  3299. # firmware reports an unusable remain%/tray_weight for a slot.
  3300. #
  3301. # Left empty since #2812. This block runs in Spoolman mode only,
  3302. # and until then the built-in table was emptied on the switch, so
  3303. # there was never anything here to read and the fallback was inert.
  3304. # Preserving those rows makes it live again, and it is keyed by slot
  3305. # rather than by spool: after a mode switch the tray may well hold
  3306. # different filament, and ``create_spool`` writes ``remaining_weight``
  3307. # unconditionally, so a stale figure would be seeded into a brand new
  3308. # Spoolman spool. Deliberately kept inert rather than deleted, so the
  3309. # intent survives for whoever revisits the cross-mode fallback.
  3310. inventory_weights: dict[tuple[int, int], float] = {}
  3311. if not await spoolman_owns_assignments(db):
  3312. try:
  3313. assign_result = await db.execute(
  3314. select(SpoolAssignment)
  3315. .options(selectinload(SpoolAssignment.spool))
  3316. .where(SpoolAssignment.printer_id == printer_id)
  3317. )
  3318. for assignment in assign_result.scalars().all():
  3319. spool = assignment.spool
  3320. if spool and spool.label_weight > 0:
  3321. remaining = max(0.0, spool.label_weight - (spool.weight_used or 0))
  3322. inventory_weights[(assignment.ams_id, assignment.tray_id)] = remaining
  3323. except Exception as e:
  3324. logger.warning("Could not load inventory weights for printer %s: %s", printer_id, e)
  3325. # Load existing Spoolman slot assignments for the no-RFID fallback path
  3326. spoolman_slot_map: dict[tuple[int, int], int] = {}
  3327. slot_map_read = False
  3328. try:
  3329. slot_result = await db.execute(
  3330. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  3331. )
  3332. for slot in slot_result.scalars().all():
  3333. spoolman_slot_map[(slot.ams_id, slot.tray_id)] = slot.spoolman_spool_id
  3334. slot_map_read = True
  3335. except Exception as e:
  3336. logger.warning("Could not load Spoolman slot assignments for printer %s: %s", printer_id, e)
  3337. # Sync each AMS tray and collect slot changes for DB persistence
  3338. synced = 0
  3339. slot_changes: list[tuple[int, int, int]] = [] # (ams_id, tray_id, spoolman_spool_id) to upsert
  3340. empty_slots: list[tuple[int, int]] = [] # (ams_id, tray_id) whose tray is now empty
  3341. spoolman_held: set[tuple] = set() # removals held for the grace period (#3186)
  3342. for ams_unit in ams_data:
  3343. if not isinstance(ams_unit, dict):
  3344. continue
  3345. ams_id = int(ams_unit.get("id", 0))
  3346. trays = ams_unit.get("tray", [])
  3347. for tray_data in trays:
  3348. if not isinstance(tray_data, dict):
  3349. continue
  3350. tray_id_raw = int(tray_data.get("id", 0))
  3351. tray = client.parse_ams_tray(ams_id, tray_data)
  3352. if not tray:
  3353. # Empty tray slot — record for local assignment cleanup
  3354. # and drop any cached unknown-tag broadcast so a
  3355. # reinserted spool re-prompts.
  3356. #
  3357. # Not during a running print: a slot that empties there
  3358. # is a filament runout, and the spool is still in the
  3359. # AMS. `spoolman_slot_assignments` is how a tag-less
  3360. # spool assigned through the Bambuddy UI is resolved at
  3361. # completion (#1459), so deleting the row mid-print
  3362. # loses the runout segment's usage — the same failure
  3363. # the internal inventory's auto-unlink had.
  3364. #
  3365. # Nor when firmware's presence bit says the slot is
  3366. # occupied. parse_ams_tray calls a tray with no type or
  3367. # no colour empty, and a spool the AMS cannot read has
  3368. # neither until something configures it — so a tag-less
  3369. # spool assigned through the UI had its row deleted by
  3370. # the first idle push after it was inserted. Same
  3371. # deletion as the internal inventory's in #3100, same
  3372. # answer, so the two modes stay in step.
  3373. #
  3374. # And only once the slot has stayed empty for the grace
  3375. # period -- the internal inventory's #3186 answer, for
  3376. # the same one-push blank.
  3377. linked_spool = spoolman_slot_map.get((ams_id, tray_id_raw))
  3378. hold_key = ("spoolman", ams_id, tray_id_raw, linked_spool)
  3379. if not printing_now and (
  3380. spool_present(tray_data) is not True or slot_unlink_grace.is_held(printer_id, hold_key)
  3381. ):
  3382. if linked_spool is None or slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  3383. empty_slots.append((ams_id, tray_id_raw))
  3384. else:
  3385. spoolman_held.add(hold_key)
  3386. logger.info(
  3387. "Spoolman slot unlink held: AMS%d-T%d (spool %d) reports empty; "
  3388. "unlinking if it is still empty in %ds",
  3389. ams_id,
  3390. tray_id_raw,
  3391. linked_spool,
  3392. int(slot_unlink_grace.GRACE_SECONDS),
  3393. )
  3394. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id_raw)
  3395. continue
  3396. spool_tag = (
  3397. tray.tray_uuid
  3398. if tray.tray_uuid and tray.tray_uuid != "00000000000000000000000000000000"
  3399. else tray.tag_uid
  3400. )
  3401. # Provide the hint only when no RFID is available
  3402. hint = spoolman_slot_map.get((ams_id, tray.tray_id)) if not spool_tag else None
  3403. try:
  3404. inv_remaining = inventory_weights.get((ams_id, tray.tray_id))
  3405. result = await client.sync_ams_tray(
  3406. tray,
  3407. printer_name,
  3408. db,
  3409. # Per-print tracking is the only weight writer (#1119).
  3410. # AMS auto-sync still maintains spool metadata / slot
  3411. # assignments but no longer touches remaining_weight.
  3412. disable_weight_sync=True,
  3413. cached_spools=cached_spools,
  3414. inventory_remaining=inv_remaining,
  3415. spoolman_spool_id_hint=hint,
  3416. auto_add_unknown_rfid=auto_add_unknown_rfid,
  3417. )
  3418. if result is None and spool_tag and not auto_add_unknown_rfid:
  3419. # Spoolman skipped auto-create per user setting — surface
  3420. # the slot so the UI can offer "+ Add to inventory".
  3421. await _broadcast_unknown_tag(
  3422. printer_id=printer_id,
  3423. ams_id=ams_id,
  3424. tray_id=tray.tray_id,
  3425. tag_uid=tray.tag_uid or "",
  3426. tray_uuid=tray.tray_uuid or "",
  3427. tray_type=tray.tray_type,
  3428. tray_color=tray.tray_color,
  3429. tray_sub_brands=tray.tray_sub_brands,
  3430. tray_count=len(trays),
  3431. )
  3432. elif result:
  3433. _clear_unknown_tag_dedup(printer_id, ams_id, tray.tray_id)
  3434. if result:
  3435. synced += 1
  3436. if result.get("id"):
  3437. slot_changes.append((ams_id, tray.tray_id, result["id"]))
  3438. # If a new spool was created, add it to the cache
  3439. # so subsequent trays can find it if they reference the same tag
  3440. spool_exists = any(s.get("id") == result["id"] for s in cached_spools)
  3441. if not spool_exists:
  3442. cached_spools.append(result)
  3443. logger.debug(
  3444. "[Printer %s] Added newly created spool %s to cache",
  3445. printer_id,
  3446. result["id"],
  3447. )
  3448. # Reconcile slot_preset_mappings (the same row internal
  3449. # mode keeps in sync via inventory + spool_tag_matcher).
  3450. # Without this the slot card surfaces the previous spool's
  3451. # preset name — same bug shape, different inventory mode.
  3452. from backend.app.services.slot_preset_writer import (
  3453. upsert_slot_preset_for_spoolman_spool,
  3454. )
  3455. await upsert_slot_preset_for_spoolman_spool(
  3456. db=db,
  3457. spoolman_spool=result,
  3458. tray_info_idx=tray.tray_info_idx or "",
  3459. tray_sub_brands=tray.tray_sub_brands or "",
  3460. tray_type=tray.tray_type or "",
  3461. printer_id=printer_id,
  3462. ams_id=ams_id,
  3463. tray_id=tray.tray_id,
  3464. )
  3465. except Exception as e:
  3466. logger.error("Error syncing AMS %s tray %s: %s", ams_id, tray.tray_id, e)
  3467. slot_unlink_grace.settle(printer_id, "spoolman", spoolman_held)
  3468. if synced > 0:
  3469. logger.info("Auto-synced %s AMS trays to Spoolman for printer %s", synced, printer_id)
  3470. # Persist slot assignment changes to the local table: only slots
  3471. # whose spool this pass changed, and only while they still hold what
  3472. # it read before awaiting Spoolman, so a spool assigned by hand in
  3473. # between stays (SSA-3). The changed slots are announced: the slot
  3474. # card reads slot_preset_mappings (rewritten above) ahead of the live
  3475. # tray, so with no event it kept the previous spool's preset name.
  3476. if slot_changes or empty_slots:
  3477. from backend.app.services.spoolman_slot_sync import write_slot_results
  3478. try:
  3479. await write_slot_results(
  3480. db,
  3481. {(printer_id, a_id, t_id): spool_id for (a_id, t_id), spool_id in spoolman_slot_map.items()}
  3482. if slot_map_read
  3483. else None,
  3484. [(printer_id, a_id, t_id, spool_id) for a_id, t_id, spool_id in slot_changes],
  3485. [(printer_id, a_id, t_id) for a_id, t_id in empty_slots],
  3486. )
  3487. except Exception as e:
  3488. await db.rollback()
  3489. logger.error("Error persisting Spoolman slot assignments for printer %s: %s", printer_id, e)
  3490. except Exception as e:
  3491. logging.getLogger(__name__).error("Spoolman AMS sync failed for printer %s: %s", printer_id, e)
  3492. # Largest finish photo attached to a notification; a bigger one is still linked.
  3493. _FINISH_PHOTO_ATTACH_MAX_BYTES = 2_500_000
  3494. async def _finish_photo_for_notification(
  3495. db, archive, archive_id: int, filename: str
  3496. ) -> tuple[str | None, bytes | None]:
  3497. """The ``{finish_photo_url}`` link and the bytes to attach, for a print's finish photo.
  3498. With authentication off the link is the archive's own photo route, as it
  3499. always was: it opens without a login and doesn't expire. With
  3500. authentication on that route needs a media token, which nothing tapping a
  3501. link in Telegram, CallMeBot or a Home Assistant notification has, so the
  3502. link only ever answered 401. The photo is then saved as a notification
  3503. photo as well (utils/notification_photos.py) and the link points there: an
  3504. unguessable name that opens this one photo and nothing else, for 3 days.
  3505. The link is relative when no External URL is set. Bytes over
  3506. ``_FINISH_PHOTO_ATTACH_MAX_BYTES`` are linked but not attached. Returns
  3507. ``(None, None)`` when the photo can't be found.
  3508. """
  3509. from backend.app.api.routes.settings import get_setting
  3510. from backend.app.core.auth import is_auth_enabled
  3511. from backend.app.utils.archive_paths import find_archive_photo
  3512. from backend.app.utils.notification_photos import save_notification_photo
  3513. log = logging.getLogger(__name__)
  3514. base = ((await get_setting(db, "external_url")) or "").strip().rstrip("/")
  3515. url: str | None = f"{base}/api/v1/archives/{archive_id}/photos/{filename}"
  3516. photo_bytes: bytes | None = None
  3517. try:
  3518. photo_path = find_archive_photo(archive, filename)
  3519. if photo_path is not None:
  3520. photo_bytes = await asyncio.to_thread(photo_path.read_bytes)
  3521. except Exception as e:
  3522. log.warning("[NOTIFY-BG] Failed to read finish photo bytes: %s", e)
  3523. try:
  3524. auth_on = await is_auth_enabled(db)
  3525. except Exception:
  3526. auth_on = True # Unknown: the archive link may need a login, so don't rely on it.
  3527. if auth_on:
  3528. url = None
  3529. if photo_bytes:
  3530. try:
  3531. name = await asyncio.to_thread(save_notification_photo, photo_bytes, "print_complete")
  3532. url = f"{base}/api/v1/notifications/photos/{name}"
  3533. except Exception as e:
  3534. log.warning("[NOTIFY-BG] Failed to save finish photo for its link: %s", e)
  3535. if photo_bytes is not None and len(photo_bytes) > _FINISH_PHOTO_ATTACH_MAX_BYTES:
  3536. log.warning("[NOTIFY-BG] Finish photo too large for attachment: %s bytes", len(photo_bytes))
  3537. return url, None
  3538. if photo_bytes:
  3539. log.info("[NOTIFY-BG] Loaded finish photo bytes: %s bytes", len(photo_bytes))
  3540. return url, photo_bytes
  3541. async def _capture_snapshot_for_notification(printer_id: int, printer, logger, *, light: bool = True) -> bytes | None:
  3542. """Capture a camera snapshot for notification image attachment.
  3543. Returns JPEG bytes (max 2.5MB) or None if capture fails or is unavailable.
  3544. Uses: external camera > buffered frame > fresh capture.
  3545. Turns the chamber light on for the picture when the printer asks for it
  3546. (#1655). ``light=False`` is for callers that capture all through a print,
  3547. where the light would flash with every frame.
  3548. """
  3549. if not printer:
  3550. return None
  3551. async with camera_light(printer if light else None):
  3552. return await _capture_snapshot_frame(printer_id, printer, logger)
  3553. async def _capture_snapshot_frame(printer_id: int, printer, logger) -> bytes | None:
  3554. """The capture behind ``_capture_snapshot_for_notification``."""
  3555. try:
  3556. from backend.app.api.routes.settings import get_setting
  3557. async with async_session() as db:
  3558. capture_enabled = await get_setting(db, "capture_finish_photo")
  3559. if capture_enabled is not None and capture_enabled.lower() != "true":
  3560. return None
  3561. # Try external camera first
  3562. if printer.external_camera_enabled and printer.external_camera_url:
  3563. logger.info("[SNAPSHOT] Capturing from external camera for printer %s", printer_id)
  3564. from backend.app.api.routes.camera import live_frame_for_capture
  3565. from backend.app.services.external_camera import capture_frame
  3566. # An external camera allows one reader, so capturing while a viewer
  3567. # is attached fails (#2707). A None here falls through to the paths
  3568. # below exactly as a failed capture did.
  3569. defer, buffered = live_frame_for_capture(printer_id)
  3570. if defer:
  3571. frame_data = buffered
  3572. else:
  3573. frame_data = await capture_frame(
  3574. printer.external_camera_url,
  3575. printer.external_camera_type or "mjpeg",
  3576. snapshot_url=printer.external_camera_snapshot_url,
  3577. )
  3578. if frame_data and len(frame_data) <= 2_500_000:
  3579. logger.info("[SNAPSHOT] External camera frame: %s bytes", len(frame_data))
  3580. return _apply_camera_rotation(frame_data, printer, logger)
  3581. # Try buffered frame from active stream
  3582. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  3583. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  3584. active_chamber = [k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")]
  3585. buffered_frame = get_buffered_frame(printer_id)
  3586. if (active_for_printer or active_chamber) and buffered_frame:
  3587. logger.info("[SNAPSHOT] Using buffered frame for printer %s: %s bytes", printer_id, len(buffered_frame))
  3588. if len(buffered_frame) <= 2_500_000:
  3589. return _apply_camera_rotation(buffered_frame, printer, logger)
  3590. # Fresh capture from printer camera
  3591. logger.info("[SNAPSHOT] Capturing fresh frame for printer %s", printer_id)
  3592. from backend.app.services.camera import capture_camera_frame_bytes
  3593. frame_data = await capture_camera_frame_bytes(
  3594. printer.ip_address, printer.access_code, printer.model, timeout=15
  3595. )
  3596. if frame_data and len(frame_data) <= 2_500_000:
  3597. logger.info("[SNAPSHOT] Fresh camera frame: %s bytes", len(frame_data))
  3598. return _apply_camera_rotation(frame_data, printer, logger)
  3599. except Exception as e:
  3600. logger.warning("[SNAPSHOT] Failed to capture snapshot for printer %s: %s", printer_id, e)
  3601. return None
  3602. async def _maybe_bank_inprint_frame(printer_id: int, layer_num: int) -> None:
  3603. """#1867: bank a recent in-print camera frame for the finish photo.
  3604. Called on every layer change and (#2547) on every print-progress advance.
  3605. Grabs one frame (throttled) into ``_inprint_frame_bank`` so the finish-photo
  3606. path has a pre-End-G-code image for prints that end with a plate swap.
  3607. Both drivers are print telemetry that stops the instant printing ends: no
  3608. further layers, and progress freezes before the End G-code (e.g. SwapMod
  3609. plate swap) executes. So the last banked frame is always the finished print,
  3610. never the swapped plate — that property is what the #1867 path relies on and
  3611. it must survive any change to the throttle below.
  3612. Layer changes alone were not enough: they stop when the *final* layer
  3613. begins, which on a three-minute last layer left the bank stale by the whole
  3614. length of that layer (#2547). Progress keeps ticking through it.
  3615. Best-effort: any failure just leaves the previous banked frame.
  3616. """
  3617. logger = logging.getLogger(__name__)
  3618. client = printer_manager.get_client(printer_id)
  3619. state = client.state if client else None
  3620. if not state or state.state != "RUNNING":
  3621. return
  3622. # Only during actual extrusion — firmware ticks layer_num during the
  3623. # pre-print calibration sequence, whose sub-stages are non-zero.
  3624. if state.mc_print_sub_stage not in (None, 0):
  3625. return
  3626. # #2547: throttled uniformly, with no last-layer exemption. The old code
  3627. # bypassed the throttle on the final layer to guarantee a fresh frame there;
  3628. # now that progress advances also drive banking, that exemption would fire a
  3629. # camera grab on every percent tick of the last layer. Bambu printers accept
  3630. # one RTSP client at a time, so each grab contends with the live view.
  3631. now = time.monotonic()
  3632. last = _inprint_frame_bank_ts.get(printer_id, 0.0)
  3633. if (now - last) < _INPRINT_BANK_MIN_INTERVAL:
  3634. return
  3635. total = state.total_layers or 0
  3636. try:
  3637. async with async_session() as db:
  3638. from backend.app.models.printer import Printer
  3639. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3640. printer = result.scalar_one_or_none()
  3641. if not printer:
  3642. return
  3643. # Reuses the notification snapshot path, which honours the
  3644. # `capture_finish_photo` setting (returns None when disabled) so we
  3645. # don't bank frames the user never asked for. Without the chamber light
  3646. # (#1655): this runs every 25 seconds for the whole print.
  3647. frame = await _capture_snapshot_for_notification(printer_id, printer, logger, light=False)
  3648. if frame:
  3649. _inprint_frame_bank[printer_id] = frame
  3650. _inprint_frame_bank_ts[printer_id] = now
  3651. logger.debug(
  3652. "[FINISH-PHOTO-BANK] banked in-print frame for printer %s at layer %s/%s (%d bytes)",
  3653. printer_id,
  3654. layer_num,
  3655. total,
  3656. len(frame),
  3657. )
  3658. except Exception as e:
  3659. logger.debug("[FINISH-PHOTO-BANK] bank failed for printer %s: %s", printer_id, e)
  3660. def _apply_camera_rotation(image_data: bytes, printer, logger) -> bytes:
  3661. """Apply camera rotation to snapshot image if configured."""
  3662. from backend.app.services.camera import apply_camera_rotation
  3663. return apply_camera_rotation(image_data, getattr(printer, "camera_rotation", 0), logger)
  3664. async def _send_print_start_notification(
  3665. printer_id: int,
  3666. data: dict,
  3667. archive_data: dict | None = None,
  3668. logger=None,
  3669. ):
  3670. """Helper to send print start notification with optional archive data."""
  3671. if logger is None:
  3672. logger = logging.getLogger(__name__)
  3673. try:
  3674. async with async_session() as db:
  3675. from backend.app.models.printer import Printer
  3676. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3677. printer = result.scalar_one_or_none()
  3678. printer_name = printer.name if printer else f"Printer {printer_id}"
  3679. # Capture camera snapshot for notification image attachment
  3680. image_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  3681. if image_data:
  3682. if archive_data is None:
  3683. archive_data = {}
  3684. archive_data["image_data"] = image_data
  3685. await notification_service.on_print_start(printer_id, printer_name, data, db, archive_data=archive_data)
  3686. # Send user-specific email notification for print start
  3687. if archive_data and archive_data.get("created_by_id"):
  3688. await notification_service.send_user_print_email(
  3689. event_type="user_print_start",
  3690. created_by_id=archive_data["created_by_id"],
  3691. printer_name=printer_name,
  3692. filename=data.get("subtask_name") or data.get("filename", "Unknown"),
  3693. db=db,
  3694. )
  3695. except Exception as e:
  3696. logger.warning("Notification on_print_start failed: %s", e)
  3697. async def _dispatch_user_print_email(
  3698. status: str,
  3699. created_by_id: int | None,
  3700. printer_name: str,
  3701. filename: str,
  3702. db,
  3703. image_data: bytes | None = None,
  3704. finish_photo_url: str | None = None,
  3705. ) -> None:
  3706. """Send a user-specific print-completion email based on print status.
  3707. Maps the normalised print status to the correct event type and delegates
  3708. to :meth:`NotificationService.send_user_print_email`. A single helper
  3709. avoids duplicating the ``if status == "completed" / elif "failed" / elif
  3710. "stopped"`` dispatch block at every call site.
  3711. Does nothing if *created_by_id* is ``None``.
  3712. """
  3713. if created_by_id is None:
  3714. return
  3715. if status == "completed":
  3716. event_type = "user_print_complete"
  3717. elif status == "failed":
  3718. event_type = "user_print_failed"
  3719. elif status in ("stopped", "aborted", "cancelled"):
  3720. event_type = "user_print_stopped"
  3721. else:
  3722. return
  3723. await notification_service.send_user_print_email(
  3724. event_type=event_type,
  3725. created_by_id=created_by_id,
  3726. printer_name=printer_name,
  3727. filename=filename,
  3728. db=db,
  3729. image_data=image_data,
  3730. finish_photo_url=finish_photo_url,
  3731. )
  3732. def _load_objects_from_archive(archive, printer_id: int, logger) -> None:
  3733. """Extract printable objects from an archive's 3MF file and store in printer state."""
  3734. try:
  3735. from backend.app.services.archive import extract_printable_objects_from_archive
  3736. client = printer_manager.get_client(printer_id)
  3737. if not client:
  3738. return
  3739. # Extract with positions for UI overlay, scoped to the plate that
  3740. # is printing — resolve_plate_id is the same resolver /cover uses,
  3741. # so the object list can't disagree with the thumbnail it is drawn
  3742. # over (#2522).
  3743. printable_objects, bbox_all = extract_printable_objects_from_archive(
  3744. app_settings.base_dir / archive.file_path,
  3745. plate_number=resolve_plate_id(client.state),
  3746. )
  3747. if printable_objects:
  3748. client.state.printable_objects = printable_objects
  3749. client.state.printable_objects_bbox_all = bbox_all
  3750. client.state.skipped_objects = []
  3751. logger.info("Loaded %s printable objects for printer %s", len(printable_objects), printer_id)
  3752. except Exception as e:
  3753. logger.debug("Failed to extract printable objects from archive: %s", e)
  3754. def _load_pauses_from_3mf(threemf_path, printer_id: int, *, warned_layers=(), passed_through_layer: int = 0) -> None:
  3755. """Read the running print's slicer pauses from its 3MF, in the background (#2984).
  3756. In a thread: a plate's G-code runs to tens of MB, and the print-start
  3757. handler must not wait for it. Kept only while the printer is still on
  3758. the print they were read for: a new print replaces the list.
  3759. ``warned_layers``: pauses this print was already warned about, kept on its
  3760. archive, so a restart does not send the heads-up again.
  3761. ``passed_through_layer``: for a print resumed after a restart, the layer it
  3762. is on. Its pauses up to there are behind it, though the printer's layer
  3763. reads 0 until it reports the next one.
  3764. """
  3765. client = printer_manager.get_client(printer_id)
  3766. if not client:
  3767. return
  3768. state = client.state
  3769. read_for = state.pause_points
  3770. try:
  3771. plate_id = resolve_plate_id(state)
  3772. except Exception:
  3773. # Never at the cost of the print-start handler this runs inside.
  3774. logging.getLogger(__name__).warning("[Printer %s] Could not tell the plate to read pauses for", printer_id)
  3775. return
  3776. async def _load() -> None:
  3777. from backend.app.utils.threemf_tools import extract_pause_points_from_3mf
  3778. points = await asyncio.to_thread(extract_pause_points_from_3mf, Path(threemf_path), plate_id)
  3779. if state.pause_points is not read_for:
  3780. return
  3781. state.pause_points = [p for p in points if p.layer > passed_through_layer]
  3782. state.pause_warned_layers = set(state.pause_warned_layers) | set(warned_layers)
  3783. if points:
  3784. logging.getLogger(__name__).info(
  3785. "Loaded %s slicer pause(s) for printer %s at layer(s) %s",
  3786. len(points),
  3787. printer_id,
  3788. [p.layer for p in points],
  3789. )
  3790. spawn_background_task(_load(), name=f"load-pauses-{printer_id}")
  3791. def _load_pauses_from_resumed_archive(archive, printer_id: int) -> None:
  3792. """Pauses for a print start taken as the resumption of a running print.
  3793. A restart while the print was paused makes its resume look like a new
  3794. print (#2984): the pauses warned about and passed are kept from before.
  3795. """
  3796. client = printer_manager.get_client(printer_id)
  3797. state = client.state if client else None
  3798. _load_pauses_from_3mf(
  3799. app_settings.base_dir / archive.file_path,
  3800. printer_id,
  3801. warned_layers=pause_eta.warned_layers_of(archive, state) if state is not None else (),
  3802. passed_through_layer=getattr(client, "layer_before_new_print", 0) if client else 0,
  3803. )
  3804. async def _restore_printable_objects(printer_id: int, state, db, logger) -> None:
  3805. """Put the skip-objects list back after a restart mid-print.
  3806. ``PrinterState.printable_objects`` is in-memory only, and the only thing
  3807. that fills it is ``_load_objects_from_archive`` on the print-start paths —
  3808. which the #1304 guard suppresses on the first RUNNING push after startup.
  3809. Everything else this hook restores (the archive, the usage-tracking session,
  3810. the timelapse baseline) was already handled; the object list was not, so a
  3811. restart mid-print took skip-objects away for the rest of that print.
  3812. Nothing recovered it either: the printer card gates its Skip button on the
  3813. object count, and the one endpoint that can rebuild the list is reachable
  3814. only from the modal that button opens.
  3815. The archive is the running print's as ``running_print_archive`` finds it:
  3816. by subtask_id, or for a print that reports none by the archive bound to it
  3817. (#3327), never by name: a leftover ``status="printing"`` row from a
  3818. completion we never saw must not hand this print someone else's objects.
  3819. After a restart that binding exists only for Bambuddy's own prints, so a
  3820. print that reports no subtask_id loads nothing rather than a guess; the
  3821. reload path on ``GET /print/objects`` covers that case on demand.
  3822. """
  3823. client = printer_manager.get_client(printer_id)
  3824. if client is None or (client.state.printable_objects and client.state.pause_points):
  3825. return
  3826. archive = await running_print_archive(db, printer_id, state, printing_only=True, by_name=False)
  3827. if archive is not None and archive.file_path:
  3828. if not client.state.printable_objects:
  3829. _load_objects_from_archive(archive, printer_id, logger)
  3830. # The slicer's pauses are in-memory only as well (#2984).
  3831. if not client.state.pause_points:
  3832. _load_pauses_from_3mf(
  3833. app_settings.base_dir / archive.file_path,
  3834. printer_id,
  3835. warned_layers=pause_eta.warned_layers_of(archive, state),
  3836. )
  3837. # Retry ladder for a fallback archive created while the printer's FTPS cool-off
  3838. # was running (#2957). The cool-off is 300s, so the first attempt is placed just
  3839. # past it; the second covers a handshake that failed again on the way back and
  3840. # armed a fresh one. Module-level so tests can shrink them.
  3841. _FALLBACK_3MF_RETRY_DELAYS_SECONDS: tuple[float, ...] = (310.0, 620.0)
  3842. # Retry ladder for the other temporary give-up: the file service answered and
  3843. # the transfer still did not finish, which at print start is usually the printer
  3844. # serving MQTT, the camera and a job upload at the same time (#3063). Nothing has
  3845. # to expire here, so the first attempt comes early -- #3063's reporter had the
  3846. # same 19MB file complete 48 seconds after the download budget ran out. The later
  3847. # two cover a printer that stays busy well into the print.
  3848. _FALLBACK_3MF_TRANSFER_RETRY_DELAYS_SECONDS: tuple[float, ...] = (60.0, 240.0, 600.0)
  3849. # printer_id -> the in-flight retry task, so print completion can cancel it.
  3850. _fallback_3mf_retry_tasks: dict[int, asyncio.Task] = {}
  3851. # printer_id -> lock serialising recovery attempts for that printer. Three callers
  3852. # can reach one archive at once: the cover endpoint (whose single-flight coalesces
  3853. # by view, so two views race), the cool-off retry task, and print completion.
  3854. # Without this they each read file_path == "" and each run a full copy, so the row
  3855. # ends up pointing at one timestamped directory while the others sit orphaned.
  3856. #
  3857. # Keyed by printer rather than archive because a printer runs one print at a time,
  3858. # which makes the two equally strong here — and it bounds the dict by printer
  3859. # count instead of needing a cleanup pass. Popping a per-archive entry cannot be
  3860. # done safely: `Lock.locked()` reads False between release and the queued waiter
  3861. # resuming, so "no waiters" is not a question this API can answer.
  3862. _fallback_recovery_locks: dict[int, asyncio.Lock] = {}
  3863. # How many of a printer's newest archives a touchscreen reprint is compared
  3864. # against (#3307). A repeat is usually of a recent job; the bound keeps a
  3865. # print start from reading a long history's extra_data.
  3866. _REPEAT_LOOKBACK = 200
  3867. async def _archived_3mf_for_repeat(
  3868. db, printer_id: int, subtask_name: str | None, gcode_file: str | None, total_layers: int | None
  3869. ):
  3870. """The newest archive of the job this printer is running again, with its 3MF.
  3871. A "Print again" on the printer's screen never names the file on the card:
  3872. it reports its own history copy under a generic name
  3873. (``file:///userdata/project_file.gcode.3mf``), the 3MF's internal title as
  3874. ``subtask_name`` and the plate as ``gcode_file`` -- and the title is often
  3875. not the card file's name, so every lookup built from it misses (#3307).
  3876. An earlier run that was archived recorded that same title and plate in its
  3877. ``_print_data``, and holds the 3MF.
  3878. Same printer, same title, same plate, a 3MF still on disk. A layer count
  3879. known on both sides must agree: the same title and plate re-sliced is
  3880. another job. None when nothing matches.
  3881. """
  3882. from backend.app.models.archive import PrintArchive
  3883. if not subtask_name or not gcode_file:
  3884. return None
  3885. rows = (
  3886. (
  3887. await db.execute(
  3888. select(PrintArchive)
  3889. .where(
  3890. PrintArchive.printer_id == printer_id,
  3891. PrintArchive.deleted_at.is_(None),
  3892. PrintArchive.file_path.isnot(None),
  3893. PrintArchive.file_path != "",
  3894. )
  3895. .order_by(PrintArchive.created_at.desc(), PrintArchive.id.desc())
  3896. .limit(_REPEAT_LOOKBACK)
  3897. )
  3898. )
  3899. .scalars()
  3900. .all()
  3901. )
  3902. for archive in rows:
  3903. extra = archive.extra_data if isinstance(archive.extra_data, dict) else {}
  3904. run = extra.get("_print_data")
  3905. if not isinstance(run, dict):
  3906. continue
  3907. if run.get("subtask_name") != subtask_name or run.get("filename") != gcode_file:
  3908. continue
  3909. if total_layers and archive.total_layers and archive.total_layers != total_layers:
  3910. continue
  3911. if not (app_settings.base_dir / archive.file_path).is_file():
  3912. continue
  3913. return archive
  3914. return None
  3915. async def _recover_fallback_archive(archive_id: int, source_3mf: Path, printer_id: int, *, price: bool = True) -> bool:
  3916. """Fill in a no-3MF archive from a 3MF that turned up later.
  3917. Returns True when the row was upgraded. Safe to call speculatively: it
  3918. verifies the archive still exists, is still a fallback, and that the file
  3919. is a readable 3MF before touching anything.
  3920. Serialised per printer — see ``_fallback_recovery_locks``.
  3921. """
  3922. lock = _fallback_recovery_locks.setdefault(printer_id, asyncio.Lock())
  3923. async with lock:
  3924. return await _recover_fallback_archive_locked(archive_id, source_3mf, printer_id, price=price)
  3925. async def _recover_fallback_archive_locked(
  3926. archive_id: int, source_3mf: Path, printer_id: int, *, price: bool = True
  3927. ) -> bool:
  3928. """The body of :func:`_recover_fallback_archive`, under its per-printer lock.
  3929. ``price`` is False when the print's end is being handled: the archive
  3930. still reads "printing" then, and its end prices it.
  3931. """
  3932. import zipfile
  3933. from backend.app.models.archive import PrintArchive
  3934. from backend.app.services.archive import ArchiveService
  3935. logger = logging.getLogger(__name__)
  3936. if not source_3mf.exists() or source_3mf.stat().st_size == 0:
  3937. return False
  3938. if not await asyncio.to_thread(zipfile.is_zipfile, source_3mf):
  3939. # A truncated or half-written download is worse than no download: it
  3940. # would replace an honest empty archive with wrong metadata.
  3941. logger.warning("[RECOVER] %s is not a readable 3MF; leaving archive %s as-is", source_3mf, archive_id)
  3942. return False
  3943. async with async_session() as db:
  3944. archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  3945. if archive is None or archive.deleted_at is not None:
  3946. return False
  3947. if archive.file_path:
  3948. # Already recovered, or never was a fallback. Either way there is a
  3949. # real 3MF attached and overwriting it is not this function's job.
  3950. return False
  3951. print_data = (archive.extra_data or {}).get("_print_data") or {}
  3952. service = ArchiveService(db)
  3953. recovered = await service.archive_print(
  3954. printer_id=printer_id,
  3955. source_file=source_3mf,
  3956. print_data={**print_data, "status": archive.status or "printing"},
  3957. subtask_id=archive.subtask_id,
  3958. update_archive_id=archive.id,
  3959. )
  3960. if recovered is None:
  3961. return False
  3962. logger.info(
  3963. "[RECOVER] Archive %s filled in from %s (%s bytes) — it started as a no-3MF fallback",
  3964. archive_id,
  3965. source_3mf,
  3966. recovered.file_size,
  3967. )
  3968. # Spoolman's tracking stored no 3MF usage at print start and would
  3969. # charge the AMS remain% change; the built-in tracker reads the archive
  3970. # at completion and needs nothing. Also at the print's end (``price``
  3971. # False): that recovery runs inside the close-out, before the charge.
  3972. try:
  3973. from backend.app.services.spoolman_tracking import attach_recovered_3mf
  3974. from backend.app.services.usage_tracker import is_completing
  3975. await attach_recovered_3mf(
  3976. printer_id,
  3977. recovered.id,
  3978. recovered.file_path,
  3979. db,
  3980. printer_manager,
  3981. plate_id=_get_start_plate_id(printer_id, recovered.id) or recovered.plate_id,
  3982. may_start=not price or (recovered.status == "printing" and not is_completing(printer_id)),
  3983. )
  3984. except Exception:
  3985. logger.warning(
  3986. "[RECOVER] Archive %s: could not add the 3MF to Spoolman's usage tracking", archive_id, exc_info=True
  3987. )
  3988. if price and recovered.status == "printing" and recovered.file_path:
  3989. # Print start had no 3MF to read the slicer's pauses from (#2984).
  3990. _load_pauses_from_3mf(app_settings.base_dir / recovered.file_path, printer_id)
  3991. if price and recovered.status == "printing":
  3992. # Print start priced it from the default rate, having no 3MF to
  3993. # estimate from; now it has one. An ended print is left alone: its
  3994. # end prices it. A print Bambuddy didn't dispatch has no start
  3995. # plate; the one the archive just read from the 3MF stands in.
  3996. _schedule_archive_cost_estimate(
  3997. printer_id,
  3998. recovered.id,
  3999. printer_manager,
  4000. ams_mapping=_get_start_ams_mapping(print_data, printer_id, recovered.id),
  4001. plate_id=_get_start_plate_id(printer_id, recovered.id) or recovered.plate_id,
  4002. session_factory=async_session,
  4003. )
  4004. # `archive_updated`, not `archive_created` — the row was already on the
  4005. # Archives page as an empty card and is now filled in, not new.
  4006. await ws_manager.send_archive_updated(
  4007. {
  4008. "id": recovered.id,
  4009. "printer_id": recovered.printer_id,
  4010. "filename": recovered.filename,
  4011. "print_name": recovered.print_name,
  4012. "status": recovered.status,
  4013. }
  4014. )
  4015. return True
  4016. async def try_recover_fallback_archive(printer_id: int, name: str, path: Path, *, price: bool = True) -> bool:
  4017. """Offer a freshly-downloaded 3MF to this printer's running fallback archive.
  4018. Called from the paths that pull a 3MF for a print that is already under way
  4019. — chiefly the cover endpoint, which downloads the very file the archive flow
  4020. could not get and, before #2957, used it for a thumbnail and nothing else.
  4021. The bytes are already local, so this costs a parse and a row update.
  4022. No-op when the running print has a real archive, which is the common case.
  4023. """
  4024. from backend.app.models.archive import PrintArchive
  4025. logger = logging.getLogger(__name__)
  4026. # `_active_prints` is keyed on the raw names seen at print start — the
  4027. # dispatch filename, the subtask name, and the subtask name plus ".3mf".
  4028. # Callers here arrive with whichever variant their own path produced, so
  4029. # match on the same normalization the download cache uses rather than on an
  4030. # exact string; that is what makes "Desktop_Goose.gcode.3mf" from the cover
  4031. # endpoint find an archive registered under "Desktop_Goose".
  4032. wanted = normalize_3mf_name(name)
  4033. archive_id = None
  4034. for (key_printer_id, key_name), value in list(_active_prints.items()):
  4035. if key_printer_id == printer_id and normalize_3mf_name(key_name) == wanted:
  4036. archive_id = value
  4037. break
  4038. if archive_id is None:
  4039. return False
  4040. async with async_session() as db:
  4041. archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  4042. # Cheap pre-check so the common case (a normal archive) does no work.
  4043. if archive is None or archive.file_path or archive.deleted_at is not None:
  4044. return False
  4045. try:
  4046. return await _recover_fallback_archive(archive_id, path, printer_id, price=price)
  4047. except Exception as e:
  4048. # Recovery is opportunistic. A failure here must never take down the
  4049. # caller, which is usually just trying to render a thumbnail.
  4050. logger.warning("[RECOVER] Could not fill in archive %s from %s: %s", archive_id, path, e)
  4051. return False
  4052. def _schedule_fallback_3mf_retry(
  4053. printer_id: int,
  4054. archive_id: int,
  4055. filenames: list[str],
  4056. delays: tuple[float, ...] | None = None,
  4057. reason: str = REASON_FTPS_COOLOFF,
  4058. ) -> None:
  4059. """Re-attempt the 3MF download after a temporary give-up.
  4060. ``reason`` says which give-up this is, and picks the default ladder: an
  4061. FTPS cool-off has to be waited out, while a transfer that timed out under
  4062. contention is worth asking about again straight away (#3063). It is only
  4063. read for the ladder and the log line -- the retry itself is identical, since
  4064. in both cases the file is on the printer and the last attempt at it failed
  4065. for a reason that does not last.
  4066. """
  4067. logger = logging.getLogger(__name__)
  4068. if delays is None:
  4069. delays = (
  4070. _FALLBACK_3MF_TRANSFER_RETRY_DELAYS_SECONDS
  4071. if reason == REASON_FTP_TRANSFER_FAILED
  4072. else _FALLBACK_3MF_RETRY_DELAYS_SECONDS
  4073. )
  4074. async def _retry() -> None:
  4075. from backend.app.models.archive import PrintArchive
  4076. from backend.app.models.printer import Printer
  4077. for delay in delays:
  4078. await asyncio.sleep(delay)
  4079. async with async_session() as db:
  4080. archive = (
  4081. await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  4082. ).scalar_one_or_none()
  4083. if archive is None or archive.deleted_at is not None or archive.file_path:
  4084. return
  4085. printer = (await db.execute(select(Printer).where(Printer.id == printer_id))).scalar_one_or_none()
  4086. if printer is None:
  4087. return
  4088. # Read the fields while the session is open rather than touching
  4089. # a detached instance minutes later, mid-download.
  4090. printer_ip = printer.ip_address
  4091. printer_code = printer.access_code
  4092. printer_model = printer.model
  4093. # Someone else may have fetched it in the meantime — the cover
  4094. # endpoint routinely does, and its copy is the same bytes.
  4095. for name in filenames:
  4096. cached = get_cached_3mf(printer_id, name)
  4097. if cached and await _recover_fallback_archive(archive_id, cached, printer_id):
  4098. return
  4099. if ftps_handshake_blocked(printer_ip):
  4100. logger.info(
  4101. "[RECOVER] Printer %s is still in its FTPS cool-off; archive %s retry deferred",
  4102. printer_id,
  4103. archive_id,
  4104. )
  4105. continue
  4106. _, _, _, ftp_timeout = await get_ftp_retry_settings()
  4107. for candidate in filenames:
  4108. # Bare name only. These come from the print-start flow, which
  4109. # already strips the path, but the local temp write must not
  4110. # depend on that holding for every future caller — a name that
  4111. # is absolute or contains ".." would otherwise escape the data
  4112. # volume via the `/` operator.
  4113. name = Path(candidate).name
  4114. if not name or name in (".", ".."):
  4115. continue
  4116. if not name.endswith(".3mf"):
  4117. name = f"{name}.3mf"
  4118. temp_path = app_settings.archive_dir / "temp" / name
  4119. temp_path.parent.mkdir(parents=True, exist_ok=True)
  4120. try:
  4121. hit = await download_file_try_paths_async(
  4122. printer_ip,
  4123. printer_code,
  4124. ftp_probe_paths(name),
  4125. temp_path,
  4126. socket_timeout=ftp_timeout,
  4127. printer_model=printer_model,
  4128. max_total_timeout=RECOVERY_DOWNLOAD_MAX_TIMEOUT,
  4129. )
  4130. except Exception as e:
  4131. logger.debug("[RECOVER] Retry download of %s failed: %s", name, e)
  4132. continue
  4133. if not hit:
  4134. continue
  4135. cache_3mf_download(printer_id, name, temp_path)
  4136. if await _recover_fallback_archive(archive_id, temp_path, printer_id):
  4137. return
  4138. logger.info("[RECOVER] Archive %s still has no 3MF after a retry", archive_id)
  4139. async def _guarded() -> None:
  4140. try:
  4141. await _retry()
  4142. except asyncio.CancelledError:
  4143. raise
  4144. except Exception as e:
  4145. logger.warning("[RECOVER] Retry task for archive %s failed: %s", archive_id, e)
  4146. finally:
  4147. if _fallback_3mf_retry_tasks.get(printer_id) is asyncio.current_task():
  4148. _fallback_3mf_retry_tasks.pop(printer_id, None)
  4149. existing = _fallback_3mf_retry_tasks.pop(printer_id, None)
  4150. if existing and not existing.done():
  4151. existing.cancel()
  4152. task = asyncio.create_task(_guarded())
  4153. _fallback_3mf_retry_tasks[printer_id] = task
  4154. logger.info(
  4155. "[RECOVER] Archive %s has no 3MF (%s) and the file should still be on printer %s; will retry in %s",
  4156. archive_id,
  4157. reason,
  4158. printer_id,
  4159. ", ".join(f"{d:g}s" for d in delays),
  4160. )
  4161. async def _ask_outcome_for_external_print(db, printer_id: int, observed_name: str | None = None) -> bool:
  4162. """Whether an archive created here should ask for the print's outcome (#1898).
  4163. Only a print Bambuddy did not dispatch reaches the archive-*creating*
  4164. branches below: a queued job already has its archive and takes the
  4165. expected-print branch, where the queue item's own ``confirm_outcome``
  4166. decides. The queue is still consulted, because a restart mid-print empties
  4167. ``_expected_prints`` — without the check the setting could override a queue
  4168. item that deliberately has the flag off. Any failure answers "don't ask":
  4169. an unwanted prompt is worse than a missing one, and this must never be the
  4170. reason a print goes unarchived.
  4171. A ``printing`` row is not proof on its own, though: Bambuddy deliberately
  4172. leaves one behind when a completion cannot be matched to it
  4173. (``_completion_belongs_to_queue_item``), and the scheduler's stranded sweep
  4174. only takes it back once the printer has sat connected and terminal for
  4175. minutes. Until then every screen-started print on that printer would
  4176. silently lose its prompt, so the row is held against ``observed_name`` by
  4177. the same comparison a completion uses: a positive disagreement means the row
  4178. is about some other run and this print is external after all.
  4179. """
  4180. logger = logging.getLogger(__name__)
  4181. try:
  4182. # A savepoint rather than a rollback of the caller's transaction on
  4183. # failure: a failed statement leaves the transaction unusable, but a
  4184. # full rollback also expires every object the caller has loaded (the
  4185. # printer, the archive it just created), and on an async session the
  4186. # next attribute read then raises instead of reloading. Everything here
  4187. # is a read, so undoing the savepoint loses nothing.
  4188. async with db.begin_nested():
  4189. return await _decide_outcome_for_external_print(db, printer_id, observed_name, logger)
  4190. except Exception as e:
  4191. logger.warning("[CALLBACK] Could not decide the outcome prompt for printer %s: %s", printer_id, e)
  4192. return False
  4193. async def _decide_outcome_for_external_print(db, printer_id: int, observed_name: str | None, logger) -> bool:
  4194. """The reads behind ``_ask_outcome_for_external_print``; see there."""
  4195. from backend.app.api.routes.settings import get_setting, setting_is_true
  4196. if not setting_is_true(await get_setting(db, "confirm_outcome_external_prints")):
  4197. return False
  4198. from backend.app.models.print_queue import PrintQueueItem
  4199. dispatched_here = await db.scalar(
  4200. select(PrintQueueItem)
  4201. .where(
  4202. PrintQueueItem.printer_id == printer_id,
  4203. PrintQueueItem.status == "printing",
  4204. )
  4205. .limit(1)
  4206. )
  4207. if dispatched_here is None:
  4208. return True
  4209. expected = await _queue_item_dispatched_name(db, dispatched_here)
  4210. observed = (observed_name or "").strip()
  4211. if not expected or not observed or _dispatched_name_matches_echo(expected, observed):
  4212. logger.info(
  4213. "[CALLBACK] Not asking for the outcome on printer %s: queue item %s is still printing, so "
  4214. "Bambuddy dispatched this run and the item's own ask-for-outcome flag decides.",
  4215. printer_id,
  4216. dispatched_here.id,
  4217. )
  4218. return False
  4219. logger.info(
  4220. "[CALLBACK] Queue item %s is still marked printing on printer %s but was dispatched as %r, not "
  4221. "%r; treating this as an externally started print.",
  4222. dispatched_here.id,
  4223. printer_id,
  4224. expected,
  4225. observed,
  4226. )
  4227. return True
  4228. async def _dispatch_outcome_confirmation_safely(
  4229. db,
  4230. printer_id: int,
  4231. printer_name: str,
  4232. data: dict,
  4233. archive_id: int,
  4234. archive_data: dict | None = None,
  4235. run_id: int | None = None,
  4236. ) -> None:
  4237. """``dispatch_outcome_confirmation`` for a caller that has more to do on ``db``.
  4238. The completion task sends the per-user print email on the same session
  4239. right after the prompt. A failed statement in the prompt leaves that
  4240. session needing a rollback, and without one the email step fails with
  4241. PendingRollbackError; the prompt is the optional part, so it must not be
  4242. the reason the email never goes out.
  4243. """
  4244. try:
  4245. await dispatch_outcome_confirmation(db, printer_id, printer_name, data, archive_id, archive_data, run_id=run_id)
  4246. except Exception as e:
  4247. logging.getLogger(__name__).error("[NOTIFY-BG] Outcome-confirmation dispatch failed: %s", e, exc_info=True)
  4248. try:
  4249. await db.rollback()
  4250. except Exception:
  4251. pass
  4252. async def _outcome_answerers(db, owner_id: int | None) -> set[int] | None:
  4253. """Who may answer a print's outcome prompt in the app: its owner and anyone
  4254. who may change everyone's archives. None (everyone on the printer) with auth
  4255. off, or for a print nobody owns."""
  4256. from sqlalchemy.orm import selectinload
  4257. from backend.app.core.auth import is_auth_enabled
  4258. from backend.app.core.permissions import Permission
  4259. from backend.app.models.user import User
  4260. if owner_id is None or not await is_auth_enabled(db):
  4261. return None
  4262. users = (await db.execute(select(User).options(selectinload(User.groups)).where(User.is_active))).scalars().all()
  4263. return {owner_id} | {u.id for u in users if u.has_permission(Permission.ARCHIVES_UPDATE_ALL.value)}
  4264. async def dispatch_outcome_confirmation(
  4265. db,
  4266. printer_id: int,
  4267. printer_name: str,
  4268. data: dict,
  4269. archive_id: int,
  4270. archive_data: dict | None = None,
  4271. *,
  4272. run_id: int | None = None,
  4273. ) -> bool:
  4274. """Emit the post-print outcome prompt for a completed print (#1898).
  4275. Gated on the run that ended (``run_id``): ``confirm_requested`` is the
  4276. opt-in (from the queue item, or from ``confirm_outcome_external_prints``
  4277. for a print Bambuddy did not start, copied onto the run at its start) and
  4278. ``user_verdict`` being unset is what makes the question still open. Mints
  4279. the run's capability token the one-tap verdict links carry: plates of one
  4280. archive each have their own question. Without a run, the archive's, as
  4281. before runs.
  4282. Lives here rather than inline in ``on_print_complete``'s notification task
  4283. because that task swallows every exception: extracted, the gate and the two
  4284. emissions can be driven by a test, which is the only thing standing between
  4285. a regression here and a farm that quietly stops asking.
  4286. Returns whether a prompt was sent.
  4287. """
  4288. logger = logging.getLogger(__name__)
  4289. from backend.app.models.archive import PrintArchive
  4290. from backend.app.models.print_log import PrintLogEntry
  4291. confirm_archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  4292. if confirm_archive is None:
  4293. return False
  4294. run = await db.get(PrintLogEntry, run_id) if run_id is not None else None
  4295. if run is not None and run.archive_id != archive_id:
  4296. run = None
  4297. # The run carries the opt-in from its start; one logged before it did
  4298. # (None) asks as its archive does.
  4299. asks = confirm_archive if run is None or run.confirm_requested is None else run
  4300. holder = run if run is not None else confirm_archive
  4301. if not (asks.confirm_requested and holder.user_verdict is None):
  4302. return False
  4303. import secrets as _secrets
  4304. # A spent token stays on the row so the one-tap route can recognise it, so
  4305. # "has a token" no longer means "answerable". A prompt going out now needs a
  4306. # live one: mint a new token whenever the stored one is already spent, or
  4307. # every button in the new message would land on the already-answered page.
  4308. if not holder.confirm_token or holder.confirm_token_used_at is not None:
  4309. holder.confirm_token = _secrets.token_urlsafe(32)
  4310. holder.confirm_token_used_at = None
  4311. await db.commit()
  4312. from backend.app.api.routes.settings import get_external_base_url, get_setting
  4313. base = await get_external_base_url(db)
  4314. if not await get_setting(db, "external_url"):
  4315. # Both the Telegram inline keyboard and the ntfy action buttons need an
  4316. # absolute URL, so an unconfigured install used to get a message body
  4317. # with two unusable relative paths and no buttons at all. The shared
  4318. # fallback at least produces tappable links; say which setting makes
  4319. # them resolve from a phone.
  4320. logger.warning(
  4321. "[#1898] No external_url configured — the outcome prompt's Good/Reject links point at %s. "
  4322. "Set Settings → External URL so they resolve away from this host.",
  4323. base,
  4324. )
  4325. token = holder.confirm_token
  4326. good_url = f"{base}/api/v1/archives/confirm/{token}/good"
  4327. reject_url = f"{base}/api/v1/archives/confirm/{token}/reject"
  4328. confirm_url = f"{base}/archives?confirm={archive_id}" + (f"&run={run.id}" if run is not None else "")
  4329. await ws_manager.send_print_confirm_request(
  4330. printer_id,
  4331. {
  4332. "archive_id": archive_id,
  4333. "run_id": run.id if run is not None else None,
  4334. "print_name": confirm_archive.print_name or confirm_archive.filename,
  4335. },
  4336. user_ids=await _outcome_answerers(db, confirm_archive.created_by_id),
  4337. )
  4338. await notification_service.on_print_confirm_request(
  4339. printer_id,
  4340. printer_name,
  4341. data,
  4342. db,
  4343. archive_data=archive_data,
  4344. good_url=good_url,
  4345. reject_url=reject_url,
  4346. confirm_url=confirm_url,
  4347. archive_id=archive_id,
  4348. run_id=run.id if run is not None else None,
  4349. )
  4350. return True
  4351. async def _printer_gone(db, printer_id: int) -> bool:
  4352. """Whether the printer was deleted. A column read: the session's loaded
  4353. Printer would otherwise answer for the database."""
  4354. from backend.app.models.printer import Printer
  4355. return await db.scalar(select(Printer.id).where(Printer.id == printer_id)) is None
  4356. async def _drop_archive_of_deleted_printer(db, printer_id: int, archive_id: int) -> bool:
  4357. """Remove the archive just written if its printer was deleted meanwhile.
  4358. The delete route sweeps the printer's archives before it commits, so a row
  4359. committed after that sweep (the check before writing passed while the
  4360. delete was still uncommitted) is left for a printer that will never report
  4361. the print's end. A row the sweep reached is the route's: deleted with the
  4362. printer's archives, or detached (printer_id NULL) when the user kept them,
  4363. and then left as it is. Either way nothing is registered for it. Run after
  4364. the archive's commit, before anything is registered for it.
  4365. """
  4366. from backend.app.models.archive import PrintArchive
  4367. if not await _printer_gone(db, printer_id):
  4368. return False
  4369. owner = await db.scalar(select(PrintArchive.printer_id).where(PrintArchive.id == archive_id))
  4370. if owner == printer_id:
  4371. logging.getLogger(__name__).info(
  4372. "Printer %s was deleted while archive %s was written; removing it", printer_id, archive_id
  4373. )
  4374. await ArchiveService(db).delete_archive(archive_id)
  4375. await forget_printer(printer_id)
  4376. return True
  4377. async def _refused_for_deleted_printer(db, printer_id: int, what: str) -> bool:
  4378. """After an IntegrityError on a print start's archive write: whether it was
  4379. the printer being deleted, forgotten then (the caller returns).
  4380. PostgreSQL: the delete route locks the printer from its archive sweep to its
  4381. commit, so an archive written or adopted for it meanwhile (one that sets
  4382. printer_id) waits and the printer's foreign key then refuses it. That is
  4383. the printer being deleted, not an error; anything else is re-raised by the
  4384. caller.
  4385. """
  4386. await db.rollback()
  4387. if not await _printer_gone(db, printer_id):
  4388. return False
  4389. logging.getLogger(__name__).info("Printer %s was deleted while %s; refused", printer_id, what)
  4390. await forget_printer(printer_id)
  4391. return True
  4392. async def _forget_if_printer_gone(printer_id: int) -> None:
  4393. """At the end of a print start: the slow steps after the last check (smart
  4394. plug, notifications, the timelapse baseline's FTP listing) can outlast the
  4395. printer's delete and its forget_printer, and then wrote state back.
  4396. (An ActivePrintSpoolman row written then is the init_db orphan sweep's on
  4397. SQLite; PostgreSQL refuses it.)
  4398. Its own session: those steps can leave the handler's needing a rollback (a
  4399. write to the archive the delete removed). Never raises: it runs inside the
  4400. fallback branch's try, whose except reports the archive as failed.
  4401. """
  4402. try:
  4403. async with async_session() as db:
  4404. gone = await _printer_gone(db, printer_id)
  4405. if gone:
  4406. await forget_printer(printer_id)
  4407. except Exception:
  4408. logging.getLogger(__name__).warning(
  4409. "Could not check whether printer %s was deleted during its print start", printer_id, exc_info=True
  4410. )
  4411. async def _announce_plate_check_failed(printer_id: int, printer_name: str, reason: str) -> None:
  4412. """Tell the UI a plate check could not run. It does not block the print,
  4413. but an unchecked plate is not an empty one."""
  4414. try:
  4415. await ws_manager.broadcast(
  4416. {"type": "plate_check_failed", "printer_id": printer_id, "printer_name": printer_name, "reason": reason}
  4417. )
  4418. except Exception as e:
  4419. logging.getLogger(__name__).warning("[PLATE CHECK] Could not announce the failed check: %s", e)
  4420. def _expected_print_keys(printer_id: int, filename: str | None, subtask_name: str | None) -> list[tuple[int, str]]:
  4421. """The ``_expected_prints`` keys a print start may be registered under."""
  4422. keys: list[tuple[int, str]] = []
  4423. if subtask_name:
  4424. keys.append((printer_id, subtask_name))
  4425. keys.append((printer_id, f"{subtask_name}.3mf"))
  4426. keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  4427. if filename:
  4428. fname = filename.split("/")[-1] if "/" in filename else filename
  4429. keys.append((printer_id, fname))
  4430. # Strip extensions to match
  4431. base = fname.replace(".gcode", "").replace(".3mf", "")
  4432. keys.append((printer_id, base))
  4433. keys.append((printer_id, f"{base}.3mf"))
  4434. return keys
  4435. async def _dispatched_by_bambuddy(db, printer_id: int, filename: str | None, subtask_name: str | None) -> bool:
  4436. """Whether this print start is one Bambuddy sent to the printer."""
  4437. if any(key in _expected_prints for key in _expected_print_keys(printer_id, filename, subtask_name)):
  4438. return True
  4439. from backend.app.models.archive import PrintArchive
  4440. from backend.app.models.print_queue import PrintQueueItem
  4441. # The queue row the scheduler sets to printing before it sends -- but only
  4442. # when it was sent as this print: a row left printing by a run whose end
  4443. # was missed names another file.
  4444. dispatched_names = (
  4445. await db.execute(
  4446. select(PrintArchive.filename)
  4447. .join(PrintQueueItem, PrintQueueItem.archive_id == PrintArchive.id)
  4448. .where(PrintQueueItem.printer_id == printer_id, PrintQueueItem.status == "printing")
  4449. )
  4450. ).scalars()
  4451. observed = [name for name in (subtask_name, (filename or "").rsplit("/", 1)[-1]) if name]
  4452. return any(
  4453. _subtask_names_match(_subtask_name_from_filename(dispatched), _subtask_name_from_filename(seen))
  4454. for dispatched in dispatched_names
  4455. if dispatched
  4456. for seen in observed
  4457. )
  4458. async def _adopt_archive_for_run(
  4459. db: AsyncSession,
  4460. archive,
  4461. printer_id: int,
  4462. *,
  4463. starting_plate_id: int | None,
  4464. effective_subtask_id: str | None,
  4465. logger: logging.Logger,
  4466. started_at: datetime | None = None,
  4467. ) -> bool:
  4468. """Start a print's run on an archive that already exists: a reprint, a
  4469. queue print of a file printed before, or the same slice sent again from the
  4470. slicer (#1634). The archive says "printing" from now, without the previous
  4471. run's outcome, answer or timelapse, and on this printer with this print's
  4472. job id. False when the printer was deleted meanwhile: the adoption is
  4473. undone or was refused, the printer is forgotten, and the caller stops.
  4474. """
  4475. from backend.app.models.archive import PrintArchive
  4476. # As it was before this adoption: put back if the printer turns
  4477. # out to have been deleted while it was adopted (below).
  4478. before_adoption = {
  4479. attr.columns[0].name: getattr(archive, attr.key)
  4480. for attr in sa_inspect(PrintArchive).column_attrs
  4481. if attr.key != "id"
  4482. }
  4483. # Update archive status to printing
  4484. archive.status = "printing"
  4485. archive.started_at = started_at or datetime.now(timezone.utc)
  4486. # The previous run's outcome is not this one's: a reprint that
  4487. # completes must not keep showing why the last one failed.
  4488. archive.failure_reason = None
  4489. archive.completed_at = None
  4490. # The previous run's answer is still on this row and the
  4491. # completion prompt is gated on ``user_verdict is None`` (#1898),
  4492. # so without a reset the second run inherits the first run's
  4493. # verdict: no prompt at all, and a green "good" badge on a run
  4494. # nobody ever judged.
  4495. if archive.confirm_requested:
  4496. archive.user_verdict = None
  4497. archive.user_verdict_source = None
  4498. archive.user_verdict_at = None
  4499. archive.confirm_token = None
  4500. archive.confirm_token_used_at = None
  4501. # Reprint of an archive reuses the source row. Without resetting
  4502. # ``timelapse_path`` _scan_for_timelapse_with_retries early-returns
  4503. # ("already has timelapse") and _capture_finish_photo_from_timelapse
  4504. # extracts the *original* print's last frame, which then ships in
  4505. # the completion notification (#1707). Clear the path so the
  4506. # scanner runs fresh; also unlink the old video file so reprints
  4507. # don't accumulate orphans in the archive directory. Photos list
  4508. # is left alone — accumulating one finish photo per run is fine.
  4509. # The print-start baseline (#2704) is stale for the same reason:
  4510. # it describes the printer before the previous run. The capture
  4511. # below overwrites it, but clear it here too so an early failure
  4512. # can't leave the scan diffing against the wrong snapshot.
  4513. #
  4514. # Unlinked only when it is the same plate printing again. Every
  4515. # plate of a Send All shares this archive, and the video on it
  4516. # can be the only copy of another plate's run -- the printer's
  4517. # was deleted once it attached (#3275). That one is left in the
  4518. # archive directory, which goes with the archive. A video whose
  4519. # plate is not on record (attached by hand, or before the
  4520. # column existed) is kept too where the 3MF has other plates,
  4521. # or cannot be read: nothing tells it apart from another plate's,
  4522. # and keeping costs one file where deleting can cost the only
  4523. # copy. On an archive with exactly one plate it is that plate's.
  4524. archive.timelapse_baseline = None
  4525. stale_timelapse_relpath = archive.timelapse_path
  4526. stale_timelapse_plate_id = archive.timelapse_plate_id
  4527. if stale_timelapse_relpath:
  4528. archive.timelapse_path = None
  4529. archive.timelapse_plate_id = None
  4530. stale_is_this_plates = stale_timelapse_plate_id == starting_plate_id
  4531. if stale_timelapse_relpath and stale_timelapse_plate_id is None and archive.file_path:
  4532. from backend.app.services.archive import plate_indexes_in_3mf
  4533. plates = await asyncio.to_thread(
  4534. plate_indexes_in_3mf,
  4535. app_settings.base_dir / archive.file_path, # SEC-PATH-OK: stored relative archive path
  4536. )
  4537. stale_is_this_plates = len(plates) == 1
  4538. if stale_timelapse_relpath and not stale_is_this_plates:
  4539. logger.info(
  4540. "Kept timelapse %s of plate %s on archive %s: plate %s is starting",
  4541. stale_timelapse_relpath,
  4542. stale_timelapse_plate_id,
  4543. archive.id,
  4544. starting_plate_id,
  4545. )
  4546. # The video is deleted once the adoption stands (below).
  4547. delete_stale_video = bool(stale_timelapse_relpath) and stale_is_this_plates
  4548. # Update on first-set OR on reprint (the queue dispatcher mints
  4549. # a fresh subtask_id per dispatch in bambu_mqtt:3647). Skipping
  4550. # the rewrite for reprints leaves the archive holding the FIRST
  4551. # run's id; if MQTT then reconnects mid-print, the reconciler
  4552. # (#1542) compares the stale stored id against the printer's
  4553. # live id, sees a mismatch, and synthesises a bogus PRINT
  4554. # COMPLETE — exactly the false-positive "Print Stopped" reported
  4555. # in #1807. Inequality check preserves the noop-on-stable-push
  4556. # behaviour the earlier `not archive.subtask_id` guard provided.
  4557. if effective_subtask_id and archive.subtask_id != effective_subtask_id:
  4558. archive.subtask_id = effective_subtask_id
  4559. # #1403 follow-up: VP-queue archives are created with
  4560. # printer_id=None at queue-add time (we don't know which
  4561. # printer will run the job yet). When the print actually
  4562. # starts on a specific printer the expected-archive lookup
  4563. # used to skip this assignment, leaving printer_id=None
  4564. # forever — which then disables the "Scan for timelapse"
  4565. # button in ArchivesPage (gated on !archive.printer_id).
  4566. if archive.printer_id != printer_id:
  4567. archive.printer_id = printer_id
  4568. adopting_id = archive.id # a rollback expires the object
  4569. try:
  4570. await db.commit()
  4571. except (IntegrityError, StaleDataError):
  4572. # The archive stays unchanged, or is gone: "delete archives"
  4573. # removed the row under the adoption, which then matched no
  4574. # row (StaleDataError) rather than being refused.
  4575. if await _refused_for_deleted_printer(db, printer_id, f"archive {adopting_id} was adopted"):
  4576. return False
  4577. raise
  4578. # The printer can be deleted between the check above and this
  4579. # commit (the route's transaction lasts seconds), and the commit
  4580. # then adopted the archive anyway: status, verdict and timelapse
  4581. # changed, and the deleted printer's id written on a row the
  4582. # route had detached or never touched. Undo the adoption.
  4583. if await _printer_gone(db, printer_id):
  4584. logger.info(
  4585. "Printer %s was deleted while archive %s was adopted; undoing the adoption",
  4586. printer_id,
  4587. archive.id,
  4588. )
  4589. # Everything as before this adoption. The printer it had
  4590. # then stays, unless it was the deleted one: the route
  4591. # detached that row (kept) or deleted it (no row matches).
  4592. previous_printer = before_adoption["printer_id"]
  4593. await db.execute(
  4594. update(PrintArchive)
  4595. .where(PrintArchive.id == archive.id)
  4596. .values(
  4597. {
  4598. **before_adoption,
  4599. "printer_id": None if previous_printer == printer_id else previous_printer,
  4600. }
  4601. )
  4602. )
  4603. await db.commit()
  4604. await forget_printer(printer_id)
  4605. return False
  4606. await _drop_replaced_plate_videos(
  4607. db,
  4608. archive.id,
  4609. starting_plate_id,
  4610. stale_timelapse_relpath if delete_stale_video else None,
  4611. logger,
  4612. )
  4613. return True
  4614. async def _claim_slicer_resend_archive(db: AsyncSession, printer_id: int, source: Path):
  4615. """The archive a print sent from the slicer adds its run to, or None for a
  4616. new one (#1634): the same G-code, so the same slice sent again in a new 3MF.
  4617. As a queue print of a file printed before (``_claim_reusable_archive`` in
  4618. the scheduler): the same owner (who started the print, nobody for a print
  4619. from the slicer) and no project, and no File Manager file, whose own prints
  4620. find its archive by the file's bytes. Not one a merge is moving. Held in
  4621. ``archive_work`` from here, with no await between, until the caller
  4622. releases it once the run is open.
  4623. Returns the archive and whether something else holds it already: a queue
  4624. dispatch uploading or printing its file, which the file must stay for.
  4625. """
  4626. from backend.app.models.archive import PrintArchive
  4627. from backend.app.services import archive_work
  4628. from backend.app.services.archive_gcode_hash import gcode_fingerprint
  4629. # Unzipping a large 3MF takes seconds: off the event loop.
  4630. fingerprint = await asyncio.to_thread(gcode_fingerprint, source)
  4631. if not fingerprint:
  4632. return None
  4633. owner_id = await _print_user_id_if_still_there(db, printer_manager.get_current_print_user(printer_id))
  4634. owner = PrintArchive.created_by_id.is_(None) if owner_id is None else PrintArchive.created_by_id == owner_id
  4635. candidates = (
  4636. await db.scalars(
  4637. select(PrintArchive)
  4638. .where(
  4639. PrintArchive.gcode_hash == fingerprint,
  4640. owner,
  4641. PrintArchive.project_id.is_(None),
  4642. PrintArchive.library_file_id.is_(None),
  4643. PrintArchive.deleted_at.is_(None),
  4644. )
  4645. .order_by(PrintArchive.id.desc())
  4646. )
  4647. ).all()
  4648. for candidate in candidates:
  4649. if archive_work.is_merging(candidate.id):
  4650. continue
  4651. in_use = archive_work.is_held(candidate.id)
  4652. archive_work.hold(candidate.id)
  4653. return candidate, in_use
  4654. return None
  4655. async def _start_slicer_resend_run(
  4656. db: AsyncSession,
  4657. archive,
  4658. printer_id: int,
  4659. *,
  4660. source: Path,
  4661. print_data: dict,
  4662. subtask_id: str | None,
  4663. started_at: datetime | None,
  4664. original_filename: str | None,
  4665. logger: logging.Logger,
  4666. file_in_use: bool = False,
  4667. ):
  4668. """Start this print's run on the archive ``_claim_slicer_resend_archive``
  4669. found (#1634). The archive takes the new file in place of its own, unless
  4670. another print of it is running, or a queue dispatch holds it
  4671. (``file_in_use``): that one still reads the file (its upload, pauses,
  4672. objects and estimate), and the G-code is the same. None when the printer
  4673. was deleted meanwhile; the caller stops.
  4674. """
  4675. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, RUN_OPEN, PrintLogEntry
  4676. printing_elsewhere = file_in_use or (
  4677. await db.scalar(
  4678. select(PrintLogEntry.id)
  4679. .where(PrintLogEntry.archive_id == archive.id, PrintLogEntry.status == RUN_OPEN)
  4680. .limit(1)
  4681. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  4682. )
  4683. is not None
  4684. )
  4685. if not printing_elsewhere:
  4686. replaced = await ArchiveService(db).archive_print(
  4687. printer_id=printer_id,
  4688. source_file=source,
  4689. print_data=print_data,
  4690. subtask_id=subtask_id,
  4691. original_filename=original_filename,
  4692. replace_archive_id=archive.id,
  4693. )
  4694. if replaced is not None:
  4695. archive = replaced
  4696. else:
  4697. # The copy failed its check, or the printer is gone (the adoption
  4698. # below says so): the run still joins, on the file it has.
  4699. logger.warning("Archive %s keeps its file: the one sent again could not be taken", archive.id)
  4700. client = printer_manager.get_client(printer_id)
  4701. if not await _adopt_archive_for_run(
  4702. db,
  4703. archive,
  4704. printer_id,
  4705. starting_plate_id=resolve_plate_id(client.state) if client else None,
  4706. effective_subtask_id=subtask_id,
  4707. logger=logger,
  4708. started_at=started_at,
  4709. ):
  4710. return None
  4711. return archive
  4712. async def on_print_start(printer_id: int, data: dict):
  4713. """Handle print start - archive the 3MF file immediately."""
  4714. logger = logging.getLogger(__name__)
  4715. # The print's start, before the 3MF download that the archive waits for
  4716. # (#3327): a minute of it would otherwise be missing from the print time.
  4717. detected_at = datetime.now(timezone.utc)
  4718. logger.info("[CALLBACK] on_print_start called for printer %s, data keys: %s", printer_id, list(data.keys()))
  4719. octoeverywhere_detection_service.reset_printer(printer_id)
  4720. try:
  4721. notify_live_activities.print_started(printer_id, printer_manager.get_status(printer_id), data)
  4722. except Exception:
  4723. logger.exception("Notify Live Activity start hook failed for printer %s", printer_id)
  4724. # Clear any stale user-stopped flag from previous print cycles
  4725. _user_stopped_printers.discard(printer_id)
  4726. _kill_switch_stopped.discard(printer_id)
  4727. # A new print starts its milestones from zero (#3211). The status path only
  4728. # resets on progress below 5 while not printing, which a printer that goes
  4729. # from FINISH at 100% straight into a new print at a preparation-phase 85%
  4730. # never shows. This callback does not fire after a Bambuddy restart (#1304)
  4731. # or on resume from pause, so it cannot repeat a milestone mid-print.
  4732. _last_progress_milestone[printer_id] = 0
  4733. _kill_switch_notification_tasks.pop(printer_id, None)
  4734. # #1721: drop any leftover pre-captured finish frame from a prior print
  4735. # so a never-consumed cache entry can't bleed into the new print's photo.
  4736. _stage22_finish_frames.pop(printer_id, None)
  4737. # #1867: same for the in-print frame bank — a queued print must not reuse
  4738. # the previous job's banked frame.
  4739. _inprint_frame_bank.pop(printer_id, None)
  4740. _inprint_frame_bank_ts.pop(printer_id, None)
  4741. # #2547: bind (or clear) the "this print ends with injected End G-code" flag.
  4742. # Unconditional, so a print Bambuddy didn't dispatch drops the previous
  4743. # print's flag instead of inheriting it.
  4744. print_dispatch_context.adopt(printer_id)
  4745. # Cancel any active bed cooldown waiter for this printer
  4746. if _bed_cool_waiters.pop(printer_id, None):
  4747. logger.info("[BED-COOL] Cancelled bed cooldown waiter for printer %s (new print started)", printer_id)
  4748. # Clear cached cover images so the new print's thumbnail is fetched fresh
  4749. from backend.app.api.routes.printers import clear_cover_cache
  4750. clear_cover_cache(printer_id)
  4751. await ws_manager.send_print_start(printer_id, data)
  4752. # Notify when the print-start AMS mapping references tray slots without spool assignments.
  4753. await notify_missing_spool_assignments_on_print_start(printer_id, data, logger)
  4754. # MQTT relay - publish print start
  4755. try:
  4756. printer_info = printer_manager.get_printer(printer_id)
  4757. if printer_info:
  4758. await mqtt_relay.on_print_start(
  4759. printer_id,
  4760. printer_info.name,
  4761. printer_info.serial_number,
  4762. data.get("filename", ""),
  4763. data.get("subtask_name", ""),
  4764. )
  4765. except Exception:
  4766. pass # Don't fail print start callback if MQTT fails
  4767. # Capture AMS tray remain%, the assignment snapshot, the dispatched plate
  4768. # and mapping, and the seeded tray-change log.
  4769. #
  4770. # Unconditional, for both inventory backends. This only *captures* — the
  4771. # writing is still split, with the internal tracker skipped at completion
  4772. # when Spoolman owns usage. Spoolman's own durable row (#1820) already
  4773. # carries its plate-scoped 3MF figures and stored mapping, but not the
  4774. # tray-change log, and that log is the only record of which spool fed
  4775. # which layers when AMS Filament Backup swaps trays mid-print. Capturing
  4776. # it on one side only would leave Spoolman users with the mid-print
  4777. # restart bug this fixes for everyone else.
  4778. try:
  4779. async with async_session() as db:
  4780. from backend.app.api.routes.settings import get_setting
  4781. from backend.app.services.usage_tracker import on_print_start as usage_on_print_start
  4782. _spoolman_on = await get_setting(db, "spoolman_enabled")
  4783. await usage_on_print_start(
  4784. printer_id,
  4785. data,
  4786. printer_manager,
  4787. db=db,
  4788. spoolman_owns_usage=bool(_spoolman_on) and _spoolman_on.lower() == "true",
  4789. )
  4790. except Exception as e:
  4791. logger.warning("Usage tracker on_print_start failed: %s", e)
  4792. # Track if notification was sent (to avoid sending twice)
  4793. notification_sent = False
  4794. # Smart plug automation: turn on plug when print starts
  4795. try:
  4796. async with async_session() as db:
  4797. await smart_plug_manager.on_print_start(printer_id, db)
  4798. except Exception as e:
  4799. logger.warning("Smart plug on_print_start failed: %s", e)
  4800. async with async_session() as db:
  4801. from backend.app.models.printer import Printer
  4802. from backend.app.services.bambu_ftp import list_files_async
  4803. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  4804. printer = result.scalar_one_or_none()
  4805. # Plate detection check - pause if objects detected on build plate
  4806. logger.info(
  4807. f"[PLATE CHECK] printer_id={printer_id}, plate_detection_enabled={printer.plate_detection_enabled if printer else 'NO PRINTER'}"
  4808. )
  4809. if printer and printer.plate_detection_enabled:
  4810. logger.info("[PLATE CHECK] ENTERING plate detection code for printer %s", printer_id)
  4811. # Release the pooled DB connection before the plate-detection camera
  4812. # work (a 2.5s light-settle sleep + FTP/camera capture). Only the
  4813. # printer SELECT has run so far — nothing to persist — so this commit
  4814. # is a data-noop that ends the read transaction and returns the
  4815. # connection to the pool during the I/O (issue #2572). expire_on_commit
  4816. # =False keeps printer.* readable; on_plate_not_empty (rare) and the
  4817. # archive lookups below re-acquire a fresh connection on next execute.
  4818. await db.commit()
  4819. try:
  4820. from backend.app.services.plate_detection import check_plate_empty
  4821. # Build ROI tuple from printer settings if available
  4822. roi = None
  4823. if all(
  4824. [
  4825. printer.plate_detection_roi_x is not None,
  4826. printer.plate_detection_roi_y is not None,
  4827. printer.plate_detection_roi_w is not None,
  4828. printer.plate_detection_roi_h is not None,
  4829. ]
  4830. ):
  4831. roi = (
  4832. printer.plate_detection_roi_x,
  4833. printer.plate_detection_roi_y,
  4834. printer.plate_detection_roi_w,
  4835. printer.plate_detection_roi_h,
  4836. )
  4837. # The light is held, not switched, so a live view opened
  4838. # meanwhile keeps it when the check lets go (SP-C11). Every
  4839. # printer, whatever the camera light mode: detection needs it.
  4840. plate_photo_data = None
  4841. objects_detected = False
  4842. check_failure: str | None = None
  4843. async with camera_light(printer, any_mode=True, settle=2.5):
  4844. logger.info("[PLATE CHECK] Running plate detection for printer %s", printer_id)
  4845. plate_result = await check_plate_empty(
  4846. printer_id=printer_id,
  4847. ip_address=printer.ip_address,
  4848. access_code=printer.access_code,
  4849. model=printer.model,
  4850. include_debug_image=False,
  4851. external_camera_url=printer.external_camera_url,
  4852. external_camera_type=printer.external_camera_type,
  4853. use_external=printer.external_camera_enabled,
  4854. roi=roi,
  4855. external_camera_snapshot_url=printer.external_camera_snapshot_url,
  4856. )
  4857. # No reference to compare against is a check that did not
  4858. # run either: say so rather than call the plate empty.
  4859. check_failure = getattr(plate_result, "failure_reason", None) or (
  4860. "not_calibrated" if plate_result.needs_calibration else None
  4861. )
  4862. objects_detected = not check_failure and not plate_result.is_empty
  4863. if objects_detected:
  4864. # Objects detected - pause the print!
  4865. logger.warning(
  4866. f"[PLATE CHECK] Objects detected on plate for printer {printer_id}! "
  4867. f"Confidence: {plate_result.confidence:.0%}, Diff: {plate_result.difference_percent:.1f}%"
  4868. )
  4869. pause_client = printer_manager.get_client(printer_id)
  4870. if pause_client:
  4871. pause_client.pause_print()
  4872. logger.info("[PLATE CHECK] Print paused for printer %s", printer_id)
  4873. # Snapshot while the light's still on — restoring it first
  4874. # would leave the notification with a dark photo.
  4875. try:
  4876. plate_photo_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  4877. except Exception as snap_err:
  4878. logger.warning(
  4879. "[PLATE CHECK] Failed to capture snapshot for printer %s: %s", printer_id, snap_err
  4880. )
  4881. if objects_detected:
  4882. # Send notification about plate not empty
  4883. await ws_manager.broadcast(
  4884. {
  4885. "type": "plate_not_empty",
  4886. "printer_id": printer_id,
  4887. "printer_name": printer.name,
  4888. "message": f"Objects detected on build plate! Print paused. (Diff: {plate_result.difference_percent:.1f}%)",
  4889. }
  4890. )
  4891. # Also send push notification
  4892. try:
  4893. await notification_service.on_plate_not_empty(
  4894. printer_id=printer_id,
  4895. printer_name=printer.name,
  4896. db=db,
  4897. difference_percent=plate_result.difference_percent,
  4898. image_data=plate_photo_data,
  4899. )
  4900. except Exception as notif_err:
  4901. logger.warning("[PLATE CHECK] Failed to send notification: %s", notif_err)
  4902. elif check_failure:
  4903. logger.warning(
  4904. "[PLATE CHECK] Plate check could not run for printer %s (%s): %s; proceeding with print",
  4905. printer_id,
  4906. check_failure,
  4907. plate_result.message,
  4908. )
  4909. await _announce_plate_check_failed(printer_id, printer.name, check_failure)
  4910. else:
  4911. logger.info("[PLATE CHECK] Plate is empty for printer %s, proceeding with print", printer_id)
  4912. except Exception as plate_err:
  4913. # Don't block print on plate detection errors
  4914. logger.warning(
  4915. "[PLATE CHECK] Plate check could not run for printer %s: %s; proceeding with print",
  4916. printer_id,
  4917. plate_err,
  4918. )
  4919. await _announce_plate_check_failed(printer_id, printer.name, "error")
  4920. if not printer:
  4921. logger.info("[CALLBACK] Skipping archive - printer not found in database")
  4922. if not notification_sent:
  4923. await _send_print_start_notification(printer_id, data, logger=logger)
  4924. return
  4925. # Skip the printer's own jobs — a calibration run is not a user's print.
  4926. # See is_internal_printer_job for what counts and why both fields are
  4927. # tested; the pressure-advance line reports as a subtask name with no
  4928. # /usr/ path, which the old prefix-only test here missed entirely.
  4929. #
  4930. # No notification either. The event describes the printer calibrating
  4931. # itself, so "Print started" is as wrong as the archive was, and the
  4932. # matching completion is suppressed in on_print_complete for the same
  4933. # reason. Checked before the auto-archive switches, whose "off" path
  4934. # still notifies.
  4935. # A /usr/ path is the printer's own file. A name alone is not proof: a
  4936. # print Bambuddy dispatched is never one, whatever it is called.
  4937. # Dispatched = an expected print, or, after a restart that lost those,
  4938. # a queue job printing on the printer.
  4939. if is_internal_printer_job(data.get("filename", ""), data.get("subtask_name", "")) and (
  4940. (data.get("filename") or "").startswith("/usr/")
  4941. or not await _dispatched_by_bambuddy(db, printer_id, data.get("filename", ""), data.get("subtask_name", ""))
  4942. ):
  4943. logger.info(
  4944. "[CALLBACK] Skipping archive — internal printer job detected: filename=%s, subtask=%s",
  4945. data.get("filename", ""),
  4946. data.get("subtask_name", ""),
  4947. )
  4948. return
  4949. # The global switch is the master: off, no printer archives prints
  4950. # started outside Bambuddy; on, each printer's own switch decides.
  4951. from backend.app.api.routes.settings import get_setting
  4952. _global_auto_archive = await get_setting(db, "auto_archive")
  4953. _global_archives = _global_auto_archive is None or _global_auto_archive.lower() == "true"
  4954. archives_prints = printer.auto_archive and _global_archives
  4955. if not archives_prints:
  4956. # auto-archive disabled — check if there's an expected print (dispatched
  4957. # by BamBuddy via queue/reprint) that already has an archive to promote.
  4958. # If so, fall through to the expected-print handling below so the archive
  4959. # is tracked in _active_prints and usage tracking works at completion.
  4960. _fn = data.get("filename", "")
  4961. _sn = data.get("subtask_name", "")
  4962. _check_keys: list[tuple[int, str]] = []
  4963. if _sn:
  4964. _check_keys += [
  4965. (printer_id, _sn),
  4966. (printer_id, f"{_sn}.3mf"),
  4967. (printer_id, f"{_sn}.gcode.3mf"),
  4968. ]
  4969. if _fn:
  4970. _base_fn = _fn.split("/")[-1] if "/" in _fn else _fn
  4971. _check_keys.append((printer_id, _base_fn))
  4972. _no_archive_base = _base_fn.replace(".gcode", "").replace(".3mf", "")
  4973. _check_keys += [
  4974. (printer_id, _no_archive_base),
  4975. (printer_id, f"{_no_archive_base}.3mf"),
  4976. ]
  4977. _has_expected = any(k in _expected_prints for k in _check_keys)
  4978. if not _has_expected:
  4979. # No expected print — truly external print (started from slicer/touchscreen).
  4980. # Say which switch it was: before 1.3.0 the global one was saved
  4981. # and never read, so an install upgraded with it off stopped
  4982. # archiving with nothing to point at it (#3477).
  4983. if not _global_archives:
  4984. logger.info(
  4985. "[CALLBACK] Skipping archive: the global 'Auto-archive prints' switch "
  4986. "(Settings > General > Archive Settings) is off, and Bambuddy did not start this print"
  4987. )
  4988. else:
  4989. logger.info(
  4990. "[CALLBACK] Skipping archive: auto-archive is off for printer %s, "
  4991. "and Bambuddy did not start this print",
  4992. printer_id,
  4993. )
  4994. if not notification_sent:
  4995. _no_archive_creator: int | None = None
  4996. for _key in _check_keys:
  4997. _expected_prints.pop(_key, None)
  4998. _expected_print_registered_at.pop(_key, None)
  4999. popped_creator = _expected_print_creators.pop(_key, None)
  5000. if _no_archive_creator is None:
  5001. _no_archive_creator = popped_creator
  5002. _creator_data = {"created_by_id": _no_archive_creator} if _no_archive_creator else None
  5003. await _send_print_start_notification(printer_id, data, _creator_data, logger)
  5004. return
  5005. else:
  5006. logger.info("[CALLBACK] auto_archive disabled but expected print found — promoting archive")
  5007. # Get the filename and subtask_name
  5008. filename = data.get("filename", "")
  5009. subtask_name = data.get("subtask_name", "")
  5010. # MQTT subtask_id uniquely identifies a print job on the printer. When
  5011. # present, it lets us match an archive across a backend restart (#972):
  5012. # same id → same print → resume the existing row instead of cancelling
  5013. # it and recreating from scratch (which loses started_at). Treat "0"
  5014. # and "" as absent — Bambu reports "0" for non-cloud / local prints.
  5015. raw_mqtt = data.get("raw_data") or {}
  5016. subtask_id = raw_mqtt.get("subtask_id")
  5017. if subtask_id is not None:
  5018. subtask_id = str(subtask_id).strip()
  5019. if subtask_id in ("", "0"):
  5020. subtask_id = None
  5021. logger.info("[CALLBACK] Print start detected - filename: %s, subtask: %s", filename, subtask_name)
  5022. if not filename and not subtask_name:
  5023. # Send notification without archive data (no filename)
  5024. logger.info("[CALLBACK] Skipping archive - no filename or subtask_name")
  5025. if not notification_sent:
  5026. await _send_print_start_notification(printer_id, data, logger=logger)
  5027. return
  5028. # Check if this is an expected print from reprint/scheduled
  5029. expected_keys = _expected_print_keys(printer_id, filename, subtask_name)
  5030. expected_archive_id = None
  5031. # Who queued it, taken with the entries it was registered beside: left
  5032. # behind once those are gone (the archive deleted mid-dispatch, or the
  5033. # start notification already sent), the expiry never saw it, and an
  5034. # unrelated later print of that name was attributed to that user (EP-2).
  5035. fallback_creator = None
  5036. for key in expected_keys:
  5037. expected_archive_id = _expected_prints.pop(key, None)
  5038. _expected_print_registered_at.pop(key, None)
  5039. if expected_archive_id:
  5040. # Clean up other possible keys for this print
  5041. for other_key in expected_keys:
  5042. _expected_prints.pop(other_key, None)
  5043. _expected_print_registered_at.pop(other_key, None)
  5044. creator = _expected_print_creators.pop(other_key, None)
  5045. if fallback_creator is None:
  5046. fallback_creator = creator
  5047. break
  5048. if expected_archive_id:
  5049. # This is a reprint/scheduled print - use existing archive, don't create new one
  5050. logger.info("Using expected archive %s for print (skipping duplicate)", expected_archive_id)
  5051. # Bambuddy dispatched it: the kill switch's fast path keeps it as
  5052. # Bambuddy's once the expected entry is consumed here.
  5053. _owned_print_archive_ids.add(expected_archive_id)
  5054. from backend.app.models.archive import PrintArchive
  5055. result = await db.execute(select(PrintArchive).where(PrintArchive.id == expected_archive_id))
  5056. archive = result.scalar_one_or_none()
  5057. # Deleted after this handler loaded it (the plate check waits
  5058. # seconds) but before the route forgot the expected print: the
  5059. # archive the route kept or deleted is left as the route left it.
  5060. if archive and await _printer_gone(db, printer_id):
  5061. logger.info(
  5062. "Printer %s was deleted before its print started; not adopting archive %s", printer_id, archive.id
  5063. )
  5064. await forget_printer(printer_id)
  5065. return
  5066. if archive:
  5067. # Persist a restart-stable id so a later restart resumes this
  5068. # archive by subtask_id instead of name-matching + duplicating
  5069. # it (#1485). The printer often hasn't echoed subtask_id back
  5070. # this soon after dispatch, so fall back to the id Bambuddy
  5071. # minted when it sent the print command. Scoped to this
  5072. # expected-print branch on purpose: an expected match means
  5073. # Bambuddy dispatched this exact print in this process, so the
  5074. # client's last-dispatch id genuinely belongs to it — using it
  5075. # for an externally-started print could mis-tag the archive.
  5076. effective_subtask_id = subtask_id
  5077. if not effective_subtask_id:
  5078. _client = printer_manager.get_client(printer_id)
  5079. _dispatched = getattr(_client, "last_dispatch_subtask_id", None) if _client else None
  5080. if _dispatched:
  5081. effective_subtask_id = str(_dispatched).strip() or None
  5082. starting_plate_id = _print_plate_ids.get((printer_id, expected_archive_id))
  5083. if not await _adopt_archive_for_run(
  5084. db,
  5085. archive,
  5086. printer_id,
  5087. starting_plate_id=starting_plate_id,
  5088. effective_subtask_id=effective_subtask_id,
  5089. logger=logger,
  5090. ):
  5091. return
  5092. # Track as active print
  5093. _bind_active_print(printer_id, archive.id, archive.filename, filename, subtask_name)
  5094. await _open_run_for(archive.id, printer_id, subtask_id)
  5095. # Start timelapse session if external camera is enabled (#1353).
  5096. # Queue / VP-dispatched prints land here in the expected-archive
  5097. # branch and used to skip start_session entirely — frames were
  5098. # never captured and the post-print stitch silently returned None.
  5099. _maybe_start_layer_timelapse(printer, printer_id, archive.id)
  5100. # Inject ams_mapping into usage tracker session — the session was created
  5101. # before expected-print promotion, so it may have ams_mapping=None when
  5102. # the MQTT request topic subscription failed (common on P1S/A1).
  5103. _stored_map = _print_ams_mappings.get((printer_id, expected_archive_id))
  5104. _stored_plate_id = _print_plate_ids.get((printer_id, expected_archive_id))
  5105. if _stored_map or _stored_plate_id is not None:
  5106. try:
  5107. from backend.app.services.usage_tracker import _active_sessions
  5108. _ut_session = _active_sessions.get(printer_id)
  5109. if _ut_session and _stored_map and not _ut_session.ams_mapping:
  5110. _ut_session.ams_mapping = _stored_map
  5111. logger.info("[CALLBACK] Injected ams_mapping into usage tracker session: %s", _stored_map)
  5112. # plate_id injection covers direct-Print of plate N of a multi-plate
  5113. # 3MF — queue prints already capture it via the on_print_start queue
  5114. # lookup, but direct-Print never goes through the queue (#1697).
  5115. if _ut_session and _stored_plate_id is not None and _ut_session.plate_id is None:
  5116. _ut_session.plate_id = _stored_plate_id
  5117. logger.info("[CALLBACK] Injected plate_id into usage tracker session: %s", _stored_plate_id)
  5118. except Exception:
  5119. pass
  5120. # Set up energy tracking (#941: persist start on archive row)
  5121. await _record_energy_start(archive, printer_id, db, context="expected-print")
  5122. await ws_manager.send_archive_updated(
  5123. {
  5124. "id": archive.id,
  5125. "status": "printing",
  5126. }
  5127. )
  5128. # Send notification with archive data (reprint/scheduled)
  5129. if not notification_sent:
  5130. # Use archive's created_by_id; fall back to the creator registered via
  5131. # register_expected_print (handles library-file-based queue items where
  5132. # the freshly-created archive has no created_by_id yet), taken above.
  5133. archive_data = {
  5134. "print_time_seconds": _run_print_time_seconds(archive, printer_id),
  5135. "created_by_id": archive.created_by_id or fallback_creator,
  5136. }
  5137. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5138. # Extract printable objects from the archived 3MF file
  5139. _load_objects_from_archive(archive, printer_id, logger)
  5140. if archive.file_path:
  5141. _load_pauses_from_3mf(app_settings.base_dir / archive.file_path, printer_id)
  5142. # Store Spoolman tracking data for per-filament usage reporting
  5143. try:
  5144. await _store_spoolman_print_data(
  5145. printer_id,
  5146. archive.id,
  5147. archive.file_path,
  5148. db,
  5149. printer_manager,
  5150. ams_mapping=_get_start_ams_mapping(data, printer_id, archive.id),
  5151. plate_id=_get_start_plate_id(printer_id, archive.id),
  5152. )
  5153. except Exception as e:
  5154. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  5155. # Price the print from its spools now rather than only at completion (#3261)
  5156. _schedule_archive_cost_estimate(
  5157. printer_id,
  5158. archive.id,
  5159. printer_manager,
  5160. ams_mapping=_get_start_ams_mapping(data, printer_id, archive.id),
  5161. plate_id=_get_start_plate_id(printer_id, archive.id),
  5162. session_factory=async_session,
  5163. )
  5164. # Capture timelapse file baseline for snapshot-diff on completion
  5165. # (mirrors the new-archive branch). Queue / VP-dispatched prints
  5166. # hit this branch — without the baseline the completion-time scan
  5167. # falls into its "take baseline now" fallback, which snapshots
  5168. # AFTER the new MP4 already exists and never matches a diff
  5169. # (#1403 follow-up — see pwostran's 2026-05-18 support bundle).
  5170. await _capture_timelapse_baseline_at_start(printer, printer_id, logger, archive_id=archive.id)
  5171. await _forget_if_printer_gone(printer_id)
  5172. return # Skip creating a new archive
  5173. # The archive was deleted after the job was queued.
  5174. if not archives_prints:
  5175. # Only the expected print brought a printer that does not
  5176. # archive this far; with it gone, nothing is archived.
  5177. return
  5178. # Otherwise archive the print anew rather than drop it, so it still
  5179. # gets its notification, usage tracking and history.
  5180. logger.warning(
  5181. "Expected archive %s for printer %s no longer exists; archiving the print anew",
  5182. expected_archive_id,
  5183. printer_id,
  5184. )
  5185. # Check if there's already a "printing" archive for this printer/file
  5186. # This prevents duplicates when backend restarts during an active print
  5187. from backend.app.models.archive import PrintArchive
  5188. existing_archive: PrintArchive | None = None
  5189. resumed_by_run = False
  5190. # Best match: this job's run, open on the printer since the print
  5191. # started. The archive row is shared by every plate of it, and its
  5192. # printer, job id and status are the latest start's, which may be
  5193. # another printer's.
  5194. if subtask_id:
  5195. from backend.app.services.print_log import open_run_on_printer
  5196. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  5197. if run is not None and run.archive_id is not None:
  5198. existing_archive = await db.scalar(
  5199. select(PrintArchive).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  5200. )
  5201. resumed_by_run = existing_archive is not None
  5202. # Preferred match: subtask_id equality. MQTT reports the same subtask_id
  5203. # across a backend restart for the same print, so this is the most
  5204. # reliable way to reattach. Only a row still printing: a cancelled one
  5205. # has been through completion (usage, history, notifications) already,
  5206. # and reopening it would count the print twice.
  5207. if subtask_id and existing_archive is None:
  5208. by_id = await db.execute(
  5209. select(PrintArchive)
  5210. .where(PrintArchive.printer_id == printer_id)
  5211. .where(PrintArchive.subtask_id == subtask_id)
  5212. .where(PrintArchive.status == "printing")
  5213. .order_by(PrintArchive.created_at.desc())
  5214. .limit(1)
  5215. )
  5216. existing_archive = by_id.scalar_one_or_none()
  5217. # Fallback match: name-based lookup. Kept as-is for prints whose
  5218. # subtask_id is missing ("0" / local / non-cloud prints).
  5219. if existing_archive is None:
  5220. check_name = subtask_name or filename.split("/")[-1].replace(".gcode", "").replace(".3mf", "")
  5221. archive_filenames = [f"{check_name}.3mf", f"{check_name}.gcode.3mf"]
  5222. # A print started from the printer's screen names its file in full,
  5223. # so the forms above double its extension. The file itself is the
  5224. # one exact match, and nothing looser is added (#3009).
  5225. if _subtask_is_the_file(subtask_name, filename):
  5226. archive_filenames.append(subtask_name)
  5227. existing = await db.execute(
  5228. select(PrintArchive)
  5229. .where(PrintArchive.printer_id == printer_id)
  5230. .where(PrintArchive.status == "printing")
  5231. .where(
  5232. or_(
  5233. PrintArchive.print_name == check_name,
  5234. PrintArchive.filename.in_(archive_filenames),
  5235. )
  5236. )
  5237. .order_by(PrintArchive.created_at.desc())
  5238. .limit(1)
  5239. )
  5240. existing_archive = existing.scalar_one_or_none()
  5241. if existing_archive:
  5242. # subtask_id match → always resume, regardless of age. Same print,
  5243. # just a backend restart.
  5244. subtask_match = resumed_by_run or bool(subtask_id and existing_archive.subtask_id == subtask_id)
  5245. if subtask_match:
  5246. logger.info("Resuming archive %s on subtask_id match (%s)", existing_archive.id, subtask_id)
  5247. _bind_active_print(printer_id, existing_archive.id, existing_archive.filename, filename, subtask_name)
  5248. await _open_run_for(existing_archive.id, printer_id, subtask_id)
  5249. _mark_owned_if_dispatched(existing_archive)
  5250. if await _run_needs_energy_start(existing_archive, printer_id, db):
  5251. await _record_energy_start(existing_archive, printer_id, db, context="subtask-resume")
  5252. if not notification_sent:
  5253. archive_data = {
  5254. "print_time_seconds": _run_print_time_seconds(existing_archive, printer_id),
  5255. "created_by_id": existing_archive.created_by_id,
  5256. }
  5257. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5258. _load_objects_from_archive(existing_archive, printer_id, logger)
  5259. if existing_archive.file_path:
  5260. _load_pauses_from_resumed_archive(existing_archive, printer_id)
  5261. return
  5262. # Name-match only (no subtask_id to anchor on): decide resume vs.
  5263. # stale from the printer's *current* progress, not wall-clock age.
  5264. # A genuinely long print used to trip a blind 4h cutoff and have its
  5265. # live archive cancelled + duplicated on every backend restart
  5266. # (#1485). If the printer reports real progress, this name-matched
  5267. # 'printing' archive IS that ongoing print — resume it whatever its
  5268. # age. Only treat it as a stale leftover when the printer clearly
  5269. # shows a different, freshly-started print: near-0% progress on an
  5270. # archive far too old to still be at 0%. Unknown progress (printer
  5271. # not connected) never cancels — resuming is the safe default.
  5272. archive_age = datetime.now(timezone.utc) - existing_archive.created_at.replace(tzinfo=timezone.utc)
  5273. live_status = printer_manager.get_status(printer_id)
  5274. live_progress = getattr(live_status, "progress", None) if live_status else None
  5275. looks_stale = (
  5276. live_progress is not None and live_progress < 1.0 and archive_age.total_seconds() > 2 * 60 * 60
  5277. )
  5278. # Two subtask ids are two prints, however young the archive: one
  5279. # whose completion was missed must not swallow the next print of
  5280. # the same file. Except Bambuddy's own reprint, which reuses the
  5281. # row and mints a fresh id (#1807): the id it last sent says so,
  5282. # and after a restart, which forgets it, this printer's queue job
  5283. # running the archive.
  5284. if (
  5285. not looks_stale
  5286. and subtask_id
  5287. and existing_archive.subtask_id
  5288. and existing_archive.subtask_id != subtask_id
  5289. ):
  5290. _client = printer_manager.get_client(printer_id)
  5291. _dispatched = str(getattr(_client, "last_dispatch_subtask_id", None) or "").strip()
  5292. if _dispatched:
  5293. looks_stale = _dispatched != subtask_id
  5294. else:
  5295. from backend.app.models.print_queue import PrintQueueItem as _PQI
  5296. queue_run = await db.scalar(
  5297. select(_PQI.id)
  5298. .where(_PQI.archive_id == existing_archive.id)
  5299. .where(_PQI.printer_id == printer_id)
  5300. .where(_PQI.status == "printing")
  5301. .limit(1)
  5302. )
  5303. looks_stale = queue_run is None
  5304. if looks_stale:
  5305. logger.warning(
  5306. "Found stale 'printing' archive %s (age: %s, printer progress %s%%, subtask %s, printer reports %s)"
  5307. " — marking cancelled and creating new archive",
  5308. existing_archive.id,
  5309. archive_age,
  5310. live_progress,
  5311. existing_archive.subtask_id,
  5312. subtask_id,
  5313. )
  5314. existing_archive.status = "cancelled"
  5315. # Canonical key, not a sentence (issue #2974). "No status update
  5316. # received" is what both stale paths actually observed; which of
  5317. # the two it was is already carried by ``status`` -- cancelled
  5318. # here, the reconciled outcome at the reconnect site -- so one
  5319. # key loses no information and gives the Statistics breakdown a
  5320. # single bucket instead of two untranslatable prose strings.
  5321. existing_archive.failure_reason = "noStatusUpdate"
  5322. # And its run, as before runs were opened at the start: a new
  5323. # print starts on the printer, so whatever is open there ended
  5324. # unseen. Left open, the next reconnect would close it again.
  5325. from backend.app.services.print_log import discard_open_runs
  5326. discarded = await discard_open_runs(db, printer_id)
  5327. await db.commit()
  5328. await _announce_discarded_runs(discarded, printer_id)
  5329. # Its print keys go with it; the new archive registers its own.
  5330. for _key in [k for k, v in _active_prints.items() if v == existing_archive.id and k[0] == printer_id]:
  5331. del _active_prints[_key]
  5332. # Fall through to create new archive (don't return)
  5333. else:
  5334. logger.info(
  5335. f"Skipping duplicate - already have printing archive {existing_archive.id} for {check_name}"
  5336. )
  5337. # Track this as the active print
  5338. _bind_active_print(printer_id, existing_archive.id, existing_archive.filename, filename, subtask_name)
  5339. _mark_owned_if_dispatched(existing_archive)
  5340. # Attach subtask_id retroactively so future restarts can resume.
  5341. # Compare for inequality (not "is empty") to also pick up reprint
  5342. # dispatches that mint a fresh id — see #1807 for the bogus
  5343. # "Print Stopped" the strict-empty guard caused on reconnect.
  5344. if subtask_id and existing_archive.subtask_id != subtask_id:
  5345. existing_archive.subtask_id = subtask_id
  5346. await db.commit()
  5347. await _open_run_for(existing_archive.id, printer_id, subtask_id)
  5348. # Also set up energy tracking if not already tracked (#941: persisted column)
  5349. if await _run_needs_energy_start(existing_archive, printer_id, db):
  5350. await _record_energy_start(existing_archive, printer_id, db, context="existing-printing")
  5351. # Send notification with archive data (existing archive)
  5352. if not notification_sent:
  5353. archive_data = {
  5354. "print_time_seconds": _run_print_time_seconds(existing_archive, printer_id),
  5355. "created_by_id": existing_archive.created_by_id,
  5356. }
  5357. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5358. # Extract printable objects from the archived 3MF file
  5359. _load_objects_from_archive(existing_archive, printer_id, logger)
  5360. if existing_archive.file_path:
  5361. _load_pauses_from_resumed_archive(existing_archive, printer_id)
  5362. return
  5363. # Build list of possible 3MF filenames to try
  5364. possible_names = []
  5365. # Bambu printers typically store files as "Name.gcode.3mf"
  5366. # The subtask_name is usually the best source for the filename
  5367. if subtask_name:
  5368. if _subtask_is_the_file(subtask_name, filename):
  5369. # A print started from the printer's own screen reports the
  5370. # file's full name, extension included. That is the file, so
  5371. # no extension is appended to it (#3009).
  5372. possible_names.append(subtask_name)
  5373. else:
  5374. # Try common Bambu naming patterns
  5375. possible_names.append(f"{subtask_name}.gcode.3mf")
  5376. possible_names.append(f"{subtask_name}.3mf")
  5377. # Try original filename with .3mf extension
  5378. if filename:
  5379. # Extract just the filename part, not the full path
  5380. fname = filename.split("/")[-1] if "/" in filename else filename
  5381. if fname.endswith(".3mf"):
  5382. possible_names.append(fname)
  5383. elif fname.endswith(".gcode"):
  5384. base = fname.rsplit(".", 1)[0]
  5385. possible_names.append(f"{base}.gcode.3mf")
  5386. possible_names.append(f"{base}.3mf")
  5387. else:
  5388. possible_names.append(f"{fname}.gcode.3mf")
  5389. possible_names.append(f"{fname}.3mf")
  5390. # Also try with spaces converted to underscores (Bambu Studio may normalize filenames)
  5391. space_variants = []
  5392. for name in possible_names:
  5393. if " " in name:
  5394. space_variants.append(name.replace(" ", "_"))
  5395. possible_names.extend(space_variants)
  5396. # Remove duplicates while preserving order
  5397. seen = set()
  5398. possible_names = [x for x in possible_names if not (x in seen or seen.add(x))]
  5399. logger.info("Trying filenames: %s", possible_names)
  5400. # Release the pooled DB connection before the 3MF FTP download. Reaching
  5401. # here means none of the expected-/existing-archive write branches ran
  5402. # (they all return earlier) — only SELECTs have executed on this path, so
  5403. # this commit persists nothing; it ends the read transaction so the
  5404. # connection returns to the pool during the download. That download tries
  5405. # up to five remote paths per candidate filename with retry/backoff and
  5406. # can run for minutes under FTP contention; holding the session across it
  5407. # pinned one pooled connection idle-in-transaction (issue #2572). No DB
  5408. # work runs during the download — the new-archive writes below re-acquire
  5409. # a fresh connection, and expire_on_commit=False keeps printer.* readable.
  5410. await db.commit()
  5411. # Try to find and download the 3MF file
  5412. temp_path = None
  5413. downloaded_filename = None
  5414. # Cache check: cover endpoint may have already pulled this 3MF during
  5415. # the print (frontend opens the card and shows the thumbnail) — reuse
  5416. # that file instead of re-downloading 36MB over the same FTP link that
  5417. # just served it (#972). The cache keys on a normalized filename so
  5418. # variants like "X", "X.3mf", "X.gcode.3mf" all collapse to one entry.
  5419. for try_filename in possible_names:
  5420. if not try_filename.endswith(".3mf"):
  5421. continue
  5422. cached = get_cached_3mf(printer_id, try_filename)
  5423. if cached:
  5424. logger.info("Reusing cached 3MF from %s (avoided duplicate FTP)", cached)
  5425. temp_path = cached
  5426. downloaded_filename = try_filename
  5427. break
  5428. # Does this printer keep the sliced file somewhere FTPS can reach? On
  5429. # H2-series and P2S the answer is routinely no — the file stays on
  5430. # internal eMMC and port 990 only ever serves external storage — and
  5431. # then the whole sweep below (six filenames x five directories x four
  5432. # retries, then the directory walk) is ~110 connections that cannot
  5433. # succeed. Skip it and say why (#2780).
  5434. storage = print_file_reachable_over_ftp(printer_manager.get_status(printer_id))
  5435. # Set when a lookup is abandoned because the printer's FTPS cool-off is
  5436. # running rather than because the file is somewhere unreachable. The
  5437. # distinction is the whole of #2957: one is permanent, the other clears
  5438. # in minutes with the file still sitting on the printer.
  5439. blocked_by_ftps_cooloff = False
  5440. # Set when a probe reached the printer and still came back without the
  5441. # file -- a timeout mid-transfer, a refused connection, anything that is
  5442. # not a clean "not here". A 550 raises FileNotOnPrinterError and is
  5443. # caught by name below, so a file that genuinely is not on the card
  5444. # leaves this False and schedules nothing. Anything else means the
  5445. # transfer, not the file, is what failed, and that does not last (#3063).
  5446. ftp_transfer_failed = False
  5447. # The narrower case of the above: the directory walk *listed* a file
  5448. # for this print and could not fetch it, so the file is on the card
  5449. # under a name that matches. The only failure that rules out an
  5450. # earlier run's 3MF standing in for it (#3307).
  5451. walk_transfer_failed = False
  5452. # Paths the sweep reached and could not transfer -- a timeout, not a
  5453. # 550. The directory walk lists the same folders and would download
  5454. # the same file again: on a slow A1 that was a second 300 s on the
  5455. # printer's one FTPS slot, at print start, for a file the recovery
  5456. # retry fetches with a deadline twice as long (#3308).
  5457. sweep_failed_paths: set[str] = set()
  5458. # The print's name, for a fallback archive whose `subtask_name` the
  5459. # plate guard below had to disown. Display only, and deliberately kept
  5460. # apart from `subtask_name`: that variable is what every file lookup
  5461. # here is built from, and once a name has been shown to fetch another
  5462. # plate's 3MF it must not key `_active_prints` either, or the cover
  5463. # endpoint hands the same contradicted file to
  5464. # `_recover_fallback_archive` and fills the row in with it (#3126).
  5465. display_name_after_plate_reject: str | None = None
  5466. # Get FTP retry settings
  5467. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  5468. # ...but "the printer put it on eMMC" is where it went, not whether we
  5469. # can read it. An H2D with a card in mirrors the job to /cache and
  5470. # serves it happily, and skipping on the URL alone cost that reporter
  5471. # every archive for two days (#2856). So ask the printer instead of
  5472. # guessing: the dispatch named the exact file, which is one connection
  5473. # walking five paths rather than the sweep's ~110. Only when the probe
  5474. # comes back empty does the verdict's reason stand.
  5475. if not storage.reachable and not downloaded_filename and storage.probe_filename:
  5476. if ftps_handshake_blocked(printer.ip_address):
  5477. # Deliberately NOT recorded as a cool-off give-up. This branch
  5478. # only runs on an unreachable verdict, and that verdict is the
  5479. # honest, permanent reason the archive is empty — the probe was
  5480. # a long shot on top of it. Blaming the cool-off here would
  5481. # schedule a retry for a file sitting on internal eMMC, which is
  5482. # the sweep #2780 removed (#2957).
  5483. logger.debug(
  5484. "Not probing for %s on printer %s: its file service is not answering over TLS",
  5485. storage.probe_filename,
  5486. printer_id,
  5487. )
  5488. else:
  5489. probe_path = app_settings.archive_dir / "temp" / storage.probe_filename
  5490. probe_path.parent.mkdir(parents=True, exist_ok=True)
  5491. try:
  5492. probe_hit = await download_file_try_paths_async(
  5493. printer.ip_address,
  5494. printer.access_code,
  5495. ftp_probe_paths(storage.probe_filename),
  5496. probe_path,
  5497. socket_timeout=ftp_timeout,
  5498. printer_model=printer.model,
  5499. )
  5500. except Exception as e:
  5501. logger.debug("3MF probe for %s failed: %s", storage.probe_filename, e)
  5502. probe_hit = False
  5503. if probe_hit:
  5504. downloaded_filename = storage.probe_filename
  5505. temp_path = probe_path
  5506. cache_3mf_download(printer_id, downloaded_filename, probe_path)
  5507. # Naming the path, not just the file: a printer that keeps
  5508. # uploads around for weeks can serve a same-named copy of an
  5509. # earlier slice, and without the directory in the log that
  5510. # mismatch is invisible rather than merely rare (#1820).
  5511. logger.info(
  5512. "Found %s at %s over FTPS for printer %s even though the printer reported %s",
  5513. downloaded_filename,
  5514. probe_hit,
  5515. printer_id,
  5516. storage.reason,
  5517. )
  5518. if not storage.reachable and not downloaded_filename:
  5519. # Same opening words whether or not a probe ran, because that is
  5520. # the phrase support asks people to grep for — only the tail says
  5521. # which of the two happened.
  5522. logger.info(
  5523. "Skipping the 3MF lookup for printer %s: %s — %s",
  5524. printer_id,
  5525. storage.reason,
  5526. "no copy of it on external storage either"
  5527. if storage.probe_filename
  5528. else "the print file is not on storage Bambuddy can read over FTPS, so no path would find it",
  5529. )
  5530. for try_filename in possible_names if not downloaded_filename and storage.reachable else []:
  5531. if not try_filename.endswith(".3mf"):
  5532. continue
  5533. # Root (/) is where BambuStudio/OrcaSlicer uploads land on A1/P1-series
  5534. # printers, so try it first — deferring it to last cost #972's reporter
  5535. # ~48 minutes of retries on /cache//model//data//data/Metadata before
  5536. # landing on the path that actually had the file.
  5537. remote_paths = [
  5538. f"/{try_filename}",
  5539. f"/cache/{try_filename}",
  5540. f"/model/{try_filename}",
  5541. f"/data/{try_filename}",
  5542. f"/data/Metadata/{try_filename}",
  5543. ]
  5544. temp_path = app_settings.archive_dir / "temp" / try_filename
  5545. temp_path.parent.mkdir(parents=True, exist_ok=True)
  5546. for remote_path in remote_paths:
  5547. if ftps_handshake_blocked(printer.ip_address):
  5548. # The printer's FTPS service is not completing a TLS
  5549. # handshake, so it has no path we could reach — walking the
  5550. # remaining candidates only re-runs the same failure
  5551. # (#2780). Fall through to the no-3MF archive now.
  5552. #
  5553. # Remember *why*, though. This is the one give-up that is
  5554. # temporary: the cool-off clears in minutes and the file was
  5555. # on the printer the whole time. The fallback archive is
  5556. # stamped with it so a retry can be scheduled, and so the
  5557. # Archives banner stops blaming storage (#2957).
  5558. blocked_by_ftps_cooloff = True
  5559. logger.warning(
  5560. "Giving up on the 3MF for printer %s: its file service is not answering over TLS",
  5561. printer_id,
  5562. )
  5563. break
  5564. logger.debug("Trying FTP download: %s", remote_path)
  5565. try:
  5566. if ftp_retry_enabled:
  5567. downloaded = await with_ftp_retry(
  5568. download_file_async,
  5569. printer.ip_address,
  5570. printer.access_code,
  5571. remote_path,
  5572. temp_path,
  5573. timeout=ftp_timeout,
  5574. socket_timeout=ftp_timeout,
  5575. printer_model=printer.model,
  5576. max_retries=ftp_retry_count,
  5577. retry_delay=ftp_retry_delay,
  5578. operation_name=f"Download 3MF from {remote_path}",
  5579. cooloff_ip=printer.ip_address,
  5580. non_retry_exceptions=(FileNotOnPrinterError,),
  5581. )
  5582. else:
  5583. downloaded = await download_file_async(
  5584. printer.ip_address,
  5585. printer.access_code,
  5586. remote_path,
  5587. temp_path,
  5588. timeout=ftp_timeout,
  5589. socket_timeout=ftp_timeout,
  5590. printer_model=printer.model,
  5591. )
  5592. if downloaded:
  5593. downloaded_filename = try_filename
  5594. logger.info("Downloaded: %s", remote_path)
  5595. # Populate shared cache so the cover endpoint (if it
  5596. # runs next) doesn't refetch the same 36MB over FTP.
  5597. cache_3mf_download(printer_id, try_filename, temp_path)
  5598. break
  5599. # with_ftp_retry returns None once it has spent its budget,
  5600. # and download_file_async returns False on a timeout, so an
  5601. # exhausted transfer arrives here rather than as an
  5602. # exception (#3063).
  5603. ftp_transfer_failed = True
  5604. sweep_failed_paths.add(remote_path)
  5605. except FileNotOnPrinterError:
  5606. # 550 — file isn't at this path. Advance to next candidate
  5607. # without burning the retry budget.
  5608. logger.debug("3MF not at %s (550), trying next path", remote_path)
  5609. except Exception as e:
  5610. ftp_transfer_failed = True
  5611. sweep_failed_paths.add(remote_path)
  5612. logger.debug("FTP download failed for %s: %s", remote_path, e)
  5613. if downloaded_filename or ftps_handshake_blocked(printer.ip_address):
  5614. break
  5615. # If still not found, try listing directories to find matching file
  5616. # Different printer models use different directory structures. Skipped
  5617. # when the printer's FTPS handshake is failing — the directory walk is
  5618. # five more connections that cannot get further than the download did.
  5619. if (
  5620. not downloaded_filename
  5621. and storage.reachable
  5622. and (filename or subtask_name)
  5623. and not ftps_handshake_blocked(printer.ip_address)
  5624. ):
  5625. search_term = (subtask_name or filename).lower().replace(".gcode", "").replace(".3mf", "")
  5626. logger.info("Direct FTP download failed, searching directories for '%s'", search_term)
  5627. search_dirs = ["/cache", "/model", "/data", "/data/Metadata", "/"]
  5628. # Every folder is listed before one file is picked, so a file named
  5629. # for the print in a later folder wins over one that merely
  5630. # contains the name in an earlier folder.
  5631. listed_paths: list[str] = []
  5632. for search_dir in search_dirs:
  5633. try:
  5634. dir_files = await list_files_async(
  5635. printer.ip_address, printer.access_code, search_dir, printer_model=printer.model
  5636. )
  5637. except Exception as e:
  5638. logger.debug("Failed to list %s: %s", search_dir, e)
  5639. continue
  5640. threemf_files = [f.get("name") for f in dir_files if f.get("name", "").endswith(".3mf")]
  5641. if threemf_files:
  5642. logger.info(
  5643. f"Found {len(threemf_files)} 3MF files in {search_dir}: {threemf_files[:5]}{'...' if len(threemf_files) > 5 else ''}"
  5644. )
  5645. listed_paths += [
  5646. posixpath.join(search_dir, f.get("name", "")) for f in dir_files if not f.get("is_directory")
  5647. ]
  5648. for remote_full_path in _3mf_files_for_print(listed_paths, search_term):
  5649. if remote_full_path in sweep_failed_paths:
  5650. # Listed for this print, and its transfer already failed:
  5651. # the same as a failed walk download, without spending a
  5652. # second deadline on it. Skipping it cannot hand the walk a
  5653. # look-alike: while a file of the print's own name is
  5654. # listed, files that merely contain the name are not.
  5655. logger.info(
  5656. "Not downloading %s again: its transfer already failed; the recovery retry will fetch it",
  5657. remote_full_path,
  5658. )
  5659. walk_transfer_failed = True
  5660. continue
  5661. fname = posixpath.basename(remote_full_path)
  5662. logger.info("Found matching file: %s", remote_full_path)
  5663. temp_path = app_settings.archive_dir / "temp" / fname
  5664. temp_path.parent.mkdir(parents=True, exist_ok=True)
  5665. try:
  5666. if ftp_retry_enabled:
  5667. downloaded = await with_ftp_retry(
  5668. download_file_async,
  5669. printer.ip_address,
  5670. printer.access_code,
  5671. remote_full_path,
  5672. temp_path,
  5673. timeout=ftp_timeout,
  5674. socket_timeout=ftp_timeout,
  5675. printer_model=printer.model,
  5676. max_retries=ftp_retry_count,
  5677. retry_delay=ftp_retry_delay,
  5678. operation_name=f"Download 3MF from {remote_full_path}",
  5679. cooloff_ip=printer.ip_address,
  5680. )
  5681. else:
  5682. downloaded = await download_file_async(
  5683. printer.ip_address,
  5684. printer.access_code,
  5685. remote_full_path,
  5686. temp_path,
  5687. timeout=ftp_timeout,
  5688. socket_timeout=ftp_timeout,
  5689. printer_model=printer.model,
  5690. )
  5691. except Exception as e:
  5692. logger.debug("Failed to download %s: %s", remote_full_path, e)
  5693. continue
  5694. if downloaded:
  5695. downloaded_filename = fname
  5696. logger.info("Found and downloaded: %s", remote_full_path)
  5697. cache_3mf_download(printer_id, fname, temp_path)
  5698. break
  5699. # The listing named the file, so it is on the card;
  5700. # only the transfer failed (#3063).
  5701. ftp_transfer_failed = True
  5702. walk_transfer_failed = True
  5703. # A "Print again" on the printer's screen names no file Bambuddy can
  5704. # find: only its history copy under a generic name, and the 3MF's
  5705. # internal title (#3307). When an earlier run of the same job was
  5706. # archived, its 3MF is the one being printed again. Not when the
  5707. # directory walk listed a file for this print and only its transfer
  5708. # failed: that file is on the card, and the retry fetches it. A
  5709. # cool-off or a failed sweep path is no such evidence -- the retry
  5710. # tries the same names, which this print's file never had.
  5711. reused_3mf_from: int | None = None
  5712. if not downloaded_filename and not walk_transfer_failed:
  5713. reported_layers = raw_mqtt.get("total_layer_num")
  5714. earlier = await _archived_3mf_for_repeat(
  5715. db,
  5716. printer_id,
  5717. subtask_name,
  5718. filename,
  5719. reported_layers if isinstance(reported_layers, int) and reported_layers > 0 else None,
  5720. )
  5721. if earlier is not None:
  5722. reuse_name = Path(earlier.filename or "").name or f"archive_{earlier.id}.3mf"
  5723. # Per printer: two printers reprinting the same job at once
  5724. # must not write one temp file (archive_print names the
  5725. # archive after original_filename, not this path).
  5726. reuse_path = app_settings.archive_dir / "temp" / f"reprint_{printer_id}_{reuse_name}"
  5727. try:
  5728. reuse_path.parent.mkdir(parents=True, exist_ok=True)
  5729. await asyncio.to_thread(shutil.copyfile, app_settings.base_dir / earlier.file_path, reuse_path)
  5730. except OSError as e:
  5731. # A copy that fails leaves the print the fallback archive
  5732. # it would have had, not no archive at all.
  5733. logger.warning("Could not copy the 3MF of archive %s for printer %s: %s", earlier.id, printer_id, e)
  5734. else:
  5735. temp_path = reuse_path
  5736. downloaded_filename = reuse_name
  5737. reused_3mf_from = earlier.id
  5738. cache_3mf_download(printer_id, reuse_name, temp_path)
  5739. logger.info(
  5740. "No 3MF on printer %s for %s (%s); using the 3MF of archive %s, an earlier run of the same job",
  5741. printer_id,
  5742. subtask_name,
  5743. filename,
  5744. earlier.id,
  5745. )
  5746. # Validate the downloaded 3MF actually matches the plate that's running
  5747. # (#1204): subtask_name lags across consecutive plates of the same model,
  5748. # so the first FTP candidate (built from subtask_name) can land on the
  5749. # previous plate's still-resident upload. Cross-check the slice_info
  5750. # plate index against the plate parsed from gcode_file (always fresh —
  5751. # it's the field whose change triggered this callback).
  5752. if downloaded_filename and temp_path:
  5753. expected_plate = parse_plate_id(filename)
  5754. actual_plate = peek_plate_index_in_3mf(temp_path) if expected_plate is not None else None
  5755. if expected_plate is not None and actual_plate is not None and actual_plate != expected_plate:
  5756. logger.warning(
  5757. "[CALLBACK] 3MF plate mismatch: downloaded %s reports plate %s but printer is "
  5758. "running plate %s — subtask_name=%r appears stale, retrying with corrected name",
  5759. downloaded_filename,
  5760. actual_plate,
  5761. expected_plate,
  5762. subtask_name,
  5763. )
  5764. corrected_subtask = swap_plate_suffix(subtask_name, expected_plate)
  5765. retry_succeeded = False
  5766. if corrected_subtask and corrected_subtask != subtask_name:
  5767. for try_filename in (f"{corrected_subtask}.gcode.3mf", f"{corrected_subtask}.3mf"):
  5768. retry_temp_path = app_settings.archive_dir / "temp" / try_filename
  5769. retry_temp_path.parent.mkdir(parents=True, exist_ok=True)
  5770. for remote_path in (
  5771. f"/{try_filename}",
  5772. f"/cache/{try_filename}",
  5773. f"/model/{try_filename}",
  5774. f"/data/{try_filename}",
  5775. f"/data/Metadata/{try_filename}",
  5776. ):
  5777. try:
  5778. if ftp_retry_enabled:
  5779. downloaded = await with_ftp_retry(
  5780. download_file_async,
  5781. printer.ip_address,
  5782. printer.access_code,
  5783. remote_path,
  5784. retry_temp_path,
  5785. timeout=ftp_timeout,
  5786. socket_timeout=ftp_timeout,
  5787. printer_model=printer.model,
  5788. max_retries=ftp_retry_count,
  5789. retry_delay=ftp_retry_delay,
  5790. operation_name=f"Re-download 3MF from {remote_path}",
  5791. cooloff_ip=printer.ip_address,
  5792. non_retry_exceptions=(FileNotOnPrinterError,),
  5793. )
  5794. else:
  5795. downloaded = await download_file_async(
  5796. printer.ip_address,
  5797. printer.access_code,
  5798. remote_path,
  5799. retry_temp_path,
  5800. timeout=ftp_timeout,
  5801. socket_timeout=ftp_timeout,
  5802. printer_model=printer.model,
  5803. )
  5804. if downloaded and peek_plate_index_in_3mf(retry_temp_path) == expected_plate:
  5805. logger.info(
  5806. "[CALLBACK] Re-download succeeded with corrected name %s "
  5807. "(plate %s) — replacing wrong file",
  5808. try_filename,
  5809. expected_plate,
  5810. )
  5811. try:
  5812. temp_path.unlink(missing_ok=True)
  5813. except OSError:
  5814. pass
  5815. temp_path = retry_temp_path
  5816. downloaded_filename = try_filename
  5817. subtask_name = corrected_subtask
  5818. cache_3mf_download(printer_id, try_filename, temp_path)
  5819. retry_succeeded = True
  5820. break
  5821. elif downloaded:
  5822. # Wrong plate again — discard and keep trying
  5823. try:
  5824. retry_temp_path.unlink(missing_ok=True)
  5825. except OSError:
  5826. pass
  5827. except FileNotOnPrinterError:
  5828. continue
  5829. except Exception as e:
  5830. logger.debug("Re-download failed for %s: %s", remote_path, e)
  5831. if retry_succeeded:
  5832. break
  5833. # If the retry didn't find a matching file, drop the wrong 3MF
  5834. # so the no-3MF fallback below creates an archive whose name
  5835. # at least reflects the right plate.
  5836. if not retry_succeeded:
  5837. logger.warning(
  5838. "[CALLBACK] Could not re-download correct plate %s — falling back to no-3MF archive",
  5839. expected_plate,
  5840. )
  5841. try:
  5842. temp_path.unlink(missing_ok=True)
  5843. except OSError:
  5844. pass
  5845. temp_path = None
  5846. downloaded_filename = None
  5847. # Whatever the sweep's transport did earlier, it is not why
  5848. # this archive ends up empty: a 3MF downloaded fine, it was
  5849. # just the wrong plate. Retrying would re-fetch that same
  5850. # contradicted file under the same stale names and hand it
  5851. # to _recover_fallback_archive, which checks that a
  5852. # candidate is a readable 3MF but not which plate it is --
  5853. # so the row would be filled in with another plate's
  5854. # filament and cost, the exact swap #2957 removed (#3063).
  5855. ftp_transfer_failed = False
  5856. # Disown the name for *lookups*: it has just been shown to
  5857. # fetch another plate's 3MF, and it keys `_active_prints`
  5858. # below, where the cover endpoint's own download of that
  5859. # same name would find this archive and fill it in with the
  5860. # file we are discarding here.
  5861. #
  5862. # Keep it for the *title*, which is a separate question.
  5863. # ``swap_plate_suffix`` returns None both for a name that
  5864. # carries no "- Plate N" / "_plate_N" suffix and for no
  5865. # name at all, and those are not the same situation: a name
  5866. # without a suffix holds no stale plate number to be wrong
  5867. # about. Blanking both uses at once dropped the project
  5868. # name too, and the row fell through to the gcode_file path
  5869. # titled "plate_1" though the real name was in hand.
  5870. # #1204's own premise is consecutive plates *of the same
  5871. # model*, so the project part is right either way (#3126).
  5872. display_name_after_plate_reject = corrected_subtask or subtask_name or None
  5873. subtask_name = corrected_subtask or ""
  5874. # Deleted while the 3MF was being fetched (that can take minutes): no
  5875. # archive, timelapse or tracking for a printer that will never report
  5876. # the print's end. forget_printer again: the download registered its
  5877. # 3MF cache entry after the delete cleared the id. (A delete committing
  5878. # while the archive is written is caught after its commit.)
  5879. if await _printer_gone(db, printer_id):
  5880. logger.info("Printer %s was deleted during the print-start download; not archiving", printer_id)
  5881. await forget_printer(printer_id)
  5882. return
  5883. if not downloaded_filename or not temp_path:
  5884. logger.warning("Could not find 3MF file for print: %s", filename or subtask_name)
  5885. # Create a fallback archive without 3MF data so the print is still tracked
  5886. # This commonly happens with P1S/A1 printers where FTP has file size limitations
  5887. try:
  5888. from backend.app.models.archive import PrintArchive
  5889. # Why the card is empty. The two temporary causes outrank the
  5890. # storage verdict because they say the sweep never got a fair
  5891. # answer: a cool-off skipped it at the transport, and a failed
  5892. # transfer reached the printer but never finished. Either way
  5893. # the file is still on the card, so reporting where the printer
  5894. # files its jobs would describe a setting that is not the
  5895. # problem (#2957, #3063).
  5896. if blocked_by_ftps_cooloff:
  5897. no_3mf_reason = REASON_FTPS_COOLOFF
  5898. elif storage.reachable and ftp_transfer_failed:
  5899. no_3mf_reason = REASON_FTP_TRANSFER_FAILED
  5900. else:
  5901. no_3mf_reason = storage.reason
  5902. # Derive print name from subtask_name or filename. The
  5903. # plate guard's disowned name comes second: it is a real name
  5904. # for a real print, and only the gcode_file path is left
  5905. # otherwise -- which titles the row "plate_1" (#3126).
  5906. print_name = subtask_name or display_name_after_plate_reject or filename
  5907. if print_name:
  5908. # Clean up the name (remove extensions, path parts)
  5909. print_name = print_name.split("/")[-1]
  5910. print_name = print_name.replace(".gcode.3mf", "").replace(".gcode", "").replace(".3mf", "")
  5911. else:
  5912. print_name = "Unknown Print"
  5913. # Recover estimated print time from MQTT (best-effort for notifications)
  5914. fallback_print_time = None
  5915. mqtt_remaining = data.get("remaining_time")
  5916. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  5917. fallback_print_time = int(mqtt_remaining)
  5918. if fallback_print_time is None:
  5919. mc_remaining = (data.get("raw_data") or {}).get("mc_remaining_time")
  5920. if mc_remaining and isinstance(mc_remaining, (int, float)) and mc_remaining > 0:
  5921. fallback_print_time = int(mc_remaining * 60)
  5922. # Best-effort filament metadata from MQTT — see
  5923. # _extract_filament_data_from_mqtt. Without this the fallback
  5924. # archive's filament fields stayed NULL even though the AMS
  5925. # state at print start was sitting right there in `data`.
  5926. # The slicer's ams_mapping (when present) narrows the result
  5927. # to slots actually used by the print (#1533).
  5928. mqtt_filament_meta = _extract_filament_data_from_mqtt(
  5929. data, _get_start_ams_mapping(data, printer_id, None)
  5930. )
  5931. # Create minimal archive entry
  5932. fallback_archive = PrintArchive(
  5933. printer_id=printer_id,
  5934. filename=filename or f"{print_name}.3mf",
  5935. file_path="", # Empty - no 3MF file available
  5936. file_size=0,
  5937. print_name=print_name,
  5938. print_time_seconds=fallback_print_time,
  5939. status="printing",
  5940. started_at=detected_at,
  5941. subtask_id=subtask_id,
  5942. confirm_requested=await _ask_outcome_for_external_print(db, printer_id, subtask_name),
  5943. filament_type=mqtt_filament_meta.get("filament_type"),
  5944. filament_color=mqtt_filament_meta.get("filament_color"),
  5945. extra_data={
  5946. "no_3mf_available": True,
  5947. # Why the card is empty, when we know -- see above. The
  5948. # banner reads this to stop telling H2/P2 owners to
  5949. # switch on a setting that is already on and would not
  5950. # have helped (#2780).
  5951. "no_3mf_reason": no_3mf_reason,
  5952. "original_subtask": subtask_name or display_name_after_plate_reject or "",
  5953. "_print_data": data,
  5954. },
  5955. )
  5956. db.add(fallback_archive)
  5957. try:
  5958. await db.commit()
  5959. except IntegrityError:
  5960. # Not "Failed to create fallback archive" below, and no
  5961. # start notification for a printer that is gone.
  5962. if await _refused_for_deleted_printer(db, printer_id, "its fallback archive was written"):
  5963. return
  5964. raise
  5965. await db.refresh(fallback_archive)
  5966. logger.info("Created fallback archive %s for %s (no 3MF available)", fallback_archive.id, print_name)
  5967. if await _drop_archive_of_deleted_printer(db, printer_id, fallback_archive.id):
  5968. return
  5969. _maybe_start_layer_timelapse(printer, printer_id, fallback_archive.id)
  5970. # Track as active print
  5971. _bind_active_print(printer_id, fallback_archive.id, fallback_archive.filename, filename, subtask_name)
  5972. await _open_run_for(fallback_archive.id, printer_id, subtask_id)
  5973. # Record starting energy if smart plug available (#941: persisted column)
  5974. await _record_energy_start(fallback_archive, printer_id, db, context="fallback")
  5975. # Send WebSocket notification
  5976. await ws_manager.send_archive_created(
  5977. {
  5978. "id": fallback_archive.id,
  5979. "printer_id": fallback_archive.printer_id,
  5980. "filename": fallback_archive.filename,
  5981. "print_name": fallback_archive.print_name,
  5982. "status": fallback_archive.status,
  5983. }
  5984. )
  5985. # MQTT relay - publish archive created
  5986. try:
  5987. await mqtt_relay.on_archive_created(
  5988. archive_id=fallback_archive.id,
  5989. print_name=fallback_archive.print_name,
  5990. printer_name=printer.name,
  5991. status=fallback_archive.status,
  5992. )
  5993. except Exception:
  5994. pass # Don't fail if MQTT fails
  5995. # Store Spoolman tracking data (may not work for fallback since no 3MF)
  5996. try:
  5997. await _store_spoolman_print_data(
  5998. printer_id,
  5999. fallback_archive.id,
  6000. fallback_archive.file_path,
  6001. db,
  6002. printer_manager,
  6003. ams_mapping=_get_start_ams_mapping(data, printer_id, fallback_archive.id),
  6004. plate_id=_get_start_plate_id(printer_id, fallback_archive.id),
  6005. )
  6006. except Exception as e:
  6007. logger.debug("[SPOOLMAN] Could not store tracking for fallback archive: %s", e)
  6008. # Both temporary give-ups are worth coming back for, and for
  6009. # the same reason: the file is on the printer and the last look
  6010. # failed at the transport rather than finding nothing. One waits
  6011. # out the handshake block (#2957), the other waits for the
  6012. # printer to stop being busy (#3063). Deliberately not scheduled
  6013. # for a storage verdict: a file on internal eMMC will not appear
  6014. # at any FTPS path however long we wait, and retrying it is
  6015. # exactly the sweep #2780 removed.
  6016. if no_3mf_reason in (REASON_FTPS_COOLOFF, REASON_FTP_TRANSFER_FAILED) and possible_names:
  6017. # `possible_names`, not the raw MQTT strings: it is the exact
  6018. # list this flow just tried, already stripped of any path
  6019. # (`filename` arrives as "/data/Metadata/plate_1.gcode" on
  6020. # some firmware) and deduped.
  6021. _schedule_fallback_3mf_retry(
  6022. printer_id=printer_id,
  6023. archive_id=fallback_archive.id,
  6024. filenames=list(possible_names),
  6025. reason=no_3mf_reason,
  6026. )
  6027. # Send notification without archive data (file not found)
  6028. if not notification_sent:
  6029. await _send_print_start_notification(printer_id, data, logger=logger)
  6030. # The same baseline the other two on_print_start branches take
  6031. # (#2704), and last for the same reason they are: it lists the
  6032. # printer's timelapse directory, so a slow card must not delay
  6033. # the _active_prints registration, the energy reading, the
  6034. # archive-created event or the start notification above it.
  6035. #
  6036. # This branch never took one, so every no-3MF archive reached
  6037. # completion with no baseline in memory and none on the row, and
  6038. # the completion scan fell into its "snapshot now" fallback --
  6039. # which runs after the printer has written the video, so the new
  6040. # file landed inside the baseline and no diff ever matched
  6041. # (#2957 follow-up).
  6042. #
  6043. # Skipped when the FTPS cool-off is what produced this fallback:
  6044. # the listing needs the same connection that just failed, so it
  6045. # could only record that the card was unreadable. The scan
  6046. # handles that case by refusing to choose between candidates.
  6047. if not blocked_by_ftps_cooloff:
  6048. await _capture_timelapse_baseline_at_start(
  6049. printer, printer_id, logger, archive_id=fallback_archive.id
  6050. )
  6051. await _forget_if_printer_gone(printer_id)
  6052. return
  6053. except Exception as e:
  6054. logger.error("Failed to create fallback archive: %s", e)
  6055. # Send notification without archive data (file not found)
  6056. if not notification_sent:
  6057. await _send_print_start_notification(printer_id, data, logger=logger)
  6058. return
  6059. # The archive of the same slice sent before, which this print adds a
  6060. # run to instead of a new archive (#1634). Held until its run is open.
  6061. joined_archive_id: int | None = None
  6062. try:
  6063. # Archive the file with status "printing"
  6064. service = ArchiveService(db)
  6065. start_print_data = {
  6066. **data,
  6067. "status": "printing",
  6068. # Where the 3MF came from when the printer had none to
  6069. # give (#3307), for whoever reads this archive later.
  6070. **({"_3mf_from_archive": reused_3mf_from} if reused_3mf_from is not None else {}),
  6071. }
  6072. # The reused copy's temp name carries the printer id; a
  6073. # download's is the file's own name, as before.
  6074. start_filename = downloaded_filename if reused_3mf_from is not None else None
  6075. try:
  6076. claimed = await _claim_slicer_resend_archive(db, printer_id, temp_path)
  6077. if claimed is not None:
  6078. joined, file_in_use = claimed
  6079. joined_archive_id = joined.id
  6080. archive = await _start_slicer_resend_run(
  6081. db,
  6082. joined,
  6083. printer_id,
  6084. source=temp_path,
  6085. print_data=start_print_data,
  6086. subtask_id=subtask_id,
  6087. started_at=detected_at,
  6088. original_filename=start_filename,
  6089. logger=logger,
  6090. file_in_use=file_in_use,
  6091. )
  6092. if archive is None:
  6093. return
  6094. else:
  6095. archive = await service.archive_print(
  6096. printer_id=printer_id,
  6097. source_file=temp_path,
  6098. print_data=start_print_data,
  6099. subtask_id=subtask_id,
  6100. started_at=detected_at,
  6101. original_filename=start_filename,
  6102. )
  6103. except IntegrityError:
  6104. # archive_print removed the folder it had copied the 3MF into.
  6105. if await _refused_for_deleted_printer(db, printer_id, "its archive was written"):
  6106. return
  6107. raise
  6108. if archive:
  6109. # Ask-for-outcome for a print Bambuddy did not dispatch (#1898).
  6110. # Set on the row rather than passed to archive_print, which also
  6111. # serves the queue dispatcher — there the queue item decides.
  6112. # Guarded because this branch has no ``except``: an unhandled
  6113. # write error would take the _active_prints registration, the
  6114. # start notification, the energy reading and the timelapse
  6115. # baseline below it with it, and a missing prompt is by far the
  6116. # cheaper failure.
  6117. archive_id = archive.id
  6118. try:
  6119. if await _ask_outcome_for_external_print(db, printer_id, subtask_name):
  6120. archive.confirm_requested = True
  6121. await db.commit()
  6122. except Exception as e:
  6123. logger.warning("Could not flag archive %s for the outcome prompt: %s", archive_id, e)
  6124. # The rollback expires every loaded object, and on an async
  6125. # session reading one afterwards raises instead of
  6126. # reloading, so the two this branch goes on to use are
  6127. # fetched again. The archive itself was committed by
  6128. # archive_print; only the flag is lost.
  6129. try:
  6130. await db.rollback()
  6131. archive = await db.get(PrintArchive, archive_id)
  6132. printer = await db.get(Printer, printer_id)
  6133. except Exception as reload_error:
  6134. logger.warning(
  6135. "Could not reload archive %s after the failed flag write: %s", archive_id, reload_error
  6136. )
  6137. if archive is None:
  6138. # Deleted meanwhile: nothing to register the print for.
  6139. return
  6140. # Not a joined archive: it is not this print's to remove, and
  6141. # its adoption already checked the printer.
  6142. if joined_archive_id is None and await _drop_archive_of_deleted_printer(db, printer_id, archive_id):
  6143. return
  6144. # Track this active print (use both original filename and downloaded filename)
  6145. _bind_active_print(printer_id, archive_id, downloaded_filename, filename, subtask_name)
  6146. await _open_run_for(archive_id, printer_id, subtask_id)
  6147. if joined_archive_id is not None:
  6148. logger.info(
  6149. "Added the print of %s to archive %s: the same G-code was sent again",
  6150. downloaded_filename,
  6151. archive_id,
  6152. )
  6153. else:
  6154. logger.info("Created archive %s for %s", archive.id, downloaded_filename)
  6155. _maybe_start_layer_timelapse(printer, printer_id, archive.id)
  6156. # Record starting energy from smart plug if available (#941: persisted column)
  6157. await _record_energy_start(archive, printer_id, db, context="auto-archive")
  6158. archive_event = {
  6159. "id": archive.id,
  6160. "printer_id": archive.printer_id,
  6161. "filename": archive.filename,
  6162. "print_name": archive.print_name,
  6163. "status": archive.status,
  6164. }
  6165. if joined_archive_id is not None:
  6166. await ws_manager.send_archive_updated(archive_event)
  6167. else:
  6168. await ws_manager.send_archive_created(archive_event)
  6169. # MQTT relay - publish archive created
  6170. try:
  6171. await mqtt_relay.on_archive_created(
  6172. archive_id=archive.id,
  6173. print_name=archive.print_name or "",
  6174. printer_name=printer.name if printer else "",
  6175. status=archive.status,
  6176. )
  6177. except Exception:
  6178. pass # Don't fail if MQTT fails
  6179. # Send notification with archive data (new archive created)
  6180. if not notification_sent:
  6181. archive_data = {
  6182. "print_time_seconds": _run_print_time_seconds(archive, printer_id),
  6183. "created_by_id": archive.created_by_id,
  6184. }
  6185. await _send_print_start_notification(printer_id, data, archive_data, logger)
  6186. # Extract printable objects for skip object functionality
  6187. try:
  6188. from backend.app.services.archive import extract_printable_objects_from_3mf
  6189. client = printer_manager.get_client(printer_id)
  6190. if client:
  6191. with open(temp_path, "rb") as f:
  6192. threemf_data = f.read()
  6193. # Extract with positions for UI overlay, scoped to the
  6194. # plate that is printing — an all-plates 3MF carries
  6195. # every plate's objects (#2522).
  6196. printable_objects, bbox_all = extract_printable_objects_from_3mf(
  6197. threemf_data,
  6198. plate_number=resolve_plate_id(client.state),
  6199. include_positions=True,
  6200. )
  6201. if printable_objects:
  6202. # Store objects in printer state
  6203. client.state.printable_objects = printable_objects
  6204. client.state.printable_objects_bbox_all = bbox_all
  6205. client.state.skipped_objects = [] # Reset skipped objects for new print
  6206. logger.info(
  6207. "Loaded %s printable objects for printer %s", len(printable_objects), printer_id
  6208. )
  6209. except Exception as e:
  6210. logger.debug("Failed to extract printable objects: %s", e)
  6211. # From the archived copy: the downloaded one may be removed
  6212. # below while the pauses are still being read.
  6213. if archive is not None and archive.file_path:
  6214. _load_pauses_from_3mf(app_settings.base_dir / archive.file_path, printer_id)
  6215. # Store Spoolman tracking data for per-filament usage reporting
  6216. try:
  6217. await _store_spoolman_print_data(
  6218. printer_id,
  6219. archive.id,
  6220. archive.file_path,
  6221. db,
  6222. printer_manager,
  6223. ams_mapping=_get_start_ams_mapping(data, printer_id, archive_id),
  6224. plate_id=_get_start_plate_id(printer_id, archive_id),
  6225. )
  6226. except Exception as e:
  6227. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  6228. # Price the print from its spools now rather than only at completion (#3261)
  6229. _schedule_archive_cost_estimate(
  6230. printer_id,
  6231. archive.id,
  6232. printer_manager,
  6233. ams_mapping=_get_start_ams_mapping(data, printer_id, archive_id),
  6234. plate_id=_get_start_plate_id(printer_id, archive_id),
  6235. session_factory=async_session,
  6236. )
  6237. # Capture timelapse file baseline for snapshot-diff on completion
  6238. await _capture_timelapse_baseline_at_start(printer, printer_id, logger, archive_id=archive.id)
  6239. await _forget_if_printer_gone(printer_id)
  6240. finally:
  6241. if joined_archive_id is not None:
  6242. from backend.app.services import archive_work
  6243. archive_work.release(joined_archive_id)
  6244. # Keep temp_path around until print completes so the cover endpoint
  6245. # can reuse it (#972). Cache eviction in on_print_complete deletes
  6246. # the file. If the cache entry was evicted early (file vanished),
  6247. # clean up any stragglers here to avoid leaking disk on retries.
  6248. cached_now = get_cached_3mf(printer_id, downloaded_filename) if downloaded_filename else None
  6249. if temp_path and temp_path.exists() and cached_now != temp_path:
  6250. temp_path.unlink()
  6251. _TIMELAPSE_VIDEO_EXTENSIONS = (".mp4", ".avi")
  6252. _TIMELAPSE_DIRECTORIES = ("/timelapse", "/timelapse/video", "/record", "/recording")
  6253. # A baseline that could not be read at print start is read again on this
  6254. # schedule while the print runs (#3412). The printer writes the video only once
  6255. # the print has ended, so a reading taken mid-print is as good as one taken at
  6256. # the start. The interval is shorter than the FTPS cool-offs (300 s), which are
  6257. # what usually makes the card unreadable, so the first reading after one ends
  6258. # is not long behind; while a cool-off runs a round costs nothing, as no
  6259. # connection is opened. The cap bounds a print that never reports an end.
  6260. #
  6261. # A round that could not read the card doubles the wait before the next, up to
  6262. # the ceiling, and after that many failed rounds in a row the re-read gives up
  6263. # and leaves the baseline unread for the completion scan to handle (#3474). Each
  6264. # connection to a hung file service can leave a session behind on the printer.
  6265. _TIMELAPSE_BASELINE_REREAD_INTERVAL_SECONDS = 120.0
  6266. _TIMELAPSE_BASELINE_REREAD_MAX_INTERVAL_SECONDS = 1800.0
  6267. _TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS = 5
  6268. _TIMELAPSE_BASELINE_REREAD_MAX_ATTEMPTS = 720
  6269. # Poll schedule for the post-print timelapse scan (#2704). Module-level so
  6270. # tests can shrink them without waiting out real delays.
  6271. #
  6272. # This replaced a fixed [5, 10, 20, 30] retry ladder, i.e. roughly 65 s of
  6273. # looking. Across 247 support bundles the attempt that found the video was #1
  6274. # 272 times, then 17 / 13 / 13 — a flat tail against the cutoff rather than a
  6275. # decaying one, which is the signature of a budget that expires while files are
  6276. # still arriving. 457 scans were scheduled and only 262 ever attached. Big
  6277. # prints make big videos and the printer writes them after the print ends, so
  6278. # the poll now runs for minutes and costs one FTP LIST per round.
  6279. _TIMELAPSE_SCAN_FIRST_DELAY_SECONDS: float = 5.0
  6280. _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS: float = 30.0
  6281. _TIMELAPSE_SCAN_TIMEOUT_SECONDS: float = 900.0
  6282. def _timelapse_scan_max_attempts() -> int:
  6283. """Round cap for the poll, derived from the wall-clock budget.
  6284. The deadline alone is not a sufficient bound: it assumes each round really
  6285. waits, which stops being true the moment ``asyncio.sleep`` is patched out,
  6286. and an FTP list that fails immediately would otherwise spin against the
  6287. printer at full speed for the whole window. Whichever bound is reached
  6288. first ends the poll.
  6289. """
  6290. if _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS <= 0:
  6291. # A zero interval makes the wall-clock budget meaningless; fall back to
  6292. # the round count the production interval would have given.
  6293. return 32
  6294. return max(1, int(_TIMELAPSE_SCAN_TIMEOUT_SECONDS // _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS) + 1)
  6295. async def _read_timelapse_baseline(printer) -> tuple[set[str], bool]:
  6296. """The video names on *printer*'s card, and whether the card was read.
  6297. ``_list_timelapse_videos`` answers ``[]`` when its connect fails rather
  6298. than raising, so a card behind the FTPS handshake cool-off, or one whose
  6299. listing timed out, is indistinguishable from one holding no videos.
  6300. Everywhere that only wants to know "is there a video yet" the difference
  6301. does not matter -- both mean "not yet, retry".
  6302. It matters where an empty listing is recorded as a *baseline*. Recording
  6303. "the card held nothing" for a card that was never read means every video on
  6304. it counts as new, and the completion scan then attaches a stale video to
  6305. this print and deletes it from the printer (#2957 follow-up, #3412). So an
  6306. card counts as read only when every directory walked answered, with a
  6307. listing or with "no such directory". One that failed may be the one
  6308. holding the videos.
  6309. One walk, one connection per directory. It used to list the card with
  6310. ``_list_timelapse_videos`` first and walk the same directories again when
  6311. that came back empty, which against a hung file service was eight dead
  6312. connections a round instead of one (#3474). A directory that fails does
  6313. not end the walk unless the failure put the printer in a cool-off: a later
  6314. one holding videos still answers the question.
  6315. """
  6316. from backend.app.services.bambu_ftp import ftps_background_blocked, list_files_result_async
  6317. ip_address = getattr(printer, "ip_address", None)
  6318. printer_id = getattr(printer, "id", None)
  6319. if not ip_address:
  6320. return set(), True
  6321. # No card in the slot: nothing to read, and nothing a video could land on.
  6322. if printer_id is not None and not external_storage_present(printer_manager.get_status(printer_id)):
  6323. return set(), True
  6324. read = True
  6325. for timelapse_path in _TIMELAPSE_DIRECTORIES:
  6326. if ftps_background_blocked(ip_address):
  6327. # Below any path: every remaining one would fail the same way.
  6328. return set(), False
  6329. listing = await list_files_result_async(
  6330. ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  6331. )
  6332. if not (listing.available or listing.missing):
  6333. read = False
  6334. continue
  6335. videos = [
  6336. f
  6337. for f in listing.files
  6338. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  6339. ]
  6340. if videos:
  6341. return {f.get("name", "") for f in videos}, True
  6342. return set(), read
  6343. async def _list_timelapse_videos(printer) -> tuple[list[dict], str | None]:
  6344. """List video files from printer's timelapse directory.
  6345. Finds MP4 (X1/A1 series) and AVI (P1 series) timelapse files.
  6346. Returns (video_files, found_path) where video_files is a list of file dicts
  6347. and found_path is the directory where they were found, or ([], None).
  6348. An empty return does not distinguish "no videos" from "could not read the
  6349. card" — see :func:`_read_timelapse_baseline`, which the two baseline
  6350. callers use instead.
  6351. """
  6352. from backend.app.services.bambu_ftp import ftps_background_blocked, list_files_async
  6353. logger = logging.getLogger(__name__)
  6354. # No card in the slot means no /timelapse to walk — four connections that
  6355. # can only fail, on a path whose failures are swallowed and so would go on
  6356. # costing time silently forever (#2780).
  6357. #
  6358. # ``getattr`` rather than ``printer.id``: every dereference below happens
  6359. # inside the loop's own try/except, so a caller that passed something
  6360. # unexpected used to get an empty listing rather than an exception. Keep
  6361. # that, instead of making this gate the first thing that can raise here.
  6362. printer_id = getattr(printer, "id", None)
  6363. if printer_id is not None and not external_storage_present(printer_manager.get_status(printer_id)):
  6364. logger.debug("[TIMELAPSE] Skipping the scan for printer %s: it reports no external storage", printer_id)
  6365. return [], None
  6366. for timelapse_path in _TIMELAPSE_DIRECTORIES:
  6367. if ftps_background_blocked(getattr(printer, "ip_address", None) or ""):
  6368. # The printer's file service is refusing or hung: every remaining
  6369. # path is another connection it may keep a session for (#3474).
  6370. logger.debug("[TIMELAPSE] Skipping the scan for printer %s: its file service is cooling off", printer_id)
  6371. return [], None
  6372. try:
  6373. found_files = await list_files_async(
  6374. printer.ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  6375. )
  6376. if found_files:
  6377. video_files = [
  6378. f
  6379. for f in found_files
  6380. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  6381. ]
  6382. if video_files:
  6383. return video_files, timelapse_path
  6384. except Exception as e:
  6385. logger.debug("[TIMELAPSE] Path %s failed: %s", timelapse_path, e)
  6386. continue
  6387. return [], None
  6388. async def _capture_timelapse_baseline_at_start(
  6389. printer, printer_id: int, logger: logging.Logger, archive_id: int | None = None
  6390. ) -> None:
  6391. """Snapshot the printer's timelapse directory at print start so the
  6392. completion-time scan can pick the new file by set-difference.
  6393. Must be called from every on_print_start path that proceeds to a real
  6394. print — both the new-archive branch and the expected-archive branch (which
  6395. queue / VP-dispatched prints take). Without a baseline,
  6396. _scan_for_timelapse_with_retries falls into its "take baseline now"
  6397. fallback that runs AFTER the new MP4 has already landed on the SD card,
  6398. so the new file ends up in the "baseline" set and no diff ever matches.
  6399. Bambu printers in LAN-only mode don't sync NTP, so mtime ordering is
  6400. unreliable — the snapshot-diff approach sidesteps that entirely.
  6401. When ``archive_id`` is known the baseline is also written to the archive
  6402. row, so it survives a restart and the manual "Scan for Timelapse" button
  6403. can run the same diff instead of falling back to clock-based matching
  6404. (#2704). Only baselines taken at print start are persisted — one taken at
  6405. completion already contains the new video and would poison a later scan.
  6406. """
  6407. names: set[str] | None = None
  6408. unread = False
  6409. _timelapse_baselines_unread.discard(printer_id)
  6410. try:
  6411. names, read = await _read_timelapse_baseline(printer)
  6412. unread = not read
  6413. _timelapse_baselines[printer_id] = names
  6414. if unread:
  6415. # Recorded anyway, deliberately, and marked. An empty baseline taken
  6416. # off a card we could not read is not evidence, but it is still the
  6417. # right *default*: Bambuddy deletes each video from the printer once
  6418. # it is attached, so the usual card holds exactly one video at
  6419. # completion and an empty baseline resolves it. Persisting NULL
  6420. # instead would send completion to take its own snapshot, by which
  6421. # point this print's video is on the card and would be swallowed by
  6422. # it. The mark keeps the scan from choosing between several videos
  6423. # (#3412), and the card is read again while the print runs.
  6424. _timelapse_baselines_unread.add(printer_id)
  6425. logger.warning(
  6426. "[TIMELAPSE] Baseline for printer %s taken while its card could not be read — "
  6427. "treating it as empty for now and reading it again during the print",
  6428. printer_id,
  6429. )
  6430. logger.info(
  6431. "[TIMELAPSE] Baseline at print start: %s video files for printer %s",
  6432. len(names),
  6433. printer_id,
  6434. )
  6435. except Exception as e:
  6436. logger.warning("[TIMELAPSE] Failed to capture baseline at print start: %s", e)
  6437. run_started_at: datetime | None = None
  6438. run_id: int | None = None
  6439. if archive_id is not None:
  6440. try:
  6441. async with async_session() as db:
  6442. from backend.app.models.archive import PrintArchive
  6443. from backend.app.services.print_log import open_run_on_printer
  6444. archive = await db.get(PrintArchive, archive_id)
  6445. if archive is not None:
  6446. # Written even when the listing failed, and then as NULL. A
  6447. # reprint reuses the archive row, so leaving the previous
  6448. # run's baseline in place would have the scan diff this print
  6449. # against the state of the printer before the *last* one —
  6450. # and a stale baseline reads as authoritative, where NULL
  6451. # correctly falls back to a fresh snapshot.
  6452. baseline = sorted(names) if names is not None else None
  6453. archive.timelapse_baseline = baseline
  6454. archive.timelapse_baseline_unread = unread
  6455. run_started_at = archive.started_at
  6456. # And on the run this print opened, whose scan reads it:
  6457. # the archive's is its latest start's, which another plate
  6458. # of it on another printer can take over.
  6459. run = await open_run_on_printer(db, printer_id)
  6460. if run is not None and run.archive_id == archive_id:
  6461. run.timelapse_baseline = baseline
  6462. run.timelapse_baseline_unread = unread
  6463. run_id = run.id
  6464. await db.commit()
  6465. except Exception as e:
  6466. # In-memory baseline still covers the normal completion path.
  6467. logger.warning("[TIMELAPSE] Failed to persist baseline for archive %s: %s", archive_id, e)
  6468. if unread and names is not None:
  6469. spawn_background_task(
  6470. _reread_unread_timelapse_baseline(printer_id, names, archive_id, run_started_at, run_id),
  6471. name=f"reread-timelapse-baseline-{printer_id}",
  6472. )
  6473. # Before the print has ended there is no video yet: a long PREPARE (heating,
  6474. # levelling) counts as much as the print itself.
  6475. _BASELINE_REREAD_STATES = _ACTIVE_PRINT_STATES | {"PREPARE", "SLICING"}
  6476. def _schedule_timelapse_scan(
  6477. archive_id: int, printer_id: int, *, plate_id: int | None, run_id: int | None = None
  6478. ) -> None:
  6479. """Start the completion scan with the print-start baseline: the names, and
  6480. whether they were taken off a card that could not be read. Taking them
  6481. ends any re-read still running for it. ``run_id`` is the print that
  6482. ended, whose video it looks for."""
  6483. unread = printer_id in _timelapse_baselines_unread
  6484. _timelapse_baselines_unread.discard(printer_id)
  6485. baseline = _timelapse_baselines.pop(printer_id, None)
  6486. from backend.app.services import archive_work
  6487. # Held while it looks: it finds its print by the archive's id, which a
  6488. # merge would move away (archive redesign P2).
  6489. spawn_background_task(
  6490. archive_work.held_while(
  6491. archive_id,
  6492. _scan_for_timelapse_with_retries(
  6493. archive_id, baseline, plate_id=plate_id, baseline_unread=unread, run_id=run_id
  6494. ),
  6495. ),
  6496. name=f"scan-timelapse-{archive_id}",
  6497. )
  6498. def _unread_baseline_round(printer_id: int, names: set[str]) -> bool | None:
  6499. """Whether to read an unread baseline again now: True to read, None to
  6500. skip this round, False once it is over -- the print it was taken for is
  6501. no longer the one on the printer, has stopped printing, or the baseline
  6502. has been read.
  6503. ``names`` is the very set object recorded at print start: completion pops
  6504. it and the next print records a new one, so identity marks the run. A
  6505. printer that is momentarily disconnected says nothing about its print, so
  6506. that only skips the round.
  6507. So does a print that is not recording a timelapse: completion only looks
  6508. for a video when one was recorded, so reading the card for it is a
  6509. connection for nothing (#3474). Skipped rather than ended, as the flag can
  6510. arrive after the print has started.
  6511. """
  6512. if _timelapse_baselines.get(printer_id) is not names or printer_id not in _timelapse_baselines_unread:
  6513. return False
  6514. state = printer_manager.get_status(printer_id)
  6515. if state is None or not state.connected:
  6516. return None
  6517. if state.state not in _BASELINE_REREAD_STATES:
  6518. return False
  6519. return True if state.timelapse else None
  6520. async def _reread_unread_timelapse_baseline(
  6521. printer_id: int,
  6522. names: set[str],
  6523. archive_id: int | None,
  6524. run_started_at: datetime | None,
  6525. run_id: int | None = None,
  6526. ) -> None:
  6527. """Read again a print-start baseline the card could not give (#3412).
  6528. The printer writes the video only once the print has ended, so the card
  6529. read at any point during the print holds exactly what it held at the
  6530. start. Each round stops once the print is no longer the one the baseline
  6531. was taken for or has stopped printing, and checks that again after the
  6532. listing returns, before anything is recorded.
  6533. A round that cannot read the card doubles the wait before the next one,
  6534. and enough of them in a row end the re-read (#3474): see
  6535. ``_TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS``.
  6536. """
  6537. from backend.app.models.archive import PrintArchive
  6538. from backend.app.models.printer import Printer
  6539. from backend.app.services.print_log import run_of
  6540. logger = logging.getLogger(__name__)
  6541. interval = _TIMELAPSE_BASELINE_REREAD_INTERVAL_SECONDS
  6542. failed_rounds = 0
  6543. for _ in range(_TIMELAPSE_BASELINE_REREAD_MAX_ATTEMPTS):
  6544. await asyncio.sleep(interval)
  6545. now = _unread_baseline_round(printer_id, names)
  6546. if now is False:
  6547. return
  6548. if now is None:
  6549. continue
  6550. try:
  6551. async with async_session() as db:
  6552. printer = await db.get(Printer, printer_id)
  6553. if printer is None:
  6554. return
  6555. fresh, read = await _read_timelapse_baseline(printer)
  6556. except Exception as e:
  6557. logger.debug("[TIMELAPSE] Re-reading the baseline for printer %s failed: %s", printer_id, e)
  6558. fresh, read = set(), False
  6559. if not read:
  6560. failed_rounds += 1
  6561. if failed_rounds >= _TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS:
  6562. logger.warning(
  6563. "[TIMELAPSE] Card of printer %s still unreadable after %s tries; no longer reading "
  6564. "the baseline for this print, it stays unread",
  6565. printer_id,
  6566. failed_rounds,
  6567. )
  6568. return
  6569. interval = min(interval * 2, _TIMELAPSE_BASELINE_REREAD_MAX_INTERVAL_SECONDS)
  6570. continue
  6571. now = _unread_baseline_round(printer_id, names)
  6572. if now is False:
  6573. return
  6574. if now is None:
  6575. # The printer dropped off while the listing was out, so nothing
  6576. # says it did not end meanwhile: keep the reading for a round that
  6577. # can vouch for it.
  6578. continue
  6579. names.update(fresh)
  6580. _timelapse_baselines_unread.discard(printer_id)
  6581. logger.info(
  6582. "[TIMELAPSE] Baseline for printer %s read during the print: %s video files",
  6583. printer_id,
  6584. len(fresh),
  6585. )
  6586. if archive_id is None:
  6587. return
  6588. try:
  6589. async with async_session() as db:
  6590. archive = await db.get(PrintArchive, archive_id)
  6591. # Only the run it was taken for: by now the row can belong to a
  6592. # reprint, whose baseline is its own.
  6593. if archive is not None and archive.started_at == run_started_at and archive.timelapse_baseline_unread:
  6594. archive.timelapse_baseline = sorted(names)
  6595. archive.timelapse_baseline_unread = False
  6596. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6597. if run is not None and run.timelapse_baseline_unread:
  6598. run.timelapse_baseline = sorted(names)
  6599. run.timelapse_baseline_unread = False
  6600. await db.commit()
  6601. except Exception as e:
  6602. # The in-memory baseline is read now, which covers completion
  6603. # unless Bambuddy restarts first.
  6604. logger.warning("[TIMELAPSE] Failed to persist the re-read baseline for archive %s: %s", archive_id, e)
  6605. return
  6606. async def _timelapse_still_wanted(db, archive, run_id: int | None, run_started_at: datetime | None) -> bool:
  6607. """Whether the print a timelapse is for still has none: its run, by id,
  6608. exists without a video and no later print took over (``superseded``).
  6609. Without a run, the archive is still on that print's start (a newer run's
  6610. video is its own) and has none."""
  6611. from backend.app.services.print_log import run_of, superseded
  6612. if run_id is not None:
  6613. run = await run_of(db, archive.id, run_id)
  6614. return run is not None and run.timelapse_path is None and not await superseded(db, run)
  6615. return archive.timelapse_path is None and archive.started_at == run_started_at
  6616. async def _scan_for_timelapse_with_retries(
  6617. archive_id: int,
  6618. baseline_names: set[str] | None = None,
  6619. *,
  6620. plate_id: int | None = None,
  6621. baseline_unread: bool = False,
  6622. run_id: int | None = None,
  6623. ):
  6624. """Poll the printer for this print's timelapse and attach it.
  6625. Snapshot diff, not timestamp matching: a printer in LAN-only mode cannot
  6626. reach Bambu's NTP server, so the clock behind both the filename and the FTP
  6627. mtime is arbitrarily wrong — one reporter's P1S was six and a half days out
  6628. (#2704). Comparing the current listing against the set of filenames that
  6629. existed when the print started needs no clock at all, because the printer
  6630. writes the video only once the print has ended.
  6631. Baseline precedence: the caller's in-memory set, then the one persisted on
  6632. the archive at print start, then a snapshot taken now. The last of those is
  6633. a poor substitute — by completion the new video may already be on the card,
  6634. in which case it lands in the "baseline" and no diff can ever match — but it
  6635. is all that is available for a print that began before Bambuddy started.
  6636. On success the video is deleted from the printer, which keeps ``/timelapse``
  6637. down to the unclaimed files and makes the next diff unambiguous.
  6638. ``baseline_unread`` marks the caller's baseline as taken off a card that
  6639. could not be read; a stored one carries the same mark on the archive.
  6640. ``run_id`` is the print that ended (archive redesign): the video is that
  6641. run's, the stored baseline is the run's own, and the poll ends once the
  6642. run has a video or is gone. Without it (no run was written), the run is
  6643. whichever the archive is on when the scan starts.
  6644. """
  6645. from backend.app.services.print_log import run_of
  6646. logger = logging.getLogger(__name__)
  6647. # Cleared when the baseline was taken off a card we could not read, so the
  6648. # attach step refuses to choose between several candidates (#2957, #3412).
  6649. baseline_trusted = True
  6650. # --- Phase 1: establish the baseline -------------------------------------
  6651. try:
  6652. async with async_session() as db:
  6653. from backend.app.models.printer import Printer
  6654. service = ArchiveService(db)
  6655. archive = await service.get_archive(archive_id)
  6656. if not archive:
  6657. logger.warning("[TIMELAPSE] Archive %s not found, aborting", archive_id)
  6658. return
  6659. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6660. if run_id is not None and run is None:
  6661. logger.info("[TIMELAPSE] Run %s of archive %s is gone, aborting", run_id, archive_id)
  6662. return
  6663. if run.timelapse_path if run is not None else archive.timelapse_path:
  6664. logger.info("[TIMELAPSE] Archive %s already has timelapse attached", archive_id)
  6665. return
  6666. # The printer the print ran on: plates of one archive can print on
  6667. # two printers, and the archive's is the one that started last.
  6668. scan_printer_id = run.printer_id if run is not None and run.printer_id else archive.printer_id
  6669. if not scan_printer_id:
  6670. logger.warning("[TIMELAPSE] Archive %s has no printer, aborting", archive_id)
  6671. return
  6672. # The run this scan is for. The archive row passes to a newer run
  6673. # when it prints again (the next plate, a reprint), and that run's
  6674. # video is its own.
  6675. run_started_at = run.started_at if run is not None else archive.started_at
  6676. # The baseline stored for it: the run's own, else the archive's
  6677. # while that is still this run's start's.
  6678. stored_baseline, stored_unread = None, False
  6679. if run is not None and run.timelapse_baseline is not None:
  6680. stored_baseline, stored_unread = run.timelapse_baseline, bool(run.timelapse_baseline_unread)
  6681. elif archive.started_at == run_started_at:
  6682. stored_baseline, stored_unread = archive.timelapse_baseline, archive.timelapse_baseline_unread
  6683. if baseline_names is not None and baseline_unread and stored_baseline is not None and not stored_unread:
  6684. # The card was read at print start, and the in-memory copy is
  6685. # one Bambuddy took after a restart off a card it could not
  6686. # read: the stored one is this run's and is evidence.
  6687. baseline_names = set(stored_baseline)
  6688. logger.info(
  6689. "[TIMELAPSE] Using stored baseline over an unread one: %s existing video files for archive %s",
  6690. len(baseline_names),
  6691. archive_id,
  6692. )
  6693. elif baseline_names is not None:
  6694. baseline_trusted = not baseline_unread
  6695. logger.info(
  6696. "[TIMELAPSE] Using print-start baseline: %s existing video files for archive %s",
  6697. len(baseline_names),
  6698. archive_id,
  6699. )
  6700. elif stored_baseline is not None:
  6701. # Persisted at print start — survives a restart mid-print.
  6702. baseline_names = set(stored_baseline)
  6703. baseline_trusted = not stored_unread
  6704. logger.info(
  6705. "[TIMELAPSE] Using stored baseline: %s existing video files for archive %s",
  6706. len(baseline_names),
  6707. archive_id,
  6708. )
  6709. else:
  6710. result = await db.execute(select(Printer).where(Printer.id == scan_printer_id))
  6711. printer = result.scalar_one_or_none()
  6712. if not printer:
  6713. logger.warning("[TIMELAPSE] Printer not found for archive %s, aborting", archive_id)
  6714. return
  6715. baseline_names, baseline_trusted = await _read_timelapse_baseline(printer)
  6716. if not baseline_trusted:
  6717. # The card is unreadable at the one moment a baseline has to
  6718. # be taken, so the empty listing means "we never looked",
  6719. # not "these are all new". Carry on with it anyway — the
  6720. # usual card holds exactly one video, which resolves
  6721. # correctly — but stop the poll from *choosing* between
  6722. # several, which is how a stale video got attached to this
  6723. # print and then deleted off the printer (#2957 follow-up).
  6724. logger.warning(
  6725. "[TIMELAPSE] Baseline for archive %s taken while printer %s's card could not be "
  6726. "read. A single new video still resolves; several will not be guessed between — "
  6727. "use Scan for Timelapse to pick one by hand",
  6728. archive_id,
  6729. scan_printer_id,
  6730. )
  6731. logger.info(
  6732. "[TIMELAPSE] Baseline snapshot (fallback): %s existing video files for archive %s",
  6733. len(baseline_names),
  6734. archive_id,
  6735. )
  6736. except Exception as e:
  6737. logger.warning("[TIMELAPSE] Failed to take baseline snapshot for archive %s: %s", archive_id, e)
  6738. return
  6739. # --- Phase 2: poll for a file that was not there when the print began -----
  6740. deadline = time.monotonic() + _TIMELAPSE_SCAN_TIMEOUT_SECONDS
  6741. max_attempts = _timelapse_scan_max_attempts()
  6742. seen_names: set[str] = set()
  6743. delay = _TIMELAPSE_SCAN_FIRST_DELAY_SECONDS
  6744. attempt = 0
  6745. while True:
  6746. await asyncio.sleep(delay)
  6747. delay = _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS
  6748. attempt += 1
  6749. try:
  6750. from backend.app.models.printer import Printer
  6751. # Read phase: fetch archive + printer in a short session and release
  6752. # the pooled connection BEFORE the FTP list/download below. Holding it
  6753. # across the FTP round-trips left one connection idle-in-transaction per
  6754. # in-flight scan (issue #2572).
  6755. async with async_session() as db:
  6756. from backend.app.services.print_log import claimed_timelapse_stems
  6757. service = ArchiveService(db)
  6758. archive = await service.get_archive(archive_id)
  6759. if not archive:
  6760. logger.warning("[TIMELAPSE] Archive %s not found, stopping poll", archive_id)
  6761. return
  6762. if not await _timelapse_still_wanted(db, archive, run_id, run_started_at):
  6763. logger.info(
  6764. "[TIMELAPSE] Archive %s has its video or is printing again, stopping the run's poll",
  6765. archive_id,
  6766. )
  6767. return
  6768. result = await db.execute(select(Printer).where(Printer.id == scan_printer_id))
  6769. printer = result.scalar_one_or_none()
  6770. if not printer:
  6771. logger.warning("[TIMELAPSE] Printer not found for archive %s, stopping poll", archive_id)
  6772. return
  6773. claimed = await claimed_timelapse_stems(db, scan_printer_id, archive_id=archive_id, run_id=run_id)
  6774. # I/O phase (no DB connection held): FTP list + download.
  6775. video_files, found_path = await _list_timelapse_videos(printer)
  6776. # The poll can run for dozens of rounds, so only narrate a round
  6777. # that saw something change. Repeating the whole listing every 30 s
  6778. # would bury the one interesting line in the support bundle.
  6779. names_now = {f.get("name", "") for f in video_files}
  6780. changed = attempt == 1 or names_now != seen_names
  6781. seen_names = names_now
  6782. speak = logger.info if changed else logger.debug
  6783. if video_files:
  6784. speak("[TIMELAPSE] Attempt %s: Found %s video files in %s", attempt, len(video_files), found_path)
  6785. if changed:
  6786. for f in video_files[:5]:
  6787. logger.info("[TIMELAPSE] - %s", f.get("name"))
  6788. attached = await _attach_first_unclaimed_timelapse(
  6789. archive_id,
  6790. printer,
  6791. video_files,
  6792. baseline_names,
  6793. claimed,
  6794. attempt,
  6795. logger,
  6796. quiet=not changed,
  6797. require_unambiguous=not baseline_trusted,
  6798. plate_id=plate_id,
  6799. run_started_at=run_started_at,
  6800. run_id=run_id,
  6801. )
  6802. if attached:
  6803. return
  6804. else:
  6805. speak("[TIMELAPSE] Attempt %s: No video files found, will retry", attempt)
  6806. except Exception as e:
  6807. logger.warning("[TIMELAPSE] Attempt %s failed with error: %s", attempt, e)
  6808. if attempt >= max_attempts or time.monotonic() >= deadline:
  6809. break
  6810. # No name-match fallback: it compared the print name against the filename,
  6811. # and Bambu firmware only ever writes "video_<timestamp>". Across 247 support
  6812. # bundles it fired 159 times and matched zero times, so all it added was a
  6813. # misleading log line before giving up.
  6814. logger.warning(
  6815. "[TIMELAPSE] No new video appeared for archive %s within %ss, giving up",
  6816. archive_id,
  6817. int(_TIMELAPSE_SCAN_TIMEOUT_SECONDS),
  6818. )
  6819. async def _attach_layer_timelapse(
  6820. archive_id: int,
  6821. timelapse_path: Path,
  6822. plate_id: int | None,
  6823. run_started_at: datetime | None,
  6824. run_id: int | None = None,
  6825. ) -> None:
  6826. """Attach the external camera's stitched timelapse to the archive.
  6827. The camera's video is the one the user set up in Bambuddy, so it wins
  6828. over the printer's own when both are on: it replaces the printer's video
  6829. (usually there first; stitching takes minutes) and that file is deleted,
  6830. leaving nothing in the archive folder unreferenced. ``run_started_at`` is
  6831. the run the frames were taken for: when the archive is printing again by
  6832. the time the stitch is done, the video is not the new run's and is
  6833. dropped. The stitched file is a temporary either way.
  6834. ``run_id`` is the print that ended (archive redesign): the video is that
  6835. run's and replaces that run's printer video; the poll by start above is
  6836. for a print without a run.
  6837. """
  6838. from backend.app.services.print_log import run_of, superseded
  6839. from backend.app.utils.safe_path import PathTraversalError, safe_join_under
  6840. logger = logging.getLogger(__name__)
  6841. try:
  6842. async with async_session() as db:
  6843. service = ArchiveService(db)
  6844. archive = await service.get_archive(archive_id)
  6845. if archive is None:
  6846. return
  6847. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6848. if run_id is not None and run is None:
  6849. logger.info("[LAYER-TL] Run %s of archive %s is gone; dropping its layer timelapse", run_id, archive_id)
  6850. return
  6851. if run is not None and await superseded(db, run):
  6852. logger.info(
  6853. "[LAYER-TL] A later print took over run %s of archive %s; dropping its layer timelapse",
  6854. run_id,
  6855. archive_id,
  6856. )
  6857. return
  6858. if run is None and archive.started_at != run_started_at:
  6859. logger.info(
  6860. "[LAYER-TL] Archive %s is printing again; dropping the previous run's layer timelapse", archive_id
  6861. )
  6862. return
  6863. holder = run if run is not None else archive
  6864. replaced = holder.timelapse_path
  6865. logger.info("[LAYER-TL] Attaching timelapse %s to archive %s", timelapse_path, archive_id)
  6866. timelapse_data = await asyncio.to_thread(timelapse_path.read_bytes)
  6867. if not await service.attach_timelapse(
  6868. archive_id, timelapse_data, "layer_timelapse.mp4", plate_id=plate_id, run_id=run_id
  6869. ):
  6870. return
  6871. attached = holder.timelapse_path
  6872. logger.info("[LAYER-TL] Layer timelapse attached successfully")
  6873. if replaced and replaced != attached:
  6874. try:
  6875. old = safe_join_under(app_settings.base_dir, replaced, http=False)
  6876. except PathTraversalError:
  6877. return
  6878. await asyncio.to_thread(old.unlink, missing_ok=True)
  6879. logger.info("[LAYER-TL] Replaced the printer's timelapse %s on archive %s", replaced, archive_id)
  6880. finally:
  6881. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  6882. # ``run_started_at`` not given: attach whichever run the archive is on.
  6883. _ANY_RUN = object()
  6884. async def _attach_first_unclaimed_timelapse(
  6885. archive_id: int,
  6886. printer,
  6887. video_files: list[dict],
  6888. baseline_names: set[str],
  6889. claimed: set[str],
  6890. attempt: int,
  6891. logger: logging.Logger,
  6892. *,
  6893. quiet: bool = False,
  6894. require_unambiguous: bool = False,
  6895. plate_id: int | None = None,
  6896. run_started_at: datetime | None | object = _ANY_RUN,
  6897. run_id: int | None = None,
  6898. ) -> bool:
  6899. """Download and attach the one video that belongs to this print.
  6900. A candidate is any file absent from the print-start baseline. More than one
  6901. can qualify when a previous print's video landed late, after this print's
  6902. baseline was taken — those are filtered out by name, because they are
  6903. already attached to another archive. Sorting the candidates instead would
  6904. mean sorting on mtime or on the filename timestamp, both of which come from
  6905. the printer's unsynced clock.
  6906. Returns True once a video is attached. The printer's copy is deleted only
  6907. after the attach succeeds on bytes whose length matched the listing.
  6908. ``quiet`` downgrades the "nothing yet" lines to DEBUG when the caller has
  6909. already seen this exact listing — the poll runs for many rounds and only the
  6910. rounds where something changed are worth an INFO line.
  6911. """
  6912. from backend.app.services.bambu_ftp import (
  6913. DownloadDeadlineExceeded,
  6914. delete_archived_timelapse,
  6915. download_file_bytes_async,
  6916. remote_file_settled,
  6917. )
  6918. speak = logger.debug if quiet else logger.info
  6919. new_files = [f for f in video_files if f.get("name", "") not in baseline_names]
  6920. if not new_files:
  6921. speak("[TIMELAPSE] Attempt %s: No new files since baseline, will retry", attempt)
  6922. return False
  6923. candidates = [f for f in new_files if Path(f.get("name", "")).stem not in claimed]
  6924. if not candidates:
  6925. speak(
  6926. "[TIMELAPSE] Attempt %s: %s new file(s), all already attached to other archives, will retry",
  6927. attempt,
  6928. len(new_files),
  6929. )
  6930. return False
  6931. if len(candidates) > 1:
  6932. if require_unambiguous:
  6933. # The baseline is not evidence -- it was taken off a card that could
  6934. # not be read -- so "new since the baseline" does not narrow these
  6935. # down at all. Taking the first would attach an arbitrary video to
  6936. # this print and then delete it from the printer.
  6937. logger.warning(
  6938. "[TIMELAPSE] Attempt %s: %s unclaimed videos (%s) and no baseline to tell them apart — "
  6939. "leaving all of them on the printer for manual selection",
  6940. attempt,
  6941. len(candidates),
  6942. ", ".join(str(f.get("name")) for f in candidates),
  6943. )
  6944. return False
  6945. logger.warning(
  6946. "[TIMELAPSE] Attempt %s: %s unclaimed new files (%s) — taking the first; "
  6947. "the rest stay on the printer for manual selection",
  6948. attempt,
  6949. len(candidates),
  6950. ", ".join(str(f.get("name")) for f in candidates),
  6951. )
  6952. target = candidates[0]
  6953. file_name = str(target.get("name") or "") # every listed entry has one
  6954. remote_path = target.get("path") or f"/timelapse/{file_name}"
  6955. logger.info(
  6956. "[TIMELAPSE] Attempt %s: New file detected: %s (downloading for archive %s)",
  6957. attempt,
  6958. file_name,
  6959. archive_id,
  6960. )
  6961. # The listing always carries a size (`list_files` skips entries it can't
  6962. # parse), but read it explicitly: the delete below is destructive and must
  6963. # depend on a size we actually had, not on one we hoped was there.
  6964. expected_size = target.get("size")
  6965. try:
  6966. timelapse_data = await download_file_bytes_async(
  6967. printer.ip_address,
  6968. printer.access_code,
  6969. remote_path,
  6970. printer_model=printer.model,
  6971. expected_size=expected_size,
  6972. )
  6973. except DownloadDeadlineExceeded:
  6974. # Too slow for its size this round; the printer keeps its copy.
  6975. timelapse_data = None
  6976. if not timelapse_data:
  6977. # Short or failed transfer. The printer keeps its copy, so the next
  6978. # round can try again — which is exactly why the delete below is
  6979. # gated on a verified download.
  6980. logger.warning("[TIMELAPSE] Attempt %s: Failed to download new file, will retry", attempt)
  6981. return False
  6982. # The length check above proves we got what the listing said, not that the
  6983. # printer had finished writing. A video still being written can be listed
  6984. # short, served short, and pass — so confirm it has stopped growing before
  6985. # committing to it and deleting the original (#2704).
  6986. if not await remote_file_settled(
  6987. printer.ip_address,
  6988. printer.access_code,
  6989. remote_path,
  6990. len(timelapse_data),
  6991. printer_model=printer.model,
  6992. ):
  6993. return False
  6994. # Write phase: attach in a fresh short-lived session. The download took a
  6995. # while: by now the archive may have its video, or be printing again for a
  6996. # newer run, whose video this is not. Then nothing is attached or deleted,
  6997. # and the poll ends (True).
  6998. async with async_session() as db:
  6999. service = ArchiveService(db)
  7000. if run_id is not None or run_started_at is not _ANY_RUN:
  7001. current = await service.get_archive(archive_id)
  7002. started = None if run_started_at is _ANY_RUN else cast("datetime | None", run_started_at)
  7003. if current is None or not await _timelapse_still_wanted(db, current, run_id, started):
  7004. logger.info(
  7005. "[TIMELAPSE] Archive %s moved on during the download (video attached or printing again); "
  7006. "%s stays on the printer",
  7007. archive_id,
  7008. file_name,
  7009. )
  7010. return True
  7011. success = await service.attach_timelapse(
  7012. archive_id, timelapse_data, file_name, plate_id=plate_id, run_id=run_id
  7013. )
  7014. if not success:
  7015. logger.warning("[TIMELAPSE] Failed to attach timelapse to archive %s", archive_id)
  7016. return False
  7017. logger.info("[TIMELAPSE] Successfully attached timelapse to archive %s", archive_id)
  7018. await delete_archived_timelapse(
  7019. printer.ip_address,
  7020. printer.access_code,
  7021. remote_path,
  7022. verified=expected_size is not None,
  7023. printer_model=printer.model,
  7024. printer_name=printer.name,
  7025. )
  7026. return True
  7027. # Defaults for the finish-photo-from-timelapse polling loop (#1397). These are
  7028. # module-level so tests can monkeypatch them down to ~0 without timing out.
  7029. _FINISH_PHOTO_TIMELAPSE_POLL_INTERVAL_SECONDS: float = 3.0
  7030. _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS: float = 60.0
  7031. # How long the *background* upgrade keeps waiting after the notification has
  7032. # already gone out (#2704 follow-up). The short bound above exists so a slow
  7033. # printer can't hold up the print-complete notification; this one exists so the
  7034. # archive still ends up with the better frame afterwards.
  7035. #
  7036. # Measured across 261 attaches in the support bundles, the video lands a median
  7037. # 13s after the print ends — but the P1 series writes MJPEG AVI rather than
  7038. # H.264 MP4 and serves it slowly, so its p90 is 167s and the worst observed case
  7039. # was 546s. Every other model was inside 26s. The long budget is therefore
  7040. # almost entirely for P1-series users; on everything else the short wait already
  7041. # wins and this task never runs.
  7042. _FINISH_PHOTO_UPGRADE_TIMEOUT_SECONDS: float = 900.0
  7043. async def _capture_finish_photo_from_timelapse(
  7044. archive_id: int,
  7045. archive_dir: Path,
  7046. timeout: float | None = None,
  7047. rotation: int = 0,
  7048. run_id: int | None = None,
  7049. ) -> tuple[str | None, bool]:
  7050. """Wait for the per-print timelapse to land on the archive and extract its
  7051. last frame as the finish photo (#1397).
  7052. Bambu firmware stops timelapse recording after the toolhead parks but
  7053. before the bed-drop end-gcode runs, so the last frame frames the finished
  7054. print correctly. A live camera grab at gcode_state=FINISH captures the
  7055. bed already lowered.
  7056. ``_scan_for_timelapse_with_retries`` runs in parallel and writes
  7057. ``archive.timelapse_path`` when the file lands. This function polls for
  7058. that field.
  7059. Returns ``(filename, still_pending)``. ``still_pending`` is True only when
  7060. the wait ran out with no video on the archive yet — i.e. the video may
  7061. still be coming and a later attempt could succeed. It is False when the
  7062. video landed (whether or not extraction worked), because in that case
  7063. waiting longer changes nothing. The caller uses that to decide between
  7064. falling back permanently and scheduling a background upgrade.
  7065. ``rotation`` is the printer's camera_rotation, applied to the extracted
  7066. still (#2708) so this source agrees with every other finish-photo source.
  7067. The archived video itself is the printer's own file and is left alone —
  7068. rotating it would mean re-encoding it.
  7069. ``run_id`` is the print that ended: its own video is the one waited for,
  7070. not the archive's, which can be another plate's.
  7071. """
  7072. import uuid
  7073. from backend.app.models.archive import PrintArchive
  7074. from backend.app.services.camera import apply_camera_rotation_to_file, extract_video_last_frame
  7075. from backend.app.services.print_log import run_of
  7076. logger = logging.getLogger(__name__)
  7077. budget = _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS if timeout is None else timeout
  7078. deadline = asyncio.get_event_loop().time() + budget
  7079. poll_interval = _FINISH_PHOTO_TIMELAPSE_POLL_INTERVAL_SECONDS
  7080. while True:
  7081. async with async_session() as db:
  7082. if run_id is not None:
  7083. run = await run_of(db, archive_id, run_id)
  7084. timelapse_relpath = run.timelapse_path if run else None
  7085. else:
  7086. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  7087. archive = result.scalar_one_or_none()
  7088. timelapse_relpath = archive.timelapse_path if archive else None
  7089. if timelapse_relpath:
  7090. video_path = app_settings.base_dir / timelapse_relpath
  7091. if video_path.exists() and video_path.stat().st_size > 0:
  7092. photos_dir = archive_dir / "photos"
  7093. photos_dir.mkdir(parents=True, exist_ok=True)
  7094. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  7095. filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  7096. output_path = photos_dir / filename
  7097. if await extract_video_last_frame(video_path, output_path):
  7098. await apply_camera_rotation_to_file(output_path, rotation, logger)
  7099. logger.info(
  7100. "[PHOTO-BG] Extracted finish photo from timelapse %s for archive %s",
  7101. video_path.name,
  7102. archive_id,
  7103. )
  7104. return filename, False
  7105. logger.warning(
  7106. "[PHOTO-BG] Timelapse %s landed but last-frame extraction failed for archive %s; falling back",
  7107. video_path.name,
  7108. archive_id,
  7109. )
  7110. return None, False
  7111. if asyncio.get_event_loop().time() >= deadline:
  7112. logger.info(
  7113. "[PHOTO-BG] Timelapse for archive %s didn't land within %.0fs; falling back to live camera",
  7114. archive_id,
  7115. budget,
  7116. )
  7117. return None, True
  7118. await asyncio.sleep(poll_interval)
  7119. async def _upgrade_finish_photo_from_timelapse(
  7120. archive_id: int,
  7121. archive_dir: Path,
  7122. rotation: int = 0,
  7123. run_id: int | None = None,
  7124. replace: str | None = None,
  7125. ) -> None:
  7126. """Put the timelapse's last frame on an archive after the fact (#2704).
  7127. The print-complete notification waits only ~60s for the video, because
  7128. holding a notification for minutes is worse than sending it with a live
  7129. camera grab. On a P1-series printer the video often lands well after that,
  7130. so the archive used to be stuck with the live grab — which is taken at
  7131. ``gcode_state=FINISH``, after the end G-code has dropped the bed, and is
  7132. the worse photo of the two.
  7133. This keeps waiting in the background and, when the video arrives, extracts
  7134. the frame and puts it *first* in the archive's photo list, so opening the
  7135. gallery shows it. ``replace`` is the live grab: the frame is written over
  7136. that file, under its name, rather than added next to it. Two finish photos
  7137. of the same moment is one too many, and the name has to survive: with
  7138. authentication off, the notification that already went out links to
  7139. ``/archives/<id>/photos/<name>``, which now shows the better photo. (With
  7140. it on, the link is to a copy in notification_photos, and attachments are
  7141. bytes, so neither cares.) Without ``replace``, or once that photo has been
  7142. deleted, the frame is added under its own name. ``run_id`` is the print it
  7143. is for, whose list leads with it too.
  7144. """
  7145. from backend.app.services.print_log import add_run_photo, run_of
  7146. logger = logging.getLogger(__name__)
  7147. filename, _ = await _capture_finish_photo_from_timelapse(
  7148. archive_id, archive_dir, timeout=_FINISH_PHOTO_UPGRADE_TIMEOUT_SECONDS, rotation=rotation, run_id=run_id
  7149. )
  7150. if not filename:
  7151. logger.info("[PHOTO-UPGRADE] No timelapse frame for archive %s; keeping the live grab", archive_id)
  7152. return
  7153. try:
  7154. async with async_session() as db:
  7155. from backend.app.models.archive import PrintArchive
  7156. archive = await db.get(PrintArchive, archive_id)
  7157. if archive is None:
  7158. return
  7159. photos = list(archive.photos or [])
  7160. if filename in photos:
  7161. return
  7162. photos_dir = archive_dir / "photos"
  7163. if replace and replace in photos and (photos_dir / replace).is_file():
  7164. # Atomic, so the gallery never reads half a JPEG.
  7165. await asyncio.to_thread(os.replace, photos_dir / filename, photos_dir / replace)
  7166. filename = replace
  7167. photos.remove(replace)
  7168. # Front of the list: PhotoGalleryModal opens at index 0.
  7169. archive.photos = [filename, *photos]
  7170. add_run_photo(await run_of(db, archive_id, run_id) if run_id is not None else None, filename, first=True)
  7171. await db.commit()
  7172. except Exception as e:
  7173. logger.warning("[PHOTO-UPGRADE] Failed to attach upgraded photo to archive %s: %s", archive_id, e)
  7174. return
  7175. logger.info(
  7176. "[PHOTO-UPGRADE] Archive %s now leads with the timelapse frame %s%s",
  7177. archive_id,
  7178. filename,
  7179. " (replacing the live grab)" if filename == replace else "",
  7180. )
  7181. await ws_manager.send_archive_updated({"id": archive_id, "photo_added": filename})
  7182. async def _restore_usage_tracking_session(printer_id: int, state, db, logger) -> None:
  7183. """Put the filament-attribution context back after a restart mid-print.
  7184. ``usage_tracker._active_sessions`` and ``PrinterState.tray_change_log``
  7185. both die with the process. The print keeps running, so at completion the
  7186. tracker would fall back to whatever the printer reports *now* — and AMS
  7187. filament backup makes "now" the substitute tray, charging the whole print
  7188. to the spool that only finished it.
  7189. The persisted row is only trusted when its print name still matches what
  7190. the printer says it is running: a row left behind by a completion we never
  7191. saw must not attach itself to the next print.
  7192. """
  7193. try:
  7194. from backend.app.api.routes.settings import get_setting
  7195. from backend.app.services.usage_tracker import (
  7196. clear_persisted_session,
  7197. get_persisted_ams_mapping,
  7198. get_persisted_print_name,
  7199. get_persisted_spool_change_log,
  7200. restore_session,
  7201. )
  7202. persisted_name = await get_persisted_print_name(db, printer_id)
  7203. current_name = (state.subtask_name or "").strip()
  7204. if persisted_name and current_name and persisted_name.strip() != current_name:
  7205. logger.info(
  7206. "[RESTART] Discarding stale print session for printer %s (%r != running %r)",
  7207. printer_id,
  7208. persisted_name,
  7209. current_name,
  7210. )
  7211. await clear_persisted_session(db, printer_id)
  7212. # Fall through to seeding: the print on the printer is real, it just
  7213. # isn't the one the row described.
  7214. persisted_log = None
  7215. else:
  7216. # Spoolman users get the tray-change log back but no in-memory
  7217. # session — see ``on_print_start`` on why that dict is load-bearing
  7218. # for the remain%-sync guard.
  7219. _spoolman_on = await get_setting(db, "spoolman_enabled")
  7220. persisted_log = await restore_session(
  7221. db,
  7222. printer_id,
  7223. register_active=not (bool(_spoolman_on) and _spoolman_on.lower() == "true"),
  7224. )
  7225. # The slot pills (#1762) read the mapping captured from the print
  7226. # command, which dies with the process as well. One this process
  7227. # has captured since is newer.
  7228. if state.print_ams_mapping is None:
  7229. state.print_ams_mapping = await get_persisted_ams_mapping(db, printer_id)
  7230. # Spools swapped on a tray before the restart (#3325); anything this
  7231. # process has recorded since goes after them.
  7232. persisted_swaps = await get_persisted_spool_change_log(db, printer_id)
  7233. if persisted_swaps:
  7234. seen = getattr(state, "spool_change_log", None) or []
  7235. state.spool_change_log = persisted_swaps + [item for item in seen if item not in persisted_swaps]
  7236. if persisted_log:
  7237. restored = [tuple(entry) for entry in persisted_log if isinstance(entry, (list, tuple)) and len(entry) == 2]
  7238. # Anything this process already observed goes after the persisted
  7239. # history — the log is ordered by layer, and a fresh process can
  7240. # only have seen changes from later in the print.
  7241. for entry in state.tray_change_log or []:
  7242. if tuple(entry) not in restored:
  7243. restored.append(tuple(entry))
  7244. state.tray_change_log = restored
  7245. tray_now = state.tray_now
  7246. if 0 <= tray_now <= 254:
  7247. if not state.tray_change_log:
  7248. # No persisted history — a print that started before this build,
  7249. # or before the row existed. Seed with the tray feeding right
  7250. # now so the remainder of the print is at least attributable to
  7251. # the right spool.
  7252. state.tray_change_log = [(tray_now, state.layer_num)]
  7253. logger.info(
  7254. "[RESTART] Seeded tray change log for printer %s: tray=%d at layer=%d",
  7255. printer_id,
  7256. tray_now,
  7257. state.layer_num,
  7258. )
  7259. # The tray handler updates ``last_loaded_tray`` on every push
  7260. # regardless of whether it logged a change, so re-align it to avoid
  7261. # a duplicate entry on the next push. Only ever with a real tray:
  7262. # ``last_loaded_tray`` is the "survives the end-of-print retract to
  7263. # 255" fallback, and writing 255 into it would defeat that.
  7264. state.last_loaded_tray = tray_now
  7265. except Exception:
  7266. # Never let attribution recovery cost the caller its timelapse
  7267. # baseline — that capture has to happen before the printer uploads
  7268. # the in-flight MP4 and there is no second chance at it.
  7269. logger.exception("[RESTART] Failed to restore usage-tracking session for printer %s", printer_id)
  7270. async def on_print_running_observed(printer_id: int, data: dict):
  7271. """Restart-recovery for a print that started before Bambuddy came up.
  7272. bambu_mqtt.py suppresses ``on_print_start`` on the first RUNNING push
  7273. after Bambuddy startup (#1304 guard, prevents duplicate archive
  7274. creation). This hook restores the persisted archive into ``_active_prints``
  7275. and captures the timelapse baseline that normally hangs off print start.
  7276. Fires once per session, in lieu of on_print_start when restart-recovery
  7277. kicks in. The printer doesn't upload the timelapse until after PRINT
  7278. COMPLETE, so a baseline captured any time during the print is still
  7279. pre-upload.
  7280. """
  7281. logger = logging.getLogger(__name__)
  7282. async with async_session() as db:
  7283. from backend.app.models.printer import Printer
  7284. state = printer_manager.get_status(printer_id)
  7285. if state is not None:
  7286. authorization = await _is_bambuddy_authorized_print(printer_id, state, db)
  7287. if authorization is True:
  7288. logger.info("[RESTART] Restored active Bambuddy print for printer %s", printer_id)
  7289. await _restore_usage_tracking_session(printer_id, state, db, logger)
  7290. await _restore_printable_objects(printer_id, state, db, logger)
  7291. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  7292. printer = result.scalar_one_or_none()
  7293. if not printer:
  7294. logger.warning(
  7295. "[TIMELAPSE] on_print_running_observed: printer %s not found in DB, skipping baseline",
  7296. printer_id,
  7297. )
  7298. return
  7299. # Avoid double-capture: ownership reconciliation above must still run when
  7300. # a baseline already exists, but the camera work itself is one-shot.
  7301. if printer_id in _timelapse_baselines:
  7302. logger.debug(
  7303. "[TIMELAPSE] on_print_running_observed: baseline already present for printer %s, skipping",
  7304. printer_id,
  7305. )
  7306. return
  7307. await _capture_timelapse_baseline_at_start(printer, printer_id, logger)
  7308. class _RunOfArchive:
  7309. """An archive as one of its runs printed it: the run's job id, start and
  7310. plate, the rest the archive's. What the reconcile judges and closes."""
  7311. def __init__(self, archive, run):
  7312. self._archive = archive
  7313. self.subtask_id = run.subtask_id
  7314. self.started_at = run.started_at
  7315. self.plate_id = run.plate_id if run.plate_id is not None else archive.plate_id
  7316. def __getattr__(self, name):
  7317. return getattr(self._archive, name)
  7318. def _is_active_archive_stale(archive, state) -> tuple[bool, str]:
  7319. """Return ``(is_stale, reason)`` for an archive in ``status="printing"``
  7320. against the printer's current MQTT state.
  7321. Reconciliation triggers (#1542 follow-up — recovers from missed PRINT
  7322. COMPLETE events, typically a print finishing during an MQTT disconnect
  7323. window followed by a smart-plug power cycle):
  7324. 1. Printer state is terminal (IDLE / FINISH / FAILED). The print is
  7325. provably not running anymore — only branch that should fire under
  7326. normal disconnect-then-reconnect timing.
  7327. 2. Printer has a different ``subtask_id`` than the archive. Bambu
  7328. firmware mints a fresh ``subtask_id`` for each print, including the
  7329. ghost replay it runs after a power cycle from a leftover SD file —
  7330. so a mismatch unambiguously means the in-DB archive is no longer
  7331. the print on the printer.
  7332. 3. Printer is running but ``subtask_name`` is empty. The printer
  7333. doesn't know what it's running; the archive's reference to it is
  7334. already broken.
  7335. Conservative on purpose: PAUSE / PREPARE / SLICING and any RUNNING state
  7336. with matching subtask_id+subtask_name is left alone. The cost of a false
  7337. positive is a print closed while it still runs: the reconcile takes its
  7338. ``_active_prints`` entry and the archive is no longer ``printing``, so
  7339. the real PRINT COMPLETE later finds no archive to close (#1679). The
  7340. cost of a false negative is the ghost-print loop in #1542. What the
  7341. archive is closed as is ``_reconciled_outcome``'s call.
  7342. Pre-push guard (#1679): when ``state.state`` is empty or ``"unknown"``,
  7343. MQTT has connected but the first ``push_status`` response hasn't been
  7344. applied yet — ``PrinterState`` is sitting on its construction defaults.
  7345. The reconcile caller in ``on_printer_status_change`` is already gated
  7346. on a real ``state.state``, so in normal operation this branch is
  7347. unreachable; it's kept as belt-and-braces for future callers and for
  7348. the narrow window where a partial state update could arrive
  7349. (``state.state`` set but ``subtask_name`` not yet populated). Returning
  7350. ``not stale`` on degenerate input is strictly conservative: a real
  7351. stale archive will still be caught by the next push_status arriving
  7352. with terminal state.
  7353. """
  7354. current_state = (state.state or "").upper()
  7355. if current_state in ("", "UNKNOWN"):
  7356. # No real push_status yet — PrinterState defaults are not evidence.
  7357. return False, ""
  7358. if current_state in ("IDLE", "FINISH", "FAILED"):
  7359. return True, f"printer state {current_state}"
  7360. # Below here the printer is in a running / pre-running state (RUNNING /
  7361. # PAUSE / PREPARE / SLICING / etc.) — decide based on subtask identity.
  7362. current_subtask_id = (state.subtask_id or "").strip()
  7363. if archive.subtask_id and current_subtask_id and archive.subtask_id != current_subtask_id:
  7364. return True, f"subtask_id changed ({archive.subtask_id!r} → {current_subtask_id!r})"
  7365. current_subtask_name = (state.subtask_name or "").strip()
  7366. if not current_subtask_name:
  7367. return True, "printer subtask_name empty"
  7368. return False, ""
  7369. async def prime_kprofile_table(printer_id: int) -> int:
  7370. """Read the printer's calibration table once per connection.
  7371. The AMS slot card shows a K value per slot (#2854). On the printers whose
  7372. trays carry no ``k`` field of their own -- the whole H2 series, whose trays
  7373. report ``cali_idx`` and nothing else -- that number can only come from
  7374. ``state.kprofiles``, and nothing used to fill it on connect. It arrived by
  7375. luck: someone opening the Profiles page or Configure Slot, a nightly GitHub
  7376. backup, or the printer answering a query BambuStudio made on the report
  7377. topic we share. A Bambuddy that nobody visited showed a card with no K
  7378. values at all.
  7379. Only the diameters actually fitted are asked for, which is one request on a
  7380. single-nozzle printer and two on a dual. Probing the four sizes blind is
  7381. what the backup does, and it is both wasteful and the thing that used to
  7382. blank the table.
  7383. Returns the number of nozzles whose table was read.
  7384. """
  7385. client = printer_manager.get_client(printer_id)
  7386. state = printer_manager.get_status(printer_id)
  7387. if client is None or state is None or not state.connected:
  7388. return 0
  7389. # Deduplicated, order preserved: a dual-nozzle printer with two 0.4s should
  7390. # ask once, and both entries are empty until the first push_status lands.
  7391. diameters = list(dict.fromkeys(n.nozzle_diameter for n in (state.nozzles or []) if n.nozzle_diameter))
  7392. if not diameters:
  7393. logging.getLogger(__name__).debug(
  7394. "[Printer %s] No nozzle diameter reported yet; leaving the K-profile table to the next reader",
  7395. printer_id,
  7396. )
  7397. return 0
  7398. primed = 0
  7399. for diameter in diameters:
  7400. try:
  7401. profiles = await client.get_kprofiles(nozzle_diameter=diameter, max_retries=2)
  7402. except Exception as exc: # noqa: BLE001
  7403. # A printer that won't answer costs the card its K values, nothing
  7404. # more — never the connection this runs on the back of.
  7405. logging.getLogger(__name__).warning(
  7406. "[Printer %s] Could not read the K-profile table for nozzle %s: %s", printer_id, diameter, exc
  7407. )
  7408. continue
  7409. primed += 1
  7410. logging.getLogger(__name__).info(
  7411. "[Printer %s] Primed K-profile table for nozzle %s: %d profiles", printer_id, diameter, len(profiles)
  7412. )
  7413. return primed
  7414. def _sd_files_in_use(state) -> set[str]:
  7415. """Names of the SD-card files the printer may be printing right now (#3009).
  7416. The post-print cleanup works out what to delete from the finished archive,
  7417. not from the printer. When reconciliation closes an old archive because the
  7418. printer has moved on to a new job, and that job is the same file reprinted
  7419. from the printer's screen, the cleanup would delete the file mid-print.
  7420. Empty unless the printer is busy: on an ordinary completion the state is
  7421. FINISH or FAILED and nothing is held back. ``gcode_file`` arrives as a bare
  7422. name, a path or a URL depending on firmware, and some report only the
  7423. plate's G-code, so the job name is matched as well, in the same spellings
  7424. the cleanup tries. Names are lower-cased; the card's FAT filesystem
  7425. ignores case.
  7426. """
  7427. def _text(value) -> str:
  7428. return value.strip() if isinstance(value, str) else ""
  7429. if state is None:
  7430. return set()
  7431. current_state = _text(getattr(state, "state", None)).upper()
  7432. if current_state in ("", "UNKNOWN", "IDLE", "FINISH", "FAILED"):
  7433. return set()
  7434. in_use: set[str] = set()
  7435. gcode_file = _text(getattr(state, "gcode_file", None))
  7436. if gcode_file:
  7437. # Drop a scheme by hand: urlparse would cut a name at "#" or "?",
  7438. # both legal in a file name, and raises on some inputs.
  7439. name = PurePosixPath(gcode_file.split("://", 1)[-1]).name
  7440. if name:
  7441. in_use.add(name.lower())
  7442. subtask_name = _text(getattr(state, "subtask_name", None))
  7443. if subtask_name:
  7444. for name in (subtask_name, subtask_name.replace(" ", "_")):
  7445. for ext in (".3mf", ".gcode"):
  7446. in_use.add(f"{name}{ext}".lower())
  7447. return in_use
  7448. def _reconciled_outcome(archive, state) -> str:
  7449. """What a stale archive is closed as at reconnect (#3310).
  7450. A print that ends while Bambuddy restarts is never seen to end: the new
  7451. MQTT client's first status is already FINISH, which is not a transition.
  7452. The printer keeps reporting the finished job's ``subtask_id`` until the
  7453. next print, so a matching id proves the FINISH or FAILED is this
  7454. archive's. Anything else -- IDLE, another job, an id of "0" (a print
  7455. started outside the cloud) or none -- says only that the print is over,
  7456. not how, and stays ``aborted``.
  7457. """
  7458. current_state = (state.state or "").upper()
  7459. if current_state not in ("FINISH", "FAILED"):
  7460. return "aborted"
  7461. archive_id = (archive.subtask_id or "").strip()
  7462. current_id = str(state.subtask_id or "").strip()
  7463. if archive_id in ("", "0") or archive_id != current_id:
  7464. return "aborted"
  7465. return "completed" if current_state == "FINISH" else "failed"
  7466. def _reconciled_end_time(archive, now: datetime, plate_id: int | None = None) -> datetime | None:
  7467. """When a print reconciled as completed most likely ended, or None (#3310).
  7468. The printer does not say when it finished, and the reconnect can be hours
  7469. later. Its start plus the slicer's estimate for the plate it printed,
  7470. never later than now, is the best guess; None when either is missing,
  7471. and the run then keeps no duration, as an aborted one does.
  7472. ``plate_id`` is the plate this run printed, from its queue job: a reprint
  7473. of another plate reuses the archive without changing ``archive.plate_id``.
  7474. """
  7475. started = archive.started_at
  7476. if not started:
  7477. return None
  7478. if started.tzinfo is None:
  7479. started = started.replace(tzinfo=timezone.utc)
  7480. plate = plate_id if plate_id is not None else archive.plate_id
  7481. estimate = None
  7482. if archive.file_path:
  7483. from backend.app.utils.threemf_tools import extract_print_time_from_3mf
  7484. full_path = (
  7485. app_settings.base_dir / archive.file_path
  7486. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  7487. try:
  7488. estimate = extract_print_time_from_3mf(full_path, plate)
  7489. except Exception as exc: # noqa: BLE001
  7490. logging.getLogger(__name__).debug(
  7491. "Plate estimate for reconciled archive %s unreadable: %s", archive.id, exc
  7492. )
  7493. if plate is None and estimate != archive.print_time_seconds:
  7494. # Without a plate the first plate's estimate is read. The archive
  7495. # carries the sum over all plates (#1593), so a difference means
  7496. # several plates and no telling which one printed.
  7497. estimate = None
  7498. elif plate is None:
  7499. # Nothing to read the plate from; the archive's own estimate, as for
  7500. # a single-plate file.
  7501. estimate = archive.print_time_seconds
  7502. if not estimate or estimate <= 0:
  7503. return None
  7504. return min(started + timedelta(seconds=int(estimate)), now)
  7505. async def reconcile_stale_active_prints(printer_id: int) -> int:
  7506. """Synthesise ``on_print_complete`` for archives whose print can't be
  7507. running on the printer anymore.
  7508. Called once per MQTT (re)connection (from on_printer_status_change when
  7509. the connected edge flips False → True) and at Bambuddy startup (from
  7510. the FastAPI lifespan). Without this, a print that completes during a
  7511. disconnect window — followed by a smart-plug-driven power cycle — leaves
  7512. the ``.3mf`` on the SD card, the firmware auto-replays it on next boot,
  7513. and Bambuddy fires a fresh PRINT START for the ghost rather than the
  7514. SD cleanup that PRINT COMPLETE was supposed to run. Repeats every
  7515. power cycle until the operator notices (#1542 follow-up). Reconciliation
  7516. closes the loop by faking the missed PRINT COMPLETE — the existing
  7517. cleanup chain handles SD-file deletion, status updates, usage tracking,
  7518. and notifications.
  7519. The status is ``_reconciled_outcome``'s: ``completed`` or ``failed``
  7520. when the printer still reports the archive's own job as finished or
  7521. failed, ``aborted`` otherwise. Nothing corrects it later: the archive is
  7522. no longer ``printing``, so a later PRINT COMPLETE finds no archive, and
  7523. after a restart the printer's FINISH is not one (#3310).
  7524. Returns the number of archives reconciled.
  7525. """
  7526. state = printer_manager.get_status(printer_id)
  7527. if not state:
  7528. return 0
  7529. # Don't reconcile while disconnected — we'd be making a decision against
  7530. # stale cached state. The connected → reconcile edge handles this.
  7531. if not state.connected:
  7532. return 0
  7533. from backend.app.models.archive import PrintArchive
  7534. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, RUN_OPEN, PrintLogEntry
  7535. reconciled = 0
  7536. async with async_session() as db:
  7537. # The prints on this printer Bambuddy saw start: their runs open here.
  7538. # The archive row is shared by every plate of a file, its printer, job
  7539. # and start the latest start's, so each is judged as its run.
  7540. open_runs = (
  7541. await db.scalars(
  7542. select(PrintLogEntry)
  7543. .where(PrintLogEntry.printer_id == printer_id, PrintLogEntry.status == RUN_OPEN)
  7544. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  7545. )
  7546. ).all()
  7547. by_run = {
  7548. archive.id: archive
  7549. for archive in (
  7550. await db.scalars(
  7551. select(PrintArchive).where(
  7552. PrintArchive.id.in_([run.archive_id for run in open_runs if run.archive_id is not None])
  7553. )
  7554. )
  7555. ).all()
  7556. }
  7557. active: list = [_RunOfArchive(by_run[run.archive_id], run) for run in open_runs if run.archive_id in by_run]
  7558. # And as before runs were opened at the start: a printing archive of
  7559. # this printer with no run open anywhere (printing since before).
  7560. archives_with_open_runs = select(PrintLogEntry.archive_id).where(
  7561. PrintLogEntry.status == RUN_OPEN, PrintLogEntry.archive_id.is_not(None)
  7562. )
  7563. result = await db.execute(
  7564. select(PrintArchive)
  7565. .where(
  7566. PrintArchive.printer_id == printer_id,
  7567. PrintArchive.status == "printing",
  7568. PrintArchive.id.not_in(archives_with_open_runs),
  7569. )
  7570. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  7571. )
  7572. active.extend(result.scalars().all())
  7573. if not active:
  7574. return 0
  7575. # The plate each run printed, from its queue job (#3310).
  7576. from backend.app.models.print_queue import PrintQueueItem
  7577. async with async_session() as db:
  7578. plate_rows = await db.execute(
  7579. select(PrintQueueItem.archive_id, PrintQueueItem.plate_id).where(
  7580. PrintQueueItem.printer_id == printer_id,
  7581. PrintQueueItem.status == "printing",
  7582. PrintQueueItem.archive_id.in_([a.id for a in active]),
  7583. )
  7584. )
  7585. run_plates = {archive_id: plate for archive_id, plate in plate_rows.all() if plate is not None}
  7586. logger = logging.getLogger(__name__)
  7587. for archive in active:
  7588. is_stale, reason = _is_active_archive_stale(archive, state)
  7589. if not is_stale:
  7590. continue
  7591. logger.info(
  7592. "[RECONCILE] Printer %s: synthesising missed PRINT COMPLETE for archive %s (%s) — %s",
  7593. printer_id,
  7594. archive.id,
  7595. archive.filename,
  7596. reason,
  7597. )
  7598. # Synthesised payload: minimal fields the on_print_complete chain
  7599. # needs. `_reconciled` marker lets downstream code distinguish this
  7600. # from a real MQTT-driven completion if it ever needs to (e.g. for
  7601. # metrics / debug logging). raw_data is the live printer state so
  7602. # the usage tracker can compare end-of-print remain% against the
  7603. # captured start values.
  7604. outcome = _reconciled_outcome(archive, state)
  7605. payload = {
  7606. "status": outcome,
  7607. "filename": archive.filename,
  7608. "subtask_name": archive.print_name or "",
  7609. "subtask_id": archive.subtask_id or "",
  7610. "raw_data": state.raw_data or {},
  7611. "_reconciled": True,
  7612. }
  7613. if isinstance(archive, _RunOfArchive):
  7614. # The archive it closes, which its names and printer may not find:
  7615. # another printer's plate of it can have started since.
  7616. payload["_reconciled_archive_id"] = archive.id
  7617. if outcome != "aborted":
  7618. # The printer's progress and layer are this job's, so a partial
  7619. # run is charged what it printed rather than nothing.
  7620. from backend.app.utils.printer_models import usage_progress
  7621. run_progress = usage_progress(
  7622. printer_manager.get_model(printer_id),
  7623. state.progress,
  7624. getattr(state, "layer_num", None),
  7625. getattr(state, "total_layers", None),
  7626. )
  7627. if run_progress:
  7628. payload["last_progress"] = run_progress
  7629. if state.layer_num:
  7630. payload["last_layer_num"] = state.layer_num
  7631. if outcome == "failed":
  7632. payload["hms_errors"] = [e.completion_payload() for e in state.hms_errors or []]
  7633. if outcome == "completed":
  7634. ended = _reconciled_end_time(archive, datetime.now(timezone.utc), run_plates.get(archive.id))
  7635. if ended:
  7636. payload["_reconciled_completed_at"] = ended
  7637. try:
  7638. await on_print_complete(printer_id, payload)
  7639. reconciled += 1
  7640. except Exception as e:
  7641. # Catch-all: a reconciliation failure must not block the
  7642. # printer's normal status flow. The archive stays in
  7643. # ``status="printing"`` and the next reconnect retries.
  7644. logger.warning(
  7645. "[RECONCILE] on_print_complete synthesis failed for archive %s: %s",
  7646. archive.id,
  7647. e,
  7648. )
  7649. return reconciled
  7650. # #2547: clearance left between the nozzle and the top of the print when the
  7651. # plate is commanded back into camera framing. The nozzle is parked away from
  7652. # the part by then, so this is belt-and-braces against a max_z_height that
  7653. # under-reports (e.g. a slicer that excludes a final Z hop).
  7654. _PLATE_RESTORE_CLEARANCE_MM = 10.0
  7655. # Feedrate for both moves. F600 is exactly what Bambu's own end G-code uses on
  7656. # this axis, so it is a proven-safe speed for the full travel.
  7657. _PLATE_RESTORE_FEEDRATE = 600
  7658. # Time allowed for the plate to reach the restored position before the camera
  7659. # grab. Sized for the ~100 mm the stock end G-code drops on X1/P1 at F600
  7660. # (10 mm/s); most H2 and P2 end G-code drops less, or not at all.
  7661. _PLATE_RESTORE_SETTLE_SECONDS = 12.0
  7662. # How long `_background_finish_photo` waits for this producer. Must cover the
  7663. # settle window plus a worst-case RTSP grab (15s), and stay below the
  7664. # notification path's own photo wait so a slow producer degrades to a
  7665. # photo-less notification rather than a missed one. The chamber light's
  7666. # snapshot delay (#1655) spends from the spare 8 seconds, which is why
  7667. # camera_light.MAX_DELAY_SECONDS stays below it.
  7668. _FINISH_PHOTO_PRODUCER_WAIT_SECONDS = _PLATE_RESTORE_SETTLE_SECONDS + 23.0
  7669. def _completion_print_keys(printer_id: int, filename: str, subtask_name: str) -> list[tuple[int, str]]:
  7670. """The ``_active_prints`` keys a finished print may be registered under.
  7671. Matching how they were registered in on_print_start. Shared by
  7672. ``on_print_complete`` and the finish-photo plate restore (#3240), which has
  7673. to find the same binding.
  7674. """
  7675. possible_keys = []
  7676. # Try subtask_name variations first (most reliable for matching)
  7677. if subtask_name:
  7678. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  7679. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  7680. possible_keys.append((printer_id, subtask_name))
  7681. # Try filename variations
  7682. if filename:
  7683. # Extract just the filename if it's a path
  7684. fname = filename.split("/")[-1] if "/" in filename else filename
  7685. if fname.endswith(".3mf"):
  7686. possible_keys.append((printer_id, fname))
  7687. elif fname.endswith(".gcode"):
  7688. base_name = fname.rsplit(".", 1)[0]
  7689. possible_keys.append((printer_id, f"{base_name}.gcode.3mf"))
  7690. possible_keys.append((printer_id, f"{base_name}.3mf"))
  7691. possible_keys.append((printer_id, fname))
  7692. else:
  7693. possible_keys.append((printer_id, f"{fname}.gcode.3mf"))
  7694. possible_keys.append((printer_id, f"{fname}.3mf"))
  7695. possible_keys.append((printer_id, fname))
  7696. # Also try full path versions
  7697. if filename.endswith(".3mf"):
  7698. possible_keys.append((printer_id, filename))
  7699. elif filename.endswith(".gcode"):
  7700. base_name = filename.rsplit(".", 1)[0]
  7701. possible_keys.append((printer_id, f"{base_name}.3mf"))
  7702. possible_keys.append((printer_id, filename))
  7703. else:
  7704. possible_keys.append((printer_id, f"{filename}.3mf"))
  7705. possible_keys.append((printer_id, filename))
  7706. return possible_keys
  7707. def _archive_name_forms(archive) -> set[str]:
  7708. """The names an archive could be echoed back as, in normalised form (#3240).
  7709. Its print name, and its filename with and without the 3MF extension.
  7710. """
  7711. forms = set()
  7712. if archive.print_name:
  7713. forms.add(_normalise_subtask_name(archive.print_name))
  7714. if archive.filename:
  7715. forms.add(_normalise_subtask_name(archive.filename))
  7716. for suffix in (".gcode.3mf", ".3mf"):
  7717. if archive.filename.lower().endswith(suffix):
  7718. forms.add(_normalise_subtask_name(archive.filename[: -len(suffix)]))
  7719. forms.discard("")
  7720. return forms
  7721. def _bound_print(printer_id: int, data: dict) -> tuple[int | None, int | None]:
  7722. """The archive and plate Bambuddy bound to the print ending on ``printer_id`` (#3240).
  7723. Read from ``_active_prints`` with the keys ``on_print_complete`` tries, and
  7724. from the plate recorded when the job was dispatched. Neither is touched:
  7725. ``on_print_complete`` pops both, which is why the finish-photo producer
  7726. calls this before its first await, while they are still there.
  7727. ``(None, None)`` for a print Bambuddy did not bind -- one it failed to
  7728. archive, or that it only knows by a name -- and on any error: the producer
  7729. calls this before the ``try`` that always releases the photo consumer, so
  7730. it must not raise.
  7731. """
  7732. try:
  7733. keys = _completion_print_keys(printer_id, str(data.get("filename") or ""), str(data.get("subtask_name") or ""))
  7734. archive_id = next((_active_prints[key] for key in keys if key in _active_prints), None)
  7735. if archive_id is None:
  7736. return None, None
  7737. return archive_id, _print_plate_ids.get((printer_id, archive_id))
  7738. except Exception:
  7739. logging.getLogger(__name__).debug("Could not read the print binding for printer %s", printer_id, exc_info=True)
  7740. return None, None
  7741. class PlateHeights(NamedTuple):
  7742. """Where the finished print is, and where its end G-code left the plate (#2547)."""
  7743. print_top: float
  7744. """The plate's ``max_z_height``: the Z the toolhead sat at for the last layer."""
  7745. end_z: float
  7746. """The last Z the end G-code moved to: where the plate is now, and where it is parked back to."""
  7747. async def _plate_heights_for_current_print(
  7748. printer_id: int,
  7749. data: dict,
  7750. logger,
  7751. archive_id: int | None,
  7752. plate_id: int | None = None,
  7753. ) -> PlateHeights | None:
  7754. """Heights of the print that just finished on ``printer_id``, or None (#2547).
  7755. These numbers become the targets of real Z moves, so every step here refuses
  7756. rather than guesses. Both have to be known: without the end Z there is no
  7757. position to park the plate back to that the printer's own G-code chose, and
  7758. the firmware won't stop a made-up one at the travel limit (#2579). A height belonging to some *other* print is the one
  7759. failure that could drive the nozzle into the model: 20 mm carried onto a
  7760. 200 mm print would command the plate up through the part.
  7761. Three independent things therefore have to agree before a height is
  7762. returned:
  7763. 1. **Binding.** ``archive_id`` is the archive Bambuddy bound to this print
  7764. when it started (see ``_bound_print``), never one found by name. A name
  7765. is not an identity: every plate of a multi-plate file shares one, and a
  7766. reprint reuses its archive row, so the newest archive with the right
  7767. name can be another plate of the same file. Support bundles show the
  7768. old name lookup doing exactly that (archive 288 for a print bound to
  7769. 169, 313 layers against 125), stopped only by the layer check below.
  7770. No binding, no move.
  7771. 2. **Identity.** The bound archive's name still has to be the finished
  7772. print's ``subtask_name``, compared in ``_normalise_subtask_name`` form
  7773. because the printer echoes spaces as underscores (#3240). Exact: not a
  7774. substring, and not the completion check's tolerance for a truncated
  7775. echo.
  7776. 3. **Corroboration.** The archive's layer count (parsed from the 3MF) has to
  7777. match the layer count the printer itself reported over MQTT for the print
  7778. that just ended. These come from genuinely different sources, so a
  7779. mismatch means the row is not this print, whatever its name says.
  7780. ``plate_id`` is the plate the job was dispatched with; a reprint of one
  7781. plate of a multi-plate file reuses the archive row without updating its
  7782. ``plate_id``. ``completed`` is accepted alongside ``printing`` only because
  7783. ``on_print_complete`` may already have flipped the status by the time we
  7784. run.
  7785. """
  7786. subtask_name = (data.get("subtask_name") or "").strip()
  7787. if not subtask_name:
  7788. # Nothing to identify the print by — refuse rather than fall back to
  7789. # "whatever ran last on this printer".
  7790. logger.info("[PLATE-RESTORE] printer %s: print has no name to match on — skipping", printer_id)
  7791. return None
  7792. if archive_id is None:
  7793. logger.info(
  7794. "[PLATE-RESTORE] printer %s: %r is not bound to an archive — skipping",
  7795. printer_id,
  7796. subtask_name,
  7797. )
  7798. return None
  7799. try:
  7800. from backend.app.models.archive import PrintArchive
  7801. from backend.app.utils.threemf_tools import extract_end_z_from_3mf, extract_max_z_height_from_3mf
  7802. async with async_session() as db:
  7803. result = await db.execute(
  7804. select(PrintArchive).where(
  7805. PrintArchive.id == archive_id,
  7806. PrintArchive.printer_id == printer_id,
  7807. PrintArchive.status.in_(("printing", "completed")),
  7808. PrintArchive.deleted_at.is_(None),
  7809. )
  7810. )
  7811. archive = result.scalar_one_or_none()
  7812. if archive is None or not archive.file_path:
  7813. logger.info("[PLATE-RESTORE] printer %s: archive %s has no usable file — skipping", printer_id, archive_id)
  7814. return None
  7815. if _normalise_subtask_name(subtask_name) not in _archive_name_forms(archive):
  7816. logger.warning(
  7817. "[PLATE-RESTORE] printer %s: archive %s is not named %r — refusing to move the plate",
  7818. printer_id,
  7819. archive_id,
  7820. subtask_name,
  7821. )
  7822. return None
  7823. client = printer_manager.get_client(printer_id)
  7824. reported_layers = getattr(getattr(client, "state", None), "total_layers", None)
  7825. if reported_layers and archive.total_layers and reported_layers != archive.total_layers:
  7826. logger.warning(
  7827. "[PLATE-RESTORE] printer %s: archive %s says %s layers but the printer reported %s "
  7828. "— refusing to move the plate on a height that may not be this print's",
  7829. printer_id,
  7830. archive.id,
  7831. archive.total_layers,
  7832. reported_layers,
  7833. )
  7834. return None
  7835. # Stored relative to the data directory, like every other archive path
  7836. # (#3240: this read a `data_dir` setting that does not exist).
  7837. path = app_settings.base_dir / archive.file_path
  7838. plate = plate_id or archive.plate_id or 1
  7839. print_top = await asyncio.to_thread(extract_max_z_height_from_3mf, path, plate)
  7840. if print_top is None:
  7841. return None
  7842. end_z = await asyncio.to_thread(extract_end_z_from_3mf, path, plate)
  7843. if end_z is None:
  7844. logger.info(
  7845. "[PLATE-RESTORE] printer %s: archive %s has no end G-code position to park at — skipping",
  7846. printer_id,
  7847. archive_id,
  7848. )
  7849. return None
  7850. return PlateHeights(print_top, end_z)
  7851. except Exception as e:
  7852. logger.debug("[PLATE-RESTORE] printer %s: no usable print height: %s", printer_id, e)
  7853. return None
  7854. async def _restore_plate_for_finish_photo(printer_id: int, max_z_height: float, logger) -> bool:
  7855. """Raise the plate back into camera framing before the finish photo (#2547).
  7856. On X1/P1/A1 Bambu's end G-code drops the plate ~100 mm as the last thing it
  7857. does, so by the time ``gcode_state`` reaches FINISH the finished print sits
  7858. far below the camera's natural framing — the complaint behind #1145, #1397
  7859. and #1565. This commands an absolute ``G1 Z`` back to just above the last
  7860. printed layer. Whether that is needed at all is ``_raise_plate_for_finish_photo``'s
  7861. call: plenty of end G-code (H2C, tall prints anywhere) leaves the plate in
  7862. view already.
  7863. Absolute, not relative, is the whole safety argument. ``max_z_height +
  7864. clearance`` is a height the toolhead was physically at seconds earlier, so
  7865. it is inside the travel limits by construction and leaves the nozzle above
  7866. the part. It is also unambiguous across model families: Z is the
  7867. nozzle-to-bed gap whether the bed moves (X1/P1/H2) or the toolhead does
  7868. (A1), so unlike the relative bed-jog path (#1334) there is no sign to get
  7869. wrong. ``M211`` is never touched — see the bed-jog docstring for why
  7870. (#2579).
  7871. Returns True if the move was sent and waited out, False if it was skipped.
  7872. """
  7873. client = printer_manager.get_client(printer_id)
  7874. if client is None:
  7875. return False
  7876. # Re-read state immediately before commanding motion. If the queue has
  7877. # already started the next print, the printer is no longer ours to move.
  7878. state = getattr(client, "state", None)
  7879. if state is None or state.state != "FINISH":
  7880. logger.info(
  7881. "[PLATE-RESTORE] printer %s is in state %s, not FINISH — skipping",
  7882. printer_id,
  7883. getattr(state, "state", "unknown"),
  7884. )
  7885. return False
  7886. target_z = max_z_height + _PLATE_RESTORE_CLEARANCE_MM
  7887. if not client.send_gcode(f"G90\nG1 Z{target_z:.2f} F{_PLATE_RESTORE_FEEDRATE}"):
  7888. logger.warning("[PLATE-RESTORE] printer %s: send failed — capturing where it is", printer_id)
  7889. return False
  7890. logger.info(
  7891. "[PLATE-RESTORE] printer %s: plate to Z%.2f (print top %.2f + %.1f clearance), settling %.0fs",
  7892. printer_id,
  7893. target_z,
  7894. max_z_height,
  7895. _PLATE_RESTORE_CLEARANCE_MM,
  7896. _PLATE_RESTORE_SETTLE_SECONDS,
  7897. )
  7898. await asyncio.sleep(_PLATE_RESTORE_SETTLE_SECONDS)
  7899. return True
  7900. def _park_plate_after_finish_photo(printer_id: int, end_z: float, logger) -> None:
  7901. """Put the plate back where the end G-code left it, after the finish photo (#2547).
  7902. ``end_z`` is the last Z the print's own end G-code moved to, never a height
  7903. worked out here. That is the safety argument: Bambu's templates keep their
  7904. end moves inside each model's travel, and the firmware does not clamp
  7905. G-code sent over MQTT (#2579). Parking at print top + 100 mm instead drove
  7906. the plate of a 238 mm H2C print to the bottom of its travel, a move Bambu's
  7907. own template had declined to make.
  7908. Fire-and-forget: if it doesn't land, the plate is merely high, and the next
  7909. print homes anyway.
  7910. """
  7911. client = printer_manager.get_client(printer_id)
  7912. state = getattr(client, "state", None) if client else None
  7913. if client is None or state is None or state.state != "FINISH":
  7914. return
  7915. client.send_gcode(f"G90\nG1 Z{end_z:.2f} F{_PLATE_RESTORE_FEEDRATE}")
  7916. logger.info("[PLATE-RESTORE] printer %s: plate back to the end G-code's Z%.2f", printer_id, end_z)
  7917. async def _raise_plate_for_finish_photo(
  7918. printer_id: int, data: dict, logger, archive_id: int | None, plate_id: int | None
  7919. ) -> float | None:
  7920. """Raise the plate into view for the finish photo if it needs it (#2547).
  7921. Returns the Z to park the plate back to once the photo is taken, or None
  7922. when the plate wasn't moved. It needs moving only when the end G-code
  7923. left it lower than the restore height; when it is already at or above it
  7924. (the H2C end G-code puts it back 2 mm under the nozzle), raising would be
  7925. a move away from the camera.
  7926. """
  7927. heights = await _plate_heights_for_current_print(printer_id, data, logger, archive_id, plate_id)
  7928. if heights is None:
  7929. logger.info("[PLATE-RESTORE] printer %s: print height unknown — capturing without restore", printer_id)
  7930. return None
  7931. if heights.end_z <= heights.print_top + _PLATE_RESTORE_CLEARANCE_MM:
  7932. logger.info(
  7933. "[PLATE-RESTORE] printer %s: end G-code left the plate at Z%.2f (print top %.2f) — already in view",
  7934. printer_id,
  7935. heights.end_z,
  7936. heights.print_top,
  7937. )
  7938. return None
  7939. if await _plate_restore_is_blocked_by_queue(printer_id):
  7940. logger.info("[PLATE-RESTORE] printer %s has queued work — skipping plate restore", printer_id)
  7941. return None
  7942. if await _restore_plate_for_finish_photo(printer_id, heights.print_top, logger):
  7943. return heights.end_z
  7944. return None
  7945. async def _plate_restore_is_blocked_by_queue(printer_id: int) -> bool:
  7946. """True if a queue item is about to take this printer (#2547).
  7947. The scheduler dispatches the next job the moment a print completes, and a
  7948. plate move interleaved with a print start is not a race worth having. The
  7949. state re-check in ``_restore_plate_for_finish_photo`` closes the tail of
  7950. this window; this closes the head of it.
  7951. """
  7952. try:
  7953. from backend.app.models.print_queue import PrintQueueItem
  7954. async with async_session() as db:
  7955. result = await db.execute(
  7956. select(PrintQueueItem.id)
  7957. .where(
  7958. PrintQueueItem.printer_id == printer_id,
  7959. PrintQueueItem.status.in_(("pending", "printing")),
  7960. )
  7961. .limit(1)
  7962. )
  7963. return result.scalar_one_or_none() is not None
  7964. except Exception as e:
  7965. # Fail closed: if we can't tell, don't move the plate.
  7966. logging.getLogger(__name__).debug(
  7967. "[PLATE-RESTORE] queue check failed for printer %s: %s — skipping restore", printer_id, e
  7968. )
  7969. return True
  7970. async def on_finish_photo_moment(printer_id: int, data: dict):
  7971. """Pre-capture a finish photo when the printer enters stage 22 / FINISH (#1721).
  7972. Fires either at the stage-22 ("Filament unloading") edge — toolhead
  7973. parked, bed not yet dropped, optimal framing — or as a FINISH-state
  7974. fallback for prints that skip stage 22 (cancel, external-spool-only,
  7975. HMS halt, firmware variants). Grabs one frame via the same
  7976. external-camera / RTSP path the post-completion fallback uses, stores
  7977. the JPEG bytes in ``_stage22_finish_frames[printer_id]``, and lets
  7978. ``_background_finish_photo`` consume the cached bytes when it runs.
  7979. Replaces the #1397 "force timelapse on at dispatch" mechanism, which
  7980. caused per-layer nozzle parking on slicer profiles with Timelapse Type
  7981. set to Smooth (#1721). No force-on now means the user's explicit
  7982. timelapse=off in the slicer send dialog is respected.
  7983. """
  7984. logger = logging.getLogger(__name__)
  7985. trigger = data.get("trigger", "unknown")
  7986. timelapse_was_active = bool(data.get("timelapse_was_active"))
  7987. logger.info(
  7988. "[FINISH-PHOTO-MOMENT] printer=%s trigger=%s timelapse_active=%s",
  7989. printer_id,
  7990. trigger,
  7991. timelapse_was_active,
  7992. )
  7993. # If a timelapse is actively recording, skip the pre-capture — the
  7994. # post-completion path will extract the last frame from the recorded
  7995. # video, which still provides the best framing (toolhead parked,
  7996. # before bed drop) without the per-layer parking side effects.
  7997. if timelapse_was_active:
  7998. logger.info(
  7999. "[FINISH-PHOTO-MOMENT] timelapse active for printer %s — skipping pre-capture (last-frame extraction will run post-completion)",
  8000. printer_id,
  8001. )
  8002. return
  8003. # #1790: register the producer-done event BEFORE the first await so the
  8004. # consumer in `_background_finish_photo` — which is dispatched back-to-back
  8005. # with us on the FINISH-state fallback path — sees it as soon as it polls.
  8006. # The `finally` below guarantees `set()` runs on every exit, including
  8007. # early returns and exceptions, so the consumer's bounded wait can't hang.
  8008. producer_done = asyncio.Event()
  8009. _stage22_finish_in_flight[printer_id] = producer_done
  8010. # #3240: also before the first await. On the FINISH-state path
  8011. # `on_print_complete` is dispatched right behind us and pops both the
  8012. # print's archive binding and its plate; read here, they are still there.
  8013. bound_archive_id, bound_plate_id = _bound_print(printer_id, data)
  8014. # #2547: the Z to park the plate back to, set once the plate has actually
  8015. # been raised, and read by the `finally` below. Declared out here so a
  8016. # failure anywhere after the move — a camera timeout, a DB error — still
  8017. # lowers the plate again.
  8018. park_z: float | None = None
  8019. try:
  8020. async with async_session() as db:
  8021. from backend.app.api.routes.settings import get_setting
  8022. from backend.app.models.printer import Printer
  8023. capture_setting = await get_setting(db, "capture_finish_photo")
  8024. if capture_setting is not None and capture_setting.lower() != "true":
  8025. logger.info("[FINISH-PHOTO-MOMENT] capture_finish_photo disabled — skipping pre-capture")
  8026. return
  8027. restore_setting = await get_setting(db, "finish_photo_restore_plate")
  8028. restore_plate_enabled = restore_setting is None or restore_setting.lower() == "true"
  8029. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8030. printer = result.scalar_one_or_none()
  8031. if printer is None:
  8032. logger.warning(
  8033. "[FINISH-PHOTO-MOMENT] printer %s not found in DB",
  8034. printer_id,
  8035. )
  8036. return
  8037. frame_bytes: bytes | None = None
  8038. # #2708: the banked frame arrives already rotated — it comes from
  8039. # `_capture_snapshot_for_notification`, which rotates before returning.
  8040. # Every other source below is a raw grab. Tracking which lets us store
  8041. # exactly one rotation in `_stage22_finish_frames` either way.
  8042. frame_already_rotated = False
  8043. # On the FINISH-state path the End G-code has already run, and two very
  8044. # different situations arrive here needing opposite answers.
  8045. #
  8046. # #1867: if Bambuddy injected End G-code into this print, a SwapMod
  8047. # snippet may have ejected the plate — the scene in front of the camera
  8048. # is no longer the finished print, and no amount of moving the plate
  8049. # brings it back. Use the banked in-print frame instead.
  8050. #
  8051. # #2547: otherwise the print is still sitting there — on X1/P1/A1 ~100 mm
  8052. # lower than the camera frames well — and the toolhead is parked out of
  8053. # the way. That is the *best* moment available on firmware that never emits
  8054. # stage 22 (H2C, A1 Mini) — so capture live, after putting the plate
  8055. # back. Preferring the bank here unconditionally, as this code used to,
  8056. # is what shipped a mid-print photo with the toolhead over the part.
  8057. if trigger == "finish_state" and print_dispatch_context.end_gcode_injected(printer_id):
  8058. banked = _inprint_frame_bank.get(printer_id)
  8059. if banked:
  8060. frame_bytes = banked
  8061. frame_already_rotated = True
  8062. logger.info(
  8063. "[FINISH-PHOTO-MOMENT] End G-code was injected — using banked in-print "
  8064. "frame (%d bytes) instead of a post-swap live grab",
  8065. len(banked),
  8066. )
  8067. else:
  8068. logger.warning(
  8069. "[FINISH-PHOTO-MOMENT] End G-code was injected for printer %s but the "
  8070. "in-print bank is empty — falling back to a live grab, which may show a "
  8071. "swapped or empty plate",
  8072. printer_id,
  8073. )
  8074. # `park_z` is set only once the plate is actually up, because the
  8075. # `finally` reads it to decide whether it owes a move back down.
  8076. #
  8077. # Never on a print whose End G-code Bambuddy injected, even when the bank
  8078. # came up empty above: that machine may have just ejected its plate, and
  8079. # driving Z into whatever a swap mechanism is doing is not a risk worth
  8080. # taking for a photo of a bed we already know may be bare.
  8081. if (
  8082. frame_bytes is None
  8083. and trigger == "finish_state"
  8084. and restore_plate_enabled
  8085. and not print_dispatch_context.end_gcode_injected(printer_id)
  8086. ):
  8087. park_z = await _raise_plate_for_finish_photo(printer_id, data, logger, bound_archive_id, bound_plate_id)
  8088. # Chamber light for the live capture below (#1655). The banked frame
  8089. # above was taken without it, like every in-print bank.
  8090. async with camera_light(printer if frame_bytes is None else None):
  8091. if frame_bytes is None and printer.external_camera_enabled and printer.external_camera_url:
  8092. from backend.app.api.routes.camera import live_frame_for_capture
  8093. from backend.app.services.external_camera import capture_frame
  8094. # #2707: this used to collide with the live view and fail, which is
  8095. # how finish-photo notifications went out with no image attached.
  8096. # Leaving frame_bytes None keeps the rest of the fallback chain.
  8097. defer, buffered = live_frame_for_capture(printer_id)
  8098. if defer:
  8099. frame_bytes = buffered
  8100. else:
  8101. frame_bytes = await capture_frame(
  8102. printer.external_camera_url,
  8103. printer.external_camera_type or "mjpeg",
  8104. snapshot_url=printer.external_camera_snapshot_url,
  8105. )
  8106. if frame_bytes:
  8107. logger.info(
  8108. "[FINISH-PHOTO-MOMENT] captured external-camera frame (%d bytes)",
  8109. len(frame_bytes),
  8110. )
  8111. elif frame_bytes is None:
  8112. from backend.app.api.routes.camera import get_buffered_frame
  8113. buffered = get_buffered_frame(printer_id)
  8114. if buffered:
  8115. frame_bytes = buffered
  8116. logger.info(
  8117. "[FINISH-PHOTO-MOMENT] used buffered RTSP frame (%d bytes)",
  8118. len(frame_bytes),
  8119. )
  8120. else:
  8121. from backend.app.services.camera import capture_camera_frame_bytes
  8122. frame_bytes = await capture_camera_frame_bytes(
  8123. ip_address=printer.ip_address,
  8124. access_code=printer.access_code,
  8125. model=printer.model,
  8126. timeout=15,
  8127. )
  8128. if frame_bytes:
  8129. logger.info(
  8130. "[FINISH-PHOTO-MOMENT] captured RTSP frame (%d bytes)",
  8131. len(frame_bytes),
  8132. )
  8133. if frame_bytes:
  8134. if not frame_already_rotated:
  8135. frame_bytes = _apply_camera_rotation(frame_bytes, printer, logger)
  8136. _stage22_finish_frames[printer_id] = frame_bytes
  8137. else:
  8138. logger.warning(
  8139. "[FINISH-PHOTO-MOMENT] no frame captured for printer %s — post-completion fallback will retry",
  8140. printer_id,
  8141. )
  8142. except Exception as e:
  8143. logger.warning(
  8144. "[FINISH-PHOTO-MOMENT] pre-capture failed for printer %s: %s",
  8145. printer_id,
  8146. e,
  8147. )
  8148. finally:
  8149. # #2547: we raised the plate, so we own lowering it — including when the
  8150. # capture above failed or threw partway through.
  8151. if park_z is not None:
  8152. try:
  8153. _park_plate_after_finish_photo(printer_id, park_z, logger)
  8154. except Exception as e:
  8155. logger.warning("[PLATE-RESTORE] printer %s: could not lower plate: %s", printer_id, e)
  8156. # #1790: always unblock the consumer's bounded wait — whether we stored
  8157. # a frame, gave up, or hit an exception. Local ref means cleanup of the
  8158. # dict entry by the consumer doesn't affect signalling.
  8159. producer_done.set()
  8160. def _subtask_name_from_filename(filename: str) -> str:
  8161. """Recover the subtask name a print command would have carried for *filename*.
  8162. The dispatcher derives the printer-facing subtask name from the archive's
  8163. file name, so stripping the extensions back off gives the value MQTT echoes
  8164. on completion. Only the two extensions Bambuddy actually stores are removed,
  8165. and in the order they nest (``.gcode.3mf``), so a model whose own name
  8166. contains a dot -- ``My.Model.3mf`` -- keeps it.
  8167. """
  8168. name = PurePosixPath(filename).name
  8169. for suffix in (".3mf", ".gcode"):
  8170. if name.lower().endswith(suffix):
  8171. name = name[: -len(suffix)]
  8172. return name
  8173. def _subtask_is_the_file(subtask_name: str, filename: str) -> bool:
  8174. """Whether the subtask name is the printed 3MF's own file name.
  8175. A print started from the printer's screen reports the same full name, with
  8176. its extension, as both subtask and file (#3009). A dispatched print reports
  8177. a bare subtask name; one that merely ends in ".3mf" can be a model named
  8178. that, whose file is ``Foo.3mf.gcode.3mf``, so it does not count.
  8179. """
  8180. if not subtask_name or not subtask_name.lower().endswith(".3mf"):
  8181. return False
  8182. return PurePosixPath((filename or "").split("://", 1)[-1]).name == subtask_name
  8183. # How the printer marks a subtask name it had to cut short. Observed on real
  8184. # hardware at ~100 characters, but the cut-off is not a fixed character count
  8185. # (a name with multibyte characters came back at 98), so match the marker
  8186. # rather than a length.
  8187. _SUBTASK_TRUNCATION_MARKER = "..."
  8188. def _normalise_subtask_name(name: str) -> str:
  8189. """Canonical form for comparing a dispatched name against MQTT's echo.
  8190. The printer does not echo the name back verbatim: it substitutes
  8191. underscores for spaces. ``H2D_Carbon_Filter_(V2)_Body & Solid Lid`` is
  8192. dispatched and ``H2D_Carbon_Filter_(V2)_Body_&_Solid_Lid`` comes back.
  8193. The 3MF lookup in this module has always known that -- it builds
  8194. space-to-underscore variants of every candidate filename, and its
  8195. directory search normalises both sides before comparing. This exists so
  8196. the completion check reads the same rule from the same place instead of
  8197. growing its own, which is exactly how it came to disagree (#2829).
  8198. A space at either end is substituted too: ``Part .gcode.3mf`` is
  8199. dispatched as ``Part `` and echoed as ``Part_``. Underscores at the ends
  8200. are dropped after the substitution so both sides agree (#3241).
  8201. """
  8202. return name.strip().replace(" ", "_").strip("_").casefold()
  8203. def _3mf_files_for_print(paths: list[str], search_term: str) -> list[str]:
  8204. """The 3MF files of the printer's folder listings that are this print's.
  8205. A file of the print's own name wins, in whichever folder. Only without one
  8206. is a file that merely contains the name taken, and then only when it is
  8207. the one such file: "Cube_large.3mf" is not "Cube"'s file, and of several
  8208. candidates none can be told apart. Paths keep the listings' order.
  8209. """
  8210. threemf = [path for path in paths if path.lower().endswith(".3mf")]
  8211. exact = [path for path in threemf if _subtask_names_match(_subtask_name_from_filename(path), search_term)]
  8212. if exact:
  8213. return exact
  8214. term = _normalise_subtask_name(search_term)
  8215. partial = [path for path in threemf if term in _normalise_subtask_name(PurePosixPath(path).name)]
  8216. if len(partial) > 1:
  8217. logging.getLogger(__name__).info(
  8218. "Several 3MF files contain %r and none is named for it: %s", search_term, partial
  8219. )
  8220. return []
  8221. return partial
  8222. def _subtask_names_match(expected: str, observed: str) -> bool:
  8223. """Whether two subtask names describe the same print.
  8224. Beyond the space/underscore substitution, the printer truncates long names
  8225. and marks the cut with ``...``. A truncated echo has to count as a match or
  8226. every print with a long name strands its queue item the same way.
  8227. """
  8228. expected_n = _normalise_subtask_name(expected)
  8229. observed_n = _normalise_subtask_name(observed)
  8230. if expected_n == observed_n:
  8231. return True
  8232. # Either side can be the truncated one: the printer truncates what it
  8233. # echoes, and an archive whose own filename was recorded from a previous
  8234. # truncated echo carries the marker too.
  8235. for full, cut in ((expected_n, observed_n), (observed_n, expected_n)):
  8236. if not cut.endswith(_SUBTASK_TRUNCATION_MARKER):
  8237. continue
  8238. # The full side has lost its edge underscores; a cut that lands right
  8239. # after one keeps it, so drop it here too (#3241). Nothing left before
  8240. # the marker is no evidence of anything, and must not match every name.
  8241. kept = cut[: -len(_SUBTASK_TRUNCATION_MARKER)].rstrip("_")
  8242. if kept and full.startswith(kept):
  8243. return True
  8244. return False
  8245. def _dispatched_name_matches_echo(expected: str, observed: str) -> bool:
  8246. """Whether the printer's subtask echo is the name a print was dispatched as.
  8247. A dispatched print is echoed under its bare name at start, but a printer
  8248. that picks a print back up -- Bambuddy reconnecting to a print already
  8249. running, or an A1 mini resuming after power-loss recovery -- can report the
  8250. file name with its extension instead (#3485). The echo is tried both as it
  8251. came and with the extensions stripped; the raw form still has to count, so
  8252. a model named ``Foo.3mf`` (file ``Foo.3mf.gcode.3mf``) keeps matching.
  8253. """
  8254. return _subtask_names_match(expected, observed) or _subtask_names_match(
  8255. expected, _subtask_name_from_filename(observed)
  8256. )
  8257. async def _queue_item_dispatched_name(db, item) -> str:
  8258. """The subtask name *item* was dispatched under, or "" when unknowable.
  8259. A row with no archive, or an archive with no file name, is unverifiable
  8260. rather than wrong; every caller answers that with its permissive branch.
  8261. """
  8262. if item.archive_id is None:
  8263. return ""
  8264. from backend.app.models.archive import PrintArchive
  8265. archive = await db.get(PrintArchive, item.archive_id)
  8266. if archive is None or not archive.filename:
  8267. return ""
  8268. return _subtask_name_from_filename(archive.filename)
  8269. async def _completion_belongs_to_queue_item(db, item, data: dict) -> bool:
  8270. """Whether this completion event is plausibly about *item*'s print.
  8271. The caller finds its queue row by printer and ``status='printing'`` alone,
  8272. which is all a completion event gives it -- there is no run identifier in
  8273. the MQTT payload to match on. That makes the lookup indiscriminate: any
  8274. completion delivered for this printer closes whichever row happens to be
  8275. printing, however unrelated. Comparing the subtask name against the archive
  8276. the row was dispatched with costs one primary-key load and rules that out.
  8277. Deliberately permissive: it answers False only on a positive disagreement
  8278. between two names we actually have. A row with no archive, an archive with
  8279. no file name, or an event with no subtask name is unverifiable rather than
  8280. wrong, and refusing those would strand the item in ``printing`` and wedge
  8281. the printer's queue -- a worse failure than the one being prevented.
  8282. """
  8283. observed = (data.get("subtask_name") or "").strip()
  8284. if not observed or item.archive_id is None:
  8285. return True
  8286. from backend.app.models.archive import PrintArchive
  8287. archive = await db.get(PrintArchive, item.archive_id)
  8288. if archive is None or not archive.filename:
  8289. return True
  8290. expected = _subtask_name_from_filename(archive.filename)
  8291. if not expected or _dispatched_name_matches_echo(expected, observed):
  8292. return True
  8293. logging.getLogger(__name__).warning(
  8294. "Ignoring print completion for queue item %s: it was dispatched as %r "
  8295. "(archive %s, %s) but the completion reports subtask %r. Leaving the item "
  8296. "printing rather than closing a run this event is not about.",
  8297. item.id,
  8298. expected,
  8299. archive.id,
  8300. archive.filename,
  8301. observed,
  8302. )
  8303. return False
  8304. async def _recover_fallback_from_cache_before_eviction(printer_id: int, data: dict) -> None:
  8305. """Spend the 3MF download cache on a still-empty fallback archive.
  8306. ``on_print_complete`` drops the cache as its first act, which deletes the
  8307. file. If the cover endpoint (or anything else) pulled the 3MF while the
  8308. print ran and the archive never got one, this is the last moment those bytes
  8309. exist (#2957).
  8310. """
  8311. logger = logging.getLogger(__name__)
  8312. names = [
  8313. n
  8314. for n in (data.get("filename"), data.get("subtask_name"), (data.get("raw_data") or {}).get("subtask_name"))
  8315. if n
  8316. ]
  8317. for name in names:
  8318. try:
  8319. cached = get_cached_3mf(printer_id, name)
  8320. # The print has ended: its completion prices the archive, so no
  8321. # print-start estimate is scheduled to race it.
  8322. if cached and await try_recover_fallback_archive(printer_id, name, cached, price=False):
  8323. return
  8324. except Exception as e:
  8325. logger.debug("[RECOVER] Pre-eviction recovery for %s failed: %s", name, e)
  8326. # Longest the print's end waits for its file to be deleted from the SD card
  8327. _SD_CLEANUP_BUDGET_SECONDS = 45.0
  8328. async def _cleanup_sd_card_after_print(
  8329. printer_id: int, subtask_name: str | None, archive_id: int | None, logger
  8330. ) -> None:
  8331. """Delete a finished print's file from the printer's SD card."""
  8332. # Cleanup: delete uploaded file from printer SD card to prevent phantom prints (Issue #374, #1542)
  8333. # The print scheduler uploads files to the SD card root (/). Some printers (e.g. P1S, A1)
  8334. # auto-start files found in root on power cycle, causing ghost prints.
  8335. # Must run before the archive_id early-return so it executes even when archiving is disabled.
  8336. try:
  8337. if subtask_name:
  8338. archive_filename: str | None = None
  8339. async with async_session() as db:
  8340. from backend.app.models.archive import PrintArchive
  8341. from backend.app.models.printer import Printer
  8342. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8343. printer = result.scalar_one_or_none()
  8344. if archive_id:
  8345. archive_row = await db.execute(select(PrintArchive.filename).where(PrintArchive.id == archive_id))
  8346. archive_filename = archive_row.scalar_one_or_none()
  8347. if printer:
  8348. from backend.app.services.bambu_ftp import DeleteResult, delete_file_async
  8349. from backend.app.utils.filename import derive_remote_filename
  8350. # Primary candidate: the exact path the dispatcher uploaded to
  8351. # (derived from archive.filename via the same rule as upload).
  8352. # Without it, a library row that ended up with a doubled
  8353. # .gcode.3mf (#1542) leaves the real file behind because the
  8354. # subtask_name + ext fallbacks below don't match what's on the
  8355. # SD card. Fallbacks remain for archive-less prints (subtask
  8356. # never resolved to an archive) and for older naming variants.
  8357. candidate_paths: list[str] = []
  8358. if archive_filename:
  8359. candidate_paths.append(f"/{derive_remote_filename(archive_filename)}")
  8360. for ext in (".3mf", ".gcode"):
  8361. fallback = f"/{subtask_name}{ext}"
  8362. if fallback not in candidate_paths:
  8363. candidate_paths.append(fallback)
  8364. # Three outcomes track across all candidates so the final log
  8365. # line reflects what actually happened. The A1 in #1721 always
  8366. # ends here with ``any_not_found=True`` and the others False
  8367. # — its firmware auto-cleans the SD card before our cleanup
  8368. # runs, every candidate FTP-DELE returns 550, and the old
  8369. # code burned 3 retries × 2 s × 3 candidates per print
  8370. # logging a misleading "may linger" WARNING on a successful
  8371. # print.
  8372. any_deleted = False
  8373. any_real_failure = False
  8374. any_not_found = False
  8375. files_in_use = _sd_files_in_use(printer_manager.get_status(printer_id))
  8376. for remote_path in candidate_paths:
  8377. if PurePosixPath(remote_path).name.lower() in files_in_use:
  8378. logger.info(
  8379. "SD card cleanup: keeping %s on printer %s, which is printing it now",
  8380. remote_path,
  8381. printer.name,
  8382. )
  8383. continue
  8384. # Retry only the FAILED case — 550 NOT_FOUND will never
  8385. # recover by waiting, so a "file isn't here" answer
  8386. # advances immediately to the next candidate without
  8387. # consuming the retry budget.
  8388. for attempt in range(1, 4):
  8389. try:
  8390. delete_result = await delete_file_async(
  8391. printer.ip_address,
  8392. printer.access_code,
  8393. remote_path,
  8394. printer_model=printer.model,
  8395. )
  8396. except Exception as e:
  8397. delete_result = DeleteResult.FAILED
  8398. logger.warning(
  8399. "SD card cleanup attempt %d/3 raised for %s: %s",
  8400. attempt,
  8401. remote_path,
  8402. e,
  8403. )
  8404. if delete_result == DeleteResult.DELETED:
  8405. any_deleted = True
  8406. logger.info("Deleted %s from printer %s SD card", remote_path, printer.name)
  8407. break
  8408. if delete_result == DeleteResult.NOT_FOUND:
  8409. any_not_found = True
  8410. break # 550 will not recover; try next candidate
  8411. # FAILED: real error — retry with backoff, then give up
  8412. if attempt < 3:
  8413. await asyncio.sleep(2)
  8414. else:
  8415. any_real_failure = True
  8416. logger.warning(
  8417. "SD card cleanup failed after 3 attempts for %s "
  8418. "(network/auth/transient error — file may linger on SD card)",
  8419. remote_path,
  8420. )
  8421. if not any_deleted and not any_real_failure and any_not_found:
  8422. # Every candidate said "not here." Either the printer
  8423. # firmware swept the SD card itself (common on A1) or the
  8424. # dispatcher's upload path doesn't match our candidate
  8425. # rule. Either way: nothing to clean up, no warning.
  8426. logger.debug(
  8427. "SD card cleanup: nothing to delete on %s — every candidate returned 550 "
  8428. "(printer likely self-cleaned)",
  8429. printer.name,
  8430. )
  8431. except Exception as e:
  8432. logger.warning("SD card file cleanup failed for printer %s: %s", printer_id, e)
  8433. async def on_print_complete(printer_id: int, data: dict):
  8434. """Handle print completion - update the archive status.
  8435. The printer counts as printing until the print's usage is booked (or this
  8436. ends), so the AMS remain% weight sync can't charge it a second time while
  8437. the usage tracker works (WU-3). Marked before the first await.
  8438. """
  8439. from backend.app.services.usage_tracker import done_completing, mark_completing
  8440. mark_completing(printer_id)
  8441. try:
  8442. await _close_out_print(printer_id, data)
  8443. finally:
  8444. done_completing(printer_id)
  8445. async def _close_out_print(printer_id: int, data: dict):
  8446. import time
  8447. logger = logging.getLogger(__name__)
  8448. start_time = time.time()
  8449. def log_timing(section: str):
  8450. elapsed = time.time() - start_time
  8451. logger.info("[TIMING] %s: %.3fs elapsed", section, elapsed)
  8452. logger.info("[CALLBACK] on_print_complete started for printer %s", printer_id)
  8453. octoeverywhere_detection_service.reset_printer(printer_id)
  8454. # A kill-switch stop sends its provider notification immediately. Keep the
  8455. # task so the later notification path can await it and avoid a duplicate;
  8456. # if that immediate attempt failed, the regular completion path retries.
  8457. kill_switch_notification_task = _kill_switch_notification_tasks.pop(printer_id, None)
  8458. # Last chance before the bytes go: if this print's archive is still an empty
  8459. # fallback and something downloaded the 3MF while it ran, fill the archive in
  8460. # now. The cover endpoint's copy lives in exactly this cache, and clearing it
  8461. # below deletes the file (#2957).
  8462. await _recover_fallback_from_cache_before_eviction(printer_id, data)
  8463. # A pending cool-off retry has nothing left to recover for — the cache is
  8464. # about to be dropped and the print is over.
  8465. retry_task = _fallback_3mf_retry_tasks.pop(printer_id, None)
  8466. if retry_task and not retry_task.done():
  8467. retry_task.cancel()
  8468. # Drop the 3MF download cache for this printer (#972). The print is over,
  8469. # nothing else legitimately needs the bytes; keeping them would only risk
  8470. # handing a stale file to the next print if it reuses the same name.
  8471. clear_3mf_cache(printer_id)
  8472. try:
  8473. ws_data = {
  8474. "status": data.get("status"),
  8475. "filename": data.get("filename"),
  8476. "subtask_name": data.get("subtask_name"),
  8477. "timelapse_was_active": data.get("timelapse_was_active"),
  8478. }
  8479. await ws_manager.send_print_complete(printer_id, ws_data)
  8480. log_timing("WebSocket send_print_complete")
  8481. except Exception as e:
  8482. logger.warning("[CALLBACK] WebSocket send_print_complete failed: %s", e)
  8483. # Capture user info before clearing (needed for print log entry)
  8484. _print_user_info = printer_manager.get_current_print_user(printer_id)
  8485. # Clear current print user tracking (Issue #206)
  8486. printer_manager.clear_current_print_user(printer_id)
  8487. # If the user explicitly stopped this print from the queue UI the printer will
  8488. # report "failed" or "aborted" via MQTT. Override that to "cancelled" so the
  8489. # correct "print stopped" notification/email is sent instead of a failure alert.
  8490. _raw_status = data.get("status", "completed")
  8491. if printer_id in _user_stopped_printers and _raw_status in ("failed", "aborted"):
  8492. logger.info(
  8493. "[CALLBACK] Overriding status '%s' -> 'cancelled' for printer %s (print was stopped from queue by user)",
  8494. _raw_status,
  8495. printer_id,
  8496. )
  8497. data = {**data, "status": "cancelled"}
  8498. elif printer_id in _kill_switch_stopped and _raw_status in ("failed", "aborted"):
  8499. logger.info(
  8500. "[CALLBACK] Overriding status '%s' -> 'cancelled' for printer %s (stopped by the printer kill switch)",
  8501. _raw_status,
  8502. printer_id,
  8503. )
  8504. data = {**data, "status": "cancelled", "_kill_switch": True}
  8505. _user_stopped_printers.discard(printer_id)
  8506. _kill_switch_stopped.discard(printer_id)
  8507. # Raise the plate-clear gate for queued dispatch (#961). Any terminal status
  8508. # may have left material on the bed: a user can cancel ten hours into a
  8509. # twelve-hour print, a printer can self-abort mid-job after a clog, and a
  8510. # touchscreen-stop reports `aborted` rather than `cancelled` because
  8511. # `_user_stopped_printers` is only populated when the user stops via the
  8512. # Bambuddy queue UI. Earlier code raised the flag only for completed/failed,
  8513. # which auto-dispatched the next queued print onto a fouled bed two seconds
  8514. # after a touchscreen-abort (#1171). Persisted to DB so the gate survives
  8515. # Auto Off power cycles and Bambuddy restarts.
  8516. try:
  8517. notify_live_activities.print_finished(printer_id, data)
  8518. except Exception:
  8519. logger.exception("Notify Live Activity completion hook failed for printer %s", printer_id)
  8520. _final_status = data.get("status", "completed")
  8521. if _final_status in ("completed", "failed", "aborted", "cancelled"):
  8522. printer_manager.set_awaiting_plate_clear(printer_id, True)
  8523. # MQTT relay - publish print complete
  8524. try:
  8525. printer_info = printer_manager.get_printer(printer_id)
  8526. if printer_info:
  8527. await mqtt_relay.on_print_complete(
  8528. printer_id,
  8529. printer_info.name,
  8530. printer_info.serial_number,
  8531. data.get("filename", ""),
  8532. data.get("subtask_name", ""),
  8533. data.get("status", "completed"),
  8534. )
  8535. except Exception:
  8536. pass # Don't fail print complete callback if MQTT fails
  8537. filename = data.get("filename", "")
  8538. subtask_name = data.get("subtask_name", "")
  8539. if not filename and not subtask_name:
  8540. # Which print ended can't be told: the printer reported no name since
  8541. # Bambuddy started (a print that failed while preparing, a calibration).
  8542. # Charge nothing, but close the print session, which otherwise kept the
  8543. # AMS weight sync off until the next print (AS-5). A print archived as
  8544. # running is closed and charged by the reconnect reconcile.
  8545. logger.warning("Print complete without filename or subtask_name")
  8546. try:
  8547. from backend.app.services.usage_tracker import discard_session
  8548. async with async_session() as db:
  8549. await discard_session(db, printer_id)
  8550. except Exception as e:
  8551. logger.warning("Failed to clear print session for printer %s: %s", printer_id, e)
  8552. return
  8553. logger.info("Print complete - filename: %s, subtask: %s, status: %s", filename, subtask_name, data.get("status"))
  8554. # Build list of possible keys to try (matching how they were registered in on_print_start)
  8555. possible_keys = _completion_print_keys(printer_id, filename, subtask_name)
  8556. # Find the archive for this print
  8557. logger.info("Looking for archive in _active_prints, keys to try: %s...", possible_keys[:5])
  8558. logger.info("Current _active_prints: %s", list(_active_prints.keys()))
  8559. archive_id = None
  8560. for key in possible_keys:
  8561. archive_id = _active_prints.pop(key, None)
  8562. if archive_id:
  8563. _owned_print_archive_ids.discard(archive_id)
  8564. logger.info("Found archive %s with key %s", archive_id, key)
  8565. # Also clean up the printer's other keys pointing to this archive;
  8566. # another printer's are a plate of it still printing there.
  8567. keys_to_remove = [k for k, v in _active_prints.items() if v == archive_id and k[0] == printer_id]
  8568. for k in keys_to_remove:
  8569. _active_prints.pop(k, None)
  8570. break
  8571. # The archive bound when the print started, before the name search below.
  8572. # Only this one may give the finish-photo plate restore a height (#3240):
  8573. # the search matches names by substring.
  8574. bound_archive_id = archive_id
  8575. if not archive_id and data.get("_reconciled") and data.get("_reconciled_archive_id"):
  8576. # The reconcile names the archive whose run on this printer it
  8577. # closes. Only while that run is open: the print's real end may have
  8578. # closed it meanwhile, or closed the print without managing to write
  8579. # it, and either way there is nothing left to end twice.
  8580. if await _archive_of_open_run(printer_id) != data["_reconciled_archive_id"]:
  8581. logger.info(
  8582. "[RECONCILE] Archive %s has no run open on printer %s any more; nothing to close",
  8583. data["_reconciled_archive_id"],
  8584. printer_id,
  8585. )
  8586. return
  8587. archive_id = data["_reconciled_archive_id"]
  8588. ending_job = str(data.get("subtask_id") or "").strip()
  8589. if not archive_id and ending_job not in ("", "0"):
  8590. # Not bound (a restart since the start): this job's run, open on the
  8591. # printer. Only by its job id: a run open there can be of the print
  8592. # running now, while the end is of an earlier one closed late (the
  8593. # restart's reconcile of a missed end).
  8594. archive_id = await _archive_of_open_run(printer_id, subtask_id=ending_job)
  8595. if archive_id:
  8596. logger.info("Found archive %s by the run of job %s open on printer %s", archive_id, ending_job, printer_id)
  8597. if not archive_id:
  8598. # Try to find by filename or subtask_name if not tracked (for prints started before app)
  8599. async with async_session() as db:
  8600. from backend.app.models.archive import PrintArchive
  8601. # Try matching by subtask_name (stored as print_name) first. The
  8602. # same name rule as the queue's completion check: a name inside
  8603. # another ("Clip" in "Cable Clip") is a different print, and the
  8604. # printer's underscores for spaces and truncation still match.
  8605. if subtask_name:
  8606. printing = await db.scalars(
  8607. select(PrintArchive)
  8608. .where(PrintArchive.printer_id == printer_id)
  8609. .where(PrintArchive.status == "printing")
  8610. .order_by(PrintArchive.created_at.desc())
  8611. )
  8612. # A print started from the printer's screen reports its file
  8613. # name, extensions included, as the subtask (#3009).
  8614. reported = {subtask_name, _subtask_name_from_filename(subtask_name)}
  8615. archive = next(
  8616. (
  8617. candidate
  8618. for candidate in printing
  8619. if any(
  8620. (candidate.print_name and _subtask_names_match(candidate.print_name, name))
  8621. or (
  8622. candidate.filename
  8623. and _subtask_names_match(_subtask_name_from_filename(candidate.filename), name)
  8624. )
  8625. for name in reported
  8626. )
  8627. ),
  8628. None,
  8629. )
  8630. if archive:
  8631. archive_id = archive.id
  8632. logger.info("Found archive %s by subtask_name match: %s", archive_id, subtask_name)
  8633. # Also try by filename
  8634. if not archive_id and filename:
  8635. result = await db.execute(
  8636. select(PrintArchive)
  8637. .where(PrintArchive.printer_id == printer_id)
  8638. .where(PrintArchive.filename == filename)
  8639. .where(PrintArchive.status == "printing")
  8640. .order_by(PrintArchive.created_at.desc())
  8641. .limit(1)
  8642. )
  8643. archive = result.scalar_one_or_none()
  8644. if archive:
  8645. archive_id = archive.id
  8646. # Bounded: a printer whose FTP hangs on DELE took up to 3 candidates x 3
  8647. # attempts x a 60 s timeout, about nine minutes, before the queue item was
  8648. # closed below. The stranded-job closer gives up after five and closed it
  8649. # first, so the job's auto-off and its billing never ran. Inline rather
  8650. # than in the background, so the next job of the same file isn't released
  8651. # (and uploaded under the name being deleted) while the cleanup still
  8652. # works on it. A delete already sent may still finish in its FTP thread
  8653. # after the cutoff, as one past delete_file_async's own timeout always did.
  8654. try:
  8655. await asyncio.wait_for(
  8656. _cleanup_sd_card_after_print(printer_id, subtask_name, archive_id, logger),
  8657. timeout=_SD_CLEANUP_BUDGET_SECONDS,
  8658. )
  8659. except TimeoutError:
  8660. logger.warning(
  8661. "SD card cleanup on printer %s took over %ss; leaving the file there and closing the print",
  8662. printer_id,
  8663. _SD_CLEANUP_BUDGET_SECONDS,
  8664. )
  8665. log_timing("SD card cleanup")
  8666. # Update queue item status early — must run before the archive_id early-return
  8667. # so queue items don't get stuck in "printing" when archive lookup fails.
  8668. # Uses run_with_retry to handle SQLite "database is locked" errors (#897).
  8669. queue_item_id = None
  8670. billing_run_id: str | None = None
  8671. billing_user_id: int | None = None
  8672. billing_cost_center_id: int | None = None
  8673. billing_plate_id: int | None = None
  8674. queue_status = None
  8675. queue_auto_off = False
  8676. try:
  8677. from backend.app.core.database import run_with_retry
  8678. from backend.app.models.print_queue import PrintQueueItem
  8679. async def _update_queue_status(db):
  8680. nonlocal billing_run_id, billing_user_id, billing_cost_center_id, billing_plate_id
  8681. nonlocal queue_item_id, queue_status, queue_auto_off
  8682. result = await db.execute(
  8683. select(PrintQueueItem)
  8684. .where(PrintQueueItem.printer_id == printer_id)
  8685. .where(PrintQueueItem.status == "printing")
  8686. )
  8687. printing_items = list(result.scalars().all())
  8688. if len(printing_items) > 1:
  8689. logger.warning(
  8690. "BUG: Multiple queue items in 'printing' status for printer %s: %s",
  8691. printer_id,
  8692. [(i.id, i.archive_id, i.library_file_id) for i in printing_items],
  8693. )
  8694. item = printing_items[0] if printing_items else None
  8695. if item is not None and not await _completion_belongs_to_queue_item(db, item, data):
  8696. return
  8697. if item:
  8698. queue_status = data.get("status", "completed")
  8699. # MQTT sends "aborted" for cancelled prints; normalise to
  8700. # "cancelled" so it matches the queue schema Literal.
  8701. if queue_status == "aborted":
  8702. queue_status = "cancelled"
  8703. item.status = queue_status
  8704. item.completed_at = datetime.now(timezone.utc)
  8705. if queue_status == "failed" and not item.error_message:
  8706. item.error_message = _format_hms_error_summary(data.get("hms_errors") or [])
  8707. # Bump usage counters on the source library file so admins can
  8708. # sort by "last printed" and (eventually) auto-purge stale
  8709. # files — #1008.
  8710. await _bump_library_file_usage_if_completed(db, item, queue_status)
  8711. await db.commit()
  8712. queue_item_id = item.id
  8713. billing_run_id = item.billing_run_id
  8714. billing_user_id = item.created_by_id
  8715. billing_cost_center_id = item.cost_center_id
  8716. billing_plate_id = item.plate_id
  8717. queue_auto_off = item.auto_off_after
  8718. logger.info("Updated queue item %s status to %s", item.id, queue_status)
  8719. await run_with_retry(_update_queue_status, label="queue status update")
  8720. # Post-commit side effects (notifications, MQTT relay, auto-off) use
  8721. # their own sessions and have their own error handling — no retry needed.
  8722. if queue_item_id is not None:
  8723. # Batch orders (#342): this run may have been the last one an order
  8724. # owed. Re-evaluate here rather than lazily on read, so a finished
  8725. # order reports itself complete without someone opening the page.
  8726. try:
  8727. from backend.app.services.print_batch import refresh_batch_status_for_item
  8728. async with async_session() as db:
  8729. await refresh_batch_status_for_item(db, queue_item_id)
  8730. await db.commit()
  8731. except Exception as e:
  8732. logger.warning("[BATCH] Failed to refresh batch status for queue item %s: %s", queue_item_id, e)
  8733. # MQTT relay - publish queue job completed
  8734. try:
  8735. printer_info = printer_manager.get_printer(printer_id)
  8736. await mqtt_relay.on_queue_job_completed(
  8737. job_id=queue_item_id,
  8738. filename=filename or subtask_name,
  8739. printer_id=printer_id,
  8740. printer_name=printer_info.name if printer_info else "Unknown",
  8741. status=queue_status,
  8742. )
  8743. except Exception:
  8744. pass # Don't fail if MQTT fails
  8745. # Check if queue is now empty and send notification
  8746. try:
  8747. from sqlalchemy import func as sa_func
  8748. async with async_session() as db:
  8749. count_result = await db.execute(
  8750. select(sa_func.count(PrintQueueItem.id)).where(PrintQueueItem.status == "pending")
  8751. )
  8752. pending_count = count_result.scalar() or 0
  8753. if pending_count == 0:
  8754. today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
  8755. completed_result = await db.execute(
  8756. select(sa_func.count(PrintQueueItem.id)).where(
  8757. PrintQueueItem.status.in_(["completed", "failed", "skipped"]),
  8758. PrintQueueItem.completed_at >= today_start,
  8759. )
  8760. )
  8761. completed_count = completed_result.scalar() or 1
  8762. await notification_service.on_queue_completed(
  8763. completed_count=completed_count,
  8764. db=db,
  8765. )
  8766. except Exception:
  8767. pass # Don't fail if notification fails
  8768. # Handle auto_off_after - power off printer if the queue item opted
  8769. # in. Delegates to the smart-plug manager so the off honours each
  8770. # plug's configured strategy (time delay or temperature threshold),
  8771. # is cancelled if the printer starts printing again, and never cuts
  8772. # power on a loaded print (#1890). Previously an inline block here
  8773. # hardcoded a 50°C / 600s cooldown wait and powered off on the
  8774. # timeout regardless of print state — cutting a touchscreen reprint.
  8775. if queue_auto_off:
  8776. try:
  8777. async with async_session() as db:
  8778. await smart_plug_manager.schedule_off_after_queue_job(printer_id, db)
  8779. except Exception as e:
  8780. logger.warning("Failed to schedule queue auto-off for printer %s: %s", printer_id, e)
  8781. except Exception as e:
  8782. logging.getLogger(__name__).warning(f"Queue item update failed: {e}")
  8783. log_timing("Queue item update")
  8784. # Register bed cooldown waiter (event-driven via on_bed_temp_update callback).
  8785. # Must run before archive_id early-return so it fires for all prints (including
  8786. # prints started from BambuStudio/touchscreen that have no archive).
  8787. if data.get("status") == "completed":
  8788. try:
  8789. from backend.app.api.routes.settings import get_setting
  8790. async with async_session() as db:
  8791. threshold_str = await get_setting(db, "bed_cooled_threshold")
  8792. threshold = float(threshold_str) if threshold_str else 35.0
  8793. # Check if any provider has on_bed_cooled enabled (skip registration if none)
  8794. async with async_session() as db:
  8795. providers = await notification_service._get_providers_for_event(db, "on_bed_cooled", printer_id)
  8796. if providers:
  8797. _bed_cool_waiters[printer_id] = {
  8798. "threshold": threshold,
  8799. "filename": filename or subtask_name or "",
  8800. "registered_at": time.time(),
  8801. }
  8802. logger.info(
  8803. "[BED-COOL] Registered waiter for printer %s (threshold: %.0f°C)",
  8804. printer_id,
  8805. threshold,
  8806. )
  8807. else:
  8808. logger.debug("[BED-COOL] No providers enabled for bed_cooled on printer %s", printer_id)
  8809. except Exception as e:
  8810. logger.warning("[BED-COOL] Failed to register waiter: %s", e)
  8811. # Capture the slicer estimate before usage tracking runs. The tracker may
  8812. # update archive.cost with this run's measured cost; billing partial runs
  8813. # against that already-partial value would discount the charge twice.
  8814. billing_planned_grams: float | None = None
  8815. billing_base_cost: float | None = None
  8816. if archive_id:
  8817. try:
  8818. async with async_session() as db:
  8819. from backend.app.models.archive import PrintArchive
  8820. billing_archive = await db.get(PrintArchive, archive_id)
  8821. if billing_archive:
  8822. billing_path = (
  8823. app_settings.base_dir / billing_archive.file_path if billing_archive.file_path else None
  8824. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  8825. billing_planned_grams, billing_base_cost = _plate_scoped_run_estimate(
  8826. billing_archive,
  8827. billing_path,
  8828. billing_plate_id
  8829. if billing_plate_id is not None
  8830. else _get_start_plate_id(printer_id, archive_id),
  8831. )
  8832. except Exception as e:
  8833. logger.warning("[FINANCE] Failed to capture planned usage for archive %s: %s", archive_id, e)
  8834. # --- Track filament consumption (must run before archive_id early-return so usage
  8835. # is recorded even when auto-archive is disabled) ---
  8836. usage_results: list[dict] = []
  8837. # Prefer ams_mapping captured from MQTT request topic (works for all print sources)
  8838. stored_ams_mapping = data.get("ams_mapping")
  8839. # Fallback to _print_ams_mappings for queue/reprint (set before print starts).
  8840. # Drained either way, like the plate below: left over, it would be read by
  8841. # the archive's next run, dispatched without a mapping.
  8842. registered_ams_mapping = _print_ams_mappings.pop((printer_id, archive_id), None) if archive_id else None
  8843. if not stored_ams_mapping:
  8844. stored_ams_mapping = registered_ams_mapping
  8845. # Always drain the plate_id register on completion — the session already
  8846. # consumed it at print-start injection; leaving it would leak into the next
  8847. # print on the same archive_id (rare but possible with reprints) (#1697).
  8848. # Capture the popped value so the completion notification can scope the
  8849. # archive-level (summed-across-plates per #1593) filament + time totals
  8850. # down to the single plate that was actually printed (#1785).
  8851. notify_plate_id: int | None = None
  8852. if archive_id:
  8853. notify_plate_id = _print_plate_ids.pop((printer_id, archive_id), None)
  8854. # The start estimate must not outlive the real cost written below (COST-4).
  8855. if archive_id:
  8856. await _settle_archive_cost_estimate(archive_id)
  8857. # Internal inventory: track AMS remain% deltas (skip if Spoolman handles usage)
  8858. try:
  8859. async with async_session() as db:
  8860. from backend.app.api.routes.settings import get_setting
  8861. _spoolman_on = await get_setting(db, "spoolman_enabled")
  8862. if not _spoolman_on or _spoolman_on.lower() != "true":
  8863. from backend.app.services.usage_tracker import on_print_complete as usage_on_print_complete
  8864. async with async_session() as db:
  8865. usage_results = await usage_on_print_complete(
  8866. printer_id,
  8867. data,
  8868. printer_manager,
  8869. db,
  8870. archive_id=archive_id,
  8871. ams_mapping=stored_ams_mapping,
  8872. )
  8873. if usage_results:
  8874. await ws_manager.broadcast(
  8875. {
  8876. "type": "spool_usage_logged",
  8877. "printer_id": printer_id,
  8878. "usage": usage_results,
  8879. }
  8880. )
  8881. log_timing("Usage tracker")
  8882. except Exception as e:
  8883. logger.warning("Usage tracker on_print_complete failed: %s", e)
  8884. # Drop the print-start context unconditionally — the Spoolman branch above
  8885. # skips the internal tracker entirely, so nothing else would clear what
  8886. # print start captured, and a row surviving its print would be restored
  8887. # onto the next one after a restart.
  8888. try:
  8889. from backend.app.services.usage_tracker import discard_session
  8890. async with async_session() as db:
  8891. await discard_session(db, printer_id)
  8892. except Exception as e:
  8893. logger.warning("Failed to clear persisted print session for printer %s: %s", printer_id, e)
  8894. # Spoolman: report filament usage (requires archive_id for tracking data lookup)
  8895. # A partial run's cost from the Spoolman spools it was charged to; the
  8896. # built-in inventory's comes in usage_results.
  8897. spoolman_run_cost: float | None = None
  8898. if archive_id:
  8899. if data.get("status") == "completed":
  8900. try:
  8901. await _report_spoolman_usage(printer_id, archive_id)
  8902. log_timing("Spoolman usage report")
  8903. except Exception as e:
  8904. logger.warning("Spoolman usage reporting failed: %s", e)
  8905. else:
  8906. # Report partial usage if tracking data exists (only stored when weight sync is disabled)
  8907. try:
  8908. async with async_session() as db:
  8909. spoolman_run_cost = await _cleanup_spoolman_tracking(
  8910. printer_id,
  8911. archive_id,
  8912. db,
  8913. last_layer_num=data.get("last_layer_num"),
  8914. last_progress=data.get("last_progress"),
  8915. status=data.get("status") or "failed",
  8916. )
  8917. except Exception as e:
  8918. logger.debug("[SPOOLMAN] Cleanup failed: %s", e)
  8919. await _release_archived_spoolman_slots(printer_id)
  8920. log_timing("Filament usage tracking")
  8921. if not archive_id:
  8922. # The printer's own calibration run has no archive by design, so this
  8923. # arrives here every time one finishes. Returning before the no-archive
  8924. # notification keeps a calibration from being announced as a finished
  8925. # print. Everything above this point has already run — the plate-clear
  8926. # gate, the queue reconciliation, the SD-card cleanup — so only the
  8927. # notification is skipped.
  8928. if is_internal_printer_job(filename, subtask_name):
  8929. logger.info(
  8930. "[CALLBACK] Internal printer job completed, no notification: filename=%s, subtask=%s",
  8931. filename,
  8932. subtask_name,
  8933. )
  8934. return
  8935. logger.warning("Could not find archive for print complete: filename=%s, subtask=%s", filename, subtask_name)
  8936. # Still send print-complete/failed/stopped notifications even without an archive.
  8937. # Try to enrich with queue/library-file data so user-specific emails work too.
  8938. async def _notify_no_archive():
  8939. try:
  8940. async with async_session() as db:
  8941. from backend.app.models.library import LibraryFile
  8942. from backend.app.models.print_queue import PrintQueueItem
  8943. from backend.app.models.printer import Printer
  8944. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8945. printer_obj = result.scalar_one_or_none()
  8946. p_name = printer_obj.name if printer_obj else f"Printer {printer_id}"
  8947. # Only the queue item this completion itself closed (the
  8948. # reconciliation above, which checks the run belongs to it).
  8949. # A print started outside Bambuddy closed none, and must not
  8950. # borrow an earlier job's owner, email or times.
  8951. no_archive_data: dict | None = None
  8952. try:
  8953. queue_item = await db.get(PrintQueueItem, queue_item_id) if queue_item_id else None
  8954. if queue_item:
  8955. no_archive_data = {"created_by_id": queue_item.created_by_id}
  8956. # How long it really ran, as the archive path reports it (#1198).
  8957. started = to_naive_utc(queue_item.started_at)
  8958. ended = to_naive_utc(queue_item.completed_at)
  8959. # Not for a completion reconciled at reconnect: it
  8960. # ended at an unknown time while the printer was
  8961. # offline, and "now" is the reconnect (DB-C9).
  8962. if (
  8963. not data.get("_reconciled")
  8964. and started
  8965. and ended
  8966. and (ended - started).total_seconds() > 0
  8967. ):
  8968. no_archive_data["actual_time_seconds"] = int((ended - started).total_seconds())
  8969. # Estimated time: the queue item caches it from its
  8970. # archive or library file; otherwise read the library
  8971. # file's metadata (LibraryFile has no column for it).
  8972. if queue_item.print_time_seconds:
  8973. no_archive_data["print_time_seconds"] = queue_item.print_time_seconds
  8974. elif queue_item.library_file_id:
  8975. lib_result = await db.execute(
  8976. select(LibraryFile).where(LibraryFile.id == queue_item.library_file_id)
  8977. )
  8978. lib_file = lib_result.scalar_one_or_none()
  8979. estimate = (
  8980. (lib_file.file_metadata or {}).get("print_time_seconds") if lib_file else None
  8981. )
  8982. if estimate:
  8983. no_archive_data["print_time_seconds"] = estimate
  8984. except Exception as lookup_err:
  8985. logger.debug(
  8986. "[NOTIFY-BG] Could not look up queue item for no-archive notification: %s", lookup_err
  8987. )
  8988. # Enrich with usage tracker results (captured in enclosing scope)
  8989. if usage_results:
  8990. if no_archive_data is None:
  8991. no_archive_data = {}
  8992. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  8993. if total_from_usage > 0:
  8994. no_archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  8995. no_archive_data["usage_results"] = usage_results
  8996. # Try MQTT remaining_time for print duration when no queue/library data
  8997. if no_archive_data and not no_archive_data.get("print_time_seconds"):
  8998. mqtt_remaining = data.get("remaining_time")
  8999. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  9000. no_archive_data["print_time_seconds"] = int(mqtt_remaining)
  9001. ps = data.get("status", "completed")
  9002. logger.info(
  9003. "[NOTIFY-BG] Sending notification without archive: printer=%s, status=%s", printer_id, ps
  9004. )
  9005. if not await _kill_switch_notification_already_sent(kill_switch_notification_task):
  9006. await notification_service.on_print_complete(
  9007. printer_id, p_name, ps, data, db, archive_data=no_archive_data
  9008. )
  9009. else:
  9010. logger.info("[NOTIFY-BG] Skipped duplicate kill-switch provider notification")
  9011. # Send user-specific email if we have a created_by_id
  9012. if no_archive_data and no_archive_data.get("created_by_id"):
  9013. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  9014. await _dispatch_user_print_email(
  9015. ps,
  9016. no_archive_data["created_by_id"],
  9017. p_name,
  9018. raw_filename,
  9019. db,
  9020. )
  9021. logger.info("[NOTIFY-BG] Completed (no-archive path)")
  9022. except Exception as e:
  9023. logger.warning("[NOTIFY-BG] Failed to send notification without archive: %s", e, exc_info=True)
  9024. spawn_background_task(_notify_no_archive(), name="notify-no-archive")
  9025. return
  9026. log_timing("Archive lookup")
  9027. # Update archive status
  9028. logger.info("[ARCHIVE] Updating archive %s status...", archive_id)
  9029. try:
  9030. async with async_session() as db:
  9031. service = ArchiveService(db)
  9032. status = data.get("status", "completed")
  9033. hms_errors = data.get("hms_errors", []) if status == "failed" else None
  9034. if hms_errors:
  9035. logger.info("[ARCHIVE] HMS errors at failure: %s", hms_errors)
  9036. failure_reason = derive_failure_reason(status, hms_errors)
  9037. if data.get("_reconciled") and (status == "aborted" or (status == "failed" and not failure_reason)):
  9038. # A reconciled completion closes out a stale archive at
  9039. # reconnect — it is not a user action, so don't mislabel it
  9040. # "userCancelled". It shares the stale-cleanup path's key
  9041. # (issue #2974) and records that the real end time is unknown,
  9042. # which is also why its logged duration is 0 (#2592). A failed
  9043. # one the printer's errors explain keeps their reason (#3310).
  9044. failure_reason = "noStatusUpdate"
  9045. elif data.get("_kill_switch"):
  9046. # Not the user's cancel: the kill switch stopped a print that did
  9047. # not come from Bambuddy.
  9048. failure_reason = "killSwitch"
  9049. if failure_reason:
  9050. logger.info("[ARCHIVE] failure_reason=%r (status=%s)", failure_reason, status)
  9051. elif status == "failed" and hms_errors:
  9052. logger.info("[ARCHIVE] HMS errors present but none matched a known failure-reason short code")
  9053. await service.update_archive_status(
  9054. archive_id,
  9055. status=status,
  9056. completed_at=(
  9057. # A reconciled completed print ended at an estimated time,
  9058. # not at the reconnect (#3310).
  9059. data.get("_reconciled_completed_at") or datetime.now(timezone.utc)
  9060. if status in ("completed", "failed", "aborted", "cancelled")
  9061. else None
  9062. ),
  9063. failure_reason=failure_reason,
  9064. )
  9065. logger.info(
  9066. "[ARCHIVE] Archive %s status updated to %s, failure_reason=%s", archive_id, status, failure_reason
  9067. )
  9068. await ws_manager.send_archive_updated(
  9069. {
  9070. "id": archive_id,
  9071. "status": status,
  9072. }
  9073. )
  9074. logger.info("[ARCHIVE] WebSocket notification sent for archive %s", archive_id)
  9075. # MQTT relay - publish archive updated
  9076. try:
  9077. await mqtt_relay.on_archive_updated(
  9078. archive_id=archive_id,
  9079. print_name=filename or subtask_name,
  9080. status=status,
  9081. )
  9082. except Exception:
  9083. pass # Don't fail if MQTT fails
  9084. except Exception as e:
  9085. logger.error("[ARCHIVE] Failed to update archive %s status: %s", archive_id, e, exc_info=True)
  9086. # Continue with other operations even if archive update fails
  9087. log_timing("Archive status update")
  9088. # Apply finance wallet charge or release reservations once. For all partial
  9089. # terminal states (failed, aborted at the printer display, or cancelled via
  9090. # Bambuddy) use this run's measured spool delta, falling back to the last
  9091. # valid printer progress. PrintArchive.filament_used_grams is the slicer
  9092. # estimate and therefore cannot represent an interrupted run.
  9093. try:
  9094. if data.get("status") in ("completed", "failed", "aborted", "cancelled"):
  9095. async with async_session() as db:
  9096. from backend.app.models.archive import PrintArchive
  9097. from backend.app.services.finance_billing import apply_print_charge_for_archive
  9098. from backend.app.services.print_log import run_being_ended
  9099. archive = await db.get(PrintArchive, archive_id)
  9100. # The run this end fills in below: charged as its own, not as
  9101. # the archive's latest dispatch (another plate of it, maybe).
  9102. charged_run = await run_being_ended(db, archive_id, printer_id, archive.started_at) if archive else None
  9103. if archive and billing_run_id is None and charged_run is None:
  9104. billing_run_id = getattr(archive, "billing_run_id", None)
  9105. if archive and archive.created_by_id is None and _print_user_info:
  9106. archive.created_by_id = await _print_user_id_if_still_there(db, _print_user_info)
  9107. await db.flush()
  9108. run_status = data.get("status", "completed")
  9109. last_progress = data.get("last_progress")
  9110. if last_progress is None:
  9111. last_progress = data.get("progress")
  9112. actual_run_grams = _compute_run_filament_grams(
  9113. run_status,
  9114. billing_planned_grams,
  9115. last_progress,
  9116. usage_results,
  9117. )
  9118. filament_usage = (actual_run_grams, billing_planned_grams) if run_status != "completed" else None
  9119. in_memory_cost_center_id = _print_cost_center_ids.pop((printer_id, archive_id), None)
  9120. charged = await apply_print_charge_for_archive(
  9121. db,
  9122. archive_id,
  9123. charged_user_id=billing_user_id,
  9124. cost_center_id=(
  9125. billing_cost_center_id if billing_cost_center_id is not None else in_memory_cost_center_id
  9126. ),
  9127. print_queue_id=queue_item_id,
  9128. print_run_id=billing_run_id,
  9129. base_cost_override=billing_base_cost,
  9130. filament_usage=filament_usage,
  9131. run_entry_id=charged_run.id if charged_run is not None else None,
  9132. run_status=run_status,
  9133. )
  9134. await db.commit()
  9135. if charged:
  9136. logger.info("[FINANCE] Applied print charge for archive %s", archive_id)
  9137. except Exception as e:
  9138. logger.warning("[FINANCE] Failed to apply print charge for archive %s: %s", archive_id, e)
  9139. printer_info = printer_manager.get_printer(printer_id)
  9140. billing_printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  9141. billing_filename = filename or subtask_name or "Unknown"
  9142. billing_error = str(e)
  9143. try:
  9144. await ws_manager.broadcast(
  9145. {
  9146. "type": "billing_charge_failed",
  9147. "printer_id": printer_id,
  9148. "printer_name": billing_printer_name,
  9149. "filename": billing_filename,
  9150. "archive_id": archive_id,
  9151. }
  9152. )
  9153. except Exception as notification_error:
  9154. logger.error(
  9155. "[FINANCE] Failed to broadcast billing error for archive %s: %s",
  9156. archive_id,
  9157. notification_error,
  9158. )
  9159. async def _notify_billing_charge_failed() -> None:
  9160. try:
  9161. async with async_session() as notification_db:
  9162. await notification_service.on_billing_charge_failed(
  9163. printer_id,
  9164. billing_printer_name,
  9165. billing_filename,
  9166. archive_id,
  9167. billing_error,
  9168. notification_db,
  9169. )
  9170. except Exception as provider_error:
  9171. logger.error(
  9172. "[FINANCE] Failed to send provider billing alert for archive %s: %s",
  9173. archive_id,
  9174. provider_error,
  9175. exc_info=True,
  9176. )
  9177. spawn_background_task(
  9178. _notify_billing_charge_failed(),
  9179. name=f"billing-charge-failed-{archive_id}",
  9180. )
  9181. log_timing("Finance charge update")
  9182. # Write independent print log entry (separate table, never touches archives)
  9183. # The run's id, for the energy work below: the archive's newest run is
  9184. # the one started last, which with plates of it on two printers need not
  9185. # be the one that just ended.
  9186. completed_run_id: int | None = None
  9187. async def _write_run(db):
  9188. # Retried on a locked SQLite database: a run left open here would be
  9189. # closed again by the next reconnect's reconcile.
  9190. nonlocal completed_run_id
  9191. from backend.app.models.archive import PrintArchive
  9192. from backend.app.services.print_log import discard_open_runs, record_archive_wear, write_log_entry
  9193. archive = await db.get(PrintArchive, archive_id)
  9194. if archive is None:
  9195. # Deleted while it printed: its run has nothing left to end on.
  9196. discarded = await discard_open_runs(db, printer_id)
  9197. await db.commit()
  9198. await _announce_discarded_runs(discarded, printer_id)
  9199. if archive:
  9200. # Back-fill created_by_id on reprint (#730): reprint reuses the
  9201. # source archive row rather than creating a new one, so an
  9202. # archive that was auto-created from a printer-initiated
  9203. # print (created_by_id=NULL) would otherwise stay unattributed
  9204. # forever. When we have a print-session user AND the archive
  9205. # has no attribution yet, credit the current user. Never
  9206. # overwrite an existing attribution — the original uploader
  9207. # keeps ownership.
  9208. _print_user_id = await _print_user_id_if_still_there(db, _print_user_info)
  9209. if archive.created_by_id is None and _print_user_id is not None:
  9210. archive.created_by_id = _print_user_id
  9211. p_info = printer_manager.get_printer(printer_id)
  9212. # Per-run actuals — written to PrintLogEntry so stats reflect
  9213. # what THIS print actually used, not the source archive's
  9214. # first-run values (#1378). Helper handles the partial-print
  9215. # math (failed / cancelled / stopped get scaled to progress
  9216. # or to tracked spool deltas).
  9217. _run_status = data.get("status", "completed")
  9218. # #2614: scope the per-run estimate to the printed plate. For a
  9219. # multi-plate 3MF dispatched one plate at a time, the archive's
  9220. # filament/cost are the whole-file totals; the PrintLogEntry must
  9221. # reflect only this plate. No effect on single-plate archives (the
  9222. # plate estimate equals the whole-file value) or on the tracker
  9223. # path (measured spool deltas win in _compute_run_filament_grams).
  9224. _est_full_path = (
  9225. app_settings.base_dir / archive.file_path if archive.file_path else None
  9226. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  9227. # The plate that printed, as billing reads it: a reprint of
  9228. # another plate must not log the archive's first plate.
  9229. _run_plate_id = billing_plate_id if billing_plate_id is not None else notify_plate_id
  9230. _est_grams, _est_cost = _plate_scoped_run_estimate(archive, _est_full_path, _run_plate_id)
  9231. if _run_plate_id is None:
  9232. _run_plate_id = archive.plate_id
  9233. _run_grams = _compute_run_filament_grams(
  9234. _run_status,
  9235. _est_grams,
  9236. data.get("last_progress", data.get("progress")),
  9237. usage_results,
  9238. )
  9239. # Per-run cost — prefer usage_results sum. For partial prints
  9240. # we deliberately skip the topup-to-estimate logic in
  9241. # usage_tracker (which assumes the print completed); the raw
  9242. # tracked-spool sum is closer to what THIS run actually cost.
  9243. _run_cost: float | None = None
  9244. if usage_results:
  9245. _run_cost = sum(r.get("cost") or 0 for r in usage_results) or None
  9246. if _run_cost is None and _run_status != "completed":
  9247. _run_cost = spoolman_run_cost
  9248. if _run_cost is None and _run_status == "completed":
  9249. _run_cost = _est_cost
  9250. from backend.app.models.printer import Printer as _Printer
  9251. _wear_rate = await db.scalar(select(_Printer.wear_cost_per_hour).where(_Printer.id == printer_id))
  9252. run_entry = await write_log_entry(
  9253. db,
  9254. archive_id=archive.id,
  9255. # Captured by _update_queue_status above; None for
  9256. # printer-initiated prints with no queue row. Batch
  9257. # cost/energy roll-up joins on it (#342).
  9258. queue_item_id=queue_item_id,
  9259. status=_run_status,
  9260. print_name=archive.print_name,
  9261. # The plate this run printed, as the estimate above takes
  9262. # it; the archive's own is the latest run's.
  9263. plate_id=_run_plate_id,
  9264. printer_name=p_info.name if p_info else None,
  9265. printer_id=printer_id,
  9266. started_at=archive.started_at,
  9267. completed_at=archive.completed_at,
  9268. filament_type=archive.filament_type,
  9269. filament_color=archive.filament_color,
  9270. filament_used_grams=_run_grams,
  9271. cost=_run_cost,
  9272. wear_cost_per_hour=_wear_rate,
  9273. failure_reason=archive.failure_reason,
  9274. thumbnail_path=archive.thumbnail_path,
  9275. created_by_id=archive.created_by_id,
  9276. created_by_username=_print_user_info.get("username") if _print_user_info else None,
  9277. # Reconciled completions have an unknown real end time —
  9278. # log 0 duration instead of the whole disconnect gap (#2592),
  9279. # unless it was estimated (#3310).
  9280. reconciled=bool(data.get("_reconciled")) and not data.get("_reconciled_completed_at"),
  9281. # Its time is the estimate or nothing, never measured (#3310).
  9282. duration_unmeasured=bool(data.get("_reconciled")),
  9283. # The real end of a print stopped while its printer was offline.
  9284. replaces_offline_stop=True,
  9285. )
  9286. await record_archive_wear(db, archive, run_entry)
  9287. await db.commit()
  9288. completed_run_id = run_entry.id
  9289. logger.info("[PRINT_LOG] Log entry written for archive %s", archive_id)
  9290. try:
  9291. from backend.app.core.database import run_with_retry
  9292. await run_with_retry(_write_run, label="print log entry")
  9293. except Exception as e:
  9294. logger.warning("[PRINT_LOG] Failed to write log entry for archive %s: %s", archive_id, e)
  9295. await _publish_run_completed(completed_run_id, printer_id)
  9296. log_timing("Print log entry")
  9297. # Run slow operations as background tasks to avoid blocking the event loop
  9298. # These operations can take 5-10+ seconds and would freeze the UI if awaited
  9299. async def _background_energy_calculation():
  9300. """Calculate and save energy usage in background.
  9301. Reads the starting kWh from the archive row (#941: persisted so a mid-print
  9302. backend restart no longer loses per-print energy data).
  9303. """
  9304. try:
  9305. logger.info("[ENERGY-BG] Starting energy calculation for archive %s", archive_id)
  9306. async with async_session() as db:
  9307. from backend.app.models.archive import PrintArchive
  9308. from backend.app.models.print_log import PrintLogEntry
  9309. from backend.app.services.print_log import energy_start_of
  9310. archive = await db.get(PrintArchive, archive_id)
  9311. if archive is None:
  9312. logger.warning("[ENERGY-BG] Archive %s no longer exists", archive_id)
  9313. return
  9314. # The ended run's own start reading: the archive's is its
  9315. # latest start's, maybe another plate's on another printer.
  9316. run_row = await db.get(PrintLogEntry, completed_run_id) if completed_run_id is not None else None
  9317. start = energy_start_of(run_row, archive)
  9318. if start is None:
  9319. logger.info("[ENERGY-BG] No start kWh recorded for archive %s's run", archive_id)
  9320. return
  9321. starting_kwh = start.kwh
  9322. candidates = await energy_plug_candidates(db, printer_id)
  9323. if not candidates:
  9324. logger.info("[ENERGY-BG] No smart plug for printer %s", printer_id)
  9325. return
  9326. # Same ordering as the start reading, so the delta below is
  9327. # against the counter that produced `starting_kwh` (#2859).
  9328. selected = await select_energy_reading(candidates, _get_plug_energy, db)
  9329. if selected is None:
  9330. logger.warning(
  9331. "[ENERGY-BG] No plug on printer %s reports a lifetime energy counter (tried: %s)",
  9332. printer_id,
  9333. ", ".join(plug.name for plug in candidates),
  9334. )
  9335. return
  9336. plug, energy = selected
  9337. logger.info("[ENERGY-BG] Energy response from plug '%s': %s", plug.name, energy)
  9338. energy_used = round(energy["total"] - starting_kwh, 4)
  9339. logger.info("[ENERGY-BG] Per-print energy: %s kWh", energy_used)
  9340. if energy_used < 0:
  9341. logger.warning(
  9342. "[ENERGY-BG] Negative energy delta for archive %s (start=%s, end=%s) — counter reset?",
  9343. archive_id,
  9344. starting_kwh,
  9345. energy["total"],
  9346. )
  9347. return
  9348. from backend.app.services.energy_price import current_price, print_energy_cost, stored_price
  9349. # Each hour of the print at the price of that hour (#1251). If
  9350. # that fails, the last price known, as before, rather than no
  9351. # energy at all. Read first: the session may be unusable after.
  9352. last_price = await stored_price(db)
  9353. try:
  9354. energy_cost_value = round(
  9355. await print_energy_cost(
  9356. db,
  9357. plug_id=start.plug_id,
  9358. start_at=start.at,
  9359. start_kwh=starting_kwh,
  9360. start_price=start.price,
  9361. end_kwh=energy["total"],
  9362. end_plug_id=plug.id,
  9363. price_now=await current_price(db),
  9364. ),
  9365. 3,
  9366. )
  9367. except Exception as e:
  9368. logger.warning("[ENERGY-BG] Hour-by-hour cost failed for archive %s: %s", archive_id, e)
  9369. energy_cost_value = round(energy_used * last_price, 3)
  9370. # First-run-only overwrite of archive.energy_kwh / energy_cost so a
  9371. # reprint doesn't visually clobber the source archive's energy data
  9372. # (#1378). Reprint energy lives in the matching PrintLogEntry below.
  9373. from sqlalchemy import func
  9374. existing_runs = await db.scalar(
  9375. select(func.count(PrintLogEntry.id)).where(PrintLogEntry.archive_id == archive_id)
  9376. )
  9377. if (existing_runs or 0) <= 1:
  9378. # 0 = legacy archive that pre-dates per-run logging; 1 = the row
  9379. # we just wrote for THIS print. Either way it's the first run.
  9380. archive.energy_kwh = energy_used
  9381. archive.energy_cost = energy_cost_value
  9382. # The ended run's energy (write_log_entry above ran before this
  9383. # background task, so its energy fields are still NULL). With
  9384. # no run written, none: the archive's newest is another print's.
  9385. if run_row is not None:
  9386. run_row.energy_kwh = energy_used
  9387. run_row.energy_cost = energy_cost_value
  9388. await db.commit()
  9389. logger.info("[ENERGY-BG] Saved: %s kWh, cost=%s", energy_used, energy_cost_value)
  9390. except Exception as e:
  9391. logger.warning("[ENERGY-BG] Failed: %s", e)
  9392. async def _background_finish_photo() -> str | None:
  9393. """Capture finish photo in background. Returns photo filename if captured."""
  9394. # #2547: the Z to park the plate back to, set once this function has
  9395. # raised the plate itself (the timelapse path, where the moment producer
  9396. # returned without doing it). Declared out here so the `finally` can
  9397. # lower it again no matter where the capture below fails.
  9398. park_z: float | None = None
  9399. try:
  9400. logger.info("[PHOTO-BG] Starting finish photo capture for archive %s", archive_id)
  9401. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  9402. # Read phase: settings + printer + archive in a short session, released
  9403. # BEFORE the capture pipeline below. The capture (timelapse last-frame,
  9404. # stage-22 wait, external-camera grab, or a fresh RTSP shot) can take
  9405. # tens of seconds; holding this session across it pinned one pooled
  9406. # connection idle-in-transaction per finishing print (issue #2572).
  9407. async with async_session() as db:
  9408. from backend.app.api.routes.settings import get_setting
  9409. from backend.app.models.archive import PrintArchive
  9410. from backend.app.models.printer import Printer
  9411. capture_enabled = await get_setting(db, "capture_finish_photo")
  9412. if capture_enabled is not None and capture_enabled.lower() != "true":
  9413. return None
  9414. if not archive_id:
  9415. return None
  9416. printer = (await db.execute(select(Printer).where(Printer.id == printer_id))).scalar_one_or_none()
  9417. archive = (
  9418. await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  9419. ).scalar_one_or_none()
  9420. if not printer or not archive:
  9421. return None
  9422. import uuid
  9423. from datetime import datetime
  9424. from backend.app.utils.archive_paths import archive_dir as resolve_archive_dir
  9425. if not archive.file_path:
  9426. logger.warning("[PHOTO-BG] Archive %s has no file_path, using fallback dir", archive_id)
  9427. archive_dir = resolve_archive_dir(archive)
  9428. photo_filename = None
  9429. # Prefer the timelapse last-frame source when a timelapse was
  9430. # recording — it captures the moment after the toolhead parks
  9431. # but before the bed drops, which the live-camera grab below
  9432. # would miss (#1397). Skipped for external cameras (those have
  9433. # their own framing and don't see a Bambu timelapse). Only
  9434. # runs when the USER explicitly enabled timelapse for this
  9435. # print — #1721 removed Bambuddy's force-on at dispatch
  9436. # because it caused per-layer nozzle parking on Smooth-mode
  9437. # slicer profiles.
  9438. prefer_timelapse_source = bool(data.get("timelapse_was_active")) and not (
  9439. printer.external_camera_enabled and printer.external_camera_url
  9440. )
  9441. timelapse_still_pending = False
  9442. if prefer_timelapse_source:
  9443. photo_filename, timelapse_still_pending = await _capture_finish_photo_from_timelapse(
  9444. archive_id=archive_id,
  9445. archive_dir=archive_dir,
  9446. rotation=getattr(printer, "camera_rotation", 0),
  9447. run_id=completed_run_id,
  9448. )
  9449. # #1721: replacement framing path — on_finish_photo_moment
  9450. # pre-captured a frame at the stage-22 / FINISH edge (toolhead
  9451. # parked, bed not yet dropped) and cached the JPEG bytes in
  9452. # _stage22_finish_frames. Consume them now so the saved photo
  9453. # has the better framing instead of the post-bed-drop angle
  9454. # the live-camera fallback below would give.
  9455. if not photo_filename:
  9456. # #1790: on the FINISH-state fallback path the producer
  9457. # task is dispatched back-to-back with this consumer, so
  9458. # a bare pop would race past with an empty result and
  9459. # the RTSP fallback below would collide with the
  9460. # producer's still-in-flight grab (single-client RTSP
  9461. # on Bambu printers). Wait for the producer to finish
  9462. # or give up before touching the cache.
  9463. #
  9464. # #2547: 20s was enough when the producer only ever grabbed a
  9465. # frame. It now also raises the plate first, which costs the
  9466. # settle window before the grab even starts — so the budget has
  9467. # to cover settle + a worst-case 15s RTSP timeout, and still sit
  9468. # under the notification's own photo wait below.
  9469. in_flight = _stage22_finish_in_flight.pop(printer_id, None)
  9470. if in_flight is not None:
  9471. try:
  9472. await asyncio.wait_for(in_flight.wait(), timeout=_FINISH_PHOTO_PRODUCER_WAIT_SECONDS)
  9473. except TimeoutError:
  9474. logger.warning(
  9475. "[PHOTO-BG] timed out waiting for stage-22 producer for printer %s — proceeding to fallback",
  9476. printer_id,
  9477. )
  9478. cached_frame = _stage22_finish_frames.pop(printer_id, None)
  9479. if cached_frame:
  9480. # Already rotated by the producer (#2708) — rotating again
  9481. # here would undo the fix on the banked-frame path, whose
  9482. # bytes reach the cache having been rotated once already.
  9483. photos_dir = archive_dir / "photos"
  9484. photos_dir.mkdir(parents=True, exist_ok=True)
  9485. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9486. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9487. photo_path = photos_dir / photo_filename
  9488. await asyncio.to_thread(photo_path.write_bytes, cached_frame)
  9489. logger.info(
  9490. "[PHOTO-BG] Saved stage-22 pre-captured frame: %s (%d bytes)",
  9491. photo_filename,
  9492. len(cached_frame),
  9493. )
  9494. # #2547: the timelapse path reaches the live grab below whenever the
  9495. # video hasn't landed in time — the documented usual outcome on
  9496. # P1-series, where transfers are slowest. `on_finish_photo_moment`
  9497. # returned early for those prints without raising the plate, so
  9498. # without this the photo that actually ships in the notification is
  9499. # of an already-dropped plate: exactly the framing #1145/#1397/#1565
  9500. # asked us to fix. The archive still gets the better video frame
  9501. # later; this is about the image the user is sent.
  9502. #
  9503. # Gated on `timelapse_was_active` precisely because that is the
  9504. # condition under which the producer skipped. On every other path it
  9505. # has already raised and lowered the plate, and repeating that here
  9506. # would be a second pointless round trip.
  9507. if (
  9508. not photo_filename
  9509. and data.get("timelapse_was_active")
  9510. and not print_dispatch_context.end_gcode_injected(printer_id)
  9511. ):
  9512. try:
  9513. async with async_session() as db:
  9514. from backend.app.api.routes.settings import get_setting
  9515. restore_setting = await get_setting(db, "finish_photo_restore_plate")
  9516. if restore_setting is None or restore_setting.lower() == "true":
  9517. park_z = await _raise_plate_for_finish_photo(
  9518. printer_id, data, logger, bound_archive_id, notify_plate_id
  9519. )
  9520. except Exception as e:
  9521. logger.warning("[PLATE-RESTORE] printer %s: restore failed: %s", printer_id, e)
  9522. # Fallback chain: external camera → buffered live frame →
  9523. # fresh RTSP capture. Only runs if the timelapse path above
  9524. # didn't already produce a photo.
  9525. # Chamber light for the live capture below (#1655).
  9526. async with camera_light(printer if not photo_filename else None):
  9527. if not photo_filename:
  9528. if printer.external_camera_enabled and printer.external_camera_url:
  9529. logger.info("[PHOTO-BG] Using external camera")
  9530. from backend.app.api.routes.camera import live_frame_for_capture
  9531. from backend.app.services.external_camera import capture_frame
  9532. # #2707: the second half of the finish-photo failure — the
  9533. # pre-capture and this fallback both collided with the live
  9534. # view. None here continues down the fallback chain.
  9535. defer, buffered = live_frame_for_capture(printer_id)
  9536. if defer:
  9537. frame_data = buffered
  9538. else:
  9539. frame_data = await capture_frame(
  9540. printer.external_camera_url,
  9541. printer.external_camera_type or "mjpeg",
  9542. snapshot_url=printer.external_camera_snapshot_url,
  9543. )
  9544. if frame_data:
  9545. frame_data = _apply_camera_rotation(frame_data, printer, logger)
  9546. photos_dir = archive_dir / "photos"
  9547. photos_dir.mkdir(parents=True, exist_ok=True)
  9548. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9549. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9550. photo_path = photos_dir / photo_filename
  9551. await asyncio.to_thread(photo_path.write_bytes, frame_data)
  9552. logger.info("[PHOTO-BG] Saved external camera frame: %s", photo_filename)
  9553. else:
  9554. # Check if camera stream is active - use buffered frame to avoid freeze
  9555. # Check both RTSP streams (_active_streams) and chamber image streams (_active_chamber_streams)
  9556. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  9557. active_chamber_for_printer = [
  9558. k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")
  9559. ]
  9560. buffered_frame = get_buffered_frame(printer_id)
  9561. if (active_for_printer or active_chamber_for_printer) and buffered_frame:
  9562. # Use frame from active stream
  9563. logger.info("[PHOTO-BG] Using buffered frame from active stream")
  9564. buffered_frame = _apply_camera_rotation(buffered_frame, printer, logger)
  9565. photos_dir = archive_dir / "photos"
  9566. photos_dir.mkdir(parents=True, exist_ok=True)
  9567. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9568. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9569. photo_path = photos_dir / photo_filename
  9570. await asyncio.to_thread(photo_path.write_bytes, buffered_frame)
  9571. logger.info("[PHOTO-BG] Saved buffered frame: %s", photo_filename)
  9572. else:
  9573. # No active stream - capture new frame
  9574. from backend.app.services.camera import capture_finish_photo
  9575. photo_filename = await capture_finish_photo(
  9576. printer_id=printer_id,
  9577. ip_address=printer.ip_address,
  9578. access_code=printer.access_code,
  9579. model=printer.model,
  9580. archive_dir=archive_dir,
  9581. rotation=getattr(printer, "camera_rotation", 0),
  9582. )
  9583. # Write phase: attach the photo in a fresh short-lived session.
  9584. if photo_filename:
  9585. async with async_session() as db:
  9586. from backend.app.models.archive import PrintArchive
  9587. from backend.app.services.print_log import add_run_photo, run_of
  9588. arch = await db.get(PrintArchive, archive_id)
  9589. # The ended print's own list takes it too: the archive's
  9590. # holds every print's.
  9591. run = await run_of(db, archive_id, completed_run_id) if completed_run_id is not None else None
  9592. if arch is not None:
  9593. # A new list: appended in place, the ORM sees no
  9594. # change and a second print's photo is never saved.
  9595. photos = list(arch.photos or [])
  9596. photos.append(photo_filename)
  9597. arch.photos = photos
  9598. add_run_photo(run, photo_filename)
  9599. await db.commit()
  9600. logger.info("[PHOTO-BG] Saved: %s", photo_filename)
  9601. # The short wait above is bounded so a slow printer can't hold up
  9602. # the print-complete notification, which is what the caller is
  9603. # blocking on. When it ran out with the video still on its way,
  9604. # keep waiting off to the side and swap the better frame into the
  9605. # archive once it arrives (#2704 follow-up) — otherwise P1-series
  9606. # users, whose videos routinely take minutes to transfer, never get
  9607. # the pre-bed-drop framing this path exists to provide.
  9608. #
  9609. # Spawned here rather than at the point the wait gave up: both this
  9610. # function and the upgrade do a read-modify-write on `photos`, and
  9611. # the live-camera fallback above can take tens of seconds. Starting
  9612. # the upgrade before that write means the two can interleave and one
  9613. # silently drops the other's entry, leaving a JPEG on disk that the
  9614. # gallery never lists.
  9615. if timelapse_still_pending:
  9616. from backend.app.services import archive_work
  9617. spawn_background_task(
  9618. archive_work.held_while(
  9619. archive_id,
  9620. _upgrade_finish_photo_from_timelapse(
  9621. archive_id,
  9622. archive_dir,
  9623. rotation=getattr(printer, "camera_rotation", 0),
  9624. run_id=completed_run_id,
  9625. replace=photo_filename,
  9626. ),
  9627. ),
  9628. name=f"finish-photo-upgrade-{archive_id}",
  9629. )
  9630. return photo_filename
  9631. except Exception as e:
  9632. logger.warning("[PHOTO-BG] Failed: %s", e)
  9633. return None
  9634. finally:
  9635. # #2547: we raised the plate, so we owe the move back down — even if
  9636. # the capture in between threw. Otherwise the user finds the print
  9637. # pinned under the nozzle.
  9638. if park_z is not None:
  9639. try:
  9640. _park_plate_after_finish_photo(printer_id, park_z, logger)
  9641. except Exception as e:
  9642. logger.warning("[PLATE-RESTORE] printer %s: could not lower plate: %s", printer_id, e)
  9643. spawn_background_task(_background_energy_calculation(), name="background-energy-calc")
  9644. # Photo capture task - result will be used by notifications
  9645. photo_task = spawn_background_task(_background_finish_photo(), name="background-finish-photo")
  9646. log_timing("Background tasks scheduled (energy, photo)")
  9647. # Also run smart plug, notifications, and maintenance as background tasks
  9648. print_status = data.get("status", "completed")
  9649. async def _background_smart_plug():
  9650. """Handle smart plug automation in background."""
  9651. try:
  9652. logger.info("[AUTO-OFF-BG] Starting smart plug automation for printer %s", printer_id)
  9653. async with async_session() as db:
  9654. await smart_plug_manager.on_print_complete(printer_id, print_status, db)
  9655. logger.info("[AUTO-OFF-BG] Completed")
  9656. except Exception as e:
  9657. logger.warning("[AUTO-OFF-BG] Failed: %s", e)
  9658. async def _background_notifications(finish_photo_filename: str | None = None):
  9659. """Send print complete notifications in background."""
  9660. try:
  9661. logger.info(
  9662. "[NOTIFY-BG] Starting notifications for printer %s, photo=%s", printer_id, finish_photo_filename
  9663. )
  9664. async with async_session() as db:
  9665. from backend.app.models.archive import PrintArchive
  9666. from backend.app.models.printer import Printer
  9667. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  9668. printer = result.scalar_one_or_none()
  9669. printer_name = printer.name if printer else f"Printer {printer_id}"
  9670. archive_data = None
  9671. if archive_id:
  9672. from backend.app.models.print_log import PrintLogEntry
  9673. archive_result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  9674. archive = archive_result.scalar_one_or_none()
  9675. # The run that ended: the archive shows its latest, which
  9676. # with another plate of it printing elsewhere is not this.
  9677. run = await db.get(PrintLogEntry, completed_run_id) if completed_run_id is not None else None
  9678. run_started = run.started_at if run is not None else archive.started_at if archive else None
  9679. run_completed = run.completed_at if run is not None else archive.completed_at if archive else None
  9680. if archive:
  9681. # Actual elapsed time from started_at/completed_at when both are
  9682. # populated (every terminal status sets completed_at after #1198).
  9683. # Falls back to None so the notification path can decide whether to
  9684. # render the slicer estimate as a last resort.
  9685. actual_time_seconds = None
  9686. # Unknown for a completion reconciled at reconnect: the
  9687. # print ended while the printer was offline and
  9688. # completed_at is the reconnect, so the gap would read
  9689. # as the duration -- 14 h for a night offline (DB-C9).
  9690. # An estimated end (#3310) gives the estimate.
  9691. if (
  9692. run_started
  9693. and run_completed
  9694. and (not data.get("_reconciled") or data.get("_reconciled_completed_at"))
  9695. ):
  9696. elapsed = (run_completed - run_started).total_seconds()
  9697. if elapsed > 0:
  9698. actual_time_seconds = int(elapsed)
  9699. archive_data = {
  9700. "print_time_seconds": archive.print_time_seconds,
  9701. "actual_time_seconds": actual_time_seconds,
  9702. "actual_filament_grams": archive.filament_used_grams,
  9703. "failure_reason": run.failure_reason if run is not None else archive.failure_reason,
  9704. "created_by_id": archive.created_by_id,
  9705. }
  9706. # Scale filament usage for partial prints
  9707. if print_status != "completed" and archive.filament_used_grams:
  9708. progress = data.get("progress") or 0
  9709. scale = _partial_progress_scale(progress)
  9710. archive_data["actual_filament_grams"] = round(archive.filament_used_grams * scale, 1)
  9711. archive_data["progress"] = progress
  9712. # Pass per-slot data from archive.extra_data
  9713. if archive.extra_data and archive.extra_data.get("filament_slots"):
  9714. slots = archive.extra_data["filament_slots"]
  9715. if print_status != "completed":
  9716. scale = _partial_progress_scale(data.get("progress"))
  9717. slots = [{**s, "used_g": round(s["used_g"] * scale, 1)} for s in slots]
  9718. archive_data["filament_slots"] = slots
  9719. # Scope project-summed totals down to the plate that was
  9720. # actually printed — see _scope_notification_archive_data_to_plate
  9721. # for the why (#1785).
  9722. archive_data = _scope_notification_archive_data_to_plate(
  9723. archive_data,
  9724. archive.file_path,
  9725. notify_plate_id,
  9726. print_status,
  9727. data.get("progress"),
  9728. app_settings.base_dir,
  9729. )
  9730. # Enrich filament_grams from usage_results when archive has no 3MF data
  9731. if not archive_data.get("actual_filament_grams") and usage_results:
  9732. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  9733. if total_from_usage > 0:
  9734. archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  9735. # Pass usage tracker results for AMS slot info in notifications
  9736. if usage_results:
  9737. archive_data["usage_results"] = usage_results
  9738. # Which archive and print it was (archive redesign).
  9739. archive_data["archive_id"] = archive_id
  9740. archive_data["run_id"] = completed_run_id
  9741. # Add finish photo URL and image bytes if available
  9742. if finish_photo_filename:
  9743. photo_url, photo_bytes = await _finish_photo_for_notification(
  9744. db, archive, archive_id, finish_photo_filename
  9745. )
  9746. if photo_url:
  9747. archive_data["finish_photo_url"] = photo_url
  9748. if photo_bytes:
  9749. archive_data["image_data"] = photo_bytes
  9750. if not await _kill_switch_notification_already_sent(kill_switch_notification_task):
  9751. await notification_service.on_print_complete(
  9752. printer_id, printer_name, print_status, data, db, archive_data=archive_data
  9753. )
  9754. else:
  9755. logger.info("[NOTIFY-BG] Skipped duplicate kill-switch provider notification")
  9756. # Post-print outcome confirmation (#1898). Runs in this
  9757. # background task so the finish photo fetched above rides
  9758. # along with the prompt.
  9759. if print_status == "completed" and archive_id:
  9760. await _dispatch_outcome_confirmation_safely(
  9761. db, printer_id, printer_name, data, archive_id, archive_data, run_id=completed_run_id
  9762. )
  9763. # Send user-specific email notification
  9764. if archive_data:
  9765. created_by_id = archive_data.get("created_by_id")
  9766. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  9767. await _dispatch_user_print_email(
  9768. print_status,
  9769. created_by_id,
  9770. printer_name,
  9771. raw_filename,
  9772. db,
  9773. image_data=archive_data.get("image_data"),
  9774. finish_photo_url=archive_data.get("finish_photo_url"),
  9775. )
  9776. logger.info("[NOTIFY-BG] Completed")
  9777. except Exception as e:
  9778. logger.error("[NOTIFY-BG] Failed: %s", e, exc_info=True)
  9779. async def _background_maintenance_check():
  9780. """Check for maintenance due in background."""
  9781. if print_status != "completed":
  9782. return
  9783. try:
  9784. logger.info("[MAINT-BG] Starting maintenance check for printer %s", printer_id)
  9785. async with async_session() as db:
  9786. from backend.app.models.printer import Printer
  9787. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  9788. printer = result.scalar_one_or_none()
  9789. printer_name = printer.name if printer else f"Printer {printer_id}"
  9790. await ensure_default_types(db)
  9791. overview = await _get_printer_maintenance_internal(printer_id, db, commit=True)
  9792. attention = [
  9793. item for item in overview.maintenance_items if item.enabled and (item.is_due or item.is_warning)
  9794. ]
  9795. items_needing_attention = [
  9796. {"name": item.maintenance_type_name, "is_due": item.is_due, "is_warning": item.is_warning}
  9797. for item in attention
  9798. ]
  9799. if items_needing_attention:
  9800. await notification_service.on_maintenance_due(printer_id, printer_name, items_needing_attention, db)
  9801. logger.info("[MAINT-BG] Sent notification: %s items need attention", len(items_needing_attention))
  9802. # MQTT relay - publish maintenance alerts
  9803. for item in attention:
  9804. current_value, threshold, unit = maintenance_alert_values(item)
  9805. try:
  9806. await mqtt_relay.on_maintenance_alert(
  9807. printer_id=printer_id,
  9808. printer_name=printer_name,
  9809. maintenance_type=item.maintenance_type_name,
  9810. current_value=current_value,
  9811. threshold=threshold,
  9812. unit=unit,
  9813. )
  9814. except Exception:
  9815. pass # Don't fail if MQTT fails
  9816. else:
  9817. logger.info("[MAINT-BG] Completed (no items need attention)")
  9818. except Exception as e:
  9819. logger.warning("[MAINT-BG] Failed: %s", e)
  9820. spawn_background_task(_background_smart_plug(), name="background-smart-plug")
  9821. spawn_background_task(_background_maintenance_check(), name="background-maintenance-check")
  9822. # Notification task waits for photo capture to complete first (with timeout).
  9823. # When a timelapse was recording, photo sourcing polls the per-print
  9824. # timelapse for up to 60s (#1397) — extend the budget so the notification
  9825. # carries the correct bed-up photo instead of falling through to the
  9826. # live-cam grab. Adds ~30s of notification latency at worst on slow links.
  9827. #
  9828. # #2547: both budgets now have to cover a plate restore as well.
  9829. #
  9830. # Without timelapse, the wait is on the moment producer, which raises the
  9831. # plate before its grab — so this has to outlast that producer's own budget.
  9832. #
  9833. # With timelapse, the capture polls up to
  9834. # `_FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS` for the video and only then
  9835. # falls back to a live grab, which is the case that raises the plate. At the
  9836. # old flat 75s that fallback was guaranteed to be cut off mid-settle, so the
  9837. # restore would have moved the plate for a photo nobody waited for.
  9838. photo_wait_timeout = (
  9839. _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS + _FINISH_PHOTO_PRODUCER_WAIT_SECONDS
  9840. if data.get("timelapse_was_active")
  9841. else _FINISH_PHOTO_PRODUCER_WAIT_SECONDS + 15
  9842. )
  9843. async def _photo_then_notify():
  9844. """Wait for photo capture, then send notification with photo URL."""
  9845. finish_photo = None
  9846. try:
  9847. finish_photo = await asyncio.wait_for(photo_task, timeout=photo_wait_timeout)
  9848. logger.info("[PHOTO-NOTIFY] Photo task returned: %s", finish_photo)
  9849. except TimeoutError:
  9850. logger.warning(
  9851. "[PHOTO-NOTIFY] Photo capture timed out after %ss, sending notification without photo",
  9852. photo_wait_timeout,
  9853. )
  9854. except Exception as e:
  9855. logger.warning("[PHOTO-NOTIFY] Photo task failed: %s", e)
  9856. try:
  9857. await _background_notifications(finish_photo)
  9858. except Exception as e:
  9859. logger.error("[PHOTO-NOTIFY] Notification sending failed: %s", e, exc_info=True)
  9860. spawn_background_task(_photo_then_notify(), name="photo-then-notify")
  9861. # Stitch external camera layer timelapse if session was active
  9862. print_status = data.get("status", "completed")
  9863. async def _background_layer_timelapse():
  9864. """Stitch layer timelapse and attach to archive."""
  9865. from backend.app.services.layer_timelapse import cancel_session, on_print_complete as tl_complete
  9866. try:
  9867. if print_status == "completed":
  9868. logger.info("[LAYER-TL] Stitching layer timelapse for printer %s", printer_id)
  9869. # The run the frames belong to, read before the stitch: it can
  9870. # take long enough for the archive to start printing again.
  9871. run_started_at = None
  9872. if archive_id:
  9873. async with async_session() as db:
  9874. run_archive = await ArchiveService(db).get_archive(archive_id)
  9875. run_started_at = run_archive.started_at if run_archive else None
  9876. timelapse_path = await tl_complete(printer_id)
  9877. if timelapse_path and archive_id:
  9878. await _attach_layer_timelapse(
  9879. archive_id, timelapse_path, notify_plate_id, run_started_at, run_id=completed_run_id
  9880. )
  9881. elif timelapse_path:
  9882. # Timelapse created but no archive - just clean up
  9883. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  9884. else:
  9885. # Print failed or cancelled - cancel timelapse session
  9886. cancel_session(printer_id)
  9887. logger.info(
  9888. "[LAYER-TL] Cancelled layer timelapse for printer %s (status: %s)", printer_id, print_status
  9889. )
  9890. except Exception as e:
  9891. logger.warning("[LAYER-TL] Failed: %s", e)
  9892. # A completed print's session left _active_sessions when its stitch
  9893. # began: the one there now belongs to the next print, which may have
  9894. # started during the stitch, and must not lose its frames.
  9895. if print_status != "completed":
  9896. try:
  9897. cancel_session(printer_id)
  9898. except Exception:
  9899. pass # Best-effort timelapse session cancellation on error
  9900. from backend.app.services import archive_work
  9901. # Held through the stitch, which can take longer than a merge waits after
  9902. # a print (archive redesign P2).
  9903. layer_timelapse = _background_layer_timelapse()
  9904. if archive_id:
  9905. layer_timelapse = archive_work.held_while(archive_id, layer_timelapse)
  9906. spawn_background_task(layer_timelapse, name="background-layer-timelapse")
  9907. log_timing("All background tasks scheduled")
  9908. # Auto-scan for timelapse if recording was active during the print. A
  9909. # failed or cancelled print gets one too: the printer writes the video of
  9910. # the run so far when it stops (measured on an H2C, #3481), and that video
  9911. # is how the user sees what went wrong. A model that writes none costs one
  9912. # poll that finds nothing new.
  9913. if archive_id and data.get("timelapse_was_active"):
  9914. logger.info("[TIMELAPSE] Timelapse was active during print, scheduling auto-scan for archive %s", archive_id)
  9915. # Schedule timelapse scan as background task with retries
  9916. # The printer needs time to encode the video after print completion
  9917. _schedule_timelapse_scan(archive_id, printer_id, plate_id=notify_plate_id, run_id=completed_run_id)
  9918. log_timing("Timelapse scan scheduled")
  9919. logger.info("[CALLBACK] on_print_complete finished for printer %s, archive %s", printer_id, archive_id)
  9920. # AMS sensor history recording
  9921. _ams_history_task: asyncio.Task | None = None
  9922. AMS_HISTORY_INTERVAL = 300 # Record every 5 minutes
  9923. AMS_HISTORY_RETENTION_DAYS = 30 # Keep data for 30 days
  9924. def _retention_days(stored: str | None, default: int) -> int:
  9925. """A history retention setting as days to keep: at least 1, the default
  9926. when unset or unreadable. A stored 0 or less (the API took one before it
  9927. had a minimum) would put the cutoff at now and delete all history."""
  9928. try:
  9929. return max(1, int(stored)) if stored is not None else default
  9930. except (TypeError, ValueError):
  9931. return default
  9932. _ams_cleanup_counter = 0 # Track recordings to trigger periodic cleanup
  9933. # Track alarm cooldowns (printer_id:ams_id:type -> last_alarm_time)
  9934. _ams_alarm_cooldown: dict[str, datetime] = {}
  9935. AMS_ALARM_COOLDOWN_MINUTES = 60 # Don't send same alarm more than once per hour
  9936. # (printer_id, ams_id) already reported as sending the drop index and no
  9937. # percentage. Logged once each so a supported printer that turns out to do this
  9938. # shows up in a support bundle rather than as a user wondering where the
  9939. # humidity reading went -- see the note at the read site below (#3140).
  9940. _ams_index_only_logged: set[tuple[int, int]] = set()
  9941. def _resolve_temp_alarm_threshold(fair_threshold: float, raw_alarm_value: str | None) -> float:
  9942. """Temperature at which the AMS alarm fires, falling back to the display band.
  9943. ``ams_temp_fair`` decides when the AMS card turns amber. It used to decide
  9944. when a notification was sent as well, which is why a room above it made the
  9945. alarm fire once an hour for as long as the weather lasted -- and the only way
  9946. to stop that was to raise the display band and lose the colour that says the
  9947. unit is warm (#2905).
  9948. Unset resolves to the fair threshold, so an install that never sets one is
  9949. unchanged. Settings storage stringifies ``None`` to the literal ``"None"``,
  9950. so that arrives here as a string and is handled by the same branch as any
  9951. other unparseable value -- there is no separate sentinel to keep in sync.
  9952. A non-positive value is refused rather than honoured: zero would alarm
  9953. permanently, and it is far more likely to be a cleared field than a
  9954. deliberate choice.
  9955. """
  9956. if raw_alarm_value is None:
  9957. return fair_threshold
  9958. try:
  9959. value = float(raw_alarm_value)
  9960. except (TypeError, ValueError):
  9961. return fair_threshold
  9962. if not math.isfinite(value) or value <= 0:
  9963. return fair_threshold
  9964. return value
  9965. # Per-AMS "drying was live at" latch that suppresses the high-temperature alarm
  9966. # through a cycle and the cool-down after it (#1802). Stored in the settings
  9967. # table rather than alongside _ams_alarm_cooldown above, because a restart
  9968. # partway through a cool-down would otherwise resume alarming about heat the
  9969. # user asked for — the same internal-timestamp-row pattern as
  9970. # support.py's debug_logging_enabled_at.
  9971. AMS_DRYING_LATCH_KEY = "ams_drying_alarm_latch"
  9972. # Upper bound on that suppression. The latch normally clears as soon as the unit
  9973. # reads at or below the threshold; see utils.ams_drying for why this cap only
  9974. # matters when it never does.
  9975. AMS_DRYING_GRACE_MINUTES = 120
  9976. async def _load_ams_drying_latch(db) -> dict[str, datetime]:
  9977. """Read the persisted per-AMS drying latch, dropping entries out of window.
  9978. Anything older than the grace cap would expire on its next visit anyway, so
  9979. discarding it here costs nothing and stops rows for deleted printers from
  9980. accumulating.
  9981. Stamps ahead of now get two defences, because a box whose clock jumps
  9982. backwards (a Pi with no RTC coming up before NTP) writes them: wildly future
  9983. ones are discarded outright, and the rest are clamped to now. Without the
  9984. clamp the cap would measure from a moment that has not happened yet and hold
  9985. the alarm quiet for the skew on top of the cap. One unnecessary notification
  9986. after a clock jump is a far better failure than an alarm silently disabled
  9987. for hours.
  9988. """
  9989. from backend.app.models.settings import Settings
  9990. result = await db.execute(select(Settings).where(Settings.key == AMS_DRYING_LATCH_KEY))
  9991. setting = result.scalar_one_or_none()
  9992. if not setting or not setting.value:
  9993. return {}
  9994. try:
  9995. raw = json.loads(setting.value)
  9996. except (ValueError, TypeError):
  9997. return {} # Corrupted row → no latch, alarms behave as they did before
  9998. if not isinstance(raw, dict):
  9999. return {}
  10000. now = datetime.now(timezone.utc)
  10001. window = timedelta(minutes=AMS_DRYING_GRACE_MINUTES)
  10002. latch: dict[str, datetime] = {}
  10003. for key, value in raw.items():
  10004. try:
  10005. stamp = datetime.fromisoformat(str(value))
  10006. except (ValueError, TypeError):
  10007. continue
  10008. if stamp.tzinfo is None:
  10009. stamp = stamp.replace(tzinfo=timezone.utc)
  10010. if not (now - window <= stamp <= now + window):
  10011. continue
  10012. # Nothing may sit in the future: suppression is measured as now minus
  10013. # the stamp, so a stamp ahead of now would extend it by the skew on top
  10014. # of the cap. Clamping the survivors keeps the cap an actual cap.
  10015. latch[str(key)] = min(stamp, now)
  10016. return latch
  10017. async def _save_ams_drying_latch(db, latch: dict[str, datetime]) -> None:
  10018. """Persist the latch, writing only when it actually changed.
  10019. Adds the session change but does not commit — the caller's own commit
  10020. carries it, so the latch lands in the same transaction as the sensor rows
  10021. that produced it.
  10022. """
  10023. from backend.app.models.settings import Settings
  10024. payload = json.dumps({key: stamp.isoformat() for key, stamp in sorted(latch.items())})
  10025. result = await db.execute(select(Settings).where(Settings.key == AMS_DRYING_LATCH_KEY))
  10026. setting = result.scalar_one_or_none()
  10027. if setting is None:
  10028. # Don't create the row on installs that never dry anything.
  10029. if payload != "{}":
  10030. db.add(Settings(key=AMS_DRYING_LATCH_KEY, value=payload))
  10031. elif setting.value != payload:
  10032. setting.value = payload
  10033. def _ams_has_filament(ams_data: dict) -> bool:
  10034. """True if this AMS unit has at least one tray slot holding filament.
  10035. Bambu firmware reports loaded slots via `tray_exist_bits`, a per-AMS hex
  10036. bitmap (one bit per tray slot — bit set = spool present). Empty AMS units
  10037. still report sensor readings, but those readings are ambient and not
  10038. actionable: no filament to dry, no humidity to push down. #1619 — gate
  10039. humidity/temperature alarms on this check so empty units don't generate
  10040. hourly noise. Sensor history still records regardless so the UI charts
  10041. stay continuous.
  10042. Fallback path inspects the `tray` array's `tray_type` fields for setups
  10043. where `tray_exist_bits` is missing (some early-connection pushall shapes).
  10044. """
  10045. bits = ams_data.get("tray_exist_bits")
  10046. if isinstance(bits, str) and bits.strip():
  10047. try:
  10048. return int(bits, 16) > 0
  10049. except ValueError:
  10050. pass
  10051. trays = ams_data.get("tray")
  10052. if isinstance(trays, list):
  10053. return any(
  10054. isinstance(t, dict) and isinstance(t.get("tray_type"), str) and t["tray_type"].strip() for t in trays
  10055. )
  10056. return False
  10057. async def record_ams_history():
  10058. """Background task to record AMS humidity and temperature data."""
  10059. logger = logging.getLogger(__name__)
  10060. # Wait a short time for MQTT connections to establish on startup
  10061. await asyncio.sleep(10)
  10062. while True:
  10063. try:
  10064. from backend.app.models.ams_history import AMSSensorHistory
  10065. from backend.app.models.printer import Printer
  10066. from backend.app.models.settings import Settings
  10067. async with async_session() as db:
  10068. # Get all active printers
  10069. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  10070. printers = result.scalars().all()
  10071. # Get alarm thresholds from settings
  10072. humidity_threshold = 60.0 # Default: fair threshold
  10073. temp_fair_threshold = 35.0 # Display band default (ams_temp_fair)
  10074. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  10075. setting = result.scalar_one_or_none()
  10076. if setting:
  10077. try:
  10078. humidity_threshold = float(setting.value)
  10079. except (ValueError, TypeError):
  10080. pass # Keep default threshold if stored value is invalid
  10081. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_fair"))
  10082. setting = result.scalar_one_or_none()
  10083. if setting:
  10084. try:
  10085. temp_fair_threshold = float(setting.value)
  10086. except (ValueError, TypeError):
  10087. pass # Keep default threshold if stored value is invalid
  10088. # The alarm gets its own threshold, seeded from the resolved fair
  10089. # value so an install that has never set one behaves exactly as
  10090. # it did before (#2905). ams_temp_fair decides when the card turns
  10091. # amber; 35 C is a reasonable place to change a colour and not a
  10092. # reasonable place to page someone. A room above it makes the
  10093. # alarm fire once an hour for as long as the weather lasts, and
  10094. # the only way to stop it was to raise the display band and lose
  10095. # the colour that says the unit is warm.
  10096. #
  10097. # An unset value is stored as the literal "None", which the except
  10098. # below swallows the same way it swallows garbage -- so the
  10099. # fallback costs nothing and needs no sentinel of its own.
  10100. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_alarm"))
  10101. setting = result.scalar_one_or_none()
  10102. temp_alarm_threshold = _resolve_temp_alarm_threshold(
  10103. temp_fair_threshold, setting.value if setting else None
  10104. )
  10105. # Per-filament humidity threshold overrides (#1605) — resolved
  10106. # per-AMS below from the loaded tray types. Reuses the same
  10107. # resolver as the auto-drying scheduler so behavior stays in
  10108. # lockstep across both consumers.
  10109. from backend.app.services.print_scheduler import PrintScheduler
  10110. per_type_humidity_thresholds: dict[str, int] = {}
  10111. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_thresholds"))
  10112. setting = result.scalar_one_or_none()
  10113. if setting and setting.value:
  10114. try:
  10115. raw = json.loads(setting.value)
  10116. if isinstance(raw, dict):
  10117. for k, v in raw.items():
  10118. try:
  10119. per_type_humidity_thresholds[str(k).upper() if k != "default" else "default"] = int(
  10120. v
  10121. )
  10122. except (TypeError, ValueError):
  10123. continue
  10124. except (ValueError, TypeError):
  10125. pass # Invalid JSON → no overrides, fall through to global threshold
  10126. # Per-AMS drying latch (#1802), loaded once per pass and written
  10127. # back below only if a unit changed it.
  10128. drying_latch = await _load_ams_drying_latch(db)
  10129. drying_latch_before = dict(drying_latch)
  10130. recorded_count = 0
  10131. for printer in printers:
  10132. # Get current state from printer manager
  10133. state = printer_manager.get_status(printer.id)
  10134. if not state or not state.connected or not state.raw_data:
  10135. continue # Skip disconnected printers - don't use stale data
  10136. raw_data = state.raw_data
  10137. if "ams" not in raw_data or not isinstance(raw_data["ams"], list):
  10138. continue
  10139. # Record data for each AMS unit
  10140. for ams_data in raw_data["ams"]:
  10141. ams_id = int(ams_data.get("id", 0))
  10142. # Percentage only. The 1-5 index is inverted, so
  10143. # charting it as a percentage drew the wettest units as
  10144. # the driest (#3140); a unit that reports no percentage
  10145. # leaves a gap in the chart instead. See
  10146. # utils/ams_humidity.
  10147. humidity = ams_humidity_percent(ams_data)
  10148. # No supported printer is known to send the index
  10149. # alone -- the report came from unsupported firmware,
  10150. # and no install has been seen using the old fallback.
  10151. # "Known" is doing work there, so say so once per unit:
  10152. # the alternative is a silent blank card.
  10153. if humidity is None and ams_data.get("humidity") is not None:
  10154. unit_key = (printer.id, ams_id)
  10155. if unit_key not in _ams_index_only_logged:
  10156. _ams_index_only_logged.add(unit_key)
  10157. logger.info(
  10158. "[%s] AMS %d reports the 1-5 humidity index but no usable humidity_raw "
  10159. "percentage. The index is inverted and is not shown as a percentage "
  10160. "(#3140), so this unit has no humidity reading, chart or alarm. "
  10161. "Please report this with the printer and AMS firmware versions.",
  10162. printer.name,
  10163. ams_id,
  10164. )
  10165. # Get temperature
  10166. temperature = None
  10167. temp_str = ams_data.get("temp")
  10168. if temp_str is not None:
  10169. try:
  10170. temperature = float(temp_str)
  10171. except (ValueError, TypeError):
  10172. pass # Skip unparseable temperature value
  10173. # Skip if no data
  10174. if humidity is None and temperature is None:
  10175. continue
  10176. # Record the data point
  10177. history = AMSSensorHistory(
  10178. printer_id=printer.id,
  10179. ams_id=ams_id,
  10180. humidity=humidity,
  10181. # Both columns hold the same reading now that the
  10182. # index can no longer reach ``humidity``. Writing it
  10183. # through the same value also stops a genuine 0%
  10184. # from being stored as NULL, which the old truthiness
  10185. # test did.
  10186. humidity_raw=humidity,
  10187. temperature=temperature,
  10188. )
  10189. db.add(history)
  10190. recorded_count += 1
  10191. # Generate AMS label and determine if it's AMS-HT (A, B, C, D or HT-A for AMS-Lite/Hub)
  10192. ams_type = unit_ams_type(ams_id, ams_data.get("info"))
  10193. is_ams_ht = ams_type == AmsType.N3S
  10194. ams_label = unit_label(ams_id, ams_type)
  10195. # Skip alarm dispatch for empty AMS units — humidity /
  10196. # temperature readings are ambient with no filament to
  10197. # protect, and the hourly notification just becomes
  10198. # noise. Sensor history was already recorded above so
  10199. # the UI charts stay continuous (#1619). Per-AMS check
  10200. # so a multi-AMS setup with one loaded + one empty
  10201. # still alarms on the loaded unit.
  10202. if not _ams_has_filament(ams_data):
  10203. continue
  10204. # Resolve per-filament humidity threshold for this AMS
  10205. # unit (#1605). Falls back to the global ams_humidity_fair
  10206. # when no per-type overrides are configured.
  10207. trays = ams_data.get("tray", []) or []
  10208. effective_humidity_threshold = float(
  10209. PrintScheduler.resolve_humidity_threshold(
  10210. trays, per_type_humidity_thresholds, int(humidity_threshold)
  10211. )
  10212. )
  10213. # Check humidity alarm (only if above threshold)
  10214. if humidity is not None and humidity > effective_humidity_threshold:
  10215. cooldown_key = f"{printer.id}:{ams_id}:humidity"
  10216. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  10217. now = datetime.now(timezone.utc)
  10218. if (
  10219. last_alarm is None
  10220. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  10221. ):
  10222. _ams_alarm_cooldown[cooldown_key] = now
  10223. logger.info(
  10224. f"Sending humidity alarm for {printer.name} {ams_label}: {humidity}% > {effective_humidity_threshold}%"
  10225. )
  10226. try:
  10227. # Call different notification method based on AMS type
  10228. if is_ams_ht:
  10229. await notification_service.on_ams_ht_humidity_high(
  10230. printer.id,
  10231. printer.name,
  10232. ams_label,
  10233. humidity,
  10234. effective_humidity_threshold,
  10235. db,
  10236. )
  10237. else:
  10238. await notification_service.on_ams_humidity_high(
  10239. printer.id,
  10240. printer.name,
  10241. ams_label,
  10242. humidity,
  10243. effective_humidity_threshold,
  10244. db,
  10245. )
  10246. except Exception as e:
  10247. logger.warning("Failed to send humidity alarm: %s", e)
  10248. # A drying cycle heats the unit far past ams_temp_fair on
  10249. # purpose — 45 C for PLA, 65 C for PETG, 85 C on an
  10250. # AMS-HT, against a 35 C default — so the alarm fired
  10251. # once an hour for the whole cycle and kept firing while
  10252. # the unit cooled back down (#1802). Latch on the
  10253. # firmware's own drying state and hold until the reading
  10254. # returns to normal. Humidity is deliberately left alone:
  10255. # it falls during drying, which is the whole point.
  10256. latch_key = f"{printer.id}:{ams_id}"
  10257. # The latch releases at `threshold`, so it takes the alarm
  10258. # number too. Handing it the display band would strand the
  10259. # latch on any unit that settles back above it -- a room
  10260. # where the AMS rests at 37.7 C never returns under a 35 C
  10261. # band, so the latch could only expire on the grace cap
  10262. # rather than releasing when the unit had actually cooled.
  10263. suppress_temp_alarm, new_latch = temperature_alarm_suppressed(
  10264. drying_active=is_drying_active(ams_data),
  10265. temperature=temperature,
  10266. threshold=temp_alarm_threshold,
  10267. latched_at=drying_latch.get(latch_key),
  10268. now=datetime.now(timezone.utc),
  10269. grace_minutes=AMS_DRYING_GRACE_MINUTES,
  10270. )
  10271. if new_latch is None:
  10272. drying_latch.pop(latch_key, None)
  10273. else:
  10274. drying_latch[latch_key] = new_latch
  10275. # Check temperature alarm (only if above threshold)
  10276. if temperature is not None and temperature > temp_alarm_threshold and not suppress_temp_alarm:
  10277. cooldown_key = f"{printer.id}:{ams_id}:temperature"
  10278. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  10279. now = datetime.now(timezone.utc)
  10280. if (
  10281. last_alarm is None
  10282. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  10283. ):
  10284. _ams_alarm_cooldown[cooldown_key] = now
  10285. logger.info(
  10286. f"Sending temperature alarm for {printer.name} {ams_label}: "
  10287. f"{temperature}°C > {temp_alarm_threshold}°C"
  10288. )
  10289. try:
  10290. # Call different notification method based on AMS type
  10291. if is_ams_ht:
  10292. # The reported threshold has to be the one
  10293. # that fired, or the message says "> 35 °C"
  10294. # while firing at 45.
  10295. await notification_service.on_ams_ht_temperature_high(
  10296. printer.id, printer.name, ams_label, temperature, temp_alarm_threshold, db
  10297. )
  10298. else:
  10299. await notification_service.on_ams_temperature_high(
  10300. printer.id, printer.name, ams_label, temperature, temp_alarm_threshold, db
  10301. )
  10302. except Exception as e:
  10303. logger.warning("Failed to send temperature alarm: %s", e)
  10304. if drying_latch != drying_latch_before:
  10305. await _save_ams_drying_latch(db, drying_latch)
  10306. await db.commit()
  10307. if recorded_count > 0:
  10308. logger.info("Recorded %s AMS sensor history entries", recorded_count)
  10309. # Periodic cleanup of old data (every ~288 recordings = ~24 hours at 5min interval)
  10310. global _ams_cleanup_counter
  10311. _ams_cleanup_counter += 1
  10312. if _ams_cleanup_counter >= 288:
  10313. _ams_cleanup_counter = 0
  10314. # Get retention days from settings
  10315. from backend.app.models.settings import Settings
  10316. result = await db.execute(select(Settings).where(Settings.key == "ams_history_retention_days"))
  10317. setting = result.scalar_one_or_none()
  10318. retention_days = _retention_days(setting.value if setting else None, AMS_HISTORY_RETENTION_DAYS)
  10319. cutoff = utcnow_naive() - timedelta(days=retention_days)
  10320. result = await db.execute(delete(AMSSensorHistory).where(AMSSensorHistory.recorded_at < cutoff))
  10321. await db.commit()
  10322. if result.rowcount > 0:
  10323. logger.info(
  10324. f"Cleaned up {result.rowcount} old AMS sensor history entries (older than {retention_days} days)"
  10325. )
  10326. # Wait until next recording interval
  10327. await asyncio.sleep(AMS_HISTORY_INTERVAL)
  10328. except asyncio.CancelledError:
  10329. break
  10330. except Exception as e:
  10331. logger.warning("AMS history recording failed: %s", e)
  10332. await asyncio.sleep(60) # Wait a bit before retrying
  10333. def start_ams_history_recording():
  10334. """Start the AMS history recording background task."""
  10335. global _ams_history_task
  10336. if _ams_history_task is None:
  10337. _ams_history_task = asyncio.create_task(record_ams_history())
  10338. logging.getLogger(__name__).info("AMS history recording started")
  10339. def stop_ams_history_recording():
  10340. """Stop the AMS history recording background task."""
  10341. global _ams_history_task
  10342. if _ams_history_task:
  10343. _ams_history_task.cancel()
  10344. _ams_history_task = None
  10345. logging.getLogger(__name__).info("AMS history recording stopped")
  10346. # Printer sensor history recording (nozzle / bed / chamber)
  10347. _printer_sensor_history_task: asyncio.Task | None = None
  10348. PRINTER_SENSOR_HISTORY_INTERVAL = 60 # Record every minute — heaters move faster than AMS humidity
  10349. PRINTER_SENSOR_HISTORY_RETENTION_DAYS = 30
  10350. _printer_sensor_cleanup_counter = 0
  10351. # Sensor kinds tracked in state.temperatures — these are the normalised keys the
  10352. # MQTT parser writes, so we don't need to handle per-model field aliases here
  10353. # (nozzle_temper / left_nozzle_temper / right_nozzle_temper / chamber_temper
  10354. # are all collapsed by services/bambu_mqtt.py before they reach this loop).
  10355. _SENSOR_KINDS = ("nozzle", "nozzle_2", "bed", "chamber")
  10356. _SENSOR_TARGET_KEYS = {
  10357. "nozzle": "nozzle_target",
  10358. "nozzle_2": "nozzle_2_target",
  10359. "bed": "bed_target",
  10360. "chamber": "chamber_target",
  10361. }
  10362. async def record_printer_sensor_history():
  10363. """Background task to record nozzle / bed / chamber readings.
  10364. Pulls from `state.temperatures` (already normalised across all printer
  10365. models by the MQTT parser) rather than re-parsing raw_data, so we get
  10366. free coverage of dual-nozzle H2D, sensor-only X1C chamber, etc.
  10367. """
  10368. logger = logging.getLogger(__name__)
  10369. await asyncio.sleep(10)
  10370. while True:
  10371. try:
  10372. from backend.app.models.printer import Printer
  10373. from backend.app.models.printer_sensor_history import PrinterSensorHistory
  10374. from backend.app.models.settings import Settings
  10375. async with async_session() as db:
  10376. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  10377. printers = result.scalars().all()
  10378. recorded_count = 0
  10379. for printer in printers:
  10380. state = printer_manager.get_status(printer.id)
  10381. if not state or not state.connected:
  10382. continue
  10383. temps = getattr(state, "temperatures", None) or {}
  10384. if not isinstance(temps, dict):
  10385. continue
  10386. for kind in _SENSOR_KINDS:
  10387. if kind not in temps:
  10388. continue
  10389. try:
  10390. value = float(temps[kind])
  10391. except (ValueError, TypeError):
  10392. continue
  10393. target_raw = temps.get(_SENSOR_TARGET_KEYS[kind])
  10394. target_val: float | None = None
  10395. if target_raw is not None:
  10396. try:
  10397. target_val = float(target_raw)
  10398. except (ValueError, TypeError):
  10399. target_val = None
  10400. db.add(
  10401. PrinterSensorHistory(
  10402. printer_id=printer.id,
  10403. sensor_kind=kind,
  10404. value=value,
  10405. target=target_val,
  10406. )
  10407. )
  10408. recorded_count += 1
  10409. await db.commit()
  10410. if recorded_count > 0:
  10411. logger.debug("Recorded %s printer sensor history entries", recorded_count)
  10412. # Periodic cleanup — once every ~24h at this interval.
  10413. global _printer_sensor_cleanup_counter
  10414. _printer_sensor_cleanup_counter += 1
  10415. cleanup_every = max(1, (24 * 60 * 60) // PRINTER_SENSOR_HISTORY_INTERVAL)
  10416. if _printer_sensor_cleanup_counter >= cleanup_every:
  10417. _printer_sensor_cleanup_counter = 0
  10418. result = await db.execute(
  10419. select(Settings).where(Settings.key == "printer_sensor_history_retention_days")
  10420. )
  10421. setting = result.scalar_one_or_none()
  10422. retention_days = _retention_days(
  10423. setting.value if setting else None, PRINTER_SENSOR_HISTORY_RETENTION_DAYS
  10424. )
  10425. cutoff = utcnow_naive() - timedelta(days=retention_days)
  10426. cleanup = await db.execute(
  10427. delete(PrinterSensorHistory).where(PrinterSensorHistory.recorded_at < cutoff)
  10428. )
  10429. await db.commit()
  10430. if cleanup.rowcount > 0:
  10431. logger.info(
  10432. "Cleaned up %s old printer sensor history entries (older than %s days)",
  10433. cleanup.rowcount,
  10434. retention_days,
  10435. )
  10436. await asyncio.sleep(PRINTER_SENSOR_HISTORY_INTERVAL)
  10437. except asyncio.CancelledError:
  10438. break
  10439. except Exception as e:
  10440. logger.warning("Printer sensor history recording failed: %s", e)
  10441. await asyncio.sleep(60)
  10442. def start_printer_sensor_history_recording():
  10443. global _printer_sensor_history_task
  10444. if _printer_sensor_history_task is None:
  10445. _printer_sensor_history_task = asyncio.create_task(record_printer_sensor_history())
  10446. logging.getLogger(__name__).info("Printer sensor history recording started")
  10447. def stop_printer_sensor_history_recording():
  10448. global _printer_sensor_history_task
  10449. if _printer_sensor_history_task:
  10450. _printer_sensor_history_task.cancel()
  10451. _printer_sensor_history_task = None
  10452. logging.getLogger(__name__).info("Printer sensor history recording stopped")
  10453. # Printer runtime tracking
  10454. _runtime_tracking_task: asyncio.Task | None = None
  10455. RUNTIME_TRACKING_INTERVAL = 30 # Update every 30 seconds
  10456. # Printers the runtime tracker has seen connected since startup. One of them
  10457. # going offline clears its timestamp, so the time it was away is not counted
  10458. # when it comes back printing (DB-C5). A printer not yet seen -- still
  10459. # connecting after a restart -- keeps it for a grace period: a print that ran
  10460. # on through a Bambuddy restart is counted from the stored timestamp, as before.
  10461. _runtime_seen_connected: set[int] = set()
  10462. # A printer never seen connected this long after the tracker started is
  10463. # offline, not still connecting: its stored timestamp predates a gap too.
  10464. RUNTIME_STARTUP_GRACE_SECONDS = 120
  10465. async def track_printer_runtime():
  10466. """Background task to track printer active runtime (RUNNING state only).
  10467. PAUSE is intentionally excluded — the runtime counter feeds hours-based
  10468. maintenance intervals (rod lubrication, belt checks, nozzle cleaning)
  10469. which track mechanical wear. Pause time has no motion and no wear, so
  10470. counting it inflates maintenance warnings (#1521).
  10471. """
  10472. logger = logging.getLogger(__name__)
  10473. # Wait for MQTT connections to establish on startup
  10474. await asyncio.sleep(15)
  10475. started = time.monotonic()
  10476. while True:
  10477. try:
  10478. from backend.app.models.printer import Printer
  10479. # Fetch printer IDs in a short-lived read-only session
  10480. async with async_session() as db:
  10481. result = await db.execute(
  10482. select(Printer.id, Printer.name, Printer.runtime_seconds, Printer.last_runtime_update).where(
  10483. Printer.is_active.is_(True)
  10484. )
  10485. )
  10486. printer_rows = result.all()
  10487. now = datetime.now(timezone.utc)
  10488. updated_count = 0
  10489. # Update each printer in its own short session to minimise write-lock
  10490. # hold time and avoid blocking critical commits like queue status
  10491. # updates (#897).
  10492. for pid, pname, runtime_secs, last_update in printer_rows:
  10493. state = printer_manager.get_status(pid)
  10494. if not state:
  10495. logger.debug("[%s] Runtime tracking: no state available", pname)
  10496. continue
  10497. if not state.connected:
  10498. logger.debug("[%s] Runtime tracking: not connected", pname)
  10499. offline = pid in _runtime_seen_connected or (
  10500. time.monotonic() - started > RUNTIME_STARTUP_GRACE_SECONDS
  10501. )
  10502. if offline and last_update is not None:
  10503. try:
  10504. async with async_session() as db:
  10505. printer = await db.get(Printer, pid)
  10506. if printer:
  10507. printer.last_runtime_update = None
  10508. await db.commit()
  10509. except Exception as e:
  10510. # Still marked: the next offline tick tries again.
  10511. logger.warning("[%s] Runtime tracking commit failed: %s", pname, e)
  10512. continue
  10513. _runtime_seen_connected.discard(pid)
  10514. continue
  10515. _runtime_seen_connected.add(pid)
  10516. needs_commit = False
  10517. new_runtime = runtime_secs
  10518. new_last_update = last_update
  10519. if state.state == "RUNNING":
  10520. if last_update:
  10521. lu = last_update if last_update.tzinfo else last_update.replace(tzinfo=timezone.utc)
  10522. elapsed = (now - lu).total_seconds()
  10523. if elapsed > 0:
  10524. new_runtime = runtime_secs + int(elapsed)
  10525. updated_count += 1
  10526. needs_commit = True
  10527. logger.debug(
  10528. f"[{pname}] Runtime tracking: added {int(elapsed)}s, "
  10529. f"total={new_runtime}s ({new_runtime / 3600:.2f}h)"
  10530. )
  10531. else:
  10532. needs_commit = True
  10533. logger.debug("[%s] Runtime tracking: first active detection", pname)
  10534. new_last_update = now
  10535. else:
  10536. if last_update is not None:
  10537. logger.debug(f"[{pname}] Runtime tracking: state={state.state}, clearing last_runtime_update")
  10538. new_last_update = None
  10539. needs_commit = True
  10540. if needs_commit:
  10541. try:
  10542. async with async_session() as db:
  10543. result = await db.execute(select(Printer).where(Printer.id == pid))
  10544. printer = result.scalar_one_or_none()
  10545. if printer:
  10546. printer.runtime_seconds = new_runtime
  10547. printer.last_runtime_update = new_last_update
  10548. await db.commit()
  10549. except Exception as e:
  10550. logger.warning("[%s] Runtime tracking commit failed: %s", pname, e)
  10551. if updated_count > 0:
  10552. logger.debug("Updated runtime for %s printer(s)", updated_count)
  10553. except asyncio.CancelledError:
  10554. logger.info("Runtime tracking cancelled")
  10555. break
  10556. except Exception as e:
  10557. logger.warning("Runtime tracking failed: %s", e)
  10558. await asyncio.sleep(RUNTIME_TRACKING_INTERVAL)
  10559. def start_runtime_tracking():
  10560. """Start the printer runtime tracking background task."""
  10561. global _runtime_tracking_task
  10562. if _runtime_tracking_task is None:
  10563. _runtime_tracking_task = asyncio.create_task(track_printer_runtime())
  10564. logging.getLogger(__name__).info("Printer runtime tracking started")
  10565. def stop_runtime_tracking():
  10566. """Stop the printer runtime tracking background task."""
  10567. global _runtime_tracking_task
  10568. if _runtime_tracking_task:
  10569. _runtime_tracking_task.cancel()
  10570. _runtime_tracking_task = None
  10571. logging.getLogger(__name__).info("Printer runtime tracking stopped")
  10572. # SpoolBuddy device watchdog
  10573. _spoolbuddy_watchdog_task: asyncio.Task | None = None
  10574. SPOOLBUDDY_WATCHDOG_INTERVAL = 15
  10575. async def _spoolbuddy_watchdog_loop():
  10576. """Periodic check for SpoolBuddy devices that have gone offline."""
  10577. from backend.app.api.routes.spoolbuddy import spoolbuddy_watchdog
  10578. while True:
  10579. try:
  10580. await spoolbuddy_watchdog()
  10581. except asyncio.CancelledError:
  10582. break
  10583. except Exception as e:
  10584. logging.getLogger(__name__).warning("SpoolBuddy watchdog failed: %s", e)
  10585. await asyncio.sleep(SPOOLBUDDY_WATCHDOG_INTERVAL)
  10586. def start_spoolbuddy_watchdog():
  10587. global _spoolbuddy_watchdog_task
  10588. if _spoolbuddy_watchdog_task is None:
  10589. _spoolbuddy_watchdog_task = asyncio.create_task(_spoolbuddy_watchdog_loop())
  10590. logging.getLogger(__name__).info("SpoolBuddy watchdog started")
  10591. def stop_spoolbuddy_watchdog():
  10592. global _spoolbuddy_watchdog_task
  10593. if _spoolbuddy_watchdog_task:
  10594. _spoolbuddy_watchdog_task.cancel()
  10595. _spoolbuddy_watchdog_task = None
  10596. logging.getLogger(__name__).info("SpoolBuddy watchdog stopped")
  10597. # Dead-MQTT-session recovery
  10598. #
  10599. # check_staleness() covers the "connected but silent" half-broken session. It
  10600. # does nothing once ``state.connected`` is False, and paho's own auto-reconnect
  10601. # is the only thing left watching at that point. When paho stops making
  10602. # progress there is no backstop at all: the #2732 bundle has a P1S drop on a
  10603. # keep-alive timeout at 02:19 and not reconnect until 11:24 — nine hours
  10604. # offline with the UI open the whole time, recovered only when something
  10605. # happened to nudge it.
  10606. #
  10607. # This loop is that backstop. It only touches printers that had a working
  10608. # session and lost it, and only when the MQTT port still answers — a printer
  10609. # that is simply switched off is left to paho, since rebuilding a client
  10610. # against an unreachable host achieves nothing and would fill the log every
  10611. # night.
  10612. _connection_watchdog_task: asyncio.Task | None = None
  10613. CONNECTION_WATCHDOG_INTERVAL = 60
  10614. # How long a printer must have been silent before we stop trusting paho.
  10615. # Comfortably above STALE_TIMEOUT (60 s) and the max reconnect backoff (30 s),
  10616. # so a session that is recovering on its own is never interrupted.
  10617. CONNECTION_WATCHDOG_OFFLINE_GRACE = 300
  10618. # Per-printer floor between rebuild attempts.
  10619. CONNECTION_WATCHDOG_RETRY_INTERVAL = 300
  10620. _connection_watchdog_last_attempt: dict[int, float] = {}
  10621. async def _recover_dead_printer_sessions() -> int:
  10622. """Rebuild MQTT clients that have been offline too long to still be trying.
  10623. Returns the number of printers a rebuild was attempted for (for tests and
  10624. for the caller's logging). Never raises: one unreachable printer must not
  10625. stop the sweep for the rest of the farm.
  10626. """
  10627. logger = logging.getLogger(__name__)
  10628. from backend.app.services.printer_diagnostic import PORT_MQTT, check_port
  10629. now = time.monotonic()
  10630. recovered = 0
  10631. for printer_id, client in list(printer_manager._clients.items()):
  10632. try:
  10633. if client.state.connected:
  10634. _connection_watchdog_last_attempt.pop(printer_id, None)
  10635. continue
  10636. # Time since the last inbound message is the age of the last known
  10637. # good session — no extra bookkeeping needed, and it is the same
  10638. # clock is_stale() reads. 0 means this client has never had one:
  10639. # that is the initial-connect path, where paho retrying is the
  10640. # correct and only behaviour, so leave it be.
  10641. last_msg = client._last_message_time
  10642. if not last_msg:
  10643. continue
  10644. offline_for = time.time() - last_msg
  10645. if offline_for < CONNECTION_WATCHDOG_OFFLINE_GRACE:
  10646. continue
  10647. last_attempt = _connection_watchdog_last_attempt.get(printer_id)
  10648. if last_attempt is not None and now - last_attempt < CONNECTION_WATCHDOG_RETRY_INTERVAL:
  10649. continue
  10650. if not await check_port(client.ip_address, PORT_MQTT):
  10651. # Switched off, unplugged, or off the network. Paho's retry is
  10652. # the right handler; say so at debug level and move on.
  10653. logger.debug(
  10654. "[#2732] Printer %s offline for %.0fs and its MQTT port is not answering "
  10655. "— leaving the reconnect to paho",
  10656. printer_id,
  10657. offline_for,
  10658. )
  10659. _connection_watchdog_last_attempt[printer_id] = now
  10660. continue
  10661. _connection_watchdog_last_attempt[printer_id] = now
  10662. recovered += 1
  10663. logger.warning(
  10664. "[#2732] Printer %s has been offline for %.0fs but answers on MQTT port %d — "
  10665. "rebuilding the client with a fresh session (last connect error: %s)",
  10666. printer_id,
  10667. offline_for,
  10668. PORT_MQTT,
  10669. client.last_connect_error or "none recorded",
  10670. )
  10671. # Async context, so this takes the hard-reset path: fresh client_id,
  10672. # paho's QoS 1 queue dropped. That matters — a project_file left
  10673. # unacked on the dead session would otherwise replay into the new
  10674. # one and trip 0500_4003 on the printer (#1136).
  10675. client.force_reconnect_stale_session(f"offline for {offline_for:.0f}s, port still answering")
  10676. except Exception as e:
  10677. logger.warning("[#2732] Connection watchdog failed for printer %s: %s", printer_id, e)
  10678. return recovered
  10679. async def _connection_watchdog_loop():
  10680. logger = logging.getLogger(__name__)
  10681. # Let the initial connects settle before judging anyone offline.
  10682. await asyncio.sleep(CONNECTION_WATCHDOG_OFFLINE_GRACE)
  10683. while True:
  10684. try:
  10685. await _recover_dead_printer_sessions()
  10686. except asyncio.CancelledError:
  10687. break
  10688. except Exception as e:
  10689. logger.warning("Connection watchdog sweep failed: %s", e)
  10690. await asyncio.sleep(CONNECTION_WATCHDOG_INTERVAL)
  10691. def start_connection_watchdog():
  10692. global _connection_watchdog_task
  10693. if _connection_watchdog_task is None:
  10694. _connection_watchdog_task = asyncio.create_task(_connection_watchdog_loop())
  10695. logging.getLogger(__name__).info("Printer connection watchdog started")
  10696. def stop_connection_watchdog():
  10697. global _connection_watchdog_task
  10698. if _connection_watchdog_task:
  10699. _connection_watchdog_task.cancel()
  10700. _connection_watchdog_task = None
  10701. _connection_watchdog_last_attempt.clear()
  10702. logging.getLogger(__name__).info("Printer connection watchdog stopped")
  10703. # Camera stream orphan cleanup
  10704. _camera_cleanup_task: asyncio.Task | None = None
  10705. CAMERA_CLEANUP_INTERVAL = 60
  10706. # Every this many camera cleanups (about an hour), the timelapse janitor too:
  10707. # kept frames of a failed stitch go after a day, not at the next restart.
  10708. TIMELAPSE_CLEANUP_EVERY = 60
  10709. async def _camera_cleanup_loop():
  10710. """Periodically clean up orphaned ffmpeg processes and timelapse leftovers."""
  10711. from backend.app.api.routes.camera import cleanup_orphaned_streams
  10712. from backend.app.services.layer_timelapse import cleanup_orphaned_timelapse_sessions
  10713. rounds = 0
  10714. while True:
  10715. try:
  10716. rounds += 1
  10717. await cleanup_orphaned_streams()
  10718. except asyncio.CancelledError:
  10719. break
  10720. except Exception as e:
  10721. logging.getLogger(__name__).warning("Camera stream cleanup failed: %s", e)
  10722. try:
  10723. if rounds % TIMELAPSE_CLEANUP_EVERY == 0:
  10724. await asyncio.to_thread(cleanup_orphaned_timelapse_sessions)
  10725. except asyncio.CancelledError:
  10726. break
  10727. except Exception as e:
  10728. logging.getLogger(__name__).warning("Timelapse frames cleanup failed: %s", e)
  10729. await asyncio.sleep(CAMERA_CLEANUP_INTERVAL)
  10730. def start_camera_cleanup():
  10731. global _camera_cleanup_task
  10732. if _camera_cleanup_task is None:
  10733. _camera_cleanup_task = asyncio.create_task(_camera_cleanup_loop())
  10734. logging.getLogger(__name__).info("Camera stream cleanup started")
  10735. def stop_camera_cleanup():
  10736. global _camera_cleanup_task
  10737. if _camera_cleanup_task:
  10738. _camera_cleanup_task.cancel()
  10739. _camera_cleanup_task = None
  10740. logging.getLogger(__name__).info("Camera stream cleanup stopped")
  10741. # ---------------------------------------------------------------------------
  10742. # Expected-print TTL eviction
  10743. # ---------------------------------------------------------------------------
  10744. def _evict_stale_expected_prints() -> None:
  10745. """Remove entries from _expected_prints / _expected_print_creators that are
  10746. older than _EXPECTED_PRINT_TTL_SECONDS.
  10747. This prevents unbounded growth when a print is registered (via
  10748. register_expected_print) but on_print_start never fires — e.g. because the
  10749. printer disconnects, the app restarts, or the print is started directly from
  10750. the printer panel without going through the queue.
  10751. """
  10752. # Use monotonic time so the TTL is unaffected by system clock adjustments
  10753. # (e.g. NTP sync, DST changes).
  10754. # A print that ended without its archive being found at completion leaves
  10755. # its id here; only running prints are owned.
  10756. _owned_print_archive_ids.intersection_update(_active_prints.values())
  10757. cutoff = time.monotonic() - _EXPECTED_PRINT_TTL_SECONDS
  10758. stale_keys = [k for k, t in _expected_print_registered_at.items() if t < cutoff]
  10759. if not stale_keys:
  10760. return
  10761. evicted: set[tuple[int, int]] = set()
  10762. for key in stale_keys:
  10763. archive_id = _expected_prints.pop(key, None)
  10764. if archive_id is not None:
  10765. evicted.add((key[0], archive_id))
  10766. _expected_print_creators.pop(key, None)
  10767. _expected_print_registered_at.pop(key, None)
  10768. # Also clean up _print_ams_mappings and _print_plate_ids for archive_ids
  10769. # that have no remaining live keys in _expected_prints (all variants
  10770. # were just evicted) and are not printing right now. Print start pops
  10771. # only the names the printer reported, so the "<base>.gcode" variant
  10772. # outlives it; without the printing check its eviction two hours after
  10773. # dispatch threw away a running print's plate, AMS mapping and cost
  10774. # centre, which completion still reads (#3275).
  10775. for printer_id, archive_id in evicted:
  10776. if not _expected_or_printing_here(printer_id, archive_id):
  10777. _drop_dispatch(printer_id, archive_id)
  10778. logging.getLogger(__name__).info(
  10779. "Evicted %d stale expected-print entries (TTL=%ds)", len(stale_keys), _EXPECTED_PRINT_TTL_SECONDS
  10780. )
  10781. async def _expected_prints_cleanup_loop() -> None:
  10782. """Background task: periodically evict stale expected-print entries."""
  10783. while True:
  10784. try:
  10785. _evict_stale_expected_prints()
  10786. except asyncio.CancelledError:
  10787. raise
  10788. except Exception as e:
  10789. logging.getLogger(__name__).warning("Expected prints cleanup failed: %s", e)
  10790. await asyncio.sleep(_EXPECTED_PRINT_CLEANUP_INTERVAL)
  10791. def start_expected_prints_cleanup() -> None:
  10792. global _expected_prints_cleanup_task
  10793. if _expected_prints_cleanup_task is None:
  10794. _expected_prints_cleanup_task = asyncio.create_task(_expected_prints_cleanup_loop())
  10795. logging.getLogger(__name__).info("Expected prints cleanup started")
  10796. def stop_expected_prints_cleanup() -> None:
  10797. global _expected_prints_cleanup_task
  10798. if _expected_prints_cleanup_task:
  10799. _expected_prints_cleanup_task.cancel()
  10800. _expected_prints_cleanup_task = None
  10801. logging.getLogger(__name__).info("Expected prints cleanup stopped")
  10802. # ---------------------------------------------------------------------------
  10803. # L-2: Periodic auth-token cleanup (stale TOTP + expired revoked JTIs)
  10804. # ---------------------------------------------------------------------------
  10805. _auth_cleanup_task: asyncio.Task | None = None
  10806. _AUTH_CLEANUP_INTERVAL = 3600 # seconds (hourly)
  10807. async def _run_auth_cleanup() -> None:
  10808. """Single cleanup pass: remove stale TOTP records, expired revoked JTIs, and old rate-limit events."""
  10809. from backend.app.core.database import async_session
  10810. from backend.app.models.auth_ephemeral import AuthEphemeralToken, AuthRateLimitEvent
  10811. from backend.app.models.user_totp import UserTOTP
  10812. now = datetime.now(timezone.utc)
  10813. # Remove unconfirmed (is_enabled=False) TOTP records older than 1 hour.
  10814. try:
  10815. async with async_session() as db:
  10816. stale_cutoff = now - timedelta(hours=1)
  10817. result = await db.execute(
  10818. select(UserTOTP).where(
  10819. UserTOTP.is_enabled.is_(False),
  10820. UserTOTP.created_at < stale_cutoff,
  10821. )
  10822. )
  10823. stale_records = result.scalars().all()
  10824. if stale_records:
  10825. for rec in stale_records:
  10826. await db.delete(rec)
  10827. await db.commit()
  10828. logging.info("Auth cleanup: removed %d stale unconfirmed TOTP record(s)", len(stale_records))
  10829. except Exception as e:
  10830. logging.warning("Auth cleanup: failed to purge stale TOTP records: %s", e)
  10831. # Remove expired revoked-JTI entries (they are no longer needed once the
  10832. # original token's exp has passed — the token would be rejected by JWT
  10833. # signature verification regardless).
  10834. try:
  10835. async with async_session() as db:
  10836. await db.execute(
  10837. delete(AuthEphemeralToken).where(
  10838. AuthEphemeralToken.token_type == "revoked_jti",
  10839. AuthEphemeralToken.expires_at < now,
  10840. )
  10841. )
  10842. await db.commit()
  10843. except Exception as e:
  10844. logging.warning("Auth cleanup: failed to purge expired revoked JTIs: %s", e)
  10845. # L-R6-B: Purge AuthRateLimitEvent rows older than the lockout window (15 min).
  10846. # Events outside this window can never affect rate-limit decisions — they only
  10847. # consume DB space. Use the same window constant as the rate limiter so the
  10848. # two are always in sync.
  10849. try:
  10850. from backend.app.api.routes.mfa import LOCKOUT_WINDOW
  10851. async with async_session() as db:
  10852. await db.execute(
  10853. delete(AuthRateLimitEvent).where(
  10854. AuthRateLimitEvent.occurred_at < now - LOCKOUT_WINDOW,
  10855. )
  10856. )
  10857. await db.commit()
  10858. except Exception as e:
  10859. logging.warning("Auth cleanup: failed to purge stale rate-limit events: %s", e)
  10860. async def _auth_cleanup_loop() -> None:
  10861. """Periodic background task: run auth cleanup every hour."""
  10862. while True:
  10863. try:
  10864. await _run_auth_cleanup()
  10865. except asyncio.CancelledError:
  10866. break
  10867. except Exception as e:
  10868. logging.warning("Auth cleanup loop error: %s", e)
  10869. await asyncio.sleep(_AUTH_CLEANUP_INTERVAL)
  10870. def start_auth_cleanup() -> None:
  10871. global _auth_cleanup_task
  10872. if _auth_cleanup_task is None:
  10873. _auth_cleanup_task = asyncio.create_task(_auth_cleanup_loop())
  10874. logging.getLogger(__name__).info("Auth periodic cleanup started")
  10875. def stop_auth_cleanup() -> None:
  10876. global _auth_cleanup_task
  10877. if _auth_cleanup_task:
  10878. _auth_cleanup_task.cancel()
  10879. _auth_cleanup_task = None
  10880. logging.getLogger(__name__).info("Auth periodic cleanup stopped")
  10881. async def initialize_wled_printers(db: AsyncSession) -> None:
  10882. """Seed WLED settings and apply offline presets before MQTT connections start."""
  10883. from backend.app.api.routes.settings import get_setting, setting_is_true
  10884. from backend.app.models.printer import Printer
  10885. wled_manager.require_plate_clear = setting_is_true(await get_setting(db, "require_plate_clear"))
  10886. result = await db.execute(select(Printer.id, Printer.wled_config, Printer.is_active))
  10887. for printer_id, config, is_active in result.all():
  10888. wled_manager.configure_printer(printer_id, config)
  10889. if not is_active and config:
  10890. wled_manager.handle_offline(printer_id)
  10891. @asynccontextmanager
  10892. async def lifespan(app: FastAPI):
  10893. # Startup
  10894. # Install Windows-only asyncio Proactor cleanup-RST filter (#1113) before
  10895. # anything else can spawn tasks that might trip it.
  10896. from backend.app.core.asyncio_handlers import install_proactor_reset_filter, warn_if_running_on_uvloop
  10897. install_proactor_reset_filter()
  10898. # Before anything opens files or sockets in bulk: a soft limit of 1024 is
  10899. # what turned descriptor exhaustion into a corrupted database (#2883).
  10900. from backend.app.core.fd_limit import raise_open_file_limit
  10901. raise_open_file_limit()
  10902. # Before init_db, so the warning is near the top of the log rather than
  10903. # below a migration run. See warn_if_running_on_uvloop for what is at stake.
  10904. warn_if_running_on_uvloop()
  10905. await init_db()
  10906. # Browser download tokens expire after five minutes. Remove abandoned
  10907. # prepared ZIPs at startup as well as before each new preparation so a
  10908. # quiet appliance cannot retain an unusable bundle indefinitely.
  10909. try:
  10910. from backend.app.services.printer_media import prune_stale_printer_file_bundles
  10911. await prune_stale_printer_file_bundles()
  10912. except Exception as exc:
  10913. logging.warning("Failed to prune stale printer download bundles: %s", exc)
  10914. # After migrations, so the is_env_managed column exists. Never raises --
  10915. # a bad BAMBUDDY_OIDC_* value is logged and skipped rather than blocking
  10916. # startup (see apply_env_oidc_provider).
  10917. from backend.app.core.oidc_env import apply_env_oidc_provider
  10918. async with async_session() as oidc_db:
  10919. await apply_env_oidc_provider(oidc_db)
  10920. # Close out batches that finished before `completed` was a reachable status
  10921. # (#342). Without this the Batches tab opens on every batch created since
  10922. # the feature shipped, all still marked active. Never blocks startup.
  10923. try:
  10924. from backend.app.services.print_batch import backfill_batch_statuses
  10925. async with async_session() as batch_db:
  10926. await backfill_batch_statuses(batch_db)
  10927. except Exception as exc:
  10928. logging.warning("[BATCH] Startup status backfill failed: %s", exc)
  10929. # Each plate's estimate of archives made before they were kept (#3375),
  10930. # read from their 3MFs in the background: the readers use the whole
  10931. # file's figures until then.
  10932. global _plate_estimates_task
  10933. _plate_estimates_task = asyncio.create_task(_fill_plate_estimates())
  10934. # The G-code fingerprint of archives made before it was kept (#1634), read
  10935. # from their files in the background: until then a slice sent again from
  10936. # the slicer archives anew, as before.
  10937. global _gcode_hashes_task
  10938. _gcode_hashes_task = asyncio.create_task(_fill_gcode_hashes())
  10939. # Register an app-scoped httpx client for Bambu Cloud services so
  10940. # per-request BambuCloudService instances reuse the same connection pool
  10941. # (important for routes like /cloud/filament-info that chain many
  10942. # get_setting_detail calls). The shared client stores no region/token
  10943. # state, so the per-request ownership pattern that fixed the region-bleed
  10944. # bug is preserved.
  10945. import httpx as _httpx
  10946. from backend.app.services.bambu_cloud import set_shared_http_client
  10947. from backend.app.services.model_providers.makerworld.service import (
  10948. set_shared_http_client as set_shared_makerworld_http_client,
  10949. )
  10950. from backend.app.services.orca_cloud import (
  10951. set_shared_http_client as set_shared_orca_http_client,
  10952. )
  10953. _shared_cloud_http_client = _httpx.AsyncClient(timeout=30.0)
  10954. set_shared_http_client(_shared_cloud_http_client)
  10955. # Reuse the same connection pool for MakerWorld — different host, same
  10956. # keep-alive pool saves a TLS handshake per request.
  10957. set_shared_makerworld_http_client(_shared_cloud_http_client)
  10958. # Same for Orca Cloud — without this the per-request OrcaCloudService()
  10959. # each spun up (and never closed) its own client, leaking sockets.
  10960. set_shared_orca_http_client(_shared_cloud_http_client)
  10961. # Fix queue items stuck with invalid "aborted" status (should be "cancelled").
  10962. # This can happen when a print was cancelled mid-print on versions before this fix.
  10963. try:
  10964. async with async_session() as db:
  10965. from backend.app.models.print_queue import PrintQueueItem
  10966. result = await db.execute(select(PrintQueueItem).where(PrintQueueItem.status == "aborted"))
  10967. aborted_items = result.scalars().all()
  10968. if aborted_items:
  10969. for item in aborted_items:
  10970. item.status = "cancelled"
  10971. await db.commit()
  10972. logging.info("Fixed %d queue item(s) with invalid 'aborted' status → 'cancelled'", len(aborted_items))
  10973. except Exception as e:
  10974. logging.warning("Failed to fix aborted queue items: %s", e)
  10975. # Restore debug logging state from previous session
  10976. await init_debug_logging()
  10977. # Set up printer manager callbacks
  10978. loop = asyncio.get_event_loop()
  10979. printer_manager.set_event_loop(loop)
  10980. printer_manager.set_status_change_callback(on_printer_status_change)
  10981. printer_manager.set_print_start_callback(on_print_start)
  10982. printer_manager.set_print_complete_callback(on_print_complete)
  10983. printer_manager.set_print_running_observed_callback(on_print_running_observed)
  10984. printer_manager.set_finish_photo_moment_callback(on_finish_photo_moment)
  10985. printer_manager.set_ams_change_callback(on_ams_change)
  10986. printer_manager.set_fts_inlet_change_callback(on_fts_inlet_change)
  10987. # Rehydrate persisted awaiting-plate-clear gate (#961) so prompts survive restarts
  10988. await printer_manager.load_awaiting_plate_clear_from_db()
  10989. # Before any printer connects, or its first report re-sends every fault
  10990. # still listed (#3486).
  10991. await _load_notified_hms_faults()
  10992. # WLED is entirely optional. Seed its in-memory per-printer configuration
  10993. # before MQTT connections begin emitting status callbacks.
  10994. async with async_session() as db:
  10995. await initialize_wled_printers(db)
  10996. # Layer change callback for external camera timelapse
  10997. async def on_layer_change(printer_id: int, layer_num: int):
  10998. """Capture timelapse frame on layer change + first layer notification."""
  10999. from backend.app.services.layer_timelapse import on_layer_change as tl_layer_change
  11000. await tl_layer_change(printer_id, layer_num)
  11001. # #1867: bank a recent in-print frame so the finish-photo path has a
  11002. # pre-End-G-code image to use instead of a live grab of a swapped plate.
  11003. # #2547 added `on_print_progress` as a second driver — this one alone
  11004. # stops firing once the final layer begins.
  11005. await _maybe_bank_inprint_frame(printer_id, layer_num)
  11006. # First layer complete notification (layer_num >= 2 means layer 1 is done).
  11007. # Gate on actual printing state — Bambu firmware ticks layer_num during
  11008. # the pre-print calibration sequence (homing / mesh-level / bed scan /
  11009. # nozzle clean), so a bare layer_num check can fire minutes before the
  11010. # first real extrusion. We require gcode_state == RUNNING and
  11011. # mc_print_sub_stage in (0 = "Printing", None) so calibration sub-stages
  11012. # (1, 9, 14, ...) are excluded. The window widens to [2, 10] because if
  11013. # the layer counter advanced past 2 during PREPARE, the next on_layer_change
  11014. # edge fires later; _first_layer_notified stays clear until we actually send
  11015. # so a deferred re-evaluation can win. See issue #1837.
  11016. if 2 <= layer_num <= 10 and not _first_layer_notified.get(printer_id, False):
  11017. client = printer_manager.get_client(printer_id)
  11018. state = client.state if client else None
  11019. if not state or state.state != "RUNNING":
  11020. return
  11021. if state.mc_print_sub_stage not in (None, 0):
  11022. return
  11023. _first_layer_notified[printer_id] = True
  11024. try:
  11025. async with async_session() as db:
  11026. from backend.app.models.printer import Printer
  11027. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  11028. printer = result.scalar_one_or_none()
  11029. if not printer:
  11030. return
  11031. printer_name = printer.name
  11032. filename = (state.subtask_name or state.gcode_file or "Unknown") if state else "Unknown"
  11033. total_layers = state.total_layers if state else 0
  11034. image_data = await _capture_snapshot_for_notification(
  11035. printer_id, printer, logging.getLogger(__name__)
  11036. )
  11037. await notification_service.on_first_layer_complete(
  11038. printer_id, printer_name, filename, total_layers, db, image_data=image_data
  11039. )
  11040. except Exception as e:
  11041. logging.getLogger(__name__).warning("First layer notification failed: %s", e)
  11042. printer_manager.set_layer_change_callback(on_layer_change)
  11043. async def on_print_progress(printer_id: int, percent: int):
  11044. """#2547: keep the in-print frame bank fresh through the final layer.
  11045. `on_layer_change` stops the moment the last layer starts, which on the
  11046. H2C capture that closed #2547 left the bank stale for the three minutes
  11047. that layer took. Progress is the only field that keeps advancing there,
  11048. and it freezes before the End G-code runs — so banking on it stays
  11049. inside the print and never sees a swapped plate.
  11050. """
  11051. client = printer_manager.get_client(printer_id)
  11052. state = client.state if client else None
  11053. await _maybe_bank_inprint_frame(printer_id, state.layer_num if state else 0)
  11054. printer_manager.set_print_progress_callback(on_print_progress)
  11055. # Event-driven bed cooldown: fires whenever bed_temper arrives via MQTT
  11056. async def on_bed_temp_update(printer_id: int, bed_temp: float):
  11057. waiter = _bed_cool_waiters.get(printer_id)
  11058. if not waiter:
  11059. return
  11060. threshold = waiter["threshold"]
  11061. if bed_temp > threshold:
  11062. return
  11063. # Bed is at or below threshold — fire notification and remove waiter
  11064. waiter_info = _bed_cool_waiters.pop(printer_id, None)
  11065. if not waiter_info:
  11066. return # Another callback already handled it
  11067. bed_cool_logger = logging.getLogger(__name__)
  11068. bed_cool_logger.info(
  11069. "[BED-COOL] Bed cooled to %.1f°C on printer %s (threshold: %.0f°C)",
  11070. bed_temp,
  11071. printer_id,
  11072. threshold,
  11073. )
  11074. try:
  11075. printer_info = printer_manager.get_printer(printer_id)
  11076. p_name = printer_info.name if printer_info else "Unknown"
  11077. async with async_session() as db:
  11078. await notification_service.on_bed_cooled(
  11079. printer_id=printer_id,
  11080. printer_name=p_name,
  11081. bed_temp=bed_temp,
  11082. threshold=threshold,
  11083. filename=waiter_info["filename"],
  11084. db=db,
  11085. )
  11086. except Exception as e:
  11087. bed_cool_logger.warning("[BED-COOL] Failed to send notification: %s", e)
  11088. printer_manager.set_bed_temp_update_callback(on_bed_temp_update)
  11089. async def on_drying_complete(printer_id: int, ams_id: int):
  11090. """Smart-plug auto-off-after-drying trigger (#1349).
  11091. Fires once per AMS unit when ``dry_time`` falls from >0 to 0. The
  11092. manager walks all plugs linked to this printer and turns off only
  11093. the ones with ``auto_off_after_drying`` enabled, after their
  11094. per-plug delay. Multiple AMS units finishing close together (e.g. a
  11095. dual-AMS dry that ends within the same MQTT push) call this once
  11096. per unit — the manager's ``_cancel_pending_off`` collapses
  11097. repeated scheduling on the same plug to one timer, so duplicate
  11098. fires are safe.
  11099. """
  11100. try:
  11101. async with async_session() as db:
  11102. await smart_plug_manager.on_drying_complete(printer_id, db)
  11103. except Exception as e:
  11104. logging.getLogger(__name__).warning(
  11105. "Failed to schedule auto-off-after-drying for printer %d (AMS %d): %s",
  11106. printer_id,
  11107. ams_id,
  11108. e,
  11109. )
  11110. printer_manager.set_drying_complete_callback(on_drying_complete)
  11111. async def on_drying_cycle_end(printer_id: int, cycle):
  11112. """Mark the spools in an AMS that just finished drying as dried (#2863)."""
  11113. from backend.app.services.spool_drying import record_drying_cycle
  11114. try:
  11115. async with async_session() as db:
  11116. await record_drying_cycle(db, printer_id, cycle)
  11117. except Exception as e:
  11118. logging.getLogger(__name__).warning(
  11119. "Failed to record drying for printer %d (AMS %d): %s",
  11120. printer_id,
  11121. cycle.ams_id,
  11122. e,
  11123. )
  11124. printer_manager.set_drying_cycle_end_callback(on_drying_cycle_end)
  11125. async def on_assignment_verified(printer_id: int, ams_id: int, tray_id: int, verified: bool, detail: dict):
  11126. """Surface the read-back result of a spool assignment to the UI (#2582).
  11127. The MQTT client confirms (or fails to confirm) that the tray telemetry
  11128. echoed back the filament id we pushed. We relay that as a websocket
  11129. event so the frontend can toast "loaded" / "assignment didn't take"
  11130. instead of the historic silent fire-and-forget, which made the
  11131. AMS→Studio hand-off feel random to users.
  11132. """
  11133. try:
  11134. from backend.app.services.spool_assignment_notifications import (
  11135. _slot_label_from_global_tray,
  11136. )
  11137. layout = printer_tray_layout(printer_id, printer_manager)
  11138. global_id = layout.tray_from_slot_key(ams_id, tray_id)
  11139. slot_label = _slot_label_from_global_tray(global_id, layout)
  11140. printer_info = printer_manager.get_printer(printer_id)
  11141. printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  11142. await ws_manager.broadcast(
  11143. {
  11144. "type": "spool_assignment_verified",
  11145. "printer_id": printer_id,
  11146. "printer_name": printer_name,
  11147. "ams_id": ams_id,
  11148. "tray_id": tray_id,
  11149. "slot": slot_label,
  11150. "verified": verified,
  11151. # Present on success: False means the filament setting landed
  11152. # but the K-profile (cali_idx) did not — the reporter's exact
  11153. # "loaded but no flow profile" symptom.
  11154. "kprofile_applied": detail.get("kprofile_applied", True),
  11155. }
  11156. )
  11157. except Exception as e:
  11158. logging.getLogger(__name__).warning(
  11159. "Failed to broadcast assignment verification for printer %d AMS%d-T%d: %s",
  11160. printer_id,
  11161. ams_id,
  11162. tray_id,
  11163. e,
  11164. )
  11165. printer_manager.set_assignment_verified_callback(on_assignment_verified)
  11166. async def on_tray_change(printer_id: int, tray_global: int, layer_num: int):
  11167. """Persist a mid-print tray change for completion-time attribution.
  11168. AMS filament backup switches trays without telling the slicer, so the
  11169. tray-change log is the only record of which spool fed which layers.
  11170. Keeping it only in memory meant a restart mid-print charged everything
  11171. to the tray that finished the job.
  11172. """
  11173. try:
  11174. from backend.app.services.usage_tracker import record_tray_change
  11175. async with async_session() as db:
  11176. await record_tray_change(db, printer_id, tray_global, layer_num)
  11177. except Exception as e:
  11178. logging.getLogger(__name__).warning(
  11179. "Failed to persist tray change for printer %d (tray=%d, layer=%d): %s",
  11180. printer_id,
  11181. tray_global,
  11182. layer_num,
  11183. e,
  11184. )
  11185. printer_manager.set_tray_change_callback(on_tray_change)
  11186. # Initialize MQTT relay from settings
  11187. async with async_session() as db:
  11188. from backend.app.api.routes.settings import get_setting
  11189. mqtt_settings = {
  11190. "mqtt_enabled": (await get_setting(db, "mqtt_enabled") or "false") == "true",
  11191. "mqtt_broker": await get_setting(db, "mqtt_broker") or "",
  11192. "mqtt_port": int(await get_setting(db, "mqtt_port") or "1883"),
  11193. "mqtt_username": await get_setting(db, "mqtt_username") or "",
  11194. "mqtt_password": await get_setting(db, "mqtt_password") or "",
  11195. "mqtt_topic_prefix": await get_setting(db, "mqtt_topic_prefix") or "bambuddy",
  11196. "mqtt_use_tls": (await get_setting(db, "mqtt_use_tls") or "false") == "true",
  11197. "panda_breath_enabled": (await get_setting(db, "panda_breath_enabled") or "false") == "true",
  11198. }
  11199. await mqtt_relay.configure(mqtt_settings)
  11200. # Restore MQTT smart plug subscriptions
  11201. if mqtt_settings.get("mqtt_enabled"):
  11202. from backend.app.models.smart_plug import SmartPlug
  11203. from backend.app.services.mqtt_smart_plug import subscribe_plug_to_mqtt
  11204. result = await db.execute(select(SmartPlug).where(SmartPlug.plug_type == "mqtt"))
  11205. mqtt_plugs = result.scalars().all()
  11206. restored = 0
  11207. for plug in mqtt_plugs:
  11208. if subscribe_plug_to_mqtt(mqtt_relay.smart_plug_service, plug):
  11209. restored += 1
  11210. if restored:
  11211. logging.info("Restored %s MQTT smart plug subscriptions", restored)
  11212. # Connect to all active printers
  11213. async with async_session() as db:
  11214. await init_printer_connections(db)
  11215. # Auto-connect to Spoolman if enabled
  11216. async with async_session() as db:
  11217. from backend.app.api.routes.settings import get_setting
  11218. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  11219. spoolman_url = await get_setting(db, "spoolman_url")
  11220. if spoolman_enabled and spoolman_enabled.lower() == "true" and spoolman_url:
  11221. try:
  11222. client = await init_spoolman_client(spoolman_url)
  11223. if await client.health_check():
  11224. logging.info("Auto-connected to Spoolman at %s", spoolman_url)
  11225. # Ensure the 'tag' extra field exists for RFID/UUID storage
  11226. field_ok = await client.ensure_tag_extra_field()
  11227. if not field_ok:
  11228. logging.error("Spoolman tag extra field registration failed — NFC tag links may not persist")
  11229. # Register the BambuStudio slicer-preset fields used by the
  11230. # spool-edit / assign flow. Spoolman rejects PATCHes with
  11231. # unknown extra keys, so these must exist before any update
  11232. # that touches them.
  11233. for field_name in ("bambu_slicer_filament", "bambu_slicer_filament_name"):
  11234. if not await client.ensure_extra_field(field_name):
  11235. logging.warning(
  11236. "Spoolman extra field %r registration failed — "
  11237. "spool slicer-preset edits will return 502",
  11238. field_name,
  11239. )
  11240. else:
  11241. logging.warning("Spoolman at %s is not reachable", spoolman_url)
  11242. except Exception as e:
  11243. logging.warning("Failed to auto-connect to Spoolman: %s", e)
  11244. # Start the print scheduler
  11245. spawn_background_task(print_scheduler.run(), name="print-scheduler")
  11246. # Start the smart plug scheduler for time-based on/off
  11247. smart_plug_manager.start_scheduler()
  11248. # Start the Home Assistant sensor poller (#1148)
  11249. ha_sensor_manager.start()
  11250. location_ha_sensor_manager.start()
  11251. # Resume any pending auto-offs that were interrupted by restart
  11252. await smart_plug_manager.resume_pending_auto_offs()
  11253. # Start the notification digest scheduler
  11254. notification_service.start_digest_scheduler()
  11255. notify_live_activities.start()
  11256. notify_widgets.start()
  11257. # Start the Telegram reaction pollers (#3046), one per bot token used by a
  11258. # provider in reactions/both mode; the notification routes resync them.
  11259. # Never fatal: a bad provider row or a DB hiccup here costs reactions
  11260. # until the next provider save, not the whole startup.
  11261. try:
  11262. await telegram_reaction_poller.start()
  11263. except Exception as e:
  11264. logging.warning("Telegram reaction poller did not start: %s", e)
  11265. # Start the GitHub backup scheduler
  11266. await github_backup_service.start_scheduler()
  11267. # Start the local backup scheduler
  11268. await local_backup_service.start_scheduler()
  11269. await obico_detection_service.start()
  11270. await octoeverywhere_detection_service.start()
  11271. # Start the library trash sweeper (#1008)
  11272. await library_trash_service.start_scheduler()
  11273. # Start the archive auto-purge sweeper (#1008 follow-up)
  11274. await archive_purge_service.start_scheduler()
  11275. # Start AMS history recording
  11276. start_ams_history_recording()
  11277. # Start printer sensor (nozzle / bed / chamber) history recording
  11278. start_printer_sensor_history_recording()
  11279. # Start printer runtime tracking
  11280. start_runtime_tracking()
  11281. # Start SpoolBuddy device watchdog
  11282. start_spoolbuddy_watchdog()
  11283. # Start camera stream orphan cleanup
  11284. start_camera_cleanup()
  11285. # Start the backstop for MQTT sessions paho has stopped recovering (#2732)
  11286. start_connection_watchdog()
  11287. # One-shot sweep for timelapse session directories orphaned by a crash
  11288. # or restart that happened mid-print (in-memory session tracking can't
  11289. # survive that, and nothing else reaps the leftover frames/output file)
  11290. try:
  11291. from backend.app.services.layer_timelapse import cleanup_orphaned_timelapse_sessions
  11292. removed = cleanup_orphaned_timelapse_sessions()
  11293. if removed:
  11294. logging.getLogger(__name__).info("Removed %d orphaned timelapse session artifact(s)", removed)
  11295. except Exception as e:
  11296. logging.getLogger(__name__).warning("Orphaned timelapse session cleanup failed: %s", e)
  11297. # Start expected-print TTL eviction (prevents memory leak when prints are
  11298. # registered but on_print_start never fires)
  11299. start_expected_prints_cleanup()
  11300. # L-2: Start periodic auth cleanup (stale TOTP + expired revoked JTIs)
  11301. start_auth_cleanup()
  11302. # Maintainer announcements: a signed feed fetched from GitHub every few hours.
  11303. from backend.app.services import announcements as announcements_service
  11304. announcements_service.start()
  11305. from backend.app.services.printer_media import start_printer_download_cleanup
  11306. start_printer_download_cleanup()
  11307. # Event-loop stall watchdog: dumps all thread stacks to stderr if the loop
  11308. # freezes (#1486 — silent "container hangs after adding a printer" reports).
  11309. from backend.app.services.loop_watchdog import start_loop_watchdog
  11310. start_loop_watchdog()
  11311. # Initialize virtual printer manager and sync from DB
  11312. from backend.app.services.virtual_printer import virtual_printer_manager
  11313. virtual_printer_manager.set_session_factory(async_session)
  11314. virtual_printer_manager.set_printer_manager(printer_manager)
  11315. try:
  11316. await virtual_printer_manager.sync_from_db()
  11317. logging.info("Virtual printer manager synced from database")
  11318. except Exception as e:
  11319. logging.warning("Failed to sync virtual printers: %s", e)
  11320. yield
  11321. # Shutdown
  11322. print_scheduler.stop()
  11323. smart_plug_manager.stop_scheduler()
  11324. ha_sensor_manager.stop()
  11325. location_ha_sensor_manager.stop()
  11326. notification_service.stop_digest_scheduler()
  11327. await notify_live_activities.close()
  11328. await notify_widgets.close()
  11329. await telegram_reaction_poller.aclose()
  11330. github_backup_service.stop_scheduler()
  11331. local_backup_service.stop_scheduler()
  11332. library_trash_service.stop_scheduler()
  11333. archive_purge_service.stop_scheduler()
  11334. obico_detection_service.stop()
  11335. octoeverywhere_detection_service.stop()
  11336. stop_ams_history_recording()
  11337. stop_printer_sensor_history_recording()
  11338. stop_runtime_tracking()
  11339. stop_spoolbuddy_watchdog()
  11340. stop_camera_cleanup()
  11341. stop_connection_watchdog()
  11342. from backend.app.services.loop_watchdog import stop_loop_watchdog
  11343. stop_loop_watchdog()
  11344. # Tear down all camera fan-out broadcasters (#1089) so subscribers exit
  11345. # cleanly rather than waiting on a queue that nothing will ever fill.
  11346. try:
  11347. from backend.app.services.camera_fanout import shutdown_all_broadcasters
  11348. await shutdown_all_broadcasters()
  11349. except Exception as e:
  11350. logging.warning("Failed to shut down camera broadcasters: %s", e)
  11351. stop_expected_prints_cleanup()
  11352. stop_auth_cleanup()
  11353. if _plate_estimates_task is not None and not _plate_estimates_task.done():
  11354. _plate_estimates_task.cancel()
  11355. if _gcode_hashes_task is not None and not _gcode_hashes_task.done():
  11356. _gcode_hashes_task.cancel()
  11357. from backend.app.services import announcements as announcements_service
  11358. announcements_service.stop()
  11359. from backend.app.services.printer_media import stop_printer_download_cleanup
  11360. await stop_printer_download_cleanup()
  11361. # Stops a timelapse editor's ffmpeg rather than leaving it to the OS.
  11362. from backend.app.services import timelapse_jobs
  11363. await timelapse_jobs.cancel_all()
  11364. printer_manager.disconnect_all()
  11365. slot_unlink_grace.reset()
  11366. await close_spoolman_client()
  11367. # Stop all virtual printer services
  11368. await virtual_printer_manager.stop_all()
  11369. await mqtt_smart_plug_service.disconnect(timeout=2)
  11370. from backend.app.services.panda_breath import panda_breath_service
  11371. await panda_breath_service.disconnect(timeout=2)
  11372. await mqtt_relay.disconnect(timeout=2)
  11373. await wled_manager.shutdown()
  11374. # Drop the shared Bambu Cloud HTTP client we registered at startup.
  11375. set_shared_http_client(None)
  11376. set_shared_makerworld_http_client(None)
  11377. set_shared_orca_http_client(None)
  11378. await _shared_cloud_http_client.aclose()
  11379. # Checkpoint WAL (SQLite only) and close all database connections
  11380. from backend.app.core.db_dialect import is_sqlite
  11381. if is_sqlite():
  11382. try:
  11383. async with engine.begin() as conn:
  11384. await conn.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
  11385. logging.info("WAL checkpoint completed")
  11386. except Exception as e:
  11387. logging.warning("WAL checkpoint failed: %s", e)
  11388. await engine.dispose()
  11389. app = FastAPI(
  11390. title=app_settings.app_name,
  11391. description="Archive and manage Bambu Lab 3MF files",
  11392. version=APP_VERSION,
  11393. lifespan=lifespan,
  11394. )
  11395. # Added first so it sits innermost, next to the routes: a refused header's value
  11396. # (often a stored token) masked in every JSON body. See core.response_redaction.
  11397. app.add_middleware(RefusedHeaderResponseMask)
  11398. def _json_safe(value):
  11399. """The validation errors with every non-finite number written as text."""
  11400. if isinstance(value, float) and not math.isfinite(value):
  11401. return str(value)
  11402. if isinstance(value, dict):
  11403. return {key: _json_safe(item) for key, item in value.items()}
  11404. if isinstance(value, list | tuple):
  11405. return [_json_safe(item) for item in value]
  11406. return value
  11407. @app.exception_handler(RequestValidationError)
  11408. async def _validation_error(_request, exc: RequestValidationError) -> JSONResponse:
  11409. """FastAPI's own 422, except that it survives a non-finite number.
  11410. A body may carry ``Infinity`` or ``NaN`` (Python's JSON reader accepts
  11411. them), and the default handler echoes the input back in its errors,
  11412. which JSON cannot hold: the request ended in an error instead of a 422.
  11413. """
  11414. return JSONResponse(status_code=422, content={"detail": _json_safe(jsonable_encoder(exc.errors()))})
  11415. # =============================================================================
  11416. # Authentication Middleware - Secures ALL API routes by default
  11417. # =============================================================================
  11418. # Public routes that don't require authentication even when auth is enabled
  11419. PUBLIC_API_ROUTES = {
  11420. # Auth routes needed before/during login
  11421. "/api/v1/auth/status",
  11422. "/api/v1/auth/login",
  11423. "/api/v1/auth/setup", # Needed for initial setup and recovery
  11424. # Advanced auth status needed for login page
  11425. "/api/v1/auth/advanced-auth/status",
  11426. "/api/v1/auth/forgot-password", # Password reset for advanced auth
  11427. "/api/v1/auth/forgot-password/confirm", # Complete password reset with token (H-6)
  11428. # 2FA routes that are called BEFORE a JWT is issued (pre-auth flow)
  11429. "/api/v1/auth/2fa/verify", # Exchange pre_auth_token + 2FA code for JWT
  11430. "/api/v1/auth/2fa/email/send", # Send OTP email (pre_auth_token based)
  11431. # OIDC routes that must be reachable without a JWT
  11432. "/api/v1/auth/oidc/providers", # Public list of enabled providers
  11433. "/api/v1/auth/oidc/callback", # Redirect target from OIDC provider
  11434. "/api/v1/auth/oidc/exchange", # Exchange short-lived OIDC token for JWT
  11435. # Connected apps: the app's server swaps a code for the user's identity,
  11436. # authenticated by its client secret rather than a login token.
  11437. "/api/v1/connect/token",
  11438. # Version check for updates (no sensitive data)
  11439. "/api/v1/updates/version",
  11440. # Metrics endpoint handles its own prometheus_token authentication
  11441. "/api/v1/metrics",
  11442. # Appliance bootstrap (#1589 follow-up): the SPA's i18n setup polls
  11443. # this BEFORE a JWT is available to pick up the firstboot wizard's
  11444. # hostname / timezone / locale and the chrony NTP-gate state. The
  11445. # response contains user-set defaults and a public sync flag — no
  11446. # secrets. Without this entry the global auth middleware returns 401
  11447. # before the route handler runs, regardless of the route's own
  11448. # "no auth required" intent.
  11449. "/api/v1/system/appliance",
  11450. # Cam Wall kiosk feed (#2531): a TV or Pi in kiosk mode has no login, so it
  11451. # authenticates with a long-lived ``camwall``-scoped token in the query
  11452. # string — exactly like the camera streams two lists below, and for the same
  11453. # reason (no header to put a JWT in). "Public" here only means the middleware
  11454. # steps aside; the route still runs RequireCamWallTokenIfAuthEnabled, which
  11455. # rejects an absent, expired, revoked, or wrong-scoped token. In particular a
  11456. # plain ``camera_stream`` token does NOT open this door.
  11457. "/api/v1/camwall/printers",
  11458. # Overlay branding: the route enforces overlay-scoped token authentication.
  11459. "/api/v1/overlay-branding/logo",
  11460. }
  11461. # Route prefixes that are public (for routes with dynamic segments)
  11462. PUBLIC_API_PREFIXES = [
  11463. # WebSocket connections handle their own auth
  11464. "/api/v1/ws",
  11465. # OIDC authorize redirects — include provider_id in path
  11466. "/api/v1/auth/oidc/authorize/",
  11467. # One-tap outcome-verdict links from push notifications (#1898). Tapped on
  11468. # a phone with no session, so no header can carry a JWT — the single-use
  11469. # capability token in the path IS the credential (same reasoning as the
  11470. # /dl/ slicer downloads below). The route grants nothing beyond writing
  11471. # good/reject on the one archive the token was minted for.
  11472. "/api/v1/archives/confirm/",
  11473. ]
  11474. # Route patterns that are public (read-only display data)
  11475. # These are checked with "in path" - needed because browsers load images/videos
  11476. # via <img src> and <video src> which don't include Authorization headers
  11477. PUBLIC_API_PATTERNS = [
  11478. # Thumbnails
  11479. "/thumbnail", # /archives/{id}/thumbnail, /library/files/{id}/thumbnail
  11480. "/plate-thumbnail/", # /archives/{id}/plate-thumbnail/{plate_id}
  11481. # Images and media
  11482. "/photos/", # /archives/{id}/photos/{filename}
  11483. "/project-image/", # /archives/{id}/project-image/{path}
  11484. "/qrcode", # /archives/{id}/qrcode
  11485. "/timelapse", # /archives/{id}/timelapse (video)
  11486. "/cover", # /printers/{id}/cover
  11487. "/icon", # /external-links/{id}/icon
  11488. # Camera (streams loaded via <img> tag)
  11489. "/camera/stream", # /printers/{id}/camera/stream
  11490. "/camera/snapshot", # /printers/{id}/camera/snapshot
  11491. # Streaming-overlay status feed (#2613): OBS loads /overlay/{id} with no login
  11492. # and this backs it, authenticated by an ``overlay``-scoped token in the query
  11493. # string (same reasoning as the camera streams above — no header to carry a
  11494. # JWT). "Public" only means the middleware steps aside; the route still runs
  11495. # RequireOverlayTokenIfAuthEnabled, which rejects an absent, expired, revoked,
  11496. # or wrong-scoped token — a camwall or camera_stream token does NOT open it.
  11497. "/overlay-status", # /printers/{id}/overlay-status
  11498. # Slicer token-authenticated downloads — protocol handlers (bambustudioopen://,
  11499. # orcaslicer://) cannot send auth headers. These endpoints validate a short-lived
  11500. # download token in the URL path instead.
  11501. "/dl/", # /archives/{id}/dl/{token}/{filename}, /library/files/{id}/dl/{token}/{filename}
  11502. # Same family, but the segment is "source-dl" — which does NOT contain "/dl/",
  11503. # and these patterns match by substring. Without its own entry the middleware
  11504. # 401s the slicer's header-less request before the route's token check runs,
  11505. # so "Open source 3MF in slicer" failed whenever auth was enabled (#3029).
  11506. "/source-dl/", # /archives/{id}/source-dl/{token}/{filename}
  11507. # Obico ML API fetches JPEG frames by one-shot nonce (issue #172 follow-up).
  11508. # The nonce itself is the credential: 32-byte random, single-use, ~30s TTL.
  11509. "/obico/cached-frame/", # /obico/cached-frame/{nonce}
  11510. ]
  11511. _security_headers_logger = logging.getLogger("backend.app.main.security_headers")
  11512. def _parse_trusted_frame_origins() -> tuple[str, ...]:
  11513. """Parse TRUSTED_FRAME_ORIGINS env var into a validated allowlist (#1191).
  11514. Format: comma-separated list of ``scheme://host[:port]`` origins.
  11515. Used by ``security_headers_middleware`` to relax ``frame-ancestors`` for
  11516. trusted same-LAN deployments (e.g. Home Assistant Webpage panel embedding
  11517. Bambuddy from a different port). Defaults to empty — strict ``'none'``.
  11518. Invalid entries are dropped with a warning rather than failing startup, so
  11519. a typo in one origin doesn't take the whole deployment down.
  11520. """
  11521. raw = os.environ.get("TRUSTED_FRAME_ORIGINS", "").strip()
  11522. if not raw:
  11523. return ()
  11524. valid: list[str] = []
  11525. for item in raw.split(","):
  11526. candidate = item.strip()
  11527. if not candidate:
  11528. continue
  11529. try:
  11530. parsed = urlparse(candidate)
  11531. except ValueError as e:
  11532. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — %s", candidate, e)
  11533. continue
  11534. if parsed.scheme not in ("http", "https"):
  11535. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — must be http(s)", candidate)
  11536. continue
  11537. if not parsed.netloc:
  11538. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — missing host", candidate)
  11539. continue
  11540. if parsed.path and parsed.path != "/":
  11541. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — paths not allowed", candidate)
  11542. continue
  11543. if parsed.query or parsed.fragment:
  11544. _security_headers_logger.warning(
  11545. "TRUSTED_FRAME_ORIGINS: dropping %r — query/fragment not allowed", candidate
  11546. )
  11547. continue
  11548. if "*" in parsed.netloc:
  11549. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — wildcards not allowed", candidate)
  11550. continue
  11551. valid.append(f"{parsed.scheme}://{parsed.netloc}")
  11552. if valid:
  11553. _security_headers_logger.info("TRUSTED_FRAME_ORIGINS: %s", ", ".join(valid))
  11554. return tuple(valid)
  11555. _TRUSTED_FRAME_ORIGINS: tuple[str, ...] = _parse_trusted_frame_origins()
  11556. def _frame_ancestors(default_value: str) -> str:
  11557. """Compose the ``frame-ancestors`` CSP directive (#1191).
  11558. ``default_value`` is the strict directive used when the operator has not
  11559. configured ``TRUSTED_FRAME_ORIGINS`` — typically ``'none'`` (catch-all and
  11560. docs) or ``'self'`` (the streaming overlay, embedded same-origin by the
  11561. Settings URL builder's preview). When trusted origins
  11562. are configured, ``'self'`` is always included so same-origin embedding never
  11563. breaks even if an operator forgets to add their own origin to the list.
  11564. """
  11565. if _TRUSTED_FRAME_ORIGINS:
  11566. return "frame-ancestors 'self' " + " ".join(_TRUSTED_FRAME_ORIGINS) + ";"
  11567. return f"frame-ancestors {default_value};"
  11568. # The two Vite-emitted worker assets that compile WebAssembly (#2976). Both
  11569. # patterns are anchored on the exact emitted name so the relaxed policies
  11570. # below can never apply to any other asset.
  11571. # src/workers/stepPreview.worker.ts -> /assets/stepPreview.worker-<hash>.js
  11572. # pdfjs-dist/legacy/build/pdf.worker.min.mjs?worker&url
  11573. # -> /assets/pdf.worker.min-<hash>.js
  11574. # Vite also emits a one-line chunk under the second name that only exports the
  11575. # worker's URL. The page imports it as a module, and a module script is run
  11576. # under the importing document's policy, never its own response's, so it
  11577. # matching as well changes nothing.
  11578. _STEP_WORKER_ASSET_RE = re.compile(r"^/assets/stepPreview\.worker-[\w-]+\.js$")
  11579. _PDF_WORKER_ASSET_RE = re.compile(r"^/assets/pdf\.worker\.min-[\w-]+\.js$")
  11580. @app.middleware("http")
  11581. async def security_headers_middleware(request, call_next):
  11582. """Add standard HTTP security headers to every response."""
  11583. # Per-request nonce stamped into `script-src` (#1460). On its own this
  11584. # changes nothing for Bambuddy's own pages — index.html has no inline
  11585. # scripts since the SW registration moved to /sw-register.js. The reason
  11586. # it's here is Cloudflare: a CF-fronted deployment has the bot-detection
  11587. # script injected into the HTML on the edge, with a fresh hash on every
  11588. # load (so hashes can't be allowlisted). When CF sees a nonce in our CSP,
  11589. # it clones the same nonce onto its injected <script>, and the inline
  11590. # script passes the policy without us needing 'unsafe-inline'. See
  11591. # https://developers.cloudflare.com/cloudflare-challenges/challenge-types/javascript-detections/#if-you-have-a-content-security-policy-csp
  11592. csp_nonce = secrets.token_urlsafe(16)
  11593. # Routes that render their own HTML need it too, or their inline script is
  11594. # blocked by the policy below. The outcome-confirmation page (#1898) is the
  11595. # one that does: it submits its own form so a verdict still costs one tap.
  11596. request.state.csp_nonce = csp_nonce
  11597. response = await call_next(request)
  11598. response.headers["X-Content-Type-Options"] = "nosniff"
  11599. # X-Frame-Options is the legacy cross-origin embedding control. Modern
  11600. # browsers honour CSP frame-ancestors instead, and the legacy
  11601. # `ALLOW-FROM <url>` syntax is deprecated and inconsistent across vendors.
  11602. # When operators have explicitly allowlisted trusted frame origins (#1191
  11603. # — typically Home Assistant on a different port), drop X-Frame-Options
  11604. # and let the CSP-side frame-ancestors directive govern embedding.
  11605. if not _TRUSTED_FRAME_ORIGINS:
  11606. response.headers["X-Frame-Options"] = "SAMEORIGIN"
  11607. response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
  11608. # Content-Security-Policy for the React SPA.
  11609. # Notes:
  11610. # - 'unsafe-inline' for style-src: React and UI libs inject inline styles at runtime.
  11611. # - connect-src ws:/wss:: MQTT/printer WebSocket connections.
  11612. # - img-src data: / blob:: base64 thumbnails and Blob-URL timelapse previews.
  11613. # - media-src blob:: timelapse video player uses Blob URLs.
  11614. # - font-src data:: some icon fonts are embedded as data URIs.
  11615. if request.url.path in ("/docs", "/redoc", "/docs/oauth2-redirect"):
  11616. # FastAPI's built-in Swagger UI / ReDoc pages load assets from
  11617. # cdn.jsdelivr.net and bootstrap with an inline <script>, so the
  11618. # default CSP would render a blank page.
  11619. response.headers["Content-Security-Policy"] = (
  11620. "default-src 'self'; "
  11621. "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
  11622. "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com; "
  11623. "img-src 'self' data: blob: https://fastapi.tiangolo.com https://cdn.redoc.ly; "
  11624. "connect-src 'self'; "
  11625. "font-src 'self' data: https://fonts.gstatic.com; "
  11626. "worker-src 'self' blob:; "
  11627. "object-src 'none'; "
  11628. "base-uri 'self'; " + _frame_ancestors("'none'")
  11629. )
  11630. elif _STEP_WORKER_ASSET_RE.match(request.url.path):
  11631. # The STEP preview worker (#2976) runs OpenCascade compiled to WASM;
  11632. # its emscripten/embind glue generates invoker functions with `new
  11633. # Function(...)`, which needs 'unsafe-eval'. Per CSP3 a dedicated
  11634. # worker is governed by the policy delivered with the WORKER SCRIPT's
  11635. # own response — not the document's — so relaxing it here confines
  11636. # eval to that DOM-less worker context. The document policy below
  11637. # stays nonce-strict, and this response header has no effect when the
  11638. # file is merely fetched (a fetch's CSP is enforced against the
  11639. # requesting document, not the resource's own headers).
  11640. response.headers["Content-Security-Policy"] = (
  11641. "default-src 'self'; "
  11642. "script-src 'self' 'wasm-unsafe-eval' 'unsafe-eval'; "
  11643. "connect-src 'self'; "
  11644. "object-src 'none'; "
  11645. "base-uri 'self'; " + _frame_ancestors("'none'")
  11646. )
  11647. elif _PDF_WORKER_ASSET_RE.match(request.url.path):
  11648. # pdf.js decodes JPEG2000/JBIG2 images and ICC colour with WebAssembly
  11649. # and fetches those modules from /assets/pdfjs/wasm/ (#2976). Same CSP3
  11650. # rule as the STEP worker above: the policy that governs a dedicated
  11651. # worker is the one delivered with its own script, so the wasm compile
  11652. # has to be permitted here rather than on the document. Unlike the STEP
  11653. # worker this one needs no JS eval, so it gets 'wasm-unsafe-eval' only.
  11654. response.headers["Content-Security-Policy"] = (
  11655. "default-src 'self'; "
  11656. "script-src 'self' 'wasm-unsafe-eval'; "
  11657. "connect-src 'self'; "
  11658. "object-src 'none'; "
  11659. "base-uri 'self'; " + _frame_ancestors("'none'")
  11660. )
  11661. else:
  11662. # The streaming overlay is embedded same-origin by the URL builder's
  11663. # preview in Settings (#1422), so this branch allows 'self'.
  11664. # Embedding from anywhere else is still refused: 'self'
  11665. # only permits a framer on this origin, which is Bambuddy's own UI, so
  11666. # a clickjacking page on another host is blocked exactly as before.
  11667. # (The overlay draws status over a camera feed and its only interactive
  11668. # element is the logo link, so there is nothing to bait a click into
  11669. # even from a same-origin framer.) Cross-origin embedding of the
  11670. # overlay — Home Assistant on another port — remains what
  11671. # TRUSTED_FRAME_ORIGINS is for, and _frame_ancestors already folds that
  11672. # allowlist in.
  11673. #
  11674. # The connected-app consent page (/connect/authorize) gets the same
  11675. # 'self': an app opened from Bambuddy's sidebar runs in an iframe, and
  11676. # its "Sign in with Bambuddy" navigates that iframe to this page. With
  11677. # 'none' the browser refuses to show it even inside Bambuddy. 'self'
  11678. # requires every ancestor to be this origin, so a foreign page -- a
  11679. # sidebar link's site included -- still cannot frame the consent
  11680. # screen to bait a click.
  11681. embeddable_same_origin = request.url.path.startswith("/overlay/") or request.url.path == "/connect/authorize"
  11682. # No 'wasm-unsafe-eval' here: nothing compiles WebAssembly on the main
  11683. # thread. Both wasm consumers — the STEP preview and pdf.js's image
  11684. # decoders (#2976) — run in dedicated workers, which CSP3 governs by
  11685. # the policy served with their own script, so each gets it in its own
  11686. # branch above and the document policy stays as strict as it was.
  11687. response.headers["Content-Security-Policy"] = (
  11688. "default-src 'self'; "
  11689. f"script-src 'self' 'nonce-{csp_nonce}'; "
  11690. "style-src 'self' 'unsafe-inline'; "
  11691. "img-src 'self' data: blob:; "
  11692. "media-src 'self' blob:; "
  11693. "connect-src 'self' ws: wss:; "
  11694. "font-src 'self' data:; "
  11695. "object-src 'none'; "
  11696. "base-uri 'self'; "
  11697. "frame-src 'self' http: https:; " + _frame_ancestors("'self'" if embeddable_same_origin else "'none'")
  11698. )
  11699. if request.url.scheme == "https":
  11700. response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
  11701. return response
  11702. @app.middleware("http")
  11703. async def auth_middleware(request, call_next):
  11704. """Enforce authentication on all API routes when auth is enabled.
  11705. This middleware provides defense-in-depth by checking auth at the API gateway level,
  11706. regardless of whether individual routes have auth dependencies.
  11707. """
  11708. from starlette.responses import JSONResponse
  11709. path = request.url.path
  11710. # Only apply to API routes
  11711. if not path.startswith("/api/"):
  11712. return await call_next(request)
  11713. # Allow public routes
  11714. if path in PUBLIC_API_ROUTES:
  11715. return await call_next(request)
  11716. # Allow public prefixes
  11717. for prefix in PUBLIC_API_PREFIXES:
  11718. if path.startswith(prefix):
  11719. return await call_next(request)
  11720. # Allow public patterns (read-only display data like thumbnails)
  11721. for pattern in PUBLIC_API_PATTERNS:
  11722. if pattern in path:
  11723. return await call_next(request)
  11724. # Check if auth is enabled. Fail CLOSED on any exception during the
  11725. # probe — GHSA-6mf4-q26m-47pv: the previous fail-open path here let
  11726. # an attacker who could force a DB exception (e.g. file-descriptor
  11727. # exhaustion via login flood) bypass auth on every protected endpoint.
  11728. # Only the probe is guarded: the request itself runs outside the try, so an
  11729. # error in a route surfaces as the 500 it is rather than as an auth outage.
  11730. try:
  11731. async with async_session() as db:
  11732. from backend.app.core.auth import is_auth_enabled
  11733. auth_enabled = await is_auth_enabled(db)
  11734. except Exception:
  11735. logging.getLogger(__name__).exception("auth_middleware: failing closed on auth-probe error from %s", path)
  11736. return JSONResponse(
  11737. status_code=503,
  11738. content={"detail": "Authentication service temporarily unavailable"},
  11739. )
  11740. if not auth_enabled:
  11741. # Auth disabled, allow all requests
  11742. return await call_next(request)
  11743. # Auth is enabled - require valid token
  11744. auth_header = request.headers.get("Authorization")
  11745. x_api_key = request.headers.get("X-API-Key")
  11746. # Check for API key auth first
  11747. if x_api_key or (auth_header and auth_header.startswith("Bearer bb_")):
  11748. # API key authentication - let the request through to be validated by route handler
  11749. # API keys are validated per-route since they have different permission levels
  11750. return await call_next(request)
  11751. # Check for JWT auth
  11752. if not auth_header or not auth_header.startswith("Bearer "):
  11753. return JSONResponse(
  11754. status_code=401,
  11755. content={"detail": "Authentication required"},
  11756. headers={"WWW-Authenticate": "Bearer"},
  11757. )
  11758. # Validate JWT token
  11759. import jwt
  11760. try:
  11761. from backend.app.core.auth import (
  11762. ALGORITHM,
  11763. SECRET_KEY,
  11764. _is_token_fresh,
  11765. get_user_by_username,
  11766. is_jti_revoked,
  11767. )
  11768. token = auth_header.replace("Bearer ", "")
  11769. payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
  11770. username = payload.get("sub")
  11771. if not username:
  11772. raise ValueError("No username in token")
  11773. jti = payload.get("jti")
  11774. if not jti:
  11775. raise ValueError("No jti in token")
  11776. iat = payload.get("iat")
  11777. # Verify user exists, is active, and token is still fresh (L-R8-A).
  11778. # Reject revoked tokens first (defense-in-depth gateway check), reusing
  11779. # this session so the gateway adds a single pooled checkout, not two (#2572).
  11780. async with async_session() as db:
  11781. if await is_jti_revoked(jti, db):
  11782. return JSONResponse(
  11783. status_code=401,
  11784. content={"detail": "Token has been revoked"},
  11785. headers={"WWW-Authenticate": "Bearer"},
  11786. )
  11787. user = await get_user_by_username(db, username)
  11788. if not user or not user.is_active:
  11789. return JSONResponse(
  11790. status_code=401,
  11791. content={"detail": "User not found or inactive"},
  11792. headers={"WWW-Authenticate": "Bearer"},
  11793. )
  11794. if not _is_token_fresh(iat, user):
  11795. return JSONResponse(
  11796. status_code=401,
  11797. content={"detail": "Token no longer valid"},
  11798. headers={"WWW-Authenticate": "Bearer"},
  11799. )
  11800. except jwt.ExpiredSignatureError:
  11801. return JSONResponse(
  11802. status_code=401,
  11803. content={"detail": "Token has expired"},
  11804. headers={"WWW-Authenticate": "Bearer"},
  11805. )
  11806. except (jwt.InvalidTokenError, ValueError):
  11807. return JSONResponse(
  11808. status_code=401,
  11809. content={"detail": "Invalid token"},
  11810. headers={"WWW-Authenticate": "Bearer"},
  11811. )
  11812. except Exception:
  11813. # Not a bad token but a failure checking it (database, pool): refuse,
  11814. # as for the probe above, rather than answer 401 -- the frontend reads
  11815. # a 401 from /auth/me as final and signs the user out.
  11816. logging.getLogger(__name__).exception("auth_middleware: failing closed on token-check error from %s", path)
  11817. return JSONResponse(
  11818. status_code=503,
  11819. content={"detail": "Authentication service temporarily unavailable"},
  11820. )
  11821. return await call_next(request)
  11822. @app.middleware("http")
  11823. async def trace_id_middleware(request, call_next):
  11824. """Stamp every HTTP request with a trace ID and echo it back.
  11825. Decorated AFTER auth_middleware on purpose: Starlette stacks
  11826. @app.middleware decorators LIFO, so the last-decorated runs first
  11827. inbound. Putting the trace stamp last makes it the OUTERMOST layer,
  11828. which means auth-middleware log lines (and every line emitted on the
  11829. way down to and back from the route handler) all carry the same
  11830. trace ID. If we put it before auth, auth's logs would be stamped
  11831. with the *previous* request's ID — useless for correlation.
  11832. Honours an inbound ``X-Trace-Id`` header so callers running their
  11833. own tracing can correlate their span IDs with our log lines, but
  11834. only if the value passes the whitelist gate in
  11835. ``backend.app.core.trace.normalise_inbound_trace_id`` — anything
  11836. rejected (too long, contains control chars, etc.) silently triggers
  11837. a freshly minted server-side ID rather than failing the request.
  11838. The minted (or echoed) ID is set on a ContextVar so that every log
  11839. record emitted during the request — application logs *and* uvicorn's
  11840. access log — carries it via TraceIDFilter, and is also written to
  11841. the ``X-Trace-Id`` response header so clients can pin a server-side
  11842. log search to the exact request they made.
  11843. """
  11844. from backend.app.core.trace import (
  11845. generate_trace_id,
  11846. normalise_inbound_trace_id,
  11847. trace_id_var,
  11848. )
  11849. inbound = normalise_inbound_trace_id(request.headers.get("X-Trace-Id"))
  11850. trace_id = inbound if inbound is not None else generate_trace_id()
  11851. token = trace_id_var.set(trace_id)
  11852. try:
  11853. response = await call_next(request)
  11854. finally:
  11855. # Reset the ContextVar so a record emitted in a totally
  11856. # unrelated background task that just happens to inherit this
  11857. # context doesn't keep referencing this request's ID forever.
  11858. # In practice ContextVar.reset is best-effort under asyncio
  11859. # task-spawn semantics, but the cost is one attribute write so
  11860. # we may as well do it.
  11861. trace_id_var.reset(token)
  11862. response.headers["X-Trace-Id"] = trace_id
  11863. return response
  11864. # API routes
  11865. app.include_router(auth.router, prefix=app_settings.api_prefix)
  11866. app.include_router(mfa.router, prefix=app_settings.api_prefix)
  11867. app.include_router(bug_report.router, prefix=app_settings.api_prefix)
  11868. app.include_router(users.router, prefix=app_settings.api_prefix)
  11869. app.include_router(groups.router, prefix=app_settings.api_prefix)
  11870. app.include_router(printers.router, prefix=app_settings.api_prefix)
  11871. app.include_router(printer_locations.router, prefix=app_settings.api_prefix)
  11872. app.include_router(archives.router, prefix=app_settings.api_prefix)
  11873. app.include_router(filaments.router, prefix=app_settings.api_prefix)
  11874. app.include_router(finance.router, prefix=app_settings.api_prefix)
  11875. app.include_router(inventory.router, prefix=app_settings.api_prefix)
  11876. app.include_router(labels.router, prefix=app_settings.api_prefix)
  11877. app.include_router(label_templates.router, prefix=app_settings.api_prefix)
  11878. app.include_router(settings_routes.router, prefix=app_settings.api_prefix)
  11879. app.include_router(overlay_branding.router, prefix=app_settings.api_prefix)
  11880. app.include_router(cloud.router, prefix=app_settings.api_prefix)
  11881. app.include_router(orca_cloud.router, prefix=app_settings.api_prefix)
  11882. app.include_router(local_presets.router, prefix=app_settings.api_prefix)
  11883. app.include_router(smart_plugs.router, prefix=app_settings.api_prefix)
  11884. app.include_router(ha_sensors.router, prefix=app_settings.api_prefix)
  11885. app.include_router(location_ha_sensors.router, prefix=app_settings.api_prefix)
  11886. app.include_router(print_log.router, prefix=app_settings.api_prefix)
  11887. app.include_router(print_queue.router, prefix=app_settings.api_prefix)
  11888. app.include_router(scheduled_dryings.router, prefix=app_settings.api_prefix)
  11889. app.include_router(kprofiles.router, prefix=app_settings.api_prefix)
  11890. app.include_router(notifications.router, prefix=app_settings.api_prefix)
  11891. app.include_router(notification_templates.router, prefix=app_settings.api_prefix)
  11892. app.include_router(user_notifications.router, prefix=app_settings.api_prefix)
  11893. app.include_router(spoolman.router, prefix=app_settings.api_prefix)
  11894. app.include_router(spoolman_inventory.router, prefix=app_settings.api_prefix)
  11895. app.include_router(updates.router, prefix=app_settings.api_prefix)
  11896. app.include_router(sponsor_prompt.router, prefix=app_settings.api_prefix)
  11897. app.include_router(announcements.router, prefix=app_settings.api_prefix)
  11898. app.include_router(maintenance.router, prefix=app_settings.api_prefix)
  11899. app.include_router(camera.router, prefix=app_settings.api_prefix)
  11900. app.include_router(camwall.router, prefix=app_settings.api_prefix)
  11901. app.include_router(external_links.router, prefix=app_settings.api_prefix)
  11902. app.include_router(projects.router, prefix=app_settings.api_prefix)
  11903. app.include_router(library.router, prefix=app_settings.api_prefix)
  11904. app.include_router(library_tags.router, prefix=app_settings.api_prefix)
  11905. app.include_router(library_trash.router, prefix=app_settings.api_prefix)
  11906. app.include_router(library_variants.router, prefix=app_settings.api_prefix)
  11907. app.include_router(slice_jobs.router, prefix=app_settings.api_prefix)
  11908. app.include_router(slicer_pipelines.router, prefix=app_settings.api_prefix)
  11909. app.include_router(pipeline_runs.pipeline_run_create_router, prefix=app_settings.api_prefix)
  11910. app.include_router(pipeline_runs.pipeline_run_router, prefix=app_settings.api_prefix)
  11911. app.include_router(slicer_presets.router, prefix=app_settings.api_prefix)
  11912. app.include_router(archive_purge.router, prefix=app_settings.api_prefix)
  11913. app.include_router(makerworld.router, prefix=app_settings.api_prefix)
  11914. app.include_router(manyfold.router, prefix=app_settings.api_prefix)
  11915. app.include_router(api_keys.router, prefix=app_settings.api_prefix)
  11916. app.include_router(connected_apps.router, prefix=app_settings.api_prefix)
  11917. app.include_router(webhook.router, prefix=app_settings.api_prefix)
  11918. app.include_router(ams_history.router, prefix=app_settings.api_prefix)
  11919. app.include_router(printer_sensor_history.router, prefix=app_settings.api_prefix)
  11920. app.include_router(system.router, prefix=app_settings.api_prefix)
  11921. app.include_router(support.router, prefix=app_settings.api_prefix)
  11922. app.include_router(websocket.router, prefix=app_settings.api_prefix)
  11923. app.include_router(discovery.router, prefix=app_settings.api_prefix)
  11924. app.include_router(pending_uploads.router, prefix=app_settings.api_prefix)
  11925. app.include_router(firmware.router, prefix=app_settings.api_prefix)
  11926. app.include_router(bulk.router, prefix=app_settings.api_prefix)
  11927. app.include_router(github_backup.router, prefix=app_settings.api_prefix)
  11928. app.include_router(local_backup.router, prefix=app_settings.api_prefix)
  11929. app.include_router(obico.router, prefix=app_settings.api_prefix)
  11930. app.include_router(octoeverywhere.router, prefix=app_settings.api_prefix)
  11931. app.include_router(metrics.router, prefix=app_settings.api_prefix)
  11932. app.include_router(virtual_printers.router, prefix=app_settings.api_prefix)
  11933. app.include_router(spoolbuddy.router, prefix=app_settings.api_prefix)
  11934. # Serve static files (React build)
  11935. if app_settings.static_dir.exists() and any(app_settings.static_dir.iterdir()):
  11936. # Compressed, and cached for good where Vite hashed the name (#3175).
  11937. app.mount(
  11938. "/assets",
  11939. AssetStaticFiles(directory=app_settings.static_dir / "assets"),
  11940. name="assets",
  11941. )
  11942. if (app_settings.static_dir / "img").exists():
  11943. app.mount(
  11944. "/img",
  11945. StaticFiles(directory=app_settings.static_dir / "img"),
  11946. name="img",
  11947. )
  11948. if (app_settings.static_dir / "icons").exists():
  11949. app.mount(
  11950. "/icons",
  11951. StaticFiles(directory=app_settings.static_dir / "icons"),
  11952. name="icons",
  11953. )
  11954. # Self-hosted Inter woff2 files (#1460). Without this mount /fonts/*.woff2
  11955. # falls through to the SPA catch-all and returns index.html, which the
  11956. # browser's font sanitizer rejects ("downloadable font: rejected by
  11957. # sanitizer").
  11958. if (app_settings.static_dir / "fonts").exists():
  11959. app.mount(
  11960. "/fonts",
  11961. StaticFiles(directory=app_settings.static_dir / "fonts"),
  11962. name="fonts",
  11963. )
  11964. @app.get("/")
  11965. async def serve_frontend():
  11966. """Serve the React frontend."""
  11967. index_file = app_settings.static_dir / "index.html"
  11968. if index_file.exists():
  11969. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  11970. return {
  11971. "message": "Bambuddy API",
  11972. "docs": "/docs",
  11973. "frontend": "Build and place React app in /static directory",
  11974. }
  11975. # index.html must always be revalidated — Vite emits content-hashed JS/CSS
  11976. # bundles (e.g. `index-JRaF_JhW.js`), so the JS itself is safe to cache
  11977. # forever, but the HTML wrapping it is the only file that knows which hash
  11978. # is current. Without explicit cache-control headers Chromium decides
  11979. # heuristically (typically 10% of the time since Last-Modified) and on
  11980. # long-running kiosks happily serves stale HTML across browser restarts.
  11981. # That stale HTML references an old bundle hash, the old bundle is also
  11982. # in the disk cache, and the user ends up running pre-update JS forever
  11983. # without ever knowing why. ``no-cache`` (revalidate every time, but a
  11984. # 304 is cheap) is the correct setting for an SPA's entry HTML.
  11985. _HTML_CACHE_HEADERS = {"Cache-Control": "no-cache, must-revalidate"}
  11986. @app.get("/health")
  11987. async def health_check():
  11988. """Health check endpoint."""
  11989. return {"status": "healthy"}
  11990. # GET + HEAD on the three PWA bootstrap routes (#1460). Scanners and a plain
  11991. # `curl -I` use HEAD; FastAPI's @app.get only registers GET, so HEAD answers
  11992. # with 405 Method Not Allowed and shows up as a "broken manifest" red herring
  11993. # in deployment debugging. One decorator per method, not api_route(methods=
  11994. # [...]): api_route gives both methods the same OpenAPI operation id.
  11995. @app.get("/manifest.json")
  11996. @app.head("/manifest.json")
  11997. async def serve_manifest():
  11998. """Serve PWA manifest."""
  11999. manifest_file = app_settings.static_dir / "manifest.json"
  12000. if manifest_file.exists():
  12001. return FileResponse(manifest_file, media_type="application/manifest+json")
  12002. return {"error": "Manifest not found"}
  12003. @app.get("/sw.js")
  12004. @app.head("/sw.js")
  12005. async def serve_service_worker():
  12006. """Serve service worker."""
  12007. sw_file = app_settings.static_dir / "sw.js"
  12008. if sw_file.exists():
  12009. return FileResponse(
  12010. sw_file,
  12011. media_type="application/javascript",
  12012. headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
  12013. )
  12014. return {"error": "Service worker not found"}
  12015. @app.get("/sw-register.js")
  12016. @app.head("/sw-register.js")
  12017. async def serve_sw_register():
  12018. """Serve the service-worker registration bootstrap script.
  12019. Served as a real JS file so the strict `script-src 'self'` CSP covers it
  12020. without needing 'unsafe-inline' or per-build hashes on the inline tag.
  12021. """
  12022. reg_file = app_settings.static_dir / "sw-register.js"
  12023. if reg_file.exists():
  12024. return FileResponse(reg_file, media_type="application/javascript")
  12025. return {"error": "sw-register.js not found"}
  12026. # ── GCode viewer static files ────────────────────────────────────────────────
  12027. # Catch-all route for React Router (must be last)
  12028. @app.get("/{full_path:path}")
  12029. async def serve_spa(full_path: str):
  12030. """Serve React app for client-side routing."""
  12031. # Don't intercept API routes - raise proper 404 so FastAPI can handle redirects
  12032. if full_path.startswith("api/"):
  12033. from fastapi import HTTPException
  12034. raise HTTPException(status_code=404, detail="Not found")
  12035. index_file = app_settings.static_dir / "index.html"
  12036. if index_file.exists():
  12037. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  12038. return {"error": "Frontend not built"}