main.py 656 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352113531135411355113561135711358113591136011361113621136311364113651136611367113681136911370113711137211373113741137511376113771137811379113801138111382113831138411385113861138711388113891139011391113921139311394113951139611397113981139911400114011140211403114041140511406114071140811409114101141111412114131141411415114161141711418114191142011421114221142311424114251142611427114281142911430114311143211433114341143511436114371143811439114401144111442114431144411445114461144711448114491145011451114521145311454114551145611457114581145911460114611146211463114641146511466114671146811469114701147111472114731147411475114761147711478114791148011481114821148311484114851148611487114881148911490114911149211493114941149511496114971149811499115001150111502115031150411505115061150711508115091151011511115121151311514115151151611517115181151911520115211152211523115241152511526115271152811529115301153111532115331153411535115361153711538115391154011541115421154311544115451154611547115481154911550115511155211553115541155511556115571155811559115601156111562115631156411565115661156711568115691157011571115721157311574115751157611577115781157911580115811158211583115841158511586115871158811589115901159111592115931159411595115961159711598115991160011601116021160311604116051160611607116081160911610116111161211613116141161511616116171161811619116201162111622116231162411625116261162711628116291163011631116321163311634116351163611637116381163911640116411164211643116441164511646116471164811649116501165111652116531165411655116561165711658116591166011661116621166311664116651166611667116681166911670116711167211673116741167511676116771167811679116801168111682116831168411685116861168711688116891169011691116921169311694116951169611697116981169911700117011170211703117041170511706117071170811709117101171111712117131171411715117161171711718117191172011721117221172311724117251172611727117281172911730117311173211733117341173511736117371173811739117401174111742117431174411745117461174711748117491175011751117521175311754117551175611757117581175911760117611176211763117641176511766117671176811769117701177111772117731177411775117761177711778117791178011781117821178311784117851178611787117881178911790117911179211793117941179511796117971179811799118001180111802118031180411805118061180711808118091181011811118121181311814118151181611817118181181911820118211182211823118241182511826118271182811829118301183111832118331183411835118361183711838118391184011841118421184311844118451184611847118481184911850118511185211853118541185511856118571185811859118601186111862118631186411865118661186711868118691187011871118721187311874118751187611877118781187911880118811188211883118841188511886118871188811889118901189111892118931189411895118961189711898118991190011901119021190311904119051190611907119081190911910119111191211913119141191511916119171191811919119201192111922119231192411925119261192711928119291193011931119321193311934119351193611937119381193911940119411194211943119441194511946119471194811949119501195111952119531195411955119561195711958119591196011961119621196311964119651196611967119681196911970119711197211973119741197511976119771197811979119801198111982119831198411985119861198711988119891199011991119921199311994119951199611997119981199912000120011200212003120041200512006120071200812009120101201112012120131201412015120161201712018120191202012021120221202312024120251202612027120281202912030120311203212033120341203512036120371203812039120401204112042120431204412045120461204712048120491205012051120521205312054120551205612057120581205912060120611206212063120641206512066120671206812069120701207112072120731207412075120761207712078120791208012081120821208312084120851208612087120881208912090120911209212093120941209512096120971209812099121001210112102121031210412105121061210712108121091211012111121121211312114121151211612117121181211912120121211212212123121241212512126121271212812129121301213112132121331213412135121361213712138121391214012141121421214312144121451214612147121481214912150121511215212153121541215512156121571215812159121601216112162121631216412165121661216712168121691217012171121721217312174121751217612177121781217912180121811218212183121841218512186121871218812189121901219112192121931219412195121961219712198121991220012201122021220312204122051220612207122081220912210122111221212213122141221512216122171221812219122201222112222122231222412225122261222712228122291223012231122321223312234122351223612237122381223912240122411224212243122441224512246122471224812249122501225112252122531225412255122561225712258122591226012261122621226312264122651226612267122681226912270122711227212273122741227512276122771227812279122801228112282122831228412285122861228712288122891229012291122921229312294122951229612297122981229912300123011230212303123041230512306123071230812309123101231112312123131231412315123161231712318123191232012321123221232312324123251232612327123281232912330123311233212333123341233512336123371233812339123401234112342123431234412345123461234712348123491235012351123521235312354123551235612357123581235912360123611236212363123641236512366123671236812369123701237112372123731237412375123761237712378123791238012381123821238312384123851238612387123881238912390123911239212393123941239512396123971239812399124001240112402124031240412405124061240712408124091241012411124121241312414124151241612417124181241912420124211242212423124241242512426124271242812429124301243112432124331243412435124361243712438124391244012441124421244312444124451244612447124481244912450124511245212453124541245512456124571245812459124601246112462124631246412465124661246712468124691247012471124721247312474124751247612477124781247912480124811248212483124841248512486124871248812489124901249112492124931249412495124961249712498124991250012501125021250312504125051250612507125081250912510125111251212513125141251512516125171251812519125201252112522125231252412525125261252712528125291253012531125321253312534125351253612537125381253912540125411254212543125441254512546125471254812549125501255112552125531255412555125561255712558125591256012561125621256312564125651256612567125681256912570125711257212573125741257512576125771257812579125801258112582125831258412585125861258712588125891259012591125921259312594125951259612597125981259912600126011260212603126041260512606126071260812609126101261112612126131261412615126161261712618126191262012621126221262312624126251262612627126281262912630126311263212633126341263512636126371263812639126401264112642126431264412645126461264712648126491265012651126521265312654126551265612657126581265912660126611266212663126641266512666126671266812669126701267112672126731267412675126761267712678126791268012681126821268312684126851268612687126881268912690126911269212693126941269512696126971269812699127001270112702127031270412705127061270712708127091271012711127121271312714127151271612717127181271912720127211272212723127241272512726127271272812729127301273112732127331273412735127361273712738127391274012741127421274312744127451274612747127481274912750127511275212753127541275512756127571275812759127601276112762127631276412765127661276712768127691277012771127721277312774127751277612777127781277912780127811278212783127841278512786127871278812789127901279112792127931279412795127961279712798127991280012801128021280312804128051280612807128081280912810128111281212813128141281512816128171281812819128201282112822128231282412825128261282712828128291283012831128321283312834128351283612837128381283912840128411284212843128441284512846128471284812849128501285112852128531285412855128561285712858128591286012861128621286312864128651286612867128681286912870128711287212873128741287512876128771287812879128801288112882128831288412885128861288712888128891289012891128921289312894128951289612897128981289912900129011290212903129041290512906129071290812909129101291112912129131291412915129161291712918129191292012921129221292312924129251292612927129281292912930129311293212933129341293512936129371293812939129401294112942129431294412945129461294712948129491295012951129521295312954129551295612957129581295912960129611296212963129641296512966129671296812969129701297112972129731297412975129761297712978129791298012981129821298312984129851298612987129881298912990129911299212993129941299512996129971299812999130001300113002130031300413005130061300713008130091301013011130121301313014130151301613017130181301913020130211302213023130241302513026130271302813029130301303113032130331303413035130361303713038130391304013041130421304313044130451304613047130481304913050130511305213053130541305513056130571305813059130601306113062130631306413065130661306713068130691307013071130721307313074130751307613077130781307913080130811308213083130841308513086130871308813089130901309113092130931309413095130961309713098130991310013101131021310313104131051310613107131081310913110131111311213113131141311513116131171311813119131201312113122131231312413125131261312713128131291313013131131321313313134131351313613137131381313913140131411314213143131441314513146131471314813149131501315113152131531315413155131561315713158131591316013161131621316313164131651316613167131681316913170131711317213173131741317513176131771317813179131801318113182131831318413185131861318713188131891319013191131921319313194131951319613197131981319913200132011320213203132041320513206132071320813209132101321113212132131321413215132161321713218132191322013221132221322313224132251322613227132281322913230132311323213233132341323513236132371323813239132401324113242132431324413245132461324713248132491325013251132521325313254132551325613257132581325913260132611326213263132641326513266132671326813269132701327113272132731327413275132761327713278132791328013281132821328313284132851328613287132881328913290132911329213293132941329513296132971329813299133001330113302133031330413305133061330713308133091331013311133121331313314133151331613317133181331913320133211332213323133241332513326133271332813329133301333113332133331333413335133361333713338133391334013341133421334313344133451334613347133481334913350133511335213353133541335513356133571335813359133601336113362133631336413365133661336713368133691337013371133721337313374133751337613377133781337913380133811338213383133841338513386133871338813389133901339113392133931339413395133961339713398133991340013401134021340313404134051340613407134081340913410134111341213413134141341513416134171341813419134201342113422134231342413425134261342713428134291343013431134321343313434134351343613437134381343913440134411344213443134441344513446134471344813449134501345113452134531345413455134561345713458134591346013461134621346313464134651346613467134681346913470134711347213473134741347513476134771347813479134801348113482134831348413485134861348713488134891349013491134921349313494134951349613497134981349913500135011350213503135041350513506135071350813509135101351113512135131351413515135161351713518135191352013521135221352313524135251352613527135281352913530135311353213533135341353513536135371353813539135401354113542135431354413545135461354713548135491355013551135521355313554135551355613557135581355913560135611356213563135641356513566135671356813569135701357113572135731357413575135761357713578135791358013581135821358313584
  1. import asyncio
  2. import json
  3. import logging
  4. import math
  5. import os
  6. import posixpath
  7. import re
  8. import secrets
  9. import shutil
  10. import time
  11. from contextlib import asynccontextmanager
  12. from datetime import datetime, timedelta, timezone
  13. from logging.handlers import RotatingFileHandler
  14. from pathlib import Path, PurePosixPath
  15. from typing import NamedTuple, cast
  16. from urllib.parse import urlparse
  17. from fastapi import FastAPI
  18. from fastapi.encoders import jsonable_encoder
  19. from fastapi.exceptions import RequestValidationError
  20. from fastapi.responses import FileResponse, JSONResponse
  21. from fastapi.staticfiles import StaticFiles
  22. from sqlalchemy import delete, func, inspect as sa_inspect, or_, select, text, update
  23. from sqlalchemy.engine import CursorResult
  24. from sqlalchemy.exc import IntegrityError
  25. from sqlalchemy.ext.asyncio import AsyncSession
  26. from sqlalchemy.orm.exc import StaleDataError
  27. from backend.app.api.routes import (
  28. ams_history,
  29. announcements,
  30. api_keys,
  31. archive_purge,
  32. archives,
  33. auth,
  34. bug_report,
  35. bulk,
  36. camera,
  37. camwall,
  38. cloud,
  39. connected_apps,
  40. discovery,
  41. external_links,
  42. filaments,
  43. finance,
  44. firmware,
  45. github_backup,
  46. groups,
  47. ha_sensors,
  48. inventory,
  49. kprofiles,
  50. label_templates,
  51. labels,
  52. library,
  53. library_tags,
  54. library_trash,
  55. library_variants,
  56. local_backup,
  57. local_presets,
  58. location_ha_sensors,
  59. maintenance,
  60. makerworld,
  61. manyfold,
  62. metrics,
  63. mfa,
  64. notification_templates,
  65. notifications,
  66. obico,
  67. octoeverywhere,
  68. orca_cloud,
  69. overlay_branding,
  70. pending_uploads,
  71. pipeline_runs,
  72. print_log,
  73. print_queue,
  74. printer_locations,
  75. printer_sensor_history,
  76. printers,
  77. projects,
  78. scheduled_dryings,
  79. settings as settings_routes,
  80. slice_jobs,
  81. slicer_pipelines,
  82. slicer_presets,
  83. smart_plugs,
  84. sponsor_prompt,
  85. spoolbuddy,
  86. spoolman,
  87. spoolman_inventory,
  88. support,
  89. system,
  90. updates,
  91. user_notifications,
  92. users,
  93. virtual_printers,
  94. webhook,
  95. websocket,
  96. )
  97. from backend.app.api.routes.maintenance import (
  98. _get_printer_maintenance_internal,
  99. ensure_default_types,
  100. maintenance_alert_values,
  101. )
  102. from backend.app.api.routes.support import init_debug_logging
  103. from backend.app.core.config import APP_VERSION, settings as app_settings
  104. from backend.app.core.database import async_session, engine, init_db
  105. from backend.app.core.static_assets import AssetStaticFiles
  106. from backend.app.core.tasks import spawn_background_task
  107. from backend.app.core.websocket import ws_manager
  108. from backend.app.services import kprofile_drift, pause_eta, print_dispatch_context, slot_unlink_grace
  109. from backend.app.services.ams_assignment_lock import get_ams_assignment_lock
  110. from backend.app.services.archive import ArchiveService, peek_plate_index_in_3mf, swap_plate_suffix
  111. from backend.app.services.archive_cost_estimate import (
  112. schedule_archive_cost_estimate as _schedule_archive_cost_estimate,
  113. settle_archive_cost_estimate as _settle_archive_cost_estimate,
  114. )
  115. from backend.app.services.archive_purge import archive_purge_service
  116. from backend.app.services.bambu_ftp import (
  117. RECOVERY_DOWNLOAD_MAX_TIMEOUT,
  118. FileNotOnPrinterError,
  119. cache_3mf_download,
  120. clear_3mf_cache,
  121. download_file_async,
  122. download_file_try_paths_async,
  123. ftps_handshake_blocked,
  124. get_cached_3mf,
  125. get_ftp_retry_settings,
  126. normalize_3mf_name,
  127. with_ftp_retry,
  128. )
  129. from backend.app.services.bambu_mqtt import PrinterState
  130. from backend.app.services.camera_light import camera_light
  131. from backend.app.services.energy_plug import energy_plug_candidates, select_energy_reading
  132. from backend.app.services.github_backup import github_backup_service
  133. from backend.app.services.ha_sensor_manager import ha_sensor_manager
  134. from backend.app.services.hms_errors import hms_fault_counts as _hms_fault_counts
  135. from backend.app.services.library_trash import library_trash_service
  136. from backend.app.services.local_backup import local_backup_service
  137. from backend.app.services.location_ha_sensor_manager import location_ha_sensor_manager
  138. from backend.app.services.mqtt_relay import mqtt_relay
  139. from backend.app.services.mqtt_smart_plug import mqtt_smart_plug_service
  140. from backend.app.services.notification_service import notification_service
  141. from backend.app.services.notify_live_activities import notify_live_activities
  142. from backend.app.services.notify_widgets import notify_widgets
  143. from backend.app.services.obico_detection import obico_detection_service
  144. from backend.app.services.octoeverywhere_detection import octoeverywhere_detection_service
  145. from backend.app.services.plug_energy import get_plug_energy as _get_plug_energy
  146. from backend.app.services.print_cost_estimate import plate_scoped_run_estimate as _plate_scoped_run_estimate
  147. from backend.app.services.print_scheduler import scheduler as print_scheduler
  148. from backend.app.services.print_storage import (
  149. REASON_FTP_TRANSFER_FAILED,
  150. REASON_FTPS_COOLOFF,
  151. external_storage_present,
  152. ftp_probe_paths,
  153. print_file_reachable_over_ftp,
  154. )
  155. from backend.app.services.printer_manager import (
  156. init_printer_connections,
  157. parse_plate_id,
  158. printer_manager,
  159. printer_state_to_dict,
  160. printer_tray_layout,
  161. resolve_plate_id,
  162. )
  163. from backend.app.services.slot_kprofile import find_slot_kprofile_for_extruder
  164. from backend.app.services.slot_nozzle import (
  165. nozzle_diameter_for_extruder,
  166. nozzle_flow_for_extruder,
  167. )
  168. from backend.app.services.smart_plug_manager import smart_plug_manager
  169. from backend.app.services.spool_assignment_notifications import (
  170. notify_missing_spool_assignments_on_print_start,
  171. )
  172. from backend.app.services.spool_filament_preset import printer_safe_filament_id
  173. from backend.app.services.spoolman import close_spoolman_client, get_spoolman_client, init_spoolman_client
  174. from backend.app.services.spoolman_tracking import (
  175. cleanup_tracking as _cleanup_spoolman_tracking,
  176. release_archived_slot_rows as _release_archived_spoolman_slots,
  177. report_usage as _report_spoolman_usage,
  178. store_print_data as _store_spoolman_print_data,
  179. )
  180. from backend.app.services.telegram_reactions import telegram_reaction_poller
  181. from backend.app.services.wled import wled_manager
  182. from backend.app.utils.ams_drying import is_drying_active, temperature_alarm_suppressed
  183. from backend.app.utils.ams_humidity import ams_humidity_percent
  184. from backend.app.utils.filament_types import printer_filament_type, support_filament_type
  185. from backend.app.utils.fts_routing import extruder_for_inlet
  186. from backend.app.utils.local_time import to_naive_utc, utcnow_naive
  187. from backend.app.utils.print_jobs import is_internal_printer_job
  188. from backend.app.utils.tray_ids import AmsType, TrayLayout, external_tray_id, unit_ams_type, unit_label
  189. # =============================================================================
  190. # Dependency Check - runs before other imports to give helpful error messages
  191. # =============================================================================
  192. def _start_error_server(missing_packages: list):
  193. """Start a minimal HTTP server to display dependency errors in browser."""
  194. import os
  195. import signal
  196. from http.server import BaseHTTPRequestHandler, HTTPServer
  197. packages_html = "".join(f"<li><code>{p}</code></li>" for p in missing_packages)
  198. html = f"""<!DOCTYPE html>
  199. <html>
  200. <head>
  201. <title>Bambuddy - Setup Required</title>
  202. <style>
  203. body {{
  204. font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
  205. background: #0f172a; color: #e2e8f0;
  206. display: flex; justify-content: center; align-items: center;
  207. min-height: 100vh; margin: 0; padding: 20px; box-sizing: border-box;
  208. }}
  209. .container {{
  210. background: #1e293b; border-radius: 12px; padding: 40px;
  211. max-width: 600px; text-align: center; box-shadow: 0 4px 20px rgba(0,0,0,0.3);
  212. }}
  213. h1 {{ color: #f87171; margin-bottom: 10px; }}
  214. h2 {{ color: #94a3b8; font-weight: normal; margin-top: 0; }}
  215. .packages {{
  216. background: #0f172a; border-radius: 8px; padding: 20px;
  217. margin: 20px 0; text-align: left;
  218. }}
  219. .packages ul {{ margin: 0; padding-left: 20px; }}
  220. .packages li {{ color: #fbbf24; margin: 8px 0; }}
  221. .command {{
  222. background: #0f172a; border-radius: 8px; padding: 15px 20px;
  223. margin: 15px 0; font-family: monospace; color: #4ade80;
  224. text-align: left; overflow-x: auto;
  225. }}
  226. .note {{ color: #94a3b8; font-size: 14px; margin-top: 20px; }}
  227. </style>
  228. </head>
  229. <body>
  230. <div class="container">
  231. <h1>Setup Required</h1>
  232. <h2>Missing Python packages</h2>
  233. <div class="packages"><ul>{packages_html}</ul></div>
  234. <p>To fix, run this command on your server:</p>
  235. <div class="command">pip install -r requirements.lock</div>
  236. <p>Or if using a virtual environment:</p>
  237. <div class="command">./venv/bin/pip install -r requirements.lock</div>
  238. <p class="note">After installing, restart Bambuddy:<br>
  239. <code>sudo systemctl restart bambuddy</code></p>
  240. </div>
  241. </body>
  242. </html>"""
  243. class ErrorHandler(BaseHTTPRequestHandler):
  244. def do_GET(self):
  245. self.send_response(503)
  246. self.send_header("Content-type", "text/html")
  247. self.end_headers()
  248. self.wfile.write(html.encode())
  249. def log_message(self, format, *args):
  250. print(f"[Error Server] {args[0]}")
  251. port = int(os.environ.get("PORT", 8000))
  252. print(f"\nStarting error server on http://0.0.0.0:{port}")
  253. print("Visit this URL in your browser to see the error details.\n")
  254. server = HTTPServer(("0.0.0.0", port), ErrorHandler) # nosec B104
  255. def shutdown(signum, frame):
  256. print("\nShutting down error server...")
  257. raise SystemExit(0)
  258. signal.signal(signal.SIGTERM, shutdown)
  259. signal.signal(signal.SIGINT, shutdown)
  260. server.serve_forever()
  261. def check_dependencies():
  262. """Check that all required packages are installed."""
  263. missing = []
  264. # Map of import name -> package name (for pip install)
  265. required = {
  266. "jwt": "PyJWT",
  267. "fastapi": "fastapi",
  268. "uvicorn": "uvicorn",
  269. "sqlalchemy": "sqlalchemy",
  270. "aiosqlite": "aiosqlite",
  271. "pydantic": "pydantic",
  272. "paho.mqtt": "paho-mqtt",
  273. }
  274. for module, package in required.items():
  275. try:
  276. __import__(module)
  277. except ImportError:
  278. missing.append(package)
  279. if missing:
  280. print("\n" + "=" * 60)
  281. print("ERROR: Missing required Python packages!")
  282. print("=" * 60)
  283. print(f"\nMissing packages: {', '.join(missing)}")
  284. print("\nTo fix, run:")
  285. print(" pip install -r requirements.lock")
  286. print("\nOr if using a virtual environment:")
  287. print(" ./venv/bin/pip install -r requirements.lock")
  288. print("=" * 60 + "\n")
  289. _start_error_server(missing)
  290. check_dependencies()
  291. # =============================================================================
  292. # Import settings first for logging configuration
  293. # Configure logging based on settings
  294. # DEBUG=true -> DEBUG level, else use LOG_LEVEL setting
  295. log_level_str = "DEBUG" if app_settings.debug else app_settings.log_level.upper()
  296. log_level = getattr(logging, log_level_str, logging.INFO)
  297. # Trace ID column ([-] when no request scope is active — startup, MQTT
  298. # callbacks, scheduled tasks not chained from a request — so the column
  299. # stays visually aligned and missing values are obvious in grep). See
  300. # backend/app/core/trace.py for the ContextVar that feeds this slot.
  301. log_format = "%(asctime)s %(levelname)s [%(name)s] [%(trace_id)s] %(message)s"
  302. # Create root logger
  303. root_logger = logging.getLogger()
  304. root_logger.setLevel(log_level)
  305. # Trace-ID injection: this filter populates record.trace_id from the
  306. # per-request ContextVar so the format string above can reference it.
  307. # Attached to each HANDLER (not the root logger) because Python's
  308. # logging semantics only invoke a logger's filters on records that
  309. # *originated* at that logger — records propagated up from child
  310. # loggers (every named logger in the app) never trigger root's filter.
  311. # Putting it on the handlers means every record any handler emits gets
  312. # trace_id injected just before the formatter runs, regardless of which
  313. # logger created the record. Without this, the formatter raises
  314. # KeyError on every child-logger record and the record is silently
  315. # dropped — which is exactly the "logs/bambuddy.log only shows logs
  316. # partially" bug we hit. See backend/app/core/trace.py for the
  317. # ContextVar the filter reads.
  318. from backend.app.core.logging_filters import RefusedHeaderRedactFilter
  319. from backend.app.core.response_redaction import RefusedHeaderResponseMask
  320. from backend.app.core.trace import TraceIDFilter
  321. _trace_id_filter = TraceIDFilter()
  322. # Console handler - always enabled
  323. console_handler = logging.StreamHandler()
  324. console_handler.setLevel(log_level)
  325. console_handler.setFormatter(logging.Formatter(log_format))
  326. console_handler.addFilter(_trace_id_filter)
  327. # A refused header's value (often a token) masked before the line is written.
  328. _refused_header_filter = RefusedHeaderRedactFilter()
  329. console_handler.addFilter(_refused_header_filter)
  330. root_logger.addHandler(console_handler)
  331. def _attach_file_handler_to_uvicorn(handler: logging.Handler) -> None:
  332. """Send uvicorn's own loggers to bambuddy.log as well.
  333. Neither reaches the root handler: uvicorn.access has propagate=False, and
  334. uvicorn.error propagates only to "uvicorn", which has propagate=False.
  335. uvicorn.access is the record of which endpoint changed server state;
  336. uvicorn.error is where an unhandled route error is logged with its
  337. traceback ("Exception in ASGI application").
  338. """
  339. for name in ("uvicorn.access", "uvicorn.error"):
  340. logging.getLogger(name).addHandler(handler)
  341. # File handler - only in production or if explicitly enabled
  342. if app_settings.log_to_file:
  343. log_file = app_settings.log_dir / "bambuddy.log"
  344. file_handler = RotatingFileHandler(
  345. log_file,
  346. maxBytes=app_settings.log_max_bytes,
  347. backupCount=app_settings.log_backup_count,
  348. encoding="utf-8",
  349. )
  350. file_handler.setLevel(log_level)
  351. file_handler.setFormatter(logging.Formatter(log_format))
  352. file_handler.addFilter(_trace_id_filter)
  353. file_handler.addFilter(_refused_header_filter)
  354. root_logger.addHandler(file_handler)
  355. logging.info("Logging to file: %s", log_file)
  356. # Pipe uvicorn's HTTP access log to bambuddy.log too. Uvicorn ships its
  357. # access logger with propagate=False by default, so without this attach
  358. # there is no on-disk record of which endpoint triggered a server-state
  359. # change — the rogue stop_print mystery on 2026-04-26 was untraceable
  360. # for exactly this reason. Filtered to write methods only
  361. # (POST/PUT/PATCH/DELETE) so the high-volume status-poll GETs from the
  362. # frontend don't churn the rotation window faster than it's useful.
  363. from backend.app.core.logging_filters import (
  364. CancelledPoolNoiseFilter,
  365. WriteRequestsOnlyFilter,
  366. )
  367. _attach_file_handler_to_uvicorn(file_handler)
  368. uvicorn_access_logger = logging.getLogger("uvicorn.access")
  369. uvicorn_access_logger.addFilter(WriteRequestsOnlyFilter())
  370. # Uvicorn's access logger has propagate=False (its own default), so the
  371. # root-attached TraceIDFilter never sees these records. Attach a
  372. # second instance directly so HTTP access lines carry the same trace
  373. # ID column as the application logs they correlate with.
  374. uvicorn_access_logger.addFilter(TraceIDFilter())
  375. # Drop SQLAlchemy connection-pool log noise that's caused by Starlette's
  376. # BaseHTTPMiddleware cancelling the inner task scope on client
  377. # disconnect (#1112). The cancel-safe `get_db` already prevents the
  378. # underlying transaction leak; this filter only suppresses the residual
  379. # log records that pre-existing pools still emit during their cleanup.
  380. logging.getLogger("sqlalchemy.pool").addFilter(CancelledPoolNoiseFilter())
  381. # Query-string tokens out of uvicorn's request and WebSocket lines, on every
  382. # handler (console included), whether or not file logging is on.
  383. from backend.app.core.logging_filters import ( # noqa: E402
  384. QueryTokenRedactFilter,
  385. ShutdownCancelledRequestFilter,
  386. )
  387. # First, so the traceback of a stream cancelled at shutdown is dropped before
  388. # the filters below render it.
  389. logging.getLogger("uvicorn.error").addFilter(ShutdownCancelledRequestFilter())
  390. for _uvicorn_logger in ("uvicorn.access", "uvicorn.error"):
  391. logging.getLogger(_uvicorn_logger).addFilter(QueryTokenRedactFilter())
  392. logging.getLogger(_uvicorn_logger).addFilter(_refused_header_filter)
  393. # These stay at WARNING even in debug mode: at INFO/DEBUG httpx/httpcore log
  394. # full request URLs, and some carry credentials (bot tokens, webhook URLs, API
  395. # keys in the query). Same as the runtime log-level toggle in support.py.
  396. logging.getLogger("httpcore").setLevel(logging.WARNING)
  397. logging.getLogger("httpx").setLevel(logging.WARNING)
  398. # aiosqlite logs every statement with its parameters at DEBUG.
  399. logging.getLogger("aiosqlite").setLevel(logging.WARNING)
  400. # Reduce noise from third-party libraries in production
  401. if not app_settings.debug:
  402. logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
  403. logging.getLogger("paho.mqtt").setLevel(logging.WARNING)
  404. logging.info("Bambuddy starting - debug=%s, log_level=%s", app_settings.debug, log_level_str)
  405. # Track active prints: {(printer_id, filename): archive_id}
  406. _active_prints: dict[tuple[int, str], int] = {}
  407. # #1721: stage-22 pre-captured finish photo bytes per printer. on_finish_photo_moment
  408. # fires when stg_cur enters 22 ("Filament unloading") at end-of-print — toolhead
  409. # parked, bed not yet dropped — and grabs a single camera frame into this cache.
  410. # `_background_finish_photo` (inside on_print_complete) consumes the cached bytes
  411. # instead of running its own grab-now chain when present, so the finish photo
  412. # captures the better-framed pre-bed-drop moment without us having to force
  413. # timelapse on at dispatch (the #1397 mechanism that caused #1721's per-layer
  414. # nozzle parking on slicer profiles with Timelapse Type = Smooth).
  415. #
  416. # #2708: the bytes in here are ALWAYS already rotated by the printer's
  417. # camera_rotation. `on_finish_photo_moment` owns that, because one of its
  418. # sources (the #1867 in-print bank) is rotated before it ever reaches the
  419. # bank and the others are not — so the consumer can't tell them apart and
  420. # must not rotate again.
  421. _stage22_finish_frames: dict[int, bytes] = {}
  422. # #1790: per-printer producer-done event. Set by `on_finish_photo_moment` in its
  423. # `finally` block (whether it captured a frame or not). The consumer in
  424. # `_background_finish_photo` waits on it before reading `_stage22_finish_frames`
  425. # so the FINISH-state fallback path — where moment and completion are dispatched
  426. # back-to-back — doesn't race past the producer with an empty pop, and the
  427. # consumer's RTSP fallback can't collide with the producer's still-in-flight RTSP
  428. # grab (Bambu printers allow only one RTSP client at a time).
  429. _stage22_finish_in_flight: dict[int, asyncio.Event] = {}
  430. # #1867: rolling "last in-print camera frame" per printer. Refreshed on
  431. # layer-change and on print-progress advances (#2547) while the model is still
  432. # printing, then consumed by the FINISH-state finish-photo path when the
  433. # dispatcher recorded that it injected End G-code into this print. Bambu
  434. # reports gcode_state=FINISH AFTER the user End G-code (e.g. SwapMod
  435. # plate-swap) has run, so a live grab there would capture the swapped/empty
  436. # plate.
  437. #
  438. # The load-bearing property: both drivers are print telemetry that stops before
  439. # the End G-code executes — no further layer_num increases, and mc_percent
  440. # freezes — so the last banked frame is always the finished print before the
  441. # swap. Anything added as a third driver must hold that same property.
  442. _inprint_frame_bank: dict[int, bytes] = {}
  443. # Monotonic timestamp of the last banked frame per printer — throttles banking
  444. # so tall prints don't add a camera grab on every layer.
  445. _inprint_frame_bank_ts: dict[int, float] = {}
  446. # Minimum seconds between banked frames, except the final object layer which
  447. # always refreshes for the best framing.
  448. _INPRINT_BANK_MIN_INTERVAL = 25.0
  449. # Per-printer "connected" edge tracker. Used by `on_printer_status_change`
  450. # to fire `reconcile_stale_active_prints` exactly once per (re)connection
  451. # (#1542 follow-up — power-cycle ghost prints). The value is True after
  452. # the first connected status update for that connection; transitions back
  453. # to False whenever we observe `state.connected = False` so the next
  454. # reconnect re-arms reconciliation. Keyed by printer_id.
  455. _printer_reconciled_since_connect: dict[int, bool] = {}
  456. # Same edge, same keying, for priming the printer's calibration table exactly
  457. # once per (re)connection. Nothing else asks for it on connect: state.kprofiles
  458. # is otherwise filled only when someone opens the Profiles page or Configure
  459. # Slot, when a GitHub backup runs, or when the printer happens to answer
  460. # somebody else's query on the report topic. Until then the AMS slot card has
  461. # no K value to show on the printers whose trays carry none of their own
  462. # (#2854 — H2-series report cali_idx and nothing more).
  463. _printer_kprofiles_primed_since_connect: dict[int, bool] = {}
  464. # Track expected prints from reprint/scheduled (skip auto-archiving for these)
  465. # {(printer_id, filename): archive_id}
  466. _expected_prints: dict[tuple[int, str], int] = {}
  467. # Archives of the running prints Bambuddy dispatched itself: adopted from an
  468. # expected print, or found with Bambuddy's dispatch marker. _active_prints holds
  469. # every print on_print_start archived, those started from Bambu Studio, Handy or
  470. # the printer's screen included, so it is no proof of ownership on its own.
  471. _owned_print_archive_ids: set[int] = set()
  472. # What a dispatch registered for its print, keyed (printer_id, archive_id):
  473. # plates of one archive print on two printers at once, each with its own.
  474. # Track AMS mapping for prints: {(printer_id, archive_id): [global_tray_id_per_slot]}
  475. # Used by usage tracker to map 3MF slots to physical AMS trays
  476. _print_ams_mappings: dict[tuple[int, int], list[int]] = {}
  477. # Track cost center selection for the current print run: {(printer_id, archive_id): cost_center_id}
  478. _print_cost_center_ids: dict[tuple[int, int], int] = {}
  479. # Track plate_id for prints from multi-plate 3MFs: {(printer_id, archive_id): plate_id}
  480. # Used by usage tracker to scope 3MF parsing to the dispatched plate (#1697).
  481. # Populated by direct-Print and queue dispatch paths; queue prints also have a
  482. # redundant queue-item lookup in on_print_start so this dict isn't load-bearing
  483. # for the queue path. Cleared on print completion or TTL eviction.
  484. _print_plate_ids: dict[tuple[int, int], int] = {}
  485. # Track progress milestones for notifications: {printer_id: last_milestone_notified}
  486. # Milestones are 25, 50, 75. Value of 0 means no milestone notified yet for current print.
  487. _last_progress_milestone: dict[int, int] = {}
  488. # Track whether first layer complete notification has been sent for current print
  489. _first_layer_notified: dict[int, bool] = {}
  490. # Track whether we already sent a kill-switch stop for the current unauthorized print
  491. _unauthorized_print_kill_sent: set[int] = set()
  492. # The MQTT status callback is a hot path. Cache the two-setting kill-switch
  493. # lookup briefly so an unknown active print does not query the database on
  494. # every status frame. A short TTL keeps settings changes responsive.
  495. _KILL_SWITCH_SETTING_CACHE_TTL_SECONDS = 5.0
  496. _kill_switch_setting_cache: tuple[bool, float] | None = None
  497. # Provider notification started when the kill switch stops a print. The later
  498. # MQTT print-complete callback awaits this task and only sends its regular
  499. # provider notification when the immediate attempt failed.
  500. _kill_switch_notification_tasks: dict[int, asyncio.Task[bool]] = {}
  501. # HMS faults already notified, with when each was last seen:
  502. # {printer_id: {fault key: timestamp}}. Tracked per fault, not per printer: a
  503. # fault that flickers beside a held notice was forgotten on every gap and
  504. # notified on every return (#3226). The window is long because the measured
  505. # gaps reach 64 s.
  506. _notified_hms_errors: dict[int, dict[str, float]] = {}
  507. _HMS_CLEAR_GRACE_SECONDS = 600.0
  508. # The print state each printer was last seen in, and the faults recorded before
  509. # it last changed, which are forgotten as soon as they are gone (see
  510. # _take_new_hms_faults).
  511. _hms_print_state: dict[int, str] = {}
  512. _hms_forget_when_gone: dict[int, set[str]] = {}
  513. # Track timelapse file baselines at print start: {printer_id: set of video filenames}
  514. # Used for snapshot-diff detection at print completion
  515. _timelapse_baselines: dict[int, set[str]] = {}
  516. # Printers whose baseline above was taken off a card that could not be read,
  517. # so it is empty without the card being empty (#3412).
  518. _timelapse_baselines_unread: set[int] = set()
  519. # Track printers waiting for bed to cool after print completion.
  520. # Event-driven: fires when bed_temper arrives via MQTT below threshold.
  521. # {printer_id: {"threshold": float, "filename": str, "registered_at": float}}
  522. _bed_cool_waiters: dict[int, dict] = {}
  523. # Track printers where the user explicitly stopped the print from the queue UI.
  524. # When on_print_complete fires with status "failed" for these printers we treat it
  525. # as "cancelled" (stopped by user) so the correct notification email is sent.
  526. _user_stopped_printers: set[int] = set()
  527. # Printers whose running print the kill switch stopped. Their completion is a
  528. # cancel with the reason "killSwitch", not the "failed" the printer reports.
  529. _kill_switch_stopped: set[int] = set()
  530. # Offline-notification edge state (#1752): fire `on_printer_offline` exactly
  531. # once when a printer transitions connected → disconnected. `_printer_last_connected`
  532. # holds the previous observation so we only fire on the True → False edge (a
  533. # False → False repeat doesn't notify; an initial False at startup doesn't
  534. # notify either, since there's no prior True). `_printer_offline_notify_tasks`
  535. # holds the per-printer pending asyncio task that fires the notification
  536. # after a debounce window — cancelled if the printer reconnects before the
  537. # window elapses, so transient MQTT blips don't flood the user.
  538. _printer_last_connected: dict[int, bool] = {}
  539. # Printers whose relay offline event went out, so the reconnect publishes the
  540. # matching online event and a blip shorter than the debounce publishes neither.
  541. _printer_offline_published: set[int] = set()
  542. _printer_offline_notify_tasks: dict[int, asyncio.Task] = {}
  543. # Debounce: a printer must stay offline this long before we notify. Sized
  544. # against the staleness path (`bambu_mqtt.py::STALE_RECONNECT_COOLDOWN = 30s`)
  545. # so a single stale-trigger cooldown isn't enough to fire — only a real
  546. # offline that survives one reconnect attempt notifies.
  547. _PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS = 60.0
  548. # HMS short-code → failure_reason key. Used by _dispatch_archive_update
  549. # when status="failed" to label the print's failure_reason in archives.
  550. #
  551. # Earlier code matched on `module` alone (e.g. "any module 0x0C HMS → Layer shift"),
  552. # which is wrong on two counts:
  553. # 1. Real layer-shift codes live in module 0x03 (see Bambu wiki), not 0x0C.
  554. # 2. Module 0x0C is "Motion Controller" — broad category that also covers cameras
  555. # and visual markers, AND the H2D firmware emits a 0x0C HMS (0C00_001B, not in
  556. # the public wiki) as part of its user-cancel sequence. Matching on the module
  557. # alone caused user-cancellations to be archived as "Layer shift" failures.
  558. # We now match by full short code only — anything not in this map leaves
  559. # failure_reason=None rather than guessing.
  560. # Values are the canonical camelCase failure-reason keys, NOT display labels
  561. # (issue #2974). The vocabulary is enforced on writes by
  562. # ``_FAILURE_REASON_KEYS`` in ``api/routes/print_log.py`` and rendered through
  563. # ``t('editArchive.failureReasons.<key>')`` on both the archive editor and the
  564. # Statistics breakdown. Storing a label here instead put a second spelling of
  565. # the same cause into one column: the Failure Analysis widget groups on the raw
  566. # value, so a print the backend classified and an identical one a user
  567. # classified counted as two different reasons, and the label form could never
  568. # be translated because there was no key for ``t()`` to resolve.
  569. _HMS_FAILURE_REASONS: dict[str, str] = {
  570. # Layer shift / step loss
  571. "0300_4057": "layerShift",
  572. "0300_4068": "layerShift",
  573. "0300_800C": "layerShift",
  574. # Filament runout (printer-side & per-AMS-slot)
  575. "0300_8004": "filamentRunout",
  576. "0700_8011": "filamentRunout",
  577. "0701_8011": "filamentRunout",
  578. "0702_8011": "filamentRunout",
  579. "0703_8011": "filamentRunout",
  580. "0704_8011": "filamentRunout",
  581. "0705_8011": "filamentRunout",
  582. "0706_8011": "filamentRunout",
  583. "0707_8011": "filamentRunout",
  584. "07FF_8011": "filamentRunout",
  585. # Clogged nozzle / extruder
  586. "0300_4006": "cloggedNozzle",
  587. "0300_8016": "cloggedNozzle",
  588. "0300_801C": "cloggedNozzle",
  589. "0700_8003": "cloggedNozzle",
  590. "0700_8007": "cloggedNozzle",
  591. "0700_8013": "cloggedNozzle",
  592. "0701_8003": "cloggedNozzle",
  593. "0701_8007": "cloggedNozzle",
  594. "0701_8013": "cloggedNozzle",
  595. "0702_8003": "cloggedNozzle",
  596. # AI print monitoring — spaghetti / the model coming off the plate.
  597. # `spaghettiDetached` is a key the archive editor already offers for this
  598. # failure mode, not a new one, so a derived reason opens the dropdown on
  599. # that option rather than blank — and survives the next save, which clears
  600. # any value the editor does not recognise.
  601. #
  602. # A module-0x0C row is safe here despite the warning above: that warning is
  603. # about matching on the module alone, and 0C00_8042 is a full short code
  604. # with a documented meaning ("The AI print monitor has detected a spaghetti
  605. # defect", hms_errors.py). The H2D cancel echo is 0C00_001B, so the two
  606. # cannot collide.
  607. #
  608. # 0300_8003's own text ends "before continuing your print", but on an X2D
  609. # it arrives with the print already paused, offering only
  610. # RESUME_PRINTING_DEFECTS / STOP_PRINTING — a halt waiting on the user.
  611. #
  612. # Two neighbours are left out on purpose, so this does not get re-derived:
  613. # * 0C00_C004 "Possible spaghetti failure was detected." — "possible"
  614. # reads as a warning about a print that is still running, not a halt.
  615. # * 0300_800A is AI monitoring too, but it reports a filament pile-up in
  616. # the waste chute. That is not the print failing.
  617. #
  618. # That line is drawn from the text, not from `severity`, because severity
  619. # cannot draw it: 0300_8003 reaches us through `print_error`, a bare
  620. # module/error word with no level in it, and bambu_mqtt.py gives every
  621. # print_error entry a flat severity=3. Matching on the short code alone is
  622. # the right shape for derive_failure_reason, not an omission.
  623. "0300_8003": "spaghettiDetached",
  624. "0C00_8042": "spaghettiDetached",
  625. }
  626. def _hms_short_code(attr: int, code: int | str) -> str:
  627. """Build the canonical "MMMM_CCCC" HMS short code from raw attr/code values."""
  628. if isinstance(code, str):
  629. code_int = int(code.replace("0x", ""), 16) if code else 0
  630. else:
  631. code_int = int(code or 0)
  632. attr_int = int(attr or 0)
  633. return f"{(attr_int >> 16) & 0xFFFF:04X}_{code_int & 0xFFFF:04X}"
  634. def derive_failure_reason(status: str, hms_errors: list[dict] | None) -> str | None:
  635. """Derive the failure_reason key for an archived print.
  636. Returns "userCancelled" for cancelled/aborted prints; for failed prints,
  637. returns the first matching key from _HMS_FAILURE_REASONS, or None when
  638. no HMS code matches (don't guess — null is honest). The keys are the
  639. archive editor's vocabulary (_FAILURE_REASON_KEYS in print_log.py) and are
  640. translated at render time.
  641. """
  642. if status in ("aborted", "cancelled"):
  643. return "userCancelled"
  644. if status != "failed":
  645. return None
  646. for err in hms_errors or []:
  647. short_code = _hms_short_code(err.get("attr", 0), err.get("code", 0))
  648. if short_code in _HMS_FAILURE_REASONS:
  649. return _HMS_FAILURE_REASONS[short_code]
  650. return None
  651. # Track created_by_id for expected prints so the user email can be sent even when
  652. # the archive itself doesn't have created_by_id set (e.g. library-file-based prints).
  653. # {(printer_id, filename): created_by_id}
  654. _expected_print_creators: dict[tuple[int, str], int] = {}
  655. # Per-printer dedup for unknown_tag WS broadcasts. Keyed by
  656. # (ams_id, tray_id) -> (tag_uid, tray_uuid); we only re-broadcast when the
  657. # tag tuple changes for the slot. Cleared when the slot is reported empty
  658. # so remove + reinsert reliably re-prompts the UI.
  659. _unknown_tag_last_broadcast: dict[int, dict[tuple[int, int], tuple[str, str]]] = {}
  660. async def _broadcast_unknown_tag(
  661. *,
  662. printer_id: int,
  663. ams_id: int,
  664. tray_id: int,
  665. tag_uid: str,
  666. tray_uuid: str,
  667. tray_type: str | None = None,
  668. tray_color: str | None = None,
  669. tray_sub_brands: str | None = None,
  670. tray_count: int | None = None,
  671. ) -> None:
  672. """Broadcast unknown_tag, deduped so repeated MQTT pushes for the same slot+tag don't spam the UI."""
  673. _logger = logging.getLogger(__name__)
  674. slot_key = (ams_id, tray_id)
  675. tag_key = (tag_uid or "", tray_uuid or "")
  676. per_printer = _unknown_tag_last_broadcast.setdefault(printer_id, {})
  677. if per_printer.get(slot_key) == tag_key:
  678. _logger.debug(
  679. "unknown_tag deduped for printer=%d AMS=%d slot=%d tag=%s",
  680. printer_id,
  681. ams_id,
  682. tray_id,
  683. tag_key[0][:8] or tag_key[1][:8] or "(none)",
  684. )
  685. return
  686. _logger.info(
  687. "unknown_tag broadcast: printer=%d AMS=%d slot=%d type=%r color=%r tag=%s",
  688. printer_id,
  689. ams_id,
  690. tray_id,
  691. tray_type,
  692. tray_color,
  693. tag_key[0][:8] or tag_key[1][:8] or "(none)",
  694. )
  695. # Broadcast first; only commit the dedup if the WS write succeeds.
  696. # If broadcast raises, the next MQTT push retries instead of being
  697. # permanently silenced by a poisoned dedup entry.
  698. await ws_manager.broadcast(
  699. {
  700. "type": "unknown_tag",
  701. "printer_id": printer_id,
  702. "ams_id": ams_id,
  703. "tray_id": tray_id,
  704. "tag_uid": tag_uid,
  705. "tray_uuid": tray_uuid,
  706. "tray_type": tray_type,
  707. "tray_color": tray_color,
  708. "tray_sub_brands": tray_sub_brands,
  709. "tray_count": tray_count,
  710. }
  711. )
  712. per_printer[slot_key] = tag_key
  713. def _clear_unknown_tag_dedup(printer_id: int, ams_id: int, tray_id: int) -> None:
  714. """Drop the cached last-broadcast tag for a slot (called when slot reports empty or gets matched)."""
  715. per_printer = _unknown_tag_last_broadcast.get(printer_id)
  716. if per_printer is None:
  717. return
  718. per_printer.pop((ams_id, tray_id), None)
  719. # TTL for expected-print entries: evict registrations older than this to prevent
  720. # unbounded growth when a print is registered but never starts (e.g. printer
  721. # disconnect, app restart, print started from the printer panel).
  722. _EXPECTED_PRINT_TTL_SECONDS: int = 2 * 60 * 60 # 2 hours
  723. # Registration timestamps used for TTL eviction: {(printer_id, filename): monotonic_time}
  724. _expected_print_registered_at: dict[tuple[int, str], float] = {}
  725. # Cleanup loop interval
  726. _EXPECTED_PRINT_CLEANUP_INTERVAL: int = 15 * 60 # 15 minutes
  727. _expected_prints_cleanup_task: asyncio.Task | None = None
  728. _ACTIVE_PRINT_STATES: set[str] = {"RUNNING", "PRINTING", "PAUSE"}
  729. def _build_status_print_keys(printer_id: int, state: PrinterState) -> list[tuple[int, str]]:
  730. """Build filename keys for matching a printer status update to Bambuddy-owned jobs."""
  731. possible_keys: list[tuple[int, str]] = []
  732. filename = (state.gcode_file or state.current_print or "").strip()
  733. subtask_name = (state.subtask_name or "").strip()
  734. if subtask_name:
  735. possible_keys.append((printer_id, subtask_name))
  736. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  737. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  738. if filename:
  739. base_name = filename.rsplit("/", 1)[-1]
  740. if base_name.endswith(".gcode.3mf"):
  741. root_name = base_name[: -len(".gcode.3mf")]
  742. possible_keys.append((printer_id, root_name))
  743. possible_keys.append((printer_id, base_name))
  744. possible_keys.append((printer_id, f"{root_name}.gcode"))
  745. possible_keys.append((printer_id, f"{root_name}.3mf"))
  746. elif base_name.endswith(".3mf"):
  747. root_name = base_name[: -len(".3mf")]
  748. possible_keys.append((printer_id, root_name))
  749. possible_keys.append((printer_id, base_name))
  750. elif base_name.endswith(".gcode"):
  751. root_name = base_name[: -len(".gcode")]
  752. possible_keys.append((printer_id, root_name))
  753. possible_keys.append((printer_id, f"{root_name}.3mf"))
  754. possible_keys.append((printer_id, base_name))
  755. else:
  756. possible_keys.append((printer_id, base_name))
  757. possible_keys.append((printer_id, f"{base_name}.3mf"))
  758. return possible_keys
  759. async def _print_user_id_if_still_there(db, print_user: dict | None) -> int | None:
  760. """The id of who started the print, unless that user has been deleted since.
  761. Deleting a user forgets them in printer_manager, but a print that ended a
  762. moment before still holds them.
  763. """
  764. if not print_user or print_user.get("user_id") is None:
  765. return None
  766. from backend.app.models.user import User
  767. return await db.scalar(select(User.id).where(User.id == print_user["user_id"]))
  768. def _is_bambuddy_authorized_print_in_memory(printer_id: int, state: PrinterState) -> bool:
  769. """Check the cheap, process-local print ownership signals."""
  770. if printer_manager.get_current_print_user(printer_id):
  771. return True
  772. return any(
  773. key in _expected_prints or _active_prints.get(key) in _owned_print_archive_ids
  774. for key in _build_status_print_keys(printer_id, state)
  775. )
  776. def _reported_print_names(printer_id: int, state: PrinterState) -> set[str]:
  777. """The names the printer reports for its print, as it echoes them (spaces become underscores)."""
  778. return {_normalise_subtask_name(name) for _pid, name in _build_status_print_keys(printer_id, state)} - {""}
  779. async def _printing_archive_without_job_id(
  780. db, printer_id: int, reported: set[str], *, dispatched_only: bool, live_only: bool = False
  781. ):
  782. """The newest printing archive on ``printer_id`` of a print that reported no job id, under a reported name.
  783. An archive with a real id of its own is a print that reported one: a "0"
  784. print is not it, even under the same name (one whose completion was missed
  785. would otherwise cover it). ``dispatched_only`` keeps to archives carrying
  786. Bambuddy's dispatch marker; ``live_only`` leaves out deleted ones.
  787. """
  788. from backend.app.models.archive import PrintArchive
  789. query = select(PrintArchive).where(
  790. PrintArchive.printer_id == printer_id,
  791. PrintArchive.status == "printing",
  792. or_(PrintArchive.subtask_id.is_(None), PrintArchive.subtask_id.in_(("", "0"))),
  793. )
  794. if dispatched_only:
  795. query = query.where(or_(PrintArchive.billing_run_id.is_not(None), PrintArchive.created_by_id.is_not(None)))
  796. if live_only:
  797. query = query.where(PrintArchive.deleted_at.is_(None))
  798. archives = await db.execute(query.order_by(PrintArchive.created_at.desc()))
  799. return next((a for a in archives.scalars() if _archive_name_forms(a) & reported), None)
  800. async def running_print_archive(db, printer_id: int, state: PrinterState, *, printing_only: bool, by_name: bool):
  801. """The archive of the print running on ``printer_id``, or None (#3327).
  802. By the print's subtask_id first, which the firmware mints per print; with
  803. ``printing_only`` only an archive that is still printing (a leftover row of
  804. a completion never seen must not stand in for another job). A print
  805. started outside the cloud (Bambu Studio over LAN, Handy, the printer's
  806. screen) reports subtask_id "0", which links it to nothing: the archive
  807. Bambuddy bound to the running print comes next, the one completion charges
  808. and photographs. A restart forgets that binding and rebinds only Bambuddy's
  809. own prints, so with ``by_name`` a "0" print last falls back to its printing
  810. archive under the name the printer reports. That can be an earlier run of
  811. the same name whose completion was missed while Bambuddy was down: good
  812. enough for a plate label or a cover, not for which objects skip-objects
  813. offers, so those callers leave it off. None until the archive exists, and
  814. never a deleted one.
  815. """
  816. from backend.app.models.archive import PrintArchive
  817. from backend.app.services.print_log import open_run_on_printer
  818. subtask_id = str(getattr(state, "subtask_id", "") or "").strip()
  819. if subtask_id not in ("", "0"):
  820. # The run open on the printer for this job: the archive row is shared
  821. # by every plate of it, its printer and job id the latest start's.
  822. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  823. if run is not None and run.archive_id is not None:
  824. archive = await db.scalar(
  825. select(PrintArchive).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  826. )
  827. if archive is not None:
  828. return archive
  829. query = select(PrintArchive).where(
  830. PrintArchive.printer_id == printer_id,
  831. PrintArchive.subtask_id == subtask_id,
  832. PrintArchive.deleted_at.is_(None),
  833. )
  834. if printing_only:
  835. query = query.where(PrintArchive.status == "printing")
  836. archive = await db.scalar(query.order_by(PrintArchive.created_at.desc()).limit(1))
  837. if archive is not None:
  838. return archive
  839. bound = next(
  840. (_active_prints[key] for key in _build_status_print_keys(printer_id, state) if key in _active_prints),
  841. None,
  842. )
  843. if bound is not None:
  844. return await db.scalar(select(PrintArchive).where(PrintArchive.id == bound, PrintArchive.deleted_at.is_(None)))
  845. if not by_name or subtask_id != "0":
  846. return None
  847. reported = _reported_print_names(printer_id, state)
  848. if not reported:
  849. return None
  850. return await _printing_archive_without_job_id(db, printer_id, reported, dispatched_only=False, live_only=True)
  851. def _bind_active_print(
  852. printer_id: int,
  853. archive_id: int,
  854. archive_filename: str | None,
  855. filename: str | None = None,
  856. subtask_name: str | None = None,
  857. ) -> None:
  858. """Track a running print under every name its completion may report.
  859. One set for every path that binds a print (new archive, expected print,
  860. restart resume, fallback): the archive's file name, the name the printer
  861. reported, and the subtask with and without ``.3mf``. Paths that registered
  862. fewer names lost the bound archive at completion, and with it the plate
  863. restore's height (#3240), whenever MQTT reported a name they had left out.
  864. """
  865. for name in {archive_filename, filename, f"{subtask_name}.3mf" if subtask_name else None, subtask_name}:
  866. if name:
  867. _active_prints[(printer_id, name)] = archive_id
  868. async def _archive_of_open_run(printer_id: int, subtask_id: str | None = None) -> int | None:
  869. """The archive of the run open on the printer, if it is still there.
  870. With ``subtask_id``, only a run of that job."""
  871. try:
  872. from backend.app.models.archive import PrintArchive
  873. from backend.app.services.print_log import open_run_on_printer
  874. async with async_session() as db:
  875. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  876. if run is None or run.archive_id is None:
  877. return None
  878. return await db.scalar(
  879. select(PrintArchive.id).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  880. )
  881. except Exception as e:
  882. logging.getLogger(__name__).warning("[RUN] Could not read the open run of printer %s: %s", printer_id, e)
  883. return None
  884. async def _announce_discarded_runs(discarded, printer_id: int) -> None:
  885. """Publish the runs ``discard_open_runs`` dropped (archive redesign):
  886. each announced by run/started, now gone without an end."""
  887. if not discarded:
  888. return
  889. try:
  890. from backend.app.models.printer import Printer
  891. async with async_session() as db:
  892. printer = (
  893. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  894. ).first()
  895. if printer is not None:
  896. for run in discarded:
  897. await mqtt_relay.on_run_discarded(run, printer.serial_number, printer.name)
  898. except Exception as e:
  899. logging.getLogger(__name__).debug("[RUN] Could not announce discarded runs on printer %s: %s", printer_id, e)
  900. async def _publish_run_completed(run_id: int | None, printer_id: int) -> None:
  901. """Publish the run a print's end just wrote on the MQTT relay (archive
  902. redesign); nothing without one. Never in the way of the completion."""
  903. if run_id is None:
  904. return
  905. try:
  906. from backend.app.models.archive import PrintArchive
  907. from backend.app.models.print_log import PrintLogEntry
  908. from backend.app.models.printer import Printer
  909. async with async_session() as db:
  910. run = await db.get(PrintLogEntry, run_id)
  911. printer = (
  912. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  913. ).first()
  914. archive = await db.get(PrintArchive, run.archive_id) if run is not None and run.archive_id else None
  915. if run is not None and printer is not None:
  916. await mqtt_relay.on_run_completed(run, archive, printer.serial_number, printer.name)
  917. except Exception as e:
  918. logging.getLogger(__name__).debug("[RUN] Could not publish the end of run %s: %s", run_id, e)
  919. async def _open_run_for(archive_id: int, printer_id: int, subtask_id: str | None = None) -> None:
  920. """Write the run of the print that just started on the archive (archive
  921. redesign). In a session of its own and never in the way of the print
  922. start: until its readers move over, the archive still carries the print.
  923. The archive's start must be committed before. ``subtask_id`` is the
  924. printer's job, which finds the run again at a resume."""
  925. try:
  926. from backend.app.models.archive import PrintArchive
  927. from backend.app.models.printer import Printer
  928. from backend.app.services.print_log import open_run, open_run_on_printer, run_snapshot
  929. async with async_session() as db:
  930. archive = await db.get(PrintArchive, archive_id)
  931. if archive is None:
  932. return
  933. printer = (
  934. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  935. ).first()
  936. printer_name = printer.name if printer is not None else None
  937. already = await open_run_on_printer(db, printer_id)
  938. before = run_snapshot(already) if already is not None else None
  939. run = await open_run(
  940. db,
  941. archive,
  942. printer_id=printer_id,
  943. printer_name=printer_name,
  944. plate_id=_print_plate_ids.get((printer_id, archive_id)),
  945. subtask_id=subtask_id,
  946. )
  947. await db.commit()
  948. # Once per print: a resume or a repeated start finds the run it opened.
  949. # A new one replaced the run still open there (open_run): that one's
  950. # print ended unseen.
  951. # Compared by start too: SQLite can give the new run the id of the one
  952. # it replaced.
  953. replaced = (
  954. run is not None and before is not None and (before.id != run.id or before.started_at != run.started_at)
  955. )
  956. if run is not None and printer is not None and (before is None or replaced):
  957. if replaced:
  958. await _announce_discarded_runs([before], printer_id)
  959. try:
  960. await mqtt_relay.on_run_started(run, archive, printer.serial_number, printer_name)
  961. except Exception:
  962. pass # Don't fail the print start if MQTT fails
  963. except Exception as e:
  964. logging.getLogger(__name__).warning("[RUN] Could not open the run of archive %s: %s", archive_id, e)
  965. async def _drop_replaced_plate_videos(
  966. db, archive_id: int, plate_id: int | None, stale_relpath: str | None, logger: logging.Logger
  967. ) -> None:
  968. """Delete the videos a reprint of a plate replaces: ``stale_relpath``, the
  969. one the archive showed (when print start judged it this plate's), and
  970. those of the archive's earlier runs of the same plate. A plate keeps one
  971. video, as when the archive held the only one; another plate's stays. The
  972. runs that held them lose them. Commits."""
  973. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, PrintLogEntry
  974. from backend.app.utils.safe_path import PathTraversalError, safe_join_under
  975. runs = select(PrintLogEntry).where(
  976. PrintLogEntry.archive_id == archive_id, PrintLogEntry.timelapse_path.is_not(None)
  977. )
  978. with_video = list(await db.scalars(runs.execution_options(**{INCLUDE_OPEN_RUNS: True})))
  979. paths = {stale_relpath} if stale_relpath else set()
  980. if plate_id is not None:
  981. paths |= {r.timelapse_path for r in with_video if r.plate_id == plate_id}
  982. # A video another plate's run holds is that plate's.
  983. paths -= {r.timelapse_path for r in with_video if r.plate_id is not None and r.plate_id != plate_id}
  984. if not paths:
  985. return
  986. for run in with_video:
  987. if run.timelapse_path in paths:
  988. run.timelapse_path = None
  989. await db.commit()
  990. for relpath in sorted(p for p in paths if p):
  991. try:
  992. stale_path = safe_join_under(app_settings.base_dir, relpath, http=False)
  993. if stale_path.is_file():
  994. stale_path.unlink()
  995. logger.info("Deleted stale timelapse %s on reprint of archive %s", relpath, archive_id)
  996. except (OSError, PathTraversalError) as e:
  997. logger.warning("Failed to delete stale timelapse %s on reprint: %s", relpath, e)
  998. def _mark_owned_if_dispatched(archive) -> None:
  999. """Remember a running print's archive as Bambuddy's when it carries a
  1000. dispatch marker Bambuddy writes itself: ``billing_run_id`` (minted per
  1001. dispatch in the scheduler) or ``created_by_id`` (from the queue item)."""
  1002. if archive.billing_run_id is not None or archive.created_by_id is not None:
  1003. _owned_print_archive_ids.add(archive.id)
  1004. async def _is_printer_kill_switch_enabled_cached() -> bool:
  1005. """Return the kill-switch setting without querying on every MQTT frame."""
  1006. global _kill_switch_setting_cache
  1007. now = time.monotonic()
  1008. if _kill_switch_setting_cache is not None:
  1009. enabled, expires_at = _kill_switch_setting_cache
  1010. if now < expires_at:
  1011. return enabled
  1012. async with async_session() as db:
  1013. from backend.app.services.finance_budget import is_printer_kill_switch_enabled
  1014. enabled = await is_printer_kill_switch_enabled(db)
  1015. _kill_switch_setting_cache = (enabled, now + _KILL_SWITCH_SETTING_CACHE_TTL_SECONDS)
  1016. return enabled
  1017. async def _is_bambuddy_authorized_print(printer_id: int, state: PrinterState, db) -> bool | None:
  1018. """Resolve whether the current print was started by Bambuddy.
  1019. ``None`` means identity is not yet safe to decide. The kill switch must
  1020. defer in that case: stopping a print is irreversible, and the first status
  1021. frames after a restart may arrive before all subtask fields are populated.
  1022. """
  1023. if _is_bambuddy_authorized_print_in_memory(printer_id, state):
  1024. return True
  1025. # The printer's built-in calibration runs (Bambuddy's Calibrate, the
  1026. # printer's screen) play gcode from its read-only system partition, where no
  1027. # user file can be. Only that path counts: a calibration *name* proves
  1028. # nothing, since Bambu Studio's pressure-advance print reports a user-style
  1029. # path, and any print can be given that name.
  1030. if str(getattr(state, "gcode_file", "") or "").startswith("/usr/"):
  1031. return True
  1032. # A1, A1 Mini, P1 and A2L report the same calibration as a bare file name,
  1033. # which nothing distinguishes from a user's file given that name, so it is
  1034. # not exempt: with the kill switch on, calibrate on those printers with the
  1035. # switch off.
  1036. possible_keys = _build_status_print_keys(printer_id, state)
  1037. # In-memory ownership is lost on every Bambuddy restart, so fall back to what
  1038. # is on disk. subtask_id is minted per print and pins the answer to the job
  1039. # actually running, rather than to an unrelated one that reuses a filename.
  1040. raw_subtask_id = getattr(state, "subtask_id", None)
  1041. subtask_id = str(raw_subtask_id).strip() if raw_subtask_id is not None else ""
  1042. if subtask_id == "":
  1043. # Not reported yet: the first frames after a restart can lack it.
  1044. return None
  1045. from backend.app.models.archive import PrintArchive
  1046. if subtask_id == "0":
  1047. # A print started outside the cloud from Bambu Studio, Handy or the
  1048. # printer's screen reports "0", so there is no id to match: one of
  1049. # Bambuddy's printing archives on this printer under the name the
  1050. # printer reports, compared as the printer echoes names (spaces become
  1051. # underscores). Nothing reported to compare yet: wait.
  1052. reported = _reported_print_names(printer_id, state)
  1053. if not reported:
  1054. return None
  1055. archive = await _printing_archive_without_job_id(db, printer_id, reported, dispatched_only=True)
  1056. else:
  1057. from backend.app.services.print_log import open_run_on_printer
  1058. # This job's run open on the printer: the archive is shared by every
  1059. # plate of it, and its status, printer and job id are the latest
  1060. # start's, which may be another printer's.
  1061. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  1062. archive = await db.get(PrintArchive, run.archive_id) if run is not None and run.archive_id else None
  1063. if archive is None:
  1064. result = await db.execute(
  1065. select(PrintArchive)
  1066. .where(
  1067. PrintArchive.printer_id == printer_id,
  1068. PrintArchive.status == "printing",
  1069. PrintArchive.subtask_id == subtask_id,
  1070. )
  1071. .order_by(PrintArchive.created_at.desc())
  1072. .limit(1)
  1073. )
  1074. archive = result.scalar_one_or_none()
  1075. # An archive row on its own proves nothing: `on_print_start` archives every
  1076. # print it observes, including ones started from Bambu Studio or Handy, and
  1077. # stamps them with the same status and subtask_id. Authorizing on its mere
  1078. # existence would disable the kill switch the moment the 3MF finishes
  1079. # downloading. Only a dispatch marker Bambuddy writes itself counts —
  1080. # `billing_run_id` (minted per dispatch in the scheduler) or `created_by_id`
  1081. # (carried over from the queue item that started it).
  1082. if archive is not None and (archive.billing_run_id is not None or archive.created_by_id is not None):
  1083. # Rehydrate the fast in-memory path for subsequent status frames. Include
  1084. # both the archive filename and every normalized key reported by MQTT.
  1085. _owned_print_archive_ids.add(archive.id)
  1086. _active_prints[(printer_id, archive.filename)] = archive.id
  1087. for key in possible_keys:
  1088. _active_prints[key] = archive.id
  1089. return True
  1090. # No dispatch marker. Before calling this someone else's print, check whether
  1091. # Bambuddy has a job of its own running on this printer: a library-file
  1092. # dispatch has no archive at send time, and an archive created seconds later
  1093. # by `on_print_start` carries neither marker. The queue row, which the
  1094. # scheduler commits to status="printing" before the MQTT send, is the one
  1095. # durable record every Bambuddy print has. It cannot be tied to this
  1096. # subtask_id, so it is grounds to defer, never to authorize — stopping a
  1097. # print is irreversible, and refusing to act costs nothing but a log line.
  1098. from backend.app.models.print_queue import PrintQueueItem
  1099. dispatched_here = await db.scalar(
  1100. select(PrintQueueItem.id)
  1101. .where(
  1102. PrintQueueItem.printer_id == printer_id,
  1103. PrintQueueItem.status == "printing",
  1104. )
  1105. .limit(1)
  1106. )
  1107. if dispatched_here is not None:
  1108. return None
  1109. return False
  1110. async def _send_kill_switch_provider_notification(
  1111. printer_id: int,
  1112. printer_name: str,
  1113. data: dict,
  1114. ) -> bool:
  1115. """Send the immediate print-stopped provider notification.
  1116. Returning a success flag lets the normal MQTT completion path retry when
  1117. this early notification could not be delivered.
  1118. """
  1119. logger = logging.getLogger(__name__)
  1120. try:
  1121. async with async_session() as db:
  1122. from backend.app.services.print_log import open_run_on_printer
  1123. # The print it stopped, for {archive_id} / {run_id}: the completion
  1124. # that would name them skips its notification after this one.
  1125. run = await open_run_on_printer(db, printer_id)
  1126. await notification_service.on_print_complete(
  1127. printer_id,
  1128. printer_name,
  1129. "stopped",
  1130. data,
  1131. db,
  1132. archive_data={"archive_id": run.archive_id, "run_id": run.id} if run is not None else None,
  1133. )
  1134. return True
  1135. except Exception as e:
  1136. logger.warning(
  1137. "[KILL SWITCH] Immediate provider notification failed for printer %s: %s",
  1138. printer_id,
  1139. e,
  1140. )
  1141. return False
  1142. async def _kill_switch_notification_already_sent(task: asyncio.Task[bool] | None) -> bool:
  1143. """Wait for an immediate kill-switch notification, if one was scheduled."""
  1144. if task is None:
  1145. return False
  1146. try:
  1147. return await task
  1148. except Exception as e:
  1149. logging.getLogger(__name__).warning("[KILL SWITCH] Notification task failed: %s", e)
  1150. return False
  1151. async def _run_needs_energy_start(archive, printer_id: int, db) -> bool:
  1152. """At a resume: whether the print's run still lacks its energy start
  1153. reading. The archive's may be an earlier print's. Commits a reading
  1154. found on the archive for this run onto it."""
  1155. try:
  1156. from backend.app.services.print_log import run_needs_energy_start
  1157. needs = await run_needs_energy_start(db, archive, printer_id=printer_id)
  1158. await db.commit()
  1159. return needs
  1160. except Exception as e:
  1161. logging.getLogger(__name__).warning("[RUN] Energy start of archive %s's run not checked: %s", archive.id, e)
  1162. return archive.energy_start_kwh is None
  1163. async def _record_energy_start(archive, printer_id: int, db, *, context: str = "") -> bool:
  1164. """Capture the smart plug lifetime counter on the archive at print start.
  1165. Persists `energy_start_kwh` on the archive row (#941) so per-print energy
  1166. tracking survives a backend restart mid-print. The print-end handler reads
  1167. this value back from the DB and computes the delta against the current
  1168. plug counter.
  1169. """
  1170. _logger = logging.getLogger(__name__)
  1171. try:
  1172. candidates = await energy_plug_candidates(db, printer_id)
  1173. if not candidates:
  1174. _logger.info("[ENERGY] No smart plug for printer %s (archive %s)", printer_id, archive.id)
  1175. return False
  1176. selected = await select_energy_reading(candidates, _get_plug_energy, db)
  1177. if selected is None:
  1178. # Naming the plugs matters here: with several linked to one printer
  1179. # this is the difference between "the meter is offline" and "you
  1180. # linked only accessories" (#2859).
  1181. _logger.warning(
  1182. "[ENERGY] No plug on printer %s reports a lifetime energy counter for archive %s (tried: %s)",
  1183. printer_id,
  1184. archive.id,
  1185. ", ".join(plug.name for plug in candidates),
  1186. )
  1187. return False
  1188. plug, energy = selected
  1189. archive.energy_start_kwh = float(energy["total"])
  1190. # With the plug's snapshots, what lets the cost follow the price over
  1191. # the print rather than take the price at its end (#1251). A price that
  1192. # can't be had must not cost the print its energy reading.
  1193. archive.energy_start_at = utcnow_naive()
  1194. archive.energy_start_plug_id = plug.id
  1195. try:
  1196. from backend.app.services.energy_price import current_price
  1197. archive.energy_start_price = await current_price(db)
  1198. except Exception as e:
  1199. _logger.warning("[ENERGY] Could not read the electricity price for archive %s: %s", archive.id, e)
  1200. archive.energy_start_price = None
  1201. try:
  1202. from backend.app.services.print_log import mirror_energy_start
  1203. await mirror_energy_start(db, archive, printer_id=printer_id)
  1204. except Exception as e:
  1205. _logger.warning("[RUN] Energy start not copied onto the run of archive %s: %s", archive.id, e)
  1206. await db.commit()
  1207. _logger.info(
  1208. "[ENERGY] Recorded starting energy%s for archive %s from plug '%s': %s kWh",
  1209. f" ({context})" if context else "",
  1210. archive.id,
  1211. plug.name,
  1212. energy["total"],
  1213. )
  1214. return True
  1215. except Exception as e:
  1216. _logger.warning("[ENERGY] Failed to record starting energy for archive %s: %s", archive.id, e)
  1217. return False
  1218. def forget_print_creator(user_id: int) -> None:
  1219. """The print-start emails of a deleted user's jobs go to nobody, not to
  1220. whoever is given the id next (SQLite reuses it)."""
  1221. for key in [key for key, creator in _expected_print_creators.items() if creator == user_id]:
  1222. del _expected_print_creators[key]
  1223. async def forget_printer(printer_id: int) -> None:
  1224. """Drop what is kept for a deleted printer's id, here and in the services.
  1225. SQLite used to give the id to the next printer created, which inherited it
  1226. all: the running print's owner (its prints passed the kill switch and were
  1227. billed to that user), the plate-clear gate, pending and suppressed
  1228. notifications, the camera feed, the plate references. Ids are no longer
  1229. reused (AUTOINCREMENT), but nothing kept for a printer that is gone should
  1230. stay. Called after the delete is committed, and again by work that was in
  1231. flight and wrote state back.
  1232. Every printer-keyed map of this module is listed here or, with the reason,
  1233. in tests/unit/test_forget_printer.py.
  1234. """
  1235. from backend.app.api.routes import camera
  1236. from backend.app.services import (
  1237. bambu_ftp,
  1238. kprofile_drift,
  1239. layer_timelapse,
  1240. plate_detection,
  1241. print_dispatch_context,
  1242. usage_tracker,
  1243. )
  1244. from backend.app.services.notify_live_activities import notify_live_activities
  1245. from backend.app.services.obico_detection import obico_detection_service
  1246. from backend.app.services.printer_file_cache import printer_file_cache
  1247. # Keyed (printer_id, filename). The archive-keyed maps of those prints go too.
  1248. archive_ids = set()
  1249. for tracked in (_active_prints, _expected_prints):
  1250. for key in [key for key in tracked if key[0] == printer_id]:
  1251. archive_ids.add(tracked.pop(key))
  1252. for archive_id in archive_ids:
  1253. _owned_print_archive_ids.discard(archive_id)
  1254. for registered in (_print_ams_mappings, _print_cost_center_ids, _print_plate_ids):
  1255. for key in [key for key in registered if key[0] == printer_id]:
  1256. del registered[key]
  1257. for tracked in (_expected_print_creators, _expected_print_registered_at):
  1258. for key in [key for key in tracked if key[0] == printer_id]:
  1259. del tracked[key]
  1260. for key in [key for key in _ams_index_only_logged if key[0] == printer_id]:
  1261. _ams_index_only_logged.discard(key)
  1262. for key in [key for key in _ams_alarm_cooldown if key.split(":", 1)[0] == str(printer_id)]:
  1263. del _ams_alarm_cooldown[key]
  1264. # Pending work that would act on the id: an offline notice, a 3MF retry.
  1265. # A kill-switch notice is about a stop that happened and is left to finish.
  1266. for tasks in (_printer_offline_notify_tasks, _fallback_3mf_retry_tasks):
  1267. task = tasks.pop(printer_id, None)
  1268. if task is not None:
  1269. task.cancel()
  1270. for keyed in (
  1271. _stage22_finish_frames,
  1272. _stage22_finish_in_flight,
  1273. _inprint_frame_bank,
  1274. _inprint_frame_bank_ts,
  1275. _printer_reconciled_since_connect,
  1276. _printer_kprofiles_primed_since_connect,
  1277. _last_progress_milestone,
  1278. _first_layer_notified,
  1279. _kill_switch_notification_tasks,
  1280. _notified_hms_errors,
  1281. _hms_print_state,
  1282. _hms_forget_when_gone,
  1283. _timelapse_baselines,
  1284. _bed_cool_waiters,
  1285. _printer_last_connected,
  1286. _unknown_tag_last_broadcast,
  1287. _last_status_broadcast,
  1288. _connection_watchdog_last_attempt,
  1289. ):
  1290. keyed.pop(printer_id, None)
  1291. for marked in (
  1292. _unauthorized_print_kill_sent,
  1293. _user_stopped_printers,
  1294. _kill_switch_stopped,
  1295. _printer_offline_published,
  1296. _nozzle_count_updated,
  1297. _runtime_seen_connected,
  1298. _timelapse_baselines_unread,
  1299. ):
  1300. marked.discard(printer_id)
  1301. printer_manager.forget_printer(printer_id)
  1302. printer_file_cache.forget_printer(printer_id)
  1303. print_dispatch_context.clear(printer_id)
  1304. kprofile_drift.forget_printer(printer_id, deleted=True)
  1305. layer_timelapse.cancel_session(printer_id)
  1306. usage_tracker.forget_printer(printer_id)
  1307. obico_detection_service.forget_printer(printer_id)
  1308. octoeverywhere_detection_service.forget_printer(printer_id)
  1309. bambu_ftp.clear_3mf_cache(printer_id)
  1310. await camera.forget_printer(printer_id)
  1311. printers.clear_cover_cache(printer_id)
  1312. print_scheduler.forget_printer(printer_id)
  1313. notify_live_activities.forget_printer(printer_id)
  1314. plate_detection.remove_printer_references(printer_id)
  1315. def register_expected_print(
  1316. printer_id: int,
  1317. filename: str,
  1318. archive_id: int,
  1319. ams_mapping: list[int] | None = None,
  1320. created_by_id: int | None = None,
  1321. cost_center_id: int | None = None,
  1322. plate_id: int | None = None,
  1323. ):
  1324. """Register an expected print from reprint/scheduled so we don't create duplicate archives."""
  1325. # Store with multiple filename variations to catch different naming patterns
  1326. _expected_prints[(printer_id, filename)] = archive_id
  1327. # Also store without .3mf extension if present
  1328. if filename.endswith(".3mf"):
  1329. base = filename[:-4]
  1330. _expected_prints[(printer_id, base)] = archive_id
  1331. _expected_prints[(printer_id, f"{base}.gcode")] = archive_id
  1332. # Store AMS mapping for usage tracking at print completion, and plate_id for
  1333. # a single-plate dispatch from a multi-plate 3MF — without it, the direct-Print
  1334. # path attributes the whole file's filament total to the spool instead of just
  1335. # the printed plate (#1697). They describe this dispatch: one that has none
  1336. # drops what an earlier dispatch of the archive left (sent, never started),
  1337. # unless a plate of the archive is printing now and still needs them.
  1338. dispatch = (printer_id, archive_id)
  1339. archive_printing = _printing_here(printer_id, archive_id)
  1340. if ams_mapping is not None:
  1341. _print_ams_mappings[dispatch] = ams_mapping
  1342. elif not archive_printing:
  1343. _print_ams_mappings.pop(dispatch, None)
  1344. if cost_center_id is not None:
  1345. _print_cost_center_ids[dispatch] = cost_center_id
  1346. elif not archive_printing:
  1347. _print_cost_center_ids.pop(dispatch, None)
  1348. if plate_id is not None:
  1349. _print_plate_ids[dispatch] = plate_id
  1350. elif not archive_printing:
  1351. _print_plate_ids.pop(dispatch, None)
  1352. # Store created_by_id so the user start email can be sent even when the archive
  1353. # itself has no created_by_id (e.g. library-file-based queue prints)
  1354. if created_by_id is not None:
  1355. _expected_print_creators[(printer_id, filename)] = created_by_id
  1356. if filename.endswith(".3mf"):
  1357. base = filename[:-4]
  1358. _expected_print_creators[(printer_id, base)] = created_by_id
  1359. _expected_print_creators[(printer_id, f"{base}.gcode")] = created_by_id
  1360. # Record registration time for TTL-based eviction
  1361. _registered_at = time.monotonic()
  1362. _expected_print_registered_at[(printer_id, filename)] = _registered_at
  1363. if filename.endswith(".3mf"):
  1364. base = filename[:-4]
  1365. _expected_print_registered_at[(printer_id, base)] = _registered_at
  1366. _expected_print_registered_at[(printer_id, f"{base}.gcode")] = _registered_at
  1367. logging.getLogger(__name__).info(
  1368. f"Registered expected print: printer={printer_id}, file={filename}, archive={archive_id}, ams_mapping={ams_mapping}, plate_id={plate_id}"
  1369. )
  1370. def unregister_expected_print(printer_id: int, filename: str, archive_id: int) -> None:
  1371. """Undo :func:`register_expected_print` when the print never went out.
  1372. Registration has to happen *before* the MQTT print command, because the
  1373. printer can report the print before the line after the send executes. So
  1374. every path that registers and then fails to send — a cancel winning the
  1375. #1853 CAS race, a ``start_print()`` that returns False, or any exception in
  1376. between — leaves an expectation for a print that will never arrive.
  1377. The TTL sweep evicts those after two hours, which is far longer than it
  1378. takes a user to react to a failed dispatch by pressing print again: that
  1379. reprint would be folded into the *old* archive and take the stale
  1380. ``ams_mapping`` / ``plate_id`` with it. Hence the explicit inverse.
  1381. Mirrors the sweep's rules, including the one that is easy to get wrong:
  1382. ``_print_ams_mappings`` / ``_print_plate_ids`` / ``_print_cost_center_ids``
  1383. are keyed by printer and archive, not by file, so they may only be dropped
  1384. once no live key of the printer still points at that archive.
  1385. """
  1386. keys = [(printer_id, filename)]
  1387. if filename.endswith(".3mf"):
  1388. base = filename[:-4]
  1389. keys.append((printer_id, base))
  1390. keys.append((printer_id, f"{base}.gcode"))
  1391. removed = False
  1392. for key in keys:
  1393. if _expected_prints.pop(key, None) is not None:
  1394. removed = True
  1395. _expected_print_creators.pop(key, None)
  1396. _expected_print_registered_at.pop(key, None)
  1397. # Still needed by another expected print of the archive on the printer,
  1398. # or a plate of it printing there, as in the sweep.
  1399. if not _expected_or_printing_here(printer_id, archive_id):
  1400. _drop_dispatch(printer_id, archive_id)
  1401. if removed:
  1402. logging.getLogger(__name__).info(
  1403. "Unregistered expected print: printer=%s, file=%s, archive=%s (print was never sent)",
  1404. printer_id,
  1405. filename,
  1406. archive_id,
  1407. )
  1408. def _compute_run_filament_grams(
  1409. status: str,
  1410. archive_filament_used_grams: float | None,
  1411. progress: float | int | None,
  1412. usage_results: list[dict] | None,
  1413. ) -> float | None:
  1414. """Per-run filament for PrintLogEntry, partial- and tracker-aware (#1378, #1390).
  1415. Priority for every status:
  1416. 1. Sum of tracked spool deltas in ``usage_results`` (AMS-measured
  1417. weight delta — same source that drives "Total Consumed" on the
  1418. Inventory page, so Stats and Inventory totals stay aligned).
  1419. 2. For ``completed``: the slicer estimate (no tracker available, fall
  1420. back to the canonical "this print used X" value).
  1421. 3. For partial statuses: ``estimate * progress%``.
  1422. 4. ``None`` if nothing is known.
  1423. """
  1424. tracked_grams = sum(r.get("weight_used") or 0 for r in (usage_results or []))
  1425. if tracked_grams > 0:
  1426. return round(tracked_grams, 1)
  1427. if status == "completed":
  1428. return archive_filament_used_grams
  1429. if archive_filament_used_grams:
  1430. scale = max(0.0, min(((progress or 0) / 100.0), 1.0))
  1431. if scale > 0:
  1432. return round(archive_filament_used_grams * scale, 1)
  1433. return None
  1434. def _printing_here(printer_id: int, archive_id: int) -> bool:
  1435. """Whether a plate of the archive is printing on the printer."""
  1436. return any(key[0] == printer_id and bound == archive_id for key, bound in _active_prints.items())
  1437. def _expected_or_printing_here(printer_id: int, archive_id: int) -> bool:
  1438. """Whether a print of the archive is expected or printing on the printer."""
  1439. return _printing_here(printer_id, archive_id) or any(
  1440. key[0] == printer_id and expected == archive_id for key, expected in _expected_prints.items()
  1441. )
  1442. def _drop_dispatch(printer_id: int, archive_id: int) -> None:
  1443. """Forget what the dispatch of the archive to the printer registered."""
  1444. dispatch = (printer_id, archive_id)
  1445. _print_ams_mappings.pop(dispatch, None)
  1446. _print_cost_center_ids.pop(dispatch, None)
  1447. _print_plate_ids.pop(dispatch, None)
  1448. def _get_start_ams_mapping(data: dict, printer_id: int, archive_id: int | None) -> list[int] | None:
  1449. """Resolve AMS mapping for print start without consuming stored queue/reprint state."""
  1450. stored_ams_mapping = data.get("ams_mapping")
  1451. if not stored_ams_mapping and archive_id:
  1452. stored_ams_mapping = _print_ams_mappings.get((printer_id, archive_id))
  1453. return stored_ams_mapping
  1454. def _get_start_plate_id(printer_id: int, archive_id: int | None) -> int | None:
  1455. """Resolve plate_id for print start without consuming stored direct-Print state.
  1456. Direct-Print of a single plate from a multi-plate 3MF registers plate_id in
  1457. ``_print_plate_ids`` at dispatch time; this lets the spoolman / usage tracker
  1458. read it back at print-start without popping (the entry is popped on print
  1459. completion or TTL eviction, mirroring ``_print_ams_mappings``).
  1460. """
  1461. if archive_id is None:
  1462. return None
  1463. return _print_plate_ids.get((printer_id, archive_id))
  1464. def _run_print_time_seconds(archive, printer_id: int) -> int | None:
  1465. """The slicer's time for the plate this print prints, for its start
  1466. notification (#3375): ``archive.print_time_seconds`` adds every plate of
  1467. the file up (#1593). The plate is the dispatch's, else the archive's. The
  1468. whole file's time when its plates are not known; None for a plate not
  1469. known of several, and the notification takes the printer's remaining
  1470. time."""
  1471. from backend.app.services.print_cost_estimate import run_estimate
  1472. plate_id = _print_plate_ids.get((printer_id, archive.id), archive.plate_id)
  1473. estimate = run_estimate(archive, plate_id)
  1474. return estimate.seconds if estimate is not None else None
  1475. _plate_estimates_task: asyncio.Task | None = None
  1476. async def _fill_plate_estimates() -> None:
  1477. from backend.app.services.print_cost_estimate import fill_plate_estimates
  1478. try:
  1479. filled = await fill_plate_estimates(async_session)
  1480. except asyncio.CancelledError:
  1481. raise
  1482. except Exception as exc:
  1483. logging.getLogger(__name__).warning("[ARCHIVE] Reading plate estimates failed: %s", exc)
  1484. return
  1485. if filled:
  1486. logging.getLogger(__name__).info("[ARCHIVE] Read the plate estimates of %d archive(s)", filled)
  1487. _gcode_hashes_task: asyncio.Task | None = None
  1488. async def _fill_gcode_hashes() -> None:
  1489. from backend.app.services.archive_gcode_hash import fill_gcode_hashes
  1490. try:
  1491. filled = await fill_gcode_hashes(async_session)
  1492. except asyncio.CancelledError:
  1493. raise
  1494. except Exception as exc:
  1495. logging.getLogger(__name__).warning("[ARCHIVE] Reading G-code fingerprints failed: %s", exc)
  1496. return
  1497. if filled:
  1498. logging.getLogger(__name__).info("[ARCHIVE] Read the G-code fingerprints of %d archive(s)", filled)
  1499. def _partial_progress_scale(progress: int | float | None) -> float:
  1500. """Clamp ``progress / 100`` into [0.0, 1.0] for partial-print scaling.
  1501. Used by every site that multiplies a "would-have-used" slicer estimate
  1502. down to "actually-used" for failed / cancelled / stopped prints. Centralised
  1503. so the three sites in ``_background_notifications`` (and the per-plate
  1504. override helper) can't drift apart on the coercion shape.
  1505. """
  1506. return max(0.0, min((progress or 0) / 100.0, 1.0))
  1507. def _scope_notification_archive_data_to_plate(
  1508. archive_data: dict,
  1509. archive_file_path: str | None,
  1510. plate_id: int | None,
  1511. print_status: str,
  1512. progress: int | float | None,
  1513. base_dir: Path,
  1514. ) -> dict:
  1515. """Override summed-across-plates totals in ``archive_data`` with the values
  1516. for ``plate_id`` so the completion notification reports what was actually
  1517. printed, not the whole project (#1785).
  1518. The 3MF parser at services/archive.py:200-264 sums ``prediction`` and
  1519. ``weight`` across every plate of a multi-plate file (#1593) — correct for
  1520. the archive card's "whole project" headline, wrong for the completion
  1521. notification of a single-plate print. The queue UI already re-reads the
  1522. 3MF per-plate at print_queue.py:272-285; this helper mirrors that for the
  1523. notification payload (filament grams, time estimate, per-slot breakdown).
  1524. No-ops when ``plate_id`` is None, the file is missing, or the 3MF carries
  1525. no per-plate values — in every fail case the original ``archive_data`` is
  1526. returned unchanged so the notification still sends.
  1527. """
  1528. if plate_id is None or not archive_file_path:
  1529. return archive_data
  1530. from backend.app.utils.threemf_tools import (
  1531. extract_filament_usage_from_3mf,
  1532. extract_print_time_from_3mf,
  1533. )
  1534. archive_path = base_dir / archive_file_path
  1535. if not archive_path.exists():
  1536. return archive_data
  1537. plate_slots = extract_filament_usage_from_3mf(archive_path, plate_id)
  1538. plate_grams = sum(f.get("used_g", 0) for f in plate_slots)
  1539. plate_time = extract_print_time_from_3mf(archive_path, plate_id)
  1540. scale = 1.0 if print_status == "completed" else _partial_progress_scale(progress)
  1541. if plate_time:
  1542. archive_data["print_time_seconds"] = plate_time
  1543. # Gate both the grams headline AND the per-slot breakdown on the same
  1544. # `plate_grams > 0` signal: if the 3MF carries per-plate filament rows but
  1545. # they all sum to zero (slicer bug / re-slice without estimate), drop back
  1546. # to the project-level grams the archive columns already provide rather
  1547. # than ship a project-level headline next to an all-zero per-plate
  1548. # breakdown.
  1549. if plate_grams > 0:
  1550. archive_data["actual_filament_grams"] = round(plate_grams * scale, 1)
  1551. archive_data["filament_slots"] = [
  1552. {
  1553. "slot_id": s.get("slot_id"),
  1554. "used_g": round((s.get("used_g") or 0) * scale, 1),
  1555. "type": s.get("type", ""),
  1556. "color": s.get("color", ""),
  1557. }
  1558. for s in plate_slots
  1559. ]
  1560. return archive_data
  1561. def _extract_filament_data_from_mqtt(data: dict, ams_mapping: list[int] | None = None) -> dict[str, str]:
  1562. """Best-effort filament metadata from the MQTT print-start snapshot.
  1563. Used when the 3MF can't be downloaded (P1S/A1/P2S firmwares lock the
  1564. file during print, see #1533) so the fallback PrintArchive still has
  1565. enough filament info to support the inventory views and AMS-expansion
  1566. planning the operator opens it for. Returns a dict with optional
  1567. ``filament_type`` and ``filament_color`` keys in the same
  1568. comma-separated format the 3MF extractor produces, so the rest of the
  1569. codebase treats the fallback archive identically to a normal one.
  1570. ``ams_mapping`` is the slicer's slot-per-print-filament list captured
  1571. from the MQTT print payload (global tray IDs, possibly -1 for VT-tray
  1572. entries). When supplied, only the slots actually consumed by this
  1573. print contribute. Without it the function falls back to every loaded
  1574. AMS slot — less accurate but still useful.
  1575. Accepts both the raw inner payload (``{"ams": {"ams": [...]}, ...}``)
  1576. that the unit tests pass directly, AND the on_print_start callback
  1577. shape (``{"raw_data": {"ams": {"ams": [...]}, ...}, ...}``) the
  1578. bambu_mqtt service hands to main.py at runtime. The original
  1579. ``_extract_filament_data_from_mqtt(data)`` shipped in #1533 only
  1580. handled the inner shape and silently returned ``{}`` for every real
  1581. print start, leaving fallback archives' filament fields NULL — the
  1582. exact regression the fix was meant to close. Reported with a log
  1583. proving the AMS state was right there at
  1584. ``data["raw_data"]["ams"]["ams"][0]["tray"][0]`` (#1533 follow-up).
  1585. """
  1586. result: dict[str, str] = {}
  1587. # Look at the on_print_start wrapper first, then the inner shape.
  1588. raw_data = (data or {}).get("raw_data")
  1589. ams_root = (raw_data or {}).get("ams") if isinstance(raw_data, dict) else None
  1590. if not isinstance(ams_root, dict):
  1591. ams_root = (data or {}).get("ams") or {}
  1592. ams_units = ams_root.get("ams") if isinstance(ams_root, dict) else None
  1593. if not isinstance(ams_units, list) or not ams_units:
  1594. return result
  1595. from backend.app.services.print_scheduler import _global_tray_id
  1596. # Map global tray id → (type, color), addressed as the print's ams_mapping
  1597. # is (utils.tray_ids).
  1598. layout = TrayLayout.from_units(ams_units)
  1599. loaded: dict[int, tuple[str, str]] = {}
  1600. for unit in ams_units:
  1601. if not isinstance(unit, dict):
  1602. continue
  1603. try:
  1604. unit_id = int(unit.get("id", 0))
  1605. except (TypeError, ValueError):
  1606. continue
  1607. for tray in unit.get("tray") or []:
  1608. if not isinstance(tray, dict):
  1609. continue
  1610. try:
  1611. tray_id = int(tray.get("id", 0))
  1612. except (TypeError, ValueError):
  1613. continue
  1614. ttype = (tray.get("tray_type") or "").strip()
  1615. # RRGGBBAA from the printer, stored as the 3MF path stores it:
  1616. # "#RRGGBB" (the Archives page's swatches and colour filter).
  1617. hex_color = (tray.get("tray_color") or "").strip().upper()[:6]
  1618. tcolor = f"#{hex_color}" if re.fullmatch(r"[0-9A-F]{6}", hex_color) else ""
  1619. if not ttype:
  1620. continue # Empty / unloaded slot.
  1621. loaded[_global_tray_id(unit_id, tray_id, layout)] = (ttype, tcolor)
  1622. if not loaded:
  1623. return result
  1624. if ams_mapping:
  1625. used_ids = [int(x) for x in ams_mapping if isinstance(x, (int, float)) and int(x) >= 0]
  1626. filaments = [loaded[g] for g in used_ids if g in loaded]
  1627. if not filaments:
  1628. return result # Mapping points entirely at slots we have no data for.
  1629. else:
  1630. filaments = [loaded[g] for g in sorted(loaded.keys())]
  1631. types_joined = ",".join(f[0] for f in filaments)
  1632. colors_joined = ",".join(f[1] for f in filaments if f[1])
  1633. # Column limits per backend/app/models/archive.py: filament_type=50,
  1634. # filament_color=200.
  1635. if types_joined:
  1636. result["filament_type"] = types_joined[:50]
  1637. if colors_joined:
  1638. result["filament_color"] = colors_joined[:200]
  1639. return result
  1640. def _maybe_start_layer_timelapse(printer, printer_id: int, archive_id: int) -> bool:
  1641. """Start a layer-timelapse session for *archive_id* when the printer has
  1642. an external camera configured. Returns True if a session was started.
  1643. Three call sites in on_print_start (expected-archive promotion, fallback
  1644. archive creation, fresh-archive creation) used to inline this same
  1645. if-block; the inline copies kept drifting (#1353 fixed only one of them
  1646. on the first pass). Centralising the conditional + call here makes the
  1647. contract testable in isolation and keeps the three sites locked in step.
  1648. """
  1649. if not (printer.external_camera_enabled and printer.external_camera_url):
  1650. return False
  1651. from backend.app.services.layer_timelapse import start_session
  1652. start_session(
  1653. printer_id,
  1654. archive_id,
  1655. printer.external_camera_url,
  1656. printer.external_camera_type or "mjpeg",
  1657. snapshot_url=printer.external_camera_snapshot_url,
  1658. rotation=getattr(printer, "camera_rotation", 0),
  1659. )
  1660. logging.getLogger(__name__).info("Started layer timelapse for printer %s, archive %s", printer_id, archive_id)
  1661. return True
  1662. def _format_hms_error_summary(hms_errors: list[dict]) -> str | None:
  1663. """Build a human-readable failure reason from MQTT hms_errors for PrintQueueItem.error_message.
  1664. Each entry has keys: code ('0x4038'), attr (32-bit int), module, severity, and
  1665. — since #2926 — the description the parser already resolved, which is preferred
  1666. when present so the queue's failure reason reads the same as the status
  1667. response. The short code still produces the bracketed label, and still
  1668. resolves the sentence for a caller whose entries predate the field. An entry
  1669. with a 16-char ``full_code`` is labelled with it instead, since the short code
  1670. of an ``hms[]`` fault is not a code anyone can look up. Falls back to the bare
  1671. label when no description is on file. Returns None for an empty list so
  1672. callers can leave error_message unset.
  1673. """
  1674. if not hms_errors:
  1675. return None
  1676. from backend.app.services.hms_errors import get_error_description
  1677. parts: list[str] = []
  1678. for err in hms_errors:
  1679. try:
  1680. # `_hms_short_code` rather than a local derivation: this one used to
  1681. # format the error without masking it to 16 bits, so an `hms[]` entry
  1682. # whose code carries an alert-level group produced a five-digit label
  1683. # like "0500_3000A" — not a code the user can look up, and never a
  1684. # catalogue key, so the sentence was lost with it.
  1685. short_code = _hms_short_code(err.get("attr", 0), err.get("code", 0))
  1686. except (TypeError, ValueError):
  1687. continue
  1688. description = err.get("description") or get_error_description(short_code)
  1689. # An `hms[]` fault's short code ("0500_000E") drops the part and level
  1690. # groups and is not a code anyone can look up; its full code in the
  1691. # printer screen's four groups is (#2728).
  1692. full_code = str(err.get("full_code") or "")
  1693. label = "-".join(full_code[i : i + 4] for i in range(0, 16, 4)) if len(full_code) == 16 else short_code
  1694. parts.append(f"[{label}] {description}" if description else f"[{label}]")
  1695. return "; ".join(parts) if parts else None
  1696. async def _bump_library_file_usage_if_completed(db, item, queue_status: str) -> None:
  1697. """Increment LibraryFile.print_count and stamp last_printed_at when a queued
  1698. print completes successfully. Gated to status=='completed': failed, cancelled
  1699. and aborted prints do not count as usage. Caller is responsible for committing
  1700. the session. No-op when the queue item has no linked library file (e.g. reprints
  1701. from an archive). See #1008."""
  1702. if queue_status != "completed" or item.library_file_id is None:
  1703. return
  1704. from backend.app.models.library import LibraryFile
  1705. lib_file = await db.scalar(select(LibraryFile).where(LibraryFile.id == item.library_file_id))
  1706. if lib_file is None:
  1707. return
  1708. lib_file.print_count = (lib_file.print_count or 0) + 1
  1709. lib_file.last_printed_at = datetime.now(timezone.utc)
  1710. def mark_printer_stopped_by_user(printer_id: int) -> None:
  1711. """Mark that the active print on this printer was stopped by the user from the queue UI.
  1712. When on_print_complete fires with status 'failed' for a printer in this set we
  1713. reclassify it as 'cancelled' so the correct 'print stopped' notification is sent
  1714. rather than a 'print failed' notification.
  1715. """
  1716. _user_stopped_printers.add(printer_id)
  1717. logging.getLogger(__name__).info("Marked printer %s as user-stopped from queue", printer_id)
  1718. _last_status_broadcast: dict[int, str] = {}
  1719. # Track printers where we've updated nozzle_count
  1720. _nozzle_count_updated: set[int] = set()
  1721. async def _maybe_notify_printer_offline(printer_id: int) -> None:
  1722. """Wait the debounce window then fire `on_printer_offline` if the printer
  1723. is still offline.
  1724. Scheduled by `on_printer_status_change` on the connected → disconnected
  1725. edge (#1752). Cancelled by the same handler if the printer reconnects
  1726. before the window elapses, so a single MQTT blip + recovery doesn't
  1727. notify. Both the staleness-detector path (`bambu_mqtt.py::check_staleness`)
  1728. and the smart-plug power-off path (`printer_manager.mark_printer_offline`)
  1729. route through the same status-change callback, so this covers both.
  1730. """
  1731. logger = logging.getLogger(__name__)
  1732. try:
  1733. await asyncio.sleep(_PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS)
  1734. still_offline = not printer_manager.is_connected(printer_id)
  1735. logger.info(
  1736. "[#1752] Printer %s offline debounce elapsed: still_offline=%s",
  1737. printer_id,
  1738. still_offline,
  1739. )
  1740. if not still_offline:
  1741. return
  1742. async with async_session() as db:
  1743. from backend.app.models.printer import Printer
  1744. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1745. printer = result.scalar_one_or_none()
  1746. if not printer:
  1747. logger.warning(
  1748. "[#1752] Printer %s missing from DB at offline-notify time; skipping",
  1749. printer_id,
  1750. )
  1751. return
  1752. # Recorded only when the relay can publish it, so the reconnect's
  1753. # online event always has an offline before it.
  1754. if mqtt_relay.enabled and mqtt_relay.connected:
  1755. _printer_offline_published.add(printer_id)
  1756. try:
  1757. await mqtt_relay.on_printer_offline(printer_id, printer.name, printer.serial_number)
  1758. except Exception as e:
  1759. logger.warning("MQTT relay offline event failed for printer %s: %s", printer_id, e)
  1760. logger.info(
  1761. "[#1752] Dispatching on_printer_offline for printer %s (%s)",
  1762. printer_id,
  1763. printer.name,
  1764. )
  1765. await notification_service.on_printer_offline(printer_id, printer.name, db)
  1766. except asyncio.CancelledError:
  1767. raise
  1768. except Exception as e:
  1769. logger.warning("Printer offline notification failed for printer %s: %s", printer_id, e)
  1770. finally:
  1771. _printer_offline_notify_tasks.pop(printer_id, None)
  1772. def _hms_notify_key(error) -> str:
  1773. """What identifies a fault for notification de-duplication.
  1774. The full code, which is unique per fault. ``attr`` alone used to be the key:
  1775. unique for a ``print_error``, but for an ``hms[]`` fault it is only the
  1776. module and part, so two faults on one part (#1840's H2C held 0500-0600-0002-0005
  1777. and -0006 together) shared a key and only the first was ever notified. An
  1778. entry without a full code falls back to attr and code together.
  1779. """
  1780. full_code = getattr(error, "full_code", "") or ""
  1781. if full_code:
  1782. return full_code.upper()
  1783. return f"{error.attr:08X}:{error.code}"
  1784. def _hms_errors_to_notify(errors: list, new_error_codes: set[str]) -> list:
  1785. """The new faults that count (see ``_hms_fault_counts``).
  1786. These go to the MQTT relay; the caller also needs a description before it
  1787. sends a notification. Level 0, Bambu's "invalid" level, never counts. This
  1788. used to read ``severity >= 2`` when severity held the part byte; on the real
  1789. level that would drop the task-stopping errors (#2728).
  1790. """
  1791. return [e for e in errors if _hms_notify_key(e) in new_error_codes and _hms_fault_counts(e)]
  1792. # What each HMS module byte is about, read off Bambu's own texts for its codes
  1793. # (hms_catalog.json). The table used to call 0x05 "Nozzle/Extruder", 0x0C
  1794. # "Motion Controller" and 0x12 "Chamber", which titled USB and MQTT errors as
  1795. # nozzle faults and an A1 mini's AMS Lite runouts as chamber errors (#3486).
  1796. _HMS_MODULE_NAMES = {
  1797. 0x03: "Print/Task", # bed, nozzle temperatures, motors, the print itself
  1798. 0x05: "System", # USB/SD, network, firmware, files, AMS communication
  1799. 0x07: "AMS/Filament",
  1800. 0x08: "Toolhead",
  1801. 0x0C: "Camera", # cameras, lidar, first-layer and AI inspection
  1802. 0x12: "AMS Lite",
  1803. 0x18: "AMS-HT",
  1804. 0x1A: "Tool Changer",
  1805. 0x29: "Air Filter",
  1806. 0x2A: "AMS 3H",
  1807. }
  1808. def _hms_error_title(module: int) -> str:
  1809. """The notification title for a fault of *module*."""
  1810. name = _HMS_MODULE_NAMES.get(module)
  1811. return f"{name} Error" if name else "Printer Error"
  1812. def _take_new_hms_faults(printer_id: int, errors: list, print_state: str | None = None) -> list:
  1813. """The faults on this printer not notified yet, and record them as notified.
  1814. Tracking is updated before anything is sent, so concurrent status callbacks
  1815. cannot notify the same fault twice. A notified fault is forgotten once it
  1816. has been gone for ``_HMS_CLEAR_GRACE_SECONDS``, so one that flickers off
  1817. and on is notified once (#3226), while one that clears and comes back much
  1818. later is notified again.
  1819. When ``print_state`` changes (a print resumed, started or finished), every
  1820. fault recorded so far is forgotten the first time it is gone, without the
  1821. wait: a runout fixed before resuming must be notified if it happens again a
  1822. few minutes later, while the printer sits paused. Not only the faults gone
  1823. at the change itself, because the update that flips the state can still
  1824. carry the old fault (a ``print_error`` entry stays until a payload brings
  1825. ``hms``).
  1826. """
  1827. now = time.time()
  1828. current = {_hms_notify_key(e) for e in errors}
  1829. seen = _notified_hms_errors.setdefault(printer_id, {})
  1830. forget_when_gone = _hms_forget_when_gone.setdefault(printer_id, set())
  1831. if print_state is not None:
  1832. print_state = print_state.upper()
  1833. if _hms_print_state.get(printer_id, print_state) != print_state:
  1834. forget_when_gone.update(seen)
  1835. _hms_print_state[printer_id] = print_state
  1836. for key, last_seen in list(seen.items()):
  1837. if key not in current and (key in forget_when_gone or now - last_seen >= _HMS_CLEAR_GRACE_SECONDS):
  1838. del seen[key]
  1839. forget_when_gone.discard(key)
  1840. new = current - seen.keys()
  1841. for key in current:
  1842. seen[key] = now
  1843. if not seen:
  1844. _notified_hms_errors.pop(printer_id, None)
  1845. if not forget_when_gone:
  1846. _hms_forget_when_gone.pop(printer_id, None)
  1847. return _hms_errors_to_notify(errors, new)
  1848. async def _take_new_hms_faults_and_save(printer_id: int, errors: list, print_state: str | None = None) -> list:
  1849. """``_take_new_hms_faults``, and store the notified set when it changed (#3486).
  1850. The set lives in memory, so without this every restart sent each fault the
  1851. printer still lists once more. A failed write only costs that protection
  1852. for the next restart, so it is logged and the notification goes ahead.
  1853. """
  1854. before = set(_notified_hms_errors.get(printer_id, ()))
  1855. new = _take_new_hms_faults(printer_id, errors, print_state)
  1856. if set(_notified_hms_errors.get(printer_id, ())) != before:
  1857. from backend.app.services.hms_notified import save_notified_hms_faults
  1858. try:
  1859. await save_notified_hms_faults(printer_id, lambda: _notified_hms_errors.get(printer_id, {}).keys())
  1860. except Exception as e:
  1861. logging.getLogger(__name__).warning(
  1862. "Could not store the notified HMS faults of printer %s: %s", printer_id, e
  1863. )
  1864. return new
  1865. async def _load_notified_hms_faults() -> None:
  1866. """Restore the faults notified before the restart (#3486).
  1867. Each counts as seen now: one still listed is not sent again, and one that
  1868. is gone is forgotten after the usual grace period.
  1869. """
  1870. from backend.app.services.hms_notified import load_notified_hms_faults
  1871. try:
  1872. loaded = await load_notified_hms_faults()
  1873. except Exception as e:
  1874. logging.getLogger(__name__).warning("Could not load the notified HMS faults: %s", e)
  1875. return
  1876. now = time.time()
  1877. for printer_id, keys in loaded.items():
  1878. seen = _notified_hms_errors.setdefault(printer_id, {})
  1879. for key in keys:
  1880. seen.setdefault(key, now)
  1881. # `stg_cur` values that mean the printer is not in a preparation stage: 0 is
  1882. # "Printing", -1 and 255 are "no stage" (#3211).
  1883. _NO_PREPARATION_STAGES = frozenset({0, -1, 255})
  1884. def _progress_milestone_to_notify(printer_id: int, state: PrinterState) -> int | None:
  1885. """The milestone (25, 50 or 75) this status update reaches for the first time, if any.
  1886. Records it as notified. Called only for a printing state with progress above 0.
  1887. Progress before the first layer is not progress through the print (#3211).
  1888. An A1 mini reports ``mc_percent`` 85 in the first frame of a print, while
  1889. still preheating the bed at layer 0, then 3, 7, 40 and 44 through its
  1890. calibration, and starts layer 1 at 45. Read as progress, that 85 sent the
  1891. 75% notification together with Print Started, and since 75 was then on
  1892. record, 25 and 50 could never fire. So nothing counts while the printer
  1893. reports a layer count but has not started layer 1.
  1894. Without a layer count -- a print whose first frame carried no
  1895. ``total_layer_num``, until the pushall that asks for it is answered
  1896. (#2702) -- the preparation stage says the same thing: before layer 1 the
  1897. A1 mini reported ``stg_cur`` 2, 4, 14 and 1 (bed preheating, calibration,
  1898. homing), and 0 from layer 1 on. -1 and 255 are "no stage". A printer that
  1899. reports neither keeps the old behaviour rather than never notifying.
  1900. """
  1901. if (state.layer_num or 0) < 1:
  1902. if (state.total_layers or 0) > 0:
  1903. return None
  1904. if state.stg_cur not in _NO_PREPARATION_STAGES:
  1905. return None
  1906. progress = state.progress or 0
  1907. current_milestone = 0
  1908. if progress >= 75:
  1909. current_milestone = 75
  1910. elif progress >= 50:
  1911. current_milestone = 50
  1912. elif progress >= 25:
  1913. current_milestone = 25
  1914. if current_milestone > _last_progress_milestone.get(printer_id, 0):
  1915. _last_progress_milestone[printer_id] = current_milestone
  1916. return current_milestone
  1917. return None
  1918. async def on_printer_status_change(printer_id: int, state: PrinterState):
  1919. """Handle printer status changes - broadcast via WebSocket."""
  1920. try:
  1921. notify_live_activities.observe(printer_id, state)
  1922. except Exception:
  1923. logging.getLogger(__name__).exception("Notify Live Activity status hook failed for printer %s", printer_id)
  1924. try:
  1925. pause_eta.observe(printer_id, state)
  1926. except Exception:
  1927. logging.getLogger(__name__).exception("Pause heads-up check failed for printer %s", printer_id)
  1928. wled_manager.handle_status(
  1929. printer_id,
  1930. state,
  1931. awaiting_plate_clear=printer_manager.is_awaiting_plate_clear(printer_id),
  1932. )
  1933. # Connected-edge reconciliation (#1542 follow-up). When the printer
  1934. # transitions disconnected → connected — which covers both Bambuddy
  1935. # startup (no prior connection) and a mid-session MQTT reconnect — fire
  1936. # `reconcile_stale_active_prints` exactly once for this connection so
  1937. # any archive still in `status="printing"` that can't actually be
  1938. # running anymore (printer IDLE / different subtask / empty subtask)
  1939. # gets a synthesised PRINT COMPLETE. Without this, a print that
  1940. # finished during a disconnect window + a smart-plug power cycle
  1941. # leaves the .3mf on the SD card and the firmware ghost-replays it on
  1942. # next boot. Reconciliation runs concurrently — it must not block the
  1943. # WebSocket dedup / broadcast logic below, and the connected edge is
  1944. # marked True BEFORE the await so concurrent status updates inside
  1945. # the same connection don't re-trigger reconciliation.
  1946. #
  1947. # Wait for a real push_status before reconciling (#1679): MQTT
  1948. # `_on_connect` broadcasts `state` IMMEDIATELY after the broker accepts
  1949. # the connection, BEFORE `_request_push_all` round-trips. At that
  1950. # instant the `PrinterState` is still on construction defaults — most
  1951. # importantly `state.state == "unknown"` and `state.subtask_name == ""`.
  1952. # If reconcile spawns here, every in-flight archive falls through to
  1953. # the empty-subtask_name trigger and gets synthesised `aborted`, which
  1954. # creates a duplicate archive on the real PRINT COMPLETE and
  1955. # double-counts filament. Gating on `state.state ∉ ("", "unknown")`
  1956. # keeps the #1542 mechanism intact: once the first real push_status
  1957. # updates `state.state` (RUNNING / IDLE / FINISH / …), this handler
  1958. # fires again with the flag still False — reconcile then runs against
  1959. # actual evidence.
  1960. state_known = bool(state.state) and state.state.upper() not in ("", "UNKNOWN")
  1961. if state.connected and state_known and not _printer_reconciled_since_connect.get(printer_id, False):
  1962. _printer_reconciled_since_connect[printer_id] = True
  1963. spawn_background_task(
  1964. reconcile_stale_active_prints(printer_id),
  1965. name=f"reconcile-stale-prints-{printer_id}",
  1966. )
  1967. elif not state.connected and _printer_reconciled_since_connect.get(printer_id, False):
  1968. # Re-arm so the next reconnect triggers reconciliation again.
  1969. _printer_reconciled_since_connect[printer_id] = False
  1970. # Same edge, for the calibration table the AMS card reads its K values from.
  1971. #
  1972. # Also gated on knowing a nozzle diameter, which is what decides *which*
  1973. # tables to ask for. A `state_known` gate alone is not enough: the first
  1974. # real push_status is what makes the state known, and the nozzle fields do
  1975. # not always arrive in it. Latching there would spend this connection's one
  1976. # attempt on a printer that could not yet say what was fitted.
  1977. nozzle_known = any(n.nozzle_diameter for n in (state.nozzles or []))
  1978. if (
  1979. state.connected
  1980. and state_known
  1981. and nozzle_known
  1982. and not _printer_kprofiles_primed_since_connect.get(printer_id, False)
  1983. ):
  1984. _printer_kprofiles_primed_since_connect[printer_id] = True
  1985. spawn_background_task(
  1986. prime_kprofile_table(printer_id),
  1987. name=f"prime-kprofiles-{printer_id}",
  1988. )
  1989. elif not state.connected and _printer_kprofiles_primed_since_connect.get(printer_id, False):
  1990. _printer_kprofiles_primed_since_connect[printer_id] = False
  1991. # A slot can lose its K-profile selection (cali_idx back to -1) with nothing
  1992. # else in the AMS report changing -- a power cycle does it to every slot --
  1993. # and on_ams_change never hears of it (#3219). Checked here, on every push
  1994. # while the printer is idle; needs_check is cheap and throttled per slot.
  1995. # Guarded: nothing here may stop the status broadcast below.
  1996. try:
  1997. if not state.connected:
  1998. kprofile_drift.forget_printer(printer_id)
  1999. elif kprofile_drift.needs_check(printer_id, state):
  2000. spawn_background_task(
  2001. kprofile_drift.reapply_lost_kprofiles(printer_id),
  2002. name=f"reapply-kprofiles-{printer_id}",
  2003. )
  2004. except Exception:
  2005. logging.getLogger(__name__).exception("[Printer %s] K-profile check failed", printer_id)
  2006. # Offline-notification edge (#1752): schedule `on_printer_offline` on
  2007. # connected → disconnected. The "back online" channel is already covered
  2008. # by the print-failure notification (firmware reports gcode_state=FAILED
  2009. # on reconnect of an interrupted print), so there is no online
  2010. # notification. The MQTT relay gets both: offline after the debounce, and
  2011. # online on the reconnect that follows a published offline.
  2012. prev_connected = _printer_last_connected.get(printer_id)
  2013. _printer_last_connected[printer_id] = state.connected
  2014. if prev_connected is True and not state.connected:
  2015. existing = _printer_offline_notify_tasks.get(printer_id)
  2016. if existing is None or existing.done():
  2017. logging.getLogger(__name__).info(
  2018. "[#1752] Printer %s connected→disconnected edge; scheduling offline notification in %.0fs",
  2019. printer_id,
  2020. _PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS,
  2021. )
  2022. _printer_offline_notify_tasks[printer_id] = asyncio.create_task(
  2023. _maybe_notify_printer_offline(printer_id),
  2024. name=f"printer-offline-notify-{printer_id}",
  2025. )
  2026. elif state.connected:
  2027. if prev_connected is False:
  2028. # A printer back from a dropped connection, often a restart, may
  2029. # take files again: don't hold it to an upload backoff from
  2030. # before (#3563). Not on Bambuddy's own start (no previous state).
  2031. client = printer_manager.get_client(printer_id)
  2032. print_scheduler.printer_reconnected(printer_id, client.ip_address if client else None)
  2033. # Kept until the relay can actually send online: with its broker down
  2034. # now, the next status update after the reconnect sends it.
  2035. if printer_id in _printer_offline_published and mqtt_relay.enabled and mqtt_relay.connected:
  2036. _printer_offline_published.discard(printer_id)
  2037. printer_info = printer_manager.get_printer(printer_id)
  2038. if printer_info:
  2039. try:
  2040. await mqtt_relay.on_printer_online(printer_id, printer_info.name, printer_info.serial_number)
  2041. except Exception as e:
  2042. logging.getLogger(__name__).warning(
  2043. "MQTT relay online event failed for printer %s: %s", printer_id, e
  2044. )
  2045. pending = _printer_offline_notify_tasks.pop(printer_id, None)
  2046. if pending is not None and not pending.done():
  2047. logging.getLogger(__name__).info(
  2048. "[#1752] Printer %s reconnected before debounce; cancelling pending offline notification",
  2049. printer_id,
  2050. )
  2051. pending.cancel()
  2052. # Only broadcast if something meaningful changed (reduce WebSocket spam)
  2053. # Include rounded temperatures to detect meaningful temp changes (within 1 degree)
  2054. temps = state.temperatures or {}
  2055. nozzle_temp = round(temps.get("nozzle", 0))
  2056. bed_temp = round(temps.get("bed", 0))
  2057. nozzle_2_temp = round(temps.get("nozzle_2", 0)) if "nozzle_2" in temps else ""
  2058. chamber_temp = round(temps.get("chamber", 0)) if "chamber" in temps else ""
  2059. # Auto-detect dual-nozzle printers from MQTT temperature data
  2060. if "nozzle_2" in temps and printer_id not in _nozzle_count_updated:
  2061. _nozzle_count_updated.add(printer_id)
  2062. # Update nozzle_count in database
  2063. async with async_session() as db:
  2064. from backend.app.models.printer import Printer
  2065. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2066. printer = result.scalar_one_or_none()
  2067. if printer and printer.nozzle_count != 2:
  2068. printer.nozzle_count = 2
  2069. await db.commit()
  2070. logging.getLogger(__name__).info(
  2071. f"Auto-detected dual-nozzle printer {printer_id}, updated nozzle_count=2"
  2072. )
  2073. # Include target temps for heating phase detection
  2074. bed_target = round(temps.get("bed_target", 0))
  2075. nozzle_target = round(temps.get("nozzle_target", 0))
  2076. # Include tray_now and vt_tray hash so external spool changes trigger broadcasts
  2077. vt_tray_key = hash(str(state.raw_data.get("vt_tray", []))) if state.raw_data else 0
  2078. # Include AMS dry_time and tray state values so drying/slot changes trigger broadcasts.
  2079. #
  2080. # dry_countdown_stalled rides along because it is the one drying signal the
  2081. # countdown itself cannot carry: the MQTT layer raises it precisely BECAUSE
  2082. # dry_time stopped moving, so on the frame that flips it every other member
  2083. # of this key is identical and the push would be deduplicated away. Mid-print
  2084. # a temperature would eventually break the tie, but a parked command on an
  2085. # idle machine changes nothing else at all — AMS temp and humidity are not in
  2086. # the key — so the badge could sit unreachable indefinitely. The flag flips at
  2087. # most once per drying cycle, so it costs no mid-print broadcast traffic.
  2088. ams_dry_key = (
  2089. tuple((a.get("dry_time", 0), bool(a.get("dry_countdown_stalled"))) for a in (state.raw_data.get("ams") or []))
  2090. if state.raw_data
  2091. else ()
  2092. )
  2093. # Include tray states so load/unload transitions (state 11→10) trigger broadcasts (#784)
  2094. #
  2095. # The filament identity fields are here because Configure Slot writes
  2096. # exactly those and nothing else. Re-configuring a slot from PLA to another
  2097. # brand or colour of PLA leaves id/tray_type/state identical, so the key
  2098. # matched, this function returned before broadcasting, and the card kept
  2099. # showing the old filament until the 30s fallback poll or a page reload —
  2100. # even though the configure route asks the printer for a fresh pushall and
  2101. # that push does carry the new values. Reset always worked, because it
  2102. # clears tray_type.
  2103. #
  2104. # These fields only change when someone configures a slot or swaps a spool,
  2105. # so unlike temperature or progress they add no broadcast traffic mid-print.
  2106. ams_tray_key = (
  2107. tuple(
  2108. (
  2109. t.get("id"),
  2110. t.get("tray_type", ""),
  2111. t.get("state"),
  2112. t.get("tray_color", ""),
  2113. t.get("tray_info_idx", ""),
  2114. t.get("tray_sub_brands", ""),
  2115. t.get("cali_idx"),
  2116. )
  2117. for a in (state.raw_data.get("ams") or [])
  2118. for t in a.get("tray", [])
  2119. )
  2120. if state.raw_data
  2121. else ()
  2122. )
  2123. # Filament Track Switch: which inlet each AMS is bound to, and whether the
  2124. # accessory is fitted at all. Neither is in ams_tray_key (it is per-tray) nor
  2125. # in the AMS change-hash (tray fields only, and widening that would fire
  2126. # spurious Spoolman syncs), so without them a "Join IN-B" on the printer
  2127. # screen changed no key at all and the card's inlet badges sat stale until a
  2128. # reload. Like the filament-backup flag, these only move when someone
  2129. # reconfigures the machine, so they add no mid-print broadcast traffic.
  2130. fts_key = (
  2131. state.fila_switch.installed if state.fila_switch else False,
  2132. tuple(sorted(state.ams_switch_inlet.items())),
  2133. # Which hotend holds which slot. Unlike the two above this does move
  2134. # mid-print, on every filament change — but only between discrete slots,
  2135. # so it adds a push per toolchange, not a stream. The AMS slot menu needs
  2136. # it live: it decides which hotend the Load dialog may offer and whether
  2137. # Unload has anything to act on.
  2138. tuple(
  2139. sorted(
  2140. ((ext, slot.ams_id, slot.slot_id, slot.has_filament) for ext, slot in state.extruder_slots.items()),
  2141. # Sort on the extruder id alone: the other members are nullable
  2142. # and comparing None with an int raises.
  2143. key=lambda entry: entry[0],
  2144. )
  2145. ),
  2146. )
  2147. status_key = (
  2148. f"{state.connected}:{state.state}:{state.progress}:{state.layer_num}:"
  2149. f"{nozzle_temp}:{bed_temp}:{nozzle_2_temp}:{chamber_temp}:"
  2150. f"{state.stg_cur}:{bed_target}:{nozzle_target}:"
  2151. f"{state.cooling_fan_speed}:{state.big_fan1_speed}:{state.big_fan2_speed}:"
  2152. f"{state.chamber_light}:{state.active_extruder}:{state.tray_now}:{vt_tray_key}:"
  2153. f"{ams_dry_key}:{ams_tray_key}:{state.door_open}:{state.ams_filament_backup}:{fts_key}"
  2154. )
  2155. # Fields that change only when the machine does (a nozzle swap, a firmware
  2156. # update, developer mode) or once per print (its AMS mapping): no traffic,
  2157. # but on an idle printer nothing else moves the key when they change.
  2158. machine_key = (
  2159. tuple((n.nozzle_type, n.nozzle_diameter) for n in (state.nozzles or [])),
  2160. state.firmware_version,
  2161. state.developer_mode,
  2162. tuple(state.print_ams_mapping) if isinstance(state.print_ams_mapping, list) else None,
  2163. # Cooling/heating, switched on the printer's screen too: on an idle
  2164. # printer nothing else moves the key (PSF-4).
  2165. state.airduct_mode,
  2166. )
  2167. status_key = f"{status_key}:{machine_key}"
  2168. is_active_print = state.state in _ACTIVE_PRINT_STATES
  2169. if not is_active_print:
  2170. _unauthorized_print_kill_sent.discard(printer_id)
  2171. elif printer_id in _unauthorized_print_kill_sent:
  2172. # stop_print() was already sent for this print; avoid all further
  2173. # ownership and settings work until the printer leaves an active state.
  2174. pass
  2175. elif _is_bambuddy_authorized_print_in_memory(printer_id, state):
  2176. # Normal Bambuddy-started prints stay entirely on the in-memory path.
  2177. _unauthorized_print_kill_sent.discard(printer_id)
  2178. else:
  2179. kill_switch_enabled = False
  2180. authorization: bool | None = None
  2181. status_logger = logging.getLogger(__name__)
  2182. try:
  2183. kill_switch_enabled = await _is_printer_kill_switch_enabled_cached()
  2184. if kill_switch_enabled:
  2185. async with async_session() as db:
  2186. authorization = await _is_bambuddy_authorized_print(printer_id, state, db)
  2187. except Exception as e:
  2188. # Fail safe: a database/reconciliation error must never turn into an
  2189. # irreversible stop of a print whose ownership is still unknown.
  2190. authorization = None
  2191. status_logger.warning(
  2192. "[KILL SWITCH] Failed to reconcile print authorization for printer %s: %s", printer_id, e
  2193. )
  2194. if not kill_switch_enabled or authorization is True:
  2195. _unauthorized_print_kill_sent.discard(printer_id)
  2196. elif authorization is None:
  2197. _unauthorized_print_kill_sent.discard(printer_id)
  2198. status_logger.debug(
  2199. "[KILL SWITCH] Deferring authorization for printer %s until archive state is reconciled",
  2200. printer_id,
  2201. )
  2202. else:
  2203. try:
  2204. stopped = printer_manager.stop_print(printer_id)
  2205. if stopped:
  2206. _unauthorized_print_kill_sent.add(printer_id)
  2207. _kill_switch_stopped.add(printer_id)
  2208. printer_info = printer_manager.get_printer(printer_id)
  2209. printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  2210. filename = state.subtask_name or state.gcode_file or state.current_print or "Unknown"
  2211. notification_data = {
  2212. "status": "stopped",
  2213. "filename": state.gcode_file or state.current_print or "",
  2214. "subtask_name": state.subtask_name or "",
  2215. "progress": state.progress,
  2216. "reason": "unauthorized_print",
  2217. }
  2218. status_logger.warning(
  2219. "[KILL SWITCH] Stopped unauthorized print on printer %s (state=%s)",
  2220. printer_id,
  2221. state.state,
  2222. )
  2223. try:
  2224. await ws_manager.broadcast(
  2225. {
  2226. "type": "kill_switch_triggered",
  2227. "printer_id": printer_id,
  2228. "printer_name": printer_name,
  2229. "filename": filename,
  2230. "reason": "unauthorized_print",
  2231. }
  2232. )
  2233. except Exception as e:
  2234. status_logger.warning(
  2235. "[KILL SWITCH] WebSocket notification failed for printer %s: %s", printer_id, e
  2236. )
  2237. previous_task = _kill_switch_notification_tasks.pop(printer_id, None)
  2238. if previous_task is not None and not previous_task.done():
  2239. previous_task.cancel()
  2240. _kill_switch_notification_tasks[printer_id] = spawn_background_task(
  2241. _send_kill_switch_provider_notification(printer_id, printer_name, notification_data),
  2242. name=f"kill-switch-notification-{printer_id}",
  2243. )
  2244. else:
  2245. status_logger.warning(
  2246. "[KILL SWITCH] Could not stop unauthorized print on printer %s (state=%s)",
  2247. printer_id,
  2248. state.state,
  2249. )
  2250. except Exception as e:
  2251. status_logger.warning(
  2252. "[KILL SWITCH] Failed to stop unauthorized print on printer %s: %s", printer_id, e
  2253. )
  2254. # MQTT relay - publish status (before dedup check - always publish to MQTT)
  2255. try:
  2256. printer_info = printer_manager.get_printer(printer_id)
  2257. if printer_info:
  2258. await mqtt_relay.on_printer_status(
  2259. printer_id,
  2260. state,
  2261. printer_info.name,
  2262. printer_info.serial_number,
  2263. printer_manager.is_awaiting_plate_clear(printer_id),
  2264. )
  2265. except Exception:
  2266. pass # Don't fail status callback if MQTT fails
  2267. if _last_status_broadcast.get(printer_id) == status_key:
  2268. return # No change, skip WebSocket broadcast
  2269. _last_status_broadcast[printer_id] = status_key
  2270. # Check for progress milestone notifications (25%, 50%, 75%)
  2271. progress = state.progress or 0
  2272. is_printing = state.state in ("RUNNING", "PRINTING")
  2273. if is_printing and progress > 0:
  2274. current_milestone = _progress_milestone_to_notify(printer_id, state)
  2275. if current_milestone is not None:
  2276. try:
  2277. from backend.app.models.printer import Printer
  2278. # Read the printer in a short session and release the connection
  2279. # BEFORE the ~15s camera snapshot below — holding it across the grab
  2280. # pinned a pooled connection per milestone, per printer (issue #2572).
  2281. async with async_session() as db:
  2282. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2283. printer = result.scalar_one_or_none()
  2284. printer_name = printer.name if printer else f"Printer {printer_id}"
  2285. filename = state.subtask_name or state.gcode_file or "Unknown"
  2286. # remaining_time is in minutes, convert to seconds for notification
  2287. remaining_time_seconds = state.remaining_time * 60 if state.remaining_time else None
  2288. # Capture camera snapshot for notification image attachment (no DB held).
  2289. image_data = await _capture_snapshot_for_notification(printer_id, printer, logging.getLogger(__name__))
  2290. # Notification send needs a session (provider/template lookups).
  2291. async with async_session() as db:
  2292. await notification_service.on_print_progress(
  2293. printer_id,
  2294. printer_name,
  2295. filename,
  2296. current_milestone,
  2297. db,
  2298. remaining_time_seconds,
  2299. image_data=image_data,
  2300. )
  2301. except Exception as e:
  2302. logging.getLogger(__name__).warning(f"Progress milestone notification failed: {e}")
  2303. elif progress < 5:
  2304. # Reset milestone tracking when print restarts or new print begins
  2305. _last_progress_milestone[printer_id] = 0
  2306. _first_layer_notified[printer_id] = False
  2307. # HMS error codes that should not trigger notifications even though they
  2308. # have known descriptions (e.g. user-initiated actions, not real errors).
  2309. _HMS_NOTIFICATION_SUPPRESS = {
  2310. "0500_400E", # Printing was cancelled (user action, not an error)
  2311. }
  2312. # Check for new HMS errors and send notifications
  2313. current_hms_errors = getattr(state, "hms_errors", []) or []
  2314. if current_hms_errors:
  2315. new_errors = await _take_new_hms_faults_and_save(printer_id, current_hms_errors, state.state)
  2316. if new_errors:
  2317. try:
  2318. from backend.app.models.printer import Printer
  2319. # Read the printer in a short session and release the connection
  2320. # BEFORE the ~15s camera snapshot below (issue #2572).
  2321. async with async_session() as db:
  2322. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2323. printer = result.scalar_one_or_none()
  2324. printer_name = printer.name if printer else f"Printer {printer_id}"
  2325. # Capture camera snapshot once for all error notifications (no DB held).
  2326. error_image_data = await _capture_snapshot_for_notification(
  2327. printer_id, printer, logging.getLogger(__name__)
  2328. )
  2329. # Notification sends need a session (provider/template lookups).
  2330. async with async_session() as db:
  2331. sent_count = 0
  2332. for error in new_errors:
  2333. # Build short code like "0700_8010"
  2334. # Mask to 16 bits to handle printers that send larger values
  2335. error_code_int = int(error.code.replace("0x", ""), 16) if error.code else 0
  2336. error_code_masked = error_code_int & 0xFFFF
  2337. short_code = f"{(error.attr >> 16) & 0xFFFF:04X}_{error_code_masked:04X}"
  2338. # Only notify for errors Bambu publishes text for — printers
  2339. # send undocumented codes that aren't real errors, and Bambu
  2340. # lists some with empty text. Resolved at parse time (#2926);
  2341. # short_code is still needed for the suppression set below.
  2342. description = error.description
  2343. if not description or short_code in _HMS_NOTIFICATION_SUPPRESS:
  2344. continue
  2345. error_type = _hms_error_title(error.module)
  2346. error_detail = description
  2347. await notification_service.on_printer_error(
  2348. printer_id, printer_name, error_type, db, error_detail, image_data=error_image_data
  2349. )
  2350. sent_count += 1
  2351. if sent_count:
  2352. logging.getLogger(__name__).info(
  2353. f"[HMS] Sent notification for {sent_count} error(s) on printer {printer_id}"
  2354. )
  2355. # Also publish to MQTT relay (no DB): every new fault that
  2356. # counts, with or without text, the same set the UI counts.
  2357. printer_info = printer_manager.get_printer(printer_id)
  2358. if printer_info:
  2359. errors_data = [
  2360. {
  2361. "code": e.code,
  2362. "attr": e.attr,
  2363. "module": e.module,
  2364. "severity": e.severity,
  2365. }
  2366. for e in new_errors
  2367. ]
  2368. await mqtt_relay.on_printer_error(
  2369. printer_id, printer_info.name, printer_info.serial_number, errors_data
  2370. )
  2371. except Exception as e:
  2372. logging.getLogger(__name__).warning(f"HMS error notification failed: {e}")
  2373. else:
  2374. # No HMS errors: nothing to send, but faults gone long enough (or gone
  2375. # across a print state change) are forgotten. Some codes, e.g. chamber
  2376. # temperature regulation during PETG prints, toggle every few seconds.
  2377. await _take_new_hms_faults_and_save(printer_id, [], state.state)
  2378. await ws_manager.send_printer_status(
  2379. printer_id,
  2380. printer_state_to_dict(
  2381. state,
  2382. printer_id,
  2383. printer_manager.get_model(printer_id),
  2384. printer_manager.get_drying_targets(printer_id),
  2385. ),
  2386. )
  2387. def _is_bambu_uuid(tray_uuid: str) -> bool:
  2388. """Check if a tray UUID looks like a valid Bambu Lab RFID UUID (non-empty, non-zero)."""
  2389. return bool(tray_uuid) and tray_uuid not in ("", "0" * len(tray_uuid))
  2390. async def on_fts_inlet_change(printer_id: int, ams_id: int, inlet: str):
  2391. """Re-point a moved AMS's K-profiles at the nozzle it now feeds.
  2392. K-profiles are per-nozzle and the printer's calibration table is numbered
  2393. per-nozzle, but a tray holds exactly one ``cali_idx``. Moving an AMS to the
  2394. switch's other inlet therefore silently invalidates every configured slot in
  2395. it: the index stays put and now resolves against the other nozzle's table.
  2396. Measured on the maintainer's H2C — one spool calibrated 0.018 on the left
  2397. and 0.020 on the right kept the left profile after the move, and a manual
  2398. RFID re-read only re-asserted the same wrong one.
  2399. Configuring a slot is a deliberate preparation step, so this re-selects
  2400. rather than re-configures: only the calibration binding moves, and only for
  2401. slots whose spool already has a stored profile for the new nozzle. A slot
  2402. Bambuddy knows nothing about is left exactly as the operator left it.
  2403. """
  2404. logger = logging.getLogger(__name__)
  2405. target_extruder = extruder_for_inlet(inlet)
  2406. if target_extruder is None:
  2407. return
  2408. client = printer_manager.get_client(printer_id)
  2409. state = printer_manager.get_status(printer_id)
  2410. if not client or not state or not state.raw_data:
  2411. return
  2412. # The nozzle the AMS now feeds -- the diameter of the TARGET extruder, not
  2413. # of nozzle 0. On a machine with two sizes fitted, moving the inlet changes
  2414. # the nozzle width, which changes both the K profile to select and the
  2415. # preset the slot should carry.
  2416. nozzle_diameter = nozzle_diameter_for_extruder(state, target_extruder, printer_manager.get_model(printer_id))
  2417. ams_raw = state.raw_data.get("ams")
  2418. ams_list = ams_raw.get("ams", []) if isinstance(ams_raw, dict) else ams_raw if isinstance(ams_raw, list) else []
  2419. unit = next((u for u in ams_list if str(u.get("id")) == str(ams_id)), None)
  2420. if not unit:
  2421. return
  2422. try:
  2423. async with async_session() as db:
  2424. for tray in unit.get("tray", []):
  2425. tray_id = int(tray.get("id", -1))
  2426. if tray_id < 0 or not tray.get("tray_type"):
  2427. continue
  2428. current_idx = tray.get("cali_idx")
  2429. profile = await find_slot_kprofile_for_extruder(
  2430. db,
  2431. printer_id,
  2432. ams_id,
  2433. tray_id,
  2434. target_extruder,
  2435. nozzle_diameter,
  2436. printer_manager.get_model(printer_id),
  2437. nozzle_flow_for_extruder(state, target_extruder, printer_manager.get_model(printer_id)),
  2438. )
  2439. if profile is None or profile.cali_idx is None:
  2440. continue
  2441. if current_idx == profile.cali_idx:
  2442. continue # Already on the right one.
  2443. logger.info(
  2444. "[Printer %s] AMS %s slot %s moved to inlet %s (nozzle %s): "
  2445. "re-selecting K-profile %s (cali_idx %s -> %s, K=%s)",
  2446. printer_id,
  2447. ams_id,
  2448. tray_id,
  2449. inlet,
  2450. target_extruder,
  2451. profile.name,
  2452. current_idx,
  2453. profile.cali_idx,
  2454. profile.k_value,
  2455. )
  2456. client.extrusion_cali_sel(
  2457. ams_id=ams_id,
  2458. tray_id=tray_id,
  2459. cali_idx=profile.cali_idx,
  2460. filament_id=printer_safe_filament_id(profile.filament_id, tray.get("tray_info_idx", "")),
  2461. nozzle_diameter=nozzle_diameter,
  2462. )
  2463. except Exception as e:
  2464. logger.warning("[Printer %s] Could not re-apply K-profiles after inlet move: %s", printer_id, e)
  2465. def _slot_feeds_print(printer_id: int, ams_id: int, tray_id: int) -> bool:
  2466. """Is this slot the one feeding the printer right now?
  2467. ``tray_now`` names one slot for the whole printer, so on a dual-nozzle
  2468. machine each extruder's own fed slot is asked as well. Only meaningful
  2469. while a print runs; the caller checks that.
  2470. """
  2471. status = printer_manager.get_status(printer_id)
  2472. if status is None:
  2473. return False
  2474. global_id = printer_tray_layout(printer_id, printer_manager).tray_from_slot_key(ams_id, tray_id)
  2475. if getattr(status, "tray_now", None) == global_id:
  2476. return True
  2477. slots = getattr(status, "extruder_slots", None) or {}
  2478. return any(slot.has_filament and slot.holds(ams_id, tray_id) for slot in slots.values())
  2479. def _tray_reports_no_filament(color: str, tray_type: str) -> bool:
  2480. """Does this tray report carry no filament data at all?
  2481. No type, and a colour that is either empty or ``00000000``: firmware sends
  2482. both for a blank slot, and ``00000000`` is what Bambuddy's own slot reset
  2483. writes. A reconnect mid-print brought it in that form and got past every
  2484. guard that only knew the empty string (#3526). A clear spool also reports
  2485. ``00000000``, but always with a type, so it is not blank.
  2486. """
  2487. return not tray_type.strip() and color.strip().upper() in ("", "00000000")
  2488. async def _unlink_stale_assignments(printer_id: int, ams_data: list, printing_now: bool) -> None:
  2489. """Unlink built-in inventory assignments whose slot no longer holds their spool.
  2490. Runs from ``on_ams_change`` and, for removals held by ``slot_unlink_grace``,
  2491. from the delayed re-check -- which is why it takes the AMS data and print
  2492. state as arguments instead of reading them from the push (#3186).
  2493. """
  2494. logger = logging.getLogger(__name__)
  2495. from backend.app.utils.color_utils import colors_similar as _colors_similar
  2496. # Auto-unlink spool assignments with stale fingerprints. Under the
  2497. # per-printer assignment lock since #3186: the held-removal re-check runs
  2498. # this outside any MQTT push, so it can now overlap one.
  2499. try:
  2500. async with get_ams_assignment_lock(printer_id), async_session() as db:
  2501. from sqlalchemy.orm import selectinload
  2502. from backend.app.api.routes.inventory import _find_tray_in_ams_data
  2503. from backend.app.models.spool import Spool as _Spool
  2504. from backend.app.models.spool_assignment import SpoolAssignment as SA
  2505. from backend.app.services.ams_slot_presence import spool_present
  2506. from backend.app.services.inventory_mode import spoolman_owns_assignments
  2507. # Built-in assignments only. Since #2812 they survive a switch to
  2508. # Spoolman mode rather than being deleted by it, and this pass ends
  2509. # in ``db.delete`` — left ungated it would unlink them one slot at a
  2510. # time as the AMS contents changed under the other mode, undoing the
  2511. # preservation more slowly but just as completely.
  2512. assignments = []
  2513. if not await spoolman_owns_assignments(db):
  2514. result = await db.execute(
  2515. select(SA)
  2516. .where(SA.printer_id == printer_id)
  2517. .options(selectinload(SA.spool).selectinload(_Spool.k_profiles))
  2518. )
  2519. assignments = result.scalars().all()
  2520. # ``printing_now`` (top of this function) keeps a runout from
  2521. # unlinking the spool that fed the print — the next idle-time pass
  2522. # unlinks it if the user really did take it out.
  2523. stale = []
  2524. # Removals this pass is holding rather than unlinking (#3186).
  2525. held: set[tuple] = set()
  2526. for assignment in assignments:
  2527. # External spool assignments (ams_id=255) live in vt_tray, not AMS data
  2528. if assignment.ams_id == 255:
  2529. ps = printer_manager.get_status(printer_id)
  2530. vt_tray_raw = ps.raw_data.get("vt_tray", []) if ps else []
  2531. ext_id = external_tray_id(assignment.tray_id) # 0→254, 1→255, rack 2-8→247-253
  2532. current_tray = None
  2533. for vt in vt_tray_raw:
  2534. if isinstance(vt, dict) and int(vt.get("id", 254)) == ext_id:
  2535. current_tray = vt
  2536. break
  2537. if not current_tray:
  2538. # vt_tray data may not have arrived yet — keep assignment
  2539. continue
  2540. else:
  2541. current_tray = _find_tray_in_ams_data(ams_data, assignment.ams_id, assignment.tray_id)
  2542. if not current_tray:
  2543. if printing_now:
  2544. logger.info(
  2545. "Auto-unlink skipped: spool %d AMS%d-T%d — slot empty during a running print (runout?)",
  2546. assignment.spool_id,
  2547. assignment.ams_id,
  2548. assignment.tray_id,
  2549. )
  2550. continue
  2551. # A whole AMS unit can drop out of one push and come back in
  2552. # the next; only a slot that stays gone is a removal (#3186).
  2553. hold_key = ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id)
  2554. if not slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2555. held.add(hold_key)
  2556. logger.info(
  2557. "Auto-unlink held: spool %d AMS%d-T%d — tray not found in AMS data; "
  2558. "unlinking if it is still gone in %ds",
  2559. assignment.spool_id,
  2560. assignment.ams_id,
  2561. assignment.tray_id,
  2562. int(slot_unlink_grace.GRACE_SECONDS),
  2563. )
  2564. continue
  2565. logger.info(
  2566. "Auto-unlink: spool %d AMS%d-T%d — tray not found in AMS data (slot empty?)",
  2567. assignment.spool_id,
  2568. assignment.ams_id,
  2569. assignment.tray_id,
  2570. )
  2571. stale.append(assignment) # Slot empty
  2572. elif _is_bambu_uuid(current_tray.get("tray_uuid", "")):
  2573. # A Bambu Lab spool is in this slot — check if it's the same spool
  2574. # that's currently assigned. If yes, keep the assignment (avoids
  2575. # unnecessary unlink/re-assign/ams_filament_setting cycle that clears
  2576. # the printer's filament preset on every startup).
  2577. tray_uuid = current_tray.get("tray_uuid", "")
  2578. tag_uid = current_tray.get("tag_uid", "")
  2579. spool = assignment.spool
  2580. spool_matches = False
  2581. if spool:
  2582. if (spool.tray_uuid and spool.tray_uuid.upper() == tray_uuid.upper()) or (
  2583. spool.tag_uid
  2584. and tag_uid
  2585. and tag_uid != "0000000000000000"
  2586. and spool.tag_uid.upper() == tag_uid.upper()
  2587. ):
  2588. spool_matches = True
  2589. if spool_matches:
  2590. # Same BL spool still in slot — keep assignment, update fingerprint if needed
  2591. cur_color = current_tray.get("tray_color", "")
  2592. cur_type = current_tray.get("tray_type", "")
  2593. fp_color = assignment.fingerprint_color or ""
  2594. fp_type = assignment.fingerprint_type or ""
  2595. if cur_color.upper() != fp_color.upper() or cur_type.upper() != fp_type.upper():
  2596. assignment.fingerprint_color = cur_color
  2597. assignment.fingerprint_type = cur_type
  2598. logger.debug(
  2599. "Auto-unlink: spool %d AMS%d-T%d — same BL spool, updated fingerprint",
  2600. assignment.spool_id,
  2601. assignment.ams_id,
  2602. assignment.tray_id,
  2603. )
  2604. continue
  2605. # Different BL spool or unrecognized — unlink so auto-assign can match
  2606. logger.info(
  2607. "Auto-unlink: spool %d AMS%d-T%d — different Bambu Lab spool detected (uuid=%s)",
  2608. assignment.spool_id,
  2609. assignment.ams_id,
  2610. assignment.tray_id,
  2611. tray_uuid,
  2612. )
  2613. stale.append(assignment)
  2614. else:
  2615. cur_color = current_tray.get("tray_color", "")
  2616. cur_type = current_tray.get("tray_type", "")
  2617. cur_state = current_tray.get("state")
  2618. fp_color = assignment.fingerprint_color or ""
  2619. fp_type = assignment.fingerprint_type or ""
  2620. # SpoolBuddy pre-config replay: fingerprint_type empty means
  2621. # the slot was empty when the user pre-assigned via SpoolBuddy
  2622. # (the firmware drops ams_filament_setting on empty slots, so
  2623. # MQTT was deferred). The moment any filament gets inserted
  2624. # — Bambu RFID, 3rd-party, or even an existing-but-now-
  2625. # reconfigured spool — fire the deferred configuration.
  2626. # The "loaded" signal is state == 11 (Bambu's "filament fed to
  2627. # extruder" code) OR, on firmwares that don't use the state
  2628. # enum meaningfully, a non-empty tray_type when state is
  2629. # NOT one of the firmware's explicit empty signals (9, 10).
  2630. # state-only was wrong for firmwares that never set 11 — A1
  2631. # Mini BMCU 01.07.02.00 and P1S Standard AMS 00.00.06.75 both
  2632. # always report state=3 — so the replay never fired for them
  2633. # (#1322). The state ∉ {9,10} guard keeps the firmware's
  2634. # explicit "empty" signals authoritative over any stale
  2635. # tray_type that might survive the relay's auto-clearing.
  2636. #
  2637. # tray_exist_bits comes first because that guard cannot tell
  2638. # a firmware "empty" from Bambuddy's own: apply_tray_exist_bits
  2639. # writes state=9 when the bit is 0 and leaves it there when the
  2640. # bit returns. A non-RFID spool inserted into a pre-assigned
  2641. # slot brings no tray_type with it, so the stale 9 made this
  2642. # expression false forever and the deferred config never fired
  2643. # — the deadlock #1322 removed from the assign path, still in
  2644. # place here (#3084, #3100).
  2645. loaded = spool_present(current_tray) is True or (
  2646. cur_state == 11 or (cur_state not in (9, 10) and cur_type.strip())
  2647. )
  2648. # Not into the slot feeding a running print: the printer refuses
  2649. # the setting for it (``ams_filament_setting refused``), and a
  2650. # spool swapped in after a runout sat through one refused send
  2651. # per AMS change. Held, it fires on the first change once the
  2652. # print is over (#3434).
  2653. if (
  2654. not fp_type.strip()
  2655. and loaded
  2656. and assignment.spool
  2657. and printing_now
  2658. and _slot_feeds_print(printer_id, assignment.ams_id, assignment.tray_id)
  2659. ):
  2660. logger.debug(
  2661. "Pre-config held for spool %d AMS%d-T%d: the slot is feeding the print",
  2662. assignment.spool_id,
  2663. assignment.ams_id,
  2664. assignment.tray_id,
  2665. )
  2666. continue
  2667. if not fp_type.strip() and loaded and assignment.spool:
  2668. # The fingerprint ends this replay. Not recorded when nothing
  2669. # could be sent (no MQTT client while the printer reconnects),
  2670. # so the next AMS change tries again. An attempt that raised is
  2671. # recorded as before: it may have sent part of the commands and
  2672. # would raise again on every change, remain% steps included.
  2673. configured: bool | None = False
  2674. try:
  2675. from backend.app.api.routes.inventory import (
  2676. apply_spool_to_slot_via_mqtt,
  2677. load_preset_lookup_user,
  2678. )
  2679. from backend.app.core.auth import is_auth_enabled
  2680. # Whoever made the assignment: their cloud sign-in
  2681. # resolves a custom preset, which nobody's did here,
  2682. # so the slot got Generic PLA on insert (#3519). With
  2683. # login off the shared sign-in applies, as before.
  2684. replay_user = None
  2685. if assignment.assigned_by_user_id is not None and await is_auth_enabled(db):
  2686. replay_user = await load_preset_lookup_user(db, assignment.assigned_by_user_id)
  2687. configured = await apply_spool_to_slot_via_mqtt(
  2688. db=db,
  2689. current_user=replay_user,
  2690. spool=assignment.spool,
  2691. printer_id=printer_id,
  2692. ams_id=assignment.ams_id,
  2693. tray_id=assignment.tray_id,
  2694. current_tray_info_idx=current_tray.get("tray_info_idx", ""),
  2695. current_tray_type=cur_type,
  2696. current_tray_sub_brands=current_tray.get("tray_sub_brands", ""),
  2697. )
  2698. except Exception:
  2699. logger.exception(
  2700. "Pre-config apply failed for spool %d on printer %d AMS%d-T%d",
  2701. assignment.spool_id,
  2702. printer_id,
  2703. assignment.ams_id,
  2704. assignment.tray_id,
  2705. )
  2706. configured = None
  2707. if configured is False:
  2708. logger.warning(
  2709. "Pre-config not sent for spool %d on printer %d AMS%d-T%d (no MQTT client); "
  2710. "retrying on the next AMS change",
  2711. assignment.spool_id,
  2712. printer_id,
  2713. assignment.ams_id,
  2714. assignment.tray_id,
  2715. )
  2716. else:
  2717. if configured:
  2718. logger.info(
  2719. "SpoolBuddy pre-config applied on insert: spool %d → printer %d AMS%d-T%d",
  2720. assignment.spool_id,
  2721. printer_id,
  2722. assignment.ams_id,
  2723. assignment.tray_id,
  2724. )
  2725. assignment.fingerprint_color = cur_color
  2726. assignment.fingerprint_type = cur_type
  2727. continue
  2728. if not _colors_similar(cur_color, fp_color) or cur_type.upper() != fp_type.upper():
  2729. # Blank tray data mid-print is a runout, not a swap: the
  2730. # firmware clears colour and type when it unloads a spool
  2731. # it just emptied. Unlinking here would erase the record
  2732. # of which spool fed the print so far.
  2733. if printing_now and _tray_reports_no_filament(cur_color, cur_type):
  2734. logger.info(
  2735. "Auto-unlink skipped: spool %d AMS%d-T%d — tray data cleared during a running print "
  2736. "(runout?)",
  2737. assignment.spool_id,
  2738. assignment.ams_id,
  2739. assignment.tray_id,
  2740. )
  2741. continue
  2742. # Same reasoning off the print, on firmware's own say-so:
  2743. # a blank tray report from a slot whose tray_exist_bits
  2744. # bit is set describes a spool the AMS cannot identify —
  2745. # a non-RFID one, or one whose slot was reset — not a
  2746. # spool that was taken out. Deleting the assignment there
  2747. # threw away the identity the user had supplied, which is
  2748. # the only place it existed (#3100). A slot the bit calls
  2749. # empty, or one that carries no bit at all, still unlinks.
  2750. #
  2751. # Unless the slot was reported empty first: a removal
  2752. # already held means a spool came out and another went
  2753. # in, so the blank report keeps the hold running below
  2754. # rather than cancelling it (#3186).
  2755. if (
  2756. spool_present(current_tray) is True
  2757. and _tray_reports_no_filament(cur_color, cur_type)
  2758. and not slot_unlink_grace.is_held(
  2759. printer_id,
  2760. ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id),
  2761. )
  2762. ):
  2763. logger.info(
  2764. "Auto-unlink skipped: spool %d AMS%d-T%d — slot still occupied, "
  2765. "tray reports no filament data yet",
  2766. assignment.spool_id,
  2767. assignment.ams_id,
  2768. assignment.tray_id,
  2769. )
  2770. continue
  2771. # A blank report the presence bit does not vouch for is
  2772. # still only one push. An idle X1C cleared a whole AMS
  2773. # unit's bits, colour and type for a moment and lost
  2774. # four saved assignments that way (#3186); unlink only
  2775. # if the slot is still blank after the grace period.
  2776. if _tray_reports_no_filament(cur_color, cur_type):
  2777. hold_key = ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id)
  2778. if not slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2779. held.add(hold_key)
  2780. logger.info(
  2781. "Auto-unlink held: spool %d AMS%d-T%d — tray reports no filament data; "
  2782. "unlinking if it is still blank in %ds",
  2783. assignment.spool_id,
  2784. assignment.ams_id,
  2785. assignment.tray_id,
  2786. int(slot_unlink_grace.GRACE_SECONDS),
  2787. )
  2788. continue
  2789. # Fingerprint mismatch — but check if tray now matches the
  2790. # assigned spool (e.g. auto-configure changed the tray).
  2791. # Both sides are reduced to the type the slot can carry
  2792. # before comparing: the assign path writes that rather
  2793. # than the spool's raw material (#2902), so a spool whose
  2794. # material is a product line — "PLA+", "HTPLA" — reports
  2795. # back as "PLA" and would otherwise fail this check and
  2796. # be auto-unlinked from the slot it was just assigned to.
  2797. # Reducing the printer's side too keeps slots configured
  2798. # by an older Bambuddy, still reporting "PLA+", matching.
  2799. spool = assignment.spool
  2800. if spool:
  2801. spool_color = (spool.rgba or "FFFFFFFF").upper()
  2802. # Two ways the assign path can have arrived at the
  2803. # slot's type, so both count as "we wrote this".
  2804. # The material column is one; the spool's preset is
  2805. # the other, and it outranks the material when the
  2806. # spool has one -- a spool whose material says PLA
  2807. # and whose preset is "Bambu PLA Aero" puts
  2808. # PLA-AERO in the slot (#2902). Read from the stored
  2809. # preset name rather than resolving the preset,
  2810. # because this runs on every AMS push and a cloud
  2811. # lookup here would be both slow and unavailable on
  2812. # the unauthenticated replay path.
  2813. spool_types = {printer_filament_type(spool.material).upper()}
  2814. if spool.slicer_filament_name:
  2815. spool_types.add(printer_filament_type(spool.slicer_filament_name).upper())
  2816. # An imported local preset stores its type outright,
  2817. # which is what the assign path used -- and the name
  2818. # above may be unset. One keyed read, and only on a
  2819. # mismatch, which is rare.
  2820. #
  2821. # slicer_filament is free text up to fifty characters,
  2822. # so the digits have to be checked against the range
  2823. # of the integer primary key they are about to be
  2824. # compared with. Postgres raises on an out-of-range
  2825. # integer rather than simply not matching, and that
  2826. # would poison this session and abandon the rest of
  2827. # the cleanup pass.
  2828. lp_ref = (spool.slicer_filament or "").strip()
  2829. if lp_ref.isdigit() and int(lp_ref) <= 2147483647:
  2830. from backend.app.models.local_preset import LocalPreset as _LP
  2831. lp_type = await db.scalar(select(_LP.filament_type).where(_LP.id == int(lp_ref)))
  2832. if lp_type:
  2833. spool_types.add(printer_filament_type(lp_type).upper())
  2834. # A support spool went in with the slicer's suffix
  2835. # ("PLA-S") that none of the above carries (#3559).
  2836. # A spool made from a support RFID tray has its
  2837. # material as "Support For PLA", so that counts too.
  2838. support_name = (
  2839. f"{spool.slicer_filament_name or ''} {spool.material or ''} {spool.subtype or ''}"
  2840. )
  2841. spool_types |= {
  2842. support_filament_type(t, spool.slicer_filament, support_name).upper()
  2843. for t in spool_types
  2844. }
  2845. if (
  2846. _colors_similar(cur_color, spool_color)
  2847. and printer_filament_type(cur_type).upper() in spool_types
  2848. ):
  2849. logger.info(
  2850. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch but tray matches spool, updating fp",
  2851. assignment.spool_id,
  2852. assignment.ams_id,
  2853. assignment.tray_id,
  2854. )
  2855. assignment.fingerprint_color = cur_color
  2856. assignment.fingerprint_type = cur_type
  2857. continue
  2858. logger.info(
  2859. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch (cur=%s/%s fp=%s/%s spool=%s/%s)",
  2860. assignment.spool_id,
  2861. assignment.ams_id,
  2862. assignment.tray_id,
  2863. cur_color,
  2864. cur_type,
  2865. fp_color,
  2866. fp_type,
  2867. spool.rgba if spool else "?",
  2868. spool.material if spool else "?",
  2869. )
  2870. stale.append(assignment) # Spool changed
  2871. slot_unlink_grace.settle(printer_id, "inventory", held)
  2872. # Snapshot slots before delete — ORM attribute access after the
  2873. # commit would refresh against a deleted row.
  2874. unlinked_slots = [(a.ams_id, a.tray_id) for a in stale]
  2875. for a in stale:
  2876. await db.delete(a)
  2877. if stale:
  2878. logger.info("Auto-unlinked %d stale spool assignments for printer %d", len(stale), printer_id)
  2879. # Commit any changes (stale deletions and/or fingerprint updates)
  2880. await db.commit()
  2881. # Tell open browsers the assignment is gone (#2575). Only the manual
  2882. # REST assign/unassign endpoints broadcast this event; without it the
  2883. # frontend's spool-assignments cache keeps rendering the unlinked
  2884. # spool on the slot until an unrelated refetch — which reads exactly
  2885. # like "the fix didn't work" (reporter verified: a browser refresh
  2886. # after the swap showed the correct state all along).
  2887. for ams_id, tray_id in unlinked_slots:
  2888. await ws_manager.broadcast(
  2889. {
  2890. "type": "spool_assignment_changed",
  2891. "printer_id": printer_id,
  2892. "ams_id": ams_id,
  2893. "tray_id": tray_id,
  2894. }
  2895. )
  2896. except Exception as e:
  2897. logger.warning("Spool assignment cleanup failed: %s", e, exc_info=True)
  2898. async def _expire_spoolman_empty_slots(printer_id: int, ams_data: list, printing_now: bool) -> None:
  2899. """Delete Spoolman slot rows whose held removal has run its grace period.
  2900. The Spoolman half of the #3186 re-check. The full sync in ``on_ams_change``
  2901. talks to Spoolman for every tray; this only needs the local rows, so it
  2902. repeats that pass's empty-slot decision -- a tray with no type or no colour
  2903. (``parse_ams_tray`` returns None for exactly those), not during a print, and
  2904. not in a slot the presence bit calls occupied -- and nothing else.
  2905. """
  2906. logger = logging.getLogger(__name__)
  2907. try:
  2908. async with async_session() as db:
  2909. from backend.app.api.routes.settings import get_setting
  2910. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  2911. from backend.app.services.ams_slot_presence import spool_present
  2912. enabled = await get_setting(db, "spoolman_enabled")
  2913. if not enabled or enabled.lower() != "true":
  2914. return
  2915. sync_mode = await get_setting(db, "spoolman_sync_mode")
  2916. if sync_mode and sync_mode != "auto":
  2917. return
  2918. trays: dict[tuple[int, int], dict] = {}
  2919. for ams_unit in ams_data or []:
  2920. if not isinstance(ams_unit, dict):
  2921. continue
  2922. for tray in ams_unit.get("tray", []):
  2923. if isinstance(tray, dict):
  2924. trays[(int(ams_unit.get("id", 0)), int(tray.get("id", 0)))] = tray
  2925. rows = (
  2926. (
  2927. await db.execute(
  2928. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  2929. )
  2930. )
  2931. .scalars()
  2932. .all()
  2933. )
  2934. held: set[tuple] = set()
  2935. expired: list[tuple[int, int]] = []
  2936. for row in rows:
  2937. tray = trays.get((row.ams_id, row.tray_id))
  2938. if tray is None or printing_now:
  2939. continue
  2940. if (tray.get("tray_type") or "").strip() and (tray.get("tray_color") or "").strip():
  2941. continue
  2942. hold_key = ("spoolman", row.ams_id, row.tray_id, row.spoolman_spool_id)
  2943. if spool_present(tray) is True and not slot_unlink_grace.is_held(printer_id, hold_key):
  2944. continue
  2945. if slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2946. expired.append((row.ams_id, row.tray_id))
  2947. else:
  2948. held.add(hold_key)
  2949. slot_unlink_grace.settle(printer_id, "spoolman", held)
  2950. if not expired:
  2951. return
  2952. # A statement rather than ORM deletes, like the sync pass: the two
  2953. # can overlap, and a row the other already removed must not fail
  2954. # this commit.
  2955. for ams_id, tray_id in expired:
  2956. await db.execute(
  2957. delete(SpoolmanSlotAssignment).where(
  2958. SpoolmanSlotAssignment.printer_id == printer_id,
  2959. SpoolmanSlotAssignment.ams_id == ams_id,
  2960. SpoolmanSlotAssignment.tray_id == tray_id,
  2961. )
  2962. )
  2963. await db.commit()
  2964. logger.info("Unlinked %d Spoolman slot(s) that stayed empty for printer %d", len(expired), printer_id)
  2965. for ams_id, tray_id in expired:
  2966. await ws_manager.broadcast(
  2967. {
  2968. "type": "spool_assignment_changed",
  2969. "printer_id": printer_id,
  2970. "ams_id": ams_id,
  2971. "tray_id": tray_id,
  2972. }
  2973. )
  2974. except Exception as e:
  2975. logger.warning("Spoolman slot re-check failed for printer %s: %s", printer_id, e, exc_info=True)
  2976. async def _recheck_held_unlinks(printer_id: int) -> None:
  2977. """Re-run just the slot cleanup against the printer's current AMS state.
  2978. Scheduled by ``slot_unlink_grace`` when it holds a removal. The unlink
  2979. passes otherwise run only when the AMS hash changes, and a slot that went
  2980. empty and stayed empty may never change it again.
  2981. """
  2982. status = printer_manager.get_status(printer_id)
  2983. # A disconnected printer's state is its last report, not a new one: acting
  2984. # on it would "confirm" a removal nobody has seen for the whole grace
  2985. # period. Leave the holds; the first push after reconnecting decides.
  2986. if status is None or not status.connected:
  2987. return
  2988. ams_raw = status.raw_data.get("ams")
  2989. ams_data = ams_raw.get("ams", []) if isinstance(ams_raw, dict) else ams_raw if isinstance(ams_raw, list) else []
  2990. printing_now = (getattr(status, "state", "") or "").upper() in ("RUNNING", "PAUSE")
  2991. await _unlink_stale_assignments(printer_id, ams_data, printing_now)
  2992. await _expire_spoolman_empty_slots(printer_id, ams_data, printing_now)
  2993. slot_unlink_grace.set_recheck(_recheck_held_unlinks)
  2994. async def on_ams_change(printer_id: int, ams_data: list):
  2995. """Handle AMS data changes - sync to Spoolman if enabled and auto mode."""
  2996. logger = logging.getLogger(__name__)
  2997. # Snapshot BEFORE any await: if a print is active, or a finished one is
  2998. # still being charged, skip weight sync later. on_print_complete may pop
  2999. # _active_sessions during our awaits (#880).
  3000. from backend.app.services.usage_tracker import printer_is_charging
  3001. _print_active = printer_is_charging(printer_id)
  3002. # A slot that reports empty while a print is running is a filament runout,
  3003. # not a spool swap: the spool is still physically in the AMS, just
  3004. # consumed. Dropping either inventory backend's slot link there loses the
  3005. # only record of which spool fed the print, so the completion path can't
  3006. # charge the runout segment to anything. Both cleanup passes below consult
  3007. # this; computed once, up front, so neither depends on the other having run.
  3008. _unlink_state = printer_manager.get_status(printer_id)
  3009. printing_now = (getattr(_unlink_state, "state", "") or "").upper() in ("RUNNING", "PAUSE")
  3010. # The session alone is not enough: a restart that finds a stale persisted
  3011. # session registers none for the print that is running.
  3012. _print_active = _print_active or printing_now
  3013. # MQTT relay - publish AMS change
  3014. try:
  3015. printer_info = printer_manager.get_printer(printer_id)
  3016. if printer_info:
  3017. await mqtt_relay.on_ams_change(printer_id, printer_info.name, printer_info.serial_number, ams_data)
  3018. except Exception:
  3019. pass # Don't fail AMS callback if MQTT fails
  3020. # Broadcast AMS change via WebSocket (bypasses status_key deduplication)
  3021. # This ensures frontend gets immediate updates when AMS slots are configured
  3022. try:
  3023. state = printer_manager.get_status(printer_id)
  3024. if state:
  3025. logger.info("[Printer %s] Broadcasting AMS change via WebSocket", printer_id)
  3026. await ws_manager.send_printer_status(
  3027. printer_id,
  3028. printer_state_to_dict(
  3029. state,
  3030. printer_id,
  3031. printer_manager.get_model(printer_id),
  3032. printer_manager.get_drying_targets(printer_id),
  3033. ),
  3034. )
  3035. except Exception as e:
  3036. logger.warning("Failed to broadcast AMS change for printer %s: %s", printer_id, e)
  3037. await _unlink_stale_assignments(printer_id, ams_data, printing_now)
  3038. # Spools whose tag this pass assigned to a slot; their other slots are
  3039. # released below, once this printer's lock is let go (#3427).
  3040. rfid_assigned: list[tuple[int, tuple[int, int, int]]] = []
  3041. # Auto-manage inventory spools from AMS tray data (skip if Spoolman manages AMS).
  3042. # Serialised per-printer via services/ams_assignment_lock: MQTT bursts can deliver
  3043. # two AMS pushes ~30 ms apart, and without the lock both callbacks read
  3044. # "no existing assignment" for the same (printer, ams, tray) and race to
  3045. # INSERT, hitting the spool_assignment_printer_id_ams_id_tray_id_key
  3046. # unique constraint on Postgres. SQLite's WAL serialises writes so the
  3047. # bug stayed latent there. See services/ams_assignment_lock for details.
  3048. try:
  3049. async with get_ams_assignment_lock(printer_id), async_session() as db:
  3050. from sqlalchemy.orm import selectinload
  3051. from backend.app.api.routes.settings import get_setting
  3052. from backend.app.models.spool import Spool
  3053. from backend.app.models.spool_assignment import SpoolAssignment as SA
  3054. from backend.app.services.spool_tag_matcher import (
  3055. auto_assign_spool,
  3056. create_spool_from_tray,
  3057. find_matching_untagged_spool,
  3058. get_spool_by_tag,
  3059. is_bambu_tag,
  3060. is_valid_tag,
  3061. link_tag_to_inventory_spool,
  3062. tag_belongs_to_archived_spool,
  3063. )
  3064. _spoolman_on = await get_setting(db, "spoolman_enabled")
  3065. _auto_add_raw = await get_setting(db, "auto_add_unknown_rfid")
  3066. _auto_add_unknown = _auto_add_raw is None or _auto_add_raw.lower() == "true"
  3067. if not _spoolman_on or _spoolman_on.lower() != "true":
  3068. for ams_unit in ams_data:
  3069. if not isinstance(ams_unit, dict):
  3070. continue
  3071. ams_id = int(ams_unit.get("id", 0))
  3072. for tray in ams_unit.get("tray", []):
  3073. if not isinstance(tray, dict):
  3074. continue
  3075. tray_id = int(tray.get("id", 0))
  3076. tag_uid = tray.get("tag_uid", "")
  3077. tray_uuid = tray.get("tray_uuid", "")
  3078. tray_info_idx = tray.get("tray_info_idx", "")
  3079. if not tray.get("tray_type"):
  3080. # Slot reported empty — drop any cached unknown-tag
  3081. # broadcast so reinserting the same spool re-prompts.
  3082. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id)
  3083. continue # Empty slot
  3084. # Check if assignment already exists for this slot
  3085. existing = await db.execute(
  3086. select(SA)
  3087. .options(selectinload(SA.spool).selectinload(Spool.k_profiles))
  3088. .where(SA.printer_id == printer_id, SA.ams_id == ams_id, SA.tray_id == tray_id)
  3089. )
  3090. existing_assignment = existing.scalar_one_or_none()
  3091. if existing_assignment:
  3092. # Sync spool weight_used from AMS remain — only INCREASE, never decrease.
  3093. # The AMS remain% is low-resolution (integer %, i.e. 10g steps for 1kg spool)
  3094. # and must not overwrite precise values from the usage tracker (3MF/G-code).
  3095. # Skip during active prints: the usage tracker handles deduction
  3096. # precisely via 3MF data on print completion. Without this guard the
  3097. # AMS remain% SET and the usage tracker ADD both fire from the same
  3098. # MQTT message, doubling the deduction (#880).
  3099. if _print_active:
  3100. continue
  3101. remain_raw = tray.get("remain")
  3102. if (
  3103. remain_raw is not None
  3104. and existing_assignment.spool
  3105. and not existing_assignment.spool.weight_locked
  3106. ):
  3107. try:
  3108. remain_val = int(remain_raw)
  3109. except (TypeError, ValueError):
  3110. remain_val = -1
  3111. if 1 <= remain_val <= 100:
  3112. lw = existing_assignment.spool.label_weight or 1000
  3113. new_used = round(lw * (100 - remain_val) / 100.0, 1)
  3114. current_used = existing_assignment.spool.weight_used or 0
  3115. if new_used > current_used + 1:
  3116. # Rechecked in the statement: the spool was loaded
  3117. # before this, and a weight set by hand (or weighed)
  3118. # and locked since then must stay (WL-6).
  3119. result = cast(
  3120. CursorResult,
  3121. await db.execute(
  3122. update(Spool)
  3123. .where(
  3124. Spool.id == existing_assignment.spool_id,
  3125. Spool.weight_locked.is_not(True),
  3126. func.coalesce(Spool.weight_used, 0) < new_used - 1,
  3127. )
  3128. .values(weight_used=new_used)
  3129. ),
  3130. )
  3131. await db.commit()
  3132. if result.rowcount:
  3133. logger.info(
  3134. "Weight sync: spool %d weight_used %s -> %s (remain=%d)",
  3135. existing_assignment.spool_id,
  3136. current_used,
  3137. new_used,
  3138. remain_val,
  3139. )
  3140. await ws_manager.broadcast({"type": "inventory_changed"})
  3141. # A K-profile the slot lost (the default K after a reset or
  3142. # power cycle) is put back by kprofile_drift on the next idle
  3143. # push; a different real profile was picked on purpose and
  3144. # stays (#3219, item 44).
  3145. continue
  3146. if is_bambu_tag(tag_uid, tray_uuid, tray_info_idx):
  3147. # BL spool with RFID tag: auto-match → inventory match → auto-create
  3148. spool = await get_spool_by_tag(db, tag_uid, tray_uuid)
  3149. # Only a spool already known by this tag is proven to be
  3150. # here; a material/colour match or a new spool is not
  3151. # grounds to take it out of other slots (#3427).
  3152. known_by_tag = spool is not None
  3153. if not spool and await tag_belongs_to_archived_spool(db, tag_uid, tray_uuid):
  3154. # An archived spool still in the AMS: the slot
  3155. # stays free rather than adopting its tag.
  3156. continue
  3157. if not spool:
  3158. # Try matching an untagged inventory spool (same material/color)
  3159. spool = await find_matching_untagged_spool(db, tray)
  3160. if spool:
  3161. await link_tag_to_inventory_spool(db, spool, tray)
  3162. elif _auto_add_unknown:
  3163. spool = await create_spool_from_tray(db, tray)
  3164. else:
  3165. # Auto-add disabled: surface the slot so the
  3166. # user can add it manually via the UI.
  3167. await _broadcast_unknown_tag(
  3168. printer_id=printer_id,
  3169. ams_id=ams_id,
  3170. tray_id=tray_id,
  3171. tag_uid=tag_uid,
  3172. tray_uuid=tray_uuid,
  3173. tray_type=tray.get("tray_type"),
  3174. tray_color=tray.get("tray_color"),
  3175. tray_sub_brands=tray.get("tray_sub_brands"),
  3176. tray_count=len(ams_unit.get("tray", [])),
  3177. )
  3178. continue
  3179. # Slot matched (existing tag, untagged inventory
  3180. # match, or freshly auto-created spool) — drop any
  3181. # stale dedup so a future tag swap re-prompts.
  3182. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id)
  3183. await auto_assign_spool(
  3184. printer_id,
  3185. ams_id,
  3186. tray_id,
  3187. spool,
  3188. printer_manager,
  3189. db,
  3190. tray_info_idx=tray_info_idx,
  3191. )
  3192. await db.commit()
  3193. if known_by_tag:
  3194. rfid_assigned.append((spool.id, (printer_id, ams_id, tray_id)))
  3195. await ws_manager.broadcast(
  3196. {
  3197. "type": "spool_auto_assigned",
  3198. "printer_id": printer_id,
  3199. "ams_id": ams_id,
  3200. "tray_id": tray_id,
  3201. "spool_id": spool.id,
  3202. }
  3203. )
  3204. logger.info(
  3205. "RFID auto-assigned spool %d to printer %d AMS%d-T%d",
  3206. spool.id,
  3207. printer_id,
  3208. ams_id,
  3209. tray_id,
  3210. )
  3211. elif is_valid_tag(tag_uid, tray_uuid) and not await tag_belongs_to_archived_spool(
  3212. db, tag_uid, tray_uuid
  3213. ):
  3214. # Non-BL spool with some tag — let user choose (not
  3215. # an archived one's, whose slot stays free)
  3216. await _broadcast_unknown_tag(
  3217. printer_id=printer_id,
  3218. ams_id=ams_id,
  3219. tray_id=tray_id,
  3220. tag_uid=tag_uid,
  3221. tray_uuid=tray_uuid,
  3222. tray_type=tray.get("tray_type"),
  3223. tray_color=tray.get("tray_color"),
  3224. tray_sub_brands=tray.get("tray_sub_brands"),
  3225. tray_count=len(ams_unit.get("tray", [])),
  3226. )
  3227. # No-tag slots (generic non-RFID filament) are left alone:
  3228. # nothing to identify, prompting "+ Add" would just create
  3229. # ghost spools with empty tags on every confirm.
  3230. except Exception as e:
  3231. logger.warning("RFID spool auto-assign failed: %s", e, exc_info=True)
  3232. if rfid_assigned:
  3233. from backend.app.services.spool_relocation import INTERNAL, release_other_slots
  3234. for spool_id, slot in rfid_assigned:
  3235. try:
  3236. await release_other_slots(INTERNAL, spool_id, slot)
  3237. except Exception as e:
  3238. logger.warning("Releasing spool %s from its other slots failed: %s", spool_id, e, exc_info=True)
  3239. try:
  3240. async with async_session() as db:
  3241. from backend.app.api.routes.settings import get_setting
  3242. from backend.app.models.printer import Printer
  3243. # Check if Spoolman is enabled
  3244. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  3245. if not spoolman_enabled or spoolman_enabled.lower() != "true":
  3246. return
  3247. # Check sync mode
  3248. sync_mode = await get_setting(db, "spoolman_sync_mode")
  3249. if sync_mode and sync_mode != "auto":
  3250. return # Only sync on auto mode
  3251. _auto_add_raw_sm = await get_setting(db, "auto_add_unknown_rfid")
  3252. auto_add_unknown_rfid = _auto_add_raw_sm is None or _auto_add_raw_sm.lower() == "true"
  3253. # `spoolman_disable_weight_sync` is deprecated (#1119) — weight is now
  3254. # always owned by per-print tracking, never by AMS auto-sync. The
  3255. # setting is still read by the settings UI for backwards compat but
  3256. # has no effect on the sync path here.
  3257. # Get Spoolman URL
  3258. spoolman_url = await get_setting(db, "spoolman_url")
  3259. if not spoolman_url:
  3260. return
  3261. # Get or create Spoolman client
  3262. client = await get_spoolman_client()
  3263. if not client:
  3264. try:
  3265. client = await init_spoolman_client(spoolman_url)
  3266. except ValueError as exc:
  3267. logger.warning("Spoolman URL %r rejected by SSRF guard: %s", spoolman_url, exc)
  3268. return
  3269. # Check if Spoolman is reachable
  3270. if not await client.health_check():
  3271. logger.warning("Spoolman not reachable at %s", spoolman_url)
  3272. return
  3273. # Get printer name for location
  3274. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3275. printer = result.scalar_one_or_none()
  3276. printer_name = printer.name if printer else f"Printer {printer_id}"
  3277. # OPTIMIZATION: Fetch all spools once before processing trays
  3278. # This eliminates redundant API calls (one per tray) when syncing multiple trays
  3279. logger.debug("[Printer %s] Fetching spools cache for AMS sync...", printer_id)
  3280. try:
  3281. cached_spools = await client.get_spools()
  3282. logger.debug("[Printer %s] Cached %d spools for batch sync", printer_id, len(cached_spools))
  3283. except Exception as e:
  3284. logger.error(
  3285. "[Printer %s] Failed to fetch spools cache after retries, aborting AMS sync: %s",
  3286. printer_id,
  3287. e,
  3288. )
  3289. return
  3290. # Load inventory weights as fallback (when AMS MQTT data lacks remain values)
  3291. from sqlalchemy.orm import selectinload
  3292. from backend.app.models.spool_assignment import SpoolAssignment
  3293. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  3294. from backend.app.services.ams_slot_presence import spool_present
  3295. from backend.app.services.inventory_mode import spoolman_owns_assignments
  3296. # Built-in remaining weight, used by sync_ams_tray only when the
  3297. # firmware reports an unusable remain%/tray_weight for a slot.
  3298. #
  3299. # Left empty since #2812. This block runs in Spoolman mode only,
  3300. # and until then the built-in table was emptied on the switch, so
  3301. # there was never anything here to read and the fallback was inert.
  3302. # Preserving those rows makes it live again, and it is keyed by slot
  3303. # rather than by spool: after a mode switch the tray may well hold
  3304. # different filament, and ``create_spool`` writes ``remaining_weight``
  3305. # unconditionally, so a stale figure would be seeded into a brand new
  3306. # Spoolman spool. Deliberately kept inert rather than deleted, so the
  3307. # intent survives for whoever revisits the cross-mode fallback.
  3308. inventory_weights: dict[tuple[int, int], float] = {}
  3309. if not await spoolman_owns_assignments(db):
  3310. try:
  3311. assign_result = await db.execute(
  3312. select(SpoolAssignment)
  3313. .options(selectinload(SpoolAssignment.spool))
  3314. .where(SpoolAssignment.printer_id == printer_id)
  3315. )
  3316. for assignment in assign_result.scalars().all():
  3317. spool = assignment.spool
  3318. if spool and spool.label_weight > 0:
  3319. remaining = max(0.0, spool.label_weight - (spool.weight_used or 0))
  3320. inventory_weights[(assignment.ams_id, assignment.tray_id)] = remaining
  3321. except Exception as e:
  3322. logger.warning("Could not load inventory weights for printer %s: %s", printer_id, e)
  3323. # Load existing Spoolman slot assignments for the no-RFID fallback path
  3324. spoolman_slot_map: dict[tuple[int, int], int] = {}
  3325. slot_map_read = False
  3326. try:
  3327. slot_result = await db.execute(
  3328. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  3329. )
  3330. for slot in slot_result.scalars().all():
  3331. spoolman_slot_map[(slot.ams_id, slot.tray_id)] = slot.spoolman_spool_id
  3332. slot_map_read = True
  3333. except Exception as e:
  3334. logger.warning("Could not load Spoolman slot assignments for printer %s: %s", printer_id, e)
  3335. # Sync each AMS tray and collect slot changes for DB persistence
  3336. synced = 0
  3337. slot_changes: list[tuple[int, int, int]] = [] # (ams_id, tray_id, spoolman_spool_id) to upsert
  3338. empty_slots: list[tuple[int, int]] = [] # (ams_id, tray_id) whose tray is now empty
  3339. spoolman_held: set[tuple] = set() # removals held for the grace period (#3186)
  3340. for ams_unit in ams_data:
  3341. if not isinstance(ams_unit, dict):
  3342. continue
  3343. ams_id = int(ams_unit.get("id", 0))
  3344. trays = ams_unit.get("tray", [])
  3345. for tray_data in trays:
  3346. if not isinstance(tray_data, dict):
  3347. continue
  3348. tray_id_raw = int(tray_data.get("id", 0))
  3349. tray = client.parse_ams_tray(ams_id, tray_data)
  3350. if not tray:
  3351. # Empty tray slot — record for local assignment cleanup
  3352. # and drop any cached unknown-tag broadcast so a
  3353. # reinserted spool re-prompts.
  3354. #
  3355. # Not during a running print: a slot that empties there
  3356. # is a filament runout, and the spool is still in the
  3357. # AMS. `spoolman_slot_assignments` is how a tag-less
  3358. # spool assigned through the Bambuddy UI is resolved at
  3359. # completion (#1459), so deleting the row mid-print
  3360. # loses the runout segment's usage — the same failure
  3361. # the internal inventory's auto-unlink had.
  3362. #
  3363. # Nor when firmware's presence bit says the slot is
  3364. # occupied. parse_ams_tray calls a tray with no type or
  3365. # no colour empty, and a spool the AMS cannot read has
  3366. # neither until something configures it — so a tag-less
  3367. # spool assigned through the UI had its row deleted by
  3368. # the first idle push after it was inserted. Same
  3369. # deletion as the internal inventory's in #3100, same
  3370. # answer, so the two modes stay in step.
  3371. #
  3372. # And only once the slot has stayed empty for the grace
  3373. # period -- the internal inventory's #3186 answer, for
  3374. # the same one-push blank.
  3375. linked_spool = spoolman_slot_map.get((ams_id, tray_id_raw))
  3376. hold_key = ("spoolman", ams_id, tray_id_raw, linked_spool)
  3377. if not printing_now and (
  3378. spool_present(tray_data) is not True or slot_unlink_grace.is_held(printer_id, hold_key)
  3379. ):
  3380. if linked_spool is None or slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  3381. empty_slots.append((ams_id, tray_id_raw))
  3382. else:
  3383. spoolman_held.add(hold_key)
  3384. logger.info(
  3385. "Spoolman slot unlink held: AMS%d-T%d (spool %d) reports empty; "
  3386. "unlinking if it is still empty in %ds",
  3387. ams_id,
  3388. tray_id_raw,
  3389. linked_spool,
  3390. int(slot_unlink_grace.GRACE_SECONDS),
  3391. )
  3392. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id_raw)
  3393. continue
  3394. spool_tag = (
  3395. tray.tray_uuid
  3396. if tray.tray_uuid and tray.tray_uuid != "00000000000000000000000000000000"
  3397. else tray.tag_uid
  3398. )
  3399. # Provide the hint only when no RFID is available
  3400. hint = spoolman_slot_map.get((ams_id, tray.tray_id)) if not spool_tag else None
  3401. try:
  3402. inv_remaining = inventory_weights.get((ams_id, tray.tray_id))
  3403. result = await client.sync_ams_tray(
  3404. tray,
  3405. printer_name,
  3406. db,
  3407. # Per-print tracking is the only weight writer (#1119).
  3408. # AMS auto-sync still maintains spool metadata / slot
  3409. # assignments but no longer touches remaining_weight.
  3410. disable_weight_sync=True,
  3411. cached_spools=cached_spools,
  3412. inventory_remaining=inv_remaining,
  3413. spoolman_spool_id_hint=hint,
  3414. auto_add_unknown_rfid=auto_add_unknown_rfid,
  3415. )
  3416. if result is None and spool_tag and not auto_add_unknown_rfid:
  3417. # Spoolman skipped auto-create per user setting — surface
  3418. # the slot so the UI can offer "+ Add to inventory".
  3419. await _broadcast_unknown_tag(
  3420. printer_id=printer_id,
  3421. ams_id=ams_id,
  3422. tray_id=tray.tray_id,
  3423. tag_uid=tray.tag_uid or "",
  3424. tray_uuid=tray.tray_uuid or "",
  3425. tray_type=tray.tray_type,
  3426. tray_color=tray.tray_color,
  3427. tray_sub_brands=tray.tray_sub_brands,
  3428. tray_count=len(trays),
  3429. )
  3430. elif result:
  3431. _clear_unknown_tag_dedup(printer_id, ams_id, tray.tray_id)
  3432. if result:
  3433. synced += 1
  3434. if result.get("id"):
  3435. slot_changes.append((ams_id, tray.tray_id, result["id"]))
  3436. # If a new spool was created, add it to the cache
  3437. # so subsequent trays can find it if they reference the same tag
  3438. spool_exists = any(s.get("id") == result["id"] for s in cached_spools)
  3439. if not spool_exists:
  3440. cached_spools.append(result)
  3441. logger.debug(
  3442. "[Printer %s] Added newly created spool %s to cache",
  3443. printer_id,
  3444. result["id"],
  3445. )
  3446. # Reconcile slot_preset_mappings (the same row internal
  3447. # mode keeps in sync via inventory + spool_tag_matcher).
  3448. # Without this the slot card surfaces the previous spool's
  3449. # preset name — same bug shape, different inventory mode.
  3450. from backend.app.services.slot_preset_writer import (
  3451. upsert_slot_preset_for_spoolman_spool,
  3452. )
  3453. await upsert_slot_preset_for_spoolman_spool(
  3454. db=db,
  3455. spoolman_spool=result,
  3456. tray_info_idx=tray.tray_info_idx or "",
  3457. tray_sub_brands=tray.tray_sub_brands or "",
  3458. tray_type=tray.tray_type or "",
  3459. printer_id=printer_id,
  3460. ams_id=ams_id,
  3461. tray_id=tray.tray_id,
  3462. )
  3463. except Exception as e:
  3464. logger.error("Error syncing AMS %s tray %s: %s", ams_id, tray.tray_id, e)
  3465. slot_unlink_grace.settle(printer_id, "spoolman", spoolman_held)
  3466. if synced > 0:
  3467. logger.info("Auto-synced %s AMS trays to Spoolman for printer %s", synced, printer_id)
  3468. # Persist slot assignment changes to the local table: only slots
  3469. # whose spool this pass changed, and only while they still hold what
  3470. # it read before awaiting Spoolman, so a spool assigned by hand in
  3471. # between stays (SSA-3). The changed slots are announced: the slot
  3472. # card reads slot_preset_mappings (rewritten above) ahead of the live
  3473. # tray, so with no event it kept the previous spool's preset name.
  3474. if slot_changes or empty_slots:
  3475. from backend.app.services.spoolman_slot_sync import write_slot_results
  3476. try:
  3477. await write_slot_results(
  3478. db,
  3479. {(printer_id, a_id, t_id): spool_id for (a_id, t_id), spool_id in spoolman_slot_map.items()}
  3480. if slot_map_read
  3481. else None,
  3482. [(printer_id, a_id, t_id, spool_id) for a_id, t_id, spool_id in slot_changes],
  3483. [(printer_id, a_id, t_id) for a_id, t_id in empty_slots],
  3484. )
  3485. except Exception as e:
  3486. await db.rollback()
  3487. logger.error("Error persisting Spoolman slot assignments for printer %s: %s", printer_id, e)
  3488. except Exception as e:
  3489. logging.getLogger(__name__).error("Spoolman AMS sync failed for printer %s: %s", printer_id, e)
  3490. # Largest finish photo attached to a notification; a bigger one is still linked.
  3491. _FINISH_PHOTO_ATTACH_MAX_BYTES = 2_500_000
  3492. async def _finish_photo_for_notification(
  3493. db, archive, archive_id: int, filename: str
  3494. ) -> tuple[str | None, bytes | None]:
  3495. """The ``{finish_photo_url}`` link and the bytes to attach, for a print's finish photo.
  3496. With authentication off the link is the archive's own photo route, as it
  3497. always was: it opens without a login and doesn't expire. With
  3498. authentication on that route needs a media token, which nothing tapping a
  3499. link in Telegram, CallMeBot or a Home Assistant notification has, so the
  3500. link only ever answered 401. The photo is then saved as a notification
  3501. photo as well (utils/notification_photos.py) and the link points there: an
  3502. unguessable name that opens this one photo and nothing else, for 3 days.
  3503. The link is relative when no External URL is set. Bytes over
  3504. ``_FINISH_PHOTO_ATTACH_MAX_BYTES`` are linked but not attached. Returns
  3505. ``(None, None)`` when the photo can't be found.
  3506. """
  3507. from backend.app.api.routes.settings import get_setting
  3508. from backend.app.core.auth import is_auth_enabled
  3509. from backend.app.utils.archive_paths import find_archive_photo
  3510. from backend.app.utils.notification_photos import save_notification_photo
  3511. log = logging.getLogger(__name__)
  3512. base = ((await get_setting(db, "external_url")) or "").strip().rstrip("/")
  3513. url: str | None = f"{base}/api/v1/archives/{archive_id}/photos/{filename}"
  3514. photo_bytes: bytes | None = None
  3515. try:
  3516. photo_path = find_archive_photo(archive, filename)
  3517. if photo_path is not None:
  3518. photo_bytes = await asyncio.to_thread(photo_path.read_bytes)
  3519. except Exception as e:
  3520. log.warning("[NOTIFY-BG] Failed to read finish photo bytes: %s", e)
  3521. try:
  3522. auth_on = await is_auth_enabled(db)
  3523. except Exception:
  3524. auth_on = True # Unknown: the archive link may need a login, so don't rely on it.
  3525. if auth_on:
  3526. url = None
  3527. if photo_bytes:
  3528. try:
  3529. name = await asyncio.to_thread(save_notification_photo, photo_bytes, "print_complete")
  3530. url = f"{base}/api/v1/notifications/photos/{name}"
  3531. except Exception as e:
  3532. log.warning("[NOTIFY-BG] Failed to save finish photo for its link: %s", e)
  3533. if photo_bytes is not None and len(photo_bytes) > _FINISH_PHOTO_ATTACH_MAX_BYTES:
  3534. log.warning("[NOTIFY-BG] Finish photo too large for attachment: %s bytes", len(photo_bytes))
  3535. return url, None
  3536. if photo_bytes:
  3537. log.info("[NOTIFY-BG] Loaded finish photo bytes: %s bytes", len(photo_bytes))
  3538. return url, photo_bytes
  3539. async def _capture_snapshot_for_notification(printer_id: int, printer, logger, *, light: bool = True) -> bytes | None:
  3540. """Capture a camera snapshot for notification image attachment.
  3541. Returns JPEG bytes (max 2.5MB) or None if capture fails or is unavailable.
  3542. Uses: external camera > buffered frame > fresh capture.
  3543. Turns the chamber light on for the picture when the printer asks for it
  3544. (#1655). ``light=False`` is for callers that capture all through a print,
  3545. where the light would flash with every frame.
  3546. """
  3547. if not printer:
  3548. return None
  3549. async with camera_light(printer if light else None):
  3550. return await _capture_snapshot_frame(printer_id, printer, logger)
  3551. async def _capture_snapshot_frame(printer_id: int, printer, logger) -> bytes | None:
  3552. """The capture behind ``_capture_snapshot_for_notification``."""
  3553. try:
  3554. from backend.app.api.routes.settings import get_setting
  3555. async with async_session() as db:
  3556. capture_enabled = await get_setting(db, "capture_finish_photo")
  3557. if capture_enabled is not None and capture_enabled.lower() != "true":
  3558. return None
  3559. # Try external camera first
  3560. if printer.external_camera_enabled and printer.external_camera_url:
  3561. logger.info("[SNAPSHOT] Capturing from external camera for printer %s", printer_id)
  3562. from backend.app.api.routes.camera import live_frame_for_capture
  3563. from backend.app.services.external_camera import capture_frame
  3564. # An external camera allows one reader, so capturing while a viewer
  3565. # is attached fails (#2707). A None here falls through to the paths
  3566. # below exactly as a failed capture did.
  3567. defer, buffered = live_frame_for_capture(printer_id)
  3568. if defer:
  3569. frame_data = buffered
  3570. else:
  3571. frame_data = await capture_frame(
  3572. printer.external_camera_url,
  3573. printer.external_camera_type or "mjpeg",
  3574. snapshot_url=printer.external_camera_snapshot_url,
  3575. )
  3576. if frame_data and len(frame_data) <= 2_500_000:
  3577. logger.info("[SNAPSHOT] External camera frame: %s bytes", len(frame_data))
  3578. return _apply_camera_rotation(frame_data, printer, logger)
  3579. # Try buffered frame from active stream
  3580. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  3581. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  3582. active_chamber = [k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")]
  3583. buffered_frame = get_buffered_frame(printer_id)
  3584. if (active_for_printer or active_chamber) and buffered_frame:
  3585. logger.info("[SNAPSHOT] Using buffered frame for printer %s: %s bytes", printer_id, len(buffered_frame))
  3586. if len(buffered_frame) <= 2_500_000:
  3587. return _apply_camera_rotation(buffered_frame, printer, logger)
  3588. # Fresh capture from printer camera
  3589. logger.info("[SNAPSHOT] Capturing fresh frame for printer %s", printer_id)
  3590. from backend.app.services.camera import capture_camera_frame_bytes
  3591. frame_data = await capture_camera_frame_bytes(
  3592. printer.ip_address, printer.access_code, printer.model, timeout=15
  3593. )
  3594. if frame_data and len(frame_data) <= 2_500_000:
  3595. logger.info("[SNAPSHOT] Fresh camera frame: %s bytes", len(frame_data))
  3596. return _apply_camera_rotation(frame_data, printer, logger)
  3597. except Exception as e:
  3598. logger.warning("[SNAPSHOT] Failed to capture snapshot for printer %s: %s", printer_id, e)
  3599. return None
  3600. async def _maybe_bank_inprint_frame(printer_id: int, layer_num: int) -> None:
  3601. """#1867: bank a recent in-print camera frame for the finish photo.
  3602. Called on every layer change and (#2547) on every print-progress advance.
  3603. Grabs one frame (throttled) into ``_inprint_frame_bank`` so the finish-photo
  3604. path has a pre-End-G-code image for prints that end with a plate swap.
  3605. Both drivers are print telemetry that stops the instant printing ends: no
  3606. further layers, and progress freezes before the End G-code (e.g. SwapMod
  3607. plate swap) executes. So the last banked frame is always the finished print,
  3608. never the swapped plate — that property is what the #1867 path relies on and
  3609. it must survive any change to the throttle below.
  3610. Layer changes alone were not enough: they stop when the *final* layer
  3611. begins, which on a three-minute last layer left the bank stale by the whole
  3612. length of that layer (#2547). Progress keeps ticking through it.
  3613. Best-effort: any failure just leaves the previous banked frame.
  3614. """
  3615. logger = logging.getLogger(__name__)
  3616. client = printer_manager.get_client(printer_id)
  3617. state = client.state if client else None
  3618. if not state or state.state != "RUNNING":
  3619. return
  3620. # Only during actual extrusion — firmware ticks layer_num during the
  3621. # pre-print calibration sequence, whose sub-stages are non-zero.
  3622. if state.mc_print_sub_stage not in (None, 0):
  3623. return
  3624. # #2547: throttled uniformly, with no last-layer exemption. The old code
  3625. # bypassed the throttle on the final layer to guarantee a fresh frame there;
  3626. # now that progress advances also drive banking, that exemption would fire a
  3627. # camera grab on every percent tick of the last layer. Bambu printers accept
  3628. # one RTSP client at a time, so each grab contends with the live view.
  3629. now = time.monotonic()
  3630. last = _inprint_frame_bank_ts.get(printer_id, 0.0)
  3631. if (now - last) < _INPRINT_BANK_MIN_INTERVAL:
  3632. return
  3633. total = state.total_layers or 0
  3634. try:
  3635. async with async_session() as db:
  3636. from backend.app.models.printer import Printer
  3637. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3638. printer = result.scalar_one_or_none()
  3639. if not printer:
  3640. return
  3641. # Reuses the notification snapshot path, which honours the
  3642. # `capture_finish_photo` setting (returns None when disabled) so we
  3643. # don't bank frames the user never asked for. Without the chamber light
  3644. # (#1655): this runs every 25 seconds for the whole print.
  3645. frame = await _capture_snapshot_for_notification(printer_id, printer, logger, light=False)
  3646. if frame:
  3647. _inprint_frame_bank[printer_id] = frame
  3648. _inprint_frame_bank_ts[printer_id] = now
  3649. logger.debug(
  3650. "[FINISH-PHOTO-BANK] banked in-print frame for printer %s at layer %s/%s (%d bytes)",
  3651. printer_id,
  3652. layer_num,
  3653. total,
  3654. len(frame),
  3655. )
  3656. except Exception as e:
  3657. logger.debug("[FINISH-PHOTO-BANK] bank failed for printer %s: %s", printer_id, e)
  3658. def _apply_camera_rotation(image_data: bytes, printer, logger) -> bytes:
  3659. """Apply camera rotation to snapshot image if configured."""
  3660. from backend.app.services.camera import apply_camera_rotation
  3661. return apply_camera_rotation(image_data, getattr(printer, "camera_rotation", 0), logger)
  3662. async def _send_print_start_notification(
  3663. printer_id: int,
  3664. data: dict,
  3665. archive_data: dict | None = None,
  3666. logger=None,
  3667. ):
  3668. """Helper to send print start notification with optional archive data."""
  3669. if logger is None:
  3670. logger = logging.getLogger(__name__)
  3671. try:
  3672. async with async_session() as db:
  3673. from backend.app.models.printer import Printer
  3674. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3675. printer = result.scalar_one_or_none()
  3676. printer_name = printer.name if printer else f"Printer {printer_id}"
  3677. # Capture camera snapshot for notification image attachment
  3678. image_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  3679. if image_data:
  3680. if archive_data is None:
  3681. archive_data = {}
  3682. archive_data["image_data"] = image_data
  3683. await notification_service.on_print_start(printer_id, printer_name, data, db, archive_data=archive_data)
  3684. # Send user-specific email notification for print start
  3685. if archive_data and archive_data.get("created_by_id"):
  3686. await notification_service.send_user_print_email(
  3687. event_type="user_print_start",
  3688. created_by_id=archive_data["created_by_id"],
  3689. printer_name=printer_name,
  3690. filename=data.get("subtask_name") or data.get("filename", "Unknown"),
  3691. db=db,
  3692. )
  3693. except Exception as e:
  3694. logger.warning("Notification on_print_start failed: %s", e)
  3695. async def _dispatch_user_print_email(
  3696. status: str,
  3697. created_by_id: int | None,
  3698. printer_name: str,
  3699. filename: str,
  3700. db,
  3701. image_data: bytes | None = None,
  3702. finish_photo_url: str | None = None,
  3703. ) -> None:
  3704. """Send a user-specific print-completion email based on print status.
  3705. Maps the normalised print status to the correct event type and delegates
  3706. to :meth:`NotificationService.send_user_print_email`. A single helper
  3707. avoids duplicating the ``if status == "completed" / elif "failed" / elif
  3708. "stopped"`` dispatch block at every call site.
  3709. Does nothing if *created_by_id* is ``None``.
  3710. """
  3711. if created_by_id is None:
  3712. return
  3713. if status == "completed":
  3714. event_type = "user_print_complete"
  3715. elif status == "failed":
  3716. event_type = "user_print_failed"
  3717. elif status in ("stopped", "aborted", "cancelled"):
  3718. event_type = "user_print_stopped"
  3719. else:
  3720. return
  3721. await notification_service.send_user_print_email(
  3722. event_type=event_type,
  3723. created_by_id=created_by_id,
  3724. printer_name=printer_name,
  3725. filename=filename,
  3726. db=db,
  3727. image_data=image_data,
  3728. finish_photo_url=finish_photo_url,
  3729. )
  3730. def _load_objects_from_archive(archive, printer_id: int, logger) -> None:
  3731. """Extract printable objects from an archive's 3MF file and store in printer state."""
  3732. try:
  3733. from backend.app.services.archive import extract_printable_objects_from_archive
  3734. client = printer_manager.get_client(printer_id)
  3735. if not client:
  3736. return
  3737. # Extract with positions for UI overlay, scoped to the plate that
  3738. # is printing — resolve_plate_id is the same resolver /cover uses,
  3739. # so the object list can't disagree with the thumbnail it is drawn
  3740. # over (#2522).
  3741. printable_objects, bbox_all = extract_printable_objects_from_archive(
  3742. app_settings.base_dir / archive.file_path,
  3743. plate_number=resolve_plate_id(client.state),
  3744. )
  3745. if printable_objects:
  3746. client.state.printable_objects = printable_objects
  3747. client.state.printable_objects_bbox_all = bbox_all
  3748. client.state.skipped_objects = []
  3749. logger.info("Loaded %s printable objects for printer %s", len(printable_objects), printer_id)
  3750. except Exception as e:
  3751. logger.debug("Failed to extract printable objects from archive: %s", e)
  3752. def _load_pauses_from_3mf(threemf_path, printer_id: int, *, warned_layers=(), passed_through_layer: int = 0) -> None:
  3753. """Read the running print's slicer pauses from its 3MF, in the background (#2984).
  3754. In a thread: a plate's G-code runs to tens of MB, and the print-start
  3755. handler must not wait for it. Kept only while the printer is still on
  3756. the print they were read for: a new print replaces the list.
  3757. ``warned_layers``: pauses this print was already warned about, kept on its
  3758. archive, so a restart does not send the heads-up again.
  3759. ``passed_through_layer``: for a print resumed after a restart, the layer it
  3760. is on. Its pauses up to there are behind it, though the printer's layer
  3761. reads 0 until it reports the next one.
  3762. """
  3763. client = printer_manager.get_client(printer_id)
  3764. if not client:
  3765. return
  3766. state = client.state
  3767. read_for = state.pause_points
  3768. try:
  3769. plate_id = resolve_plate_id(state)
  3770. except Exception:
  3771. # Never at the cost of the print-start handler this runs inside.
  3772. logging.getLogger(__name__).warning("[Printer %s] Could not tell the plate to read pauses for", printer_id)
  3773. return
  3774. async def _load() -> None:
  3775. from backend.app.utils.threemf_tools import extract_pause_points_from_3mf
  3776. points = await asyncio.to_thread(extract_pause_points_from_3mf, Path(threemf_path), plate_id)
  3777. if state.pause_points is not read_for:
  3778. return
  3779. state.pause_points = [p for p in points if p.layer > passed_through_layer]
  3780. state.pause_warned_layers = set(state.pause_warned_layers) | set(warned_layers)
  3781. if points:
  3782. logging.getLogger(__name__).info(
  3783. "Loaded %s slicer pause(s) for printer %s at layer(s) %s",
  3784. len(points),
  3785. printer_id,
  3786. [p.layer for p in points],
  3787. )
  3788. spawn_background_task(_load(), name=f"load-pauses-{printer_id}")
  3789. def _load_pauses_from_resumed_archive(archive, printer_id: int) -> None:
  3790. """Pauses for a print start taken as the resumption of a running print.
  3791. A restart while the print was paused makes its resume look like a new
  3792. print (#2984): the pauses warned about and passed are kept from before.
  3793. """
  3794. client = printer_manager.get_client(printer_id)
  3795. state = client.state if client else None
  3796. _load_pauses_from_3mf(
  3797. app_settings.base_dir / archive.file_path,
  3798. printer_id,
  3799. warned_layers=pause_eta.warned_layers_of(archive, state) if state is not None else (),
  3800. passed_through_layer=getattr(client, "layer_before_new_print", 0) if client else 0,
  3801. )
  3802. async def _restore_printable_objects(printer_id: int, state, db, logger) -> None:
  3803. """Put the skip-objects list back after a restart mid-print.
  3804. ``PrinterState.printable_objects`` is in-memory only, and the only thing
  3805. that fills it is ``_load_objects_from_archive`` on the print-start paths —
  3806. which the #1304 guard suppresses on the first RUNNING push after startup.
  3807. Everything else this hook restores (the archive, the usage-tracking session,
  3808. the timelapse baseline) was already handled; the object list was not, so a
  3809. restart mid-print took skip-objects away for the rest of that print.
  3810. Nothing recovered it either: the printer card gates its Skip button on the
  3811. object count, and the one endpoint that can rebuild the list is reachable
  3812. only from the modal that button opens.
  3813. The archive is the running print's as ``running_print_archive`` finds it:
  3814. by subtask_id, or for a print that reports none by the archive bound to it
  3815. (#3327), never by name: a leftover ``status="printing"`` row from a
  3816. completion we never saw must not hand this print someone else's objects.
  3817. After a restart that binding exists only for Bambuddy's own prints, so a
  3818. print that reports no subtask_id loads nothing rather than a guess; the
  3819. reload path on ``GET /print/objects`` covers that case on demand.
  3820. """
  3821. client = printer_manager.get_client(printer_id)
  3822. if client is None or (client.state.printable_objects and client.state.pause_points):
  3823. return
  3824. archive = await running_print_archive(db, printer_id, state, printing_only=True, by_name=False)
  3825. if archive is not None and archive.file_path:
  3826. if not client.state.printable_objects:
  3827. _load_objects_from_archive(archive, printer_id, logger)
  3828. # The slicer's pauses are in-memory only as well (#2984).
  3829. if not client.state.pause_points:
  3830. _load_pauses_from_3mf(
  3831. app_settings.base_dir / archive.file_path,
  3832. printer_id,
  3833. warned_layers=pause_eta.warned_layers_of(archive, state),
  3834. )
  3835. # Retry ladder for a fallback archive created while the printer's FTPS cool-off
  3836. # was running (#2957). The cool-off is 300s, so the first attempt is placed just
  3837. # past it; the second covers a handshake that failed again on the way back and
  3838. # armed a fresh one. Module-level so tests can shrink them.
  3839. _FALLBACK_3MF_RETRY_DELAYS_SECONDS: tuple[float, ...] = (310.0, 620.0)
  3840. # Retry ladder for the other temporary give-up: the file service answered and
  3841. # the transfer still did not finish, which at print start is usually the printer
  3842. # serving MQTT, the camera and a job upload at the same time (#3063). Nothing has
  3843. # to expire here, so the first attempt comes early -- #3063's reporter had the
  3844. # same 19MB file complete 48 seconds after the download budget ran out. The later
  3845. # two cover a printer that stays busy well into the print.
  3846. _FALLBACK_3MF_TRANSFER_RETRY_DELAYS_SECONDS: tuple[float, ...] = (60.0, 240.0, 600.0)
  3847. # printer_id -> the in-flight retry task, so print completion can cancel it.
  3848. _fallback_3mf_retry_tasks: dict[int, asyncio.Task] = {}
  3849. # printer_id -> lock serialising recovery attempts for that printer. Three callers
  3850. # can reach one archive at once: the cover endpoint (whose single-flight coalesces
  3851. # by view, so two views race), the cool-off retry task, and print completion.
  3852. # Without this they each read file_path == "" and each run a full copy, so the row
  3853. # ends up pointing at one timestamped directory while the others sit orphaned.
  3854. #
  3855. # Keyed by printer rather than archive because a printer runs one print at a time,
  3856. # which makes the two equally strong here — and it bounds the dict by printer
  3857. # count instead of needing a cleanup pass. Popping a per-archive entry cannot be
  3858. # done safely: `Lock.locked()` reads False between release and the queued waiter
  3859. # resuming, so "no waiters" is not a question this API can answer.
  3860. _fallback_recovery_locks: dict[int, asyncio.Lock] = {}
  3861. # How many of a printer's newest archives a touchscreen reprint is compared
  3862. # against (#3307). A repeat is usually of a recent job; the bound keeps a
  3863. # print start from reading a long history's extra_data.
  3864. _REPEAT_LOOKBACK = 200
  3865. async def _archived_3mf_for_repeat(
  3866. db, printer_id: int, subtask_name: str | None, gcode_file: str | None, total_layers: int | None
  3867. ):
  3868. """The newest archive of the job this printer is running again, with its 3MF.
  3869. A "Print again" on the printer's screen never names the file on the card:
  3870. it reports its own history copy under a generic name
  3871. (``file:///userdata/project_file.gcode.3mf``), the 3MF's internal title as
  3872. ``subtask_name`` and the plate as ``gcode_file`` -- and the title is often
  3873. not the card file's name, so every lookup built from it misses (#3307).
  3874. An earlier run that was archived recorded that same title and plate in its
  3875. ``_print_data``, and holds the 3MF.
  3876. Same printer, same title, same plate, a 3MF still on disk. A layer count
  3877. known on both sides must agree: the same title and plate re-sliced is
  3878. another job. None when nothing matches.
  3879. """
  3880. from backend.app.models.archive import PrintArchive
  3881. if not subtask_name or not gcode_file:
  3882. return None
  3883. rows = (
  3884. (
  3885. await db.execute(
  3886. select(PrintArchive)
  3887. .where(
  3888. PrintArchive.printer_id == printer_id,
  3889. PrintArchive.deleted_at.is_(None),
  3890. PrintArchive.file_path.isnot(None),
  3891. PrintArchive.file_path != "",
  3892. )
  3893. .order_by(PrintArchive.created_at.desc(), PrintArchive.id.desc())
  3894. .limit(_REPEAT_LOOKBACK)
  3895. )
  3896. )
  3897. .scalars()
  3898. .all()
  3899. )
  3900. for archive in rows:
  3901. extra = archive.extra_data if isinstance(archive.extra_data, dict) else {}
  3902. run = extra.get("_print_data")
  3903. if not isinstance(run, dict):
  3904. continue
  3905. if run.get("subtask_name") != subtask_name or run.get("filename") != gcode_file:
  3906. continue
  3907. if total_layers and archive.total_layers and archive.total_layers != total_layers:
  3908. continue
  3909. if not (app_settings.base_dir / archive.file_path).is_file():
  3910. continue
  3911. return archive
  3912. return None
  3913. async def _recover_fallback_archive(archive_id: int, source_3mf: Path, printer_id: int, *, price: bool = True) -> bool:
  3914. """Fill in a no-3MF archive from a 3MF that turned up later.
  3915. Returns True when the row was upgraded. Safe to call speculatively: it
  3916. verifies the archive still exists, is still a fallback, and that the file
  3917. is a readable 3MF before touching anything.
  3918. Serialised per printer — see ``_fallback_recovery_locks``.
  3919. """
  3920. lock = _fallback_recovery_locks.setdefault(printer_id, asyncio.Lock())
  3921. async with lock:
  3922. return await _recover_fallback_archive_locked(archive_id, source_3mf, printer_id, price=price)
  3923. async def _recover_fallback_archive_locked(
  3924. archive_id: int, source_3mf: Path, printer_id: int, *, price: bool = True
  3925. ) -> bool:
  3926. """The body of :func:`_recover_fallback_archive`, under its per-printer lock.
  3927. ``price`` is False when the print's end is being handled: the archive
  3928. still reads "printing" then, and its end prices it.
  3929. """
  3930. import zipfile
  3931. from backend.app.models.archive import PrintArchive
  3932. from backend.app.services.archive import ArchiveService
  3933. logger = logging.getLogger(__name__)
  3934. if not source_3mf.exists() or source_3mf.stat().st_size == 0:
  3935. return False
  3936. if not await asyncio.to_thread(zipfile.is_zipfile, source_3mf):
  3937. # A truncated or half-written download is worse than no download: it
  3938. # would replace an honest empty archive with wrong metadata.
  3939. logger.warning("[RECOVER] %s is not a readable 3MF; leaving archive %s as-is", source_3mf, archive_id)
  3940. return False
  3941. async with async_session() as db:
  3942. archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  3943. if archive is None or archive.deleted_at is not None:
  3944. return False
  3945. if archive.file_path:
  3946. # Already recovered, or never was a fallback. Either way there is a
  3947. # real 3MF attached and overwriting it is not this function's job.
  3948. return False
  3949. print_data = (archive.extra_data or {}).get("_print_data") or {}
  3950. service = ArchiveService(db)
  3951. recovered = await service.archive_print(
  3952. printer_id=printer_id,
  3953. source_file=source_3mf,
  3954. print_data={**print_data, "status": archive.status or "printing"},
  3955. subtask_id=archive.subtask_id,
  3956. update_archive_id=archive.id,
  3957. )
  3958. if recovered is None:
  3959. return False
  3960. logger.info(
  3961. "[RECOVER] Archive %s filled in from %s (%s bytes) — it started as a no-3MF fallback",
  3962. archive_id,
  3963. source_3mf,
  3964. recovered.file_size,
  3965. )
  3966. # Spoolman's tracking stored no 3MF usage at print start and would
  3967. # charge the AMS remain% change; the built-in tracker reads the archive
  3968. # at completion and needs nothing. Also at the print's end (``price``
  3969. # False): that recovery runs inside the close-out, before the charge.
  3970. try:
  3971. from backend.app.services.spoolman_tracking import attach_recovered_3mf
  3972. from backend.app.services.usage_tracker import is_completing
  3973. await attach_recovered_3mf(
  3974. printer_id,
  3975. recovered.id,
  3976. recovered.file_path,
  3977. db,
  3978. printer_manager,
  3979. plate_id=_get_start_plate_id(printer_id, recovered.id) or recovered.plate_id,
  3980. may_start=not price or (recovered.status == "printing" and not is_completing(printer_id)),
  3981. )
  3982. except Exception:
  3983. logger.warning(
  3984. "[RECOVER] Archive %s: could not add the 3MF to Spoolman's usage tracking", archive_id, exc_info=True
  3985. )
  3986. if price and recovered.status == "printing" and recovered.file_path:
  3987. # Print start had no 3MF to read the slicer's pauses from (#2984).
  3988. _load_pauses_from_3mf(app_settings.base_dir / recovered.file_path, printer_id)
  3989. if price and recovered.status == "printing":
  3990. # Print start priced it from the default rate, having no 3MF to
  3991. # estimate from; now it has one. An ended print is left alone: its
  3992. # end prices it. A print Bambuddy didn't dispatch has no start
  3993. # plate; the one the archive just read from the 3MF stands in.
  3994. _schedule_archive_cost_estimate(
  3995. printer_id,
  3996. recovered.id,
  3997. printer_manager,
  3998. ams_mapping=_get_start_ams_mapping(print_data, printer_id, recovered.id),
  3999. plate_id=_get_start_plate_id(printer_id, recovered.id) or recovered.plate_id,
  4000. session_factory=async_session,
  4001. )
  4002. # `archive_updated`, not `archive_created` — the row was already on the
  4003. # Archives page as an empty card and is now filled in, not new.
  4004. await ws_manager.send_archive_updated(
  4005. {
  4006. "id": recovered.id,
  4007. "printer_id": recovered.printer_id,
  4008. "filename": recovered.filename,
  4009. "print_name": recovered.print_name,
  4010. "status": recovered.status,
  4011. }
  4012. )
  4013. return True
  4014. async def try_recover_fallback_archive(printer_id: int, name: str, path: Path, *, price: bool = True) -> bool:
  4015. """Offer a freshly-downloaded 3MF to this printer's running fallback archive.
  4016. Called from the paths that pull a 3MF for a print that is already under way
  4017. — chiefly the cover endpoint, which downloads the very file the archive flow
  4018. could not get and, before #2957, used it for a thumbnail and nothing else.
  4019. The bytes are already local, so this costs a parse and a row update.
  4020. No-op when the running print has a real archive, which is the common case.
  4021. """
  4022. from backend.app.models.archive import PrintArchive
  4023. logger = logging.getLogger(__name__)
  4024. # `_active_prints` is keyed on the raw names seen at print start — the
  4025. # dispatch filename, the subtask name, and the subtask name plus ".3mf".
  4026. # Callers here arrive with whichever variant their own path produced, so
  4027. # match on the same normalization the download cache uses rather than on an
  4028. # exact string; that is what makes "Desktop_Goose.gcode.3mf" from the cover
  4029. # endpoint find an archive registered under "Desktop_Goose".
  4030. wanted = normalize_3mf_name(name)
  4031. archive_id = None
  4032. for (key_printer_id, key_name), value in list(_active_prints.items()):
  4033. if key_printer_id == printer_id and normalize_3mf_name(key_name) == wanted:
  4034. archive_id = value
  4035. break
  4036. if archive_id is None:
  4037. return False
  4038. async with async_session() as db:
  4039. archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  4040. # Cheap pre-check so the common case (a normal archive) does no work.
  4041. if archive is None or archive.file_path or archive.deleted_at is not None:
  4042. return False
  4043. try:
  4044. return await _recover_fallback_archive(archive_id, path, printer_id, price=price)
  4045. except Exception as e:
  4046. # Recovery is opportunistic. A failure here must never take down the
  4047. # caller, which is usually just trying to render a thumbnail.
  4048. logger.warning("[RECOVER] Could not fill in archive %s from %s: %s", archive_id, path, e)
  4049. return False
  4050. def _schedule_fallback_3mf_retry(
  4051. printer_id: int,
  4052. archive_id: int,
  4053. filenames: list[str],
  4054. delays: tuple[float, ...] | None = None,
  4055. reason: str = REASON_FTPS_COOLOFF,
  4056. ) -> None:
  4057. """Re-attempt the 3MF download after a temporary give-up.
  4058. ``reason`` says which give-up this is, and picks the default ladder: an
  4059. FTPS cool-off has to be waited out, while a transfer that timed out under
  4060. contention is worth asking about again straight away (#3063). It is only
  4061. read for the ladder and the log line -- the retry itself is identical, since
  4062. in both cases the file is on the printer and the last attempt at it failed
  4063. for a reason that does not last.
  4064. """
  4065. logger = logging.getLogger(__name__)
  4066. if delays is None:
  4067. delays = (
  4068. _FALLBACK_3MF_TRANSFER_RETRY_DELAYS_SECONDS
  4069. if reason == REASON_FTP_TRANSFER_FAILED
  4070. else _FALLBACK_3MF_RETRY_DELAYS_SECONDS
  4071. )
  4072. async def _retry() -> None:
  4073. from backend.app.models.archive import PrintArchive
  4074. from backend.app.models.printer import Printer
  4075. for delay in delays:
  4076. await asyncio.sleep(delay)
  4077. async with async_session() as db:
  4078. archive = (
  4079. await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  4080. ).scalar_one_or_none()
  4081. if archive is None or archive.deleted_at is not None or archive.file_path:
  4082. return
  4083. printer = (await db.execute(select(Printer).where(Printer.id == printer_id))).scalar_one_or_none()
  4084. if printer is None:
  4085. return
  4086. # Read the fields while the session is open rather than touching
  4087. # a detached instance minutes later, mid-download.
  4088. printer_ip = printer.ip_address
  4089. printer_code = printer.access_code
  4090. printer_model = printer.model
  4091. # Someone else may have fetched it in the meantime — the cover
  4092. # endpoint routinely does, and its copy is the same bytes.
  4093. for name in filenames:
  4094. cached = get_cached_3mf(printer_id, name)
  4095. if cached and await _recover_fallback_archive(archive_id, cached, printer_id):
  4096. return
  4097. if ftps_handshake_blocked(printer_ip):
  4098. logger.info(
  4099. "[RECOVER] Printer %s is still in its FTPS cool-off; archive %s retry deferred",
  4100. printer_id,
  4101. archive_id,
  4102. )
  4103. continue
  4104. _, _, _, ftp_timeout = await get_ftp_retry_settings()
  4105. for candidate in filenames:
  4106. # Bare name only. These come from the print-start flow, which
  4107. # already strips the path, but the local temp write must not
  4108. # depend on that holding for every future caller — a name that
  4109. # is absolute or contains ".." would otherwise escape the data
  4110. # volume via the `/` operator.
  4111. name = Path(candidate).name
  4112. if not name or name in (".", ".."):
  4113. continue
  4114. if not name.endswith(".3mf"):
  4115. name = f"{name}.3mf"
  4116. temp_path = app_settings.archive_dir / "temp" / name
  4117. temp_path.parent.mkdir(parents=True, exist_ok=True)
  4118. try:
  4119. hit = await download_file_try_paths_async(
  4120. printer_ip,
  4121. printer_code,
  4122. ftp_probe_paths(name),
  4123. temp_path,
  4124. socket_timeout=ftp_timeout,
  4125. printer_model=printer_model,
  4126. max_total_timeout=RECOVERY_DOWNLOAD_MAX_TIMEOUT,
  4127. )
  4128. except Exception as e:
  4129. logger.debug("[RECOVER] Retry download of %s failed: %s", name, e)
  4130. continue
  4131. if not hit:
  4132. continue
  4133. cache_3mf_download(printer_id, name, temp_path)
  4134. if await _recover_fallback_archive(archive_id, temp_path, printer_id):
  4135. return
  4136. logger.info("[RECOVER] Archive %s still has no 3MF after a retry", archive_id)
  4137. async def _guarded() -> None:
  4138. try:
  4139. await _retry()
  4140. except asyncio.CancelledError:
  4141. raise
  4142. except Exception as e:
  4143. logger.warning("[RECOVER] Retry task for archive %s failed: %s", archive_id, e)
  4144. finally:
  4145. if _fallback_3mf_retry_tasks.get(printer_id) is asyncio.current_task():
  4146. _fallback_3mf_retry_tasks.pop(printer_id, None)
  4147. existing = _fallback_3mf_retry_tasks.pop(printer_id, None)
  4148. if existing and not existing.done():
  4149. existing.cancel()
  4150. task = asyncio.create_task(_guarded())
  4151. _fallback_3mf_retry_tasks[printer_id] = task
  4152. logger.info(
  4153. "[RECOVER] Archive %s has no 3MF (%s) and the file should still be on printer %s; will retry in %s",
  4154. archive_id,
  4155. reason,
  4156. printer_id,
  4157. ", ".join(f"{d:g}s" for d in delays),
  4158. )
  4159. async def _ask_outcome_for_external_print(db, printer_id: int, observed_name: str | None = None) -> bool:
  4160. """Whether an archive created here should ask for the print's outcome (#1898).
  4161. Only a print Bambuddy did not dispatch reaches the archive-*creating*
  4162. branches below: a queued job already has its archive and takes the
  4163. expected-print branch, where the queue item's own ``confirm_outcome``
  4164. decides. The queue is still consulted, because a restart mid-print empties
  4165. ``_expected_prints`` — without the check the setting could override a queue
  4166. item that deliberately has the flag off. Any failure answers "don't ask":
  4167. an unwanted prompt is worse than a missing one, and this must never be the
  4168. reason a print goes unarchived.
  4169. A ``printing`` row is not proof on its own, though: Bambuddy deliberately
  4170. leaves one behind when a completion cannot be matched to it
  4171. (``_completion_belongs_to_queue_item``), and the scheduler's stranded sweep
  4172. only takes it back once the printer has sat connected and terminal for
  4173. minutes. Until then every screen-started print on that printer would
  4174. silently lose its prompt, so the row is held against ``observed_name`` by
  4175. the same comparison a completion uses: a positive disagreement means the row
  4176. is about some other run and this print is external after all.
  4177. """
  4178. logger = logging.getLogger(__name__)
  4179. try:
  4180. # A savepoint rather than a rollback of the caller's transaction on
  4181. # failure: a failed statement leaves the transaction unusable, but a
  4182. # full rollback also expires every object the caller has loaded (the
  4183. # printer, the archive it just created), and on an async session the
  4184. # next attribute read then raises instead of reloading. Everything here
  4185. # is a read, so undoing the savepoint loses nothing.
  4186. async with db.begin_nested():
  4187. return await _decide_outcome_for_external_print(db, printer_id, observed_name, logger)
  4188. except Exception as e:
  4189. logger.warning("[CALLBACK] Could not decide the outcome prompt for printer %s: %s", printer_id, e)
  4190. return False
  4191. async def _decide_outcome_for_external_print(db, printer_id: int, observed_name: str | None, logger) -> bool:
  4192. """The reads behind ``_ask_outcome_for_external_print``; see there."""
  4193. from backend.app.api.routes.settings import get_setting, setting_is_true
  4194. if not setting_is_true(await get_setting(db, "confirm_outcome_external_prints")):
  4195. return False
  4196. from backend.app.models.print_queue import PrintQueueItem
  4197. dispatched_here = await db.scalar(
  4198. select(PrintQueueItem)
  4199. .where(
  4200. PrintQueueItem.printer_id == printer_id,
  4201. PrintQueueItem.status == "printing",
  4202. )
  4203. .limit(1)
  4204. )
  4205. if dispatched_here is None:
  4206. return True
  4207. expected = await _queue_item_dispatched_name(db, dispatched_here)
  4208. observed = (observed_name or "").strip()
  4209. if not expected or not observed or _dispatched_name_matches_echo(expected, observed):
  4210. logger.info(
  4211. "[CALLBACK] Not asking for the outcome on printer %s: queue item %s is still printing, so "
  4212. "Bambuddy dispatched this run and the item's own ask-for-outcome flag decides.",
  4213. printer_id,
  4214. dispatched_here.id,
  4215. )
  4216. return False
  4217. logger.info(
  4218. "[CALLBACK] Queue item %s is still marked printing on printer %s but was dispatched as %r, not "
  4219. "%r; treating this as an externally started print.",
  4220. dispatched_here.id,
  4221. printer_id,
  4222. expected,
  4223. observed,
  4224. )
  4225. return True
  4226. async def _dispatch_outcome_confirmation_safely(
  4227. db,
  4228. printer_id: int,
  4229. printer_name: str,
  4230. data: dict,
  4231. archive_id: int,
  4232. archive_data: dict | None = None,
  4233. run_id: int | None = None,
  4234. ) -> None:
  4235. """``dispatch_outcome_confirmation`` for a caller that has more to do on ``db``.
  4236. The completion task sends the per-user print email on the same session
  4237. right after the prompt. A failed statement in the prompt leaves that
  4238. session needing a rollback, and without one the email step fails with
  4239. PendingRollbackError; the prompt is the optional part, so it must not be
  4240. the reason the email never goes out.
  4241. """
  4242. try:
  4243. await dispatch_outcome_confirmation(db, printer_id, printer_name, data, archive_id, archive_data, run_id=run_id)
  4244. except Exception as e:
  4245. logging.getLogger(__name__).error("[NOTIFY-BG] Outcome-confirmation dispatch failed: %s", e, exc_info=True)
  4246. try:
  4247. await db.rollback()
  4248. except Exception:
  4249. pass
  4250. async def _outcome_answerers(db, owner_id: int | None) -> set[int] | None:
  4251. """Who may answer a print's outcome prompt in the app: its owner and anyone
  4252. who may change everyone's archives. None (everyone on the printer) with auth
  4253. off, or for a print nobody owns."""
  4254. from sqlalchemy.orm import selectinload
  4255. from backend.app.core.auth import is_auth_enabled
  4256. from backend.app.core.permissions import Permission
  4257. from backend.app.models.user import User
  4258. if owner_id is None or not await is_auth_enabled(db):
  4259. return None
  4260. users = (await db.execute(select(User).options(selectinload(User.groups)).where(User.is_active))).scalars().all()
  4261. return {owner_id} | {u.id for u in users if u.has_permission(Permission.ARCHIVES_UPDATE_ALL.value)}
  4262. async def dispatch_outcome_confirmation(
  4263. db,
  4264. printer_id: int,
  4265. printer_name: str,
  4266. data: dict,
  4267. archive_id: int,
  4268. archive_data: dict | None = None,
  4269. *,
  4270. run_id: int | None = None,
  4271. ) -> bool:
  4272. """Emit the post-print outcome prompt for a completed print (#1898).
  4273. Gated on the run that ended (``run_id``): ``confirm_requested`` is the
  4274. opt-in (from the queue item, or from ``confirm_outcome_external_prints``
  4275. for a print Bambuddy did not start, copied onto the run at its start) and
  4276. ``user_verdict`` being unset is what makes the question still open. Mints
  4277. the run's capability token the one-tap verdict links carry: plates of one
  4278. archive each have their own question. Without a run, the archive's, as
  4279. before runs.
  4280. Lives here rather than inline in ``on_print_complete``'s notification task
  4281. because that task swallows every exception: extracted, the gate and the two
  4282. emissions can be driven by a test, which is the only thing standing between
  4283. a regression here and a farm that quietly stops asking.
  4284. Returns whether a prompt was sent.
  4285. """
  4286. logger = logging.getLogger(__name__)
  4287. from backend.app.models.archive import PrintArchive
  4288. from backend.app.models.print_log import PrintLogEntry
  4289. confirm_archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  4290. if confirm_archive is None:
  4291. return False
  4292. run = await db.get(PrintLogEntry, run_id) if run_id is not None else None
  4293. if run is not None and run.archive_id != archive_id:
  4294. run = None
  4295. # The run carries the opt-in from its start; one logged before it did
  4296. # (None) asks as its archive does.
  4297. asks = confirm_archive if run is None or run.confirm_requested is None else run
  4298. holder = run if run is not None else confirm_archive
  4299. if not (asks.confirm_requested and holder.user_verdict is None):
  4300. return False
  4301. import secrets as _secrets
  4302. # A spent token stays on the row so the one-tap route can recognise it, so
  4303. # "has a token" no longer means "answerable". A prompt going out now needs a
  4304. # live one: mint a new token whenever the stored one is already spent, or
  4305. # every button in the new message would land on the already-answered page.
  4306. if not holder.confirm_token or holder.confirm_token_used_at is not None:
  4307. holder.confirm_token = _secrets.token_urlsafe(32)
  4308. holder.confirm_token_used_at = None
  4309. await db.commit()
  4310. from backend.app.api.routes.settings import get_external_base_url, get_setting
  4311. base = await get_external_base_url(db)
  4312. if not await get_setting(db, "external_url"):
  4313. # Both the Telegram inline keyboard and the ntfy action buttons need an
  4314. # absolute URL, so an unconfigured install used to get a message body
  4315. # with two unusable relative paths and no buttons at all. The shared
  4316. # fallback at least produces tappable links; say which setting makes
  4317. # them resolve from a phone.
  4318. logger.warning(
  4319. "[#1898] No external_url configured — the outcome prompt's Good/Reject links point at %s. "
  4320. "Set Settings → External URL so they resolve away from this host.",
  4321. base,
  4322. )
  4323. token = holder.confirm_token
  4324. good_url = f"{base}/api/v1/archives/confirm/{token}/good"
  4325. reject_url = f"{base}/api/v1/archives/confirm/{token}/reject"
  4326. confirm_url = f"{base}/archives?confirm={archive_id}" + (f"&run={run.id}" if run is not None else "")
  4327. await ws_manager.send_print_confirm_request(
  4328. printer_id,
  4329. {
  4330. "archive_id": archive_id,
  4331. "run_id": run.id if run is not None else None,
  4332. "print_name": confirm_archive.print_name or confirm_archive.filename,
  4333. },
  4334. user_ids=await _outcome_answerers(db, confirm_archive.created_by_id),
  4335. )
  4336. await notification_service.on_print_confirm_request(
  4337. printer_id,
  4338. printer_name,
  4339. data,
  4340. db,
  4341. archive_data=archive_data,
  4342. good_url=good_url,
  4343. reject_url=reject_url,
  4344. confirm_url=confirm_url,
  4345. archive_id=archive_id,
  4346. run_id=run.id if run is not None else None,
  4347. )
  4348. return True
  4349. async def _printer_gone(db, printer_id: int) -> bool:
  4350. """Whether the printer was deleted. A column read: the session's loaded
  4351. Printer would otherwise answer for the database."""
  4352. from backend.app.models.printer import Printer
  4353. return await db.scalar(select(Printer.id).where(Printer.id == printer_id)) is None
  4354. async def _drop_archive_of_deleted_printer(db, printer_id: int, archive_id: int) -> bool:
  4355. """Remove the archive just written if its printer was deleted meanwhile.
  4356. The delete route sweeps the printer's archives before it commits, so a row
  4357. committed after that sweep (the check before writing passed while the
  4358. delete was still uncommitted) is left for a printer that will never report
  4359. the print's end. A row the sweep reached is the route's: deleted with the
  4360. printer's archives, or detached (printer_id NULL) when the user kept them,
  4361. and then left as it is. Either way nothing is registered for it. Run after
  4362. the archive's commit, before anything is registered for it.
  4363. """
  4364. from backend.app.models.archive import PrintArchive
  4365. if not await _printer_gone(db, printer_id):
  4366. return False
  4367. owner = await db.scalar(select(PrintArchive.printer_id).where(PrintArchive.id == archive_id))
  4368. if owner == printer_id:
  4369. logging.getLogger(__name__).info(
  4370. "Printer %s was deleted while archive %s was written; removing it", printer_id, archive_id
  4371. )
  4372. await ArchiveService(db).delete_archive(archive_id)
  4373. await forget_printer(printer_id)
  4374. return True
  4375. async def _refused_for_deleted_printer(db, printer_id: int, what: str) -> bool:
  4376. """After an IntegrityError on a print start's archive write: whether it was
  4377. the printer being deleted, forgotten then (the caller returns).
  4378. PostgreSQL: the delete route locks the printer from its archive sweep to its
  4379. commit, so an archive written or adopted for it meanwhile (one that sets
  4380. printer_id) waits and the printer's foreign key then refuses it. That is
  4381. the printer being deleted, not an error; anything else is re-raised by the
  4382. caller.
  4383. """
  4384. await db.rollback()
  4385. if not await _printer_gone(db, printer_id):
  4386. return False
  4387. logging.getLogger(__name__).info("Printer %s was deleted while %s; refused", printer_id, what)
  4388. await forget_printer(printer_id)
  4389. return True
  4390. async def _forget_if_printer_gone(printer_id: int) -> None:
  4391. """At the end of a print start: the slow steps after the last check (smart
  4392. plug, notifications, the timelapse baseline's FTP listing) can outlast the
  4393. printer's delete and its forget_printer, and then wrote state back.
  4394. (An ActivePrintSpoolman row written then is the init_db orphan sweep's on
  4395. SQLite; PostgreSQL refuses it.)
  4396. Its own session: those steps can leave the handler's needing a rollback (a
  4397. write to the archive the delete removed). Never raises: it runs inside the
  4398. fallback branch's try, whose except reports the archive as failed.
  4399. """
  4400. try:
  4401. async with async_session() as db:
  4402. gone = await _printer_gone(db, printer_id)
  4403. if gone:
  4404. await forget_printer(printer_id)
  4405. except Exception:
  4406. logging.getLogger(__name__).warning(
  4407. "Could not check whether printer %s was deleted during its print start", printer_id, exc_info=True
  4408. )
  4409. async def _announce_plate_check_failed(printer_id: int, printer_name: str, reason: str) -> None:
  4410. """Tell the UI a plate check could not run. It does not block the print,
  4411. but an unchecked plate is not an empty one."""
  4412. try:
  4413. await ws_manager.broadcast(
  4414. {"type": "plate_check_failed", "printer_id": printer_id, "printer_name": printer_name, "reason": reason}
  4415. )
  4416. except Exception as e:
  4417. logging.getLogger(__name__).warning("[PLATE CHECK] Could not announce the failed check: %s", e)
  4418. def _expected_print_keys(printer_id: int, filename: str | None, subtask_name: str | None) -> list[tuple[int, str]]:
  4419. """The ``_expected_prints`` keys a print start may be registered under."""
  4420. keys: list[tuple[int, str]] = []
  4421. if subtask_name:
  4422. keys.append((printer_id, subtask_name))
  4423. keys.append((printer_id, f"{subtask_name}.3mf"))
  4424. keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  4425. if filename:
  4426. fname = filename.split("/")[-1] if "/" in filename else filename
  4427. keys.append((printer_id, fname))
  4428. # Strip extensions to match
  4429. base = fname.replace(".gcode", "").replace(".3mf", "")
  4430. keys.append((printer_id, base))
  4431. keys.append((printer_id, f"{base}.3mf"))
  4432. return keys
  4433. async def _dispatched_by_bambuddy(db, printer_id: int, filename: str | None, subtask_name: str | None) -> bool:
  4434. """Whether this print start is one Bambuddy sent to the printer."""
  4435. if any(key in _expected_prints for key in _expected_print_keys(printer_id, filename, subtask_name)):
  4436. return True
  4437. from backend.app.models.archive import PrintArchive
  4438. from backend.app.models.print_queue import PrintQueueItem
  4439. # The queue row the scheduler sets to printing before it sends -- but only
  4440. # when it was sent as this print: a row left printing by a run whose end
  4441. # was missed names another file.
  4442. dispatched_names = (
  4443. await db.execute(
  4444. select(PrintArchive.filename)
  4445. .join(PrintQueueItem, PrintQueueItem.archive_id == PrintArchive.id)
  4446. .where(PrintQueueItem.printer_id == printer_id, PrintQueueItem.status == "printing")
  4447. )
  4448. ).scalars()
  4449. observed = [name for name in (subtask_name, (filename or "").rsplit("/", 1)[-1]) if name]
  4450. return any(
  4451. _subtask_names_match(_subtask_name_from_filename(dispatched), _subtask_name_from_filename(seen))
  4452. for dispatched in dispatched_names
  4453. if dispatched
  4454. for seen in observed
  4455. )
  4456. async def _adopt_archive_for_run(
  4457. db: AsyncSession,
  4458. archive,
  4459. printer_id: int,
  4460. *,
  4461. starting_plate_id: int | None,
  4462. effective_subtask_id: str | None,
  4463. logger: logging.Logger,
  4464. started_at: datetime | None = None,
  4465. ) -> bool:
  4466. """Start a print's run on an archive that already exists: a reprint, a
  4467. queue print of a file printed before, or the same slice sent again from the
  4468. slicer (#1634). The archive says "printing" from now, without the previous
  4469. run's outcome, answer or timelapse, and on this printer with this print's
  4470. job id. False when the printer was deleted meanwhile: the adoption is
  4471. undone or was refused, the printer is forgotten, and the caller stops.
  4472. """
  4473. from backend.app.models.archive import PrintArchive
  4474. # As it was before this adoption: put back if the printer turns
  4475. # out to have been deleted while it was adopted (below).
  4476. before_adoption = {
  4477. attr.columns[0].name: getattr(archive, attr.key)
  4478. for attr in sa_inspect(PrintArchive).column_attrs
  4479. if attr.key != "id"
  4480. }
  4481. # Update archive status to printing
  4482. archive.status = "printing"
  4483. archive.started_at = started_at or datetime.now(timezone.utc)
  4484. # The previous run's outcome is not this one's: a reprint that
  4485. # completes must not keep showing why the last one failed.
  4486. archive.failure_reason = None
  4487. archive.completed_at = None
  4488. # The previous run's answer is still on this row and the
  4489. # completion prompt is gated on ``user_verdict is None`` (#1898),
  4490. # so without a reset the second run inherits the first run's
  4491. # verdict: no prompt at all, and a green "good" badge on a run
  4492. # nobody ever judged.
  4493. if archive.confirm_requested:
  4494. archive.user_verdict = None
  4495. archive.user_verdict_source = None
  4496. archive.user_verdict_at = None
  4497. archive.confirm_token = None
  4498. archive.confirm_token_used_at = None
  4499. # Reprint of an archive reuses the source row. Without resetting
  4500. # ``timelapse_path`` _scan_for_timelapse_with_retries early-returns
  4501. # ("already has timelapse") and _capture_finish_photo_from_timelapse
  4502. # extracts the *original* print's last frame, which then ships in
  4503. # the completion notification (#1707). Clear the path so the
  4504. # scanner runs fresh; also unlink the old video file so reprints
  4505. # don't accumulate orphans in the archive directory. Photos list
  4506. # is left alone — accumulating one finish photo per run is fine.
  4507. # The print-start baseline (#2704) is stale for the same reason:
  4508. # it describes the printer before the previous run. The capture
  4509. # below overwrites it, but clear it here too so an early failure
  4510. # can't leave the scan diffing against the wrong snapshot.
  4511. #
  4512. # Unlinked only when it is the same plate printing again. Every
  4513. # plate of a Send All shares this archive, and the video on it
  4514. # can be the only copy of another plate's run -- the printer's
  4515. # was deleted once it attached (#3275). That one is left in the
  4516. # archive directory, which goes with the archive. A video whose
  4517. # plate is not on record (attached by hand, or before the
  4518. # column existed) is kept too where the 3MF has other plates,
  4519. # or cannot be read: nothing tells it apart from another plate's,
  4520. # and keeping costs one file where deleting can cost the only
  4521. # copy. On an archive with exactly one plate it is that plate's.
  4522. archive.timelapse_baseline = None
  4523. stale_timelapse_relpath = archive.timelapse_path
  4524. stale_timelapse_plate_id = archive.timelapse_plate_id
  4525. if stale_timelapse_relpath:
  4526. archive.timelapse_path = None
  4527. archive.timelapse_plate_id = None
  4528. stale_is_this_plates = stale_timelapse_plate_id == starting_plate_id
  4529. if stale_timelapse_relpath and stale_timelapse_plate_id is None and archive.file_path:
  4530. from backend.app.services.archive import plate_indexes_in_3mf
  4531. plates = await asyncio.to_thread(
  4532. plate_indexes_in_3mf,
  4533. app_settings.base_dir / archive.file_path, # SEC-PATH-OK: stored relative archive path
  4534. )
  4535. stale_is_this_plates = len(plates) == 1
  4536. if stale_timelapse_relpath and not stale_is_this_plates:
  4537. logger.info(
  4538. "Kept timelapse %s of plate %s on archive %s: plate %s is starting",
  4539. stale_timelapse_relpath,
  4540. stale_timelapse_plate_id,
  4541. archive.id,
  4542. starting_plate_id,
  4543. )
  4544. # The video is deleted once the adoption stands (below).
  4545. delete_stale_video = bool(stale_timelapse_relpath) and stale_is_this_plates
  4546. # Update on first-set OR on reprint (the queue dispatcher mints
  4547. # a fresh subtask_id per dispatch in bambu_mqtt:3647). Skipping
  4548. # the rewrite for reprints leaves the archive holding the FIRST
  4549. # run's id; if MQTT then reconnects mid-print, the reconciler
  4550. # (#1542) compares the stale stored id against the printer's
  4551. # live id, sees a mismatch, and synthesises a bogus PRINT
  4552. # COMPLETE — exactly the false-positive "Print Stopped" reported
  4553. # in #1807. Inequality check preserves the noop-on-stable-push
  4554. # behaviour the earlier `not archive.subtask_id` guard provided.
  4555. if effective_subtask_id and archive.subtask_id != effective_subtask_id:
  4556. archive.subtask_id = effective_subtask_id
  4557. # #1403 follow-up: VP-queue archives are created with
  4558. # printer_id=None at queue-add time (we don't know which
  4559. # printer will run the job yet). When the print actually
  4560. # starts on a specific printer the expected-archive lookup
  4561. # used to skip this assignment, leaving printer_id=None
  4562. # forever — which then disables the "Scan for timelapse"
  4563. # button in ArchivesPage (gated on !archive.printer_id).
  4564. if archive.printer_id != printer_id:
  4565. archive.printer_id = printer_id
  4566. adopting_id = archive.id # a rollback expires the object
  4567. try:
  4568. await db.commit()
  4569. except (IntegrityError, StaleDataError):
  4570. # The archive stays unchanged, or is gone: "delete archives"
  4571. # removed the row under the adoption, which then matched no
  4572. # row (StaleDataError) rather than being refused.
  4573. if await _refused_for_deleted_printer(db, printer_id, f"archive {adopting_id} was adopted"):
  4574. return False
  4575. raise
  4576. # The printer can be deleted between the check above and this
  4577. # commit (the route's transaction lasts seconds), and the commit
  4578. # then adopted the archive anyway: status, verdict and timelapse
  4579. # changed, and the deleted printer's id written on a row the
  4580. # route had detached or never touched. Undo the adoption.
  4581. if await _printer_gone(db, printer_id):
  4582. logger.info(
  4583. "Printer %s was deleted while archive %s was adopted; undoing the adoption",
  4584. printer_id,
  4585. archive.id,
  4586. )
  4587. # Everything as before this adoption. The printer it had
  4588. # then stays, unless it was the deleted one: the route
  4589. # detached that row (kept) or deleted it (no row matches).
  4590. previous_printer = before_adoption["printer_id"]
  4591. await db.execute(
  4592. update(PrintArchive)
  4593. .where(PrintArchive.id == archive.id)
  4594. .values(
  4595. {
  4596. **before_adoption,
  4597. "printer_id": None if previous_printer == printer_id else previous_printer,
  4598. }
  4599. )
  4600. )
  4601. await db.commit()
  4602. await forget_printer(printer_id)
  4603. return False
  4604. await _drop_replaced_plate_videos(
  4605. db,
  4606. archive.id,
  4607. starting_plate_id,
  4608. stale_timelapse_relpath if delete_stale_video else None,
  4609. logger,
  4610. )
  4611. return True
  4612. async def _claim_slicer_resend_archive(db: AsyncSession, printer_id: int, source: Path):
  4613. """The archive a print sent from the slicer adds its run to, or None for a
  4614. new one (#1634): the same G-code, so the same slice sent again in a new 3MF.
  4615. As a queue print of a file printed before (``_claim_reusable_archive`` in
  4616. the scheduler): the same owner (who started the print, nobody for a print
  4617. from the slicer) and no project, and no File Manager file, whose own prints
  4618. find its archive by the file's bytes. Not one a merge is moving. Held in
  4619. ``archive_work`` from here, with no await between, until the caller
  4620. releases it once the run is open.
  4621. Returns the archive and whether something else holds it already: a queue
  4622. dispatch uploading or printing its file, which the file must stay for.
  4623. """
  4624. from backend.app.models.archive import PrintArchive
  4625. from backend.app.services import archive_work
  4626. from backend.app.services.archive_gcode_hash import gcode_fingerprint
  4627. # Unzipping a large 3MF takes seconds: off the event loop.
  4628. fingerprint = await asyncio.to_thread(gcode_fingerprint, source)
  4629. if not fingerprint:
  4630. return None
  4631. owner_id = await _print_user_id_if_still_there(db, printer_manager.get_current_print_user(printer_id))
  4632. owner = PrintArchive.created_by_id.is_(None) if owner_id is None else PrintArchive.created_by_id == owner_id
  4633. candidates = (
  4634. await db.scalars(
  4635. select(PrintArchive)
  4636. .where(
  4637. PrintArchive.gcode_hash == fingerprint,
  4638. owner,
  4639. PrintArchive.project_id.is_(None),
  4640. PrintArchive.library_file_id.is_(None),
  4641. PrintArchive.deleted_at.is_(None),
  4642. )
  4643. .order_by(PrintArchive.id.desc())
  4644. )
  4645. ).all()
  4646. for candidate in candidates:
  4647. if archive_work.is_merging(candidate.id):
  4648. continue
  4649. in_use = archive_work.is_held(candidate.id)
  4650. archive_work.hold(candidate.id)
  4651. return candidate, in_use
  4652. return None
  4653. async def _start_slicer_resend_run(
  4654. db: AsyncSession,
  4655. archive,
  4656. printer_id: int,
  4657. *,
  4658. source: Path,
  4659. print_data: dict,
  4660. subtask_id: str | None,
  4661. started_at: datetime | None,
  4662. original_filename: str | None,
  4663. logger: logging.Logger,
  4664. file_in_use: bool = False,
  4665. ):
  4666. """Start this print's run on the archive ``_claim_slicer_resend_archive``
  4667. found (#1634). The archive takes the new file in place of its own, unless
  4668. another print of it is running, or a queue dispatch holds it
  4669. (``file_in_use``): that one still reads the file (its upload, pauses,
  4670. objects and estimate), and the G-code is the same. None when the printer
  4671. was deleted meanwhile; the caller stops.
  4672. """
  4673. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, RUN_OPEN, PrintLogEntry
  4674. printing_elsewhere = file_in_use or (
  4675. await db.scalar(
  4676. select(PrintLogEntry.id)
  4677. .where(PrintLogEntry.archive_id == archive.id, PrintLogEntry.status == RUN_OPEN)
  4678. .limit(1)
  4679. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  4680. )
  4681. is not None
  4682. )
  4683. if not printing_elsewhere:
  4684. replaced = await ArchiveService(db).archive_print(
  4685. printer_id=printer_id,
  4686. source_file=source,
  4687. print_data=print_data,
  4688. subtask_id=subtask_id,
  4689. original_filename=original_filename,
  4690. replace_archive_id=archive.id,
  4691. )
  4692. if replaced is not None:
  4693. archive = replaced
  4694. else:
  4695. # The copy failed its check, or the printer is gone (the adoption
  4696. # below says so): the run still joins, on the file it has.
  4697. logger.warning("Archive %s keeps its file: the one sent again could not be taken", archive.id)
  4698. client = printer_manager.get_client(printer_id)
  4699. if not await _adopt_archive_for_run(
  4700. db,
  4701. archive,
  4702. printer_id,
  4703. starting_plate_id=resolve_plate_id(client.state) if client else None,
  4704. effective_subtask_id=subtask_id,
  4705. logger=logger,
  4706. started_at=started_at,
  4707. ):
  4708. return None
  4709. return archive
  4710. async def on_print_start(printer_id: int, data: dict):
  4711. """Handle print start - archive the 3MF file immediately."""
  4712. logger = logging.getLogger(__name__)
  4713. # The print's start, before the 3MF download that the archive waits for
  4714. # (#3327): a minute of it would otherwise be missing from the print time.
  4715. detected_at = datetime.now(timezone.utc)
  4716. logger.info("[CALLBACK] on_print_start called for printer %s, data keys: %s", printer_id, list(data.keys()))
  4717. octoeverywhere_detection_service.reset_printer(printer_id)
  4718. try:
  4719. notify_live_activities.print_started(printer_id, printer_manager.get_status(printer_id), data)
  4720. except Exception:
  4721. logger.exception("Notify Live Activity start hook failed for printer %s", printer_id)
  4722. # Clear any stale user-stopped flag from previous print cycles
  4723. _user_stopped_printers.discard(printer_id)
  4724. _kill_switch_stopped.discard(printer_id)
  4725. # A new print starts its milestones from zero (#3211). The status path only
  4726. # resets on progress below 5 while not printing, which a printer that goes
  4727. # from FINISH at 100% straight into a new print at a preparation-phase 85%
  4728. # never shows. This callback does not fire after a Bambuddy restart (#1304)
  4729. # or on resume from pause, so it cannot repeat a milestone mid-print.
  4730. _last_progress_milestone[printer_id] = 0
  4731. _kill_switch_notification_tasks.pop(printer_id, None)
  4732. # #1721: drop any leftover pre-captured finish frame from a prior print
  4733. # so a never-consumed cache entry can't bleed into the new print's photo.
  4734. _stage22_finish_frames.pop(printer_id, None)
  4735. # #1867: same for the in-print frame bank — a queued print must not reuse
  4736. # the previous job's banked frame.
  4737. _inprint_frame_bank.pop(printer_id, None)
  4738. _inprint_frame_bank_ts.pop(printer_id, None)
  4739. # #2547: bind (or clear) the "this print ends with injected End G-code" flag.
  4740. # Unconditional, so a print Bambuddy didn't dispatch drops the previous
  4741. # print's flag instead of inheriting it.
  4742. print_dispatch_context.adopt(printer_id)
  4743. # Cancel any active bed cooldown waiter for this printer
  4744. if _bed_cool_waiters.pop(printer_id, None):
  4745. logger.info("[BED-COOL] Cancelled bed cooldown waiter for printer %s (new print started)", printer_id)
  4746. # Clear cached cover images so the new print's thumbnail is fetched fresh
  4747. from backend.app.api.routes.printers import clear_cover_cache
  4748. clear_cover_cache(printer_id)
  4749. await ws_manager.send_print_start(printer_id, data)
  4750. # Notify when the print-start AMS mapping references tray slots without spool assignments.
  4751. await notify_missing_spool_assignments_on_print_start(printer_id, data, logger)
  4752. # MQTT relay - publish print start
  4753. try:
  4754. printer_info = printer_manager.get_printer(printer_id)
  4755. if printer_info:
  4756. await mqtt_relay.on_print_start(
  4757. printer_id,
  4758. printer_info.name,
  4759. printer_info.serial_number,
  4760. data.get("filename", ""),
  4761. data.get("subtask_name", ""),
  4762. )
  4763. except Exception:
  4764. pass # Don't fail print start callback if MQTT fails
  4765. # Capture AMS tray remain%, the assignment snapshot, the dispatched plate
  4766. # and mapping, and the seeded tray-change log.
  4767. #
  4768. # Unconditional, for both inventory backends. This only *captures* — the
  4769. # writing is still split, with the internal tracker skipped at completion
  4770. # when Spoolman owns usage. Spoolman's own durable row (#1820) already
  4771. # carries its plate-scoped 3MF figures and stored mapping, but not the
  4772. # tray-change log, and that log is the only record of which spool fed
  4773. # which layers when AMS Filament Backup swaps trays mid-print. Capturing
  4774. # it on one side only would leave Spoolman users with the mid-print
  4775. # restart bug this fixes for everyone else.
  4776. try:
  4777. async with async_session() as db:
  4778. from backend.app.api.routes.settings import get_setting
  4779. from backend.app.services.usage_tracker import on_print_start as usage_on_print_start
  4780. _spoolman_on = await get_setting(db, "spoolman_enabled")
  4781. await usage_on_print_start(
  4782. printer_id,
  4783. data,
  4784. printer_manager,
  4785. db=db,
  4786. spoolman_owns_usage=bool(_spoolman_on) and _spoolman_on.lower() == "true",
  4787. )
  4788. except Exception as e:
  4789. logger.warning("Usage tracker on_print_start failed: %s", e)
  4790. # Track if notification was sent (to avoid sending twice)
  4791. notification_sent = False
  4792. # Smart plug automation: turn on plug when print starts
  4793. try:
  4794. async with async_session() as db:
  4795. await smart_plug_manager.on_print_start(printer_id, db)
  4796. except Exception as e:
  4797. logger.warning("Smart plug on_print_start failed: %s", e)
  4798. async with async_session() as db:
  4799. from backend.app.models.printer import Printer
  4800. from backend.app.services.bambu_ftp import list_files_async
  4801. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  4802. printer = result.scalar_one_or_none()
  4803. # Plate detection check - pause if objects detected on build plate
  4804. logger.info(
  4805. f"[PLATE CHECK] printer_id={printer_id}, plate_detection_enabled={printer.plate_detection_enabled if printer else 'NO PRINTER'}"
  4806. )
  4807. if printer and printer.plate_detection_enabled:
  4808. logger.info("[PLATE CHECK] ENTERING plate detection code for printer %s", printer_id)
  4809. # Release the pooled DB connection before the plate-detection camera
  4810. # work (a 2.5s light-settle sleep + FTP/camera capture). Only the
  4811. # printer SELECT has run so far — nothing to persist — so this commit
  4812. # is a data-noop that ends the read transaction and returns the
  4813. # connection to the pool during the I/O (issue #2572). expire_on_commit
  4814. # =False keeps printer.* readable; on_plate_not_empty (rare) and the
  4815. # archive lookups below re-acquire a fresh connection on next execute.
  4816. await db.commit()
  4817. try:
  4818. from backend.app.services.plate_detection import check_plate_empty
  4819. # Build ROI tuple from printer settings if available
  4820. roi = None
  4821. if all(
  4822. [
  4823. printer.plate_detection_roi_x is not None,
  4824. printer.plate_detection_roi_y is not None,
  4825. printer.plate_detection_roi_w is not None,
  4826. printer.plate_detection_roi_h is not None,
  4827. ]
  4828. ):
  4829. roi = (
  4830. printer.plate_detection_roi_x,
  4831. printer.plate_detection_roi_y,
  4832. printer.plate_detection_roi_w,
  4833. printer.plate_detection_roi_h,
  4834. )
  4835. # The light is held, not switched, so a live view opened
  4836. # meanwhile keeps it when the check lets go (SP-C11). Every
  4837. # printer, whatever the camera light mode: detection needs it.
  4838. plate_photo_data = None
  4839. objects_detected = False
  4840. check_failure: str | None = None
  4841. async with camera_light(printer, any_mode=True, settle=2.5):
  4842. logger.info("[PLATE CHECK] Running plate detection for printer %s", printer_id)
  4843. plate_result = await check_plate_empty(
  4844. printer_id=printer_id,
  4845. ip_address=printer.ip_address,
  4846. access_code=printer.access_code,
  4847. model=printer.model,
  4848. include_debug_image=False,
  4849. external_camera_url=printer.external_camera_url,
  4850. external_camera_type=printer.external_camera_type,
  4851. use_external=printer.external_camera_enabled,
  4852. roi=roi,
  4853. external_camera_snapshot_url=printer.external_camera_snapshot_url,
  4854. )
  4855. # No reference to compare against is a check that did not
  4856. # run either: say so rather than call the plate empty.
  4857. check_failure = getattr(plate_result, "failure_reason", None) or (
  4858. "not_calibrated" if plate_result.needs_calibration else None
  4859. )
  4860. objects_detected = not check_failure and not plate_result.is_empty
  4861. if objects_detected:
  4862. # Objects detected - pause the print!
  4863. logger.warning(
  4864. f"[PLATE CHECK] Objects detected on plate for printer {printer_id}! "
  4865. f"Confidence: {plate_result.confidence:.0%}, Diff: {plate_result.difference_percent:.1f}%"
  4866. )
  4867. pause_client = printer_manager.get_client(printer_id)
  4868. if pause_client:
  4869. pause_client.pause_print()
  4870. logger.info("[PLATE CHECK] Print paused for printer %s", printer_id)
  4871. # Snapshot while the light's still on — restoring it first
  4872. # would leave the notification with a dark photo.
  4873. try:
  4874. plate_photo_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  4875. except Exception as snap_err:
  4876. logger.warning(
  4877. "[PLATE CHECK] Failed to capture snapshot for printer %s: %s", printer_id, snap_err
  4878. )
  4879. if objects_detected:
  4880. # Send notification about plate not empty
  4881. await ws_manager.broadcast(
  4882. {
  4883. "type": "plate_not_empty",
  4884. "printer_id": printer_id,
  4885. "printer_name": printer.name,
  4886. "message": f"Objects detected on build plate! Print paused. (Diff: {plate_result.difference_percent:.1f}%)",
  4887. }
  4888. )
  4889. # Also send push notification
  4890. try:
  4891. await notification_service.on_plate_not_empty(
  4892. printer_id=printer_id,
  4893. printer_name=printer.name,
  4894. db=db,
  4895. difference_percent=plate_result.difference_percent,
  4896. image_data=plate_photo_data,
  4897. )
  4898. except Exception as notif_err:
  4899. logger.warning("[PLATE CHECK] Failed to send notification: %s", notif_err)
  4900. elif check_failure:
  4901. logger.warning(
  4902. "[PLATE CHECK] Plate check could not run for printer %s (%s): %s; proceeding with print",
  4903. printer_id,
  4904. check_failure,
  4905. plate_result.message,
  4906. )
  4907. await _announce_plate_check_failed(printer_id, printer.name, check_failure)
  4908. else:
  4909. logger.info("[PLATE CHECK] Plate is empty for printer %s, proceeding with print", printer_id)
  4910. except Exception as plate_err:
  4911. # Don't block print on plate detection errors
  4912. logger.warning(
  4913. "[PLATE CHECK] Plate check could not run for printer %s: %s; proceeding with print",
  4914. printer_id,
  4915. plate_err,
  4916. )
  4917. await _announce_plate_check_failed(printer_id, printer.name, "error")
  4918. if not printer:
  4919. logger.info("[CALLBACK] Skipping archive - printer not found in database")
  4920. if not notification_sent:
  4921. await _send_print_start_notification(printer_id, data, logger=logger)
  4922. return
  4923. # Skip the printer's own jobs — a calibration run is not a user's print.
  4924. # See is_internal_printer_job for what counts and why both fields are
  4925. # tested; the pressure-advance line reports as a subtask name with no
  4926. # /usr/ path, which the old prefix-only test here missed entirely.
  4927. #
  4928. # No notification either. The event describes the printer calibrating
  4929. # itself, so "Print started" is as wrong as the archive was, and the
  4930. # matching completion is suppressed in on_print_complete for the same
  4931. # reason. Checked before the auto-archive switches, whose "off" path
  4932. # still notifies.
  4933. # A /usr/ path is the printer's own file. A name alone is not proof: a
  4934. # print Bambuddy dispatched is never one, whatever it is called.
  4935. # Dispatched = an expected print, or, after a restart that lost those,
  4936. # a queue job printing on the printer.
  4937. if is_internal_printer_job(data.get("filename", ""), data.get("subtask_name", "")) and (
  4938. (data.get("filename") or "").startswith("/usr/")
  4939. or not await _dispatched_by_bambuddy(db, printer_id, data.get("filename", ""), data.get("subtask_name", ""))
  4940. ):
  4941. logger.info(
  4942. "[CALLBACK] Skipping archive — internal printer job detected: filename=%s, subtask=%s",
  4943. data.get("filename", ""),
  4944. data.get("subtask_name", ""),
  4945. )
  4946. return
  4947. # The global switch is the master: off, no printer archives prints
  4948. # started outside Bambuddy; on, each printer's own switch decides.
  4949. from backend.app.api.routes.settings import get_setting
  4950. _global_auto_archive = await get_setting(db, "auto_archive")
  4951. _global_archives = _global_auto_archive is None or _global_auto_archive.lower() == "true"
  4952. archives_prints = printer.auto_archive and _global_archives
  4953. if not archives_prints:
  4954. # auto-archive disabled — check if there's an expected print (dispatched
  4955. # by BamBuddy via queue/reprint) that already has an archive to promote.
  4956. # If so, fall through to the expected-print handling below so the archive
  4957. # is tracked in _active_prints and usage tracking works at completion.
  4958. _fn = data.get("filename", "")
  4959. _sn = data.get("subtask_name", "")
  4960. _check_keys: list[tuple[int, str]] = []
  4961. if _sn:
  4962. _check_keys += [
  4963. (printer_id, _sn),
  4964. (printer_id, f"{_sn}.3mf"),
  4965. (printer_id, f"{_sn}.gcode.3mf"),
  4966. ]
  4967. if _fn:
  4968. _base_fn = _fn.split("/")[-1] if "/" in _fn else _fn
  4969. _check_keys.append((printer_id, _base_fn))
  4970. _no_archive_base = _base_fn.replace(".gcode", "").replace(".3mf", "")
  4971. _check_keys += [
  4972. (printer_id, _no_archive_base),
  4973. (printer_id, f"{_no_archive_base}.3mf"),
  4974. ]
  4975. _has_expected = any(k in _expected_prints for k in _check_keys)
  4976. if not _has_expected:
  4977. # No expected print — truly external print (started from slicer/touchscreen).
  4978. # Say which switch it was: before 1.3.0 the global one was saved
  4979. # and never read, so an install upgraded with it off stopped
  4980. # archiving with nothing to point at it (#3477).
  4981. if not _global_archives:
  4982. logger.info(
  4983. "[CALLBACK] Skipping archive: the global 'Auto-archive prints' switch "
  4984. "(Settings > General > Archive Settings) is off, and Bambuddy did not start this print"
  4985. )
  4986. else:
  4987. logger.info(
  4988. "[CALLBACK] Skipping archive: auto-archive is off for printer %s, "
  4989. "and Bambuddy did not start this print",
  4990. printer_id,
  4991. )
  4992. if not notification_sent:
  4993. _no_archive_creator: int | None = None
  4994. for _key in _check_keys:
  4995. _expected_prints.pop(_key, None)
  4996. _expected_print_registered_at.pop(_key, None)
  4997. popped_creator = _expected_print_creators.pop(_key, None)
  4998. if _no_archive_creator is None:
  4999. _no_archive_creator = popped_creator
  5000. _creator_data = {"created_by_id": _no_archive_creator} if _no_archive_creator else None
  5001. await _send_print_start_notification(printer_id, data, _creator_data, logger)
  5002. return
  5003. else:
  5004. logger.info("[CALLBACK] auto_archive disabled but expected print found — promoting archive")
  5005. # Get the filename and subtask_name
  5006. filename = data.get("filename", "")
  5007. subtask_name = data.get("subtask_name", "")
  5008. # MQTT subtask_id uniquely identifies a print job on the printer. When
  5009. # present, it lets us match an archive across a backend restart (#972):
  5010. # same id → same print → resume the existing row instead of cancelling
  5011. # it and recreating from scratch (which loses started_at). Treat "0"
  5012. # and "" as absent — Bambu reports "0" for non-cloud / local prints.
  5013. raw_mqtt = data.get("raw_data") or {}
  5014. subtask_id = raw_mqtt.get("subtask_id")
  5015. if subtask_id is not None:
  5016. subtask_id = str(subtask_id).strip()
  5017. if subtask_id in ("", "0"):
  5018. subtask_id = None
  5019. logger.info("[CALLBACK] Print start detected - filename: %s, subtask: %s", filename, subtask_name)
  5020. if not filename and not subtask_name:
  5021. # Send notification without archive data (no filename)
  5022. logger.info("[CALLBACK] Skipping archive - no filename or subtask_name")
  5023. if not notification_sent:
  5024. await _send_print_start_notification(printer_id, data, logger=logger)
  5025. return
  5026. # Check if this is an expected print from reprint/scheduled
  5027. expected_keys = _expected_print_keys(printer_id, filename, subtask_name)
  5028. expected_archive_id = None
  5029. # Who queued it, taken with the entries it was registered beside: left
  5030. # behind once those are gone (the archive deleted mid-dispatch, or the
  5031. # start notification already sent), the expiry never saw it, and an
  5032. # unrelated later print of that name was attributed to that user (EP-2).
  5033. fallback_creator = None
  5034. for key in expected_keys:
  5035. expected_archive_id = _expected_prints.pop(key, None)
  5036. _expected_print_registered_at.pop(key, None)
  5037. if expected_archive_id:
  5038. # Clean up other possible keys for this print
  5039. for other_key in expected_keys:
  5040. _expected_prints.pop(other_key, None)
  5041. _expected_print_registered_at.pop(other_key, None)
  5042. creator = _expected_print_creators.pop(other_key, None)
  5043. if fallback_creator is None:
  5044. fallback_creator = creator
  5045. break
  5046. if expected_archive_id:
  5047. # This is a reprint/scheduled print - use existing archive, don't create new one
  5048. logger.info("Using expected archive %s for print (skipping duplicate)", expected_archive_id)
  5049. # Bambuddy dispatched it: the kill switch's fast path keeps it as
  5050. # Bambuddy's once the expected entry is consumed here.
  5051. _owned_print_archive_ids.add(expected_archive_id)
  5052. from backend.app.models.archive import PrintArchive
  5053. result = await db.execute(select(PrintArchive).where(PrintArchive.id == expected_archive_id))
  5054. archive = result.scalar_one_or_none()
  5055. # Deleted after this handler loaded it (the plate check waits
  5056. # seconds) but before the route forgot the expected print: the
  5057. # archive the route kept or deleted is left as the route left it.
  5058. if archive and await _printer_gone(db, printer_id):
  5059. logger.info(
  5060. "Printer %s was deleted before its print started; not adopting archive %s", printer_id, archive.id
  5061. )
  5062. await forget_printer(printer_id)
  5063. return
  5064. if archive:
  5065. # Persist a restart-stable id so a later restart resumes this
  5066. # archive by subtask_id instead of name-matching + duplicating
  5067. # it (#1485). The printer often hasn't echoed subtask_id back
  5068. # this soon after dispatch, so fall back to the id Bambuddy
  5069. # minted when it sent the print command. Scoped to this
  5070. # expected-print branch on purpose: an expected match means
  5071. # Bambuddy dispatched this exact print in this process, so the
  5072. # client's last-dispatch id genuinely belongs to it — using it
  5073. # for an externally-started print could mis-tag the archive.
  5074. effective_subtask_id = subtask_id
  5075. if not effective_subtask_id:
  5076. _client = printer_manager.get_client(printer_id)
  5077. _dispatched = getattr(_client, "last_dispatch_subtask_id", None) if _client else None
  5078. if _dispatched:
  5079. effective_subtask_id = str(_dispatched).strip() or None
  5080. starting_plate_id = _print_plate_ids.get((printer_id, expected_archive_id))
  5081. if not await _adopt_archive_for_run(
  5082. db,
  5083. archive,
  5084. printer_id,
  5085. starting_plate_id=starting_plate_id,
  5086. effective_subtask_id=effective_subtask_id,
  5087. logger=logger,
  5088. ):
  5089. return
  5090. # Track as active print
  5091. _bind_active_print(printer_id, archive.id, archive.filename, filename, subtask_name)
  5092. await _open_run_for(archive.id, printer_id, subtask_id)
  5093. # Start timelapse session if external camera is enabled (#1353).
  5094. # Queue / VP-dispatched prints land here in the expected-archive
  5095. # branch and used to skip start_session entirely — frames were
  5096. # never captured and the post-print stitch silently returned None.
  5097. _maybe_start_layer_timelapse(printer, printer_id, archive.id)
  5098. # Inject ams_mapping into usage tracker session — the session was created
  5099. # before expected-print promotion, so it may have ams_mapping=None when
  5100. # the MQTT request topic subscription failed (common on P1S/A1).
  5101. _stored_map = _print_ams_mappings.get((printer_id, expected_archive_id))
  5102. _stored_plate_id = _print_plate_ids.get((printer_id, expected_archive_id))
  5103. if _stored_map or _stored_plate_id is not None:
  5104. try:
  5105. from backend.app.services.usage_tracker import _active_sessions
  5106. _ut_session = _active_sessions.get(printer_id)
  5107. if _ut_session and _stored_map and not _ut_session.ams_mapping:
  5108. _ut_session.ams_mapping = _stored_map
  5109. logger.info("[CALLBACK] Injected ams_mapping into usage tracker session: %s", _stored_map)
  5110. # plate_id injection covers direct-Print of plate N of a multi-plate
  5111. # 3MF — queue prints already capture it via the on_print_start queue
  5112. # lookup, but direct-Print never goes through the queue (#1697).
  5113. if _ut_session and _stored_plate_id is not None and _ut_session.plate_id is None:
  5114. _ut_session.plate_id = _stored_plate_id
  5115. logger.info("[CALLBACK] Injected plate_id into usage tracker session: %s", _stored_plate_id)
  5116. except Exception:
  5117. pass
  5118. # Set up energy tracking (#941: persist start on archive row)
  5119. await _record_energy_start(archive, printer_id, db, context="expected-print")
  5120. await ws_manager.send_archive_updated(
  5121. {
  5122. "id": archive.id,
  5123. "status": "printing",
  5124. }
  5125. )
  5126. # Send notification with archive data (reprint/scheduled)
  5127. if not notification_sent:
  5128. # Use archive's created_by_id; fall back to the creator registered via
  5129. # register_expected_print (handles library-file-based queue items where
  5130. # the freshly-created archive has no created_by_id yet), taken above.
  5131. archive_data = {
  5132. "print_time_seconds": _run_print_time_seconds(archive, printer_id),
  5133. "created_by_id": archive.created_by_id or fallback_creator,
  5134. }
  5135. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5136. # Extract printable objects from the archived 3MF file
  5137. _load_objects_from_archive(archive, printer_id, logger)
  5138. if archive.file_path:
  5139. _load_pauses_from_3mf(app_settings.base_dir / archive.file_path, printer_id)
  5140. # Store Spoolman tracking data for per-filament usage reporting
  5141. try:
  5142. await _store_spoolman_print_data(
  5143. printer_id,
  5144. archive.id,
  5145. archive.file_path,
  5146. db,
  5147. printer_manager,
  5148. ams_mapping=_get_start_ams_mapping(data, printer_id, archive.id),
  5149. plate_id=_get_start_plate_id(printer_id, archive.id),
  5150. )
  5151. except Exception as e:
  5152. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  5153. # Price the print from its spools now rather than only at completion (#3261)
  5154. _schedule_archive_cost_estimate(
  5155. printer_id,
  5156. archive.id,
  5157. printer_manager,
  5158. ams_mapping=_get_start_ams_mapping(data, printer_id, archive.id),
  5159. plate_id=_get_start_plate_id(printer_id, archive.id),
  5160. session_factory=async_session,
  5161. )
  5162. # Capture timelapse file baseline for snapshot-diff on completion
  5163. # (mirrors the new-archive branch). Queue / VP-dispatched prints
  5164. # hit this branch — without the baseline the completion-time scan
  5165. # falls into its "take baseline now" fallback, which snapshots
  5166. # AFTER the new MP4 already exists and never matches a diff
  5167. # (#1403 follow-up — see pwostran's 2026-05-18 support bundle).
  5168. await _capture_timelapse_baseline_at_start(printer, printer_id, logger, archive_id=archive.id)
  5169. await _forget_if_printer_gone(printer_id)
  5170. return # Skip creating a new archive
  5171. # The archive was deleted after the job was queued.
  5172. if not archives_prints:
  5173. # Only the expected print brought a printer that does not
  5174. # archive this far; with it gone, nothing is archived.
  5175. return
  5176. # Otherwise archive the print anew rather than drop it, so it still
  5177. # gets its notification, usage tracking and history.
  5178. logger.warning(
  5179. "Expected archive %s for printer %s no longer exists; archiving the print anew",
  5180. expected_archive_id,
  5181. printer_id,
  5182. )
  5183. # Check if there's already a "printing" archive for this printer/file
  5184. # This prevents duplicates when backend restarts during an active print
  5185. from backend.app.models.archive import PrintArchive
  5186. existing_archive: PrintArchive | None = None
  5187. resumed_by_run = False
  5188. # Best match: this job's run, open on the printer since the print
  5189. # started. The archive row is shared by every plate of it, and its
  5190. # printer, job id and status are the latest start's, which may be
  5191. # another printer's.
  5192. if subtask_id:
  5193. from backend.app.services.print_log import open_run_on_printer
  5194. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  5195. if run is not None and run.archive_id is not None:
  5196. existing_archive = await db.scalar(
  5197. select(PrintArchive).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  5198. )
  5199. resumed_by_run = existing_archive is not None
  5200. # Preferred match: subtask_id equality. MQTT reports the same subtask_id
  5201. # across a backend restart for the same print, so this is the most
  5202. # reliable way to reattach. Only a row still printing: a cancelled one
  5203. # has been through completion (usage, history, notifications) already,
  5204. # and reopening it would count the print twice.
  5205. if subtask_id and existing_archive is None:
  5206. by_id = await db.execute(
  5207. select(PrintArchive)
  5208. .where(PrintArchive.printer_id == printer_id)
  5209. .where(PrintArchive.subtask_id == subtask_id)
  5210. .where(PrintArchive.status == "printing")
  5211. .order_by(PrintArchive.created_at.desc())
  5212. .limit(1)
  5213. )
  5214. existing_archive = by_id.scalar_one_or_none()
  5215. # Fallback match: name-based lookup. Kept as-is for prints whose
  5216. # subtask_id is missing ("0" / local / non-cloud prints).
  5217. if existing_archive is None:
  5218. check_name = subtask_name or filename.split("/")[-1].replace(".gcode", "").replace(".3mf", "")
  5219. archive_filenames = [f"{check_name}.3mf", f"{check_name}.gcode.3mf"]
  5220. # A print started from the printer's screen names its file in full,
  5221. # so the forms above double its extension. The file itself is the
  5222. # one exact match, and nothing looser is added (#3009).
  5223. if _subtask_is_the_file(subtask_name, filename):
  5224. archive_filenames.append(subtask_name)
  5225. existing = await db.execute(
  5226. select(PrintArchive)
  5227. .where(PrintArchive.printer_id == printer_id)
  5228. .where(PrintArchive.status == "printing")
  5229. .where(
  5230. or_(
  5231. PrintArchive.print_name == check_name,
  5232. PrintArchive.filename.in_(archive_filenames),
  5233. )
  5234. )
  5235. .order_by(PrintArchive.created_at.desc())
  5236. .limit(1)
  5237. )
  5238. existing_archive = existing.scalar_one_or_none()
  5239. if existing_archive:
  5240. # subtask_id match → always resume, regardless of age. Same print,
  5241. # just a backend restart.
  5242. subtask_match = resumed_by_run or bool(subtask_id and existing_archive.subtask_id == subtask_id)
  5243. if subtask_match:
  5244. logger.info("Resuming archive %s on subtask_id match (%s)", existing_archive.id, subtask_id)
  5245. _bind_active_print(printer_id, existing_archive.id, existing_archive.filename, filename, subtask_name)
  5246. await _open_run_for(existing_archive.id, printer_id, subtask_id)
  5247. _mark_owned_if_dispatched(existing_archive)
  5248. if await _run_needs_energy_start(existing_archive, printer_id, db):
  5249. await _record_energy_start(existing_archive, printer_id, db, context="subtask-resume")
  5250. if not notification_sent:
  5251. archive_data = {
  5252. "print_time_seconds": _run_print_time_seconds(existing_archive, printer_id),
  5253. "created_by_id": existing_archive.created_by_id,
  5254. }
  5255. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5256. _load_objects_from_archive(existing_archive, printer_id, logger)
  5257. if existing_archive.file_path:
  5258. _load_pauses_from_resumed_archive(existing_archive, printer_id)
  5259. return
  5260. # Name-match only (no subtask_id to anchor on): decide resume vs.
  5261. # stale from the printer's *current* progress, not wall-clock age.
  5262. # A genuinely long print used to trip a blind 4h cutoff and have its
  5263. # live archive cancelled + duplicated on every backend restart
  5264. # (#1485). If the printer reports real progress, this name-matched
  5265. # 'printing' archive IS that ongoing print — resume it whatever its
  5266. # age. Only treat it as a stale leftover when the printer clearly
  5267. # shows a different, freshly-started print: near-0% progress on an
  5268. # archive far too old to still be at 0%. Unknown progress (printer
  5269. # not connected) never cancels — resuming is the safe default.
  5270. archive_age = datetime.now(timezone.utc) - existing_archive.created_at.replace(tzinfo=timezone.utc)
  5271. live_status = printer_manager.get_status(printer_id)
  5272. live_progress = getattr(live_status, "progress", None) if live_status else None
  5273. looks_stale = (
  5274. live_progress is not None and live_progress < 1.0 and archive_age.total_seconds() > 2 * 60 * 60
  5275. )
  5276. # Two subtask ids are two prints, however young the archive: one
  5277. # whose completion was missed must not swallow the next print of
  5278. # the same file. Except Bambuddy's own reprint, which reuses the
  5279. # row and mints a fresh id (#1807): the id it last sent says so,
  5280. # and after a restart, which forgets it, this printer's queue job
  5281. # running the archive.
  5282. if (
  5283. not looks_stale
  5284. and subtask_id
  5285. and existing_archive.subtask_id
  5286. and existing_archive.subtask_id != subtask_id
  5287. ):
  5288. _client = printer_manager.get_client(printer_id)
  5289. _dispatched = str(getattr(_client, "last_dispatch_subtask_id", None) or "").strip()
  5290. if _dispatched:
  5291. looks_stale = _dispatched != subtask_id
  5292. else:
  5293. from backend.app.models.print_queue import PrintQueueItem as _PQI
  5294. queue_run = await db.scalar(
  5295. select(_PQI.id)
  5296. .where(_PQI.archive_id == existing_archive.id)
  5297. .where(_PQI.printer_id == printer_id)
  5298. .where(_PQI.status == "printing")
  5299. .limit(1)
  5300. )
  5301. looks_stale = queue_run is None
  5302. if looks_stale:
  5303. logger.warning(
  5304. "Found stale 'printing' archive %s (age: %s, printer progress %s%%, subtask %s, printer reports %s)"
  5305. " — marking cancelled and creating new archive",
  5306. existing_archive.id,
  5307. archive_age,
  5308. live_progress,
  5309. existing_archive.subtask_id,
  5310. subtask_id,
  5311. )
  5312. existing_archive.status = "cancelled"
  5313. # Canonical key, not a sentence (issue #2974). "No status update
  5314. # received" is what both stale paths actually observed; which of
  5315. # the two it was is already carried by ``status`` -- cancelled
  5316. # here, the reconciled outcome at the reconnect site -- so one
  5317. # key loses no information and gives the Statistics breakdown a
  5318. # single bucket instead of two untranslatable prose strings.
  5319. existing_archive.failure_reason = "noStatusUpdate"
  5320. # And its run, as before runs were opened at the start: a new
  5321. # print starts on the printer, so whatever is open there ended
  5322. # unseen. Left open, the next reconnect would close it again.
  5323. from backend.app.services.print_log import discard_open_runs
  5324. discarded = await discard_open_runs(db, printer_id)
  5325. await db.commit()
  5326. await _announce_discarded_runs(discarded, printer_id)
  5327. # Its print keys go with it; the new archive registers its own.
  5328. for _key in [k for k, v in _active_prints.items() if v == existing_archive.id and k[0] == printer_id]:
  5329. del _active_prints[_key]
  5330. # Fall through to create new archive (don't return)
  5331. else:
  5332. logger.info(
  5333. f"Skipping duplicate - already have printing archive {existing_archive.id} for {check_name}"
  5334. )
  5335. # Track this as the active print
  5336. _bind_active_print(printer_id, existing_archive.id, existing_archive.filename, filename, subtask_name)
  5337. _mark_owned_if_dispatched(existing_archive)
  5338. # Attach subtask_id retroactively so future restarts can resume.
  5339. # Compare for inequality (not "is empty") to also pick up reprint
  5340. # dispatches that mint a fresh id — see #1807 for the bogus
  5341. # "Print Stopped" the strict-empty guard caused on reconnect.
  5342. if subtask_id and existing_archive.subtask_id != subtask_id:
  5343. existing_archive.subtask_id = subtask_id
  5344. await db.commit()
  5345. await _open_run_for(existing_archive.id, printer_id, subtask_id)
  5346. # Also set up energy tracking if not already tracked (#941: persisted column)
  5347. if await _run_needs_energy_start(existing_archive, printer_id, db):
  5348. await _record_energy_start(existing_archive, printer_id, db, context="existing-printing")
  5349. # Send notification with archive data (existing archive)
  5350. if not notification_sent:
  5351. archive_data = {
  5352. "print_time_seconds": _run_print_time_seconds(existing_archive, printer_id),
  5353. "created_by_id": existing_archive.created_by_id,
  5354. }
  5355. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5356. # Extract printable objects from the archived 3MF file
  5357. _load_objects_from_archive(existing_archive, printer_id, logger)
  5358. if existing_archive.file_path:
  5359. _load_pauses_from_resumed_archive(existing_archive, printer_id)
  5360. return
  5361. # Build list of possible 3MF filenames to try
  5362. possible_names = []
  5363. # Bambu printers typically store files as "Name.gcode.3mf"
  5364. # The subtask_name is usually the best source for the filename
  5365. if subtask_name:
  5366. if _subtask_is_the_file(subtask_name, filename):
  5367. # A print started from the printer's own screen reports the
  5368. # file's full name, extension included. That is the file, so
  5369. # no extension is appended to it (#3009).
  5370. possible_names.append(subtask_name)
  5371. else:
  5372. # Try common Bambu naming patterns
  5373. possible_names.append(f"{subtask_name}.gcode.3mf")
  5374. possible_names.append(f"{subtask_name}.3mf")
  5375. # Try original filename with .3mf extension
  5376. if filename:
  5377. # Extract just the filename part, not the full path
  5378. fname = filename.split("/")[-1] if "/" in filename else filename
  5379. if fname.endswith(".3mf"):
  5380. possible_names.append(fname)
  5381. elif fname.endswith(".gcode"):
  5382. base = fname.rsplit(".", 1)[0]
  5383. possible_names.append(f"{base}.gcode.3mf")
  5384. possible_names.append(f"{base}.3mf")
  5385. else:
  5386. possible_names.append(f"{fname}.gcode.3mf")
  5387. possible_names.append(f"{fname}.3mf")
  5388. # Also try with spaces converted to underscores (Bambu Studio may normalize filenames)
  5389. space_variants = []
  5390. for name in possible_names:
  5391. if " " in name:
  5392. space_variants.append(name.replace(" ", "_"))
  5393. possible_names.extend(space_variants)
  5394. # Remove duplicates while preserving order
  5395. seen = set()
  5396. possible_names = [x for x in possible_names if not (x in seen or seen.add(x))]
  5397. logger.info("Trying filenames: %s", possible_names)
  5398. # Release the pooled DB connection before the 3MF FTP download. Reaching
  5399. # here means none of the expected-/existing-archive write branches ran
  5400. # (they all return earlier) — only SELECTs have executed on this path, so
  5401. # this commit persists nothing; it ends the read transaction so the
  5402. # connection returns to the pool during the download. That download tries
  5403. # up to five remote paths per candidate filename with retry/backoff and
  5404. # can run for minutes under FTP contention; holding the session across it
  5405. # pinned one pooled connection idle-in-transaction (issue #2572). No DB
  5406. # work runs during the download — the new-archive writes below re-acquire
  5407. # a fresh connection, and expire_on_commit=False keeps printer.* readable.
  5408. await db.commit()
  5409. # Try to find and download the 3MF file
  5410. temp_path = None
  5411. downloaded_filename = None
  5412. # Cache check: cover endpoint may have already pulled this 3MF during
  5413. # the print (frontend opens the card and shows the thumbnail) — reuse
  5414. # that file instead of re-downloading 36MB over the same FTP link that
  5415. # just served it (#972). The cache keys on a normalized filename so
  5416. # variants like "X", "X.3mf", "X.gcode.3mf" all collapse to one entry.
  5417. for try_filename in possible_names:
  5418. if not try_filename.endswith(".3mf"):
  5419. continue
  5420. cached = get_cached_3mf(printer_id, try_filename)
  5421. if cached:
  5422. logger.info("Reusing cached 3MF from %s (avoided duplicate FTP)", cached)
  5423. temp_path = cached
  5424. downloaded_filename = try_filename
  5425. break
  5426. # Does this printer keep the sliced file somewhere FTPS can reach? On
  5427. # H2-series and P2S the answer is routinely no — the file stays on
  5428. # internal eMMC and port 990 only ever serves external storage — and
  5429. # then the whole sweep below (six filenames x five directories x four
  5430. # retries, then the directory walk) is ~110 connections that cannot
  5431. # succeed. Skip it and say why (#2780).
  5432. storage = print_file_reachable_over_ftp(printer_manager.get_status(printer_id))
  5433. # Set when a lookup is abandoned because the printer's FTPS cool-off is
  5434. # running rather than because the file is somewhere unreachable. The
  5435. # distinction is the whole of #2957: one is permanent, the other clears
  5436. # in minutes with the file still sitting on the printer.
  5437. blocked_by_ftps_cooloff = False
  5438. # Set when a probe reached the printer and still came back without the
  5439. # file -- a timeout mid-transfer, a refused connection, anything that is
  5440. # not a clean "not here". A 550 raises FileNotOnPrinterError and is
  5441. # caught by name below, so a file that genuinely is not on the card
  5442. # leaves this False and schedules nothing. Anything else means the
  5443. # transfer, not the file, is what failed, and that does not last (#3063).
  5444. ftp_transfer_failed = False
  5445. # The narrower case of the above: the directory walk *listed* a file
  5446. # for this print and could not fetch it, so the file is on the card
  5447. # under a name that matches. The only failure that rules out an
  5448. # earlier run's 3MF standing in for it (#3307).
  5449. walk_transfer_failed = False
  5450. # Paths the sweep reached and could not transfer -- a timeout, not a
  5451. # 550. The directory walk lists the same folders and would download
  5452. # the same file again: on a slow A1 that was a second 300 s on the
  5453. # printer's one FTPS slot, at print start, for a file the recovery
  5454. # retry fetches with a deadline twice as long (#3308).
  5455. sweep_failed_paths: set[str] = set()
  5456. # The print's name, for a fallback archive whose `subtask_name` the
  5457. # plate guard below had to disown. Display only, and deliberately kept
  5458. # apart from `subtask_name`: that variable is what every file lookup
  5459. # here is built from, and once a name has been shown to fetch another
  5460. # plate's 3MF it must not key `_active_prints` either, or the cover
  5461. # endpoint hands the same contradicted file to
  5462. # `_recover_fallback_archive` and fills the row in with it (#3126).
  5463. display_name_after_plate_reject: str | None = None
  5464. # Get FTP retry settings
  5465. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  5466. # ...but "the printer put it on eMMC" is where it went, not whether we
  5467. # can read it. An H2D with a card in mirrors the job to /cache and
  5468. # serves it happily, and skipping on the URL alone cost that reporter
  5469. # every archive for two days (#2856). So ask the printer instead of
  5470. # guessing: the dispatch named the exact file, which is one connection
  5471. # walking five paths rather than the sweep's ~110. Only when the probe
  5472. # comes back empty does the verdict's reason stand.
  5473. if not storage.reachable and not downloaded_filename and storage.probe_filename:
  5474. if ftps_handshake_blocked(printer.ip_address):
  5475. # Deliberately NOT recorded as a cool-off give-up. This branch
  5476. # only runs on an unreachable verdict, and that verdict is the
  5477. # honest, permanent reason the archive is empty — the probe was
  5478. # a long shot on top of it. Blaming the cool-off here would
  5479. # schedule a retry for a file sitting on internal eMMC, which is
  5480. # the sweep #2780 removed (#2957).
  5481. logger.debug(
  5482. "Not probing for %s on printer %s: its file service is not answering over TLS",
  5483. storage.probe_filename,
  5484. printer_id,
  5485. )
  5486. else:
  5487. probe_path = app_settings.archive_dir / "temp" / storage.probe_filename
  5488. probe_path.parent.mkdir(parents=True, exist_ok=True)
  5489. try:
  5490. probe_hit = await download_file_try_paths_async(
  5491. printer.ip_address,
  5492. printer.access_code,
  5493. ftp_probe_paths(storage.probe_filename),
  5494. probe_path,
  5495. socket_timeout=ftp_timeout,
  5496. printer_model=printer.model,
  5497. )
  5498. except Exception as e:
  5499. logger.debug("3MF probe for %s failed: %s", storage.probe_filename, e)
  5500. probe_hit = False
  5501. if probe_hit:
  5502. downloaded_filename = storage.probe_filename
  5503. temp_path = probe_path
  5504. cache_3mf_download(printer_id, downloaded_filename, probe_path)
  5505. # Naming the path, not just the file: a printer that keeps
  5506. # uploads around for weeks can serve a same-named copy of an
  5507. # earlier slice, and without the directory in the log that
  5508. # mismatch is invisible rather than merely rare (#1820).
  5509. logger.info(
  5510. "Found %s at %s over FTPS for printer %s even though the printer reported %s",
  5511. downloaded_filename,
  5512. probe_hit,
  5513. printer_id,
  5514. storage.reason,
  5515. )
  5516. if not storage.reachable and not downloaded_filename:
  5517. # Same opening words whether or not a probe ran, because that is
  5518. # the phrase support asks people to grep for — only the tail says
  5519. # which of the two happened.
  5520. logger.info(
  5521. "Skipping the 3MF lookup for printer %s: %s — %s",
  5522. printer_id,
  5523. storage.reason,
  5524. "no copy of it on external storage either"
  5525. if storage.probe_filename
  5526. else "the print file is not on storage Bambuddy can read over FTPS, so no path would find it",
  5527. )
  5528. for try_filename in possible_names if not downloaded_filename and storage.reachable else []:
  5529. if not try_filename.endswith(".3mf"):
  5530. continue
  5531. # Root (/) is where BambuStudio/OrcaSlicer uploads land on A1/P1-series
  5532. # printers, so try it first — deferring it to last cost #972's reporter
  5533. # ~48 minutes of retries on /cache//model//data//data/Metadata before
  5534. # landing on the path that actually had the file.
  5535. remote_paths = [
  5536. f"/{try_filename}",
  5537. f"/cache/{try_filename}",
  5538. f"/model/{try_filename}",
  5539. f"/data/{try_filename}",
  5540. f"/data/Metadata/{try_filename}",
  5541. ]
  5542. temp_path = app_settings.archive_dir / "temp" / try_filename
  5543. temp_path.parent.mkdir(parents=True, exist_ok=True)
  5544. for remote_path in remote_paths:
  5545. if ftps_handshake_blocked(printer.ip_address):
  5546. # The printer's FTPS service is not completing a TLS
  5547. # handshake, so it has no path we could reach — walking the
  5548. # remaining candidates only re-runs the same failure
  5549. # (#2780). Fall through to the no-3MF archive now.
  5550. #
  5551. # Remember *why*, though. This is the one give-up that is
  5552. # temporary: the cool-off clears in minutes and the file was
  5553. # on the printer the whole time. The fallback archive is
  5554. # stamped with it so a retry can be scheduled, and so the
  5555. # Archives banner stops blaming storage (#2957).
  5556. blocked_by_ftps_cooloff = True
  5557. logger.warning(
  5558. "Giving up on the 3MF for printer %s: its file service is not answering over TLS",
  5559. printer_id,
  5560. )
  5561. break
  5562. logger.debug("Trying FTP download: %s", remote_path)
  5563. try:
  5564. if ftp_retry_enabled:
  5565. downloaded = await with_ftp_retry(
  5566. download_file_async,
  5567. printer.ip_address,
  5568. printer.access_code,
  5569. remote_path,
  5570. temp_path,
  5571. timeout=ftp_timeout,
  5572. socket_timeout=ftp_timeout,
  5573. printer_model=printer.model,
  5574. max_retries=ftp_retry_count,
  5575. retry_delay=ftp_retry_delay,
  5576. operation_name=f"Download 3MF from {remote_path}",
  5577. cooloff_ip=printer.ip_address,
  5578. non_retry_exceptions=(FileNotOnPrinterError,),
  5579. )
  5580. else:
  5581. downloaded = await download_file_async(
  5582. printer.ip_address,
  5583. printer.access_code,
  5584. remote_path,
  5585. temp_path,
  5586. timeout=ftp_timeout,
  5587. socket_timeout=ftp_timeout,
  5588. printer_model=printer.model,
  5589. )
  5590. if downloaded:
  5591. downloaded_filename = try_filename
  5592. logger.info("Downloaded: %s", remote_path)
  5593. # Populate shared cache so the cover endpoint (if it
  5594. # runs next) doesn't refetch the same 36MB over FTP.
  5595. cache_3mf_download(printer_id, try_filename, temp_path)
  5596. break
  5597. # with_ftp_retry returns None once it has spent its budget,
  5598. # and download_file_async returns False on a timeout, so an
  5599. # exhausted transfer arrives here rather than as an
  5600. # exception (#3063).
  5601. ftp_transfer_failed = True
  5602. sweep_failed_paths.add(remote_path)
  5603. except FileNotOnPrinterError:
  5604. # 550 — file isn't at this path. Advance to next candidate
  5605. # without burning the retry budget.
  5606. logger.debug("3MF not at %s (550), trying next path", remote_path)
  5607. except Exception as e:
  5608. ftp_transfer_failed = True
  5609. sweep_failed_paths.add(remote_path)
  5610. logger.debug("FTP download failed for %s: %s", remote_path, e)
  5611. if downloaded_filename or ftps_handshake_blocked(printer.ip_address):
  5612. break
  5613. # If still not found, try listing directories to find matching file
  5614. # Different printer models use different directory structures. Skipped
  5615. # when the printer's FTPS handshake is failing — the directory walk is
  5616. # five more connections that cannot get further than the download did.
  5617. if (
  5618. not downloaded_filename
  5619. and storage.reachable
  5620. and (filename or subtask_name)
  5621. and not ftps_handshake_blocked(printer.ip_address)
  5622. ):
  5623. search_term = (subtask_name or filename).lower().replace(".gcode", "").replace(".3mf", "")
  5624. logger.info("Direct FTP download failed, searching directories for '%s'", search_term)
  5625. search_dirs = ["/cache", "/model", "/data", "/data/Metadata", "/"]
  5626. # Every folder is listed before one file is picked, so a file named
  5627. # for the print in a later folder wins over one that merely
  5628. # contains the name in an earlier folder.
  5629. listed_paths: list[str] = []
  5630. for search_dir in search_dirs:
  5631. try:
  5632. dir_files = await list_files_async(
  5633. printer.ip_address, printer.access_code, search_dir, printer_model=printer.model
  5634. )
  5635. except Exception as e:
  5636. logger.debug("Failed to list %s: %s", search_dir, e)
  5637. continue
  5638. threemf_files = [f.get("name") for f in dir_files if f.get("name", "").endswith(".3mf")]
  5639. if threemf_files:
  5640. logger.info(
  5641. f"Found {len(threemf_files)} 3MF files in {search_dir}: {threemf_files[:5]}{'...' if len(threemf_files) > 5 else ''}"
  5642. )
  5643. listed_paths += [
  5644. posixpath.join(search_dir, f.get("name", "")) for f in dir_files if not f.get("is_directory")
  5645. ]
  5646. for remote_full_path in _3mf_files_for_print(listed_paths, search_term):
  5647. if remote_full_path in sweep_failed_paths:
  5648. # Listed for this print, and its transfer already failed:
  5649. # the same as a failed walk download, without spending a
  5650. # second deadline on it. Skipping it cannot hand the walk a
  5651. # look-alike: while a file of the print's own name is
  5652. # listed, files that merely contain the name are not.
  5653. logger.info(
  5654. "Not downloading %s again: its transfer already failed; the recovery retry will fetch it",
  5655. remote_full_path,
  5656. )
  5657. walk_transfer_failed = True
  5658. continue
  5659. fname = posixpath.basename(remote_full_path)
  5660. logger.info("Found matching file: %s", remote_full_path)
  5661. temp_path = app_settings.archive_dir / "temp" / fname
  5662. temp_path.parent.mkdir(parents=True, exist_ok=True)
  5663. try:
  5664. if ftp_retry_enabled:
  5665. downloaded = await with_ftp_retry(
  5666. download_file_async,
  5667. printer.ip_address,
  5668. printer.access_code,
  5669. remote_full_path,
  5670. temp_path,
  5671. timeout=ftp_timeout,
  5672. socket_timeout=ftp_timeout,
  5673. printer_model=printer.model,
  5674. max_retries=ftp_retry_count,
  5675. retry_delay=ftp_retry_delay,
  5676. operation_name=f"Download 3MF from {remote_full_path}",
  5677. cooloff_ip=printer.ip_address,
  5678. )
  5679. else:
  5680. downloaded = await download_file_async(
  5681. printer.ip_address,
  5682. printer.access_code,
  5683. remote_full_path,
  5684. temp_path,
  5685. timeout=ftp_timeout,
  5686. socket_timeout=ftp_timeout,
  5687. printer_model=printer.model,
  5688. )
  5689. except Exception as e:
  5690. logger.debug("Failed to download %s: %s", remote_full_path, e)
  5691. continue
  5692. if downloaded:
  5693. downloaded_filename = fname
  5694. logger.info("Found and downloaded: %s", remote_full_path)
  5695. cache_3mf_download(printer_id, fname, temp_path)
  5696. break
  5697. # The listing named the file, so it is on the card;
  5698. # only the transfer failed (#3063).
  5699. ftp_transfer_failed = True
  5700. walk_transfer_failed = True
  5701. # A "Print again" on the printer's screen names no file Bambuddy can
  5702. # find: only its history copy under a generic name, and the 3MF's
  5703. # internal title (#3307). When an earlier run of the same job was
  5704. # archived, its 3MF is the one being printed again. Not when the
  5705. # directory walk listed a file for this print and only its transfer
  5706. # failed: that file is on the card, and the retry fetches it. A
  5707. # cool-off or a failed sweep path is no such evidence -- the retry
  5708. # tries the same names, which this print's file never had.
  5709. reused_3mf_from: int | None = None
  5710. if not downloaded_filename and not walk_transfer_failed:
  5711. reported_layers = raw_mqtt.get("total_layer_num")
  5712. earlier = await _archived_3mf_for_repeat(
  5713. db,
  5714. printer_id,
  5715. subtask_name,
  5716. filename,
  5717. reported_layers if isinstance(reported_layers, int) and reported_layers > 0 else None,
  5718. )
  5719. if earlier is not None:
  5720. reuse_name = Path(earlier.filename or "").name or f"archive_{earlier.id}.3mf"
  5721. # Per printer: two printers reprinting the same job at once
  5722. # must not write one temp file (archive_print names the
  5723. # archive after original_filename, not this path).
  5724. reuse_path = app_settings.archive_dir / "temp" / f"reprint_{printer_id}_{reuse_name}"
  5725. try:
  5726. reuse_path.parent.mkdir(parents=True, exist_ok=True)
  5727. await asyncio.to_thread(shutil.copyfile, app_settings.base_dir / earlier.file_path, reuse_path)
  5728. except OSError as e:
  5729. # A copy that fails leaves the print the fallback archive
  5730. # it would have had, not no archive at all.
  5731. logger.warning("Could not copy the 3MF of archive %s for printer %s: %s", earlier.id, printer_id, e)
  5732. else:
  5733. temp_path = reuse_path
  5734. downloaded_filename = reuse_name
  5735. reused_3mf_from = earlier.id
  5736. cache_3mf_download(printer_id, reuse_name, temp_path)
  5737. logger.info(
  5738. "No 3MF on printer %s for %s (%s); using the 3MF of archive %s, an earlier run of the same job",
  5739. printer_id,
  5740. subtask_name,
  5741. filename,
  5742. earlier.id,
  5743. )
  5744. # Validate the downloaded 3MF actually matches the plate that's running
  5745. # (#1204): subtask_name lags across consecutive plates of the same model,
  5746. # so the first FTP candidate (built from subtask_name) can land on the
  5747. # previous plate's still-resident upload. Cross-check the slice_info
  5748. # plate index against the plate parsed from gcode_file (always fresh —
  5749. # it's the field whose change triggered this callback).
  5750. if downloaded_filename and temp_path:
  5751. expected_plate = parse_plate_id(filename)
  5752. actual_plate = peek_plate_index_in_3mf(temp_path) if expected_plate is not None else None
  5753. if expected_plate is not None and actual_plate is not None and actual_plate != expected_plate:
  5754. logger.warning(
  5755. "[CALLBACK] 3MF plate mismatch: downloaded %s reports plate %s but printer is "
  5756. "running plate %s — subtask_name=%r appears stale, retrying with corrected name",
  5757. downloaded_filename,
  5758. actual_plate,
  5759. expected_plate,
  5760. subtask_name,
  5761. )
  5762. corrected_subtask = swap_plate_suffix(subtask_name, expected_plate)
  5763. retry_succeeded = False
  5764. if corrected_subtask and corrected_subtask != subtask_name:
  5765. for try_filename in (f"{corrected_subtask}.gcode.3mf", f"{corrected_subtask}.3mf"):
  5766. retry_temp_path = app_settings.archive_dir / "temp" / try_filename
  5767. retry_temp_path.parent.mkdir(parents=True, exist_ok=True)
  5768. for remote_path in (
  5769. f"/{try_filename}",
  5770. f"/cache/{try_filename}",
  5771. f"/model/{try_filename}",
  5772. f"/data/{try_filename}",
  5773. f"/data/Metadata/{try_filename}",
  5774. ):
  5775. try:
  5776. if ftp_retry_enabled:
  5777. downloaded = await with_ftp_retry(
  5778. download_file_async,
  5779. printer.ip_address,
  5780. printer.access_code,
  5781. remote_path,
  5782. retry_temp_path,
  5783. timeout=ftp_timeout,
  5784. socket_timeout=ftp_timeout,
  5785. printer_model=printer.model,
  5786. max_retries=ftp_retry_count,
  5787. retry_delay=ftp_retry_delay,
  5788. operation_name=f"Re-download 3MF from {remote_path}",
  5789. cooloff_ip=printer.ip_address,
  5790. non_retry_exceptions=(FileNotOnPrinterError,),
  5791. )
  5792. else:
  5793. downloaded = await download_file_async(
  5794. printer.ip_address,
  5795. printer.access_code,
  5796. remote_path,
  5797. retry_temp_path,
  5798. timeout=ftp_timeout,
  5799. socket_timeout=ftp_timeout,
  5800. printer_model=printer.model,
  5801. )
  5802. if downloaded and peek_plate_index_in_3mf(retry_temp_path) == expected_plate:
  5803. logger.info(
  5804. "[CALLBACK] Re-download succeeded with corrected name %s "
  5805. "(plate %s) — replacing wrong file",
  5806. try_filename,
  5807. expected_plate,
  5808. )
  5809. try:
  5810. temp_path.unlink(missing_ok=True)
  5811. except OSError:
  5812. pass
  5813. temp_path = retry_temp_path
  5814. downloaded_filename = try_filename
  5815. subtask_name = corrected_subtask
  5816. cache_3mf_download(printer_id, try_filename, temp_path)
  5817. retry_succeeded = True
  5818. break
  5819. elif downloaded:
  5820. # Wrong plate again — discard and keep trying
  5821. try:
  5822. retry_temp_path.unlink(missing_ok=True)
  5823. except OSError:
  5824. pass
  5825. except FileNotOnPrinterError:
  5826. continue
  5827. except Exception as e:
  5828. logger.debug("Re-download failed for %s: %s", remote_path, e)
  5829. if retry_succeeded:
  5830. break
  5831. # If the retry didn't find a matching file, drop the wrong 3MF
  5832. # so the no-3MF fallback below creates an archive whose name
  5833. # at least reflects the right plate.
  5834. if not retry_succeeded:
  5835. logger.warning(
  5836. "[CALLBACK] Could not re-download correct plate %s — falling back to no-3MF archive",
  5837. expected_plate,
  5838. )
  5839. try:
  5840. temp_path.unlink(missing_ok=True)
  5841. except OSError:
  5842. pass
  5843. temp_path = None
  5844. downloaded_filename = None
  5845. # Whatever the sweep's transport did earlier, it is not why
  5846. # this archive ends up empty: a 3MF downloaded fine, it was
  5847. # just the wrong plate. Retrying would re-fetch that same
  5848. # contradicted file under the same stale names and hand it
  5849. # to _recover_fallback_archive, which checks that a
  5850. # candidate is a readable 3MF but not which plate it is --
  5851. # so the row would be filled in with another plate's
  5852. # filament and cost, the exact swap #2957 removed (#3063).
  5853. ftp_transfer_failed = False
  5854. # Disown the name for *lookups*: it has just been shown to
  5855. # fetch another plate's 3MF, and it keys `_active_prints`
  5856. # below, where the cover endpoint's own download of that
  5857. # same name would find this archive and fill it in with the
  5858. # file we are discarding here.
  5859. #
  5860. # Keep it for the *title*, which is a separate question.
  5861. # ``swap_plate_suffix`` returns None both for a name that
  5862. # carries no "- Plate N" / "_plate_N" suffix and for no
  5863. # name at all, and those are not the same situation: a name
  5864. # without a suffix holds no stale plate number to be wrong
  5865. # about. Blanking both uses at once dropped the project
  5866. # name too, and the row fell through to the gcode_file path
  5867. # titled "plate_1" though the real name was in hand.
  5868. # #1204's own premise is consecutive plates *of the same
  5869. # model*, so the project part is right either way (#3126).
  5870. display_name_after_plate_reject = corrected_subtask or subtask_name or None
  5871. subtask_name = corrected_subtask or ""
  5872. # Deleted while the 3MF was being fetched (that can take minutes): no
  5873. # archive, timelapse or tracking for a printer that will never report
  5874. # the print's end. forget_printer again: the download registered its
  5875. # 3MF cache entry after the delete cleared the id. (A delete committing
  5876. # while the archive is written is caught after its commit.)
  5877. if await _printer_gone(db, printer_id):
  5878. logger.info("Printer %s was deleted during the print-start download; not archiving", printer_id)
  5879. await forget_printer(printer_id)
  5880. return
  5881. if not downloaded_filename or not temp_path:
  5882. logger.warning("Could not find 3MF file for print: %s", filename or subtask_name)
  5883. # Create a fallback archive without 3MF data so the print is still tracked
  5884. # This commonly happens with P1S/A1 printers where FTP has file size limitations
  5885. try:
  5886. from backend.app.models.archive import PrintArchive
  5887. # Why the card is empty. The two temporary causes outrank the
  5888. # storage verdict because they say the sweep never got a fair
  5889. # answer: a cool-off skipped it at the transport, and a failed
  5890. # transfer reached the printer but never finished. Either way
  5891. # the file is still on the card, so reporting where the printer
  5892. # files its jobs would describe a setting that is not the
  5893. # problem (#2957, #3063).
  5894. if blocked_by_ftps_cooloff:
  5895. no_3mf_reason = REASON_FTPS_COOLOFF
  5896. elif storage.reachable and ftp_transfer_failed:
  5897. no_3mf_reason = REASON_FTP_TRANSFER_FAILED
  5898. else:
  5899. no_3mf_reason = storage.reason
  5900. # Derive print name from subtask_name or filename. The
  5901. # plate guard's disowned name comes second: it is a real name
  5902. # for a real print, and only the gcode_file path is left
  5903. # otherwise -- which titles the row "plate_1" (#3126).
  5904. print_name = subtask_name or display_name_after_plate_reject or filename
  5905. if print_name:
  5906. # Clean up the name (remove extensions, path parts)
  5907. print_name = print_name.split("/")[-1]
  5908. print_name = print_name.replace(".gcode.3mf", "").replace(".gcode", "").replace(".3mf", "")
  5909. else:
  5910. print_name = "Unknown Print"
  5911. # Recover estimated print time from MQTT (best-effort for notifications)
  5912. fallback_print_time = None
  5913. mqtt_remaining = data.get("remaining_time")
  5914. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  5915. fallback_print_time = int(mqtt_remaining)
  5916. if fallback_print_time is None:
  5917. mc_remaining = (data.get("raw_data") or {}).get("mc_remaining_time")
  5918. if mc_remaining and isinstance(mc_remaining, (int, float)) and mc_remaining > 0:
  5919. fallback_print_time = int(mc_remaining * 60)
  5920. # Best-effort filament metadata from MQTT — see
  5921. # _extract_filament_data_from_mqtt. Without this the fallback
  5922. # archive's filament fields stayed NULL even though the AMS
  5923. # state at print start was sitting right there in `data`.
  5924. # The slicer's ams_mapping (when present) narrows the result
  5925. # to slots actually used by the print (#1533).
  5926. mqtt_filament_meta = _extract_filament_data_from_mqtt(
  5927. data, _get_start_ams_mapping(data, printer_id, None)
  5928. )
  5929. # Create minimal archive entry
  5930. fallback_archive = PrintArchive(
  5931. printer_id=printer_id,
  5932. filename=filename or f"{print_name}.3mf",
  5933. file_path="", # Empty - no 3MF file available
  5934. file_size=0,
  5935. print_name=print_name,
  5936. print_time_seconds=fallback_print_time,
  5937. status="printing",
  5938. started_at=detected_at,
  5939. subtask_id=subtask_id,
  5940. confirm_requested=await _ask_outcome_for_external_print(db, printer_id, subtask_name),
  5941. filament_type=mqtt_filament_meta.get("filament_type"),
  5942. filament_color=mqtt_filament_meta.get("filament_color"),
  5943. extra_data={
  5944. "no_3mf_available": True,
  5945. # Why the card is empty, when we know -- see above. The
  5946. # banner reads this to stop telling H2/P2 owners to
  5947. # switch on a setting that is already on and would not
  5948. # have helped (#2780).
  5949. "no_3mf_reason": no_3mf_reason,
  5950. "original_subtask": subtask_name or display_name_after_plate_reject or "",
  5951. "_print_data": data,
  5952. },
  5953. )
  5954. db.add(fallback_archive)
  5955. try:
  5956. await db.commit()
  5957. except IntegrityError:
  5958. # Not "Failed to create fallback archive" below, and no
  5959. # start notification for a printer that is gone.
  5960. if await _refused_for_deleted_printer(db, printer_id, "its fallback archive was written"):
  5961. return
  5962. raise
  5963. await db.refresh(fallback_archive)
  5964. logger.info("Created fallback archive %s for %s (no 3MF available)", fallback_archive.id, print_name)
  5965. if await _drop_archive_of_deleted_printer(db, printer_id, fallback_archive.id):
  5966. return
  5967. _maybe_start_layer_timelapse(printer, printer_id, fallback_archive.id)
  5968. # Track as active print
  5969. _bind_active_print(printer_id, fallback_archive.id, fallback_archive.filename, filename, subtask_name)
  5970. await _open_run_for(fallback_archive.id, printer_id, subtask_id)
  5971. # Record starting energy if smart plug available (#941: persisted column)
  5972. await _record_energy_start(fallback_archive, printer_id, db, context="fallback")
  5973. # Send WebSocket notification
  5974. await ws_manager.send_archive_created(
  5975. {
  5976. "id": fallback_archive.id,
  5977. "printer_id": fallback_archive.printer_id,
  5978. "filename": fallback_archive.filename,
  5979. "print_name": fallback_archive.print_name,
  5980. "status": fallback_archive.status,
  5981. }
  5982. )
  5983. # MQTT relay - publish archive created
  5984. try:
  5985. await mqtt_relay.on_archive_created(
  5986. archive_id=fallback_archive.id,
  5987. print_name=fallback_archive.print_name,
  5988. printer_name=printer.name,
  5989. status=fallback_archive.status,
  5990. )
  5991. except Exception:
  5992. pass # Don't fail if MQTT fails
  5993. # Store Spoolman tracking data (may not work for fallback since no 3MF)
  5994. try:
  5995. await _store_spoolman_print_data(
  5996. printer_id,
  5997. fallback_archive.id,
  5998. fallback_archive.file_path,
  5999. db,
  6000. printer_manager,
  6001. ams_mapping=_get_start_ams_mapping(data, printer_id, fallback_archive.id),
  6002. plate_id=_get_start_plate_id(printer_id, fallback_archive.id),
  6003. )
  6004. except Exception as e:
  6005. logger.debug("[SPOOLMAN] Could not store tracking for fallback archive: %s", e)
  6006. # Both temporary give-ups are worth coming back for, and for
  6007. # the same reason: the file is on the printer and the last look
  6008. # failed at the transport rather than finding nothing. One waits
  6009. # out the handshake block (#2957), the other waits for the
  6010. # printer to stop being busy (#3063). Deliberately not scheduled
  6011. # for a storage verdict: a file on internal eMMC will not appear
  6012. # at any FTPS path however long we wait, and retrying it is
  6013. # exactly the sweep #2780 removed.
  6014. if no_3mf_reason in (REASON_FTPS_COOLOFF, REASON_FTP_TRANSFER_FAILED) and possible_names:
  6015. # `possible_names`, not the raw MQTT strings: it is the exact
  6016. # list this flow just tried, already stripped of any path
  6017. # (`filename` arrives as "/data/Metadata/plate_1.gcode" on
  6018. # some firmware) and deduped.
  6019. _schedule_fallback_3mf_retry(
  6020. printer_id=printer_id,
  6021. archive_id=fallback_archive.id,
  6022. filenames=list(possible_names),
  6023. reason=no_3mf_reason,
  6024. )
  6025. # Send notification without archive data (file not found)
  6026. if not notification_sent:
  6027. await _send_print_start_notification(printer_id, data, logger=logger)
  6028. # The same baseline the other two on_print_start branches take
  6029. # (#2704), and last for the same reason they are: it lists the
  6030. # printer's timelapse directory, so a slow card must not delay
  6031. # the _active_prints registration, the energy reading, the
  6032. # archive-created event or the start notification above it.
  6033. #
  6034. # This branch never took one, so every no-3MF archive reached
  6035. # completion with no baseline in memory and none on the row, and
  6036. # the completion scan fell into its "snapshot now" fallback --
  6037. # which runs after the printer has written the video, so the new
  6038. # file landed inside the baseline and no diff ever matched
  6039. # (#2957 follow-up).
  6040. #
  6041. # Skipped when the FTPS cool-off is what produced this fallback:
  6042. # the listing needs the same connection that just failed, so it
  6043. # could only record that the card was unreadable. The scan
  6044. # handles that case by refusing to choose between candidates.
  6045. if not blocked_by_ftps_cooloff:
  6046. await _capture_timelapse_baseline_at_start(
  6047. printer, printer_id, logger, archive_id=fallback_archive.id
  6048. )
  6049. await _forget_if_printer_gone(printer_id)
  6050. return
  6051. except Exception as e:
  6052. logger.error("Failed to create fallback archive: %s", e)
  6053. # Send notification without archive data (file not found)
  6054. if not notification_sent:
  6055. await _send_print_start_notification(printer_id, data, logger=logger)
  6056. return
  6057. # The archive of the same slice sent before, which this print adds a
  6058. # run to instead of a new archive (#1634). Held until its run is open.
  6059. joined_archive_id: int | None = None
  6060. try:
  6061. # Archive the file with status "printing"
  6062. service = ArchiveService(db)
  6063. start_print_data = {
  6064. **data,
  6065. "status": "printing",
  6066. # Where the 3MF came from when the printer had none to
  6067. # give (#3307), for whoever reads this archive later.
  6068. **({"_3mf_from_archive": reused_3mf_from} if reused_3mf_from is not None else {}),
  6069. }
  6070. # The reused copy's temp name carries the printer id; a
  6071. # download's is the file's own name, as before.
  6072. start_filename = downloaded_filename if reused_3mf_from is not None else None
  6073. try:
  6074. claimed = await _claim_slicer_resend_archive(db, printer_id, temp_path)
  6075. if claimed is not None:
  6076. joined, file_in_use = claimed
  6077. joined_archive_id = joined.id
  6078. archive = await _start_slicer_resend_run(
  6079. db,
  6080. joined,
  6081. printer_id,
  6082. source=temp_path,
  6083. print_data=start_print_data,
  6084. subtask_id=subtask_id,
  6085. started_at=detected_at,
  6086. original_filename=start_filename,
  6087. logger=logger,
  6088. file_in_use=file_in_use,
  6089. )
  6090. if archive is None:
  6091. return
  6092. else:
  6093. archive = await service.archive_print(
  6094. printer_id=printer_id,
  6095. source_file=temp_path,
  6096. print_data=start_print_data,
  6097. subtask_id=subtask_id,
  6098. started_at=detected_at,
  6099. original_filename=start_filename,
  6100. )
  6101. except IntegrityError:
  6102. # archive_print removed the folder it had copied the 3MF into.
  6103. if await _refused_for_deleted_printer(db, printer_id, "its archive was written"):
  6104. return
  6105. raise
  6106. if archive:
  6107. # Ask-for-outcome for a print Bambuddy did not dispatch (#1898).
  6108. # Set on the row rather than passed to archive_print, which also
  6109. # serves the queue dispatcher — there the queue item decides.
  6110. # Guarded because this branch has no ``except``: an unhandled
  6111. # write error would take the _active_prints registration, the
  6112. # start notification, the energy reading and the timelapse
  6113. # baseline below it with it, and a missing prompt is by far the
  6114. # cheaper failure.
  6115. archive_id = archive.id
  6116. try:
  6117. if await _ask_outcome_for_external_print(db, printer_id, subtask_name):
  6118. archive.confirm_requested = True
  6119. await db.commit()
  6120. except Exception as e:
  6121. logger.warning("Could not flag archive %s for the outcome prompt: %s", archive_id, e)
  6122. # The rollback expires every loaded object, and on an async
  6123. # session reading one afterwards raises instead of
  6124. # reloading, so the two this branch goes on to use are
  6125. # fetched again. The archive itself was committed by
  6126. # archive_print; only the flag is lost.
  6127. try:
  6128. await db.rollback()
  6129. archive = await db.get(PrintArchive, archive_id)
  6130. printer = await db.get(Printer, printer_id)
  6131. except Exception as reload_error:
  6132. logger.warning(
  6133. "Could not reload archive %s after the failed flag write: %s", archive_id, reload_error
  6134. )
  6135. if archive is None:
  6136. # Deleted meanwhile: nothing to register the print for.
  6137. return
  6138. # Not a joined archive: it is not this print's to remove, and
  6139. # its adoption already checked the printer.
  6140. if joined_archive_id is None and await _drop_archive_of_deleted_printer(db, printer_id, archive_id):
  6141. return
  6142. # Track this active print (use both original filename and downloaded filename)
  6143. _bind_active_print(printer_id, archive_id, downloaded_filename, filename, subtask_name)
  6144. await _open_run_for(archive_id, printer_id, subtask_id)
  6145. if joined_archive_id is not None:
  6146. logger.info(
  6147. "Added the print of %s to archive %s: the same G-code was sent again",
  6148. downloaded_filename,
  6149. archive_id,
  6150. )
  6151. else:
  6152. logger.info("Created archive %s for %s", archive.id, downloaded_filename)
  6153. _maybe_start_layer_timelapse(printer, printer_id, archive.id)
  6154. # Record starting energy from smart plug if available (#941: persisted column)
  6155. await _record_energy_start(archive, printer_id, db, context="auto-archive")
  6156. archive_event = {
  6157. "id": archive.id,
  6158. "printer_id": archive.printer_id,
  6159. "filename": archive.filename,
  6160. "print_name": archive.print_name,
  6161. "status": archive.status,
  6162. }
  6163. if joined_archive_id is not None:
  6164. await ws_manager.send_archive_updated(archive_event)
  6165. else:
  6166. await ws_manager.send_archive_created(archive_event)
  6167. # MQTT relay - publish archive created
  6168. try:
  6169. await mqtt_relay.on_archive_created(
  6170. archive_id=archive.id,
  6171. print_name=archive.print_name or "",
  6172. printer_name=printer.name if printer else "",
  6173. status=archive.status,
  6174. )
  6175. except Exception:
  6176. pass # Don't fail if MQTT fails
  6177. # Send notification with archive data (new archive created)
  6178. if not notification_sent:
  6179. archive_data = {
  6180. "print_time_seconds": _run_print_time_seconds(archive, printer_id),
  6181. "created_by_id": archive.created_by_id,
  6182. }
  6183. await _send_print_start_notification(printer_id, data, archive_data, logger)
  6184. # Extract printable objects for skip object functionality
  6185. try:
  6186. from backend.app.services.archive import extract_printable_objects_from_3mf
  6187. client = printer_manager.get_client(printer_id)
  6188. if client:
  6189. with open(temp_path, "rb") as f:
  6190. threemf_data = f.read()
  6191. # Extract with positions for UI overlay, scoped to the
  6192. # plate that is printing — an all-plates 3MF carries
  6193. # every plate's objects (#2522).
  6194. printable_objects, bbox_all = extract_printable_objects_from_3mf(
  6195. threemf_data,
  6196. plate_number=resolve_plate_id(client.state),
  6197. include_positions=True,
  6198. )
  6199. if printable_objects:
  6200. # Store objects in printer state
  6201. client.state.printable_objects = printable_objects
  6202. client.state.printable_objects_bbox_all = bbox_all
  6203. client.state.skipped_objects = [] # Reset skipped objects for new print
  6204. logger.info(
  6205. "Loaded %s printable objects for printer %s", len(printable_objects), printer_id
  6206. )
  6207. except Exception as e:
  6208. logger.debug("Failed to extract printable objects: %s", e)
  6209. # From the archived copy: the downloaded one may be removed
  6210. # below while the pauses are still being read.
  6211. if archive is not None and archive.file_path:
  6212. _load_pauses_from_3mf(app_settings.base_dir / archive.file_path, printer_id)
  6213. # Store Spoolman tracking data for per-filament usage reporting
  6214. try:
  6215. await _store_spoolman_print_data(
  6216. printer_id,
  6217. archive.id,
  6218. archive.file_path,
  6219. db,
  6220. printer_manager,
  6221. ams_mapping=_get_start_ams_mapping(data, printer_id, archive_id),
  6222. plate_id=_get_start_plate_id(printer_id, archive_id),
  6223. )
  6224. except Exception as e:
  6225. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  6226. # Price the print from its spools now rather than only at completion (#3261)
  6227. _schedule_archive_cost_estimate(
  6228. printer_id,
  6229. archive.id,
  6230. printer_manager,
  6231. ams_mapping=_get_start_ams_mapping(data, printer_id, archive_id),
  6232. plate_id=_get_start_plate_id(printer_id, archive_id),
  6233. session_factory=async_session,
  6234. )
  6235. # Capture timelapse file baseline for snapshot-diff on completion
  6236. await _capture_timelapse_baseline_at_start(printer, printer_id, logger, archive_id=archive.id)
  6237. await _forget_if_printer_gone(printer_id)
  6238. finally:
  6239. if joined_archive_id is not None:
  6240. from backend.app.services import archive_work
  6241. archive_work.release(joined_archive_id)
  6242. # Keep temp_path around until print completes so the cover endpoint
  6243. # can reuse it (#972). Cache eviction in on_print_complete deletes
  6244. # the file. If the cache entry was evicted early (file vanished),
  6245. # clean up any stragglers here to avoid leaking disk on retries.
  6246. cached_now = get_cached_3mf(printer_id, downloaded_filename) if downloaded_filename else None
  6247. if temp_path and temp_path.exists() and cached_now != temp_path:
  6248. temp_path.unlink()
  6249. _TIMELAPSE_VIDEO_EXTENSIONS = (".mp4", ".avi")
  6250. _TIMELAPSE_DIRECTORIES = ("/timelapse", "/timelapse/video", "/record", "/recording")
  6251. # A baseline that could not be read at print start is read again on this
  6252. # schedule while the print runs (#3412). The printer writes the video only once
  6253. # the print has ended, so a reading taken mid-print is as good as one taken at
  6254. # the start. The interval is shorter than the FTPS cool-offs (300 s), which are
  6255. # what usually makes the card unreadable, so the first reading after one ends
  6256. # is not long behind; while a cool-off runs a round costs nothing, as no
  6257. # connection is opened. The cap bounds a print that never reports an end.
  6258. #
  6259. # A round that could not read the card doubles the wait before the next, up to
  6260. # the ceiling, and after that many failed rounds in a row the re-read gives up
  6261. # and leaves the baseline unread for the completion scan to handle (#3474). Each
  6262. # connection to a hung file service can leave a session behind on the printer.
  6263. _TIMELAPSE_BASELINE_REREAD_INTERVAL_SECONDS = 120.0
  6264. _TIMELAPSE_BASELINE_REREAD_MAX_INTERVAL_SECONDS = 1800.0
  6265. _TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS = 5
  6266. _TIMELAPSE_BASELINE_REREAD_MAX_ATTEMPTS = 720
  6267. # Poll schedule for the post-print timelapse scan (#2704). Module-level so
  6268. # tests can shrink them without waiting out real delays.
  6269. #
  6270. # This replaced a fixed [5, 10, 20, 30] retry ladder, i.e. roughly 65 s of
  6271. # looking. Across 247 support bundles the attempt that found the video was #1
  6272. # 272 times, then 17 / 13 / 13 — a flat tail against the cutoff rather than a
  6273. # decaying one, which is the signature of a budget that expires while files are
  6274. # still arriving. 457 scans were scheduled and only 262 ever attached. Big
  6275. # prints make big videos and the printer writes them after the print ends, so
  6276. # the poll now runs for minutes and costs one FTP LIST per round.
  6277. _TIMELAPSE_SCAN_FIRST_DELAY_SECONDS: float = 5.0
  6278. _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS: float = 30.0
  6279. _TIMELAPSE_SCAN_TIMEOUT_SECONDS: float = 900.0
  6280. def _timelapse_scan_max_attempts() -> int:
  6281. """Round cap for the poll, derived from the wall-clock budget.
  6282. The deadline alone is not a sufficient bound: it assumes each round really
  6283. waits, which stops being true the moment ``asyncio.sleep`` is patched out,
  6284. and an FTP list that fails immediately would otherwise spin against the
  6285. printer at full speed for the whole window. Whichever bound is reached
  6286. first ends the poll.
  6287. """
  6288. if _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS <= 0:
  6289. # A zero interval makes the wall-clock budget meaningless; fall back to
  6290. # the round count the production interval would have given.
  6291. return 32
  6292. return max(1, int(_TIMELAPSE_SCAN_TIMEOUT_SECONDS // _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS) + 1)
  6293. async def _read_timelapse_baseline(printer) -> tuple[set[str], bool]:
  6294. """The video names on *printer*'s card, and whether the card was read.
  6295. ``_list_timelapse_videos`` answers ``[]`` when its connect fails rather
  6296. than raising, so a card behind the FTPS handshake cool-off, or one whose
  6297. listing timed out, is indistinguishable from one holding no videos.
  6298. Everywhere that only wants to know "is there a video yet" the difference
  6299. does not matter -- both mean "not yet, retry".
  6300. It matters where an empty listing is recorded as a *baseline*. Recording
  6301. "the card held nothing" for a card that was never read means every video on
  6302. it counts as new, and the completion scan then attaches a stale video to
  6303. this print and deletes it from the printer (#2957 follow-up, #3412). So an
  6304. card counts as read only when every directory walked answered, with a
  6305. listing or with "no such directory". One that failed may be the one
  6306. holding the videos.
  6307. One walk, one connection per directory. It used to list the card with
  6308. ``_list_timelapse_videos`` first and walk the same directories again when
  6309. that came back empty, which against a hung file service was eight dead
  6310. connections a round instead of one (#3474). A directory that fails does
  6311. not end the walk unless the failure put the printer in a cool-off: a later
  6312. one holding videos still answers the question.
  6313. """
  6314. from backend.app.services.bambu_ftp import ftps_background_blocked, list_files_result_async
  6315. ip_address = getattr(printer, "ip_address", None)
  6316. printer_id = getattr(printer, "id", None)
  6317. if not ip_address:
  6318. return set(), True
  6319. # No card in the slot: nothing to read, and nothing a video could land on.
  6320. if printer_id is not None and not external_storage_present(printer_manager.get_status(printer_id)):
  6321. return set(), True
  6322. read = True
  6323. for timelapse_path in _TIMELAPSE_DIRECTORIES:
  6324. if ftps_background_blocked(ip_address):
  6325. # Below any path: every remaining one would fail the same way.
  6326. return set(), False
  6327. listing = await list_files_result_async(
  6328. ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  6329. )
  6330. if not (listing.available or listing.missing):
  6331. read = False
  6332. continue
  6333. videos = [
  6334. f
  6335. for f in listing.files
  6336. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  6337. ]
  6338. if videos:
  6339. return {f.get("name", "") for f in videos}, True
  6340. return set(), read
  6341. async def _list_timelapse_videos(printer) -> tuple[list[dict], str | None]:
  6342. """List video files from printer's timelapse directory.
  6343. Finds MP4 (X1/A1 series) and AVI (P1 series) timelapse files.
  6344. Returns (video_files, found_path) where video_files is a list of file dicts
  6345. and found_path is the directory where they were found, or ([], None).
  6346. An empty return does not distinguish "no videos" from "could not read the
  6347. card" — see :func:`_read_timelapse_baseline`, which the two baseline
  6348. callers use instead.
  6349. """
  6350. from backend.app.services.bambu_ftp import ftps_background_blocked, list_files_async
  6351. logger = logging.getLogger(__name__)
  6352. # No card in the slot means no /timelapse to walk — four connections that
  6353. # can only fail, on a path whose failures are swallowed and so would go on
  6354. # costing time silently forever (#2780).
  6355. #
  6356. # ``getattr`` rather than ``printer.id``: every dereference below happens
  6357. # inside the loop's own try/except, so a caller that passed something
  6358. # unexpected used to get an empty listing rather than an exception. Keep
  6359. # that, instead of making this gate the first thing that can raise here.
  6360. printer_id = getattr(printer, "id", None)
  6361. if printer_id is not None and not external_storage_present(printer_manager.get_status(printer_id)):
  6362. logger.debug("[TIMELAPSE] Skipping the scan for printer %s: it reports no external storage", printer_id)
  6363. return [], None
  6364. for timelapse_path in _TIMELAPSE_DIRECTORIES:
  6365. if ftps_background_blocked(getattr(printer, "ip_address", None) or ""):
  6366. # The printer's file service is refusing or hung: every remaining
  6367. # path is another connection it may keep a session for (#3474).
  6368. logger.debug("[TIMELAPSE] Skipping the scan for printer %s: its file service is cooling off", printer_id)
  6369. return [], None
  6370. try:
  6371. found_files = await list_files_async(
  6372. printer.ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  6373. )
  6374. if found_files:
  6375. video_files = [
  6376. f
  6377. for f in found_files
  6378. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  6379. ]
  6380. if video_files:
  6381. return video_files, timelapse_path
  6382. except Exception as e:
  6383. logger.debug("[TIMELAPSE] Path %s failed: %s", timelapse_path, e)
  6384. continue
  6385. return [], None
  6386. async def _capture_timelapse_baseline_at_start(
  6387. printer, printer_id: int, logger: logging.Logger, archive_id: int | None = None
  6388. ) -> None:
  6389. """Snapshot the printer's timelapse directory at print start so the
  6390. completion-time scan can pick the new file by set-difference.
  6391. Must be called from every on_print_start path that proceeds to a real
  6392. print — both the new-archive branch and the expected-archive branch (which
  6393. queue / VP-dispatched prints take). Without a baseline,
  6394. _scan_for_timelapse_with_retries falls into its "take baseline now"
  6395. fallback that runs AFTER the new MP4 has already landed on the SD card,
  6396. so the new file ends up in the "baseline" set and no diff ever matches.
  6397. Bambu printers in LAN-only mode don't sync NTP, so mtime ordering is
  6398. unreliable — the snapshot-diff approach sidesteps that entirely.
  6399. When ``archive_id`` is known the baseline is also written to the archive
  6400. row, so it survives a restart and the manual "Scan for Timelapse" button
  6401. can run the same diff instead of falling back to clock-based matching
  6402. (#2704). Only baselines taken at print start are persisted — one taken at
  6403. completion already contains the new video and would poison a later scan.
  6404. """
  6405. names: set[str] | None = None
  6406. unread = False
  6407. _timelapse_baselines_unread.discard(printer_id)
  6408. try:
  6409. names, read = await _read_timelapse_baseline(printer)
  6410. unread = not read
  6411. _timelapse_baselines[printer_id] = names
  6412. if unread:
  6413. # Recorded anyway, deliberately, and marked. An empty baseline taken
  6414. # off a card we could not read is not evidence, but it is still the
  6415. # right *default*: Bambuddy deletes each video from the printer once
  6416. # it is attached, so the usual card holds exactly one video at
  6417. # completion and an empty baseline resolves it. Persisting NULL
  6418. # instead would send completion to take its own snapshot, by which
  6419. # point this print's video is on the card and would be swallowed by
  6420. # it. The mark keeps the scan from choosing between several videos
  6421. # (#3412), and the card is read again while the print runs.
  6422. _timelapse_baselines_unread.add(printer_id)
  6423. logger.warning(
  6424. "[TIMELAPSE] Baseline for printer %s taken while its card could not be read — "
  6425. "treating it as empty for now and reading it again during the print",
  6426. printer_id,
  6427. )
  6428. logger.info(
  6429. "[TIMELAPSE] Baseline at print start: %s video files for printer %s",
  6430. len(names),
  6431. printer_id,
  6432. )
  6433. except Exception as e:
  6434. logger.warning("[TIMELAPSE] Failed to capture baseline at print start: %s", e)
  6435. run_started_at: datetime | None = None
  6436. run_id: int | None = None
  6437. if archive_id is not None:
  6438. try:
  6439. async with async_session() as db:
  6440. from backend.app.models.archive import PrintArchive
  6441. from backend.app.services.print_log import open_run_on_printer
  6442. archive = await db.get(PrintArchive, archive_id)
  6443. if archive is not None:
  6444. # Written even when the listing failed, and then as NULL. A
  6445. # reprint reuses the archive row, so leaving the previous
  6446. # run's baseline in place would have the scan diff this print
  6447. # against the state of the printer before the *last* one —
  6448. # and a stale baseline reads as authoritative, where NULL
  6449. # correctly falls back to a fresh snapshot.
  6450. baseline = sorted(names) if names is not None else None
  6451. archive.timelapse_baseline = baseline
  6452. archive.timelapse_baseline_unread = unread
  6453. run_started_at = archive.started_at
  6454. # And on the run this print opened, whose scan reads it:
  6455. # the archive's is its latest start's, which another plate
  6456. # of it on another printer can take over.
  6457. run = await open_run_on_printer(db, printer_id)
  6458. if run is not None and run.archive_id == archive_id:
  6459. run.timelapse_baseline = baseline
  6460. run.timelapse_baseline_unread = unread
  6461. run_id = run.id
  6462. await db.commit()
  6463. except Exception as e:
  6464. # In-memory baseline still covers the normal completion path.
  6465. logger.warning("[TIMELAPSE] Failed to persist baseline for archive %s: %s", archive_id, e)
  6466. if unread and names is not None:
  6467. spawn_background_task(
  6468. _reread_unread_timelapse_baseline(printer_id, names, archive_id, run_started_at, run_id),
  6469. name=f"reread-timelapse-baseline-{printer_id}",
  6470. )
  6471. # Before the print has ended there is no video yet: a long PREPARE (heating,
  6472. # levelling) counts as much as the print itself.
  6473. _BASELINE_REREAD_STATES = _ACTIVE_PRINT_STATES | {"PREPARE", "SLICING"}
  6474. def _schedule_timelapse_scan(
  6475. archive_id: int, printer_id: int, *, plate_id: int | None, run_id: int | None = None
  6476. ) -> None:
  6477. """Start the completion scan with the print-start baseline: the names, and
  6478. whether they were taken off a card that could not be read. Taking them
  6479. ends any re-read still running for it. ``run_id`` is the print that
  6480. ended, whose video it looks for."""
  6481. unread = printer_id in _timelapse_baselines_unread
  6482. _timelapse_baselines_unread.discard(printer_id)
  6483. baseline = _timelapse_baselines.pop(printer_id, None)
  6484. from backend.app.services import archive_work
  6485. # Held while it looks: it finds its print by the archive's id, which a
  6486. # merge would move away (archive redesign P2).
  6487. spawn_background_task(
  6488. archive_work.held_while(
  6489. archive_id,
  6490. _scan_for_timelapse_with_retries(
  6491. archive_id, baseline, plate_id=plate_id, baseline_unread=unread, run_id=run_id
  6492. ),
  6493. ),
  6494. name=f"scan-timelapse-{archive_id}",
  6495. )
  6496. def _unread_baseline_round(printer_id: int, names: set[str]) -> bool | None:
  6497. """Whether to read an unread baseline again now: True to read, None to
  6498. skip this round, False once it is over -- the print it was taken for is
  6499. no longer the one on the printer, has stopped printing, or the baseline
  6500. has been read.
  6501. ``names`` is the very set object recorded at print start: completion pops
  6502. it and the next print records a new one, so identity marks the run. A
  6503. printer that is momentarily disconnected says nothing about its print, so
  6504. that only skips the round.
  6505. So does a print that is not recording a timelapse: completion only looks
  6506. for a video when one was recorded, so reading the card for it is a
  6507. connection for nothing (#3474). Skipped rather than ended, as the flag can
  6508. arrive after the print has started.
  6509. """
  6510. if _timelapse_baselines.get(printer_id) is not names or printer_id not in _timelapse_baselines_unread:
  6511. return False
  6512. state = printer_manager.get_status(printer_id)
  6513. if state is None or not state.connected:
  6514. return None
  6515. if state.state not in _BASELINE_REREAD_STATES:
  6516. return False
  6517. return True if state.timelapse else None
  6518. async def _reread_unread_timelapse_baseline(
  6519. printer_id: int,
  6520. names: set[str],
  6521. archive_id: int | None,
  6522. run_started_at: datetime | None,
  6523. run_id: int | None = None,
  6524. ) -> None:
  6525. """Read again a print-start baseline the card could not give (#3412).
  6526. The printer writes the video only once the print has ended, so the card
  6527. read at any point during the print holds exactly what it held at the
  6528. start. Each round stops once the print is no longer the one the baseline
  6529. was taken for or has stopped printing, and checks that again after the
  6530. listing returns, before anything is recorded.
  6531. A round that cannot read the card doubles the wait before the next one,
  6532. and enough of them in a row end the re-read (#3474): see
  6533. ``_TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS``.
  6534. """
  6535. from backend.app.models.archive import PrintArchive
  6536. from backend.app.models.printer import Printer
  6537. from backend.app.services.print_log import run_of
  6538. logger = logging.getLogger(__name__)
  6539. interval = _TIMELAPSE_BASELINE_REREAD_INTERVAL_SECONDS
  6540. failed_rounds = 0
  6541. for _ in range(_TIMELAPSE_BASELINE_REREAD_MAX_ATTEMPTS):
  6542. await asyncio.sleep(interval)
  6543. now = _unread_baseline_round(printer_id, names)
  6544. if now is False:
  6545. return
  6546. if now is None:
  6547. continue
  6548. try:
  6549. async with async_session() as db:
  6550. printer = await db.get(Printer, printer_id)
  6551. if printer is None:
  6552. return
  6553. fresh, read = await _read_timelapse_baseline(printer)
  6554. except Exception as e:
  6555. logger.debug("[TIMELAPSE] Re-reading the baseline for printer %s failed: %s", printer_id, e)
  6556. fresh, read = set(), False
  6557. if not read:
  6558. failed_rounds += 1
  6559. if failed_rounds >= _TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS:
  6560. logger.warning(
  6561. "[TIMELAPSE] Card of printer %s still unreadable after %s tries; no longer reading "
  6562. "the baseline for this print, it stays unread",
  6563. printer_id,
  6564. failed_rounds,
  6565. )
  6566. return
  6567. interval = min(interval * 2, _TIMELAPSE_BASELINE_REREAD_MAX_INTERVAL_SECONDS)
  6568. continue
  6569. now = _unread_baseline_round(printer_id, names)
  6570. if now is False:
  6571. return
  6572. if now is None:
  6573. # The printer dropped off while the listing was out, so nothing
  6574. # says it did not end meanwhile: keep the reading for a round that
  6575. # can vouch for it.
  6576. continue
  6577. names.update(fresh)
  6578. _timelapse_baselines_unread.discard(printer_id)
  6579. logger.info(
  6580. "[TIMELAPSE] Baseline for printer %s read during the print: %s video files",
  6581. printer_id,
  6582. len(fresh),
  6583. )
  6584. if archive_id is None:
  6585. return
  6586. try:
  6587. async with async_session() as db:
  6588. archive = await db.get(PrintArchive, archive_id)
  6589. # Only the run it was taken for: by now the row can belong to a
  6590. # reprint, whose baseline is its own.
  6591. if archive is not None and archive.started_at == run_started_at and archive.timelapse_baseline_unread:
  6592. archive.timelapse_baseline = sorted(names)
  6593. archive.timelapse_baseline_unread = False
  6594. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6595. if run is not None and run.timelapse_baseline_unread:
  6596. run.timelapse_baseline = sorted(names)
  6597. run.timelapse_baseline_unread = False
  6598. await db.commit()
  6599. except Exception as e:
  6600. # The in-memory baseline is read now, which covers completion
  6601. # unless Bambuddy restarts first.
  6602. logger.warning("[TIMELAPSE] Failed to persist the re-read baseline for archive %s: %s", archive_id, e)
  6603. return
  6604. async def _timelapse_still_wanted(db, archive, run_id: int | None, run_started_at: datetime | None) -> bool:
  6605. """Whether the print a timelapse is for still has none: its run, by id,
  6606. exists without a video and no later print took over (``superseded``).
  6607. Without a run, the archive is still on that print's start (a newer run's
  6608. video is its own) and has none."""
  6609. from backend.app.services.print_log import run_of, superseded
  6610. if run_id is not None:
  6611. run = await run_of(db, archive.id, run_id)
  6612. return run is not None and run.timelapse_path is None and not await superseded(db, run)
  6613. return archive.timelapse_path is None and archive.started_at == run_started_at
  6614. async def _scan_for_timelapse_with_retries(
  6615. archive_id: int,
  6616. baseline_names: set[str] | None = None,
  6617. *,
  6618. plate_id: int | None = None,
  6619. baseline_unread: bool = False,
  6620. run_id: int | None = None,
  6621. ):
  6622. """Poll the printer for this print's timelapse and attach it.
  6623. Snapshot diff, not timestamp matching: a printer in LAN-only mode cannot
  6624. reach Bambu's NTP server, so the clock behind both the filename and the FTP
  6625. mtime is arbitrarily wrong — one reporter's P1S was six and a half days out
  6626. (#2704). Comparing the current listing against the set of filenames that
  6627. existed when the print started needs no clock at all, because the printer
  6628. writes the video only once the print has ended.
  6629. Baseline precedence: the caller's in-memory set, then the one persisted on
  6630. the archive at print start, then a snapshot taken now. The last of those is
  6631. a poor substitute — by completion the new video may already be on the card,
  6632. in which case it lands in the "baseline" and no diff can ever match — but it
  6633. is all that is available for a print that began before Bambuddy started.
  6634. On success the video is deleted from the printer, which keeps ``/timelapse``
  6635. down to the unclaimed files and makes the next diff unambiguous.
  6636. ``baseline_unread`` marks the caller's baseline as taken off a card that
  6637. could not be read; a stored one carries the same mark on the archive.
  6638. ``run_id`` is the print that ended (archive redesign): the video is that
  6639. run's, the stored baseline is the run's own, and the poll ends once the
  6640. run has a video or is gone. Without it (no run was written), the run is
  6641. whichever the archive is on when the scan starts.
  6642. """
  6643. from backend.app.services.print_log import run_of
  6644. logger = logging.getLogger(__name__)
  6645. # Cleared when the baseline was taken off a card we could not read, so the
  6646. # attach step refuses to choose between several candidates (#2957, #3412).
  6647. baseline_trusted = True
  6648. # --- Phase 1: establish the baseline -------------------------------------
  6649. try:
  6650. async with async_session() as db:
  6651. from backend.app.models.printer import Printer
  6652. service = ArchiveService(db)
  6653. archive = await service.get_archive(archive_id)
  6654. if not archive:
  6655. logger.warning("[TIMELAPSE] Archive %s not found, aborting", archive_id)
  6656. return
  6657. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6658. if run_id is not None and run is None:
  6659. logger.info("[TIMELAPSE] Run %s of archive %s is gone, aborting", run_id, archive_id)
  6660. return
  6661. if run.timelapse_path if run is not None else archive.timelapse_path:
  6662. logger.info("[TIMELAPSE] Archive %s already has timelapse attached", archive_id)
  6663. return
  6664. # The printer the print ran on: plates of one archive can print on
  6665. # two printers, and the archive's is the one that started last.
  6666. scan_printer_id = run.printer_id if run is not None and run.printer_id else archive.printer_id
  6667. if not scan_printer_id:
  6668. logger.warning("[TIMELAPSE] Archive %s has no printer, aborting", archive_id)
  6669. return
  6670. # The run this scan is for. The archive row passes to a newer run
  6671. # when it prints again (the next plate, a reprint), and that run's
  6672. # video is its own.
  6673. run_started_at = run.started_at if run is not None else archive.started_at
  6674. # The baseline stored for it: the run's own, else the archive's
  6675. # while that is still this run's start's.
  6676. stored_baseline, stored_unread = None, False
  6677. if run is not None and run.timelapse_baseline is not None:
  6678. stored_baseline, stored_unread = run.timelapse_baseline, bool(run.timelapse_baseline_unread)
  6679. elif archive.started_at == run_started_at:
  6680. stored_baseline, stored_unread = archive.timelapse_baseline, archive.timelapse_baseline_unread
  6681. if baseline_names is not None and baseline_unread and stored_baseline is not None and not stored_unread:
  6682. # The card was read at print start, and the in-memory copy is
  6683. # one Bambuddy took after a restart off a card it could not
  6684. # read: the stored one is this run's and is evidence.
  6685. baseline_names = set(stored_baseline)
  6686. logger.info(
  6687. "[TIMELAPSE] Using stored baseline over an unread one: %s existing video files for archive %s",
  6688. len(baseline_names),
  6689. archive_id,
  6690. )
  6691. elif baseline_names is not None:
  6692. baseline_trusted = not baseline_unread
  6693. logger.info(
  6694. "[TIMELAPSE] Using print-start baseline: %s existing video files for archive %s",
  6695. len(baseline_names),
  6696. archive_id,
  6697. )
  6698. elif stored_baseline is not None:
  6699. # Persisted at print start — survives a restart mid-print.
  6700. baseline_names = set(stored_baseline)
  6701. baseline_trusted = not stored_unread
  6702. logger.info(
  6703. "[TIMELAPSE] Using stored baseline: %s existing video files for archive %s",
  6704. len(baseline_names),
  6705. archive_id,
  6706. )
  6707. else:
  6708. result = await db.execute(select(Printer).where(Printer.id == scan_printer_id))
  6709. printer = result.scalar_one_or_none()
  6710. if not printer:
  6711. logger.warning("[TIMELAPSE] Printer not found for archive %s, aborting", archive_id)
  6712. return
  6713. baseline_names, baseline_trusted = await _read_timelapse_baseline(printer)
  6714. if not baseline_trusted:
  6715. # The card is unreadable at the one moment a baseline has to
  6716. # be taken, so the empty listing means "we never looked",
  6717. # not "these are all new". Carry on with it anyway — the
  6718. # usual card holds exactly one video, which resolves
  6719. # correctly — but stop the poll from *choosing* between
  6720. # several, which is how a stale video got attached to this
  6721. # print and then deleted off the printer (#2957 follow-up).
  6722. logger.warning(
  6723. "[TIMELAPSE] Baseline for archive %s taken while printer %s's card could not be "
  6724. "read. A single new video still resolves; several will not be guessed between — "
  6725. "use Scan for Timelapse to pick one by hand",
  6726. archive_id,
  6727. scan_printer_id,
  6728. )
  6729. logger.info(
  6730. "[TIMELAPSE] Baseline snapshot (fallback): %s existing video files for archive %s",
  6731. len(baseline_names),
  6732. archive_id,
  6733. )
  6734. except Exception as e:
  6735. logger.warning("[TIMELAPSE] Failed to take baseline snapshot for archive %s: %s", archive_id, e)
  6736. return
  6737. # --- Phase 2: poll for a file that was not there when the print began -----
  6738. deadline = time.monotonic() + _TIMELAPSE_SCAN_TIMEOUT_SECONDS
  6739. max_attempts = _timelapse_scan_max_attempts()
  6740. seen_names: set[str] = set()
  6741. delay = _TIMELAPSE_SCAN_FIRST_DELAY_SECONDS
  6742. attempt = 0
  6743. while True:
  6744. await asyncio.sleep(delay)
  6745. delay = _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS
  6746. attempt += 1
  6747. try:
  6748. from backend.app.models.printer import Printer
  6749. # Read phase: fetch archive + printer in a short session and release
  6750. # the pooled connection BEFORE the FTP list/download below. Holding it
  6751. # across the FTP round-trips left one connection idle-in-transaction per
  6752. # in-flight scan (issue #2572).
  6753. async with async_session() as db:
  6754. from backend.app.services.print_log import claimed_timelapse_stems
  6755. service = ArchiveService(db)
  6756. archive = await service.get_archive(archive_id)
  6757. if not archive:
  6758. logger.warning("[TIMELAPSE] Archive %s not found, stopping poll", archive_id)
  6759. return
  6760. if not await _timelapse_still_wanted(db, archive, run_id, run_started_at):
  6761. logger.info(
  6762. "[TIMELAPSE] Archive %s has its video or is printing again, stopping the run's poll",
  6763. archive_id,
  6764. )
  6765. return
  6766. result = await db.execute(select(Printer).where(Printer.id == scan_printer_id))
  6767. printer = result.scalar_one_or_none()
  6768. if not printer:
  6769. logger.warning("[TIMELAPSE] Printer not found for archive %s, stopping poll", archive_id)
  6770. return
  6771. claimed = await claimed_timelapse_stems(db, scan_printer_id, archive_id=archive_id, run_id=run_id)
  6772. # I/O phase (no DB connection held): FTP list + download.
  6773. video_files, found_path = await _list_timelapse_videos(printer)
  6774. # The poll can run for dozens of rounds, so only narrate a round
  6775. # that saw something change. Repeating the whole listing every 30 s
  6776. # would bury the one interesting line in the support bundle.
  6777. names_now = {f.get("name", "") for f in video_files}
  6778. changed = attempt == 1 or names_now != seen_names
  6779. seen_names = names_now
  6780. speak = logger.info if changed else logger.debug
  6781. if video_files:
  6782. speak("[TIMELAPSE] Attempt %s: Found %s video files in %s", attempt, len(video_files), found_path)
  6783. if changed:
  6784. for f in video_files[:5]:
  6785. logger.info("[TIMELAPSE] - %s", f.get("name"))
  6786. attached = await _attach_first_unclaimed_timelapse(
  6787. archive_id,
  6788. printer,
  6789. video_files,
  6790. baseline_names,
  6791. claimed,
  6792. attempt,
  6793. logger,
  6794. quiet=not changed,
  6795. require_unambiguous=not baseline_trusted,
  6796. plate_id=plate_id,
  6797. run_started_at=run_started_at,
  6798. run_id=run_id,
  6799. )
  6800. if attached:
  6801. return
  6802. else:
  6803. speak("[TIMELAPSE] Attempt %s: No video files found, will retry", attempt)
  6804. except Exception as e:
  6805. logger.warning("[TIMELAPSE] Attempt %s failed with error: %s", attempt, e)
  6806. if attempt >= max_attempts or time.monotonic() >= deadline:
  6807. break
  6808. # No name-match fallback: it compared the print name against the filename,
  6809. # and Bambu firmware only ever writes "video_<timestamp>". Across 247 support
  6810. # bundles it fired 159 times and matched zero times, so all it added was a
  6811. # misleading log line before giving up.
  6812. logger.warning(
  6813. "[TIMELAPSE] No new video appeared for archive %s within %ss, giving up",
  6814. archive_id,
  6815. int(_TIMELAPSE_SCAN_TIMEOUT_SECONDS),
  6816. )
  6817. async def _attach_layer_timelapse(
  6818. archive_id: int,
  6819. timelapse_path: Path,
  6820. plate_id: int | None,
  6821. run_started_at: datetime | None,
  6822. run_id: int | None = None,
  6823. ) -> None:
  6824. """Attach the external camera's stitched timelapse to the archive.
  6825. The camera's video is the one the user set up in Bambuddy, so it wins
  6826. over the printer's own when both are on: it replaces the printer's video
  6827. (usually there first; stitching takes minutes) and that file is deleted,
  6828. leaving nothing in the archive folder unreferenced. ``run_started_at`` is
  6829. the run the frames were taken for: when the archive is printing again by
  6830. the time the stitch is done, the video is not the new run's and is
  6831. dropped. The stitched file is a temporary either way.
  6832. ``run_id`` is the print that ended (archive redesign): the video is that
  6833. run's and replaces that run's printer video; the poll by start above is
  6834. for a print without a run.
  6835. """
  6836. from backend.app.services.print_log import run_of, superseded
  6837. from backend.app.utils.safe_path import PathTraversalError, safe_join_under
  6838. logger = logging.getLogger(__name__)
  6839. try:
  6840. async with async_session() as db:
  6841. service = ArchiveService(db)
  6842. archive = await service.get_archive(archive_id)
  6843. if archive is None:
  6844. return
  6845. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6846. if run_id is not None and run is None:
  6847. logger.info("[LAYER-TL] Run %s of archive %s is gone; dropping its layer timelapse", run_id, archive_id)
  6848. return
  6849. if run is not None and await superseded(db, run):
  6850. logger.info(
  6851. "[LAYER-TL] A later print took over run %s of archive %s; dropping its layer timelapse",
  6852. run_id,
  6853. archive_id,
  6854. )
  6855. return
  6856. if run is None and archive.started_at != run_started_at:
  6857. logger.info(
  6858. "[LAYER-TL] Archive %s is printing again; dropping the previous run's layer timelapse", archive_id
  6859. )
  6860. return
  6861. holder = run if run is not None else archive
  6862. replaced = holder.timelapse_path
  6863. logger.info("[LAYER-TL] Attaching timelapse %s to archive %s", timelapse_path, archive_id)
  6864. timelapse_data = await asyncio.to_thread(timelapse_path.read_bytes)
  6865. if not await service.attach_timelapse(
  6866. archive_id, timelapse_data, "layer_timelapse.mp4", plate_id=plate_id, run_id=run_id
  6867. ):
  6868. return
  6869. attached = holder.timelapse_path
  6870. logger.info("[LAYER-TL] Layer timelapse attached successfully")
  6871. if replaced and replaced != attached:
  6872. try:
  6873. old = safe_join_under(app_settings.base_dir, replaced, http=False)
  6874. except PathTraversalError:
  6875. return
  6876. await asyncio.to_thread(old.unlink, missing_ok=True)
  6877. logger.info("[LAYER-TL] Replaced the printer's timelapse %s on archive %s", replaced, archive_id)
  6878. finally:
  6879. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  6880. # ``run_started_at`` not given: attach whichever run the archive is on.
  6881. _ANY_RUN = object()
  6882. async def _attach_first_unclaimed_timelapse(
  6883. archive_id: int,
  6884. printer,
  6885. video_files: list[dict],
  6886. baseline_names: set[str],
  6887. claimed: set[str],
  6888. attempt: int,
  6889. logger: logging.Logger,
  6890. *,
  6891. quiet: bool = False,
  6892. require_unambiguous: bool = False,
  6893. plate_id: int | None = None,
  6894. run_started_at: datetime | None | object = _ANY_RUN,
  6895. run_id: int | None = None,
  6896. ) -> bool:
  6897. """Download and attach the one video that belongs to this print.
  6898. A candidate is any file absent from the print-start baseline. More than one
  6899. can qualify when a previous print's video landed late, after this print's
  6900. baseline was taken — those are filtered out by name, because they are
  6901. already attached to another archive. Sorting the candidates instead would
  6902. mean sorting on mtime or on the filename timestamp, both of which come from
  6903. the printer's unsynced clock.
  6904. Returns True once a video is attached. The printer's copy is deleted only
  6905. after the attach succeeds on bytes whose length matched the listing.
  6906. ``quiet`` downgrades the "nothing yet" lines to DEBUG when the caller has
  6907. already seen this exact listing — the poll runs for many rounds and only the
  6908. rounds where something changed are worth an INFO line.
  6909. """
  6910. from backend.app.services.bambu_ftp import (
  6911. DownloadDeadlineExceeded,
  6912. delete_archived_timelapse,
  6913. download_file_bytes_async,
  6914. remote_file_settled,
  6915. )
  6916. speak = logger.debug if quiet else logger.info
  6917. new_files = [f for f in video_files if f.get("name", "") not in baseline_names]
  6918. if not new_files:
  6919. speak("[TIMELAPSE] Attempt %s: No new files since baseline, will retry", attempt)
  6920. return False
  6921. candidates = [f for f in new_files if Path(f.get("name", "")).stem not in claimed]
  6922. if not candidates:
  6923. speak(
  6924. "[TIMELAPSE] Attempt %s: %s new file(s), all already attached to other archives, will retry",
  6925. attempt,
  6926. len(new_files),
  6927. )
  6928. return False
  6929. if len(candidates) > 1:
  6930. if require_unambiguous:
  6931. # The baseline is not evidence -- it was taken off a card that could
  6932. # not be read -- so "new since the baseline" does not narrow these
  6933. # down at all. Taking the first would attach an arbitrary video to
  6934. # this print and then delete it from the printer.
  6935. logger.warning(
  6936. "[TIMELAPSE] Attempt %s: %s unclaimed videos (%s) and no baseline to tell them apart — "
  6937. "leaving all of them on the printer for manual selection",
  6938. attempt,
  6939. len(candidates),
  6940. ", ".join(str(f.get("name")) for f in candidates),
  6941. )
  6942. return False
  6943. logger.warning(
  6944. "[TIMELAPSE] Attempt %s: %s unclaimed new files (%s) — taking the first; "
  6945. "the rest stay on the printer for manual selection",
  6946. attempt,
  6947. len(candidates),
  6948. ", ".join(str(f.get("name")) for f in candidates),
  6949. )
  6950. target = candidates[0]
  6951. file_name = str(target.get("name") or "") # every listed entry has one
  6952. remote_path = target.get("path") or f"/timelapse/{file_name}"
  6953. logger.info(
  6954. "[TIMELAPSE] Attempt %s: New file detected: %s (downloading for archive %s)",
  6955. attempt,
  6956. file_name,
  6957. archive_id,
  6958. )
  6959. # The listing always carries a size (`list_files` skips entries it can't
  6960. # parse), but read it explicitly: the delete below is destructive and must
  6961. # depend on a size we actually had, not on one we hoped was there.
  6962. expected_size = target.get("size")
  6963. try:
  6964. timelapse_data = await download_file_bytes_async(
  6965. printer.ip_address,
  6966. printer.access_code,
  6967. remote_path,
  6968. printer_model=printer.model,
  6969. expected_size=expected_size,
  6970. )
  6971. except DownloadDeadlineExceeded:
  6972. # Too slow for its size this round; the printer keeps its copy.
  6973. timelapse_data = None
  6974. if not timelapse_data:
  6975. # Short or failed transfer. The printer keeps its copy, so the next
  6976. # round can try again — which is exactly why the delete below is
  6977. # gated on a verified download.
  6978. logger.warning("[TIMELAPSE] Attempt %s: Failed to download new file, will retry", attempt)
  6979. return False
  6980. # The length check above proves we got what the listing said, not that the
  6981. # printer had finished writing. A video still being written can be listed
  6982. # short, served short, and pass — so confirm it has stopped growing before
  6983. # committing to it and deleting the original (#2704).
  6984. if not await remote_file_settled(
  6985. printer.ip_address,
  6986. printer.access_code,
  6987. remote_path,
  6988. len(timelapse_data),
  6989. printer_model=printer.model,
  6990. ):
  6991. return False
  6992. # Write phase: attach in a fresh short-lived session. The download took a
  6993. # while: by now the archive may have its video, or be printing again for a
  6994. # newer run, whose video this is not. Then nothing is attached or deleted,
  6995. # and the poll ends (True).
  6996. async with async_session() as db:
  6997. service = ArchiveService(db)
  6998. if run_id is not None or run_started_at is not _ANY_RUN:
  6999. current = await service.get_archive(archive_id)
  7000. started = None if run_started_at is _ANY_RUN else cast("datetime | None", run_started_at)
  7001. if current is None or not await _timelapse_still_wanted(db, current, run_id, started):
  7002. logger.info(
  7003. "[TIMELAPSE] Archive %s moved on during the download (video attached or printing again); "
  7004. "%s stays on the printer",
  7005. archive_id,
  7006. file_name,
  7007. )
  7008. return True
  7009. success = await service.attach_timelapse(
  7010. archive_id, timelapse_data, file_name, plate_id=plate_id, run_id=run_id
  7011. )
  7012. if not success:
  7013. logger.warning("[TIMELAPSE] Failed to attach timelapse to archive %s", archive_id)
  7014. return False
  7015. logger.info("[TIMELAPSE] Successfully attached timelapse to archive %s", archive_id)
  7016. await delete_archived_timelapse(
  7017. printer.ip_address,
  7018. printer.access_code,
  7019. remote_path,
  7020. verified=expected_size is not None,
  7021. printer_model=printer.model,
  7022. printer_name=printer.name,
  7023. )
  7024. return True
  7025. # Defaults for the finish-photo-from-timelapse polling loop (#1397). These are
  7026. # module-level so tests can monkeypatch them down to ~0 without timing out.
  7027. _FINISH_PHOTO_TIMELAPSE_POLL_INTERVAL_SECONDS: float = 3.0
  7028. _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS: float = 60.0
  7029. # How long the *background* upgrade keeps waiting after the notification has
  7030. # already gone out (#2704 follow-up). The short bound above exists so a slow
  7031. # printer can't hold up the print-complete notification; this one exists so the
  7032. # archive still ends up with the better frame afterwards.
  7033. #
  7034. # Measured across 261 attaches in the support bundles, the video lands a median
  7035. # 13s after the print ends — but the P1 series writes MJPEG AVI rather than
  7036. # H.264 MP4 and serves it slowly, so its p90 is 167s and the worst observed case
  7037. # was 546s. Every other model was inside 26s. The long budget is therefore
  7038. # almost entirely for P1-series users; on everything else the short wait already
  7039. # wins and this task never runs.
  7040. _FINISH_PHOTO_UPGRADE_TIMEOUT_SECONDS: float = 900.0
  7041. async def _capture_finish_photo_from_timelapse(
  7042. archive_id: int,
  7043. archive_dir: Path,
  7044. timeout: float | None = None,
  7045. rotation: int = 0,
  7046. run_id: int | None = None,
  7047. ) -> tuple[str | None, bool]:
  7048. """Wait for the per-print timelapse to land on the archive and extract its
  7049. last frame as the finish photo (#1397).
  7050. Bambu firmware stops timelapse recording after the toolhead parks but
  7051. before the bed-drop end-gcode runs, so the last frame frames the finished
  7052. print correctly. A live camera grab at gcode_state=FINISH captures the
  7053. bed already lowered.
  7054. ``_scan_for_timelapse_with_retries`` runs in parallel and writes
  7055. ``archive.timelapse_path`` when the file lands. This function polls for
  7056. that field.
  7057. Returns ``(filename, still_pending)``. ``still_pending`` is True only when
  7058. the wait ran out with no video on the archive yet — i.e. the video may
  7059. still be coming and a later attempt could succeed. It is False when the
  7060. video landed (whether or not extraction worked), because in that case
  7061. waiting longer changes nothing. The caller uses that to decide between
  7062. falling back permanently and scheduling a background upgrade.
  7063. ``rotation`` is the printer's camera_rotation, applied to the extracted
  7064. still (#2708) so this source agrees with every other finish-photo source.
  7065. The archived video itself is the printer's own file and is left alone —
  7066. rotating it would mean re-encoding it.
  7067. ``run_id`` is the print that ended: its own video is the one waited for,
  7068. not the archive's, which can be another plate's.
  7069. """
  7070. import uuid
  7071. from backend.app.models.archive import PrintArchive
  7072. from backend.app.services.camera import apply_camera_rotation_to_file, extract_video_last_frame
  7073. from backend.app.services.print_log import run_of
  7074. logger = logging.getLogger(__name__)
  7075. budget = _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS if timeout is None else timeout
  7076. deadline = asyncio.get_event_loop().time() + budget
  7077. poll_interval = _FINISH_PHOTO_TIMELAPSE_POLL_INTERVAL_SECONDS
  7078. while True:
  7079. async with async_session() as db:
  7080. if run_id is not None:
  7081. run = await run_of(db, archive_id, run_id)
  7082. timelapse_relpath = run.timelapse_path if run else None
  7083. else:
  7084. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  7085. archive = result.scalar_one_or_none()
  7086. timelapse_relpath = archive.timelapse_path if archive else None
  7087. if timelapse_relpath:
  7088. video_path = app_settings.base_dir / timelapse_relpath
  7089. if video_path.exists() and video_path.stat().st_size > 0:
  7090. photos_dir = archive_dir / "photos"
  7091. photos_dir.mkdir(parents=True, exist_ok=True)
  7092. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  7093. filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  7094. output_path = photos_dir / filename
  7095. if await extract_video_last_frame(video_path, output_path):
  7096. await apply_camera_rotation_to_file(output_path, rotation, logger)
  7097. logger.info(
  7098. "[PHOTO-BG] Extracted finish photo from timelapse %s for archive %s",
  7099. video_path.name,
  7100. archive_id,
  7101. )
  7102. return filename, False
  7103. logger.warning(
  7104. "[PHOTO-BG] Timelapse %s landed but last-frame extraction failed for archive %s; falling back",
  7105. video_path.name,
  7106. archive_id,
  7107. )
  7108. return None, False
  7109. if asyncio.get_event_loop().time() >= deadline:
  7110. logger.info(
  7111. "[PHOTO-BG] Timelapse for archive %s didn't land within %.0fs; falling back to live camera",
  7112. archive_id,
  7113. budget,
  7114. )
  7115. return None, True
  7116. await asyncio.sleep(poll_interval)
  7117. async def _upgrade_finish_photo_from_timelapse(
  7118. archive_id: int,
  7119. archive_dir: Path,
  7120. rotation: int = 0,
  7121. run_id: int | None = None,
  7122. replace: str | None = None,
  7123. ) -> None:
  7124. """Put the timelapse's last frame on an archive after the fact (#2704).
  7125. The print-complete notification waits only ~60s for the video, because
  7126. holding a notification for minutes is worse than sending it with a live
  7127. camera grab. On a P1-series printer the video often lands well after that,
  7128. so the archive used to be stuck with the live grab — which is taken at
  7129. ``gcode_state=FINISH``, after the end G-code has dropped the bed, and is
  7130. the worse photo of the two.
  7131. This keeps waiting in the background and, when the video arrives, extracts
  7132. the frame and puts it *first* in the archive's photo list, so opening the
  7133. gallery shows it. ``replace`` is the live grab: the frame is written over
  7134. that file, under its name, rather than added next to it. Two finish photos
  7135. of the same moment is one too many, and the name has to survive: with
  7136. authentication off, the notification that already went out links to
  7137. ``/archives/<id>/photos/<name>``, which now shows the better photo. (With
  7138. it on, the link is to a copy in notification_photos, and attachments are
  7139. bytes, so neither cares.) Without ``replace``, or once that photo has been
  7140. deleted, the frame is added under its own name. ``run_id`` is the print it
  7141. is for, whose list leads with it too.
  7142. """
  7143. from backend.app.services.print_log import add_run_photo, run_of
  7144. logger = logging.getLogger(__name__)
  7145. filename, _ = await _capture_finish_photo_from_timelapse(
  7146. archive_id, archive_dir, timeout=_FINISH_PHOTO_UPGRADE_TIMEOUT_SECONDS, rotation=rotation, run_id=run_id
  7147. )
  7148. if not filename:
  7149. logger.info("[PHOTO-UPGRADE] No timelapse frame for archive %s; keeping the live grab", archive_id)
  7150. return
  7151. try:
  7152. async with async_session() as db:
  7153. from backend.app.models.archive import PrintArchive
  7154. archive = await db.get(PrintArchive, archive_id)
  7155. if archive is None:
  7156. return
  7157. photos = list(archive.photos or [])
  7158. if filename in photos:
  7159. return
  7160. photos_dir = archive_dir / "photos"
  7161. if replace and replace in photos and (photos_dir / replace).is_file():
  7162. # Atomic, so the gallery never reads half a JPEG.
  7163. await asyncio.to_thread(os.replace, photos_dir / filename, photos_dir / replace)
  7164. filename = replace
  7165. photos.remove(replace)
  7166. # Front of the list: PhotoGalleryModal opens at index 0.
  7167. archive.photos = [filename, *photos]
  7168. add_run_photo(await run_of(db, archive_id, run_id) if run_id is not None else None, filename, first=True)
  7169. await db.commit()
  7170. except Exception as e:
  7171. logger.warning("[PHOTO-UPGRADE] Failed to attach upgraded photo to archive %s: %s", archive_id, e)
  7172. return
  7173. logger.info(
  7174. "[PHOTO-UPGRADE] Archive %s now leads with the timelapse frame %s%s",
  7175. archive_id,
  7176. filename,
  7177. " (replacing the live grab)" if filename == replace else "",
  7178. )
  7179. await ws_manager.send_archive_updated({"id": archive_id, "photo_added": filename})
  7180. async def _restore_usage_tracking_session(printer_id: int, state, db, logger) -> None:
  7181. """Put the filament-attribution context back after a restart mid-print.
  7182. ``usage_tracker._active_sessions`` and ``PrinterState.tray_change_log``
  7183. both die with the process. The print keeps running, so at completion the
  7184. tracker would fall back to whatever the printer reports *now* — and AMS
  7185. filament backup makes "now" the substitute tray, charging the whole print
  7186. to the spool that only finished it.
  7187. The persisted row is only trusted when its print name still matches what
  7188. the printer says it is running: a row left behind by a completion we never
  7189. saw must not attach itself to the next print.
  7190. """
  7191. try:
  7192. from backend.app.api.routes.settings import get_setting
  7193. from backend.app.services.usage_tracker import (
  7194. clear_persisted_session,
  7195. get_persisted_ams_mapping,
  7196. get_persisted_print_name,
  7197. get_persisted_spool_change_log,
  7198. restore_session,
  7199. )
  7200. persisted_name = await get_persisted_print_name(db, printer_id)
  7201. current_name = (state.subtask_name or "").strip()
  7202. if persisted_name and current_name and persisted_name.strip() != current_name:
  7203. logger.info(
  7204. "[RESTART] Discarding stale print session for printer %s (%r != running %r)",
  7205. printer_id,
  7206. persisted_name,
  7207. current_name,
  7208. )
  7209. await clear_persisted_session(db, printer_id)
  7210. # Fall through to seeding: the print on the printer is real, it just
  7211. # isn't the one the row described.
  7212. persisted_log = None
  7213. else:
  7214. # Spoolman users get the tray-change log back but no in-memory
  7215. # session — see ``on_print_start`` on why that dict is load-bearing
  7216. # for the remain%-sync guard.
  7217. _spoolman_on = await get_setting(db, "spoolman_enabled")
  7218. persisted_log = await restore_session(
  7219. db,
  7220. printer_id,
  7221. register_active=not (bool(_spoolman_on) and _spoolman_on.lower() == "true"),
  7222. )
  7223. # The slot pills (#1762) read the mapping captured from the print
  7224. # command, which dies with the process as well. One this process
  7225. # has captured since is newer.
  7226. if state.print_ams_mapping is None:
  7227. state.print_ams_mapping = await get_persisted_ams_mapping(db, printer_id)
  7228. # Spools swapped on a tray before the restart (#3325); anything this
  7229. # process has recorded since goes after them.
  7230. persisted_swaps = await get_persisted_spool_change_log(db, printer_id)
  7231. if persisted_swaps:
  7232. seen = getattr(state, "spool_change_log", None) or []
  7233. state.spool_change_log = persisted_swaps + [item for item in seen if item not in persisted_swaps]
  7234. if persisted_log:
  7235. restored = [tuple(entry) for entry in persisted_log if isinstance(entry, (list, tuple)) and len(entry) == 2]
  7236. # Anything this process already observed goes after the persisted
  7237. # history — the log is ordered by layer, and a fresh process can
  7238. # only have seen changes from later in the print.
  7239. for entry in state.tray_change_log or []:
  7240. if tuple(entry) not in restored:
  7241. restored.append(tuple(entry))
  7242. state.tray_change_log = restored
  7243. tray_now = state.tray_now
  7244. if 0 <= tray_now <= 254:
  7245. if not state.tray_change_log:
  7246. # No persisted history — a print that started before this build,
  7247. # or before the row existed. Seed with the tray feeding right
  7248. # now so the remainder of the print is at least attributable to
  7249. # the right spool.
  7250. state.tray_change_log = [(tray_now, state.layer_num)]
  7251. logger.info(
  7252. "[RESTART] Seeded tray change log for printer %s: tray=%d at layer=%d",
  7253. printer_id,
  7254. tray_now,
  7255. state.layer_num,
  7256. )
  7257. # The tray handler updates ``last_loaded_tray`` on every push
  7258. # regardless of whether it logged a change, so re-align it to avoid
  7259. # a duplicate entry on the next push. Only ever with a real tray:
  7260. # ``last_loaded_tray`` is the "survives the end-of-print retract to
  7261. # 255" fallback, and writing 255 into it would defeat that.
  7262. state.last_loaded_tray = tray_now
  7263. except Exception:
  7264. # Never let attribution recovery cost the caller its timelapse
  7265. # baseline — that capture has to happen before the printer uploads
  7266. # the in-flight MP4 and there is no second chance at it.
  7267. logger.exception("[RESTART] Failed to restore usage-tracking session for printer %s", printer_id)
  7268. async def on_print_running_observed(printer_id: int, data: dict):
  7269. """Restart-recovery for a print that started before Bambuddy came up.
  7270. bambu_mqtt.py suppresses ``on_print_start`` on the first RUNNING push
  7271. after Bambuddy startup (#1304 guard, prevents duplicate archive
  7272. creation). This hook restores the persisted archive into ``_active_prints``
  7273. and captures the timelapse baseline that normally hangs off print start.
  7274. Fires once per session, in lieu of on_print_start when restart-recovery
  7275. kicks in. The printer doesn't upload the timelapse until after PRINT
  7276. COMPLETE, so a baseline captured any time during the print is still
  7277. pre-upload.
  7278. """
  7279. logger = logging.getLogger(__name__)
  7280. async with async_session() as db:
  7281. from backend.app.models.printer import Printer
  7282. state = printer_manager.get_status(printer_id)
  7283. if state is not None:
  7284. authorization = await _is_bambuddy_authorized_print(printer_id, state, db)
  7285. if authorization is True:
  7286. logger.info("[RESTART] Restored active Bambuddy print for printer %s", printer_id)
  7287. await _restore_usage_tracking_session(printer_id, state, db, logger)
  7288. await _restore_printable_objects(printer_id, state, db, logger)
  7289. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  7290. printer = result.scalar_one_or_none()
  7291. if not printer:
  7292. logger.warning(
  7293. "[TIMELAPSE] on_print_running_observed: printer %s not found in DB, skipping baseline",
  7294. printer_id,
  7295. )
  7296. return
  7297. # Avoid double-capture: ownership reconciliation above must still run when
  7298. # a baseline already exists, but the camera work itself is one-shot.
  7299. if printer_id in _timelapse_baselines:
  7300. logger.debug(
  7301. "[TIMELAPSE] on_print_running_observed: baseline already present for printer %s, skipping",
  7302. printer_id,
  7303. )
  7304. return
  7305. await _capture_timelapse_baseline_at_start(printer, printer_id, logger)
  7306. class _RunOfArchive:
  7307. """An archive as one of its runs printed it: the run's job id, start and
  7308. plate, the rest the archive's. What the reconcile judges and closes."""
  7309. def __init__(self, archive, run):
  7310. self._archive = archive
  7311. self.subtask_id = run.subtask_id
  7312. self.started_at = run.started_at
  7313. self.plate_id = run.plate_id if run.plate_id is not None else archive.plate_id
  7314. def __getattr__(self, name):
  7315. return getattr(self._archive, name)
  7316. def _is_active_archive_stale(archive, state) -> tuple[bool, str]:
  7317. """Return ``(is_stale, reason)`` for an archive in ``status="printing"``
  7318. against the printer's current MQTT state.
  7319. Reconciliation triggers (#1542 follow-up — recovers from missed PRINT
  7320. COMPLETE events, typically a print finishing during an MQTT disconnect
  7321. window followed by a smart-plug power cycle):
  7322. 1. Printer state is terminal (IDLE / FINISH / FAILED). The print is
  7323. provably not running anymore — only branch that should fire under
  7324. normal disconnect-then-reconnect timing.
  7325. 2. Printer has a different ``subtask_id`` than the archive. Bambu
  7326. firmware mints a fresh ``subtask_id`` for each print, including the
  7327. ghost replay it runs after a power cycle from a leftover SD file —
  7328. so a mismatch unambiguously means the in-DB archive is no longer
  7329. the print on the printer.
  7330. 3. Printer is running but ``subtask_name`` is empty. The printer
  7331. doesn't know what it's running; the archive's reference to it is
  7332. already broken.
  7333. Conservative on purpose: PAUSE / PREPARE / SLICING and any RUNNING state
  7334. with matching subtask_id+subtask_name is left alone. The cost of a false
  7335. positive is a print closed while it still runs: the reconcile takes its
  7336. ``_active_prints`` entry and the archive is no longer ``printing``, so
  7337. the real PRINT COMPLETE later finds no archive to close (#1679). The
  7338. cost of a false negative is the ghost-print loop in #1542. What the
  7339. archive is closed as is ``_reconciled_outcome``'s call.
  7340. Pre-push guard (#1679): when ``state.state`` is empty or ``"unknown"``,
  7341. MQTT has connected but the first ``push_status`` response hasn't been
  7342. applied yet — ``PrinterState`` is sitting on its construction defaults.
  7343. The reconcile caller in ``on_printer_status_change`` is already gated
  7344. on a real ``state.state``, so in normal operation this branch is
  7345. unreachable; it's kept as belt-and-braces for future callers and for
  7346. the narrow window where a partial state update could arrive
  7347. (``state.state`` set but ``subtask_name`` not yet populated). Returning
  7348. ``not stale`` on degenerate input is strictly conservative: a real
  7349. stale archive will still be caught by the next push_status arriving
  7350. with terminal state.
  7351. """
  7352. current_state = (state.state or "").upper()
  7353. if current_state in ("", "UNKNOWN"):
  7354. # No real push_status yet — PrinterState defaults are not evidence.
  7355. return False, ""
  7356. if current_state in ("IDLE", "FINISH", "FAILED"):
  7357. return True, f"printer state {current_state}"
  7358. # Below here the printer is in a running / pre-running state (RUNNING /
  7359. # PAUSE / PREPARE / SLICING / etc.) — decide based on subtask identity.
  7360. current_subtask_id = (state.subtask_id or "").strip()
  7361. if archive.subtask_id and current_subtask_id and archive.subtask_id != current_subtask_id:
  7362. return True, f"subtask_id changed ({archive.subtask_id!r} → {current_subtask_id!r})"
  7363. current_subtask_name = (state.subtask_name or "").strip()
  7364. if not current_subtask_name:
  7365. return True, "printer subtask_name empty"
  7366. return False, ""
  7367. async def prime_kprofile_table(printer_id: int) -> int:
  7368. """Read the printer's calibration table once per connection.
  7369. The AMS slot card shows a K value per slot (#2854). On the printers whose
  7370. trays carry no ``k`` field of their own -- the whole H2 series, whose trays
  7371. report ``cali_idx`` and nothing else -- that number can only come from
  7372. ``state.kprofiles``, and nothing used to fill it on connect. It arrived by
  7373. luck: someone opening the Profiles page or Configure Slot, a nightly GitHub
  7374. backup, or the printer answering a query BambuStudio made on the report
  7375. topic we share. A Bambuddy that nobody visited showed a card with no K
  7376. values at all.
  7377. Only the diameters actually fitted are asked for, which is one request on a
  7378. single-nozzle printer and two on a dual. Probing the four sizes blind is
  7379. what the backup does, and it is both wasteful and the thing that used to
  7380. blank the table.
  7381. Returns the number of nozzles whose table was read.
  7382. """
  7383. client = printer_manager.get_client(printer_id)
  7384. state = printer_manager.get_status(printer_id)
  7385. if client is None or state is None or not state.connected:
  7386. return 0
  7387. # Deduplicated, order preserved: a dual-nozzle printer with two 0.4s should
  7388. # ask once, and both entries are empty until the first push_status lands.
  7389. diameters = list(dict.fromkeys(n.nozzle_diameter for n in (state.nozzles or []) if n.nozzle_diameter))
  7390. if not diameters:
  7391. logging.getLogger(__name__).debug(
  7392. "[Printer %s] No nozzle diameter reported yet; leaving the K-profile table to the next reader",
  7393. printer_id,
  7394. )
  7395. return 0
  7396. primed = 0
  7397. for diameter in diameters:
  7398. try:
  7399. profiles = await client.get_kprofiles(nozzle_diameter=diameter, max_retries=2)
  7400. except Exception as exc: # noqa: BLE001
  7401. # A printer that won't answer costs the card its K values, nothing
  7402. # more — never the connection this runs on the back of.
  7403. logging.getLogger(__name__).warning(
  7404. "[Printer %s] Could not read the K-profile table for nozzle %s: %s", printer_id, diameter, exc
  7405. )
  7406. continue
  7407. primed += 1
  7408. logging.getLogger(__name__).info(
  7409. "[Printer %s] Primed K-profile table for nozzle %s: %d profiles", printer_id, diameter, len(profiles)
  7410. )
  7411. return primed
  7412. def _sd_files_in_use(state) -> set[str]:
  7413. """Names of the SD-card files the printer may be printing right now (#3009).
  7414. The post-print cleanup works out what to delete from the finished archive,
  7415. not from the printer. When reconciliation closes an old archive because the
  7416. printer has moved on to a new job, and that job is the same file reprinted
  7417. from the printer's screen, the cleanup would delete the file mid-print.
  7418. Empty unless the printer is busy: on an ordinary completion the state is
  7419. FINISH or FAILED and nothing is held back. ``gcode_file`` arrives as a bare
  7420. name, a path or a URL depending on firmware, and some report only the
  7421. plate's G-code, so the job name is matched as well, in the same spellings
  7422. the cleanup tries. Names are lower-cased; the card's FAT filesystem
  7423. ignores case.
  7424. """
  7425. def _text(value) -> str:
  7426. return value.strip() if isinstance(value, str) else ""
  7427. if state is None:
  7428. return set()
  7429. current_state = _text(getattr(state, "state", None)).upper()
  7430. if current_state in ("", "UNKNOWN", "IDLE", "FINISH", "FAILED"):
  7431. return set()
  7432. in_use: set[str] = set()
  7433. gcode_file = _text(getattr(state, "gcode_file", None))
  7434. if gcode_file:
  7435. # Drop a scheme by hand: urlparse would cut a name at "#" or "?",
  7436. # both legal in a file name, and raises on some inputs.
  7437. name = PurePosixPath(gcode_file.split("://", 1)[-1]).name
  7438. if name:
  7439. in_use.add(name.lower())
  7440. subtask_name = _text(getattr(state, "subtask_name", None))
  7441. if subtask_name:
  7442. for name in (subtask_name, subtask_name.replace(" ", "_")):
  7443. for ext in (".3mf", ".gcode"):
  7444. in_use.add(f"{name}{ext}".lower())
  7445. return in_use
  7446. def _reconciled_outcome(archive, state) -> str:
  7447. """What a stale archive is closed as at reconnect (#3310).
  7448. A print that ends while Bambuddy restarts is never seen to end: the new
  7449. MQTT client's first status is already FINISH, which is not a transition.
  7450. The printer keeps reporting the finished job's ``subtask_id`` until the
  7451. next print, so a matching id proves the FINISH or FAILED is this
  7452. archive's. Anything else -- IDLE, another job, an id of "0" (a print
  7453. started outside the cloud) or none -- says only that the print is over,
  7454. not how, and stays ``aborted``.
  7455. """
  7456. current_state = (state.state or "").upper()
  7457. if current_state not in ("FINISH", "FAILED"):
  7458. return "aborted"
  7459. archive_id = (archive.subtask_id or "").strip()
  7460. current_id = str(state.subtask_id or "").strip()
  7461. if archive_id in ("", "0") or archive_id != current_id:
  7462. return "aborted"
  7463. return "completed" if current_state == "FINISH" else "failed"
  7464. def _reconciled_end_time(archive, now: datetime, plate_id: int | None = None) -> datetime | None:
  7465. """When a print reconciled as completed most likely ended, or None (#3310).
  7466. The printer does not say when it finished, and the reconnect can be hours
  7467. later. Its start plus the slicer's estimate for the plate it printed,
  7468. never later than now, is the best guess; None when either is missing,
  7469. and the run then keeps no duration, as an aborted one does.
  7470. ``plate_id`` is the plate this run printed, from its queue job: a reprint
  7471. of another plate reuses the archive without changing ``archive.plate_id``.
  7472. """
  7473. started = archive.started_at
  7474. if not started:
  7475. return None
  7476. if started.tzinfo is None:
  7477. started = started.replace(tzinfo=timezone.utc)
  7478. plate = plate_id if plate_id is not None else archive.plate_id
  7479. estimate = None
  7480. if archive.file_path:
  7481. from backend.app.utils.threemf_tools import extract_print_time_from_3mf
  7482. full_path = (
  7483. app_settings.base_dir / archive.file_path
  7484. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  7485. try:
  7486. estimate = extract_print_time_from_3mf(full_path, plate)
  7487. except Exception as exc: # noqa: BLE001
  7488. logging.getLogger(__name__).debug(
  7489. "Plate estimate for reconciled archive %s unreadable: %s", archive.id, exc
  7490. )
  7491. if plate is None and estimate != archive.print_time_seconds:
  7492. # Without a plate the first plate's estimate is read. The archive
  7493. # carries the sum over all plates (#1593), so a difference means
  7494. # several plates and no telling which one printed.
  7495. estimate = None
  7496. elif plate is None:
  7497. # Nothing to read the plate from; the archive's own estimate, as for
  7498. # a single-plate file.
  7499. estimate = archive.print_time_seconds
  7500. if not estimate or estimate <= 0:
  7501. return None
  7502. return min(started + timedelta(seconds=int(estimate)), now)
  7503. async def reconcile_stale_active_prints(printer_id: int) -> int:
  7504. """Synthesise ``on_print_complete`` for archives whose print can't be
  7505. running on the printer anymore.
  7506. Called once per MQTT (re)connection (from on_printer_status_change when
  7507. the connected edge flips False → True) and at Bambuddy startup (from
  7508. the FastAPI lifespan). Without this, a print that completes during a
  7509. disconnect window — followed by a smart-plug-driven power cycle — leaves
  7510. the ``.3mf`` on the SD card, the firmware auto-replays it on next boot,
  7511. and Bambuddy fires a fresh PRINT START for the ghost rather than the
  7512. SD cleanup that PRINT COMPLETE was supposed to run. Repeats every
  7513. power cycle until the operator notices (#1542 follow-up). Reconciliation
  7514. closes the loop by faking the missed PRINT COMPLETE — the existing
  7515. cleanup chain handles SD-file deletion, status updates, usage tracking,
  7516. and notifications.
  7517. The status is ``_reconciled_outcome``'s: ``completed`` or ``failed``
  7518. when the printer still reports the archive's own job as finished or
  7519. failed, ``aborted`` otherwise. Nothing corrects it later: the archive is
  7520. no longer ``printing``, so a later PRINT COMPLETE finds no archive, and
  7521. after a restart the printer's FINISH is not one (#3310).
  7522. Returns the number of archives reconciled.
  7523. """
  7524. state = printer_manager.get_status(printer_id)
  7525. if not state:
  7526. return 0
  7527. # Don't reconcile while disconnected — we'd be making a decision against
  7528. # stale cached state. The connected → reconcile edge handles this.
  7529. if not state.connected:
  7530. return 0
  7531. from backend.app.models.archive import PrintArchive
  7532. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, RUN_OPEN, PrintLogEntry
  7533. reconciled = 0
  7534. async with async_session() as db:
  7535. # The prints on this printer Bambuddy saw start: their runs open here.
  7536. # The archive row is shared by every plate of a file, its printer, job
  7537. # and start the latest start's, so each is judged as its run.
  7538. open_runs = (
  7539. await db.scalars(
  7540. select(PrintLogEntry)
  7541. .where(PrintLogEntry.printer_id == printer_id, PrintLogEntry.status == RUN_OPEN)
  7542. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  7543. )
  7544. ).all()
  7545. by_run = {
  7546. archive.id: archive
  7547. for archive in (
  7548. await db.scalars(
  7549. select(PrintArchive).where(
  7550. PrintArchive.id.in_([run.archive_id for run in open_runs if run.archive_id is not None])
  7551. )
  7552. )
  7553. ).all()
  7554. }
  7555. active: list = [_RunOfArchive(by_run[run.archive_id], run) for run in open_runs if run.archive_id in by_run]
  7556. # And as before runs were opened at the start: a printing archive of
  7557. # this printer with no run open anywhere (printing since before).
  7558. archives_with_open_runs = select(PrintLogEntry.archive_id).where(
  7559. PrintLogEntry.status == RUN_OPEN, PrintLogEntry.archive_id.is_not(None)
  7560. )
  7561. result = await db.execute(
  7562. select(PrintArchive)
  7563. .where(
  7564. PrintArchive.printer_id == printer_id,
  7565. PrintArchive.status == "printing",
  7566. PrintArchive.id.not_in(archives_with_open_runs),
  7567. )
  7568. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  7569. )
  7570. active.extend(result.scalars().all())
  7571. if not active:
  7572. return 0
  7573. # The plate each run printed, from its queue job (#3310).
  7574. from backend.app.models.print_queue import PrintQueueItem
  7575. async with async_session() as db:
  7576. plate_rows = await db.execute(
  7577. select(PrintQueueItem.archive_id, PrintQueueItem.plate_id).where(
  7578. PrintQueueItem.printer_id == printer_id,
  7579. PrintQueueItem.status == "printing",
  7580. PrintQueueItem.archive_id.in_([a.id for a in active]),
  7581. )
  7582. )
  7583. run_plates = {archive_id: plate for archive_id, plate in plate_rows.all() if plate is not None}
  7584. logger = logging.getLogger(__name__)
  7585. for archive in active:
  7586. is_stale, reason = _is_active_archive_stale(archive, state)
  7587. if not is_stale:
  7588. continue
  7589. logger.info(
  7590. "[RECONCILE] Printer %s: synthesising missed PRINT COMPLETE for archive %s (%s) — %s",
  7591. printer_id,
  7592. archive.id,
  7593. archive.filename,
  7594. reason,
  7595. )
  7596. # Synthesised payload: minimal fields the on_print_complete chain
  7597. # needs. `_reconciled` marker lets downstream code distinguish this
  7598. # from a real MQTT-driven completion if it ever needs to (e.g. for
  7599. # metrics / debug logging). raw_data is the live printer state so
  7600. # the usage tracker can compare end-of-print remain% against the
  7601. # captured start values.
  7602. outcome = _reconciled_outcome(archive, state)
  7603. payload = {
  7604. "status": outcome,
  7605. "filename": archive.filename,
  7606. "subtask_name": archive.print_name or "",
  7607. "subtask_id": archive.subtask_id or "",
  7608. "raw_data": state.raw_data or {},
  7609. "_reconciled": True,
  7610. }
  7611. if isinstance(archive, _RunOfArchive):
  7612. # The archive it closes, which its names and printer may not find:
  7613. # another printer's plate of it can have started since.
  7614. payload["_reconciled_archive_id"] = archive.id
  7615. if outcome != "aborted":
  7616. # The printer's progress and layer are this job's, so a partial
  7617. # run is charged what it printed rather than nothing.
  7618. from backend.app.utils.printer_models import usage_progress
  7619. run_progress = usage_progress(
  7620. printer_manager.get_model(printer_id),
  7621. state.progress,
  7622. getattr(state, "layer_num", None),
  7623. getattr(state, "total_layers", None),
  7624. )
  7625. if run_progress:
  7626. payload["last_progress"] = run_progress
  7627. if state.layer_num:
  7628. payload["last_layer_num"] = state.layer_num
  7629. if outcome == "failed":
  7630. payload["hms_errors"] = [e.completion_payload() for e in state.hms_errors or []]
  7631. if outcome == "completed":
  7632. ended = _reconciled_end_time(archive, datetime.now(timezone.utc), run_plates.get(archive.id))
  7633. if ended:
  7634. payload["_reconciled_completed_at"] = ended
  7635. try:
  7636. await on_print_complete(printer_id, payload)
  7637. reconciled += 1
  7638. except Exception as e:
  7639. # Catch-all: a reconciliation failure must not block the
  7640. # printer's normal status flow. The archive stays in
  7641. # ``status="printing"`` and the next reconnect retries.
  7642. logger.warning(
  7643. "[RECONCILE] on_print_complete synthesis failed for archive %s: %s",
  7644. archive.id,
  7645. e,
  7646. )
  7647. return reconciled
  7648. # #2547: clearance left between the nozzle and the top of the print when the
  7649. # plate is commanded back into camera framing. The nozzle is parked away from
  7650. # the part by then, so this is belt-and-braces against a max_z_height that
  7651. # under-reports (e.g. a slicer that excludes a final Z hop).
  7652. _PLATE_RESTORE_CLEARANCE_MM = 10.0
  7653. # Feedrate for both moves. F600 is exactly what Bambu's own end G-code uses on
  7654. # this axis, so it is a proven-safe speed for the full travel.
  7655. _PLATE_RESTORE_FEEDRATE = 600
  7656. # Time allowed for the plate to reach the restored position before the camera
  7657. # grab. Sized for the ~100 mm the stock end G-code drops on X1/P1 at F600
  7658. # (10 mm/s); most H2 and P2 end G-code drops less, or not at all.
  7659. _PLATE_RESTORE_SETTLE_SECONDS = 12.0
  7660. # How long `_background_finish_photo` waits for this producer. Must cover the
  7661. # settle window plus a worst-case RTSP grab (15s), and stay below the
  7662. # notification path's own photo wait so a slow producer degrades to a
  7663. # photo-less notification rather than a missed one. The chamber light's
  7664. # snapshot delay (#1655) spends from the spare 8 seconds, which is why
  7665. # camera_light.MAX_DELAY_SECONDS stays below it.
  7666. _FINISH_PHOTO_PRODUCER_WAIT_SECONDS = _PLATE_RESTORE_SETTLE_SECONDS + 23.0
  7667. def _completion_print_keys(printer_id: int, filename: str, subtask_name: str) -> list[tuple[int, str]]:
  7668. """The ``_active_prints`` keys a finished print may be registered under.
  7669. Matching how they were registered in on_print_start. Shared by
  7670. ``on_print_complete`` and the finish-photo plate restore (#3240), which has
  7671. to find the same binding.
  7672. """
  7673. possible_keys = []
  7674. # Try subtask_name variations first (most reliable for matching)
  7675. if subtask_name:
  7676. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  7677. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  7678. possible_keys.append((printer_id, subtask_name))
  7679. # Try filename variations
  7680. if filename:
  7681. # Extract just the filename if it's a path
  7682. fname = filename.split("/")[-1] if "/" in filename else filename
  7683. if fname.endswith(".3mf"):
  7684. possible_keys.append((printer_id, fname))
  7685. elif fname.endswith(".gcode"):
  7686. base_name = fname.rsplit(".", 1)[0]
  7687. possible_keys.append((printer_id, f"{base_name}.gcode.3mf"))
  7688. possible_keys.append((printer_id, f"{base_name}.3mf"))
  7689. possible_keys.append((printer_id, fname))
  7690. else:
  7691. possible_keys.append((printer_id, f"{fname}.gcode.3mf"))
  7692. possible_keys.append((printer_id, f"{fname}.3mf"))
  7693. possible_keys.append((printer_id, fname))
  7694. # Also try full path versions
  7695. if filename.endswith(".3mf"):
  7696. possible_keys.append((printer_id, filename))
  7697. elif filename.endswith(".gcode"):
  7698. base_name = filename.rsplit(".", 1)[0]
  7699. possible_keys.append((printer_id, f"{base_name}.3mf"))
  7700. possible_keys.append((printer_id, filename))
  7701. else:
  7702. possible_keys.append((printer_id, f"{filename}.3mf"))
  7703. possible_keys.append((printer_id, filename))
  7704. return possible_keys
  7705. def _archive_name_forms(archive) -> set[str]:
  7706. """The names an archive could be echoed back as, in normalised form (#3240).
  7707. Its print name, and its filename with and without the 3MF extension.
  7708. """
  7709. forms = set()
  7710. if archive.print_name:
  7711. forms.add(_normalise_subtask_name(archive.print_name))
  7712. if archive.filename:
  7713. forms.add(_normalise_subtask_name(archive.filename))
  7714. for suffix in (".gcode.3mf", ".3mf"):
  7715. if archive.filename.lower().endswith(suffix):
  7716. forms.add(_normalise_subtask_name(archive.filename[: -len(suffix)]))
  7717. forms.discard("")
  7718. return forms
  7719. def _bound_print(printer_id: int, data: dict) -> tuple[int | None, int | None]:
  7720. """The archive and plate Bambuddy bound to the print ending on ``printer_id`` (#3240).
  7721. Read from ``_active_prints`` with the keys ``on_print_complete`` tries, and
  7722. from the plate recorded when the job was dispatched. Neither is touched:
  7723. ``on_print_complete`` pops both, which is why the finish-photo producer
  7724. calls this before its first await, while they are still there.
  7725. ``(None, None)`` for a print Bambuddy did not bind -- one it failed to
  7726. archive, or that it only knows by a name -- and on any error: the producer
  7727. calls this before the ``try`` that always releases the photo consumer, so
  7728. it must not raise.
  7729. """
  7730. try:
  7731. keys = _completion_print_keys(printer_id, str(data.get("filename") or ""), str(data.get("subtask_name") or ""))
  7732. archive_id = next((_active_prints[key] for key in keys if key in _active_prints), None)
  7733. if archive_id is None:
  7734. return None, None
  7735. return archive_id, _print_plate_ids.get((printer_id, archive_id))
  7736. except Exception:
  7737. logging.getLogger(__name__).debug("Could not read the print binding for printer %s", printer_id, exc_info=True)
  7738. return None, None
  7739. class PlateHeights(NamedTuple):
  7740. """Where the finished print is, and where its end G-code left the plate (#2547)."""
  7741. print_top: float
  7742. """The plate's ``max_z_height``: the Z the toolhead sat at for the last layer."""
  7743. end_z: float
  7744. """The last Z the end G-code moved to: where the plate is now, and where it is parked back to."""
  7745. async def _plate_heights_for_current_print(
  7746. printer_id: int,
  7747. data: dict,
  7748. logger,
  7749. archive_id: int | None,
  7750. plate_id: int | None = None,
  7751. ) -> PlateHeights | None:
  7752. """Heights of the print that just finished on ``printer_id``, or None (#2547).
  7753. These numbers become the targets of real Z moves, so every step here refuses
  7754. rather than guesses. Both have to be known: without the end Z there is no
  7755. position to park the plate back to that the printer's own G-code chose, and
  7756. the firmware won't stop a made-up one at the travel limit (#2579). A height belonging to some *other* print is the one
  7757. failure that could drive the nozzle into the model: 20 mm carried onto a
  7758. 200 mm print would command the plate up through the part.
  7759. Three independent things therefore have to agree before a height is
  7760. returned:
  7761. 1. **Binding.** ``archive_id`` is the archive Bambuddy bound to this print
  7762. when it started (see ``_bound_print``), never one found by name. A name
  7763. is not an identity: every plate of a multi-plate file shares one, and a
  7764. reprint reuses its archive row, so the newest archive with the right
  7765. name can be another plate of the same file. Support bundles show the
  7766. old name lookup doing exactly that (archive 288 for a print bound to
  7767. 169, 313 layers against 125), stopped only by the layer check below.
  7768. No binding, no move.
  7769. 2. **Identity.** The bound archive's name still has to be the finished
  7770. print's ``subtask_name``, compared in ``_normalise_subtask_name`` form
  7771. because the printer echoes spaces as underscores (#3240). Exact: not a
  7772. substring, and not the completion check's tolerance for a truncated
  7773. echo.
  7774. 3. **Corroboration.** The archive's layer count (parsed from the 3MF) has to
  7775. match the layer count the printer itself reported over MQTT for the print
  7776. that just ended. These come from genuinely different sources, so a
  7777. mismatch means the row is not this print, whatever its name says.
  7778. ``plate_id`` is the plate the job was dispatched with; a reprint of one
  7779. plate of a multi-plate file reuses the archive row without updating its
  7780. ``plate_id``. ``completed`` is accepted alongside ``printing`` only because
  7781. ``on_print_complete`` may already have flipped the status by the time we
  7782. run.
  7783. """
  7784. subtask_name = (data.get("subtask_name") or "").strip()
  7785. if not subtask_name:
  7786. # Nothing to identify the print by — refuse rather than fall back to
  7787. # "whatever ran last on this printer".
  7788. logger.info("[PLATE-RESTORE] printer %s: print has no name to match on — skipping", printer_id)
  7789. return None
  7790. if archive_id is None:
  7791. logger.info(
  7792. "[PLATE-RESTORE] printer %s: %r is not bound to an archive — skipping",
  7793. printer_id,
  7794. subtask_name,
  7795. )
  7796. return None
  7797. try:
  7798. from backend.app.models.archive import PrintArchive
  7799. from backend.app.utils.threemf_tools import extract_end_z_from_3mf, extract_max_z_height_from_3mf
  7800. async with async_session() as db:
  7801. result = await db.execute(
  7802. select(PrintArchive).where(
  7803. PrintArchive.id == archive_id,
  7804. PrintArchive.printer_id == printer_id,
  7805. PrintArchive.status.in_(("printing", "completed")),
  7806. PrintArchive.deleted_at.is_(None),
  7807. )
  7808. )
  7809. archive = result.scalar_one_or_none()
  7810. if archive is None or not archive.file_path:
  7811. logger.info("[PLATE-RESTORE] printer %s: archive %s has no usable file — skipping", printer_id, archive_id)
  7812. return None
  7813. if _normalise_subtask_name(subtask_name) not in _archive_name_forms(archive):
  7814. logger.warning(
  7815. "[PLATE-RESTORE] printer %s: archive %s is not named %r — refusing to move the plate",
  7816. printer_id,
  7817. archive_id,
  7818. subtask_name,
  7819. )
  7820. return None
  7821. client = printer_manager.get_client(printer_id)
  7822. reported_layers = getattr(getattr(client, "state", None), "total_layers", None)
  7823. if reported_layers and archive.total_layers and reported_layers != archive.total_layers:
  7824. logger.warning(
  7825. "[PLATE-RESTORE] printer %s: archive %s says %s layers but the printer reported %s "
  7826. "— refusing to move the plate on a height that may not be this print's",
  7827. printer_id,
  7828. archive.id,
  7829. archive.total_layers,
  7830. reported_layers,
  7831. )
  7832. return None
  7833. # Stored relative to the data directory, like every other archive path
  7834. # (#3240: this read a `data_dir` setting that does not exist).
  7835. path = app_settings.base_dir / archive.file_path
  7836. plate = plate_id or archive.plate_id or 1
  7837. print_top = await asyncio.to_thread(extract_max_z_height_from_3mf, path, plate)
  7838. if print_top is None:
  7839. return None
  7840. end_z = await asyncio.to_thread(extract_end_z_from_3mf, path, plate)
  7841. if end_z is None:
  7842. logger.info(
  7843. "[PLATE-RESTORE] printer %s: archive %s has no end G-code position to park at — skipping",
  7844. printer_id,
  7845. archive_id,
  7846. )
  7847. return None
  7848. return PlateHeights(print_top, end_z)
  7849. except Exception as e:
  7850. logger.debug("[PLATE-RESTORE] printer %s: no usable print height: %s", printer_id, e)
  7851. return None
  7852. async def _restore_plate_for_finish_photo(printer_id: int, max_z_height: float, logger) -> bool:
  7853. """Raise the plate back into camera framing before the finish photo (#2547).
  7854. On X1/P1/A1 Bambu's end G-code drops the plate ~100 mm as the last thing it
  7855. does, so by the time ``gcode_state`` reaches FINISH the finished print sits
  7856. far below the camera's natural framing — the complaint behind #1145, #1397
  7857. and #1565. This commands an absolute ``G1 Z`` back to just above the last
  7858. printed layer. Whether that is needed at all is ``_raise_plate_for_finish_photo``'s
  7859. call: plenty of end G-code (H2C, tall prints anywhere) leaves the plate in
  7860. view already.
  7861. Absolute, not relative, is the whole safety argument. ``max_z_height +
  7862. clearance`` is a height the toolhead was physically at seconds earlier, so
  7863. it is inside the travel limits by construction and leaves the nozzle above
  7864. the part. It is also unambiguous across model families: Z is the
  7865. nozzle-to-bed gap whether the bed moves (X1/P1/H2) or the toolhead does
  7866. (A1), so unlike the relative bed-jog path (#1334) there is no sign to get
  7867. wrong. ``M211`` is never touched — see the bed-jog docstring for why
  7868. (#2579).
  7869. Returns True if the move was sent and waited out, False if it was skipped.
  7870. """
  7871. client = printer_manager.get_client(printer_id)
  7872. if client is None:
  7873. return False
  7874. # Re-read state immediately before commanding motion. If the queue has
  7875. # already started the next print, the printer is no longer ours to move.
  7876. state = getattr(client, "state", None)
  7877. if state is None or state.state != "FINISH":
  7878. logger.info(
  7879. "[PLATE-RESTORE] printer %s is in state %s, not FINISH — skipping",
  7880. printer_id,
  7881. getattr(state, "state", "unknown"),
  7882. )
  7883. return False
  7884. target_z = max_z_height + _PLATE_RESTORE_CLEARANCE_MM
  7885. if not client.send_gcode(f"G90\nG1 Z{target_z:.2f} F{_PLATE_RESTORE_FEEDRATE}"):
  7886. logger.warning("[PLATE-RESTORE] printer %s: send failed — capturing where it is", printer_id)
  7887. return False
  7888. logger.info(
  7889. "[PLATE-RESTORE] printer %s: plate to Z%.2f (print top %.2f + %.1f clearance), settling %.0fs",
  7890. printer_id,
  7891. target_z,
  7892. max_z_height,
  7893. _PLATE_RESTORE_CLEARANCE_MM,
  7894. _PLATE_RESTORE_SETTLE_SECONDS,
  7895. )
  7896. await asyncio.sleep(_PLATE_RESTORE_SETTLE_SECONDS)
  7897. return True
  7898. def _park_plate_after_finish_photo(printer_id: int, end_z: float, logger) -> None:
  7899. """Put the plate back where the end G-code left it, after the finish photo (#2547).
  7900. ``end_z`` is the last Z the print's own end G-code moved to, never a height
  7901. worked out here. That is the safety argument: Bambu's templates keep their
  7902. end moves inside each model's travel, and the firmware does not clamp
  7903. G-code sent over MQTT (#2579). Parking at print top + 100 mm instead drove
  7904. the plate of a 238 mm H2C print to the bottom of its travel, a move Bambu's
  7905. own template had declined to make.
  7906. Fire-and-forget: if it doesn't land, the plate is merely high, and the next
  7907. print homes anyway.
  7908. """
  7909. client = printer_manager.get_client(printer_id)
  7910. state = getattr(client, "state", None) if client else None
  7911. if client is None or state is None or state.state != "FINISH":
  7912. return
  7913. client.send_gcode(f"G90\nG1 Z{end_z:.2f} F{_PLATE_RESTORE_FEEDRATE}")
  7914. logger.info("[PLATE-RESTORE] printer %s: plate back to the end G-code's Z%.2f", printer_id, end_z)
  7915. async def _raise_plate_for_finish_photo(
  7916. printer_id: int, data: dict, logger, archive_id: int | None, plate_id: int | None
  7917. ) -> float | None:
  7918. """Raise the plate into view for the finish photo if it needs it (#2547).
  7919. Returns the Z to park the plate back to once the photo is taken, or None
  7920. when the plate wasn't moved. It needs moving only when the end G-code
  7921. left it lower than the restore height; when it is already at or above it
  7922. (the H2C end G-code puts it back 2 mm under the nozzle), raising would be
  7923. a move away from the camera.
  7924. """
  7925. heights = await _plate_heights_for_current_print(printer_id, data, logger, archive_id, plate_id)
  7926. if heights is None:
  7927. logger.info("[PLATE-RESTORE] printer %s: print height unknown — capturing without restore", printer_id)
  7928. return None
  7929. if heights.end_z <= heights.print_top + _PLATE_RESTORE_CLEARANCE_MM:
  7930. logger.info(
  7931. "[PLATE-RESTORE] printer %s: end G-code left the plate at Z%.2f (print top %.2f) — already in view",
  7932. printer_id,
  7933. heights.end_z,
  7934. heights.print_top,
  7935. )
  7936. return None
  7937. if await _plate_restore_is_blocked_by_queue(printer_id):
  7938. logger.info("[PLATE-RESTORE] printer %s has queued work — skipping plate restore", printer_id)
  7939. return None
  7940. if await _restore_plate_for_finish_photo(printer_id, heights.print_top, logger):
  7941. return heights.end_z
  7942. return None
  7943. async def _plate_restore_is_blocked_by_queue(printer_id: int) -> bool:
  7944. """True if a queue item is about to take this printer (#2547).
  7945. The scheduler dispatches the next job the moment a print completes, and a
  7946. plate move interleaved with a print start is not a race worth having. The
  7947. state re-check in ``_restore_plate_for_finish_photo`` closes the tail of
  7948. this window; this closes the head of it.
  7949. """
  7950. try:
  7951. from backend.app.models.print_queue import PrintQueueItem
  7952. async with async_session() as db:
  7953. result = await db.execute(
  7954. select(PrintQueueItem.id)
  7955. .where(
  7956. PrintQueueItem.printer_id == printer_id,
  7957. PrintQueueItem.status.in_(("pending", "printing")),
  7958. )
  7959. .limit(1)
  7960. )
  7961. return result.scalar_one_or_none() is not None
  7962. except Exception as e:
  7963. # Fail closed: if we can't tell, don't move the plate.
  7964. logging.getLogger(__name__).debug(
  7965. "[PLATE-RESTORE] queue check failed for printer %s: %s — skipping restore", printer_id, e
  7966. )
  7967. return True
  7968. async def on_finish_photo_moment(printer_id: int, data: dict):
  7969. """Pre-capture a finish photo when the printer enters stage 22 / FINISH (#1721).
  7970. Fires either at the stage-22 ("Filament unloading") edge — toolhead
  7971. parked, bed not yet dropped, optimal framing — or as a FINISH-state
  7972. fallback for prints that skip stage 22 (cancel, external-spool-only,
  7973. HMS halt, firmware variants). Grabs one frame via the same
  7974. external-camera / RTSP path the post-completion fallback uses, stores
  7975. the JPEG bytes in ``_stage22_finish_frames[printer_id]``, and lets
  7976. ``_background_finish_photo`` consume the cached bytes when it runs.
  7977. Replaces the #1397 "force timelapse on at dispatch" mechanism, which
  7978. caused per-layer nozzle parking on slicer profiles with Timelapse Type
  7979. set to Smooth (#1721). No force-on now means the user's explicit
  7980. timelapse=off in the slicer send dialog is respected.
  7981. """
  7982. logger = logging.getLogger(__name__)
  7983. trigger = data.get("trigger", "unknown")
  7984. timelapse_was_active = bool(data.get("timelapse_was_active"))
  7985. logger.info(
  7986. "[FINISH-PHOTO-MOMENT] printer=%s trigger=%s timelapse_active=%s",
  7987. printer_id,
  7988. trigger,
  7989. timelapse_was_active,
  7990. )
  7991. # If a timelapse is actively recording, skip the pre-capture — the
  7992. # post-completion path will extract the last frame from the recorded
  7993. # video, which still provides the best framing (toolhead parked,
  7994. # before bed drop) without the per-layer parking side effects.
  7995. if timelapse_was_active:
  7996. logger.info(
  7997. "[FINISH-PHOTO-MOMENT] timelapse active for printer %s — skipping pre-capture (last-frame extraction will run post-completion)",
  7998. printer_id,
  7999. )
  8000. return
  8001. # #1790: register the producer-done event BEFORE the first await so the
  8002. # consumer in `_background_finish_photo` — which is dispatched back-to-back
  8003. # with us on the FINISH-state fallback path — sees it as soon as it polls.
  8004. # The `finally` below guarantees `set()` runs on every exit, including
  8005. # early returns and exceptions, so the consumer's bounded wait can't hang.
  8006. producer_done = asyncio.Event()
  8007. _stage22_finish_in_flight[printer_id] = producer_done
  8008. # #3240: also before the first await. On the FINISH-state path
  8009. # `on_print_complete` is dispatched right behind us and pops both the
  8010. # print's archive binding and its plate; read here, they are still there.
  8011. bound_archive_id, bound_plate_id = _bound_print(printer_id, data)
  8012. # #2547: the Z to park the plate back to, set once the plate has actually
  8013. # been raised, and read by the `finally` below. Declared out here so a
  8014. # failure anywhere after the move — a camera timeout, a DB error — still
  8015. # lowers the plate again.
  8016. park_z: float | None = None
  8017. try:
  8018. async with async_session() as db:
  8019. from backend.app.api.routes.settings import get_setting
  8020. from backend.app.models.printer import Printer
  8021. capture_setting = await get_setting(db, "capture_finish_photo")
  8022. if capture_setting is not None and capture_setting.lower() != "true":
  8023. logger.info("[FINISH-PHOTO-MOMENT] capture_finish_photo disabled — skipping pre-capture")
  8024. return
  8025. restore_setting = await get_setting(db, "finish_photo_restore_plate")
  8026. restore_plate_enabled = restore_setting is None or restore_setting.lower() == "true"
  8027. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8028. printer = result.scalar_one_or_none()
  8029. if printer is None:
  8030. logger.warning(
  8031. "[FINISH-PHOTO-MOMENT] printer %s not found in DB",
  8032. printer_id,
  8033. )
  8034. return
  8035. frame_bytes: bytes | None = None
  8036. # #2708: the banked frame arrives already rotated — it comes from
  8037. # `_capture_snapshot_for_notification`, which rotates before returning.
  8038. # Every other source below is a raw grab. Tracking which lets us store
  8039. # exactly one rotation in `_stage22_finish_frames` either way.
  8040. frame_already_rotated = False
  8041. # On the FINISH-state path the End G-code has already run, and two very
  8042. # different situations arrive here needing opposite answers.
  8043. #
  8044. # #1867: if Bambuddy injected End G-code into this print, a SwapMod
  8045. # snippet may have ejected the plate — the scene in front of the camera
  8046. # is no longer the finished print, and no amount of moving the plate
  8047. # brings it back. Use the banked in-print frame instead.
  8048. #
  8049. # #2547: otherwise the print is still sitting there — on X1/P1/A1 ~100 mm
  8050. # lower than the camera frames well — and the toolhead is parked out of
  8051. # the way. That is the *best* moment available on firmware that never emits
  8052. # stage 22 (H2C, A1 Mini) — so capture live, after putting the plate
  8053. # back. Preferring the bank here unconditionally, as this code used to,
  8054. # is what shipped a mid-print photo with the toolhead over the part.
  8055. if trigger == "finish_state" and print_dispatch_context.end_gcode_injected(printer_id):
  8056. banked = _inprint_frame_bank.get(printer_id)
  8057. if banked:
  8058. frame_bytes = banked
  8059. frame_already_rotated = True
  8060. logger.info(
  8061. "[FINISH-PHOTO-MOMENT] End G-code was injected — using banked in-print "
  8062. "frame (%d bytes) instead of a post-swap live grab",
  8063. len(banked),
  8064. )
  8065. else:
  8066. logger.warning(
  8067. "[FINISH-PHOTO-MOMENT] End G-code was injected for printer %s but the "
  8068. "in-print bank is empty — falling back to a live grab, which may show a "
  8069. "swapped or empty plate",
  8070. printer_id,
  8071. )
  8072. # `park_z` is set only once the plate is actually up, because the
  8073. # `finally` reads it to decide whether it owes a move back down.
  8074. #
  8075. # Never on a print whose End G-code Bambuddy injected, even when the bank
  8076. # came up empty above: that machine may have just ejected its plate, and
  8077. # driving Z into whatever a swap mechanism is doing is not a risk worth
  8078. # taking for a photo of a bed we already know may be bare.
  8079. if (
  8080. frame_bytes is None
  8081. and trigger == "finish_state"
  8082. and restore_plate_enabled
  8083. and not print_dispatch_context.end_gcode_injected(printer_id)
  8084. ):
  8085. park_z = await _raise_plate_for_finish_photo(printer_id, data, logger, bound_archive_id, bound_plate_id)
  8086. # Chamber light for the live capture below (#1655). The banked frame
  8087. # above was taken without it, like every in-print bank.
  8088. async with camera_light(printer if frame_bytes is None else None):
  8089. if frame_bytes is None and printer.external_camera_enabled and printer.external_camera_url:
  8090. from backend.app.api.routes.camera import live_frame_for_capture
  8091. from backend.app.services.external_camera import capture_frame
  8092. # #2707: this used to collide with the live view and fail, which is
  8093. # how finish-photo notifications went out with no image attached.
  8094. # Leaving frame_bytes None keeps the rest of the fallback chain.
  8095. defer, buffered = live_frame_for_capture(printer_id)
  8096. if defer:
  8097. frame_bytes = buffered
  8098. else:
  8099. frame_bytes = await capture_frame(
  8100. printer.external_camera_url,
  8101. printer.external_camera_type or "mjpeg",
  8102. snapshot_url=printer.external_camera_snapshot_url,
  8103. )
  8104. if frame_bytes:
  8105. logger.info(
  8106. "[FINISH-PHOTO-MOMENT] captured external-camera frame (%d bytes)",
  8107. len(frame_bytes),
  8108. )
  8109. elif frame_bytes is None:
  8110. from backend.app.api.routes.camera import get_buffered_frame
  8111. buffered = get_buffered_frame(printer_id)
  8112. if buffered:
  8113. frame_bytes = buffered
  8114. logger.info(
  8115. "[FINISH-PHOTO-MOMENT] used buffered RTSP frame (%d bytes)",
  8116. len(frame_bytes),
  8117. )
  8118. else:
  8119. from backend.app.services.camera import capture_camera_frame_bytes
  8120. frame_bytes = await capture_camera_frame_bytes(
  8121. ip_address=printer.ip_address,
  8122. access_code=printer.access_code,
  8123. model=printer.model,
  8124. timeout=15,
  8125. )
  8126. if frame_bytes:
  8127. logger.info(
  8128. "[FINISH-PHOTO-MOMENT] captured RTSP frame (%d bytes)",
  8129. len(frame_bytes),
  8130. )
  8131. if frame_bytes:
  8132. if not frame_already_rotated:
  8133. frame_bytes = _apply_camera_rotation(frame_bytes, printer, logger)
  8134. _stage22_finish_frames[printer_id] = frame_bytes
  8135. else:
  8136. logger.warning(
  8137. "[FINISH-PHOTO-MOMENT] no frame captured for printer %s — post-completion fallback will retry",
  8138. printer_id,
  8139. )
  8140. except Exception as e:
  8141. logger.warning(
  8142. "[FINISH-PHOTO-MOMENT] pre-capture failed for printer %s: %s",
  8143. printer_id,
  8144. e,
  8145. )
  8146. finally:
  8147. # #2547: we raised the plate, so we own lowering it — including when the
  8148. # capture above failed or threw partway through.
  8149. if park_z is not None:
  8150. try:
  8151. _park_plate_after_finish_photo(printer_id, park_z, logger)
  8152. except Exception as e:
  8153. logger.warning("[PLATE-RESTORE] printer %s: could not lower plate: %s", printer_id, e)
  8154. # #1790: always unblock the consumer's bounded wait — whether we stored
  8155. # a frame, gave up, or hit an exception. Local ref means cleanup of the
  8156. # dict entry by the consumer doesn't affect signalling.
  8157. producer_done.set()
  8158. def _subtask_name_from_filename(filename: str) -> str:
  8159. """Recover the subtask name a print command would have carried for *filename*.
  8160. The dispatcher derives the printer-facing subtask name from the archive's
  8161. file name, so stripping the extensions back off gives the value MQTT echoes
  8162. on completion. Only the two extensions Bambuddy actually stores are removed,
  8163. and in the order they nest (``.gcode.3mf``), so a model whose own name
  8164. contains a dot -- ``My.Model.3mf`` -- keeps it.
  8165. """
  8166. name = PurePosixPath(filename).name
  8167. for suffix in (".3mf", ".gcode"):
  8168. if name.lower().endswith(suffix):
  8169. name = name[: -len(suffix)]
  8170. return name
  8171. def _subtask_is_the_file(subtask_name: str, filename: str) -> bool:
  8172. """Whether the subtask name is the printed 3MF's own file name.
  8173. A print started from the printer's screen reports the same full name, with
  8174. its extension, as both subtask and file (#3009). A dispatched print reports
  8175. a bare subtask name; one that merely ends in ".3mf" can be a model named
  8176. that, whose file is ``Foo.3mf.gcode.3mf``, so it does not count.
  8177. """
  8178. if not subtask_name or not subtask_name.lower().endswith(".3mf"):
  8179. return False
  8180. return PurePosixPath((filename or "").split("://", 1)[-1]).name == subtask_name
  8181. # How the printer marks a subtask name it had to cut short. Observed on real
  8182. # hardware at ~100 characters, but the cut-off is not a fixed character count
  8183. # (a name with multibyte characters came back at 98), so match the marker
  8184. # rather than a length.
  8185. _SUBTASK_TRUNCATION_MARKER = "..."
  8186. def _normalise_subtask_name(name: str) -> str:
  8187. """Canonical form for comparing a dispatched name against MQTT's echo.
  8188. The printer does not echo the name back verbatim: it substitutes
  8189. underscores for spaces. ``H2D_Carbon_Filter_(V2)_Body & Solid Lid`` is
  8190. dispatched and ``H2D_Carbon_Filter_(V2)_Body_&_Solid_Lid`` comes back.
  8191. The 3MF lookup in this module has always known that -- it builds
  8192. space-to-underscore variants of every candidate filename, and its
  8193. directory search normalises both sides before comparing. This exists so
  8194. the completion check reads the same rule from the same place instead of
  8195. growing its own, which is exactly how it came to disagree (#2829).
  8196. A space at either end is substituted too: ``Part .gcode.3mf`` is
  8197. dispatched as ``Part `` and echoed as ``Part_``. Underscores at the ends
  8198. are dropped after the substitution so both sides agree (#3241).
  8199. """
  8200. return name.strip().replace(" ", "_").strip("_").casefold()
  8201. def _3mf_files_for_print(paths: list[str], search_term: str) -> list[str]:
  8202. """The 3MF files of the printer's folder listings that are this print's.
  8203. A file of the print's own name wins, in whichever folder. Only without one
  8204. is a file that merely contains the name taken, and then only when it is
  8205. the one such file: "Cube_large.3mf" is not "Cube"'s file, and of several
  8206. candidates none can be told apart. Paths keep the listings' order.
  8207. """
  8208. threemf = [path for path in paths if path.lower().endswith(".3mf")]
  8209. exact = [path for path in threemf if _subtask_names_match(_subtask_name_from_filename(path), search_term)]
  8210. if exact:
  8211. return exact
  8212. term = _normalise_subtask_name(search_term)
  8213. partial = [path for path in threemf if term in _normalise_subtask_name(PurePosixPath(path).name)]
  8214. if len(partial) > 1:
  8215. logging.getLogger(__name__).info(
  8216. "Several 3MF files contain %r and none is named for it: %s", search_term, partial
  8217. )
  8218. return []
  8219. return partial
  8220. def _subtask_names_match(expected: str, observed: str) -> bool:
  8221. """Whether two subtask names describe the same print.
  8222. Beyond the space/underscore substitution, the printer truncates long names
  8223. and marks the cut with ``...``. A truncated echo has to count as a match or
  8224. every print with a long name strands its queue item the same way.
  8225. """
  8226. expected_n = _normalise_subtask_name(expected)
  8227. observed_n = _normalise_subtask_name(observed)
  8228. if expected_n == observed_n:
  8229. return True
  8230. # Either side can be the truncated one: the printer truncates what it
  8231. # echoes, and an archive whose own filename was recorded from a previous
  8232. # truncated echo carries the marker too.
  8233. for full, cut in ((expected_n, observed_n), (observed_n, expected_n)):
  8234. if not cut.endswith(_SUBTASK_TRUNCATION_MARKER):
  8235. continue
  8236. # The full side has lost its edge underscores; a cut that lands right
  8237. # after one keeps it, so drop it here too (#3241). Nothing left before
  8238. # the marker is no evidence of anything, and must not match every name.
  8239. kept = cut[: -len(_SUBTASK_TRUNCATION_MARKER)].rstrip("_")
  8240. if kept and full.startswith(kept):
  8241. return True
  8242. return False
  8243. def _dispatched_name_matches_echo(expected: str, observed: str) -> bool:
  8244. """Whether the printer's subtask echo is the name a print was dispatched as.
  8245. A dispatched print is echoed under its bare name at start, but a printer
  8246. that picks a print back up -- Bambuddy reconnecting to a print already
  8247. running, or an A1 mini resuming after power-loss recovery -- can report the
  8248. file name with its extension instead (#3485). The echo is tried both as it
  8249. came and with the extensions stripped; the raw form still has to count, so
  8250. a model named ``Foo.3mf`` (file ``Foo.3mf.gcode.3mf``) keeps matching.
  8251. """
  8252. return _subtask_names_match(expected, observed) or _subtask_names_match(
  8253. expected, _subtask_name_from_filename(observed)
  8254. )
  8255. async def _queue_item_dispatched_name(db, item) -> str:
  8256. """The subtask name *item* was dispatched under, or "" when unknowable.
  8257. A row with no archive, or an archive with no file name, is unverifiable
  8258. rather than wrong; every caller answers that with its permissive branch.
  8259. """
  8260. if item.archive_id is None:
  8261. return ""
  8262. from backend.app.models.archive import PrintArchive
  8263. archive = await db.get(PrintArchive, item.archive_id)
  8264. if archive is None or not archive.filename:
  8265. return ""
  8266. return _subtask_name_from_filename(archive.filename)
  8267. async def _completion_belongs_to_queue_item(db, item, data: dict) -> bool:
  8268. """Whether this completion event is plausibly about *item*'s print.
  8269. The caller finds its queue row by printer and ``status='printing'`` alone,
  8270. which is all a completion event gives it -- there is no run identifier in
  8271. the MQTT payload to match on. That makes the lookup indiscriminate: any
  8272. completion delivered for this printer closes whichever row happens to be
  8273. printing, however unrelated. Comparing the subtask name against the archive
  8274. the row was dispatched with costs one primary-key load and rules that out.
  8275. Deliberately permissive: it answers False only on a positive disagreement
  8276. between two names we actually have. A row with no archive, an archive with
  8277. no file name, or an event with no subtask name is unverifiable rather than
  8278. wrong, and refusing those would strand the item in ``printing`` and wedge
  8279. the printer's queue -- a worse failure than the one being prevented.
  8280. """
  8281. observed = (data.get("subtask_name") or "").strip()
  8282. if not observed or item.archive_id is None:
  8283. return True
  8284. from backend.app.models.archive import PrintArchive
  8285. archive = await db.get(PrintArchive, item.archive_id)
  8286. if archive is None or not archive.filename:
  8287. return True
  8288. expected = _subtask_name_from_filename(archive.filename)
  8289. if not expected or _dispatched_name_matches_echo(expected, observed):
  8290. return True
  8291. logging.getLogger(__name__).warning(
  8292. "Ignoring print completion for queue item %s: it was dispatched as %r "
  8293. "(archive %s, %s) but the completion reports subtask %r. Leaving the item "
  8294. "printing rather than closing a run this event is not about.",
  8295. item.id,
  8296. expected,
  8297. archive.id,
  8298. archive.filename,
  8299. observed,
  8300. )
  8301. return False
  8302. async def _recover_fallback_from_cache_before_eviction(printer_id: int, data: dict) -> None:
  8303. """Spend the 3MF download cache on a still-empty fallback archive.
  8304. ``on_print_complete`` drops the cache as its first act, which deletes the
  8305. file. If the cover endpoint (or anything else) pulled the 3MF while the
  8306. print ran and the archive never got one, this is the last moment those bytes
  8307. exist (#2957).
  8308. """
  8309. logger = logging.getLogger(__name__)
  8310. names = [
  8311. n
  8312. for n in (data.get("filename"), data.get("subtask_name"), (data.get("raw_data") or {}).get("subtask_name"))
  8313. if n
  8314. ]
  8315. for name in names:
  8316. try:
  8317. cached = get_cached_3mf(printer_id, name)
  8318. # The print has ended: its completion prices the archive, so no
  8319. # print-start estimate is scheduled to race it.
  8320. if cached and await try_recover_fallback_archive(printer_id, name, cached, price=False):
  8321. return
  8322. except Exception as e:
  8323. logger.debug("[RECOVER] Pre-eviction recovery for %s failed: %s", name, e)
  8324. # Longest the print's end waits for its file to be deleted from the SD card
  8325. _SD_CLEANUP_BUDGET_SECONDS = 45.0
  8326. async def _cleanup_sd_card_after_print(
  8327. printer_id: int, subtask_name: str | None, archive_id: int | None, logger
  8328. ) -> None:
  8329. """Delete a finished print's file from the printer's SD card."""
  8330. # Cleanup: delete uploaded file from printer SD card to prevent phantom prints (Issue #374, #1542)
  8331. # The print scheduler uploads files to the SD card root (/). Some printers (e.g. P1S, A1)
  8332. # auto-start files found in root on power cycle, causing ghost prints.
  8333. # Must run before the archive_id early-return so it executes even when archiving is disabled.
  8334. try:
  8335. if subtask_name:
  8336. archive_filename: str | None = None
  8337. async with async_session() as db:
  8338. from backend.app.models.archive import PrintArchive
  8339. from backend.app.models.printer import Printer
  8340. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8341. printer = result.scalar_one_or_none()
  8342. if archive_id:
  8343. archive_row = await db.execute(select(PrintArchive.filename).where(PrintArchive.id == archive_id))
  8344. archive_filename = archive_row.scalar_one_or_none()
  8345. if printer:
  8346. from backend.app.services.bambu_ftp import DeleteResult, delete_file_async
  8347. from backend.app.utils.filename import derive_remote_filename
  8348. # Primary candidate: the exact path the dispatcher uploaded to
  8349. # (derived from archive.filename via the same rule as upload).
  8350. # Without it, a library row that ended up with a doubled
  8351. # .gcode.3mf (#1542) leaves the real file behind because the
  8352. # subtask_name + ext fallbacks below don't match what's on the
  8353. # SD card. Fallbacks remain for archive-less prints (subtask
  8354. # never resolved to an archive) and for older naming variants.
  8355. candidate_paths: list[str] = []
  8356. if archive_filename:
  8357. candidate_paths.append(f"/{derive_remote_filename(archive_filename)}")
  8358. for ext in (".3mf", ".gcode"):
  8359. fallback = f"/{subtask_name}{ext}"
  8360. if fallback not in candidate_paths:
  8361. candidate_paths.append(fallback)
  8362. # Three outcomes track across all candidates so the final log
  8363. # line reflects what actually happened. The A1 in #1721 always
  8364. # ends here with ``any_not_found=True`` and the others False
  8365. # — its firmware auto-cleans the SD card before our cleanup
  8366. # runs, every candidate FTP-DELE returns 550, and the old
  8367. # code burned 3 retries × 2 s × 3 candidates per print
  8368. # logging a misleading "may linger" WARNING on a successful
  8369. # print.
  8370. any_deleted = False
  8371. any_real_failure = False
  8372. any_not_found = False
  8373. files_in_use = _sd_files_in_use(printer_manager.get_status(printer_id))
  8374. for remote_path in candidate_paths:
  8375. if PurePosixPath(remote_path).name.lower() in files_in_use:
  8376. logger.info(
  8377. "SD card cleanup: keeping %s on printer %s, which is printing it now",
  8378. remote_path,
  8379. printer.name,
  8380. )
  8381. continue
  8382. # Retry only the FAILED case — 550 NOT_FOUND will never
  8383. # recover by waiting, so a "file isn't here" answer
  8384. # advances immediately to the next candidate without
  8385. # consuming the retry budget.
  8386. for attempt in range(1, 4):
  8387. try:
  8388. delete_result = await delete_file_async(
  8389. printer.ip_address,
  8390. printer.access_code,
  8391. remote_path,
  8392. printer_model=printer.model,
  8393. )
  8394. except Exception as e:
  8395. delete_result = DeleteResult.FAILED
  8396. logger.warning(
  8397. "SD card cleanup attempt %d/3 raised for %s: %s",
  8398. attempt,
  8399. remote_path,
  8400. e,
  8401. )
  8402. if delete_result == DeleteResult.DELETED:
  8403. any_deleted = True
  8404. logger.info("Deleted %s from printer %s SD card", remote_path, printer.name)
  8405. break
  8406. if delete_result == DeleteResult.NOT_FOUND:
  8407. any_not_found = True
  8408. break # 550 will not recover; try next candidate
  8409. # FAILED: real error — retry with backoff, then give up
  8410. if attempt < 3:
  8411. await asyncio.sleep(2)
  8412. else:
  8413. any_real_failure = True
  8414. logger.warning(
  8415. "SD card cleanup failed after 3 attempts for %s "
  8416. "(network/auth/transient error — file may linger on SD card)",
  8417. remote_path,
  8418. )
  8419. if not any_deleted and not any_real_failure and any_not_found:
  8420. # Every candidate said "not here." Either the printer
  8421. # firmware swept the SD card itself (common on A1) or the
  8422. # dispatcher's upload path doesn't match our candidate
  8423. # rule. Either way: nothing to clean up, no warning.
  8424. logger.debug(
  8425. "SD card cleanup: nothing to delete on %s — every candidate returned 550 "
  8426. "(printer likely self-cleaned)",
  8427. printer.name,
  8428. )
  8429. except Exception as e:
  8430. logger.warning("SD card file cleanup failed for printer %s: %s", printer_id, e)
  8431. async def on_print_complete(printer_id: int, data: dict):
  8432. """Handle print completion - update the archive status.
  8433. The printer counts as printing until the print's usage is booked (or this
  8434. ends), so the AMS remain% weight sync can't charge it a second time while
  8435. the usage tracker works (WU-3). Marked before the first await.
  8436. """
  8437. from backend.app.services.usage_tracker import done_completing, mark_completing
  8438. mark_completing(printer_id)
  8439. try:
  8440. await _close_out_print(printer_id, data)
  8441. finally:
  8442. done_completing(printer_id)
  8443. async def _close_out_print(printer_id: int, data: dict):
  8444. import time
  8445. logger = logging.getLogger(__name__)
  8446. start_time = time.time()
  8447. def log_timing(section: str):
  8448. elapsed = time.time() - start_time
  8449. logger.info("[TIMING] %s: %.3fs elapsed", section, elapsed)
  8450. logger.info("[CALLBACK] on_print_complete started for printer %s", printer_id)
  8451. octoeverywhere_detection_service.reset_printer(printer_id)
  8452. # A kill-switch stop sends its provider notification immediately. Keep the
  8453. # task so the later notification path can await it and avoid a duplicate;
  8454. # if that immediate attempt failed, the regular completion path retries.
  8455. kill_switch_notification_task = _kill_switch_notification_tasks.pop(printer_id, None)
  8456. # Last chance before the bytes go: if this print's archive is still an empty
  8457. # fallback and something downloaded the 3MF while it ran, fill the archive in
  8458. # now. The cover endpoint's copy lives in exactly this cache, and clearing it
  8459. # below deletes the file (#2957).
  8460. await _recover_fallback_from_cache_before_eviction(printer_id, data)
  8461. # A pending cool-off retry has nothing left to recover for — the cache is
  8462. # about to be dropped and the print is over.
  8463. retry_task = _fallback_3mf_retry_tasks.pop(printer_id, None)
  8464. if retry_task and not retry_task.done():
  8465. retry_task.cancel()
  8466. # Drop the 3MF download cache for this printer (#972). The print is over,
  8467. # nothing else legitimately needs the bytes; keeping them would only risk
  8468. # handing a stale file to the next print if it reuses the same name.
  8469. clear_3mf_cache(printer_id)
  8470. try:
  8471. ws_data = {
  8472. "status": data.get("status"),
  8473. "filename": data.get("filename"),
  8474. "subtask_name": data.get("subtask_name"),
  8475. "timelapse_was_active": data.get("timelapse_was_active"),
  8476. }
  8477. await ws_manager.send_print_complete(printer_id, ws_data)
  8478. log_timing("WebSocket send_print_complete")
  8479. except Exception as e:
  8480. logger.warning("[CALLBACK] WebSocket send_print_complete failed: %s", e)
  8481. # Capture user info before clearing (needed for print log entry)
  8482. _print_user_info = printer_manager.get_current_print_user(printer_id)
  8483. # Clear current print user tracking (Issue #206)
  8484. printer_manager.clear_current_print_user(printer_id)
  8485. # If the user explicitly stopped this print from the queue UI the printer will
  8486. # report "failed" or "aborted" via MQTT. Override that to "cancelled" so the
  8487. # correct "print stopped" notification/email is sent instead of a failure alert.
  8488. _raw_status = data.get("status", "completed")
  8489. if printer_id in _user_stopped_printers and _raw_status in ("failed", "aborted"):
  8490. logger.info(
  8491. "[CALLBACK] Overriding status '%s' -> 'cancelled' for printer %s (print was stopped from queue by user)",
  8492. _raw_status,
  8493. printer_id,
  8494. )
  8495. data = {**data, "status": "cancelled"}
  8496. elif printer_id in _kill_switch_stopped and _raw_status in ("failed", "aborted"):
  8497. logger.info(
  8498. "[CALLBACK] Overriding status '%s' -> 'cancelled' for printer %s (stopped by the printer kill switch)",
  8499. _raw_status,
  8500. printer_id,
  8501. )
  8502. data = {**data, "status": "cancelled", "_kill_switch": True}
  8503. _user_stopped_printers.discard(printer_id)
  8504. _kill_switch_stopped.discard(printer_id)
  8505. # Raise the plate-clear gate for queued dispatch (#961). Any terminal status
  8506. # may have left material on the bed: a user can cancel ten hours into a
  8507. # twelve-hour print, a printer can self-abort mid-job after a clog, and a
  8508. # touchscreen-stop reports `aborted` rather than `cancelled` because
  8509. # `_user_stopped_printers` is only populated when the user stops via the
  8510. # Bambuddy queue UI. Earlier code raised the flag only for completed/failed,
  8511. # which auto-dispatched the next queued print onto a fouled bed two seconds
  8512. # after a touchscreen-abort (#1171). Persisted to DB so the gate survives
  8513. # Auto Off power cycles and Bambuddy restarts.
  8514. try:
  8515. notify_live_activities.print_finished(printer_id, data)
  8516. except Exception:
  8517. logger.exception("Notify Live Activity completion hook failed for printer %s", printer_id)
  8518. _final_status = data.get("status", "completed")
  8519. if _final_status in ("completed", "failed", "aborted", "cancelled"):
  8520. printer_manager.set_awaiting_plate_clear(printer_id, True)
  8521. # MQTT relay - publish print complete
  8522. try:
  8523. printer_info = printer_manager.get_printer(printer_id)
  8524. if printer_info:
  8525. await mqtt_relay.on_print_complete(
  8526. printer_id,
  8527. printer_info.name,
  8528. printer_info.serial_number,
  8529. data.get("filename", ""),
  8530. data.get("subtask_name", ""),
  8531. data.get("status", "completed"),
  8532. )
  8533. except Exception:
  8534. pass # Don't fail print complete callback if MQTT fails
  8535. filename = data.get("filename", "")
  8536. subtask_name = data.get("subtask_name", "")
  8537. if not filename and not subtask_name:
  8538. # Which print ended can't be told: the printer reported no name since
  8539. # Bambuddy started (a print that failed while preparing, a calibration).
  8540. # Charge nothing, but close the print session, which otherwise kept the
  8541. # AMS weight sync off until the next print (AS-5). A print archived as
  8542. # running is closed and charged by the reconnect reconcile.
  8543. logger.warning("Print complete without filename or subtask_name")
  8544. try:
  8545. from backend.app.services.usage_tracker import discard_session
  8546. async with async_session() as db:
  8547. await discard_session(db, printer_id)
  8548. except Exception as e:
  8549. logger.warning("Failed to clear print session for printer %s: %s", printer_id, e)
  8550. return
  8551. logger.info("Print complete - filename: %s, subtask: %s, status: %s", filename, subtask_name, data.get("status"))
  8552. # Build list of possible keys to try (matching how they were registered in on_print_start)
  8553. possible_keys = _completion_print_keys(printer_id, filename, subtask_name)
  8554. # Find the archive for this print
  8555. logger.info("Looking for archive in _active_prints, keys to try: %s...", possible_keys[:5])
  8556. logger.info("Current _active_prints: %s", list(_active_prints.keys()))
  8557. archive_id = None
  8558. for key in possible_keys:
  8559. archive_id = _active_prints.pop(key, None)
  8560. if archive_id:
  8561. _owned_print_archive_ids.discard(archive_id)
  8562. logger.info("Found archive %s with key %s", archive_id, key)
  8563. # Also clean up the printer's other keys pointing to this archive;
  8564. # another printer's are a plate of it still printing there.
  8565. keys_to_remove = [k for k, v in _active_prints.items() if v == archive_id and k[0] == printer_id]
  8566. for k in keys_to_remove:
  8567. _active_prints.pop(k, None)
  8568. break
  8569. # The archive bound when the print started, before the name search below.
  8570. # Only this one may give the finish-photo plate restore a height (#3240):
  8571. # the search matches names by substring.
  8572. bound_archive_id = archive_id
  8573. if not archive_id and data.get("_reconciled") and data.get("_reconciled_archive_id"):
  8574. # The reconcile names the archive whose run on this printer it
  8575. # closes. Only while that run is open: the print's real end may have
  8576. # closed it meanwhile, or closed the print without managing to write
  8577. # it, and either way there is nothing left to end twice.
  8578. if await _archive_of_open_run(printer_id) != data["_reconciled_archive_id"]:
  8579. logger.info(
  8580. "[RECONCILE] Archive %s has no run open on printer %s any more; nothing to close",
  8581. data["_reconciled_archive_id"],
  8582. printer_id,
  8583. )
  8584. return
  8585. archive_id = data["_reconciled_archive_id"]
  8586. ending_job = str(data.get("subtask_id") or "").strip()
  8587. if not archive_id and ending_job not in ("", "0"):
  8588. # Not bound (a restart since the start): this job's run, open on the
  8589. # printer. Only by its job id: a run open there can be of the print
  8590. # running now, while the end is of an earlier one closed late (the
  8591. # restart's reconcile of a missed end).
  8592. archive_id = await _archive_of_open_run(printer_id, subtask_id=ending_job)
  8593. if archive_id:
  8594. logger.info("Found archive %s by the run of job %s open on printer %s", archive_id, ending_job, printer_id)
  8595. if not archive_id:
  8596. # Try to find by filename or subtask_name if not tracked (for prints started before app)
  8597. async with async_session() as db:
  8598. from backend.app.models.archive import PrintArchive
  8599. # Try matching by subtask_name (stored as print_name) first. The
  8600. # same name rule as the queue's completion check: a name inside
  8601. # another ("Clip" in "Cable Clip") is a different print, and the
  8602. # printer's underscores for spaces and truncation still match.
  8603. if subtask_name:
  8604. printing = await db.scalars(
  8605. select(PrintArchive)
  8606. .where(PrintArchive.printer_id == printer_id)
  8607. .where(PrintArchive.status == "printing")
  8608. .order_by(PrintArchive.created_at.desc())
  8609. )
  8610. # A print started from the printer's screen reports its file
  8611. # name, extensions included, as the subtask (#3009).
  8612. reported = {subtask_name, _subtask_name_from_filename(subtask_name)}
  8613. archive = next(
  8614. (
  8615. candidate
  8616. for candidate in printing
  8617. if any(
  8618. (candidate.print_name and _subtask_names_match(candidate.print_name, name))
  8619. or (
  8620. candidate.filename
  8621. and _subtask_names_match(_subtask_name_from_filename(candidate.filename), name)
  8622. )
  8623. for name in reported
  8624. )
  8625. ),
  8626. None,
  8627. )
  8628. if archive:
  8629. archive_id = archive.id
  8630. logger.info("Found archive %s by subtask_name match: %s", archive_id, subtask_name)
  8631. # Also try by filename
  8632. if not archive_id and filename:
  8633. result = await db.execute(
  8634. select(PrintArchive)
  8635. .where(PrintArchive.printer_id == printer_id)
  8636. .where(PrintArchive.filename == filename)
  8637. .where(PrintArchive.status == "printing")
  8638. .order_by(PrintArchive.created_at.desc())
  8639. .limit(1)
  8640. )
  8641. archive = result.scalar_one_or_none()
  8642. if archive:
  8643. archive_id = archive.id
  8644. # Bounded: a printer whose FTP hangs on DELE took up to 3 candidates x 3
  8645. # attempts x a 60 s timeout, about nine minutes, before the queue item was
  8646. # closed below. The stranded-job closer gives up after five and closed it
  8647. # first, so the job's auto-off and its billing never ran. Inline rather
  8648. # than in the background, so the next job of the same file isn't released
  8649. # (and uploaded under the name being deleted) while the cleanup still
  8650. # works on it. A delete already sent may still finish in its FTP thread
  8651. # after the cutoff, as one past delete_file_async's own timeout always did.
  8652. try:
  8653. await asyncio.wait_for(
  8654. _cleanup_sd_card_after_print(printer_id, subtask_name, archive_id, logger),
  8655. timeout=_SD_CLEANUP_BUDGET_SECONDS,
  8656. )
  8657. except TimeoutError:
  8658. logger.warning(
  8659. "SD card cleanup on printer %s took over %ss; leaving the file there and closing the print",
  8660. printer_id,
  8661. _SD_CLEANUP_BUDGET_SECONDS,
  8662. )
  8663. log_timing("SD card cleanup")
  8664. # Update queue item status early — must run before the archive_id early-return
  8665. # so queue items don't get stuck in "printing" when archive lookup fails.
  8666. # Uses run_with_retry to handle SQLite "database is locked" errors (#897).
  8667. queue_item_id = None
  8668. billing_run_id: str | None = None
  8669. billing_user_id: int | None = None
  8670. billing_cost_center_id: int | None = None
  8671. billing_plate_id: int | None = None
  8672. queue_status = None
  8673. queue_auto_off = False
  8674. try:
  8675. from backend.app.core.database import run_with_retry
  8676. from backend.app.models.print_queue import PrintQueueItem
  8677. async def _update_queue_status(db):
  8678. nonlocal billing_run_id, billing_user_id, billing_cost_center_id, billing_plate_id
  8679. nonlocal queue_item_id, queue_status, queue_auto_off
  8680. result = await db.execute(
  8681. select(PrintQueueItem)
  8682. .where(PrintQueueItem.printer_id == printer_id)
  8683. .where(PrintQueueItem.status == "printing")
  8684. )
  8685. printing_items = list(result.scalars().all())
  8686. if len(printing_items) > 1:
  8687. logger.warning(
  8688. "BUG: Multiple queue items in 'printing' status for printer %s: %s",
  8689. printer_id,
  8690. [(i.id, i.archive_id, i.library_file_id) for i in printing_items],
  8691. )
  8692. item = printing_items[0] if printing_items else None
  8693. if item is not None and not await _completion_belongs_to_queue_item(db, item, data):
  8694. return
  8695. if item:
  8696. queue_status = data.get("status", "completed")
  8697. # MQTT sends "aborted" for cancelled prints; normalise to
  8698. # "cancelled" so it matches the queue schema Literal.
  8699. if queue_status == "aborted":
  8700. queue_status = "cancelled"
  8701. item.status = queue_status
  8702. item.completed_at = datetime.now(timezone.utc)
  8703. if queue_status == "failed" and not item.error_message:
  8704. item.error_message = _format_hms_error_summary(data.get("hms_errors") or [])
  8705. # Bump usage counters on the source library file so admins can
  8706. # sort by "last printed" and (eventually) auto-purge stale
  8707. # files — #1008.
  8708. await _bump_library_file_usage_if_completed(db, item, queue_status)
  8709. await db.commit()
  8710. queue_item_id = item.id
  8711. billing_run_id = item.billing_run_id
  8712. billing_user_id = item.created_by_id
  8713. billing_cost_center_id = item.cost_center_id
  8714. billing_plate_id = item.plate_id
  8715. queue_auto_off = item.auto_off_after
  8716. logger.info("Updated queue item %s status to %s", item.id, queue_status)
  8717. await run_with_retry(_update_queue_status, label="queue status update")
  8718. # Post-commit side effects (notifications, MQTT relay, auto-off) use
  8719. # their own sessions and have their own error handling — no retry needed.
  8720. if queue_item_id is not None:
  8721. # Batch orders (#342): this run may have been the last one an order
  8722. # owed. Re-evaluate here rather than lazily on read, so a finished
  8723. # order reports itself complete without someone opening the page.
  8724. try:
  8725. from backend.app.services.print_batch import refresh_batch_status_for_item
  8726. async with async_session() as db:
  8727. await refresh_batch_status_for_item(db, queue_item_id)
  8728. await db.commit()
  8729. except Exception as e:
  8730. logger.warning("[BATCH] Failed to refresh batch status for queue item %s: %s", queue_item_id, e)
  8731. # MQTT relay - publish queue job completed
  8732. try:
  8733. printer_info = printer_manager.get_printer(printer_id)
  8734. await mqtt_relay.on_queue_job_completed(
  8735. job_id=queue_item_id,
  8736. filename=filename or subtask_name,
  8737. printer_id=printer_id,
  8738. printer_name=printer_info.name if printer_info else "Unknown",
  8739. status=queue_status,
  8740. )
  8741. except Exception:
  8742. pass # Don't fail if MQTT fails
  8743. # Check if queue is now empty and send notification
  8744. try:
  8745. from sqlalchemy import func as sa_func
  8746. async with async_session() as db:
  8747. count_result = await db.execute(
  8748. select(sa_func.count(PrintQueueItem.id)).where(PrintQueueItem.status == "pending")
  8749. )
  8750. pending_count = count_result.scalar() or 0
  8751. if pending_count == 0:
  8752. today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
  8753. completed_result = await db.execute(
  8754. select(sa_func.count(PrintQueueItem.id)).where(
  8755. PrintQueueItem.status.in_(["completed", "failed", "skipped"]),
  8756. PrintQueueItem.completed_at >= today_start,
  8757. )
  8758. )
  8759. completed_count = completed_result.scalar() or 1
  8760. await notification_service.on_queue_completed(
  8761. completed_count=completed_count,
  8762. db=db,
  8763. )
  8764. except Exception:
  8765. pass # Don't fail if notification fails
  8766. # Handle auto_off_after - power off printer if the queue item opted
  8767. # in. Delegates to the smart-plug manager so the off honours each
  8768. # plug's configured strategy (time delay or temperature threshold),
  8769. # is cancelled if the printer starts printing again, and never cuts
  8770. # power on a loaded print (#1890). Previously an inline block here
  8771. # hardcoded a 50°C / 600s cooldown wait and powered off on the
  8772. # timeout regardless of print state — cutting a touchscreen reprint.
  8773. if queue_auto_off:
  8774. try:
  8775. async with async_session() as db:
  8776. await smart_plug_manager.schedule_off_after_queue_job(printer_id, db)
  8777. except Exception as e:
  8778. logger.warning("Failed to schedule queue auto-off for printer %s: %s", printer_id, e)
  8779. except Exception as e:
  8780. logging.getLogger(__name__).warning(f"Queue item update failed: {e}")
  8781. log_timing("Queue item update")
  8782. # Register bed cooldown waiter (event-driven via on_bed_temp_update callback).
  8783. # Must run before archive_id early-return so it fires for all prints (including
  8784. # prints started from BambuStudio/touchscreen that have no archive).
  8785. if data.get("status") == "completed":
  8786. try:
  8787. from backend.app.api.routes.settings import get_setting
  8788. async with async_session() as db:
  8789. threshold_str = await get_setting(db, "bed_cooled_threshold")
  8790. threshold = float(threshold_str) if threshold_str else 35.0
  8791. # Check if any provider has on_bed_cooled enabled (skip registration if none)
  8792. async with async_session() as db:
  8793. providers = await notification_service._get_providers_for_event(db, "on_bed_cooled", printer_id)
  8794. if providers:
  8795. _bed_cool_waiters[printer_id] = {
  8796. "threshold": threshold,
  8797. "filename": filename or subtask_name or "",
  8798. "registered_at": time.time(),
  8799. }
  8800. logger.info(
  8801. "[BED-COOL] Registered waiter for printer %s (threshold: %.0f°C)",
  8802. printer_id,
  8803. threshold,
  8804. )
  8805. else:
  8806. logger.debug("[BED-COOL] No providers enabled for bed_cooled on printer %s", printer_id)
  8807. except Exception as e:
  8808. logger.warning("[BED-COOL] Failed to register waiter: %s", e)
  8809. # Capture the slicer estimate before usage tracking runs. The tracker may
  8810. # update archive.cost with this run's measured cost; billing partial runs
  8811. # against that already-partial value would discount the charge twice.
  8812. billing_planned_grams: float | None = None
  8813. billing_base_cost: float | None = None
  8814. if archive_id:
  8815. try:
  8816. async with async_session() as db:
  8817. from backend.app.models.archive import PrintArchive
  8818. billing_archive = await db.get(PrintArchive, archive_id)
  8819. if billing_archive:
  8820. billing_path = (
  8821. app_settings.base_dir / billing_archive.file_path if billing_archive.file_path else None
  8822. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  8823. billing_planned_grams, billing_base_cost = _plate_scoped_run_estimate(
  8824. billing_archive,
  8825. billing_path,
  8826. billing_plate_id
  8827. if billing_plate_id is not None
  8828. else _get_start_plate_id(printer_id, archive_id),
  8829. )
  8830. except Exception as e:
  8831. logger.warning("[FINANCE] Failed to capture planned usage for archive %s: %s", archive_id, e)
  8832. # --- Track filament consumption (must run before archive_id early-return so usage
  8833. # is recorded even when auto-archive is disabled) ---
  8834. usage_results: list[dict] = []
  8835. # Prefer ams_mapping captured from MQTT request topic (works for all print sources)
  8836. stored_ams_mapping = data.get("ams_mapping")
  8837. # Fallback to _print_ams_mappings for queue/reprint (set before print starts).
  8838. # Drained either way, like the plate below: left over, it would be read by
  8839. # the archive's next run, dispatched without a mapping.
  8840. registered_ams_mapping = _print_ams_mappings.pop((printer_id, archive_id), None) if archive_id else None
  8841. if not stored_ams_mapping:
  8842. stored_ams_mapping = registered_ams_mapping
  8843. # Always drain the plate_id register on completion — the session already
  8844. # consumed it at print-start injection; leaving it would leak into the next
  8845. # print on the same archive_id (rare but possible with reprints) (#1697).
  8846. # Capture the popped value so the completion notification can scope the
  8847. # archive-level (summed-across-plates per #1593) filament + time totals
  8848. # down to the single plate that was actually printed (#1785).
  8849. notify_plate_id: int | None = None
  8850. if archive_id:
  8851. notify_plate_id = _print_plate_ids.pop((printer_id, archive_id), None)
  8852. # The start estimate must not outlive the real cost written below (COST-4).
  8853. if archive_id:
  8854. await _settle_archive_cost_estimate(archive_id)
  8855. # Internal inventory: track AMS remain% deltas (skip if Spoolman handles usage)
  8856. try:
  8857. async with async_session() as db:
  8858. from backend.app.api.routes.settings import get_setting
  8859. _spoolman_on = await get_setting(db, "spoolman_enabled")
  8860. if not _spoolman_on or _spoolman_on.lower() != "true":
  8861. from backend.app.services.usage_tracker import on_print_complete as usage_on_print_complete
  8862. async with async_session() as db:
  8863. usage_results = await usage_on_print_complete(
  8864. printer_id,
  8865. data,
  8866. printer_manager,
  8867. db,
  8868. archive_id=archive_id,
  8869. ams_mapping=stored_ams_mapping,
  8870. )
  8871. if usage_results:
  8872. await ws_manager.broadcast(
  8873. {
  8874. "type": "spool_usage_logged",
  8875. "printer_id": printer_id,
  8876. "usage": usage_results,
  8877. }
  8878. )
  8879. log_timing("Usage tracker")
  8880. except Exception as e:
  8881. logger.warning("Usage tracker on_print_complete failed: %s", e)
  8882. # Drop the print-start context unconditionally — the Spoolman branch above
  8883. # skips the internal tracker entirely, so nothing else would clear what
  8884. # print start captured, and a row surviving its print would be restored
  8885. # onto the next one after a restart.
  8886. try:
  8887. from backend.app.services.usage_tracker import discard_session
  8888. async with async_session() as db:
  8889. await discard_session(db, printer_id)
  8890. except Exception as e:
  8891. logger.warning("Failed to clear persisted print session for printer %s: %s", printer_id, e)
  8892. # Spoolman: report filament usage (requires archive_id for tracking data lookup)
  8893. # A partial run's cost from the Spoolman spools it was charged to; the
  8894. # built-in inventory's comes in usage_results.
  8895. spoolman_run_cost: float | None = None
  8896. if archive_id:
  8897. if data.get("status") == "completed":
  8898. try:
  8899. await _report_spoolman_usage(printer_id, archive_id)
  8900. log_timing("Spoolman usage report")
  8901. except Exception as e:
  8902. logger.warning("Spoolman usage reporting failed: %s", e)
  8903. else:
  8904. # Report partial usage if tracking data exists (only stored when weight sync is disabled)
  8905. try:
  8906. async with async_session() as db:
  8907. spoolman_run_cost = await _cleanup_spoolman_tracking(
  8908. printer_id,
  8909. archive_id,
  8910. db,
  8911. last_layer_num=data.get("last_layer_num"),
  8912. last_progress=data.get("last_progress"),
  8913. status=data.get("status") or "failed",
  8914. )
  8915. except Exception as e:
  8916. logger.debug("[SPOOLMAN] Cleanup failed: %s", e)
  8917. await _release_archived_spoolman_slots(printer_id)
  8918. log_timing("Filament usage tracking")
  8919. if not archive_id:
  8920. # The printer's own calibration run has no archive by design, so this
  8921. # arrives here every time one finishes. Returning before the no-archive
  8922. # notification keeps a calibration from being announced as a finished
  8923. # print. Everything above this point has already run — the plate-clear
  8924. # gate, the queue reconciliation, the SD-card cleanup — so only the
  8925. # notification is skipped.
  8926. if is_internal_printer_job(filename, subtask_name):
  8927. logger.info(
  8928. "[CALLBACK] Internal printer job completed, no notification: filename=%s, subtask=%s",
  8929. filename,
  8930. subtask_name,
  8931. )
  8932. return
  8933. logger.warning("Could not find archive for print complete: filename=%s, subtask=%s", filename, subtask_name)
  8934. # Still send print-complete/failed/stopped notifications even without an archive.
  8935. # Try to enrich with queue/library-file data so user-specific emails work too.
  8936. async def _notify_no_archive():
  8937. try:
  8938. async with async_session() as db:
  8939. from backend.app.models.library import LibraryFile
  8940. from backend.app.models.print_queue import PrintQueueItem
  8941. from backend.app.models.printer import Printer
  8942. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8943. printer_obj = result.scalar_one_or_none()
  8944. p_name = printer_obj.name if printer_obj else f"Printer {printer_id}"
  8945. # Only the queue item this completion itself closed (the
  8946. # reconciliation above, which checks the run belongs to it).
  8947. # A print started outside Bambuddy closed none, and must not
  8948. # borrow an earlier job's owner, email or times.
  8949. no_archive_data: dict | None = None
  8950. try:
  8951. queue_item = await db.get(PrintQueueItem, queue_item_id) if queue_item_id else None
  8952. if queue_item:
  8953. no_archive_data = {"created_by_id": queue_item.created_by_id}
  8954. # How long it really ran, as the archive path reports it (#1198).
  8955. started = to_naive_utc(queue_item.started_at)
  8956. ended = to_naive_utc(queue_item.completed_at)
  8957. # Not for a completion reconciled at reconnect: it
  8958. # ended at an unknown time while the printer was
  8959. # offline, and "now" is the reconnect (DB-C9).
  8960. if (
  8961. not data.get("_reconciled")
  8962. and started
  8963. and ended
  8964. and (ended - started).total_seconds() > 0
  8965. ):
  8966. no_archive_data["actual_time_seconds"] = int((ended - started).total_seconds())
  8967. # Estimated time: the queue item caches it from its
  8968. # archive or library file; otherwise read the library
  8969. # file's metadata (LibraryFile has no column for it).
  8970. if queue_item.print_time_seconds:
  8971. no_archive_data["print_time_seconds"] = queue_item.print_time_seconds
  8972. elif queue_item.library_file_id:
  8973. lib_result = await db.execute(
  8974. select(LibraryFile).where(LibraryFile.id == queue_item.library_file_id)
  8975. )
  8976. lib_file = lib_result.scalar_one_or_none()
  8977. estimate = (
  8978. (lib_file.file_metadata or {}).get("print_time_seconds") if lib_file else None
  8979. )
  8980. if estimate:
  8981. no_archive_data["print_time_seconds"] = estimate
  8982. except Exception as lookup_err:
  8983. logger.debug(
  8984. "[NOTIFY-BG] Could not look up queue item for no-archive notification: %s", lookup_err
  8985. )
  8986. # Enrich with usage tracker results (captured in enclosing scope)
  8987. if usage_results:
  8988. if no_archive_data is None:
  8989. no_archive_data = {}
  8990. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  8991. if total_from_usage > 0:
  8992. no_archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  8993. no_archive_data["usage_results"] = usage_results
  8994. # Try MQTT remaining_time for print duration when no queue/library data
  8995. if no_archive_data and not no_archive_data.get("print_time_seconds"):
  8996. mqtt_remaining = data.get("remaining_time")
  8997. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  8998. no_archive_data["print_time_seconds"] = int(mqtt_remaining)
  8999. ps = data.get("status", "completed")
  9000. logger.info(
  9001. "[NOTIFY-BG] Sending notification without archive: printer=%s, status=%s", printer_id, ps
  9002. )
  9003. if not await _kill_switch_notification_already_sent(kill_switch_notification_task):
  9004. await notification_service.on_print_complete(
  9005. printer_id, p_name, ps, data, db, archive_data=no_archive_data
  9006. )
  9007. else:
  9008. logger.info("[NOTIFY-BG] Skipped duplicate kill-switch provider notification")
  9009. # Send user-specific email if we have a created_by_id
  9010. if no_archive_data and no_archive_data.get("created_by_id"):
  9011. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  9012. await _dispatch_user_print_email(
  9013. ps,
  9014. no_archive_data["created_by_id"],
  9015. p_name,
  9016. raw_filename,
  9017. db,
  9018. )
  9019. logger.info("[NOTIFY-BG] Completed (no-archive path)")
  9020. except Exception as e:
  9021. logger.warning("[NOTIFY-BG] Failed to send notification without archive: %s", e, exc_info=True)
  9022. spawn_background_task(_notify_no_archive(), name="notify-no-archive")
  9023. return
  9024. log_timing("Archive lookup")
  9025. # Update archive status
  9026. logger.info("[ARCHIVE] Updating archive %s status...", archive_id)
  9027. try:
  9028. async with async_session() as db:
  9029. service = ArchiveService(db)
  9030. status = data.get("status", "completed")
  9031. hms_errors = data.get("hms_errors", []) if status == "failed" else None
  9032. if hms_errors:
  9033. logger.info("[ARCHIVE] HMS errors at failure: %s", hms_errors)
  9034. failure_reason = derive_failure_reason(status, hms_errors)
  9035. if data.get("_reconciled") and (status == "aborted" or (status == "failed" and not failure_reason)):
  9036. # A reconciled completion closes out a stale archive at
  9037. # reconnect — it is not a user action, so don't mislabel it
  9038. # "userCancelled". It shares the stale-cleanup path's key
  9039. # (issue #2974) and records that the real end time is unknown,
  9040. # which is also why its logged duration is 0 (#2592). A failed
  9041. # one the printer's errors explain keeps their reason (#3310).
  9042. failure_reason = "noStatusUpdate"
  9043. elif data.get("_kill_switch"):
  9044. # Not the user's cancel: the kill switch stopped a print that did
  9045. # not come from Bambuddy.
  9046. failure_reason = "killSwitch"
  9047. if failure_reason:
  9048. logger.info("[ARCHIVE] failure_reason=%r (status=%s)", failure_reason, status)
  9049. elif status == "failed" and hms_errors:
  9050. logger.info("[ARCHIVE] HMS errors present but none matched a known failure-reason short code")
  9051. await service.update_archive_status(
  9052. archive_id,
  9053. status=status,
  9054. completed_at=(
  9055. # A reconciled completed print ended at an estimated time,
  9056. # not at the reconnect (#3310).
  9057. data.get("_reconciled_completed_at") or datetime.now(timezone.utc)
  9058. if status in ("completed", "failed", "aborted", "cancelled")
  9059. else None
  9060. ),
  9061. failure_reason=failure_reason,
  9062. )
  9063. logger.info(
  9064. "[ARCHIVE] Archive %s status updated to %s, failure_reason=%s", archive_id, status, failure_reason
  9065. )
  9066. await ws_manager.send_archive_updated(
  9067. {
  9068. "id": archive_id,
  9069. "status": status,
  9070. }
  9071. )
  9072. logger.info("[ARCHIVE] WebSocket notification sent for archive %s", archive_id)
  9073. # MQTT relay - publish archive updated
  9074. try:
  9075. await mqtt_relay.on_archive_updated(
  9076. archive_id=archive_id,
  9077. print_name=filename or subtask_name,
  9078. status=status,
  9079. )
  9080. except Exception:
  9081. pass # Don't fail if MQTT fails
  9082. except Exception as e:
  9083. logger.error("[ARCHIVE] Failed to update archive %s status: %s", archive_id, e, exc_info=True)
  9084. # Continue with other operations even if archive update fails
  9085. log_timing("Archive status update")
  9086. # Apply finance wallet charge or release reservations once. For all partial
  9087. # terminal states (failed, aborted at the printer display, or cancelled via
  9088. # Bambuddy) use this run's measured spool delta, falling back to the last
  9089. # valid printer progress. PrintArchive.filament_used_grams is the slicer
  9090. # estimate and therefore cannot represent an interrupted run.
  9091. try:
  9092. if data.get("status") in ("completed", "failed", "aborted", "cancelled"):
  9093. async with async_session() as db:
  9094. from backend.app.models.archive import PrintArchive
  9095. from backend.app.services.finance_billing import apply_print_charge_for_archive
  9096. from backend.app.services.print_log import run_being_ended
  9097. archive = await db.get(PrintArchive, archive_id)
  9098. # The run this end fills in below: charged as its own, not as
  9099. # the archive's latest dispatch (another plate of it, maybe).
  9100. charged_run = await run_being_ended(db, archive_id, printer_id, archive.started_at) if archive else None
  9101. if archive and billing_run_id is None and charged_run is None:
  9102. billing_run_id = getattr(archive, "billing_run_id", None)
  9103. if archive and archive.created_by_id is None and _print_user_info:
  9104. archive.created_by_id = await _print_user_id_if_still_there(db, _print_user_info)
  9105. await db.flush()
  9106. run_status = data.get("status", "completed")
  9107. last_progress = data.get("last_progress")
  9108. if last_progress is None:
  9109. last_progress = data.get("progress")
  9110. actual_run_grams = _compute_run_filament_grams(
  9111. run_status,
  9112. billing_planned_grams,
  9113. last_progress,
  9114. usage_results,
  9115. )
  9116. filament_usage = (actual_run_grams, billing_planned_grams) if run_status != "completed" else None
  9117. in_memory_cost_center_id = _print_cost_center_ids.pop((printer_id, archive_id), None)
  9118. charged = await apply_print_charge_for_archive(
  9119. db,
  9120. archive_id,
  9121. charged_user_id=billing_user_id,
  9122. cost_center_id=(
  9123. billing_cost_center_id if billing_cost_center_id is not None else in_memory_cost_center_id
  9124. ),
  9125. print_queue_id=queue_item_id,
  9126. print_run_id=billing_run_id,
  9127. base_cost_override=billing_base_cost,
  9128. filament_usage=filament_usage,
  9129. run_entry_id=charged_run.id if charged_run is not None else None,
  9130. run_status=run_status,
  9131. )
  9132. await db.commit()
  9133. if charged:
  9134. logger.info("[FINANCE] Applied print charge for archive %s", archive_id)
  9135. except Exception as e:
  9136. logger.warning("[FINANCE] Failed to apply print charge for archive %s: %s", archive_id, e)
  9137. printer_info = printer_manager.get_printer(printer_id)
  9138. billing_printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  9139. billing_filename = filename or subtask_name or "Unknown"
  9140. billing_error = str(e)
  9141. try:
  9142. await ws_manager.broadcast(
  9143. {
  9144. "type": "billing_charge_failed",
  9145. "printer_id": printer_id,
  9146. "printer_name": billing_printer_name,
  9147. "filename": billing_filename,
  9148. "archive_id": archive_id,
  9149. }
  9150. )
  9151. except Exception as notification_error:
  9152. logger.error(
  9153. "[FINANCE] Failed to broadcast billing error for archive %s: %s",
  9154. archive_id,
  9155. notification_error,
  9156. )
  9157. async def _notify_billing_charge_failed() -> None:
  9158. try:
  9159. async with async_session() as notification_db:
  9160. await notification_service.on_billing_charge_failed(
  9161. printer_id,
  9162. billing_printer_name,
  9163. billing_filename,
  9164. archive_id,
  9165. billing_error,
  9166. notification_db,
  9167. )
  9168. except Exception as provider_error:
  9169. logger.error(
  9170. "[FINANCE] Failed to send provider billing alert for archive %s: %s",
  9171. archive_id,
  9172. provider_error,
  9173. exc_info=True,
  9174. )
  9175. spawn_background_task(
  9176. _notify_billing_charge_failed(),
  9177. name=f"billing-charge-failed-{archive_id}",
  9178. )
  9179. log_timing("Finance charge update")
  9180. # Write independent print log entry (separate table, never touches archives)
  9181. # The run's id, for the energy work below: the archive's newest run is
  9182. # the one started last, which with plates of it on two printers need not
  9183. # be the one that just ended.
  9184. completed_run_id: int | None = None
  9185. async def _write_run(db):
  9186. # Retried on a locked SQLite database: a run left open here would be
  9187. # closed again by the next reconnect's reconcile.
  9188. nonlocal completed_run_id
  9189. from backend.app.models.archive import PrintArchive
  9190. from backend.app.services.print_log import discard_open_runs, record_archive_wear, write_log_entry
  9191. archive = await db.get(PrintArchive, archive_id)
  9192. if archive is None:
  9193. # Deleted while it printed: its run has nothing left to end on.
  9194. discarded = await discard_open_runs(db, printer_id)
  9195. await db.commit()
  9196. await _announce_discarded_runs(discarded, printer_id)
  9197. if archive:
  9198. # Back-fill created_by_id on reprint (#730): reprint reuses the
  9199. # source archive row rather than creating a new one, so an
  9200. # archive that was auto-created from a printer-initiated
  9201. # print (created_by_id=NULL) would otherwise stay unattributed
  9202. # forever. When we have a print-session user AND the archive
  9203. # has no attribution yet, credit the current user. Never
  9204. # overwrite an existing attribution — the original uploader
  9205. # keeps ownership.
  9206. _print_user_id = await _print_user_id_if_still_there(db, _print_user_info)
  9207. if archive.created_by_id is None and _print_user_id is not None:
  9208. archive.created_by_id = _print_user_id
  9209. p_info = printer_manager.get_printer(printer_id)
  9210. # Per-run actuals — written to PrintLogEntry so stats reflect
  9211. # what THIS print actually used, not the source archive's
  9212. # first-run values (#1378). Helper handles the partial-print
  9213. # math (failed / cancelled / stopped get scaled to progress
  9214. # or to tracked spool deltas).
  9215. _run_status = data.get("status", "completed")
  9216. # #2614: scope the per-run estimate to the printed plate. For a
  9217. # multi-plate 3MF dispatched one plate at a time, the archive's
  9218. # filament/cost are the whole-file totals; the PrintLogEntry must
  9219. # reflect only this plate. No effect on single-plate archives (the
  9220. # plate estimate equals the whole-file value) or on the tracker
  9221. # path (measured spool deltas win in _compute_run_filament_grams).
  9222. _est_full_path = (
  9223. app_settings.base_dir / archive.file_path if archive.file_path else None
  9224. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  9225. # The plate that printed, as billing reads it: a reprint of
  9226. # another plate must not log the archive's first plate.
  9227. _run_plate_id = billing_plate_id if billing_plate_id is not None else notify_plate_id
  9228. _est_grams, _est_cost = _plate_scoped_run_estimate(archive, _est_full_path, _run_plate_id)
  9229. if _run_plate_id is None:
  9230. _run_plate_id = archive.plate_id
  9231. _run_grams = _compute_run_filament_grams(
  9232. _run_status,
  9233. _est_grams,
  9234. data.get("last_progress", data.get("progress")),
  9235. usage_results,
  9236. )
  9237. # Per-run cost — prefer usage_results sum. For partial prints
  9238. # we deliberately skip the topup-to-estimate logic in
  9239. # usage_tracker (which assumes the print completed); the raw
  9240. # tracked-spool sum is closer to what THIS run actually cost.
  9241. _run_cost: float | None = None
  9242. if usage_results:
  9243. _run_cost = sum(r.get("cost") or 0 for r in usage_results) or None
  9244. if _run_cost is None and _run_status != "completed":
  9245. _run_cost = spoolman_run_cost
  9246. if _run_cost is None and _run_status == "completed":
  9247. _run_cost = _est_cost
  9248. from backend.app.models.printer import Printer as _Printer
  9249. _wear_rate = await db.scalar(select(_Printer.wear_cost_per_hour).where(_Printer.id == printer_id))
  9250. run_entry = await write_log_entry(
  9251. db,
  9252. archive_id=archive.id,
  9253. # Captured by _update_queue_status above; None for
  9254. # printer-initiated prints with no queue row. Batch
  9255. # cost/energy roll-up joins on it (#342).
  9256. queue_item_id=queue_item_id,
  9257. status=_run_status,
  9258. print_name=archive.print_name,
  9259. # The plate this run printed, as the estimate above takes
  9260. # it; the archive's own is the latest run's.
  9261. plate_id=_run_plate_id,
  9262. printer_name=p_info.name if p_info else None,
  9263. printer_id=printer_id,
  9264. started_at=archive.started_at,
  9265. completed_at=archive.completed_at,
  9266. filament_type=archive.filament_type,
  9267. filament_color=archive.filament_color,
  9268. filament_used_grams=_run_grams,
  9269. cost=_run_cost,
  9270. wear_cost_per_hour=_wear_rate,
  9271. failure_reason=archive.failure_reason,
  9272. thumbnail_path=archive.thumbnail_path,
  9273. created_by_id=archive.created_by_id,
  9274. created_by_username=_print_user_info.get("username") if _print_user_info else None,
  9275. # Reconciled completions have an unknown real end time —
  9276. # log 0 duration instead of the whole disconnect gap (#2592),
  9277. # unless it was estimated (#3310).
  9278. reconciled=bool(data.get("_reconciled")) and not data.get("_reconciled_completed_at"),
  9279. # Its time is the estimate or nothing, never measured (#3310).
  9280. duration_unmeasured=bool(data.get("_reconciled")),
  9281. # The real end of a print stopped while its printer was offline.
  9282. replaces_offline_stop=True,
  9283. )
  9284. await record_archive_wear(db, archive, run_entry)
  9285. await db.commit()
  9286. completed_run_id = run_entry.id
  9287. logger.info("[PRINT_LOG] Log entry written for archive %s", archive_id)
  9288. try:
  9289. from backend.app.core.database import run_with_retry
  9290. await run_with_retry(_write_run, label="print log entry")
  9291. except Exception as e:
  9292. logger.warning("[PRINT_LOG] Failed to write log entry for archive %s: %s", archive_id, e)
  9293. await _publish_run_completed(completed_run_id, printer_id)
  9294. log_timing("Print log entry")
  9295. # Run slow operations as background tasks to avoid blocking the event loop
  9296. # These operations can take 5-10+ seconds and would freeze the UI if awaited
  9297. async def _background_energy_calculation():
  9298. """Calculate and save energy usage in background.
  9299. Reads the starting kWh from the archive row (#941: persisted so a mid-print
  9300. backend restart no longer loses per-print energy data).
  9301. """
  9302. try:
  9303. logger.info("[ENERGY-BG] Starting energy calculation for archive %s", archive_id)
  9304. async with async_session() as db:
  9305. from backend.app.models.archive import PrintArchive
  9306. from backend.app.models.print_log import PrintLogEntry
  9307. from backend.app.services.print_log import energy_start_of
  9308. archive = await db.get(PrintArchive, archive_id)
  9309. if archive is None:
  9310. logger.warning("[ENERGY-BG] Archive %s no longer exists", archive_id)
  9311. return
  9312. # The ended run's own start reading: the archive's is its
  9313. # latest start's, maybe another plate's on another printer.
  9314. run_row = await db.get(PrintLogEntry, completed_run_id) if completed_run_id is not None else None
  9315. start = energy_start_of(run_row, archive)
  9316. if start is None:
  9317. logger.info("[ENERGY-BG] No start kWh recorded for archive %s's run", archive_id)
  9318. return
  9319. starting_kwh = start.kwh
  9320. candidates = await energy_plug_candidates(db, printer_id)
  9321. if not candidates:
  9322. logger.info("[ENERGY-BG] No smart plug for printer %s", printer_id)
  9323. return
  9324. # Same ordering as the start reading, so the delta below is
  9325. # against the counter that produced `starting_kwh` (#2859).
  9326. selected = await select_energy_reading(candidates, _get_plug_energy, db)
  9327. if selected is None:
  9328. logger.warning(
  9329. "[ENERGY-BG] No plug on printer %s reports a lifetime energy counter (tried: %s)",
  9330. printer_id,
  9331. ", ".join(plug.name for plug in candidates),
  9332. )
  9333. return
  9334. plug, energy = selected
  9335. logger.info("[ENERGY-BG] Energy response from plug '%s': %s", plug.name, energy)
  9336. energy_used = round(energy["total"] - starting_kwh, 4)
  9337. logger.info("[ENERGY-BG] Per-print energy: %s kWh", energy_used)
  9338. if energy_used < 0:
  9339. logger.warning(
  9340. "[ENERGY-BG] Negative energy delta for archive %s (start=%s, end=%s) — counter reset?",
  9341. archive_id,
  9342. starting_kwh,
  9343. energy["total"],
  9344. )
  9345. return
  9346. from backend.app.services.energy_price import current_price, print_energy_cost, stored_price
  9347. # Each hour of the print at the price of that hour (#1251). If
  9348. # that fails, the last price known, as before, rather than no
  9349. # energy at all. Read first: the session may be unusable after.
  9350. last_price = await stored_price(db)
  9351. try:
  9352. energy_cost_value = round(
  9353. await print_energy_cost(
  9354. db,
  9355. plug_id=start.plug_id,
  9356. start_at=start.at,
  9357. start_kwh=starting_kwh,
  9358. start_price=start.price,
  9359. end_kwh=energy["total"],
  9360. end_plug_id=plug.id,
  9361. price_now=await current_price(db),
  9362. ),
  9363. 3,
  9364. )
  9365. except Exception as e:
  9366. logger.warning("[ENERGY-BG] Hour-by-hour cost failed for archive %s: %s", archive_id, e)
  9367. energy_cost_value = round(energy_used * last_price, 3)
  9368. # First-run-only overwrite of archive.energy_kwh / energy_cost so a
  9369. # reprint doesn't visually clobber the source archive's energy data
  9370. # (#1378). Reprint energy lives in the matching PrintLogEntry below.
  9371. from sqlalchemy import func
  9372. existing_runs = await db.scalar(
  9373. select(func.count(PrintLogEntry.id)).where(PrintLogEntry.archive_id == archive_id)
  9374. )
  9375. if (existing_runs or 0) <= 1:
  9376. # 0 = legacy archive that pre-dates per-run logging; 1 = the row
  9377. # we just wrote for THIS print. Either way it's the first run.
  9378. archive.energy_kwh = energy_used
  9379. archive.energy_cost = energy_cost_value
  9380. # The ended run's energy (write_log_entry above ran before this
  9381. # background task, so its energy fields are still NULL). With
  9382. # no run written, none: the archive's newest is another print's.
  9383. if run_row is not None:
  9384. run_row.energy_kwh = energy_used
  9385. run_row.energy_cost = energy_cost_value
  9386. await db.commit()
  9387. logger.info("[ENERGY-BG] Saved: %s kWh, cost=%s", energy_used, energy_cost_value)
  9388. except Exception as e:
  9389. logger.warning("[ENERGY-BG] Failed: %s", e)
  9390. async def _background_finish_photo() -> str | None:
  9391. """Capture finish photo in background. Returns photo filename if captured."""
  9392. # #2547: the Z to park the plate back to, set once this function has
  9393. # raised the plate itself (the timelapse path, where the moment producer
  9394. # returned without doing it). Declared out here so the `finally` can
  9395. # lower it again no matter where the capture below fails.
  9396. park_z: float | None = None
  9397. try:
  9398. logger.info("[PHOTO-BG] Starting finish photo capture for archive %s", archive_id)
  9399. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  9400. # Read phase: settings + printer + archive in a short session, released
  9401. # BEFORE the capture pipeline below. The capture (timelapse last-frame,
  9402. # stage-22 wait, external-camera grab, or a fresh RTSP shot) can take
  9403. # tens of seconds; holding this session across it pinned one pooled
  9404. # connection idle-in-transaction per finishing print (issue #2572).
  9405. async with async_session() as db:
  9406. from backend.app.api.routes.settings import get_setting
  9407. from backend.app.models.archive import PrintArchive
  9408. from backend.app.models.printer import Printer
  9409. capture_enabled = await get_setting(db, "capture_finish_photo")
  9410. if capture_enabled is not None and capture_enabled.lower() != "true":
  9411. return None
  9412. if not archive_id:
  9413. return None
  9414. printer = (await db.execute(select(Printer).where(Printer.id == printer_id))).scalar_one_or_none()
  9415. archive = (
  9416. await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  9417. ).scalar_one_or_none()
  9418. if not printer or not archive:
  9419. return None
  9420. import uuid
  9421. from datetime import datetime
  9422. from backend.app.utils.archive_paths import archive_dir as resolve_archive_dir
  9423. if not archive.file_path:
  9424. logger.warning("[PHOTO-BG] Archive %s has no file_path, using fallback dir", archive_id)
  9425. archive_dir = resolve_archive_dir(archive)
  9426. photo_filename = None
  9427. # Prefer the timelapse last-frame source when a timelapse was
  9428. # recording — it captures the moment after the toolhead parks
  9429. # but before the bed drops, which the live-camera grab below
  9430. # would miss (#1397). Skipped for external cameras (those have
  9431. # their own framing and don't see a Bambu timelapse). Only
  9432. # runs when the USER explicitly enabled timelapse for this
  9433. # print — #1721 removed Bambuddy's force-on at dispatch
  9434. # because it caused per-layer nozzle parking on Smooth-mode
  9435. # slicer profiles.
  9436. prefer_timelapse_source = bool(data.get("timelapse_was_active")) and not (
  9437. printer.external_camera_enabled and printer.external_camera_url
  9438. )
  9439. timelapse_still_pending = False
  9440. if prefer_timelapse_source:
  9441. photo_filename, timelapse_still_pending = await _capture_finish_photo_from_timelapse(
  9442. archive_id=archive_id,
  9443. archive_dir=archive_dir,
  9444. rotation=getattr(printer, "camera_rotation", 0),
  9445. run_id=completed_run_id,
  9446. )
  9447. # #1721: replacement framing path — on_finish_photo_moment
  9448. # pre-captured a frame at the stage-22 / FINISH edge (toolhead
  9449. # parked, bed not yet dropped) and cached the JPEG bytes in
  9450. # _stage22_finish_frames. Consume them now so the saved photo
  9451. # has the better framing instead of the post-bed-drop angle
  9452. # the live-camera fallback below would give.
  9453. if not photo_filename:
  9454. # #1790: on the FINISH-state fallback path the producer
  9455. # task is dispatched back-to-back with this consumer, so
  9456. # a bare pop would race past with an empty result and
  9457. # the RTSP fallback below would collide with the
  9458. # producer's still-in-flight grab (single-client RTSP
  9459. # on Bambu printers). Wait for the producer to finish
  9460. # or give up before touching the cache.
  9461. #
  9462. # #2547: 20s was enough when the producer only ever grabbed a
  9463. # frame. It now also raises the plate first, which costs the
  9464. # settle window before the grab even starts — so the budget has
  9465. # to cover settle + a worst-case 15s RTSP timeout, and still sit
  9466. # under the notification's own photo wait below.
  9467. in_flight = _stage22_finish_in_flight.pop(printer_id, None)
  9468. if in_flight is not None:
  9469. try:
  9470. await asyncio.wait_for(in_flight.wait(), timeout=_FINISH_PHOTO_PRODUCER_WAIT_SECONDS)
  9471. except TimeoutError:
  9472. logger.warning(
  9473. "[PHOTO-BG] timed out waiting for stage-22 producer for printer %s — proceeding to fallback",
  9474. printer_id,
  9475. )
  9476. cached_frame = _stage22_finish_frames.pop(printer_id, None)
  9477. if cached_frame:
  9478. # Already rotated by the producer (#2708) — rotating again
  9479. # here would undo the fix on the banked-frame path, whose
  9480. # bytes reach the cache having been rotated once already.
  9481. photos_dir = archive_dir / "photos"
  9482. photos_dir.mkdir(parents=True, exist_ok=True)
  9483. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9484. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9485. photo_path = photos_dir / photo_filename
  9486. await asyncio.to_thread(photo_path.write_bytes, cached_frame)
  9487. logger.info(
  9488. "[PHOTO-BG] Saved stage-22 pre-captured frame: %s (%d bytes)",
  9489. photo_filename,
  9490. len(cached_frame),
  9491. )
  9492. # #2547: the timelapse path reaches the live grab below whenever the
  9493. # video hasn't landed in time — the documented usual outcome on
  9494. # P1-series, where transfers are slowest. `on_finish_photo_moment`
  9495. # returned early for those prints without raising the plate, so
  9496. # without this the photo that actually ships in the notification is
  9497. # of an already-dropped plate: exactly the framing #1145/#1397/#1565
  9498. # asked us to fix. The archive still gets the better video frame
  9499. # later; this is about the image the user is sent.
  9500. #
  9501. # Gated on `timelapse_was_active` precisely because that is the
  9502. # condition under which the producer skipped. On every other path it
  9503. # has already raised and lowered the plate, and repeating that here
  9504. # would be a second pointless round trip.
  9505. if (
  9506. not photo_filename
  9507. and data.get("timelapse_was_active")
  9508. and not print_dispatch_context.end_gcode_injected(printer_id)
  9509. ):
  9510. try:
  9511. async with async_session() as db:
  9512. from backend.app.api.routes.settings import get_setting
  9513. restore_setting = await get_setting(db, "finish_photo_restore_plate")
  9514. if restore_setting is None or restore_setting.lower() == "true":
  9515. park_z = await _raise_plate_for_finish_photo(
  9516. printer_id, data, logger, bound_archive_id, notify_plate_id
  9517. )
  9518. except Exception as e:
  9519. logger.warning("[PLATE-RESTORE] printer %s: restore failed: %s", printer_id, e)
  9520. # Fallback chain: external camera → buffered live frame →
  9521. # fresh RTSP capture. Only runs if the timelapse path above
  9522. # didn't already produce a photo.
  9523. # Chamber light for the live capture below (#1655).
  9524. async with camera_light(printer if not photo_filename else None):
  9525. if not photo_filename:
  9526. if printer.external_camera_enabled and printer.external_camera_url:
  9527. logger.info("[PHOTO-BG] Using external camera")
  9528. from backend.app.api.routes.camera import live_frame_for_capture
  9529. from backend.app.services.external_camera import capture_frame
  9530. # #2707: the second half of the finish-photo failure — the
  9531. # pre-capture and this fallback both collided with the live
  9532. # view. None here continues down the fallback chain.
  9533. defer, buffered = live_frame_for_capture(printer_id)
  9534. if defer:
  9535. frame_data = buffered
  9536. else:
  9537. frame_data = await capture_frame(
  9538. printer.external_camera_url,
  9539. printer.external_camera_type or "mjpeg",
  9540. snapshot_url=printer.external_camera_snapshot_url,
  9541. )
  9542. if frame_data:
  9543. frame_data = _apply_camera_rotation(frame_data, printer, logger)
  9544. photos_dir = archive_dir / "photos"
  9545. photos_dir.mkdir(parents=True, exist_ok=True)
  9546. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9547. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9548. photo_path = photos_dir / photo_filename
  9549. await asyncio.to_thread(photo_path.write_bytes, frame_data)
  9550. logger.info("[PHOTO-BG] Saved external camera frame: %s", photo_filename)
  9551. else:
  9552. # Check if camera stream is active - use buffered frame to avoid freeze
  9553. # Check both RTSP streams (_active_streams) and chamber image streams (_active_chamber_streams)
  9554. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  9555. active_chamber_for_printer = [
  9556. k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")
  9557. ]
  9558. buffered_frame = get_buffered_frame(printer_id)
  9559. if (active_for_printer or active_chamber_for_printer) and buffered_frame:
  9560. # Use frame from active stream
  9561. logger.info("[PHOTO-BG] Using buffered frame from active stream")
  9562. buffered_frame = _apply_camera_rotation(buffered_frame, printer, logger)
  9563. photos_dir = archive_dir / "photos"
  9564. photos_dir.mkdir(parents=True, exist_ok=True)
  9565. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9566. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9567. photo_path = photos_dir / photo_filename
  9568. await asyncio.to_thread(photo_path.write_bytes, buffered_frame)
  9569. logger.info("[PHOTO-BG] Saved buffered frame: %s", photo_filename)
  9570. else:
  9571. # No active stream - capture new frame
  9572. from backend.app.services.camera import capture_finish_photo
  9573. photo_filename = await capture_finish_photo(
  9574. printer_id=printer_id,
  9575. ip_address=printer.ip_address,
  9576. access_code=printer.access_code,
  9577. model=printer.model,
  9578. archive_dir=archive_dir,
  9579. rotation=getattr(printer, "camera_rotation", 0),
  9580. )
  9581. # Write phase: attach the photo in a fresh short-lived session.
  9582. if photo_filename:
  9583. async with async_session() as db:
  9584. from backend.app.models.archive import PrintArchive
  9585. from backend.app.services.print_log import add_run_photo, run_of
  9586. arch = await db.get(PrintArchive, archive_id)
  9587. # The ended print's own list takes it too: the archive's
  9588. # holds every print's.
  9589. run = await run_of(db, archive_id, completed_run_id) if completed_run_id is not None else None
  9590. if arch is not None:
  9591. # A new list: appended in place, the ORM sees no
  9592. # change and a second print's photo is never saved.
  9593. photos = list(arch.photos or [])
  9594. photos.append(photo_filename)
  9595. arch.photos = photos
  9596. add_run_photo(run, photo_filename)
  9597. await db.commit()
  9598. logger.info("[PHOTO-BG] Saved: %s", photo_filename)
  9599. # The short wait above is bounded so a slow printer can't hold up
  9600. # the print-complete notification, which is what the caller is
  9601. # blocking on. When it ran out with the video still on its way,
  9602. # keep waiting off to the side and swap the better frame into the
  9603. # archive once it arrives (#2704 follow-up) — otherwise P1-series
  9604. # users, whose videos routinely take minutes to transfer, never get
  9605. # the pre-bed-drop framing this path exists to provide.
  9606. #
  9607. # Spawned here rather than at the point the wait gave up: both this
  9608. # function and the upgrade do a read-modify-write on `photos`, and
  9609. # the live-camera fallback above can take tens of seconds. Starting
  9610. # the upgrade before that write means the two can interleave and one
  9611. # silently drops the other's entry, leaving a JPEG on disk that the
  9612. # gallery never lists.
  9613. if timelapse_still_pending:
  9614. from backend.app.services import archive_work
  9615. spawn_background_task(
  9616. archive_work.held_while(
  9617. archive_id,
  9618. _upgrade_finish_photo_from_timelapse(
  9619. archive_id,
  9620. archive_dir,
  9621. rotation=getattr(printer, "camera_rotation", 0),
  9622. run_id=completed_run_id,
  9623. replace=photo_filename,
  9624. ),
  9625. ),
  9626. name=f"finish-photo-upgrade-{archive_id}",
  9627. )
  9628. return photo_filename
  9629. except Exception as e:
  9630. logger.warning("[PHOTO-BG] Failed: %s", e)
  9631. return None
  9632. finally:
  9633. # #2547: we raised the plate, so we owe the move back down — even if
  9634. # the capture in between threw. Otherwise the user finds the print
  9635. # pinned under the nozzle.
  9636. if park_z is not None:
  9637. try:
  9638. _park_plate_after_finish_photo(printer_id, park_z, logger)
  9639. except Exception as e:
  9640. logger.warning("[PLATE-RESTORE] printer %s: could not lower plate: %s", printer_id, e)
  9641. spawn_background_task(_background_energy_calculation(), name="background-energy-calc")
  9642. # Photo capture task - result will be used by notifications
  9643. photo_task = spawn_background_task(_background_finish_photo(), name="background-finish-photo")
  9644. log_timing("Background tasks scheduled (energy, photo)")
  9645. # Also run smart plug, notifications, and maintenance as background tasks
  9646. print_status = data.get("status", "completed")
  9647. async def _background_smart_plug():
  9648. """Handle smart plug automation in background."""
  9649. try:
  9650. logger.info("[AUTO-OFF-BG] Starting smart plug automation for printer %s", printer_id)
  9651. async with async_session() as db:
  9652. await smart_plug_manager.on_print_complete(printer_id, print_status, db)
  9653. logger.info("[AUTO-OFF-BG] Completed")
  9654. except Exception as e:
  9655. logger.warning("[AUTO-OFF-BG] Failed: %s", e)
  9656. async def _background_notifications(finish_photo_filename: str | None = None):
  9657. """Send print complete notifications in background."""
  9658. try:
  9659. logger.info(
  9660. "[NOTIFY-BG] Starting notifications for printer %s, photo=%s", printer_id, finish_photo_filename
  9661. )
  9662. async with async_session() as db:
  9663. from backend.app.models.archive import PrintArchive
  9664. from backend.app.models.printer import Printer
  9665. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  9666. printer = result.scalar_one_or_none()
  9667. printer_name = printer.name if printer else f"Printer {printer_id}"
  9668. archive_data = None
  9669. if archive_id:
  9670. from backend.app.models.print_log import PrintLogEntry
  9671. archive_result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  9672. archive = archive_result.scalar_one_or_none()
  9673. # The run that ended: the archive shows its latest, which
  9674. # with another plate of it printing elsewhere is not this.
  9675. run = await db.get(PrintLogEntry, completed_run_id) if completed_run_id is not None else None
  9676. run_started = run.started_at if run is not None else archive.started_at if archive else None
  9677. run_completed = run.completed_at if run is not None else archive.completed_at if archive else None
  9678. if archive:
  9679. # Actual elapsed time from started_at/completed_at when both are
  9680. # populated (every terminal status sets completed_at after #1198).
  9681. # Falls back to None so the notification path can decide whether to
  9682. # render the slicer estimate as a last resort.
  9683. actual_time_seconds = None
  9684. # Unknown for a completion reconciled at reconnect: the
  9685. # print ended while the printer was offline and
  9686. # completed_at is the reconnect, so the gap would read
  9687. # as the duration -- 14 h for a night offline (DB-C9).
  9688. # An estimated end (#3310) gives the estimate.
  9689. if (
  9690. run_started
  9691. and run_completed
  9692. and (not data.get("_reconciled") or data.get("_reconciled_completed_at"))
  9693. ):
  9694. elapsed = (run_completed - run_started).total_seconds()
  9695. if elapsed > 0:
  9696. actual_time_seconds = int(elapsed)
  9697. archive_data = {
  9698. "print_time_seconds": archive.print_time_seconds,
  9699. "actual_time_seconds": actual_time_seconds,
  9700. "actual_filament_grams": archive.filament_used_grams,
  9701. "failure_reason": run.failure_reason if run is not None else archive.failure_reason,
  9702. "created_by_id": archive.created_by_id,
  9703. }
  9704. # Scale filament usage for partial prints
  9705. if print_status != "completed" and archive.filament_used_grams:
  9706. progress = data.get("progress") or 0
  9707. scale = _partial_progress_scale(progress)
  9708. archive_data["actual_filament_grams"] = round(archive.filament_used_grams * scale, 1)
  9709. archive_data["progress"] = progress
  9710. # Pass per-slot data from archive.extra_data
  9711. if archive.extra_data and archive.extra_data.get("filament_slots"):
  9712. slots = archive.extra_data["filament_slots"]
  9713. if print_status != "completed":
  9714. scale = _partial_progress_scale(data.get("progress"))
  9715. slots = [{**s, "used_g": round(s["used_g"] * scale, 1)} for s in slots]
  9716. archive_data["filament_slots"] = slots
  9717. # Scope project-summed totals down to the plate that was
  9718. # actually printed — see _scope_notification_archive_data_to_plate
  9719. # for the why (#1785).
  9720. archive_data = _scope_notification_archive_data_to_plate(
  9721. archive_data,
  9722. archive.file_path,
  9723. notify_plate_id,
  9724. print_status,
  9725. data.get("progress"),
  9726. app_settings.base_dir,
  9727. )
  9728. # Enrich filament_grams from usage_results when archive has no 3MF data
  9729. if not archive_data.get("actual_filament_grams") and usage_results:
  9730. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  9731. if total_from_usage > 0:
  9732. archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  9733. # Pass usage tracker results for AMS slot info in notifications
  9734. if usage_results:
  9735. archive_data["usage_results"] = usage_results
  9736. # Which archive and print it was (archive redesign).
  9737. archive_data["archive_id"] = archive_id
  9738. archive_data["run_id"] = completed_run_id
  9739. # Add finish photo URL and image bytes if available
  9740. if finish_photo_filename:
  9741. photo_url, photo_bytes = await _finish_photo_for_notification(
  9742. db, archive, archive_id, finish_photo_filename
  9743. )
  9744. if photo_url:
  9745. archive_data["finish_photo_url"] = photo_url
  9746. if photo_bytes:
  9747. archive_data["image_data"] = photo_bytes
  9748. if not await _kill_switch_notification_already_sent(kill_switch_notification_task):
  9749. await notification_service.on_print_complete(
  9750. printer_id, printer_name, print_status, data, db, archive_data=archive_data
  9751. )
  9752. else:
  9753. logger.info("[NOTIFY-BG] Skipped duplicate kill-switch provider notification")
  9754. # Post-print outcome confirmation (#1898). Runs in this
  9755. # background task so the finish photo fetched above rides
  9756. # along with the prompt.
  9757. if print_status == "completed" and archive_id:
  9758. await _dispatch_outcome_confirmation_safely(
  9759. db, printer_id, printer_name, data, archive_id, archive_data, run_id=completed_run_id
  9760. )
  9761. # Send user-specific email notification
  9762. if archive_data:
  9763. created_by_id = archive_data.get("created_by_id")
  9764. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  9765. await _dispatch_user_print_email(
  9766. print_status,
  9767. created_by_id,
  9768. printer_name,
  9769. raw_filename,
  9770. db,
  9771. image_data=archive_data.get("image_data"),
  9772. finish_photo_url=archive_data.get("finish_photo_url"),
  9773. )
  9774. logger.info("[NOTIFY-BG] Completed")
  9775. except Exception as e:
  9776. logger.error("[NOTIFY-BG] Failed: %s", e, exc_info=True)
  9777. async def _background_maintenance_check():
  9778. """Check for maintenance due in background."""
  9779. if print_status != "completed":
  9780. return
  9781. try:
  9782. logger.info("[MAINT-BG] Starting maintenance check for printer %s", printer_id)
  9783. async with async_session() as db:
  9784. from backend.app.models.printer import Printer
  9785. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  9786. printer = result.scalar_one_or_none()
  9787. printer_name = printer.name if printer else f"Printer {printer_id}"
  9788. await ensure_default_types(db)
  9789. overview = await _get_printer_maintenance_internal(printer_id, db, commit=True)
  9790. attention = [
  9791. item for item in overview.maintenance_items if item.enabled and (item.is_due or item.is_warning)
  9792. ]
  9793. items_needing_attention = [
  9794. {"name": item.maintenance_type_name, "is_due": item.is_due, "is_warning": item.is_warning}
  9795. for item in attention
  9796. ]
  9797. if items_needing_attention:
  9798. await notification_service.on_maintenance_due(printer_id, printer_name, items_needing_attention, db)
  9799. logger.info("[MAINT-BG] Sent notification: %s items need attention", len(items_needing_attention))
  9800. # MQTT relay - publish maintenance alerts
  9801. for item in attention:
  9802. current_value, threshold, unit = maintenance_alert_values(item)
  9803. try:
  9804. await mqtt_relay.on_maintenance_alert(
  9805. printer_id=printer_id,
  9806. printer_name=printer_name,
  9807. maintenance_type=item.maintenance_type_name,
  9808. current_value=current_value,
  9809. threshold=threshold,
  9810. unit=unit,
  9811. )
  9812. except Exception:
  9813. pass # Don't fail if MQTT fails
  9814. else:
  9815. logger.info("[MAINT-BG] Completed (no items need attention)")
  9816. except Exception as e:
  9817. logger.warning("[MAINT-BG] Failed: %s", e)
  9818. spawn_background_task(_background_smart_plug(), name="background-smart-plug")
  9819. spawn_background_task(_background_maintenance_check(), name="background-maintenance-check")
  9820. # Notification task waits for photo capture to complete first (with timeout).
  9821. # When a timelapse was recording, photo sourcing polls the per-print
  9822. # timelapse for up to 60s (#1397) — extend the budget so the notification
  9823. # carries the correct bed-up photo instead of falling through to the
  9824. # live-cam grab. Adds ~30s of notification latency at worst on slow links.
  9825. #
  9826. # #2547: both budgets now have to cover a plate restore as well.
  9827. #
  9828. # Without timelapse, the wait is on the moment producer, which raises the
  9829. # plate before its grab — so this has to outlast that producer's own budget.
  9830. #
  9831. # With timelapse, the capture polls up to
  9832. # `_FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS` for the video and only then
  9833. # falls back to a live grab, which is the case that raises the plate. At the
  9834. # old flat 75s that fallback was guaranteed to be cut off mid-settle, so the
  9835. # restore would have moved the plate for a photo nobody waited for.
  9836. photo_wait_timeout = (
  9837. _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS + _FINISH_PHOTO_PRODUCER_WAIT_SECONDS
  9838. if data.get("timelapse_was_active")
  9839. else _FINISH_PHOTO_PRODUCER_WAIT_SECONDS + 15
  9840. )
  9841. async def _photo_then_notify():
  9842. """Wait for photo capture, then send notification with photo URL."""
  9843. finish_photo = None
  9844. try:
  9845. finish_photo = await asyncio.wait_for(photo_task, timeout=photo_wait_timeout)
  9846. logger.info("[PHOTO-NOTIFY] Photo task returned: %s", finish_photo)
  9847. except TimeoutError:
  9848. logger.warning(
  9849. "[PHOTO-NOTIFY] Photo capture timed out after %ss, sending notification without photo",
  9850. photo_wait_timeout,
  9851. )
  9852. except Exception as e:
  9853. logger.warning("[PHOTO-NOTIFY] Photo task failed: %s", e)
  9854. try:
  9855. await _background_notifications(finish_photo)
  9856. except Exception as e:
  9857. logger.error("[PHOTO-NOTIFY] Notification sending failed: %s", e, exc_info=True)
  9858. spawn_background_task(_photo_then_notify(), name="photo-then-notify")
  9859. # Stitch external camera layer timelapse if session was active
  9860. print_status = data.get("status", "completed")
  9861. async def _background_layer_timelapse():
  9862. """Stitch layer timelapse and attach to archive."""
  9863. from backend.app.services.layer_timelapse import cancel_session, on_print_complete as tl_complete
  9864. try:
  9865. if print_status == "completed":
  9866. logger.info("[LAYER-TL] Stitching layer timelapse for printer %s", printer_id)
  9867. # The run the frames belong to, read before the stitch: it can
  9868. # take long enough for the archive to start printing again.
  9869. run_started_at = None
  9870. if archive_id:
  9871. async with async_session() as db:
  9872. run_archive = await ArchiveService(db).get_archive(archive_id)
  9873. run_started_at = run_archive.started_at if run_archive else None
  9874. timelapse_path = await tl_complete(printer_id)
  9875. if timelapse_path and archive_id:
  9876. await _attach_layer_timelapse(
  9877. archive_id, timelapse_path, notify_plate_id, run_started_at, run_id=completed_run_id
  9878. )
  9879. elif timelapse_path:
  9880. # Timelapse created but no archive - just clean up
  9881. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  9882. else:
  9883. # Print failed or cancelled - cancel timelapse session
  9884. cancel_session(printer_id)
  9885. logger.info(
  9886. "[LAYER-TL] Cancelled layer timelapse for printer %s (status: %s)", printer_id, print_status
  9887. )
  9888. except Exception as e:
  9889. logger.warning("[LAYER-TL] Failed: %s", e)
  9890. # A completed print's session left _active_sessions when its stitch
  9891. # began: the one there now belongs to the next print, which may have
  9892. # started during the stitch, and must not lose its frames.
  9893. if print_status != "completed":
  9894. try:
  9895. cancel_session(printer_id)
  9896. except Exception:
  9897. pass # Best-effort timelapse session cancellation on error
  9898. from backend.app.services import archive_work
  9899. # Held through the stitch, which can take longer than a merge waits after
  9900. # a print (archive redesign P2).
  9901. layer_timelapse = _background_layer_timelapse()
  9902. if archive_id:
  9903. layer_timelapse = archive_work.held_while(archive_id, layer_timelapse)
  9904. spawn_background_task(layer_timelapse, name="background-layer-timelapse")
  9905. log_timing("All background tasks scheduled")
  9906. # Auto-scan for timelapse if recording was active during the print. A
  9907. # failed or cancelled print gets one too: the printer writes the video of
  9908. # the run so far when it stops (measured on an H2C, #3481), and that video
  9909. # is how the user sees what went wrong. A model that writes none costs one
  9910. # poll that finds nothing new.
  9911. if archive_id and data.get("timelapse_was_active"):
  9912. logger.info("[TIMELAPSE] Timelapse was active during print, scheduling auto-scan for archive %s", archive_id)
  9913. # Schedule timelapse scan as background task with retries
  9914. # The printer needs time to encode the video after print completion
  9915. _schedule_timelapse_scan(archive_id, printer_id, plate_id=notify_plate_id, run_id=completed_run_id)
  9916. log_timing("Timelapse scan scheduled")
  9917. logger.info("[CALLBACK] on_print_complete finished for printer %s, archive %s", printer_id, archive_id)
  9918. # AMS sensor history recording
  9919. _ams_history_task: asyncio.Task | None = None
  9920. AMS_HISTORY_INTERVAL = 300 # Record every 5 minutes
  9921. AMS_HISTORY_RETENTION_DAYS = 30 # Keep data for 30 days
  9922. def _retention_days(stored: str | None, default: int) -> int:
  9923. """A history retention setting as days to keep: at least 1, the default
  9924. when unset or unreadable. A stored 0 or less (the API took one before it
  9925. had a minimum) would put the cutoff at now and delete all history."""
  9926. try:
  9927. return max(1, int(stored)) if stored is not None else default
  9928. except (TypeError, ValueError):
  9929. return default
  9930. _ams_cleanup_counter = 0 # Track recordings to trigger periodic cleanup
  9931. # Track alarm cooldowns (printer_id:ams_id:type -> last_alarm_time)
  9932. _ams_alarm_cooldown: dict[str, datetime] = {}
  9933. AMS_ALARM_COOLDOWN_MINUTES = 60 # Don't send same alarm more than once per hour
  9934. # (printer_id, ams_id) already reported as sending the drop index and no
  9935. # percentage. Logged once each so a supported printer that turns out to do this
  9936. # shows up in a support bundle rather than as a user wondering where the
  9937. # humidity reading went -- see the note at the read site below (#3140).
  9938. _ams_index_only_logged: set[tuple[int, int]] = set()
  9939. def _resolve_temp_alarm_threshold(fair_threshold: float, raw_alarm_value: str | None) -> float:
  9940. """Temperature at which the AMS alarm fires, falling back to the display band.
  9941. ``ams_temp_fair`` decides when the AMS card turns amber. It used to decide
  9942. when a notification was sent as well, which is why a room above it made the
  9943. alarm fire once an hour for as long as the weather lasted -- and the only way
  9944. to stop that was to raise the display band and lose the colour that says the
  9945. unit is warm (#2905).
  9946. Unset resolves to the fair threshold, so an install that never sets one is
  9947. unchanged. Settings storage stringifies ``None`` to the literal ``"None"``,
  9948. so that arrives here as a string and is handled by the same branch as any
  9949. other unparseable value -- there is no separate sentinel to keep in sync.
  9950. A non-positive value is refused rather than honoured: zero would alarm
  9951. permanently, and it is far more likely to be a cleared field than a
  9952. deliberate choice.
  9953. """
  9954. if raw_alarm_value is None:
  9955. return fair_threshold
  9956. try:
  9957. value = float(raw_alarm_value)
  9958. except (TypeError, ValueError):
  9959. return fair_threshold
  9960. if not math.isfinite(value) or value <= 0:
  9961. return fair_threshold
  9962. return value
  9963. # Per-AMS "drying was live at" latch that suppresses the high-temperature alarm
  9964. # through a cycle and the cool-down after it (#1802). Stored in the settings
  9965. # table rather than alongside _ams_alarm_cooldown above, because a restart
  9966. # partway through a cool-down would otherwise resume alarming about heat the
  9967. # user asked for — the same internal-timestamp-row pattern as
  9968. # support.py's debug_logging_enabled_at.
  9969. AMS_DRYING_LATCH_KEY = "ams_drying_alarm_latch"
  9970. # Upper bound on that suppression. The latch normally clears as soon as the unit
  9971. # reads at or below the threshold; see utils.ams_drying for why this cap only
  9972. # matters when it never does.
  9973. AMS_DRYING_GRACE_MINUTES = 120
  9974. async def _load_ams_drying_latch(db) -> dict[str, datetime]:
  9975. """Read the persisted per-AMS drying latch, dropping entries out of window.
  9976. Anything older than the grace cap would expire on its next visit anyway, so
  9977. discarding it here costs nothing and stops rows for deleted printers from
  9978. accumulating.
  9979. Stamps ahead of now get two defences, because a box whose clock jumps
  9980. backwards (a Pi with no RTC coming up before NTP) writes them: wildly future
  9981. ones are discarded outright, and the rest are clamped to now. Without the
  9982. clamp the cap would measure from a moment that has not happened yet and hold
  9983. the alarm quiet for the skew on top of the cap. One unnecessary notification
  9984. after a clock jump is a far better failure than an alarm silently disabled
  9985. for hours.
  9986. """
  9987. from backend.app.models.settings import Settings
  9988. result = await db.execute(select(Settings).where(Settings.key == AMS_DRYING_LATCH_KEY))
  9989. setting = result.scalar_one_or_none()
  9990. if not setting or not setting.value:
  9991. return {}
  9992. try:
  9993. raw = json.loads(setting.value)
  9994. except (ValueError, TypeError):
  9995. return {} # Corrupted row → no latch, alarms behave as they did before
  9996. if not isinstance(raw, dict):
  9997. return {}
  9998. now = datetime.now(timezone.utc)
  9999. window = timedelta(minutes=AMS_DRYING_GRACE_MINUTES)
  10000. latch: dict[str, datetime] = {}
  10001. for key, value in raw.items():
  10002. try:
  10003. stamp = datetime.fromisoformat(str(value))
  10004. except (ValueError, TypeError):
  10005. continue
  10006. if stamp.tzinfo is None:
  10007. stamp = stamp.replace(tzinfo=timezone.utc)
  10008. if not (now - window <= stamp <= now + window):
  10009. continue
  10010. # Nothing may sit in the future: suppression is measured as now minus
  10011. # the stamp, so a stamp ahead of now would extend it by the skew on top
  10012. # of the cap. Clamping the survivors keeps the cap an actual cap.
  10013. latch[str(key)] = min(stamp, now)
  10014. return latch
  10015. async def _save_ams_drying_latch(db, latch: dict[str, datetime]) -> None:
  10016. """Persist the latch, writing only when it actually changed.
  10017. Adds the session change but does not commit — the caller's own commit
  10018. carries it, so the latch lands in the same transaction as the sensor rows
  10019. that produced it.
  10020. """
  10021. from backend.app.models.settings import Settings
  10022. payload = json.dumps({key: stamp.isoformat() for key, stamp in sorted(latch.items())})
  10023. result = await db.execute(select(Settings).where(Settings.key == AMS_DRYING_LATCH_KEY))
  10024. setting = result.scalar_one_or_none()
  10025. if setting is None:
  10026. # Don't create the row on installs that never dry anything.
  10027. if payload != "{}":
  10028. db.add(Settings(key=AMS_DRYING_LATCH_KEY, value=payload))
  10029. elif setting.value != payload:
  10030. setting.value = payload
  10031. def _ams_has_filament(ams_data: dict) -> bool:
  10032. """True if this AMS unit has at least one tray slot holding filament.
  10033. Bambu firmware reports loaded slots via `tray_exist_bits`, a per-AMS hex
  10034. bitmap (one bit per tray slot — bit set = spool present). Empty AMS units
  10035. still report sensor readings, but those readings are ambient and not
  10036. actionable: no filament to dry, no humidity to push down. #1619 — gate
  10037. humidity/temperature alarms on this check so empty units don't generate
  10038. hourly noise. Sensor history still records regardless so the UI charts
  10039. stay continuous.
  10040. Fallback path inspects the `tray` array's `tray_type` fields for setups
  10041. where `tray_exist_bits` is missing (some early-connection pushall shapes).
  10042. """
  10043. bits = ams_data.get("tray_exist_bits")
  10044. if isinstance(bits, str) and bits.strip():
  10045. try:
  10046. return int(bits, 16) > 0
  10047. except ValueError:
  10048. pass
  10049. trays = ams_data.get("tray")
  10050. if isinstance(trays, list):
  10051. return any(
  10052. isinstance(t, dict) and isinstance(t.get("tray_type"), str) and t["tray_type"].strip() for t in trays
  10053. )
  10054. return False
  10055. async def record_ams_history():
  10056. """Background task to record AMS humidity and temperature data."""
  10057. logger = logging.getLogger(__name__)
  10058. # Wait a short time for MQTT connections to establish on startup
  10059. await asyncio.sleep(10)
  10060. while True:
  10061. try:
  10062. from backend.app.models.ams_history import AMSSensorHistory
  10063. from backend.app.models.printer import Printer
  10064. from backend.app.models.settings import Settings
  10065. async with async_session() as db:
  10066. # Get all active printers
  10067. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  10068. printers = result.scalars().all()
  10069. # Get alarm thresholds from settings
  10070. humidity_threshold = 60.0 # Default: fair threshold
  10071. temp_fair_threshold = 35.0 # Display band default (ams_temp_fair)
  10072. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  10073. setting = result.scalar_one_or_none()
  10074. if setting:
  10075. try:
  10076. humidity_threshold = float(setting.value)
  10077. except (ValueError, TypeError):
  10078. pass # Keep default threshold if stored value is invalid
  10079. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_fair"))
  10080. setting = result.scalar_one_or_none()
  10081. if setting:
  10082. try:
  10083. temp_fair_threshold = float(setting.value)
  10084. except (ValueError, TypeError):
  10085. pass # Keep default threshold if stored value is invalid
  10086. # The alarm gets its own threshold, seeded from the resolved fair
  10087. # value so an install that has never set one behaves exactly as
  10088. # it did before (#2905). ams_temp_fair decides when the card turns
  10089. # amber; 35 C is a reasonable place to change a colour and not a
  10090. # reasonable place to page someone. A room above it makes the
  10091. # alarm fire once an hour for as long as the weather lasts, and
  10092. # the only way to stop it was to raise the display band and lose
  10093. # the colour that says the unit is warm.
  10094. #
  10095. # An unset value is stored as the literal "None", which the except
  10096. # below swallows the same way it swallows garbage -- so the
  10097. # fallback costs nothing and needs no sentinel of its own.
  10098. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_alarm"))
  10099. setting = result.scalar_one_or_none()
  10100. temp_alarm_threshold = _resolve_temp_alarm_threshold(
  10101. temp_fair_threshold, setting.value if setting else None
  10102. )
  10103. # Per-filament humidity threshold overrides (#1605) — resolved
  10104. # per-AMS below from the loaded tray types. Reuses the same
  10105. # resolver as the auto-drying scheduler so behavior stays in
  10106. # lockstep across both consumers.
  10107. from backend.app.services.print_scheduler import PrintScheduler
  10108. per_type_humidity_thresholds: dict[str, int] = {}
  10109. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_thresholds"))
  10110. setting = result.scalar_one_or_none()
  10111. if setting and setting.value:
  10112. try:
  10113. raw = json.loads(setting.value)
  10114. if isinstance(raw, dict):
  10115. for k, v in raw.items():
  10116. try:
  10117. per_type_humidity_thresholds[str(k).upper() if k != "default" else "default"] = int(
  10118. v
  10119. )
  10120. except (TypeError, ValueError):
  10121. continue
  10122. except (ValueError, TypeError):
  10123. pass # Invalid JSON → no overrides, fall through to global threshold
  10124. # Per-AMS drying latch (#1802), loaded once per pass and written
  10125. # back below only if a unit changed it.
  10126. drying_latch = await _load_ams_drying_latch(db)
  10127. drying_latch_before = dict(drying_latch)
  10128. recorded_count = 0
  10129. for printer in printers:
  10130. # Get current state from printer manager
  10131. state = printer_manager.get_status(printer.id)
  10132. if not state or not state.connected or not state.raw_data:
  10133. continue # Skip disconnected printers - don't use stale data
  10134. raw_data = state.raw_data
  10135. if "ams" not in raw_data or not isinstance(raw_data["ams"], list):
  10136. continue
  10137. # Record data for each AMS unit
  10138. for ams_data in raw_data["ams"]:
  10139. ams_id = int(ams_data.get("id", 0))
  10140. # Percentage only. The 1-5 index is inverted, so
  10141. # charting it as a percentage drew the wettest units as
  10142. # the driest (#3140); a unit that reports no percentage
  10143. # leaves a gap in the chart instead. See
  10144. # utils/ams_humidity.
  10145. humidity = ams_humidity_percent(ams_data)
  10146. # No supported printer is known to send the index
  10147. # alone -- the report came from unsupported firmware,
  10148. # and no install has been seen using the old fallback.
  10149. # "Known" is doing work there, so say so once per unit:
  10150. # the alternative is a silent blank card.
  10151. if humidity is None and ams_data.get("humidity") is not None:
  10152. unit_key = (printer.id, ams_id)
  10153. if unit_key not in _ams_index_only_logged:
  10154. _ams_index_only_logged.add(unit_key)
  10155. logger.info(
  10156. "[%s] AMS %d reports the 1-5 humidity index but no usable humidity_raw "
  10157. "percentage. The index is inverted and is not shown as a percentage "
  10158. "(#3140), so this unit has no humidity reading, chart or alarm. "
  10159. "Please report this with the printer and AMS firmware versions.",
  10160. printer.name,
  10161. ams_id,
  10162. )
  10163. # Get temperature
  10164. temperature = None
  10165. temp_str = ams_data.get("temp")
  10166. if temp_str is not None:
  10167. try:
  10168. temperature = float(temp_str)
  10169. except (ValueError, TypeError):
  10170. pass # Skip unparseable temperature value
  10171. # Skip if no data
  10172. if humidity is None and temperature is None:
  10173. continue
  10174. # Record the data point
  10175. history = AMSSensorHistory(
  10176. printer_id=printer.id,
  10177. ams_id=ams_id,
  10178. humidity=humidity,
  10179. # Both columns hold the same reading now that the
  10180. # index can no longer reach ``humidity``. Writing it
  10181. # through the same value also stops a genuine 0%
  10182. # from being stored as NULL, which the old truthiness
  10183. # test did.
  10184. humidity_raw=humidity,
  10185. temperature=temperature,
  10186. )
  10187. db.add(history)
  10188. recorded_count += 1
  10189. # Generate AMS label and determine if it's AMS-HT (A, B, C, D or HT-A for AMS-Lite/Hub)
  10190. ams_type = unit_ams_type(ams_id, ams_data.get("info"))
  10191. is_ams_ht = ams_type == AmsType.N3S
  10192. ams_label = unit_label(ams_id, ams_type)
  10193. # Skip alarm dispatch for empty AMS units — humidity /
  10194. # temperature readings are ambient with no filament to
  10195. # protect, and the hourly notification just becomes
  10196. # noise. Sensor history was already recorded above so
  10197. # the UI charts stay continuous (#1619). Per-AMS check
  10198. # so a multi-AMS setup with one loaded + one empty
  10199. # still alarms on the loaded unit.
  10200. if not _ams_has_filament(ams_data):
  10201. continue
  10202. # Resolve per-filament humidity threshold for this AMS
  10203. # unit (#1605). Falls back to the global ams_humidity_fair
  10204. # when no per-type overrides are configured.
  10205. trays = ams_data.get("tray", []) or []
  10206. effective_humidity_threshold = float(
  10207. PrintScheduler.resolve_humidity_threshold(
  10208. trays, per_type_humidity_thresholds, int(humidity_threshold)
  10209. )
  10210. )
  10211. # Check humidity alarm (only if above threshold)
  10212. if humidity is not None and humidity > effective_humidity_threshold:
  10213. cooldown_key = f"{printer.id}:{ams_id}:humidity"
  10214. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  10215. now = datetime.now(timezone.utc)
  10216. if (
  10217. last_alarm is None
  10218. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  10219. ):
  10220. _ams_alarm_cooldown[cooldown_key] = now
  10221. logger.info(
  10222. f"Sending humidity alarm for {printer.name} {ams_label}: {humidity}% > {effective_humidity_threshold}%"
  10223. )
  10224. try:
  10225. # Call different notification method based on AMS type
  10226. if is_ams_ht:
  10227. await notification_service.on_ams_ht_humidity_high(
  10228. printer.id,
  10229. printer.name,
  10230. ams_label,
  10231. humidity,
  10232. effective_humidity_threshold,
  10233. db,
  10234. )
  10235. else:
  10236. await notification_service.on_ams_humidity_high(
  10237. printer.id,
  10238. printer.name,
  10239. ams_label,
  10240. humidity,
  10241. effective_humidity_threshold,
  10242. db,
  10243. )
  10244. except Exception as e:
  10245. logger.warning("Failed to send humidity alarm: %s", e)
  10246. # A drying cycle heats the unit far past ams_temp_fair on
  10247. # purpose — 45 C for PLA, 65 C for PETG, 85 C on an
  10248. # AMS-HT, against a 35 C default — so the alarm fired
  10249. # once an hour for the whole cycle and kept firing while
  10250. # the unit cooled back down (#1802). Latch on the
  10251. # firmware's own drying state and hold until the reading
  10252. # returns to normal. Humidity is deliberately left alone:
  10253. # it falls during drying, which is the whole point.
  10254. latch_key = f"{printer.id}:{ams_id}"
  10255. # The latch releases at `threshold`, so it takes the alarm
  10256. # number too. Handing it the display band would strand the
  10257. # latch on any unit that settles back above it -- a room
  10258. # where the AMS rests at 37.7 C never returns under a 35 C
  10259. # band, so the latch could only expire on the grace cap
  10260. # rather than releasing when the unit had actually cooled.
  10261. suppress_temp_alarm, new_latch = temperature_alarm_suppressed(
  10262. drying_active=is_drying_active(ams_data),
  10263. temperature=temperature,
  10264. threshold=temp_alarm_threshold,
  10265. latched_at=drying_latch.get(latch_key),
  10266. now=datetime.now(timezone.utc),
  10267. grace_minutes=AMS_DRYING_GRACE_MINUTES,
  10268. )
  10269. if new_latch is None:
  10270. drying_latch.pop(latch_key, None)
  10271. else:
  10272. drying_latch[latch_key] = new_latch
  10273. # Check temperature alarm (only if above threshold)
  10274. if temperature is not None and temperature > temp_alarm_threshold and not suppress_temp_alarm:
  10275. cooldown_key = f"{printer.id}:{ams_id}:temperature"
  10276. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  10277. now = datetime.now(timezone.utc)
  10278. if (
  10279. last_alarm is None
  10280. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  10281. ):
  10282. _ams_alarm_cooldown[cooldown_key] = now
  10283. logger.info(
  10284. f"Sending temperature alarm for {printer.name} {ams_label}: "
  10285. f"{temperature}°C > {temp_alarm_threshold}°C"
  10286. )
  10287. try:
  10288. # Call different notification method based on AMS type
  10289. if is_ams_ht:
  10290. # The reported threshold has to be the one
  10291. # that fired, or the message says "> 35 °C"
  10292. # while firing at 45.
  10293. await notification_service.on_ams_ht_temperature_high(
  10294. printer.id, printer.name, ams_label, temperature, temp_alarm_threshold, db
  10295. )
  10296. else:
  10297. await notification_service.on_ams_temperature_high(
  10298. printer.id, printer.name, ams_label, temperature, temp_alarm_threshold, db
  10299. )
  10300. except Exception as e:
  10301. logger.warning("Failed to send temperature alarm: %s", e)
  10302. if drying_latch != drying_latch_before:
  10303. await _save_ams_drying_latch(db, drying_latch)
  10304. await db.commit()
  10305. if recorded_count > 0:
  10306. logger.info("Recorded %s AMS sensor history entries", recorded_count)
  10307. # Periodic cleanup of old data (every ~288 recordings = ~24 hours at 5min interval)
  10308. global _ams_cleanup_counter
  10309. _ams_cleanup_counter += 1
  10310. if _ams_cleanup_counter >= 288:
  10311. _ams_cleanup_counter = 0
  10312. # Get retention days from settings
  10313. from backend.app.models.settings import Settings
  10314. result = await db.execute(select(Settings).where(Settings.key == "ams_history_retention_days"))
  10315. setting = result.scalar_one_or_none()
  10316. retention_days = _retention_days(setting.value if setting else None, AMS_HISTORY_RETENTION_DAYS)
  10317. cutoff = utcnow_naive() - timedelta(days=retention_days)
  10318. result = await db.execute(delete(AMSSensorHistory).where(AMSSensorHistory.recorded_at < cutoff))
  10319. await db.commit()
  10320. if result.rowcount > 0:
  10321. logger.info(
  10322. f"Cleaned up {result.rowcount} old AMS sensor history entries (older than {retention_days} days)"
  10323. )
  10324. # Wait until next recording interval
  10325. await asyncio.sleep(AMS_HISTORY_INTERVAL)
  10326. except asyncio.CancelledError:
  10327. break
  10328. except Exception as e:
  10329. logger.warning("AMS history recording failed: %s", e)
  10330. await asyncio.sleep(60) # Wait a bit before retrying
  10331. def start_ams_history_recording():
  10332. """Start the AMS history recording background task."""
  10333. global _ams_history_task
  10334. if _ams_history_task is None:
  10335. _ams_history_task = asyncio.create_task(record_ams_history())
  10336. logging.getLogger(__name__).info("AMS history recording started")
  10337. def stop_ams_history_recording():
  10338. """Stop the AMS history recording background task."""
  10339. global _ams_history_task
  10340. if _ams_history_task:
  10341. _ams_history_task.cancel()
  10342. _ams_history_task = None
  10343. logging.getLogger(__name__).info("AMS history recording stopped")
  10344. # Printer sensor history recording (nozzle / bed / chamber)
  10345. _printer_sensor_history_task: asyncio.Task | None = None
  10346. PRINTER_SENSOR_HISTORY_INTERVAL = 60 # Record every minute — heaters move faster than AMS humidity
  10347. PRINTER_SENSOR_HISTORY_RETENTION_DAYS = 30
  10348. _printer_sensor_cleanup_counter = 0
  10349. # Sensor kinds tracked in state.temperatures — these are the normalised keys the
  10350. # MQTT parser writes, so we don't need to handle per-model field aliases here
  10351. # (nozzle_temper / left_nozzle_temper / right_nozzle_temper / chamber_temper
  10352. # are all collapsed by services/bambu_mqtt.py before they reach this loop).
  10353. _SENSOR_KINDS = ("nozzle", "nozzle_2", "bed", "chamber")
  10354. _SENSOR_TARGET_KEYS = {
  10355. "nozzle": "nozzle_target",
  10356. "nozzle_2": "nozzle_2_target",
  10357. "bed": "bed_target",
  10358. "chamber": "chamber_target",
  10359. }
  10360. async def record_printer_sensor_history():
  10361. """Background task to record nozzle / bed / chamber readings.
  10362. Pulls from `state.temperatures` (already normalised across all printer
  10363. models by the MQTT parser) rather than re-parsing raw_data, so we get
  10364. free coverage of dual-nozzle H2D, sensor-only X1C chamber, etc.
  10365. """
  10366. logger = logging.getLogger(__name__)
  10367. await asyncio.sleep(10)
  10368. while True:
  10369. try:
  10370. from backend.app.models.printer import Printer
  10371. from backend.app.models.printer_sensor_history import PrinterSensorHistory
  10372. from backend.app.models.settings import Settings
  10373. async with async_session() as db:
  10374. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  10375. printers = result.scalars().all()
  10376. recorded_count = 0
  10377. for printer in printers:
  10378. state = printer_manager.get_status(printer.id)
  10379. if not state or not state.connected:
  10380. continue
  10381. temps = getattr(state, "temperatures", None) or {}
  10382. if not isinstance(temps, dict):
  10383. continue
  10384. for kind in _SENSOR_KINDS:
  10385. if kind not in temps:
  10386. continue
  10387. try:
  10388. value = float(temps[kind])
  10389. except (ValueError, TypeError):
  10390. continue
  10391. target_raw = temps.get(_SENSOR_TARGET_KEYS[kind])
  10392. target_val: float | None = None
  10393. if target_raw is not None:
  10394. try:
  10395. target_val = float(target_raw)
  10396. except (ValueError, TypeError):
  10397. target_val = None
  10398. db.add(
  10399. PrinterSensorHistory(
  10400. printer_id=printer.id,
  10401. sensor_kind=kind,
  10402. value=value,
  10403. target=target_val,
  10404. )
  10405. )
  10406. recorded_count += 1
  10407. await db.commit()
  10408. if recorded_count > 0:
  10409. logger.debug("Recorded %s printer sensor history entries", recorded_count)
  10410. # Periodic cleanup — once every ~24h at this interval.
  10411. global _printer_sensor_cleanup_counter
  10412. _printer_sensor_cleanup_counter += 1
  10413. cleanup_every = max(1, (24 * 60 * 60) // PRINTER_SENSOR_HISTORY_INTERVAL)
  10414. if _printer_sensor_cleanup_counter >= cleanup_every:
  10415. _printer_sensor_cleanup_counter = 0
  10416. result = await db.execute(
  10417. select(Settings).where(Settings.key == "printer_sensor_history_retention_days")
  10418. )
  10419. setting = result.scalar_one_or_none()
  10420. retention_days = _retention_days(
  10421. setting.value if setting else None, PRINTER_SENSOR_HISTORY_RETENTION_DAYS
  10422. )
  10423. cutoff = utcnow_naive() - timedelta(days=retention_days)
  10424. cleanup = await db.execute(
  10425. delete(PrinterSensorHistory).where(PrinterSensorHistory.recorded_at < cutoff)
  10426. )
  10427. await db.commit()
  10428. if cleanup.rowcount > 0:
  10429. logger.info(
  10430. "Cleaned up %s old printer sensor history entries (older than %s days)",
  10431. cleanup.rowcount,
  10432. retention_days,
  10433. )
  10434. await asyncio.sleep(PRINTER_SENSOR_HISTORY_INTERVAL)
  10435. except asyncio.CancelledError:
  10436. break
  10437. except Exception as e:
  10438. logger.warning("Printer sensor history recording failed: %s", e)
  10439. await asyncio.sleep(60)
  10440. def start_printer_sensor_history_recording():
  10441. global _printer_sensor_history_task
  10442. if _printer_sensor_history_task is None:
  10443. _printer_sensor_history_task = asyncio.create_task(record_printer_sensor_history())
  10444. logging.getLogger(__name__).info("Printer sensor history recording started")
  10445. def stop_printer_sensor_history_recording():
  10446. global _printer_sensor_history_task
  10447. if _printer_sensor_history_task:
  10448. _printer_sensor_history_task.cancel()
  10449. _printer_sensor_history_task = None
  10450. logging.getLogger(__name__).info("Printer sensor history recording stopped")
  10451. # Printer runtime tracking
  10452. _runtime_tracking_task: asyncio.Task | None = None
  10453. RUNTIME_TRACKING_INTERVAL = 30 # Update every 30 seconds
  10454. # Printers the runtime tracker has seen connected since startup. One of them
  10455. # going offline clears its timestamp, so the time it was away is not counted
  10456. # when it comes back printing (DB-C5). A printer not yet seen -- still
  10457. # connecting after a restart -- keeps it for a grace period: a print that ran
  10458. # on through a Bambuddy restart is counted from the stored timestamp, as before.
  10459. _runtime_seen_connected: set[int] = set()
  10460. # A printer never seen connected this long after the tracker started is
  10461. # offline, not still connecting: its stored timestamp predates a gap too.
  10462. RUNTIME_STARTUP_GRACE_SECONDS = 120
  10463. async def track_printer_runtime():
  10464. """Background task to track printer active runtime (RUNNING state only).
  10465. PAUSE is intentionally excluded — the runtime counter feeds hours-based
  10466. maintenance intervals (rod lubrication, belt checks, nozzle cleaning)
  10467. which track mechanical wear. Pause time has no motion and no wear, so
  10468. counting it inflates maintenance warnings (#1521).
  10469. """
  10470. logger = logging.getLogger(__name__)
  10471. # Wait for MQTT connections to establish on startup
  10472. await asyncio.sleep(15)
  10473. started = time.monotonic()
  10474. while True:
  10475. try:
  10476. from backend.app.models.printer import Printer
  10477. # Fetch printer IDs in a short-lived read-only session
  10478. async with async_session() as db:
  10479. result = await db.execute(
  10480. select(Printer.id, Printer.name, Printer.runtime_seconds, Printer.last_runtime_update).where(
  10481. Printer.is_active.is_(True)
  10482. )
  10483. )
  10484. printer_rows = result.all()
  10485. now = datetime.now(timezone.utc)
  10486. updated_count = 0
  10487. # Update each printer in its own short session to minimise write-lock
  10488. # hold time and avoid blocking critical commits like queue status
  10489. # updates (#897).
  10490. for pid, pname, runtime_secs, last_update in printer_rows:
  10491. state = printer_manager.get_status(pid)
  10492. if not state:
  10493. logger.debug("[%s] Runtime tracking: no state available", pname)
  10494. continue
  10495. if not state.connected:
  10496. logger.debug("[%s] Runtime tracking: not connected", pname)
  10497. offline = pid in _runtime_seen_connected or (
  10498. time.monotonic() - started > RUNTIME_STARTUP_GRACE_SECONDS
  10499. )
  10500. if offline and last_update is not None:
  10501. try:
  10502. async with async_session() as db:
  10503. printer = await db.get(Printer, pid)
  10504. if printer:
  10505. printer.last_runtime_update = None
  10506. await db.commit()
  10507. except Exception as e:
  10508. # Still marked: the next offline tick tries again.
  10509. logger.warning("[%s] Runtime tracking commit failed: %s", pname, e)
  10510. continue
  10511. _runtime_seen_connected.discard(pid)
  10512. continue
  10513. _runtime_seen_connected.add(pid)
  10514. needs_commit = False
  10515. new_runtime = runtime_secs
  10516. new_last_update = last_update
  10517. if state.state == "RUNNING":
  10518. if last_update:
  10519. lu = last_update if last_update.tzinfo else last_update.replace(tzinfo=timezone.utc)
  10520. elapsed = (now - lu).total_seconds()
  10521. if elapsed > 0:
  10522. new_runtime = runtime_secs + int(elapsed)
  10523. updated_count += 1
  10524. needs_commit = True
  10525. logger.debug(
  10526. f"[{pname}] Runtime tracking: added {int(elapsed)}s, "
  10527. f"total={new_runtime}s ({new_runtime / 3600:.2f}h)"
  10528. )
  10529. else:
  10530. needs_commit = True
  10531. logger.debug("[%s] Runtime tracking: first active detection", pname)
  10532. new_last_update = now
  10533. else:
  10534. if last_update is not None:
  10535. logger.debug(f"[{pname}] Runtime tracking: state={state.state}, clearing last_runtime_update")
  10536. new_last_update = None
  10537. needs_commit = True
  10538. if needs_commit:
  10539. try:
  10540. async with async_session() as db:
  10541. result = await db.execute(select(Printer).where(Printer.id == pid))
  10542. printer = result.scalar_one_or_none()
  10543. if printer:
  10544. printer.runtime_seconds = new_runtime
  10545. printer.last_runtime_update = new_last_update
  10546. await db.commit()
  10547. except Exception as e:
  10548. logger.warning("[%s] Runtime tracking commit failed: %s", pname, e)
  10549. if updated_count > 0:
  10550. logger.debug("Updated runtime for %s printer(s)", updated_count)
  10551. except asyncio.CancelledError:
  10552. logger.info("Runtime tracking cancelled")
  10553. break
  10554. except Exception as e:
  10555. logger.warning("Runtime tracking failed: %s", e)
  10556. await asyncio.sleep(RUNTIME_TRACKING_INTERVAL)
  10557. def start_runtime_tracking():
  10558. """Start the printer runtime tracking background task."""
  10559. global _runtime_tracking_task
  10560. if _runtime_tracking_task is None:
  10561. _runtime_tracking_task = asyncio.create_task(track_printer_runtime())
  10562. logging.getLogger(__name__).info("Printer runtime tracking started")
  10563. def stop_runtime_tracking():
  10564. """Stop the printer runtime tracking background task."""
  10565. global _runtime_tracking_task
  10566. if _runtime_tracking_task:
  10567. _runtime_tracking_task.cancel()
  10568. _runtime_tracking_task = None
  10569. logging.getLogger(__name__).info("Printer runtime tracking stopped")
  10570. # SpoolBuddy device watchdog
  10571. _spoolbuddy_watchdog_task: asyncio.Task | None = None
  10572. SPOOLBUDDY_WATCHDOG_INTERVAL = 15
  10573. async def _spoolbuddy_watchdog_loop():
  10574. """Periodic check for SpoolBuddy devices that have gone offline."""
  10575. from backend.app.api.routes.spoolbuddy import spoolbuddy_watchdog
  10576. while True:
  10577. try:
  10578. await spoolbuddy_watchdog()
  10579. except asyncio.CancelledError:
  10580. break
  10581. except Exception as e:
  10582. logging.getLogger(__name__).warning("SpoolBuddy watchdog failed: %s", e)
  10583. await asyncio.sleep(SPOOLBUDDY_WATCHDOG_INTERVAL)
  10584. def start_spoolbuddy_watchdog():
  10585. global _spoolbuddy_watchdog_task
  10586. if _spoolbuddy_watchdog_task is None:
  10587. _spoolbuddy_watchdog_task = asyncio.create_task(_spoolbuddy_watchdog_loop())
  10588. logging.getLogger(__name__).info("SpoolBuddy watchdog started")
  10589. def stop_spoolbuddy_watchdog():
  10590. global _spoolbuddy_watchdog_task
  10591. if _spoolbuddy_watchdog_task:
  10592. _spoolbuddy_watchdog_task.cancel()
  10593. _spoolbuddy_watchdog_task = None
  10594. logging.getLogger(__name__).info("SpoolBuddy watchdog stopped")
  10595. # Dead-MQTT-session recovery
  10596. #
  10597. # check_staleness() covers the "connected but silent" half-broken session. It
  10598. # does nothing once ``state.connected`` is False, and paho's own auto-reconnect
  10599. # is the only thing left watching at that point. When paho stops making
  10600. # progress there is no backstop at all: the #2732 bundle has a P1S drop on a
  10601. # keep-alive timeout at 02:19 and not reconnect until 11:24 — nine hours
  10602. # offline with the UI open the whole time, recovered only when something
  10603. # happened to nudge it.
  10604. #
  10605. # This loop is that backstop. It only touches printers that had a working
  10606. # session and lost it, and only when the MQTT port still answers — a printer
  10607. # that is simply switched off is left to paho, since rebuilding a client
  10608. # against an unreachable host achieves nothing and would fill the log every
  10609. # night.
  10610. _connection_watchdog_task: asyncio.Task | None = None
  10611. CONNECTION_WATCHDOG_INTERVAL = 60
  10612. # How long a printer must have been silent before we stop trusting paho.
  10613. # Comfortably above STALE_TIMEOUT (60 s) and the max reconnect backoff (30 s),
  10614. # so a session that is recovering on its own is never interrupted.
  10615. CONNECTION_WATCHDOG_OFFLINE_GRACE = 300
  10616. # Per-printer floor between rebuild attempts.
  10617. CONNECTION_WATCHDOG_RETRY_INTERVAL = 300
  10618. _connection_watchdog_last_attempt: dict[int, float] = {}
  10619. async def _recover_dead_printer_sessions() -> int:
  10620. """Rebuild MQTT clients that have been offline too long to still be trying.
  10621. Returns the number of printers a rebuild was attempted for (for tests and
  10622. for the caller's logging). Never raises: one unreachable printer must not
  10623. stop the sweep for the rest of the farm.
  10624. """
  10625. logger = logging.getLogger(__name__)
  10626. from backend.app.services.printer_diagnostic import PORT_MQTT, check_port
  10627. now = time.monotonic()
  10628. recovered = 0
  10629. for printer_id, client in list(printer_manager._clients.items()):
  10630. try:
  10631. if client.state.connected:
  10632. _connection_watchdog_last_attempt.pop(printer_id, None)
  10633. continue
  10634. # Time since the last inbound message is the age of the last known
  10635. # good session — no extra bookkeeping needed, and it is the same
  10636. # clock is_stale() reads. 0 means this client has never had one:
  10637. # that is the initial-connect path, where paho retrying is the
  10638. # correct and only behaviour, so leave it be.
  10639. last_msg = client._last_message_time
  10640. if not last_msg:
  10641. continue
  10642. offline_for = time.time() - last_msg
  10643. if offline_for < CONNECTION_WATCHDOG_OFFLINE_GRACE:
  10644. continue
  10645. last_attempt = _connection_watchdog_last_attempt.get(printer_id)
  10646. if last_attempt is not None and now - last_attempt < CONNECTION_WATCHDOG_RETRY_INTERVAL:
  10647. continue
  10648. if not await check_port(client.ip_address, PORT_MQTT):
  10649. # Switched off, unplugged, or off the network. Paho's retry is
  10650. # the right handler; say so at debug level and move on.
  10651. logger.debug(
  10652. "[#2732] Printer %s offline for %.0fs and its MQTT port is not answering "
  10653. "— leaving the reconnect to paho",
  10654. printer_id,
  10655. offline_for,
  10656. )
  10657. _connection_watchdog_last_attempt[printer_id] = now
  10658. continue
  10659. _connection_watchdog_last_attempt[printer_id] = now
  10660. recovered += 1
  10661. logger.warning(
  10662. "[#2732] Printer %s has been offline for %.0fs but answers on MQTT port %d — "
  10663. "rebuilding the client with a fresh session (last connect error: %s)",
  10664. printer_id,
  10665. offline_for,
  10666. PORT_MQTT,
  10667. client.last_connect_error or "none recorded",
  10668. )
  10669. # Async context, so this takes the hard-reset path: fresh client_id,
  10670. # paho's QoS 1 queue dropped. That matters — a project_file left
  10671. # unacked on the dead session would otherwise replay into the new
  10672. # one and trip 0500_4003 on the printer (#1136).
  10673. client.force_reconnect_stale_session(f"offline for {offline_for:.0f}s, port still answering")
  10674. except Exception as e:
  10675. logger.warning("[#2732] Connection watchdog failed for printer %s: %s", printer_id, e)
  10676. return recovered
  10677. async def _connection_watchdog_loop():
  10678. logger = logging.getLogger(__name__)
  10679. # Let the initial connects settle before judging anyone offline.
  10680. await asyncio.sleep(CONNECTION_WATCHDOG_OFFLINE_GRACE)
  10681. while True:
  10682. try:
  10683. await _recover_dead_printer_sessions()
  10684. except asyncio.CancelledError:
  10685. break
  10686. except Exception as e:
  10687. logger.warning("Connection watchdog sweep failed: %s", e)
  10688. await asyncio.sleep(CONNECTION_WATCHDOG_INTERVAL)
  10689. def start_connection_watchdog():
  10690. global _connection_watchdog_task
  10691. if _connection_watchdog_task is None:
  10692. _connection_watchdog_task = asyncio.create_task(_connection_watchdog_loop())
  10693. logging.getLogger(__name__).info("Printer connection watchdog started")
  10694. def stop_connection_watchdog():
  10695. global _connection_watchdog_task
  10696. if _connection_watchdog_task:
  10697. _connection_watchdog_task.cancel()
  10698. _connection_watchdog_task = None
  10699. _connection_watchdog_last_attempt.clear()
  10700. logging.getLogger(__name__).info("Printer connection watchdog stopped")
  10701. # Camera stream orphan cleanup
  10702. _camera_cleanup_task: asyncio.Task | None = None
  10703. CAMERA_CLEANUP_INTERVAL = 60
  10704. # Every this many camera cleanups (about an hour), the timelapse janitor too:
  10705. # kept frames of a failed stitch go after a day, not at the next restart.
  10706. TIMELAPSE_CLEANUP_EVERY = 60
  10707. async def _camera_cleanup_loop():
  10708. """Periodically clean up orphaned ffmpeg processes and timelapse leftovers."""
  10709. from backend.app.api.routes.camera import cleanup_orphaned_streams
  10710. from backend.app.services.layer_timelapse import cleanup_orphaned_timelapse_sessions
  10711. rounds = 0
  10712. while True:
  10713. try:
  10714. rounds += 1
  10715. await cleanup_orphaned_streams()
  10716. except asyncio.CancelledError:
  10717. break
  10718. except Exception as e:
  10719. logging.getLogger(__name__).warning("Camera stream cleanup failed: %s", e)
  10720. try:
  10721. if rounds % TIMELAPSE_CLEANUP_EVERY == 0:
  10722. await asyncio.to_thread(cleanup_orphaned_timelapse_sessions)
  10723. except asyncio.CancelledError:
  10724. break
  10725. except Exception as e:
  10726. logging.getLogger(__name__).warning("Timelapse frames cleanup failed: %s", e)
  10727. await asyncio.sleep(CAMERA_CLEANUP_INTERVAL)
  10728. def start_camera_cleanup():
  10729. global _camera_cleanup_task
  10730. if _camera_cleanup_task is None:
  10731. _camera_cleanup_task = asyncio.create_task(_camera_cleanup_loop())
  10732. logging.getLogger(__name__).info("Camera stream cleanup started")
  10733. def stop_camera_cleanup():
  10734. global _camera_cleanup_task
  10735. if _camera_cleanup_task:
  10736. _camera_cleanup_task.cancel()
  10737. _camera_cleanup_task = None
  10738. logging.getLogger(__name__).info("Camera stream cleanup stopped")
  10739. # ---------------------------------------------------------------------------
  10740. # Expected-print TTL eviction
  10741. # ---------------------------------------------------------------------------
  10742. def _evict_stale_expected_prints() -> None:
  10743. """Remove entries from _expected_prints / _expected_print_creators that are
  10744. older than _EXPECTED_PRINT_TTL_SECONDS.
  10745. This prevents unbounded growth when a print is registered (via
  10746. register_expected_print) but on_print_start never fires — e.g. because the
  10747. printer disconnects, the app restarts, or the print is started directly from
  10748. the printer panel without going through the queue.
  10749. """
  10750. # Use monotonic time so the TTL is unaffected by system clock adjustments
  10751. # (e.g. NTP sync, DST changes).
  10752. # A print that ended without its archive being found at completion leaves
  10753. # its id here; only running prints are owned.
  10754. _owned_print_archive_ids.intersection_update(_active_prints.values())
  10755. cutoff = time.monotonic() - _EXPECTED_PRINT_TTL_SECONDS
  10756. stale_keys = [k for k, t in _expected_print_registered_at.items() if t < cutoff]
  10757. if not stale_keys:
  10758. return
  10759. evicted: set[tuple[int, int]] = set()
  10760. for key in stale_keys:
  10761. archive_id = _expected_prints.pop(key, None)
  10762. if archive_id is not None:
  10763. evicted.add((key[0], archive_id))
  10764. _expected_print_creators.pop(key, None)
  10765. _expected_print_registered_at.pop(key, None)
  10766. # Also clean up _print_ams_mappings and _print_plate_ids for archive_ids
  10767. # that have no remaining live keys in _expected_prints (all variants
  10768. # were just evicted) and are not printing right now. Print start pops
  10769. # only the names the printer reported, so the "<base>.gcode" variant
  10770. # outlives it; without the printing check its eviction two hours after
  10771. # dispatch threw away a running print's plate, AMS mapping and cost
  10772. # centre, which completion still reads (#3275).
  10773. for printer_id, archive_id in evicted:
  10774. if not _expected_or_printing_here(printer_id, archive_id):
  10775. _drop_dispatch(printer_id, archive_id)
  10776. logging.getLogger(__name__).info(
  10777. "Evicted %d stale expected-print entries (TTL=%ds)", len(stale_keys), _EXPECTED_PRINT_TTL_SECONDS
  10778. )
  10779. async def _expected_prints_cleanup_loop() -> None:
  10780. """Background task: periodically evict stale expected-print entries."""
  10781. while True:
  10782. try:
  10783. _evict_stale_expected_prints()
  10784. except asyncio.CancelledError:
  10785. raise
  10786. except Exception as e:
  10787. logging.getLogger(__name__).warning("Expected prints cleanup failed: %s", e)
  10788. await asyncio.sleep(_EXPECTED_PRINT_CLEANUP_INTERVAL)
  10789. def start_expected_prints_cleanup() -> None:
  10790. global _expected_prints_cleanup_task
  10791. if _expected_prints_cleanup_task is None:
  10792. _expected_prints_cleanup_task = asyncio.create_task(_expected_prints_cleanup_loop())
  10793. logging.getLogger(__name__).info("Expected prints cleanup started")
  10794. def stop_expected_prints_cleanup() -> None:
  10795. global _expected_prints_cleanup_task
  10796. if _expected_prints_cleanup_task:
  10797. _expected_prints_cleanup_task.cancel()
  10798. _expected_prints_cleanup_task = None
  10799. logging.getLogger(__name__).info("Expected prints cleanup stopped")
  10800. # ---------------------------------------------------------------------------
  10801. # L-2: Periodic auth-token cleanup (stale TOTP + expired revoked JTIs)
  10802. # ---------------------------------------------------------------------------
  10803. _auth_cleanup_task: asyncio.Task | None = None
  10804. _AUTH_CLEANUP_INTERVAL = 3600 # seconds (hourly)
  10805. async def _run_auth_cleanup() -> None:
  10806. """Single cleanup pass: remove stale TOTP records, expired revoked JTIs, and old rate-limit events."""
  10807. from backend.app.core.database import async_session
  10808. from backend.app.models.auth_ephemeral import AuthEphemeralToken, AuthRateLimitEvent
  10809. from backend.app.models.user_totp import UserTOTP
  10810. now = datetime.now(timezone.utc)
  10811. # Remove unconfirmed (is_enabled=False) TOTP records older than 1 hour.
  10812. try:
  10813. async with async_session() as db:
  10814. stale_cutoff = now - timedelta(hours=1)
  10815. result = await db.execute(
  10816. select(UserTOTP).where(
  10817. UserTOTP.is_enabled.is_(False),
  10818. UserTOTP.created_at < stale_cutoff,
  10819. )
  10820. )
  10821. stale_records = result.scalars().all()
  10822. if stale_records:
  10823. for rec in stale_records:
  10824. await db.delete(rec)
  10825. await db.commit()
  10826. logging.info("Auth cleanup: removed %d stale unconfirmed TOTP record(s)", len(stale_records))
  10827. except Exception as e:
  10828. logging.warning("Auth cleanup: failed to purge stale TOTP records: %s", e)
  10829. # Remove expired revoked-JTI entries (they are no longer needed once the
  10830. # original token's exp has passed — the token would be rejected by JWT
  10831. # signature verification regardless).
  10832. try:
  10833. async with async_session() as db:
  10834. await db.execute(
  10835. delete(AuthEphemeralToken).where(
  10836. AuthEphemeralToken.token_type == "revoked_jti",
  10837. AuthEphemeralToken.expires_at < now,
  10838. )
  10839. )
  10840. await db.commit()
  10841. except Exception as e:
  10842. logging.warning("Auth cleanup: failed to purge expired revoked JTIs: %s", e)
  10843. # L-R6-B: Purge AuthRateLimitEvent rows older than the lockout window (15 min).
  10844. # Events outside this window can never affect rate-limit decisions — they only
  10845. # consume DB space. Use the same window constant as the rate limiter so the
  10846. # two are always in sync.
  10847. try:
  10848. from backend.app.api.routes.mfa import LOCKOUT_WINDOW
  10849. async with async_session() as db:
  10850. await db.execute(
  10851. delete(AuthRateLimitEvent).where(
  10852. AuthRateLimitEvent.occurred_at < now - LOCKOUT_WINDOW,
  10853. )
  10854. )
  10855. await db.commit()
  10856. except Exception as e:
  10857. logging.warning("Auth cleanup: failed to purge stale rate-limit events: %s", e)
  10858. async def _auth_cleanup_loop() -> None:
  10859. """Periodic background task: run auth cleanup every hour."""
  10860. while True:
  10861. try:
  10862. await _run_auth_cleanup()
  10863. except asyncio.CancelledError:
  10864. break
  10865. except Exception as e:
  10866. logging.warning("Auth cleanup loop error: %s", e)
  10867. await asyncio.sleep(_AUTH_CLEANUP_INTERVAL)
  10868. def start_auth_cleanup() -> None:
  10869. global _auth_cleanup_task
  10870. if _auth_cleanup_task is None:
  10871. _auth_cleanup_task = asyncio.create_task(_auth_cleanup_loop())
  10872. logging.getLogger(__name__).info("Auth periodic cleanup started")
  10873. def stop_auth_cleanup() -> None:
  10874. global _auth_cleanup_task
  10875. if _auth_cleanup_task:
  10876. _auth_cleanup_task.cancel()
  10877. _auth_cleanup_task = None
  10878. logging.getLogger(__name__).info("Auth periodic cleanup stopped")
  10879. async def initialize_wled_printers(db: AsyncSession) -> None:
  10880. """Seed WLED settings and apply offline presets before MQTT connections start."""
  10881. from backend.app.api.routes.settings import get_setting, setting_is_true
  10882. from backend.app.models.printer import Printer
  10883. wled_manager.require_plate_clear = setting_is_true(await get_setting(db, "require_plate_clear"))
  10884. result = await db.execute(select(Printer.id, Printer.wled_config, Printer.is_active))
  10885. for printer_id, config, is_active in result.all():
  10886. wled_manager.configure_printer(printer_id, config)
  10887. if not is_active and config:
  10888. wled_manager.handle_offline(printer_id)
  10889. @asynccontextmanager
  10890. async def lifespan(app: FastAPI):
  10891. # Startup
  10892. # Install Windows-only asyncio Proactor cleanup-RST filter (#1113) before
  10893. # anything else can spawn tasks that might trip it.
  10894. from backend.app.core.asyncio_handlers import install_proactor_reset_filter, warn_if_running_on_uvloop
  10895. install_proactor_reset_filter()
  10896. # Before anything opens files or sockets in bulk: a soft limit of 1024 is
  10897. # what turned descriptor exhaustion into a corrupted database (#2883).
  10898. from backend.app.core.fd_limit import raise_open_file_limit
  10899. raise_open_file_limit()
  10900. # Before init_db, so the warning is near the top of the log rather than
  10901. # below a migration run. See warn_if_running_on_uvloop for what is at stake.
  10902. warn_if_running_on_uvloop()
  10903. await init_db()
  10904. # Browser download tokens expire after five minutes. Remove abandoned
  10905. # prepared ZIPs at startup as well as before each new preparation so a
  10906. # quiet appliance cannot retain an unusable bundle indefinitely.
  10907. try:
  10908. from backend.app.services.printer_media import prune_stale_printer_file_bundles
  10909. await prune_stale_printer_file_bundles()
  10910. except Exception as exc:
  10911. logging.warning("Failed to prune stale printer download bundles: %s", exc)
  10912. # After migrations, so the is_env_managed column exists. Never raises --
  10913. # a bad BAMBUDDY_OIDC_* value is logged and skipped rather than blocking
  10914. # startup (see apply_env_oidc_provider).
  10915. from backend.app.core.oidc_env import apply_env_oidc_provider
  10916. async with async_session() as oidc_db:
  10917. await apply_env_oidc_provider(oidc_db)
  10918. # Close out batches that finished before `completed` was a reachable status
  10919. # (#342). Without this the Batches tab opens on every batch created since
  10920. # the feature shipped, all still marked active. Never blocks startup.
  10921. try:
  10922. from backend.app.services.print_batch import backfill_batch_statuses
  10923. async with async_session() as batch_db:
  10924. await backfill_batch_statuses(batch_db)
  10925. except Exception as exc:
  10926. logging.warning("[BATCH] Startup status backfill failed: %s", exc)
  10927. # Each plate's estimate of archives made before they were kept (#3375),
  10928. # read from their 3MFs in the background: the readers use the whole
  10929. # file's figures until then.
  10930. global _plate_estimates_task
  10931. _plate_estimates_task = asyncio.create_task(_fill_plate_estimates())
  10932. # The G-code fingerprint of archives made before it was kept (#1634), read
  10933. # from their files in the background: until then a slice sent again from
  10934. # the slicer archives anew, as before.
  10935. global _gcode_hashes_task
  10936. _gcode_hashes_task = asyncio.create_task(_fill_gcode_hashes())
  10937. # Register an app-scoped httpx client for Bambu Cloud services so
  10938. # per-request BambuCloudService instances reuse the same connection pool
  10939. # (important for routes like /cloud/filament-info that chain many
  10940. # get_setting_detail calls). The shared client stores no region/token
  10941. # state, so the per-request ownership pattern that fixed the region-bleed
  10942. # bug is preserved.
  10943. import httpx as _httpx
  10944. from backend.app.services.bambu_cloud import set_shared_http_client
  10945. from backend.app.services.model_providers.makerworld.service import (
  10946. set_shared_http_client as set_shared_makerworld_http_client,
  10947. )
  10948. from backend.app.services.orca_cloud import (
  10949. set_shared_http_client as set_shared_orca_http_client,
  10950. )
  10951. _shared_cloud_http_client = _httpx.AsyncClient(timeout=30.0)
  10952. set_shared_http_client(_shared_cloud_http_client)
  10953. # Reuse the same connection pool for MakerWorld — different host, same
  10954. # keep-alive pool saves a TLS handshake per request.
  10955. set_shared_makerworld_http_client(_shared_cloud_http_client)
  10956. # Same for Orca Cloud — without this the per-request OrcaCloudService()
  10957. # each spun up (and never closed) its own client, leaking sockets.
  10958. set_shared_orca_http_client(_shared_cloud_http_client)
  10959. # Fix queue items stuck with invalid "aborted" status (should be "cancelled").
  10960. # This can happen when a print was cancelled mid-print on versions before this fix.
  10961. try:
  10962. async with async_session() as db:
  10963. from backend.app.models.print_queue import PrintQueueItem
  10964. result = await db.execute(select(PrintQueueItem).where(PrintQueueItem.status == "aborted"))
  10965. aborted_items = result.scalars().all()
  10966. if aborted_items:
  10967. for item in aborted_items:
  10968. item.status = "cancelled"
  10969. await db.commit()
  10970. logging.info("Fixed %d queue item(s) with invalid 'aborted' status → 'cancelled'", len(aborted_items))
  10971. except Exception as e:
  10972. logging.warning("Failed to fix aborted queue items: %s", e)
  10973. # Restore debug logging state from previous session
  10974. await init_debug_logging()
  10975. # Set up printer manager callbacks
  10976. loop = asyncio.get_event_loop()
  10977. printer_manager.set_event_loop(loop)
  10978. printer_manager.set_status_change_callback(on_printer_status_change)
  10979. printer_manager.set_print_start_callback(on_print_start)
  10980. printer_manager.set_print_complete_callback(on_print_complete)
  10981. printer_manager.set_print_running_observed_callback(on_print_running_observed)
  10982. printer_manager.set_finish_photo_moment_callback(on_finish_photo_moment)
  10983. printer_manager.set_ams_change_callback(on_ams_change)
  10984. printer_manager.set_fts_inlet_change_callback(on_fts_inlet_change)
  10985. # Rehydrate persisted awaiting-plate-clear gate (#961) so prompts survive restarts
  10986. await printer_manager.load_awaiting_plate_clear_from_db()
  10987. # Before any printer connects, or its first report re-sends every fault
  10988. # still listed (#3486).
  10989. await _load_notified_hms_faults()
  10990. # WLED is entirely optional. Seed its in-memory per-printer configuration
  10991. # before MQTT connections begin emitting status callbacks.
  10992. async with async_session() as db:
  10993. await initialize_wled_printers(db)
  10994. # Layer change callback for external camera timelapse
  10995. async def on_layer_change(printer_id: int, layer_num: int):
  10996. """Capture timelapse frame on layer change + first layer notification."""
  10997. from backend.app.services.layer_timelapse import on_layer_change as tl_layer_change
  10998. await tl_layer_change(printer_id, layer_num)
  10999. # #1867: bank a recent in-print frame so the finish-photo path has a
  11000. # pre-End-G-code image to use instead of a live grab of a swapped plate.
  11001. # #2547 added `on_print_progress` as a second driver — this one alone
  11002. # stops firing once the final layer begins.
  11003. await _maybe_bank_inprint_frame(printer_id, layer_num)
  11004. # First layer complete notification (layer_num >= 2 means layer 1 is done).
  11005. # Gate on actual printing state — Bambu firmware ticks layer_num during
  11006. # the pre-print calibration sequence (homing / mesh-level / bed scan /
  11007. # nozzle clean), so a bare layer_num check can fire minutes before the
  11008. # first real extrusion. We require gcode_state == RUNNING and
  11009. # mc_print_sub_stage in (0 = "Printing", None) so calibration sub-stages
  11010. # (1, 9, 14, ...) are excluded. The window widens to [2, 10] because if
  11011. # the layer counter advanced past 2 during PREPARE, the next on_layer_change
  11012. # edge fires later; _first_layer_notified stays clear until we actually send
  11013. # so a deferred re-evaluation can win. See issue #1837.
  11014. if 2 <= layer_num <= 10 and not _first_layer_notified.get(printer_id, False):
  11015. client = printer_manager.get_client(printer_id)
  11016. state = client.state if client else None
  11017. if not state or state.state != "RUNNING":
  11018. return
  11019. if state.mc_print_sub_stage not in (None, 0):
  11020. return
  11021. _first_layer_notified[printer_id] = True
  11022. try:
  11023. async with async_session() as db:
  11024. from backend.app.models.printer import Printer
  11025. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  11026. printer = result.scalar_one_or_none()
  11027. if not printer:
  11028. return
  11029. printer_name = printer.name
  11030. filename = (state.subtask_name or state.gcode_file or "Unknown") if state else "Unknown"
  11031. total_layers = state.total_layers if state else 0
  11032. image_data = await _capture_snapshot_for_notification(
  11033. printer_id, printer, logging.getLogger(__name__)
  11034. )
  11035. await notification_service.on_first_layer_complete(
  11036. printer_id, printer_name, filename, total_layers, db, image_data=image_data
  11037. )
  11038. except Exception as e:
  11039. logging.getLogger(__name__).warning("First layer notification failed: %s", e)
  11040. printer_manager.set_layer_change_callback(on_layer_change)
  11041. async def on_print_progress(printer_id: int, percent: int):
  11042. """#2547: keep the in-print frame bank fresh through the final layer.
  11043. `on_layer_change` stops the moment the last layer starts, which on the
  11044. H2C capture that closed #2547 left the bank stale for the three minutes
  11045. that layer took. Progress is the only field that keeps advancing there,
  11046. and it freezes before the End G-code runs — so banking on it stays
  11047. inside the print and never sees a swapped plate.
  11048. """
  11049. client = printer_manager.get_client(printer_id)
  11050. state = client.state if client else None
  11051. await _maybe_bank_inprint_frame(printer_id, state.layer_num if state else 0)
  11052. printer_manager.set_print_progress_callback(on_print_progress)
  11053. # Event-driven bed cooldown: fires whenever bed_temper arrives via MQTT
  11054. async def on_bed_temp_update(printer_id: int, bed_temp: float):
  11055. waiter = _bed_cool_waiters.get(printer_id)
  11056. if not waiter:
  11057. return
  11058. threshold = waiter["threshold"]
  11059. if bed_temp > threshold:
  11060. return
  11061. # Bed is at or below threshold — fire notification and remove waiter
  11062. waiter_info = _bed_cool_waiters.pop(printer_id, None)
  11063. if not waiter_info:
  11064. return # Another callback already handled it
  11065. bed_cool_logger = logging.getLogger(__name__)
  11066. bed_cool_logger.info(
  11067. "[BED-COOL] Bed cooled to %.1f°C on printer %s (threshold: %.0f°C)",
  11068. bed_temp,
  11069. printer_id,
  11070. threshold,
  11071. )
  11072. try:
  11073. printer_info = printer_manager.get_printer(printer_id)
  11074. p_name = printer_info.name if printer_info else "Unknown"
  11075. async with async_session() as db:
  11076. await notification_service.on_bed_cooled(
  11077. printer_id=printer_id,
  11078. printer_name=p_name,
  11079. bed_temp=bed_temp,
  11080. threshold=threshold,
  11081. filename=waiter_info["filename"],
  11082. db=db,
  11083. )
  11084. except Exception as e:
  11085. bed_cool_logger.warning("[BED-COOL] Failed to send notification: %s", e)
  11086. printer_manager.set_bed_temp_update_callback(on_bed_temp_update)
  11087. async def on_drying_complete(printer_id: int, ams_id: int):
  11088. """Smart-plug auto-off-after-drying trigger (#1349).
  11089. Fires once per AMS unit when ``dry_time`` falls from >0 to 0. The
  11090. manager walks all plugs linked to this printer and turns off only
  11091. the ones with ``auto_off_after_drying`` enabled, after their
  11092. per-plug delay. Multiple AMS units finishing close together (e.g. a
  11093. dual-AMS dry that ends within the same MQTT push) call this once
  11094. per unit — the manager's ``_cancel_pending_off`` collapses
  11095. repeated scheduling on the same plug to one timer, so duplicate
  11096. fires are safe.
  11097. """
  11098. try:
  11099. async with async_session() as db:
  11100. await smart_plug_manager.on_drying_complete(printer_id, db)
  11101. except Exception as e:
  11102. logging.getLogger(__name__).warning(
  11103. "Failed to schedule auto-off-after-drying for printer %d (AMS %d): %s",
  11104. printer_id,
  11105. ams_id,
  11106. e,
  11107. )
  11108. printer_manager.set_drying_complete_callback(on_drying_complete)
  11109. async def on_drying_cycle_end(printer_id: int, cycle):
  11110. """Mark the spools in an AMS that just finished drying as dried (#2863)."""
  11111. from backend.app.services.spool_drying import record_drying_cycle
  11112. try:
  11113. async with async_session() as db:
  11114. await record_drying_cycle(db, printer_id, cycle)
  11115. except Exception as e:
  11116. logging.getLogger(__name__).warning(
  11117. "Failed to record drying for printer %d (AMS %d): %s",
  11118. printer_id,
  11119. cycle.ams_id,
  11120. e,
  11121. )
  11122. printer_manager.set_drying_cycle_end_callback(on_drying_cycle_end)
  11123. async def on_assignment_verified(printer_id: int, ams_id: int, tray_id: int, verified: bool, detail: dict):
  11124. """Surface the read-back result of a spool assignment to the UI (#2582).
  11125. The MQTT client confirms (or fails to confirm) that the tray telemetry
  11126. echoed back the filament id we pushed. We relay that as a websocket
  11127. event so the frontend can toast "loaded" / "assignment didn't take"
  11128. instead of the historic silent fire-and-forget, which made the
  11129. AMS→Studio hand-off feel random to users.
  11130. """
  11131. try:
  11132. from backend.app.services.spool_assignment_notifications import (
  11133. _slot_label_from_global_tray,
  11134. )
  11135. layout = printer_tray_layout(printer_id, printer_manager)
  11136. global_id = layout.tray_from_slot_key(ams_id, tray_id)
  11137. slot_label = _slot_label_from_global_tray(global_id, layout)
  11138. printer_info = printer_manager.get_printer(printer_id)
  11139. printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  11140. await ws_manager.broadcast(
  11141. {
  11142. "type": "spool_assignment_verified",
  11143. "printer_id": printer_id,
  11144. "printer_name": printer_name,
  11145. "ams_id": ams_id,
  11146. "tray_id": tray_id,
  11147. "slot": slot_label,
  11148. "verified": verified,
  11149. # Present on success: False means the filament setting landed
  11150. # but the K-profile (cali_idx) did not — the reporter's exact
  11151. # "loaded but no flow profile" symptom.
  11152. "kprofile_applied": detail.get("kprofile_applied", True),
  11153. }
  11154. )
  11155. except Exception as e:
  11156. logging.getLogger(__name__).warning(
  11157. "Failed to broadcast assignment verification for printer %d AMS%d-T%d: %s",
  11158. printer_id,
  11159. ams_id,
  11160. tray_id,
  11161. e,
  11162. )
  11163. printer_manager.set_assignment_verified_callback(on_assignment_verified)
  11164. async def on_tray_change(printer_id: int, tray_global: int, layer_num: int):
  11165. """Persist a mid-print tray change for completion-time attribution.
  11166. AMS filament backup switches trays without telling the slicer, so the
  11167. tray-change log is the only record of which spool fed which layers.
  11168. Keeping it only in memory meant a restart mid-print charged everything
  11169. to the tray that finished the job.
  11170. """
  11171. try:
  11172. from backend.app.services.usage_tracker import record_tray_change
  11173. async with async_session() as db:
  11174. await record_tray_change(db, printer_id, tray_global, layer_num)
  11175. except Exception as e:
  11176. logging.getLogger(__name__).warning(
  11177. "Failed to persist tray change for printer %d (tray=%d, layer=%d): %s",
  11178. printer_id,
  11179. tray_global,
  11180. layer_num,
  11181. e,
  11182. )
  11183. printer_manager.set_tray_change_callback(on_tray_change)
  11184. # Initialize MQTT relay from settings
  11185. async with async_session() as db:
  11186. from backend.app.api.routes.settings import get_setting
  11187. mqtt_settings = {
  11188. "mqtt_enabled": (await get_setting(db, "mqtt_enabled") or "false") == "true",
  11189. "mqtt_broker": await get_setting(db, "mqtt_broker") or "",
  11190. "mqtt_port": int(await get_setting(db, "mqtt_port") or "1883"),
  11191. "mqtt_username": await get_setting(db, "mqtt_username") or "",
  11192. "mqtt_password": await get_setting(db, "mqtt_password") or "",
  11193. "mqtt_topic_prefix": await get_setting(db, "mqtt_topic_prefix") or "bambuddy",
  11194. "mqtt_use_tls": (await get_setting(db, "mqtt_use_tls") or "false") == "true",
  11195. }
  11196. await mqtt_relay.configure(mqtt_settings)
  11197. # Restore MQTT smart plug subscriptions
  11198. if mqtt_settings.get("mqtt_enabled"):
  11199. from backend.app.models.smart_plug import SmartPlug
  11200. from backend.app.services.mqtt_smart_plug import subscribe_plug_to_mqtt
  11201. result = await db.execute(select(SmartPlug).where(SmartPlug.plug_type == "mqtt"))
  11202. mqtt_plugs = result.scalars().all()
  11203. restored = 0
  11204. for plug in mqtt_plugs:
  11205. if subscribe_plug_to_mqtt(mqtt_relay.smart_plug_service, plug):
  11206. restored += 1
  11207. if restored:
  11208. logging.info("Restored %s MQTT smart plug subscriptions", restored)
  11209. # Connect to all active printers
  11210. async with async_session() as db:
  11211. await init_printer_connections(db)
  11212. # Auto-connect to Spoolman if enabled
  11213. async with async_session() as db:
  11214. from backend.app.api.routes.settings import get_setting
  11215. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  11216. spoolman_url = await get_setting(db, "spoolman_url")
  11217. if spoolman_enabled and spoolman_enabled.lower() == "true" and spoolman_url:
  11218. try:
  11219. client = await init_spoolman_client(spoolman_url)
  11220. if await client.health_check():
  11221. logging.info("Auto-connected to Spoolman at %s", spoolman_url)
  11222. # Ensure the 'tag' extra field exists for RFID/UUID storage
  11223. field_ok = await client.ensure_tag_extra_field()
  11224. if not field_ok:
  11225. logging.error("Spoolman tag extra field registration failed — NFC tag links may not persist")
  11226. # Register the BambuStudio slicer-preset fields used by the
  11227. # spool-edit / assign flow. Spoolman rejects PATCHes with
  11228. # unknown extra keys, so these must exist before any update
  11229. # that touches them.
  11230. for field_name in ("bambu_slicer_filament", "bambu_slicer_filament_name"):
  11231. if not await client.ensure_extra_field(field_name):
  11232. logging.warning(
  11233. "Spoolman extra field %r registration failed — "
  11234. "spool slicer-preset edits will return 502",
  11235. field_name,
  11236. )
  11237. else:
  11238. logging.warning("Spoolman at %s is not reachable", spoolman_url)
  11239. except Exception as e:
  11240. logging.warning("Failed to auto-connect to Spoolman: %s", e)
  11241. # Start the print scheduler
  11242. spawn_background_task(print_scheduler.run(), name="print-scheduler")
  11243. # Start the smart plug scheduler for time-based on/off
  11244. smart_plug_manager.start_scheduler()
  11245. # Start the Home Assistant sensor poller (#1148)
  11246. ha_sensor_manager.start()
  11247. location_ha_sensor_manager.start()
  11248. # Resume any pending auto-offs that were interrupted by restart
  11249. await smart_plug_manager.resume_pending_auto_offs()
  11250. # Start the notification digest scheduler
  11251. notification_service.start_digest_scheduler()
  11252. notify_live_activities.start()
  11253. notify_widgets.start()
  11254. # Start the Telegram reaction pollers (#3046), one per bot token used by a
  11255. # provider in reactions/both mode; the notification routes resync them.
  11256. # Never fatal: a bad provider row or a DB hiccup here costs reactions
  11257. # until the next provider save, not the whole startup.
  11258. try:
  11259. await telegram_reaction_poller.start()
  11260. except Exception as e:
  11261. logging.warning("Telegram reaction poller did not start: %s", e)
  11262. # Start the GitHub backup scheduler
  11263. await github_backup_service.start_scheduler()
  11264. # Start the local backup scheduler
  11265. await local_backup_service.start_scheduler()
  11266. await obico_detection_service.start()
  11267. await octoeverywhere_detection_service.start()
  11268. # Start the library trash sweeper (#1008)
  11269. await library_trash_service.start_scheduler()
  11270. # Start the archive auto-purge sweeper (#1008 follow-up)
  11271. await archive_purge_service.start_scheduler()
  11272. # Start AMS history recording
  11273. start_ams_history_recording()
  11274. # Start printer sensor (nozzle / bed / chamber) history recording
  11275. start_printer_sensor_history_recording()
  11276. # Start printer runtime tracking
  11277. start_runtime_tracking()
  11278. # Start SpoolBuddy device watchdog
  11279. start_spoolbuddy_watchdog()
  11280. # Start camera stream orphan cleanup
  11281. start_camera_cleanup()
  11282. # Start the backstop for MQTT sessions paho has stopped recovering (#2732)
  11283. start_connection_watchdog()
  11284. # One-shot sweep for timelapse session directories orphaned by a crash
  11285. # or restart that happened mid-print (in-memory session tracking can't
  11286. # survive that, and nothing else reaps the leftover frames/output file)
  11287. try:
  11288. from backend.app.services.layer_timelapse import cleanup_orphaned_timelapse_sessions
  11289. removed = cleanup_orphaned_timelapse_sessions()
  11290. if removed:
  11291. logging.getLogger(__name__).info("Removed %d orphaned timelapse session artifact(s)", removed)
  11292. except Exception as e:
  11293. logging.getLogger(__name__).warning("Orphaned timelapse session cleanup failed: %s", e)
  11294. # Start expected-print TTL eviction (prevents memory leak when prints are
  11295. # registered but on_print_start never fires)
  11296. start_expected_prints_cleanup()
  11297. # L-2: Start periodic auth cleanup (stale TOTP + expired revoked JTIs)
  11298. start_auth_cleanup()
  11299. # Maintainer announcements: a signed feed fetched from GitHub every few hours.
  11300. from backend.app.services import announcements as announcements_service
  11301. announcements_service.start()
  11302. from backend.app.services.printer_media import start_printer_download_cleanup
  11303. start_printer_download_cleanup()
  11304. # Event-loop stall watchdog: dumps all thread stacks to stderr if the loop
  11305. # freezes (#1486 — silent "container hangs after adding a printer" reports).
  11306. from backend.app.services.loop_watchdog import start_loop_watchdog
  11307. start_loop_watchdog()
  11308. # Initialize virtual printer manager and sync from DB
  11309. from backend.app.services.virtual_printer import virtual_printer_manager
  11310. virtual_printer_manager.set_session_factory(async_session)
  11311. virtual_printer_manager.set_printer_manager(printer_manager)
  11312. try:
  11313. await virtual_printer_manager.sync_from_db()
  11314. logging.info("Virtual printer manager synced from database")
  11315. except Exception as e:
  11316. logging.warning("Failed to sync virtual printers: %s", e)
  11317. yield
  11318. # Shutdown
  11319. print_scheduler.stop()
  11320. smart_plug_manager.stop_scheduler()
  11321. ha_sensor_manager.stop()
  11322. location_ha_sensor_manager.stop()
  11323. notification_service.stop_digest_scheduler()
  11324. await notify_live_activities.close()
  11325. await notify_widgets.close()
  11326. await telegram_reaction_poller.aclose()
  11327. github_backup_service.stop_scheduler()
  11328. local_backup_service.stop_scheduler()
  11329. library_trash_service.stop_scheduler()
  11330. archive_purge_service.stop_scheduler()
  11331. obico_detection_service.stop()
  11332. octoeverywhere_detection_service.stop()
  11333. stop_ams_history_recording()
  11334. stop_printer_sensor_history_recording()
  11335. stop_runtime_tracking()
  11336. stop_spoolbuddy_watchdog()
  11337. stop_camera_cleanup()
  11338. stop_connection_watchdog()
  11339. from backend.app.services.loop_watchdog import stop_loop_watchdog
  11340. stop_loop_watchdog()
  11341. # Tear down all camera fan-out broadcasters (#1089) so subscribers exit
  11342. # cleanly rather than waiting on a queue that nothing will ever fill.
  11343. try:
  11344. from backend.app.services.camera_fanout import shutdown_all_broadcasters
  11345. await shutdown_all_broadcasters()
  11346. except Exception as e:
  11347. logging.warning("Failed to shut down camera broadcasters: %s", e)
  11348. stop_expected_prints_cleanup()
  11349. stop_auth_cleanup()
  11350. if _plate_estimates_task is not None and not _plate_estimates_task.done():
  11351. _plate_estimates_task.cancel()
  11352. if _gcode_hashes_task is not None and not _gcode_hashes_task.done():
  11353. _gcode_hashes_task.cancel()
  11354. from backend.app.services import announcements as announcements_service
  11355. announcements_service.stop()
  11356. from backend.app.services.printer_media import stop_printer_download_cleanup
  11357. await stop_printer_download_cleanup()
  11358. # Stops a timelapse editor's ffmpeg rather than leaving it to the OS.
  11359. from backend.app.services import timelapse_jobs
  11360. await timelapse_jobs.cancel_all()
  11361. printer_manager.disconnect_all()
  11362. slot_unlink_grace.reset()
  11363. await close_spoolman_client()
  11364. # Stop all virtual printer services
  11365. await virtual_printer_manager.stop_all()
  11366. await mqtt_smart_plug_service.disconnect(timeout=2)
  11367. await mqtt_relay.disconnect(timeout=2)
  11368. await wled_manager.shutdown()
  11369. # Drop the shared Bambu Cloud HTTP client we registered at startup.
  11370. set_shared_http_client(None)
  11371. set_shared_makerworld_http_client(None)
  11372. set_shared_orca_http_client(None)
  11373. await _shared_cloud_http_client.aclose()
  11374. # Checkpoint WAL (SQLite only) and close all database connections
  11375. from backend.app.core.db_dialect import is_sqlite
  11376. if is_sqlite():
  11377. try:
  11378. async with engine.begin() as conn:
  11379. await conn.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
  11380. logging.info("WAL checkpoint completed")
  11381. except Exception as e:
  11382. logging.warning("WAL checkpoint failed: %s", e)
  11383. await engine.dispose()
  11384. app = FastAPI(
  11385. title=app_settings.app_name,
  11386. description="Archive and manage Bambu Lab 3MF files",
  11387. version=APP_VERSION,
  11388. lifespan=lifespan,
  11389. )
  11390. # Added first so it sits innermost, next to the routes: a refused header's value
  11391. # (often a stored token) masked in every JSON body. See core.response_redaction.
  11392. app.add_middleware(RefusedHeaderResponseMask)
  11393. def _json_safe(value):
  11394. """The validation errors with every non-finite number written as text."""
  11395. if isinstance(value, float) and not math.isfinite(value):
  11396. return str(value)
  11397. if isinstance(value, dict):
  11398. return {key: _json_safe(item) for key, item in value.items()}
  11399. if isinstance(value, list | tuple):
  11400. return [_json_safe(item) for item in value]
  11401. return value
  11402. @app.exception_handler(RequestValidationError)
  11403. async def _validation_error(_request, exc: RequestValidationError) -> JSONResponse:
  11404. """FastAPI's own 422, except that it survives a non-finite number.
  11405. A body may carry ``Infinity`` or ``NaN`` (Python's JSON reader accepts
  11406. them), and the default handler echoes the input back in its errors,
  11407. which JSON cannot hold: the request ended in an error instead of a 422.
  11408. """
  11409. return JSONResponse(status_code=422, content={"detail": _json_safe(jsonable_encoder(exc.errors()))})
  11410. # =============================================================================
  11411. # Authentication Middleware - Secures ALL API routes by default
  11412. # =============================================================================
  11413. # Public routes that don't require authentication even when auth is enabled
  11414. PUBLIC_API_ROUTES = {
  11415. # Auth routes needed before/during login
  11416. "/api/v1/auth/status",
  11417. "/api/v1/auth/login",
  11418. "/api/v1/auth/setup", # Needed for initial setup and recovery
  11419. # Advanced auth status needed for login page
  11420. "/api/v1/auth/advanced-auth/status",
  11421. "/api/v1/auth/forgot-password", # Password reset for advanced auth
  11422. "/api/v1/auth/forgot-password/confirm", # Complete password reset with token (H-6)
  11423. # 2FA routes that are called BEFORE a JWT is issued (pre-auth flow)
  11424. "/api/v1/auth/2fa/verify", # Exchange pre_auth_token + 2FA code for JWT
  11425. "/api/v1/auth/2fa/email/send", # Send OTP email (pre_auth_token based)
  11426. # OIDC routes that must be reachable without a JWT
  11427. "/api/v1/auth/oidc/providers", # Public list of enabled providers
  11428. "/api/v1/auth/oidc/callback", # Redirect target from OIDC provider
  11429. "/api/v1/auth/oidc/exchange", # Exchange short-lived OIDC token for JWT
  11430. # Connected apps: the app's server swaps a code for the user's identity,
  11431. # authenticated by its client secret rather than a login token.
  11432. "/api/v1/connect/token",
  11433. # Version check for updates (no sensitive data)
  11434. "/api/v1/updates/version",
  11435. # Metrics endpoint handles its own prometheus_token authentication
  11436. "/api/v1/metrics",
  11437. # Appliance bootstrap (#1589 follow-up): the SPA's i18n setup polls
  11438. # this BEFORE a JWT is available to pick up the firstboot wizard's
  11439. # hostname / timezone / locale and the chrony NTP-gate state. The
  11440. # response contains user-set defaults and a public sync flag — no
  11441. # secrets. Without this entry the global auth middleware returns 401
  11442. # before the route handler runs, regardless of the route's own
  11443. # "no auth required" intent.
  11444. "/api/v1/system/appliance",
  11445. # Cam Wall kiosk feed (#2531): a TV or Pi in kiosk mode has no login, so it
  11446. # authenticates with a long-lived ``camwall``-scoped token in the query
  11447. # string — exactly like the camera streams two lists below, and for the same
  11448. # reason (no header to put a JWT in). "Public" here only means the middleware
  11449. # steps aside; the route still runs RequireCamWallTokenIfAuthEnabled, which
  11450. # rejects an absent, expired, revoked, or wrong-scoped token. In particular a
  11451. # plain ``camera_stream`` token does NOT open this door.
  11452. "/api/v1/camwall/printers",
  11453. # Overlay branding: the route enforces overlay-scoped token authentication.
  11454. "/api/v1/overlay-branding/logo",
  11455. }
  11456. # Route prefixes that are public (for routes with dynamic segments)
  11457. PUBLIC_API_PREFIXES = [
  11458. # WebSocket connections handle their own auth
  11459. "/api/v1/ws",
  11460. # OIDC authorize redirects — include provider_id in path
  11461. "/api/v1/auth/oidc/authorize/",
  11462. # One-tap outcome-verdict links from push notifications (#1898). Tapped on
  11463. # a phone with no session, so no header can carry a JWT — the single-use
  11464. # capability token in the path IS the credential (same reasoning as the
  11465. # /dl/ slicer downloads below). The route grants nothing beyond writing
  11466. # good/reject on the one archive the token was minted for.
  11467. "/api/v1/archives/confirm/",
  11468. ]
  11469. # Route patterns that are public (read-only display data)
  11470. # These are checked with "in path" - needed because browsers load images/videos
  11471. # via <img src> and <video src> which don't include Authorization headers
  11472. PUBLIC_API_PATTERNS = [
  11473. # Thumbnails
  11474. "/thumbnail", # /archives/{id}/thumbnail, /library/files/{id}/thumbnail
  11475. "/plate-thumbnail/", # /archives/{id}/plate-thumbnail/{plate_id}
  11476. # Images and media
  11477. "/photos/", # /archives/{id}/photos/{filename}
  11478. "/project-image/", # /archives/{id}/project-image/{path}
  11479. "/qrcode", # /archives/{id}/qrcode
  11480. "/timelapse", # /archives/{id}/timelapse (video)
  11481. "/cover", # /printers/{id}/cover
  11482. "/icon", # /external-links/{id}/icon
  11483. # Camera (streams loaded via <img> tag)
  11484. "/camera/stream", # /printers/{id}/camera/stream
  11485. "/camera/snapshot", # /printers/{id}/camera/snapshot
  11486. # Streaming-overlay status feed (#2613): OBS loads /overlay/{id} with no login
  11487. # and this backs it, authenticated by an ``overlay``-scoped token in the query
  11488. # string (same reasoning as the camera streams above — no header to carry a
  11489. # JWT). "Public" only means the middleware steps aside; the route still runs
  11490. # RequireOverlayTokenIfAuthEnabled, which rejects an absent, expired, revoked,
  11491. # or wrong-scoped token — a camwall or camera_stream token does NOT open it.
  11492. "/overlay-status", # /printers/{id}/overlay-status
  11493. # Slicer token-authenticated downloads — protocol handlers (bambustudioopen://,
  11494. # orcaslicer://) cannot send auth headers. These endpoints validate a short-lived
  11495. # download token in the URL path instead.
  11496. "/dl/", # /archives/{id}/dl/{token}/{filename}, /library/files/{id}/dl/{token}/{filename}
  11497. # Same family, but the segment is "source-dl" — which does NOT contain "/dl/",
  11498. # and these patterns match by substring. Without its own entry the middleware
  11499. # 401s the slicer's header-less request before the route's token check runs,
  11500. # so "Open source 3MF in slicer" failed whenever auth was enabled (#3029).
  11501. "/source-dl/", # /archives/{id}/source-dl/{token}/{filename}
  11502. # Obico ML API fetches JPEG frames by one-shot nonce (issue #172 follow-up).
  11503. # The nonce itself is the credential: 32-byte random, single-use, ~30s TTL.
  11504. "/obico/cached-frame/", # /obico/cached-frame/{nonce}
  11505. ]
  11506. _security_headers_logger = logging.getLogger("backend.app.main.security_headers")
  11507. def _parse_trusted_frame_origins() -> tuple[str, ...]:
  11508. """Parse TRUSTED_FRAME_ORIGINS env var into a validated allowlist (#1191).
  11509. Format: comma-separated list of ``scheme://host[:port]`` origins.
  11510. Used by ``security_headers_middleware`` to relax ``frame-ancestors`` for
  11511. trusted same-LAN deployments (e.g. Home Assistant Webpage panel embedding
  11512. Bambuddy from a different port). Defaults to empty — strict ``'none'``.
  11513. Invalid entries are dropped with a warning rather than failing startup, so
  11514. a typo in one origin doesn't take the whole deployment down.
  11515. """
  11516. raw = os.environ.get("TRUSTED_FRAME_ORIGINS", "").strip()
  11517. if not raw:
  11518. return ()
  11519. valid: list[str] = []
  11520. for item in raw.split(","):
  11521. candidate = item.strip()
  11522. if not candidate:
  11523. continue
  11524. try:
  11525. parsed = urlparse(candidate)
  11526. except ValueError as e:
  11527. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — %s", candidate, e)
  11528. continue
  11529. if parsed.scheme not in ("http", "https"):
  11530. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — must be http(s)", candidate)
  11531. continue
  11532. if not parsed.netloc:
  11533. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — missing host", candidate)
  11534. continue
  11535. if parsed.path and parsed.path != "/":
  11536. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — paths not allowed", candidate)
  11537. continue
  11538. if parsed.query or parsed.fragment:
  11539. _security_headers_logger.warning(
  11540. "TRUSTED_FRAME_ORIGINS: dropping %r — query/fragment not allowed", candidate
  11541. )
  11542. continue
  11543. if "*" in parsed.netloc:
  11544. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — wildcards not allowed", candidate)
  11545. continue
  11546. valid.append(f"{parsed.scheme}://{parsed.netloc}")
  11547. if valid:
  11548. _security_headers_logger.info("TRUSTED_FRAME_ORIGINS: %s", ", ".join(valid))
  11549. return tuple(valid)
  11550. _TRUSTED_FRAME_ORIGINS: tuple[str, ...] = _parse_trusted_frame_origins()
  11551. def _frame_ancestors(default_value: str) -> str:
  11552. """Compose the ``frame-ancestors`` CSP directive (#1191).
  11553. ``default_value`` is the strict directive used when the operator has not
  11554. configured ``TRUSTED_FRAME_ORIGINS`` — typically ``'none'`` (catch-all and
  11555. docs) or ``'self'`` (the streaming overlay, embedded same-origin by the
  11556. Settings URL builder's preview). When trusted origins
  11557. are configured, ``'self'`` is always included so same-origin embedding never
  11558. breaks even if an operator forgets to add their own origin to the list.
  11559. """
  11560. if _TRUSTED_FRAME_ORIGINS:
  11561. return "frame-ancestors 'self' " + " ".join(_TRUSTED_FRAME_ORIGINS) + ";"
  11562. return f"frame-ancestors {default_value};"
  11563. # The two Vite-emitted worker assets that compile WebAssembly (#2976). Both
  11564. # patterns are anchored on the exact emitted name so the relaxed policies
  11565. # below can never apply to any other asset.
  11566. # src/workers/stepPreview.worker.ts -> /assets/stepPreview.worker-<hash>.js
  11567. # pdfjs-dist/legacy/build/pdf.worker.min.mjs?worker&url
  11568. # -> /assets/pdf.worker.min-<hash>.js
  11569. # Vite also emits a one-line chunk under the second name that only exports the
  11570. # worker's URL. The page imports it as a module, and a module script is run
  11571. # under the importing document's policy, never its own response's, so it
  11572. # matching as well changes nothing.
  11573. _STEP_WORKER_ASSET_RE = re.compile(r"^/assets/stepPreview\.worker-[\w-]+\.js$")
  11574. _PDF_WORKER_ASSET_RE = re.compile(r"^/assets/pdf\.worker\.min-[\w-]+\.js$")
  11575. @app.middleware("http")
  11576. async def security_headers_middleware(request, call_next):
  11577. """Add standard HTTP security headers to every response."""
  11578. # Per-request nonce stamped into `script-src` (#1460). On its own this
  11579. # changes nothing for Bambuddy's own pages — index.html has no inline
  11580. # scripts since the SW registration moved to /sw-register.js. The reason
  11581. # it's here is Cloudflare: a CF-fronted deployment has the bot-detection
  11582. # script injected into the HTML on the edge, with a fresh hash on every
  11583. # load (so hashes can't be allowlisted). When CF sees a nonce in our CSP,
  11584. # it clones the same nonce onto its injected <script>, and the inline
  11585. # script passes the policy without us needing 'unsafe-inline'. See
  11586. # https://developers.cloudflare.com/cloudflare-challenges/challenge-types/javascript-detections/#if-you-have-a-content-security-policy-csp
  11587. csp_nonce = secrets.token_urlsafe(16)
  11588. # Routes that render their own HTML need it too, or their inline script is
  11589. # blocked by the policy below. The outcome-confirmation page (#1898) is the
  11590. # one that does: it submits its own form so a verdict still costs one tap.
  11591. request.state.csp_nonce = csp_nonce
  11592. response = await call_next(request)
  11593. response.headers["X-Content-Type-Options"] = "nosniff"
  11594. # X-Frame-Options is the legacy cross-origin embedding control. Modern
  11595. # browsers honour CSP frame-ancestors instead, and the legacy
  11596. # `ALLOW-FROM <url>` syntax is deprecated and inconsistent across vendors.
  11597. # When operators have explicitly allowlisted trusted frame origins (#1191
  11598. # — typically Home Assistant on a different port), drop X-Frame-Options
  11599. # and let the CSP-side frame-ancestors directive govern embedding.
  11600. if not _TRUSTED_FRAME_ORIGINS:
  11601. response.headers["X-Frame-Options"] = "SAMEORIGIN"
  11602. response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
  11603. # Content-Security-Policy for the React SPA.
  11604. # Notes:
  11605. # - 'unsafe-inline' for style-src: React and UI libs inject inline styles at runtime.
  11606. # - connect-src ws:/wss:: MQTT/printer WebSocket connections.
  11607. # - img-src data: / blob:: base64 thumbnails and Blob-URL timelapse previews.
  11608. # - media-src blob:: timelapse video player uses Blob URLs.
  11609. # - font-src data:: some icon fonts are embedded as data URIs.
  11610. if request.url.path in ("/docs", "/redoc", "/docs/oauth2-redirect"):
  11611. # FastAPI's built-in Swagger UI / ReDoc pages load assets from
  11612. # cdn.jsdelivr.net and bootstrap with an inline <script>, so the
  11613. # default CSP would render a blank page.
  11614. response.headers["Content-Security-Policy"] = (
  11615. "default-src 'self'; "
  11616. "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
  11617. "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com; "
  11618. "img-src 'self' data: blob: https://fastapi.tiangolo.com https://cdn.redoc.ly; "
  11619. "connect-src 'self'; "
  11620. "font-src 'self' data: https://fonts.gstatic.com; "
  11621. "worker-src 'self' blob:; "
  11622. "object-src 'none'; "
  11623. "base-uri 'self'; " + _frame_ancestors("'none'")
  11624. )
  11625. elif _STEP_WORKER_ASSET_RE.match(request.url.path):
  11626. # The STEP preview worker (#2976) runs OpenCascade compiled to WASM;
  11627. # its emscripten/embind glue generates invoker functions with `new
  11628. # Function(...)`, which needs 'unsafe-eval'. Per CSP3 a dedicated
  11629. # worker is governed by the policy delivered with the WORKER SCRIPT's
  11630. # own response — not the document's — so relaxing it here confines
  11631. # eval to that DOM-less worker context. The document policy below
  11632. # stays nonce-strict, and this response header has no effect when the
  11633. # file is merely fetched (a fetch's CSP is enforced against the
  11634. # requesting document, not the resource's own headers).
  11635. response.headers["Content-Security-Policy"] = (
  11636. "default-src 'self'; "
  11637. "script-src 'self' 'wasm-unsafe-eval' 'unsafe-eval'; "
  11638. "connect-src 'self'; "
  11639. "object-src 'none'; "
  11640. "base-uri 'self'; " + _frame_ancestors("'none'")
  11641. )
  11642. elif _PDF_WORKER_ASSET_RE.match(request.url.path):
  11643. # pdf.js decodes JPEG2000/JBIG2 images and ICC colour with WebAssembly
  11644. # and fetches those modules from /assets/pdfjs/wasm/ (#2976). Same CSP3
  11645. # rule as the STEP worker above: the policy that governs a dedicated
  11646. # worker is the one delivered with its own script, so the wasm compile
  11647. # has to be permitted here rather than on the document. Unlike the STEP
  11648. # worker this one needs no JS eval, so it gets 'wasm-unsafe-eval' only.
  11649. response.headers["Content-Security-Policy"] = (
  11650. "default-src 'self'; "
  11651. "script-src 'self' 'wasm-unsafe-eval'; "
  11652. "connect-src 'self'; "
  11653. "object-src 'none'; "
  11654. "base-uri 'self'; " + _frame_ancestors("'none'")
  11655. )
  11656. else:
  11657. # The streaming overlay is embedded same-origin by the URL builder's
  11658. # preview in Settings (#1422), so this branch allows 'self'.
  11659. # Embedding from anywhere else is still refused: 'self'
  11660. # only permits a framer on this origin, which is Bambuddy's own UI, so
  11661. # a clickjacking page on another host is blocked exactly as before.
  11662. # (The overlay draws status over a camera feed and its only interactive
  11663. # element is the logo link, so there is nothing to bait a click into
  11664. # even from a same-origin framer.) Cross-origin embedding of the
  11665. # overlay — Home Assistant on another port — remains what
  11666. # TRUSTED_FRAME_ORIGINS is for, and _frame_ancestors already folds that
  11667. # allowlist in.
  11668. #
  11669. # The connected-app consent page (/connect/authorize) gets the same
  11670. # 'self': an app opened from Bambuddy's sidebar runs in an iframe, and
  11671. # its "Sign in with Bambuddy" navigates that iframe to this page. With
  11672. # 'none' the browser refuses to show it even inside Bambuddy. 'self'
  11673. # requires every ancestor to be this origin, so a foreign page -- a
  11674. # sidebar link's site included -- still cannot frame the consent
  11675. # screen to bait a click.
  11676. embeddable_same_origin = request.url.path.startswith("/overlay/") or request.url.path == "/connect/authorize"
  11677. # No 'wasm-unsafe-eval' here: nothing compiles WebAssembly on the main
  11678. # thread. Both wasm consumers — the STEP preview and pdf.js's image
  11679. # decoders (#2976) — run in dedicated workers, which CSP3 governs by
  11680. # the policy served with their own script, so each gets it in its own
  11681. # branch above and the document policy stays as strict as it was.
  11682. response.headers["Content-Security-Policy"] = (
  11683. "default-src 'self'; "
  11684. f"script-src 'self' 'nonce-{csp_nonce}'; "
  11685. "style-src 'self' 'unsafe-inline'; "
  11686. "img-src 'self' data: blob:; "
  11687. "media-src 'self' blob:; "
  11688. "connect-src 'self' ws: wss:; "
  11689. "font-src 'self' data:; "
  11690. "object-src 'none'; "
  11691. "base-uri 'self'; "
  11692. "frame-src 'self' http: https:; " + _frame_ancestors("'self'" if embeddable_same_origin else "'none'")
  11693. )
  11694. if request.url.scheme == "https":
  11695. response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
  11696. return response
  11697. @app.middleware("http")
  11698. async def auth_middleware(request, call_next):
  11699. """Enforce authentication on all API routes when auth is enabled.
  11700. This middleware provides defense-in-depth by checking auth at the API gateway level,
  11701. regardless of whether individual routes have auth dependencies.
  11702. """
  11703. from starlette.responses import JSONResponse
  11704. path = request.url.path
  11705. # Only apply to API routes
  11706. if not path.startswith("/api/"):
  11707. return await call_next(request)
  11708. # Allow public routes
  11709. if path in PUBLIC_API_ROUTES:
  11710. return await call_next(request)
  11711. # Allow public prefixes
  11712. for prefix in PUBLIC_API_PREFIXES:
  11713. if path.startswith(prefix):
  11714. return await call_next(request)
  11715. # Allow public patterns (read-only display data like thumbnails)
  11716. for pattern in PUBLIC_API_PATTERNS:
  11717. if pattern in path:
  11718. return await call_next(request)
  11719. # Check if auth is enabled. Fail CLOSED on any exception during the
  11720. # probe — GHSA-6mf4-q26m-47pv: the previous fail-open path here let
  11721. # an attacker who could force a DB exception (e.g. file-descriptor
  11722. # exhaustion via login flood) bypass auth on every protected endpoint.
  11723. # Only the probe is guarded: the request itself runs outside the try, so an
  11724. # error in a route surfaces as the 500 it is rather than as an auth outage.
  11725. try:
  11726. async with async_session() as db:
  11727. from backend.app.core.auth import is_auth_enabled
  11728. auth_enabled = await is_auth_enabled(db)
  11729. except Exception:
  11730. logging.getLogger(__name__).exception("auth_middleware: failing closed on auth-probe error from %s", path)
  11731. return JSONResponse(
  11732. status_code=503,
  11733. content={"detail": "Authentication service temporarily unavailable"},
  11734. )
  11735. if not auth_enabled:
  11736. # Auth disabled, allow all requests
  11737. return await call_next(request)
  11738. # Auth is enabled - require valid token
  11739. auth_header = request.headers.get("Authorization")
  11740. x_api_key = request.headers.get("X-API-Key")
  11741. # Check for API key auth first
  11742. if x_api_key or (auth_header and auth_header.startswith("Bearer bb_")):
  11743. # API key authentication - let the request through to be validated by route handler
  11744. # API keys are validated per-route since they have different permission levels
  11745. return await call_next(request)
  11746. # Check for JWT auth
  11747. if not auth_header or not auth_header.startswith("Bearer "):
  11748. return JSONResponse(
  11749. status_code=401,
  11750. content={"detail": "Authentication required"},
  11751. headers={"WWW-Authenticate": "Bearer"},
  11752. )
  11753. # Validate JWT token
  11754. import jwt
  11755. try:
  11756. from backend.app.core.auth import (
  11757. ALGORITHM,
  11758. SECRET_KEY,
  11759. _is_token_fresh,
  11760. get_user_by_username,
  11761. is_jti_revoked,
  11762. )
  11763. token = auth_header.replace("Bearer ", "")
  11764. payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
  11765. username = payload.get("sub")
  11766. if not username:
  11767. raise ValueError("No username in token")
  11768. jti = payload.get("jti")
  11769. if not jti:
  11770. raise ValueError("No jti in token")
  11771. iat = payload.get("iat")
  11772. # Verify user exists, is active, and token is still fresh (L-R8-A).
  11773. # Reject revoked tokens first (defense-in-depth gateway check), reusing
  11774. # this session so the gateway adds a single pooled checkout, not two (#2572).
  11775. async with async_session() as db:
  11776. if await is_jti_revoked(jti, db):
  11777. return JSONResponse(
  11778. status_code=401,
  11779. content={"detail": "Token has been revoked"},
  11780. headers={"WWW-Authenticate": "Bearer"},
  11781. )
  11782. user = await get_user_by_username(db, username)
  11783. if not user or not user.is_active:
  11784. return JSONResponse(
  11785. status_code=401,
  11786. content={"detail": "User not found or inactive"},
  11787. headers={"WWW-Authenticate": "Bearer"},
  11788. )
  11789. if not _is_token_fresh(iat, user):
  11790. return JSONResponse(
  11791. status_code=401,
  11792. content={"detail": "Token no longer valid"},
  11793. headers={"WWW-Authenticate": "Bearer"},
  11794. )
  11795. except jwt.ExpiredSignatureError:
  11796. return JSONResponse(
  11797. status_code=401,
  11798. content={"detail": "Token has expired"},
  11799. headers={"WWW-Authenticate": "Bearer"},
  11800. )
  11801. except (jwt.InvalidTokenError, ValueError):
  11802. return JSONResponse(
  11803. status_code=401,
  11804. content={"detail": "Invalid token"},
  11805. headers={"WWW-Authenticate": "Bearer"},
  11806. )
  11807. except Exception:
  11808. # Not a bad token but a failure checking it (database, pool): refuse,
  11809. # as for the probe above, rather than answer 401 -- the frontend reads
  11810. # a 401 from /auth/me as final and signs the user out.
  11811. logging.getLogger(__name__).exception("auth_middleware: failing closed on token-check error from %s", path)
  11812. return JSONResponse(
  11813. status_code=503,
  11814. content={"detail": "Authentication service temporarily unavailable"},
  11815. )
  11816. return await call_next(request)
  11817. @app.middleware("http")
  11818. async def trace_id_middleware(request, call_next):
  11819. """Stamp every HTTP request with a trace ID and echo it back.
  11820. Decorated AFTER auth_middleware on purpose: Starlette stacks
  11821. @app.middleware decorators LIFO, so the last-decorated runs first
  11822. inbound. Putting the trace stamp last makes it the OUTERMOST layer,
  11823. which means auth-middleware log lines (and every line emitted on the
  11824. way down to and back from the route handler) all carry the same
  11825. trace ID. If we put it before auth, auth's logs would be stamped
  11826. with the *previous* request's ID — useless for correlation.
  11827. Honours an inbound ``X-Trace-Id`` header so callers running their
  11828. own tracing can correlate their span IDs with our log lines, but
  11829. only if the value passes the whitelist gate in
  11830. ``backend.app.core.trace.normalise_inbound_trace_id`` — anything
  11831. rejected (too long, contains control chars, etc.) silently triggers
  11832. a freshly minted server-side ID rather than failing the request.
  11833. The minted (or echoed) ID is set on a ContextVar so that every log
  11834. record emitted during the request — application logs *and* uvicorn's
  11835. access log — carries it via TraceIDFilter, and is also written to
  11836. the ``X-Trace-Id`` response header so clients can pin a server-side
  11837. log search to the exact request they made.
  11838. """
  11839. from backend.app.core.trace import (
  11840. generate_trace_id,
  11841. normalise_inbound_trace_id,
  11842. trace_id_var,
  11843. )
  11844. inbound = normalise_inbound_trace_id(request.headers.get("X-Trace-Id"))
  11845. trace_id = inbound if inbound is not None else generate_trace_id()
  11846. token = trace_id_var.set(trace_id)
  11847. try:
  11848. response = await call_next(request)
  11849. finally:
  11850. # Reset the ContextVar so a record emitted in a totally
  11851. # unrelated background task that just happens to inherit this
  11852. # context doesn't keep referencing this request's ID forever.
  11853. # In practice ContextVar.reset is best-effort under asyncio
  11854. # task-spawn semantics, but the cost is one attribute write so
  11855. # we may as well do it.
  11856. trace_id_var.reset(token)
  11857. response.headers["X-Trace-Id"] = trace_id
  11858. return response
  11859. # API routes
  11860. app.include_router(auth.router, prefix=app_settings.api_prefix)
  11861. app.include_router(mfa.router, prefix=app_settings.api_prefix)
  11862. app.include_router(bug_report.router, prefix=app_settings.api_prefix)
  11863. app.include_router(users.router, prefix=app_settings.api_prefix)
  11864. app.include_router(groups.router, prefix=app_settings.api_prefix)
  11865. app.include_router(printers.router, prefix=app_settings.api_prefix)
  11866. app.include_router(printer_locations.router, prefix=app_settings.api_prefix)
  11867. app.include_router(archives.router, prefix=app_settings.api_prefix)
  11868. app.include_router(filaments.router, prefix=app_settings.api_prefix)
  11869. app.include_router(finance.router, prefix=app_settings.api_prefix)
  11870. app.include_router(inventory.router, prefix=app_settings.api_prefix)
  11871. app.include_router(labels.router, prefix=app_settings.api_prefix)
  11872. app.include_router(label_templates.router, prefix=app_settings.api_prefix)
  11873. app.include_router(settings_routes.router, prefix=app_settings.api_prefix)
  11874. app.include_router(overlay_branding.router, prefix=app_settings.api_prefix)
  11875. app.include_router(cloud.router, prefix=app_settings.api_prefix)
  11876. app.include_router(orca_cloud.router, prefix=app_settings.api_prefix)
  11877. app.include_router(local_presets.router, prefix=app_settings.api_prefix)
  11878. app.include_router(smart_plugs.router, prefix=app_settings.api_prefix)
  11879. app.include_router(ha_sensors.router, prefix=app_settings.api_prefix)
  11880. app.include_router(location_ha_sensors.router, prefix=app_settings.api_prefix)
  11881. app.include_router(print_log.router, prefix=app_settings.api_prefix)
  11882. app.include_router(print_queue.router, prefix=app_settings.api_prefix)
  11883. app.include_router(scheduled_dryings.router, prefix=app_settings.api_prefix)
  11884. app.include_router(kprofiles.router, prefix=app_settings.api_prefix)
  11885. app.include_router(notifications.router, prefix=app_settings.api_prefix)
  11886. app.include_router(notification_templates.router, prefix=app_settings.api_prefix)
  11887. app.include_router(user_notifications.router, prefix=app_settings.api_prefix)
  11888. app.include_router(spoolman.router, prefix=app_settings.api_prefix)
  11889. app.include_router(spoolman_inventory.router, prefix=app_settings.api_prefix)
  11890. app.include_router(updates.router, prefix=app_settings.api_prefix)
  11891. app.include_router(sponsor_prompt.router, prefix=app_settings.api_prefix)
  11892. app.include_router(announcements.router, prefix=app_settings.api_prefix)
  11893. app.include_router(maintenance.router, prefix=app_settings.api_prefix)
  11894. app.include_router(camera.router, prefix=app_settings.api_prefix)
  11895. app.include_router(camwall.router, prefix=app_settings.api_prefix)
  11896. app.include_router(external_links.router, prefix=app_settings.api_prefix)
  11897. app.include_router(projects.router, prefix=app_settings.api_prefix)
  11898. app.include_router(library.router, prefix=app_settings.api_prefix)
  11899. app.include_router(library_tags.router, prefix=app_settings.api_prefix)
  11900. app.include_router(library_trash.router, prefix=app_settings.api_prefix)
  11901. app.include_router(library_variants.router, prefix=app_settings.api_prefix)
  11902. app.include_router(slice_jobs.router, prefix=app_settings.api_prefix)
  11903. app.include_router(slicer_pipelines.router, prefix=app_settings.api_prefix)
  11904. app.include_router(pipeline_runs.pipeline_run_create_router, prefix=app_settings.api_prefix)
  11905. app.include_router(pipeline_runs.pipeline_run_router, prefix=app_settings.api_prefix)
  11906. app.include_router(slicer_presets.router, prefix=app_settings.api_prefix)
  11907. app.include_router(archive_purge.router, prefix=app_settings.api_prefix)
  11908. app.include_router(makerworld.router, prefix=app_settings.api_prefix)
  11909. app.include_router(manyfold.router, prefix=app_settings.api_prefix)
  11910. app.include_router(api_keys.router, prefix=app_settings.api_prefix)
  11911. app.include_router(connected_apps.router, prefix=app_settings.api_prefix)
  11912. app.include_router(webhook.router, prefix=app_settings.api_prefix)
  11913. app.include_router(ams_history.router, prefix=app_settings.api_prefix)
  11914. app.include_router(printer_sensor_history.router, prefix=app_settings.api_prefix)
  11915. app.include_router(system.router, prefix=app_settings.api_prefix)
  11916. app.include_router(support.router, prefix=app_settings.api_prefix)
  11917. app.include_router(websocket.router, prefix=app_settings.api_prefix)
  11918. app.include_router(discovery.router, prefix=app_settings.api_prefix)
  11919. app.include_router(pending_uploads.router, prefix=app_settings.api_prefix)
  11920. app.include_router(firmware.router, prefix=app_settings.api_prefix)
  11921. app.include_router(bulk.router, prefix=app_settings.api_prefix)
  11922. app.include_router(github_backup.router, prefix=app_settings.api_prefix)
  11923. app.include_router(local_backup.router, prefix=app_settings.api_prefix)
  11924. app.include_router(obico.router, prefix=app_settings.api_prefix)
  11925. app.include_router(octoeverywhere.router, prefix=app_settings.api_prefix)
  11926. app.include_router(metrics.router, prefix=app_settings.api_prefix)
  11927. app.include_router(virtual_printers.router, prefix=app_settings.api_prefix)
  11928. app.include_router(spoolbuddy.router, prefix=app_settings.api_prefix)
  11929. # Serve static files (React build)
  11930. if app_settings.static_dir.exists() and any(app_settings.static_dir.iterdir()):
  11931. # Compressed, and cached for good where Vite hashed the name (#3175).
  11932. app.mount(
  11933. "/assets",
  11934. AssetStaticFiles(directory=app_settings.static_dir / "assets"),
  11935. name="assets",
  11936. )
  11937. if (app_settings.static_dir / "img").exists():
  11938. app.mount(
  11939. "/img",
  11940. StaticFiles(directory=app_settings.static_dir / "img"),
  11941. name="img",
  11942. )
  11943. if (app_settings.static_dir / "icons").exists():
  11944. app.mount(
  11945. "/icons",
  11946. StaticFiles(directory=app_settings.static_dir / "icons"),
  11947. name="icons",
  11948. )
  11949. # Self-hosted Inter woff2 files (#1460). Without this mount /fonts/*.woff2
  11950. # falls through to the SPA catch-all and returns index.html, which the
  11951. # browser's font sanitizer rejects ("downloadable font: rejected by
  11952. # sanitizer").
  11953. if (app_settings.static_dir / "fonts").exists():
  11954. app.mount(
  11955. "/fonts",
  11956. StaticFiles(directory=app_settings.static_dir / "fonts"),
  11957. name="fonts",
  11958. )
  11959. @app.get("/")
  11960. async def serve_frontend():
  11961. """Serve the React frontend."""
  11962. index_file = app_settings.static_dir / "index.html"
  11963. if index_file.exists():
  11964. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  11965. return {
  11966. "message": "Bambuddy API",
  11967. "docs": "/docs",
  11968. "frontend": "Build and place React app in /static directory",
  11969. }
  11970. # index.html must always be revalidated — Vite emits content-hashed JS/CSS
  11971. # bundles (e.g. `index-JRaF_JhW.js`), so the JS itself is safe to cache
  11972. # forever, but the HTML wrapping it is the only file that knows which hash
  11973. # is current. Without explicit cache-control headers Chromium decides
  11974. # heuristically (typically 10% of the time since Last-Modified) and on
  11975. # long-running kiosks happily serves stale HTML across browser restarts.
  11976. # That stale HTML references an old bundle hash, the old bundle is also
  11977. # in the disk cache, and the user ends up running pre-update JS forever
  11978. # without ever knowing why. ``no-cache`` (revalidate every time, but a
  11979. # 304 is cheap) is the correct setting for an SPA's entry HTML.
  11980. _HTML_CACHE_HEADERS = {"Cache-Control": "no-cache, must-revalidate"}
  11981. @app.get("/health")
  11982. async def health_check():
  11983. """Health check endpoint."""
  11984. return {"status": "healthy"}
  11985. # GET + HEAD on the three PWA bootstrap routes (#1460). Scanners and a plain
  11986. # `curl -I` use HEAD; FastAPI's @app.get only registers GET, so HEAD answers
  11987. # with 405 Method Not Allowed and shows up as a "broken manifest" red herring
  11988. # in deployment debugging. One decorator per method, not api_route(methods=
  11989. # [...]): api_route gives both methods the same OpenAPI operation id.
  11990. @app.get("/manifest.json")
  11991. @app.head("/manifest.json")
  11992. async def serve_manifest():
  11993. """Serve PWA manifest."""
  11994. manifest_file = app_settings.static_dir / "manifest.json"
  11995. if manifest_file.exists():
  11996. return FileResponse(manifest_file, media_type="application/manifest+json")
  11997. return {"error": "Manifest not found"}
  11998. @app.get("/sw.js")
  11999. @app.head("/sw.js")
  12000. async def serve_service_worker():
  12001. """Serve service worker."""
  12002. sw_file = app_settings.static_dir / "sw.js"
  12003. if sw_file.exists():
  12004. return FileResponse(
  12005. sw_file,
  12006. media_type="application/javascript",
  12007. headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
  12008. )
  12009. return {"error": "Service worker not found"}
  12010. @app.get("/sw-register.js")
  12011. @app.head("/sw-register.js")
  12012. async def serve_sw_register():
  12013. """Serve the service-worker registration bootstrap script.
  12014. Served as a real JS file so the strict `script-src 'self'` CSP covers it
  12015. without needing 'unsafe-inline' or per-build hashes on the inline tag.
  12016. """
  12017. reg_file = app_settings.static_dir / "sw-register.js"
  12018. if reg_file.exists():
  12019. return FileResponse(reg_file, media_type="application/javascript")
  12020. return {"error": "sw-register.js not found"}
  12021. # ── GCode viewer static files ────────────────────────────────────────────────
  12022. # Catch-all route for React Router (must be last)
  12023. @app.get("/{full_path:path}")
  12024. async def serve_spa(full_path: str):
  12025. """Serve React app for client-side routing."""
  12026. # Don't intercept API routes - raise proper 404 so FastAPI can handle redirects
  12027. if full_path.startswith("api/"):
  12028. from fastapi import HTTPException
  12029. raise HTTPException(status_code=404, detail="Not found")
  12030. index_file = app_settings.static_dir / "index.html"
  12031. if index_file.exists():
  12032. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  12033. return {"error": "Frontend not built"}