main.py 655 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246924792489249925092519252925392549255925692579258925992609261926292639264926592669267926892699270927192729273927492759276927792789279928092819282928392849285928692879288928992909291929292939294929592969297929892999300930193029303930493059306930793089309931093119312931393149315931693179318931993209321932293239324932593269327932893299330933193329333933493359336933793389339934093419342934393449345934693479348934993509351935293539354935593569357935893599360936193629363936493659366936793689369937093719372937393749375937693779378937993809381938293839384938593869387938893899390939193929393939493959396939793989399940094019402940394049405940694079408940994109411941294139414941594169417941894199420942194229423942494259426942794289429943094319432943394349435943694379438943994409441944294439444944594469447944894499450945194529453945494559456945794589459946094619462946394649465946694679468946994709471947294739474947594769477947894799480948194829483948494859486948794889489949094919492949394949495949694979498949995009501950295039504950595069507950895099510951195129513951495159516951795189519952095219522952395249525952695279528952995309531953295339534953595369537953895399540954195429543954495459546954795489549955095519552955395549555955695579558955995609561956295639564956595669567956895699570957195729573957495759576957795789579958095819582958395849585958695879588958995909591959295939594959595969597959895999600960196029603960496059606960796089609961096119612961396149615961696179618961996209621962296239624962596269627962896299630963196329633963496359636963796389639964096419642964396449645964696479648964996509651965296539654965596569657965896599660966196629663966496659666966796689669967096719672967396749675967696779678967996809681968296839684968596869687968896899690969196929693969496959696969796989699970097019702970397049705970697079708970997109711971297139714971597169717971897199720972197229723972497259726972797289729973097319732973397349735973697379738973997409741974297439744974597469747974897499750975197529753975497559756975797589759976097619762976397649765976697679768976997709771977297739774977597769777977897799780978197829783978497859786978797889789979097919792979397949795979697979798979998009801980298039804980598069807980898099810981198129813981498159816981798189819982098219822982398249825982698279828982998309831983298339834983598369837983898399840984198429843984498459846984798489849985098519852985398549855985698579858985998609861986298639864986598669867986898699870987198729873987498759876987798789879988098819882988398849885988698879888988998909891989298939894989598969897989898999900990199029903990499059906990799089909991099119912991399149915991699179918991999209921992299239924992599269927992899299930993199329933993499359936993799389939994099419942994399449945994699479948994999509951995299539954995599569957995899599960996199629963996499659966996799689969997099719972997399749975997699779978997999809981998299839984998599869987998899899990999199929993999499959996999799989999100001000110002100031000410005100061000710008100091001010011100121001310014100151001610017100181001910020100211002210023100241002510026100271002810029100301003110032100331003410035100361003710038100391004010041100421004310044100451004610047100481004910050100511005210053100541005510056100571005810059100601006110062100631006410065100661006710068100691007010071100721007310074100751007610077100781007910080100811008210083100841008510086100871008810089100901009110092100931009410095100961009710098100991010010101101021010310104101051010610107101081010910110101111011210113101141011510116101171011810119101201012110122101231012410125101261012710128101291013010131101321013310134101351013610137101381013910140101411014210143101441014510146101471014810149101501015110152101531015410155101561015710158101591016010161101621016310164101651016610167101681016910170101711017210173101741017510176101771017810179101801018110182101831018410185101861018710188101891019010191101921019310194101951019610197101981019910200102011020210203102041020510206102071020810209102101021110212102131021410215102161021710218102191022010221102221022310224102251022610227102281022910230102311023210233102341023510236102371023810239102401024110242102431024410245102461024710248102491025010251102521025310254102551025610257102581025910260102611026210263102641026510266102671026810269102701027110272102731027410275102761027710278102791028010281102821028310284102851028610287102881028910290102911029210293102941029510296102971029810299103001030110302103031030410305103061030710308103091031010311103121031310314103151031610317103181031910320103211032210323103241032510326103271032810329103301033110332103331033410335103361033710338103391034010341103421034310344103451034610347103481034910350103511035210353103541035510356103571035810359103601036110362103631036410365103661036710368103691037010371103721037310374103751037610377103781037910380103811038210383103841038510386103871038810389103901039110392103931039410395103961039710398103991040010401104021040310404104051040610407104081040910410104111041210413104141041510416104171041810419104201042110422104231042410425104261042710428104291043010431104321043310434104351043610437104381043910440104411044210443104441044510446104471044810449104501045110452104531045410455104561045710458104591046010461104621046310464104651046610467104681046910470104711047210473104741047510476104771047810479104801048110482104831048410485104861048710488104891049010491104921049310494104951049610497104981049910500105011050210503105041050510506105071050810509105101051110512105131051410515105161051710518105191052010521105221052310524105251052610527105281052910530105311053210533105341053510536105371053810539105401054110542105431054410545105461054710548105491055010551105521055310554105551055610557105581055910560105611056210563105641056510566105671056810569105701057110572105731057410575105761057710578105791058010581105821058310584105851058610587105881058910590105911059210593105941059510596105971059810599106001060110602106031060410605106061060710608106091061010611106121061310614106151061610617106181061910620106211062210623106241062510626106271062810629106301063110632106331063410635106361063710638106391064010641106421064310644106451064610647106481064910650106511065210653106541065510656106571065810659106601066110662106631066410665106661066710668106691067010671106721067310674106751067610677106781067910680106811068210683106841068510686106871068810689106901069110692106931069410695106961069710698106991070010701107021070310704107051070610707107081070910710107111071210713107141071510716107171071810719107201072110722107231072410725107261072710728107291073010731107321073310734107351073610737107381073910740107411074210743107441074510746107471074810749107501075110752107531075410755107561075710758107591076010761107621076310764107651076610767107681076910770107711077210773107741077510776107771077810779107801078110782107831078410785107861078710788107891079010791107921079310794107951079610797107981079910800108011080210803108041080510806108071080810809108101081110812108131081410815108161081710818108191082010821108221082310824108251082610827108281082910830108311083210833108341083510836108371083810839108401084110842108431084410845108461084710848108491085010851108521085310854108551085610857108581085910860108611086210863108641086510866108671086810869108701087110872108731087410875108761087710878108791088010881108821088310884108851088610887108881088910890108911089210893108941089510896108971089810899109001090110902109031090410905109061090710908109091091010911109121091310914109151091610917109181091910920109211092210923109241092510926109271092810929109301093110932109331093410935109361093710938109391094010941109421094310944109451094610947109481094910950109511095210953109541095510956109571095810959109601096110962109631096410965109661096710968109691097010971109721097310974109751097610977109781097910980109811098210983109841098510986109871098810989109901099110992109931099410995109961099710998109991100011001110021100311004110051100611007110081100911010110111101211013110141101511016110171101811019110201102111022110231102411025110261102711028110291103011031110321103311034110351103611037110381103911040110411104211043110441104511046110471104811049110501105111052110531105411055110561105711058110591106011061110621106311064110651106611067110681106911070110711107211073110741107511076110771107811079110801108111082110831108411085110861108711088110891109011091110921109311094110951109611097110981109911100111011110211103111041110511106111071110811109111101111111112111131111411115111161111711118111191112011121111221112311124111251112611127111281112911130111311113211133111341113511136111371113811139111401114111142111431114411145111461114711148111491115011151111521115311154111551115611157111581115911160111611116211163111641116511166111671116811169111701117111172111731117411175111761117711178111791118011181111821118311184111851118611187111881118911190111911119211193111941119511196111971119811199112001120111202112031120411205112061120711208112091121011211112121121311214112151121611217112181121911220112211122211223112241122511226112271122811229112301123111232112331123411235112361123711238112391124011241112421124311244112451124611247112481124911250112511125211253112541125511256112571125811259112601126111262112631126411265112661126711268112691127011271112721127311274112751127611277112781127911280112811128211283112841128511286112871128811289112901129111292112931129411295112961129711298112991130011301113021130311304113051130611307113081130911310113111131211313113141131511316113171131811319113201132111322113231132411325113261132711328113291133011331113321133311334113351133611337113381133911340113411134211343113441134511346113471134811349113501135111352113531135411355113561135711358113591136011361113621136311364113651136611367113681136911370113711137211373113741137511376113771137811379113801138111382113831138411385113861138711388113891139011391113921139311394113951139611397113981139911400114011140211403114041140511406114071140811409114101141111412114131141411415114161141711418114191142011421114221142311424114251142611427114281142911430114311143211433114341143511436114371143811439114401144111442114431144411445114461144711448114491145011451114521145311454114551145611457114581145911460114611146211463114641146511466114671146811469114701147111472114731147411475114761147711478114791148011481114821148311484114851148611487114881148911490114911149211493114941149511496114971149811499115001150111502115031150411505115061150711508115091151011511115121151311514115151151611517115181151911520115211152211523115241152511526115271152811529115301153111532115331153411535115361153711538115391154011541115421154311544115451154611547115481154911550115511155211553115541155511556115571155811559115601156111562115631156411565115661156711568115691157011571115721157311574115751157611577115781157911580115811158211583115841158511586115871158811589115901159111592115931159411595115961159711598115991160011601116021160311604116051160611607116081160911610116111161211613116141161511616116171161811619116201162111622116231162411625116261162711628116291163011631116321163311634116351163611637116381163911640116411164211643116441164511646116471164811649116501165111652116531165411655116561165711658116591166011661116621166311664116651166611667116681166911670116711167211673116741167511676116771167811679116801168111682116831168411685116861168711688116891169011691116921169311694116951169611697116981169911700117011170211703117041170511706117071170811709117101171111712117131171411715117161171711718117191172011721117221172311724117251172611727117281172911730117311173211733117341173511736117371173811739117401174111742117431174411745117461174711748117491175011751117521175311754117551175611757117581175911760117611176211763117641176511766117671176811769117701177111772117731177411775117761177711778117791178011781117821178311784117851178611787117881178911790117911179211793117941179511796117971179811799118001180111802118031180411805118061180711808118091181011811118121181311814118151181611817118181181911820118211182211823118241182511826118271182811829118301183111832118331183411835118361183711838118391184011841118421184311844118451184611847118481184911850118511185211853118541185511856118571185811859118601186111862118631186411865118661186711868118691187011871118721187311874118751187611877118781187911880118811188211883118841188511886118871188811889118901189111892118931189411895118961189711898118991190011901119021190311904119051190611907119081190911910119111191211913119141191511916119171191811919119201192111922119231192411925119261192711928119291193011931119321193311934119351193611937119381193911940119411194211943119441194511946119471194811949119501195111952119531195411955119561195711958119591196011961119621196311964119651196611967119681196911970119711197211973119741197511976119771197811979119801198111982119831198411985119861198711988119891199011991119921199311994119951199611997119981199912000120011200212003120041200512006120071200812009120101201112012120131201412015120161201712018120191202012021120221202312024120251202612027120281202912030120311203212033120341203512036120371203812039120401204112042120431204412045120461204712048120491205012051120521205312054120551205612057120581205912060120611206212063120641206512066120671206812069120701207112072120731207412075120761207712078120791208012081120821208312084120851208612087120881208912090120911209212093120941209512096120971209812099121001210112102121031210412105121061210712108121091211012111121121211312114121151211612117121181211912120121211212212123121241212512126121271212812129121301213112132121331213412135121361213712138121391214012141121421214312144121451214612147121481214912150121511215212153121541215512156121571215812159121601216112162121631216412165121661216712168121691217012171121721217312174121751217612177121781217912180121811218212183121841218512186121871218812189121901219112192121931219412195121961219712198121991220012201122021220312204122051220612207122081220912210122111221212213122141221512216122171221812219122201222112222122231222412225122261222712228122291223012231122321223312234122351223612237122381223912240122411224212243122441224512246122471224812249122501225112252122531225412255122561225712258122591226012261122621226312264122651226612267122681226912270122711227212273122741227512276122771227812279122801228112282122831228412285122861228712288122891229012291122921229312294122951229612297122981229912300123011230212303123041230512306123071230812309123101231112312123131231412315123161231712318123191232012321123221232312324123251232612327123281232912330123311233212333123341233512336123371233812339123401234112342123431234412345123461234712348123491235012351123521235312354123551235612357123581235912360123611236212363123641236512366123671236812369123701237112372123731237412375123761237712378123791238012381123821238312384123851238612387123881238912390123911239212393123941239512396123971239812399124001240112402124031240412405124061240712408124091241012411124121241312414124151241612417124181241912420124211242212423124241242512426124271242812429124301243112432124331243412435124361243712438124391244012441124421244312444124451244612447124481244912450124511245212453124541245512456124571245812459124601246112462124631246412465124661246712468124691247012471124721247312474124751247612477124781247912480124811248212483124841248512486124871248812489124901249112492124931249412495124961249712498124991250012501125021250312504125051250612507125081250912510125111251212513125141251512516125171251812519125201252112522125231252412525125261252712528125291253012531125321253312534125351253612537125381253912540125411254212543125441254512546125471254812549125501255112552125531255412555125561255712558125591256012561125621256312564125651256612567125681256912570125711257212573125741257512576125771257812579125801258112582125831258412585125861258712588125891259012591125921259312594125951259612597125981259912600126011260212603126041260512606126071260812609126101261112612126131261412615126161261712618126191262012621126221262312624126251262612627126281262912630126311263212633126341263512636126371263812639126401264112642126431264412645126461264712648126491265012651126521265312654126551265612657126581265912660126611266212663126641266512666126671266812669126701267112672126731267412675126761267712678126791268012681126821268312684126851268612687126881268912690126911269212693126941269512696126971269812699127001270112702127031270412705127061270712708127091271012711127121271312714127151271612717127181271912720127211272212723127241272512726127271272812729127301273112732127331273412735127361273712738127391274012741127421274312744127451274612747127481274912750127511275212753127541275512756127571275812759127601276112762127631276412765127661276712768127691277012771127721277312774127751277612777127781277912780127811278212783127841278512786127871278812789127901279112792127931279412795127961279712798127991280012801128021280312804128051280612807128081280912810128111281212813128141281512816128171281812819128201282112822128231282412825128261282712828128291283012831128321283312834128351283612837128381283912840128411284212843128441284512846128471284812849128501285112852128531285412855128561285712858128591286012861128621286312864128651286612867128681286912870128711287212873128741287512876128771287812879128801288112882128831288412885128861288712888128891289012891128921289312894128951289612897128981289912900129011290212903129041290512906129071290812909129101291112912129131291412915129161291712918129191292012921129221292312924129251292612927129281292912930129311293212933129341293512936129371293812939129401294112942129431294412945129461294712948129491295012951129521295312954129551295612957129581295912960129611296212963129641296512966129671296812969129701297112972129731297412975129761297712978129791298012981129821298312984129851298612987129881298912990129911299212993129941299512996129971299812999130001300113002130031300413005130061300713008130091301013011130121301313014130151301613017130181301913020130211302213023130241302513026130271302813029130301303113032130331303413035130361303713038130391304013041130421304313044130451304613047130481304913050130511305213053130541305513056130571305813059130601306113062130631306413065130661306713068130691307013071130721307313074130751307613077130781307913080130811308213083130841308513086130871308813089130901309113092130931309413095130961309713098130991310013101131021310313104131051310613107131081310913110131111311213113131141311513116131171311813119131201312113122131231312413125131261312713128131291313013131131321313313134131351313613137131381313913140131411314213143131441314513146131471314813149131501315113152131531315413155131561315713158131591316013161131621316313164131651316613167131681316913170131711317213173131741317513176131771317813179131801318113182131831318413185131861318713188131891319013191131921319313194131951319613197131981319913200132011320213203132041320513206132071320813209132101321113212132131321413215132161321713218132191322013221132221322313224132251322613227132281322913230132311323213233132341323513236132371323813239132401324113242132431324413245132461324713248132491325013251132521325313254132551325613257132581325913260132611326213263132641326513266132671326813269132701327113272132731327413275132761327713278132791328013281132821328313284132851328613287132881328913290132911329213293132941329513296132971329813299133001330113302133031330413305133061330713308133091331013311133121331313314133151331613317133181331913320133211332213323133241332513326133271332813329133301333113332133331333413335133361333713338133391334013341133421334313344133451334613347133481334913350133511335213353133541335513356133571335813359133601336113362133631336413365133661336713368133691337013371133721337313374133751337613377133781337913380133811338213383133841338513386133871338813389133901339113392133931339413395133961339713398133991340013401134021340313404134051340613407134081340913410134111341213413134141341513416134171341813419134201342113422134231342413425134261342713428134291343013431134321343313434134351343613437134381343913440134411344213443134441344513446134471344813449134501345113452134531345413455134561345713458134591346013461134621346313464134651346613467134681346913470134711347213473134741347513476134771347813479134801348113482134831348413485134861348713488134891349013491134921349313494134951349613497134981349913500135011350213503135041350513506135071350813509135101351113512135131351413515135161351713518135191352013521135221352313524135251352613527135281352913530135311353213533135341353513536135371353813539135401354113542135431354413545135461354713548135491355013551135521355313554135551355613557135581355913560135611356213563135641356513566135671356813569135701357113572135731357413575135761357713578
  1. import asyncio
  2. import json
  3. import logging
  4. import math
  5. import os
  6. import posixpath
  7. import re
  8. import secrets
  9. import shutil
  10. import time
  11. from contextlib import asynccontextmanager
  12. from datetime import datetime, timedelta, timezone
  13. from logging.handlers import RotatingFileHandler
  14. from pathlib import Path, PurePosixPath
  15. from typing import NamedTuple, cast
  16. from urllib.parse import urlparse
  17. from fastapi import FastAPI
  18. from fastapi.encoders import jsonable_encoder
  19. from fastapi.exceptions import RequestValidationError
  20. from fastapi.responses import FileResponse, JSONResponse
  21. from fastapi.staticfiles import StaticFiles
  22. from sqlalchemy import delete, func, inspect as sa_inspect, or_, select, text, update
  23. from sqlalchemy.engine import CursorResult
  24. from sqlalchemy.exc import IntegrityError
  25. from sqlalchemy.ext.asyncio import AsyncSession
  26. from sqlalchemy.orm.exc import StaleDataError
  27. from backend.app.api.routes import (
  28. ams_history,
  29. announcements,
  30. api_keys,
  31. archive_purge,
  32. archives,
  33. auth,
  34. bug_report,
  35. bulk,
  36. camera,
  37. camwall,
  38. cloud,
  39. connected_apps,
  40. discovery,
  41. external_links,
  42. filaments,
  43. finance,
  44. firmware,
  45. github_backup,
  46. groups,
  47. ha_sensors,
  48. inventory,
  49. kprofiles,
  50. label_templates,
  51. labels,
  52. library,
  53. library_tags,
  54. library_trash,
  55. library_variants,
  56. local_backup,
  57. local_presets,
  58. location_ha_sensors,
  59. maintenance,
  60. makerworld,
  61. manyfold,
  62. metrics,
  63. mfa,
  64. notification_templates,
  65. notifications,
  66. obico,
  67. octoeverywhere,
  68. orca_cloud,
  69. overlay_branding,
  70. pending_uploads,
  71. pipeline_runs,
  72. print_log,
  73. print_queue,
  74. printer_locations,
  75. printer_sensor_history,
  76. printers,
  77. projects,
  78. scheduled_dryings,
  79. settings as settings_routes,
  80. slice_jobs,
  81. slicer_pipelines,
  82. slicer_presets,
  83. smart_plugs,
  84. sponsor_prompt,
  85. spoolbuddy,
  86. spoolman,
  87. spoolman_inventory,
  88. support,
  89. system,
  90. updates,
  91. user_notifications,
  92. users,
  93. virtual_printers,
  94. webhook,
  95. websocket,
  96. )
  97. from backend.app.api.routes.maintenance import (
  98. _get_printer_maintenance_internal,
  99. ensure_default_types,
  100. maintenance_alert_values,
  101. )
  102. from backend.app.api.routes.support import init_debug_logging
  103. from backend.app.core.config import APP_VERSION, settings as app_settings
  104. from backend.app.core.database import async_session, engine, init_db
  105. from backend.app.core.static_assets import AssetStaticFiles
  106. from backend.app.core.tasks import spawn_background_task
  107. from backend.app.core.websocket import ws_manager
  108. from backend.app.services import kprofile_drift, pause_eta, print_dispatch_context, slot_unlink_grace
  109. from backend.app.services.ams_assignment_lock import get_ams_assignment_lock
  110. from backend.app.services.archive import ArchiveService, peek_plate_index_in_3mf, swap_plate_suffix
  111. from backend.app.services.archive_cost_estimate import (
  112. schedule_archive_cost_estimate as _schedule_archive_cost_estimate,
  113. settle_archive_cost_estimate as _settle_archive_cost_estimate,
  114. )
  115. from backend.app.services.archive_purge import archive_purge_service
  116. from backend.app.services.bambu_ftp import (
  117. RECOVERY_DOWNLOAD_MAX_TIMEOUT,
  118. FileNotOnPrinterError,
  119. cache_3mf_download,
  120. clear_3mf_cache,
  121. download_file_async,
  122. download_file_try_paths_async,
  123. ftps_handshake_blocked,
  124. get_cached_3mf,
  125. get_ftp_retry_settings,
  126. normalize_3mf_name,
  127. with_ftp_retry,
  128. )
  129. from backend.app.services.bambu_mqtt import PrinterState
  130. from backend.app.services.camera_light import camera_light
  131. from backend.app.services.energy_plug import energy_plug_candidates, select_energy_reading
  132. from backend.app.services.github_backup import github_backup_service
  133. from backend.app.services.ha_sensor_manager import ha_sensor_manager
  134. from backend.app.services.hms_errors import hms_fault_counts as _hms_fault_counts
  135. from backend.app.services.library_trash import library_trash_service
  136. from backend.app.services.local_backup import local_backup_service
  137. from backend.app.services.location_ha_sensor_manager import location_ha_sensor_manager
  138. from backend.app.services.mqtt_relay import mqtt_relay
  139. from backend.app.services.mqtt_smart_plug import mqtt_smart_plug_service
  140. from backend.app.services.notification_service import notification_service
  141. from backend.app.services.notify_live_activities import notify_live_activities
  142. from backend.app.services.notify_widgets import notify_widgets
  143. from backend.app.services.obico_detection import obico_detection_service
  144. from backend.app.services.octoeverywhere_detection import octoeverywhere_detection_service
  145. from backend.app.services.plug_energy import get_plug_energy as _get_plug_energy
  146. from backend.app.services.print_cost_estimate import plate_scoped_run_estimate as _plate_scoped_run_estimate
  147. from backend.app.services.print_scheduler import scheduler as print_scheduler
  148. from backend.app.services.print_storage import (
  149. REASON_FTP_TRANSFER_FAILED,
  150. REASON_FTPS_COOLOFF,
  151. external_storage_present,
  152. ftp_probe_paths,
  153. print_file_reachable_over_ftp,
  154. )
  155. from backend.app.services.printer_manager import (
  156. init_printer_connections,
  157. parse_plate_id,
  158. printer_manager,
  159. printer_state_to_dict,
  160. printer_tray_layout,
  161. resolve_plate_id,
  162. )
  163. from backend.app.services.slot_kprofile import find_slot_kprofile_for_extruder
  164. from backend.app.services.slot_nozzle import (
  165. nozzle_diameter_for_extruder,
  166. nozzle_flow_for_extruder,
  167. )
  168. from backend.app.services.smart_plug_manager import smart_plug_manager
  169. from backend.app.services.spool_assignment_notifications import (
  170. notify_missing_spool_assignments_on_print_start,
  171. )
  172. from backend.app.services.spool_filament_preset import printer_safe_filament_id
  173. from backend.app.services.spoolman import close_spoolman_client, get_spoolman_client, init_spoolman_client
  174. from backend.app.services.spoolman_tracking import (
  175. cleanup_tracking as _cleanup_spoolman_tracking,
  176. release_archived_slot_rows as _release_archived_spoolman_slots,
  177. report_usage as _report_spoolman_usage,
  178. store_print_data as _store_spoolman_print_data,
  179. )
  180. from backend.app.services.telegram_reactions import telegram_reaction_poller
  181. from backend.app.services.wled import wled_manager
  182. from backend.app.utils.ams_drying import is_drying_active, temperature_alarm_suppressed
  183. from backend.app.utils.ams_humidity import ams_humidity_percent
  184. from backend.app.utils.filament_types import printer_filament_type, support_filament_type
  185. from backend.app.utils.fts_routing import extruder_for_inlet
  186. from backend.app.utils.local_time import to_naive_utc, utcnow_naive
  187. from backend.app.utils.print_jobs import is_internal_printer_job
  188. from backend.app.utils.tray_ids import AmsType, TrayLayout, external_tray_id, unit_ams_type, unit_label
  189. # =============================================================================
  190. # Dependency Check - runs before other imports to give helpful error messages
  191. # =============================================================================
  192. def _start_error_server(missing_packages: list):
  193. """Start a minimal HTTP server to display dependency errors in browser."""
  194. import os
  195. import signal
  196. from http.server import BaseHTTPRequestHandler, HTTPServer
  197. packages_html = "".join(f"<li><code>{p}</code></li>" for p in missing_packages)
  198. html = f"""<!DOCTYPE html>
  199. <html>
  200. <head>
  201. <title>Bambuddy - Setup Required</title>
  202. <style>
  203. body {{
  204. font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
  205. background: #0f172a; color: #e2e8f0;
  206. display: flex; justify-content: center; align-items: center;
  207. min-height: 100vh; margin: 0; padding: 20px; box-sizing: border-box;
  208. }}
  209. .container {{
  210. background: #1e293b; border-radius: 12px; padding: 40px;
  211. max-width: 600px; text-align: center; box-shadow: 0 4px 20px rgba(0,0,0,0.3);
  212. }}
  213. h1 {{ color: #f87171; margin-bottom: 10px; }}
  214. h2 {{ color: #94a3b8; font-weight: normal; margin-top: 0; }}
  215. .packages {{
  216. background: #0f172a; border-radius: 8px; padding: 20px;
  217. margin: 20px 0; text-align: left;
  218. }}
  219. .packages ul {{ margin: 0; padding-left: 20px; }}
  220. .packages li {{ color: #fbbf24; margin: 8px 0; }}
  221. .command {{
  222. background: #0f172a; border-radius: 8px; padding: 15px 20px;
  223. margin: 15px 0; font-family: monospace; color: #4ade80;
  224. text-align: left; overflow-x: auto;
  225. }}
  226. .note {{ color: #94a3b8; font-size: 14px; margin-top: 20px; }}
  227. </style>
  228. </head>
  229. <body>
  230. <div class="container">
  231. <h1>Setup Required</h1>
  232. <h2>Missing Python packages</h2>
  233. <div class="packages"><ul>{packages_html}</ul></div>
  234. <p>To fix, run this command on your server:</p>
  235. <div class="command">pip install -r requirements.lock</div>
  236. <p>Or if using a virtual environment:</p>
  237. <div class="command">./venv/bin/pip install -r requirements.lock</div>
  238. <p class="note">After installing, restart Bambuddy:<br>
  239. <code>sudo systemctl restart bambuddy</code></p>
  240. </div>
  241. </body>
  242. </html>"""
  243. class ErrorHandler(BaseHTTPRequestHandler):
  244. def do_GET(self):
  245. self.send_response(503)
  246. self.send_header("Content-type", "text/html")
  247. self.end_headers()
  248. self.wfile.write(html.encode())
  249. def log_message(self, format, *args):
  250. print(f"[Error Server] {args[0]}")
  251. port = int(os.environ.get("PORT", 8000))
  252. print(f"\nStarting error server on http://0.0.0.0:{port}")
  253. print("Visit this URL in your browser to see the error details.\n")
  254. server = HTTPServer(("0.0.0.0", port), ErrorHandler) # nosec B104
  255. def shutdown(signum, frame):
  256. print("\nShutting down error server...")
  257. raise SystemExit(0)
  258. signal.signal(signal.SIGTERM, shutdown)
  259. signal.signal(signal.SIGINT, shutdown)
  260. server.serve_forever()
  261. def check_dependencies():
  262. """Check that all required packages are installed."""
  263. missing = []
  264. # Map of import name -> package name (for pip install)
  265. required = {
  266. "jwt": "PyJWT",
  267. "fastapi": "fastapi",
  268. "uvicorn": "uvicorn",
  269. "sqlalchemy": "sqlalchemy",
  270. "aiosqlite": "aiosqlite",
  271. "pydantic": "pydantic",
  272. "paho.mqtt": "paho-mqtt",
  273. }
  274. for module, package in required.items():
  275. try:
  276. __import__(module)
  277. except ImportError:
  278. missing.append(package)
  279. if missing:
  280. print("\n" + "=" * 60)
  281. print("ERROR: Missing required Python packages!")
  282. print("=" * 60)
  283. print(f"\nMissing packages: {', '.join(missing)}")
  284. print("\nTo fix, run:")
  285. print(" pip install -r requirements.lock")
  286. print("\nOr if using a virtual environment:")
  287. print(" ./venv/bin/pip install -r requirements.lock")
  288. print("=" * 60 + "\n")
  289. _start_error_server(missing)
  290. check_dependencies()
  291. # =============================================================================
  292. # Import settings first for logging configuration
  293. # Configure logging based on settings
  294. # DEBUG=true -> DEBUG level, else use LOG_LEVEL setting
  295. log_level_str = "DEBUG" if app_settings.debug else app_settings.log_level.upper()
  296. log_level = getattr(logging, log_level_str, logging.INFO)
  297. # Trace ID column ([-] when no request scope is active — startup, MQTT
  298. # callbacks, scheduled tasks not chained from a request — so the column
  299. # stays visually aligned and missing values are obvious in grep). See
  300. # backend/app/core/trace.py for the ContextVar that feeds this slot.
  301. log_format = "%(asctime)s %(levelname)s [%(name)s] [%(trace_id)s] %(message)s"
  302. # Create root logger
  303. root_logger = logging.getLogger()
  304. root_logger.setLevel(log_level)
  305. # Trace-ID injection: this filter populates record.trace_id from the
  306. # per-request ContextVar so the format string above can reference it.
  307. # Attached to each HANDLER (not the root logger) because Python's
  308. # logging semantics only invoke a logger's filters on records that
  309. # *originated* at that logger — records propagated up from child
  310. # loggers (every named logger in the app) never trigger root's filter.
  311. # Putting it on the handlers means every record any handler emits gets
  312. # trace_id injected just before the formatter runs, regardless of which
  313. # logger created the record. Without this, the formatter raises
  314. # KeyError on every child-logger record and the record is silently
  315. # dropped — which is exactly the "logs/bambuddy.log only shows logs
  316. # partially" bug we hit. See backend/app/core/trace.py for the
  317. # ContextVar the filter reads.
  318. from backend.app.core.logging_filters import RefusedHeaderRedactFilter
  319. from backend.app.core.response_redaction import RefusedHeaderResponseMask
  320. from backend.app.core.trace import TraceIDFilter
  321. _trace_id_filter = TraceIDFilter()
  322. # Console handler - always enabled
  323. console_handler = logging.StreamHandler()
  324. console_handler.setLevel(log_level)
  325. console_handler.setFormatter(logging.Formatter(log_format))
  326. console_handler.addFilter(_trace_id_filter)
  327. # A refused header's value (often a token) masked before the line is written.
  328. _refused_header_filter = RefusedHeaderRedactFilter()
  329. console_handler.addFilter(_refused_header_filter)
  330. root_logger.addHandler(console_handler)
  331. def _attach_file_handler_to_uvicorn(handler: logging.Handler) -> None:
  332. """Send uvicorn's own loggers to bambuddy.log as well.
  333. Neither reaches the root handler: uvicorn.access has propagate=False, and
  334. uvicorn.error propagates only to "uvicorn", which has propagate=False.
  335. uvicorn.access is the record of which endpoint changed server state;
  336. uvicorn.error is where an unhandled route error is logged with its
  337. traceback ("Exception in ASGI application").
  338. """
  339. for name in ("uvicorn.access", "uvicorn.error"):
  340. logging.getLogger(name).addHandler(handler)
  341. # File handler - only in production or if explicitly enabled
  342. if app_settings.log_to_file:
  343. log_file = app_settings.log_dir / "bambuddy.log"
  344. file_handler = RotatingFileHandler(
  345. log_file,
  346. maxBytes=app_settings.log_max_bytes,
  347. backupCount=app_settings.log_backup_count,
  348. encoding="utf-8",
  349. )
  350. file_handler.setLevel(log_level)
  351. file_handler.setFormatter(logging.Formatter(log_format))
  352. file_handler.addFilter(_trace_id_filter)
  353. file_handler.addFilter(_refused_header_filter)
  354. root_logger.addHandler(file_handler)
  355. logging.info("Logging to file: %s", log_file)
  356. # Pipe uvicorn's HTTP access log to bambuddy.log too. Uvicorn ships its
  357. # access logger with propagate=False by default, so without this attach
  358. # there is no on-disk record of which endpoint triggered a server-state
  359. # change — the rogue stop_print mystery on 2026-04-26 was untraceable
  360. # for exactly this reason. Filtered to write methods only
  361. # (POST/PUT/PATCH/DELETE) so the high-volume status-poll GETs from the
  362. # frontend don't churn the rotation window faster than it's useful.
  363. from backend.app.core.logging_filters import (
  364. CancelledPoolNoiseFilter,
  365. WriteRequestsOnlyFilter,
  366. )
  367. _attach_file_handler_to_uvicorn(file_handler)
  368. uvicorn_access_logger = logging.getLogger("uvicorn.access")
  369. uvicorn_access_logger.addFilter(WriteRequestsOnlyFilter())
  370. # Uvicorn's access logger has propagate=False (its own default), so the
  371. # root-attached TraceIDFilter never sees these records. Attach a
  372. # second instance directly so HTTP access lines carry the same trace
  373. # ID column as the application logs they correlate with.
  374. uvicorn_access_logger.addFilter(TraceIDFilter())
  375. # Drop SQLAlchemy connection-pool log noise that's caused by Starlette's
  376. # BaseHTTPMiddleware cancelling the inner task scope on client
  377. # disconnect (#1112). The cancel-safe `get_db` already prevents the
  378. # underlying transaction leak; this filter only suppresses the residual
  379. # log records that pre-existing pools still emit during their cleanup.
  380. logging.getLogger("sqlalchemy.pool").addFilter(CancelledPoolNoiseFilter())
  381. # Query-string tokens out of uvicorn's request and WebSocket lines, on every
  382. # handler (console included), whether or not file logging is on.
  383. from backend.app.core.logging_filters import ( # noqa: E402
  384. QueryTokenRedactFilter,
  385. ShutdownCancelledRequestFilter,
  386. )
  387. # First, so the traceback of a stream cancelled at shutdown is dropped before
  388. # the filters below render it.
  389. logging.getLogger("uvicorn.error").addFilter(ShutdownCancelledRequestFilter())
  390. for _uvicorn_logger in ("uvicorn.access", "uvicorn.error"):
  391. logging.getLogger(_uvicorn_logger).addFilter(QueryTokenRedactFilter())
  392. logging.getLogger(_uvicorn_logger).addFilter(_refused_header_filter)
  393. # These stay at WARNING even in debug mode: at INFO/DEBUG httpx/httpcore log
  394. # full request URLs, and some carry credentials (bot tokens, webhook URLs, API
  395. # keys in the query). Same as the runtime log-level toggle in support.py.
  396. logging.getLogger("httpcore").setLevel(logging.WARNING)
  397. logging.getLogger("httpx").setLevel(logging.WARNING)
  398. # aiosqlite logs every statement with its parameters at DEBUG.
  399. logging.getLogger("aiosqlite").setLevel(logging.WARNING)
  400. # Reduce noise from third-party libraries in production
  401. if not app_settings.debug:
  402. logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
  403. logging.getLogger("paho.mqtt").setLevel(logging.WARNING)
  404. logging.info("Bambuddy starting - debug=%s, log_level=%s", app_settings.debug, log_level_str)
  405. # Track active prints: {(printer_id, filename): archive_id}
  406. _active_prints: dict[tuple[int, str], int] = {}
  407. # #1721: stage-22 pre-captured finish photo bytes per printer. on_finish_photo_moment
  408. # fires when stg_cur enters 22 ("Filament unloading") at end-of-print — toolhead
  409. # parked, bed not yet dropped — and grabs a single camera frame into this cache.
  410. # `_background_finish_photo` (inside on_print_complete) consumes the cached bytes
  411. # instead of running its own grab-now chain when present, so the finish photo
  412. # captures the better-framed pre-bed-drop moment without us having to force
  413. # timelapse on at dispatch (the #1397 mechanism that caused #1721's per-layer
  414. # nozzle parking on slicer profiles with Timelapse Type = Smooth).
  415. #
  416. # #2708: the bytes in here are ALWAYS already rotated by the printer's
  417. # camera_rotation. `on_finish_photo_moment` owns that, because one of its
  418. # sources (the #1867 in-print bank) is rotated before it ever reaches the
  419. # bank and the others are not — so the consumer can't tell them apart and
  420. # must not rotate again.
  421. _stage22_finish_frames: dict[int, bytes] = {}
  422. # #1790: per-printer producer-done event. Set by `on_finish_photo_moment` in its
  423. # `finally` block (whether it captured a frame or not). The consumer in
  424. # `_background_finish_photo` waits on it before reading `_stage22_finish_frames`
  425. # so the FINISH-state fallback path — where moment and completion are dispatched
  426. # back-to-back — doesn't race past the producer with an empty pop, and the
  427. # consumer's RTSP fallback can't collide with the producer's still-in-flight RTSP
  428. # grab (Bambu printers allow only one RTSP client at a time).
  429. _stage22_finish_in_flight: dict[int, asyncio.Event] = {}
  430. # #1867: rolling "last in-print camera frame" per printer. Refreshed on
  431. # layer-change and on print-progress advances (#2547) while the model is still
  432. # printing, then consumed by the FINISH-state finish-photo path when the
  433. # dispatcher recorded that it injected End G-code into this print. Bambu
  434. # reports gcode_state=FINISH AFTER the user End G-code (e.g. SwapMod
  435. # plate-swap) has run, so a live grab there would capture the swapped/empty
  436. # plate.
  437. #
  438. # The load-bearing property: both drivers are print telemetry that stops before
  439. # the End G-code executes — no further layer_num increases, and mc_percent
  440. # freezes — so the last banked frame is always the finished print before the
  441. # swap. Anything added as a third driver must hold that same property.
  442. _inprint_frame_bank: dict[int, bytes] = {}
  443. # Monotonic timestamp of the last banked frame per printer — throttles banking
  444. # so tall prints don't add a camera grab on every layer.
  445. _inprint_frame_bank_ts: dict[int, float] = {}
  446. # Minimum seconds between banked frames, except the final object layer which
  447. # always refreshes for the best framing.
  448. _INPRINT_BANK_MIN_INTERVAL = 25.0
  449. # Per-printer "connected" edge tracker. Used by `on_printer_status_change`
  450. # to fire `reconcile_stale_active_prints` exactly once per (re)connection
  451. # (#1542 follow-up — power-cycle ghost prints). The value is True after
  452. # the first connected status update for that connection; transitions back
  453. # to False whenever we observe `state.connected = False` so the next
  454. # reconnect re-arms reconciliation. Keyed by printer_id.
  455. _printer_reconciled_since_connect: dict[int, bool] = {}
  456. # Same edge, same keying, for priming the printer's calibration table exactly
  457. # once per (re)connection. Nothing else asks for it on connect: state.kprofiles
  458. # is otherwise filled only when someone opens the Profiles page or Configure
  459. # Slot, when a GitHub backup runs, or when the printer happens to answer
  460. # somebody else's query on the report topic. Until then the AMS slot card has
  461. # no K value to show on the printers whose trays carry none of their own
  462. # (#2854 — H2-series report cali_idx and nothing more).
  463. _printer_kprofiles_primed_since_connect: dict[int, bool] = {}
  464. # Track expected prints from reprint/scheduled (skip auto-archiving for these)
  465. # {(printer_id, filename): archive_id}
  466. _expected_prints: dict[tuple[int, str], int] = {}
  467. # Archives of the running prints Bambuddy dispatched itself: adopted from an
  468. # expected print, or found with Bambuddy's dispatch marker. _active_prints holds
  469. # every print on_print_start archived, those started from Bambu Studio, Handy or
  470. # the printer's screen included, so it is no proof of ownership on its own.
  471. _owned_print_archive_ids: set[int] = set()
  472. # What a dispatch registered for its print, keyed (printer_id, archive_id):
  473. # plates of one archive print on two printers at once, each with its own.
  474. # Track AMS mapping for prints: {(printer_id, archive_id): [global_tray_id_per_slot]}
  475. # Used by usage tracker to map 3MF slots to physical AMS trays
  476. _print_ams_mappings: dict[tuple[int, int], list[int]] = {}
  477. # Track cost center selection for the current print run: {(printer_id, archive_id): cost_center_id}
  478. _print_cost_center_ids: dict[tuple[int, int], int] = {}
  479. # Track plate_id for prints from multi-plate 3MFs: {(printer_id, archive_id): plate_id}
  480. # Used by usage tracker to scope 3MF parsing to the dispatched plate (#1697).
  481. # Populated by direct-Print and queue dispatch paths; queue prints also have a
  482. # redundant queue-item lookup in on_print_start so this dict isn't load-bearing
  483. # for the queue path. Cleared on print completion or TTL eviction.
  484. _print_plate_ids: dict[tuple[int, int], int] = {}
  485. # Track progress milestones for notifications: {printer_id: last_milestone_notified}
  486. # Milestones are 25, 50, 75. Value of 0 means no milestone notified yet for current print.
  487. _last_progress_milestone: dict[int, int] = {}
  488. # Track whether first layer complete notification has been sent for current print
  489. _first_layer_notified: dict[int, bool] = {}
  490. # Track whether we already sent a kill-switch stop for the current unauthorized print
  491. _unauthorized_print_kill_sent: set[int] = set()
  492. # The MQTT status callback is a hot path. Cache the two-setting kill-switch
  493. # lookup briefly so an unknown active print does not query the database on
  494. # every status frame. A short TTL keeps settings changes responsive.
  495. _KILL_SWITCH_SETTING_CACHE_TTL_SECONDS = 5.0
  496. _kill_switch_setting_cache: tuple[bool, float] | None = None
  497. # Provider notification started when the kill switch stops a print. The later
  498. # MQTT print-complete callback awaits this task and only sends its regular
  499. # provider notification when the immediate attempt failed.
  500. _kill_switch_notification_tasks: dict[int, asyncio.Task[bool]] = {}
  501. # HMS faults already notified, with when each was last seen:
  502. # {printer_id: {fault key: timestamp}}. Tracked per fault, not per printer: a
  503. # fault that flickers beside a held notice was forgotten on every gap and
  504. # notified on every return (#3226). The window is long because the measured
  505. # gaps reach 64 s.
  506. _notified_hms_errors: dict[int, dict[str, float]] = {}
  507. _HMS_CLEAR_GRACE_SECONDS = 600.0
  508. # The print state each printer was last seen in, and the faults recorded before
  509. # it last changed, which are forgotten as soon as they are gone (see
  510. # _take_new_hms_faults).
  511. _hms_print_state: dict[int, str] = {}
  512. _hms_forget_when_gone: dict[int, set[str]] = {}
  513. # Track timelapse file baselines at print start: {printer_id: set of video filenames}
  514. # Used for snapshot-diff detection at print completion
  515. _timelapse_baselines: dict[int, set[str]] = {}
  516. # Printers whose baseline above was taken off a card that could not be read,
  517. # so it is empty without the card being empty (#3412).
  518. _timelapse_baselines_unread: set[int] = set()
  519. # Track printers waiting for bed to cool after print completion.
  520. # Event-driven: fires when bed_temper arrives via MQTT below threshold.
  521. # {printer_id: {"threshold": float, "filename": str, "registered_at": float}}
  522. _bed_cool_waiters: dict[int, dict] = {}
  523. # Track printers where the user explicitly stopped the print from the queue UI.
  524. # When on_print_complete fires with status "failed" for these printers we treat it
  525. # as "cancelled" (stopped by user) so the correct notification email is sent.
  526. _user_stopped_printers: set[int] = set()
  527. # Printers whose running print the kill switch stopped. Their completion is a
  528. # cancel with the reason "killSwitch", not the "failed" the printer reports.
  529. _kill_switch_stopped: set[int] = set()
  530. # Offline-notification edge state (#1752): fire `on_printer_offline` exactly
  531. # once when a printer transitions connected → disconnected. `_printer_last_connected`
  532. # holds the previous observation so we only fire on the True → False edge (a
  533. # False → False repeat doesn't notify; an initial False at startup doesn't
  534. # notify either, since there's no prior True). `_printer_offline_notify_tasks`
  535. # holds the per-printer pending asyncio task that fires the notification
  536. # after a debounce window — cancelled if the printer reconnects before the
  537. # window elapses, so transient MQTT blips don't flood the user.
  538. _printer_last_connected: dict[int, bool] = {}
  539. # Printers whose relay offline event went out, so the reconnect publishes the
  540. # matching online event and a blip shorter than the debounce publishes neither.
  541. _printer_offline_published: set[int] = set()
  542. _printer_offline_notify_tasks: dict[int, asyncio.Task] = {}
  543. # Debounce: a printer must stay offline this long before we notify. Sized
  544. # against the staleness path (`bambu_mqtt.py::STALE_RECONNECT_COOLDOWN = 30s`)
  545. # so a single stale-trigger cooldown isn't enough to fire — only a real
  546. # offline that survives one reconnect attempt notifies.
  547. _PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS = 60.0
  548. # HMS short-code → failure_reason key. Used by _dispatch_archive_update
  549. # when status="failed" to label the print's failure_reason in archives.
  550. #
  551. # Earlier code matched on `module` alone (e.g. "any module 0x0C HMS → Layer shift"),
  552. # which is wrong on two counts:
  553. # 1. Real layer-shift codes live in module 0x03 (see Bambu wiki), not 0x0C.
  554. # 2. Module 0x0C is "Motion Controller" — broad category that also covers cameras
  555. # and visual markers, AND the H2D firmware emits a 0x0C HMS (0C00_001B, not in
  556. # the public wiki) as part of its user-cancel sequence. Matching on the module
  557. # alone caused user-cancellations to be archived as "Layer shift" failures.
  558. # We now match by full short code only — anything not in this map leaves
  559. # failure_reason=None rather than guessing.
  560. # Values are the canonical camelCase failure-reason keys, NOT display labels
  561. # (issue #2974). The vocabulary is enforced on writes by
  562. # ``_FAILURE_REASON_KEYS`` in ``api/routes/print_log.py`` and rendered through
  563. # ``t('editArchive.failureReasons.<key>')`` on both the archive editor and the
  564. # Statistics breakdown. Storing a label here instead put a second spelling of
  565. # the same cause into one column: the Failure Analysis widget groups on the raw
  566. # value, so a print the backend classified and an identical one a user
  567. # classified counted as two different reasons, and the label form could never
  568. # be translated because there was no key for ``t()`` to resolve.
  569. _HMS_FAILURE_REASONS: dict[str, str] = {
  570. # Layer shift / step loss
  571. "0300_4057": "layerShift",
  572. "0300_4068": "layerShift",
  573. "0300_800C": "layerShift",
  574. # Filament runout (printer-side & per-AMS-slot)
  575. "0300_8004": "filamentRunout",
  576. "0700_8011": "filamentRunout",
  577. "0701_8011": "filamentRunout",
  578. "0702_8011": "filamentRunout",
  579. "0703_8011": "filamentRunout",
  580. "0704_8011": "filamentRunout",
  581. "0705_8011": "filamentRunout",
  582. "0706_8011": "filamentRunout",
  583. "0707_8011": "filamentRunout",
  584. "07FF_8011": "filamentRunout",
  585. # Clogged nozzle / extruder
  586. "0300_4006": "cloggedNozzle",
  587. "0300_8016": "cloggedNozzle",
  588. "0300_801C": "cloggedNozzle",
  589. "0700_8003": "cloggedNozzle",
  590. "0700_8007": "cloggedNozzle",
  591. "0700_8013": "cloggedNozzle",
  592. "0701_8003": "cloggedNozzle",
  593. "0701_8007": "cloggedNozzle",
  594. "0701_8013": "cloggedNozzle",
  595. "0702_8003": "cloggedNozzle",
  596. # AI print monitoring — spaghetti / the model coming off the plate.
  597. # `spaghettiDetached` is a key the archive editor already offers for this
  598. # failure mode, not a new one, so a derived reason opens the dropdown on
  599. # that option rather than blank — and survives the next save, which clears
  600. # any value the editor does not recognise.
  601. #
  602. # A module-0x0C row is safe here despite the warning above: that warning is
  603. # about matching on the module alone, and 0C00_8042 is a full short code
  604. # with a documented meaning ("The AI print monitor has detected a spaghetti
  605. # defect", hms_errors.py). The H2D cancel echo is 0C00_001B, so the two
  606. # cannot collide.
  607. #
  608. # 0300_8003's own text ends "before continuing your print", but on an X2D
  609. # it arrives with the print already paused, offering only
  610. # RESUME_PRINTING_DEFECTS / STOP_PRINTING — a halt waiting on the user.
  611. #
  612. # Two neighbours are left out on purpose, so this does not get re-derived:
  613. # * 0C00_C004 "Possible spaghetti failure was detected." — "possible"
  614. # reads as a warning about a print that is still running, not a halt.
  615. # * 0300_800A is AI monitoring too, but it reports a filament pile-up in
  616. # the waste chute. That is not the print failing.
  617. #
  618. # That line is drawn from the text, not from `severity`, because severity
  619. # cannot draw it: 0300_8003 reaches us through `print_error`, a bare
  620. # module/error word with no level in it, and bambu_mqtt.py gives every
  621. # print_error entry a flat severity=3. Matching on the short code alone is
  622. # the right shape for derive_failure_reason, not an omission.
  623. "0300_8003": "spaghettiDetached",
  624. "0C00_8042": "spaghettiDetached",
  625. }
  626. def _hms_short_code(attr: int, code: int | str) -> str:
  627. """Build the canonical "MMMM_CCCC" HMS short code from raw attr/code values."""
  628. if isinstance(code, str):
  629. code_int = int(code.replace("0x", ""), 16) if code else 0
  630. else:
  631. code_int = int(code or 0)
  632. attr_int = int(attr or 0)
  633. return f"{(attr_int >> 16) & 0xFFFF:04X}_{code_int & 0xFFFF:04X}"
  634. def derive_failure_reason(status: str, hms_errors: list[dict] | None) -> str | None:
  635. """Derive the failure_reason key for an archived print.
  636. Returns "userCancelled" for cancelled/aborted prints; for failed prints,
  637. returns the first matching key from _HMS_FAILURE_REASONS, or None when
  638. no HMS code matches (don't guess — null is honest). The keys are the
  639. archive editor's vocabulary (_FAILURE_REASON_KEYS in print_log.py) and are
  640. translated at render time.
  641. """
  642. if status in ("aborted", "cancelled"):
  643. return "userCancelled"
  644. if status != "failed":
  645. return None
  646. for err in hms_errors or []:
  647. short_code = _hms_short_code(err.get("attr", 0), err.get("code", 0))
  648. if short_code in _HMS_FAILURE_REASONS:
  649. return _HMS_FAILURE_REASONS[short_code]
  650. return None
  651. # Track created_by_id for expected prints so the user email can be sent even when
  652. # the archive itself doesn't have created_by_id set (e.g. library-file-based prints).
  653. # {(printer_id, filename): created_by_id}
  654. _expected_print_creators: dict[tuple[int, str], int] = {}
  655. # Per-printer dedup for unknown_tag WS broadcasts. Keyed by
  656. # (ams_id, tray_id) -> (tag_uid, tray_uuid); we only re-broadcast when the
  657. # tag tuple changes for the slot. Cleared when the slot is reported empty
  658. # so remove + reinsert reliably re-prompts the UI.
  659. _unknown_tag_last_broadcast: dict[int, dict[tuple[int, int], tuple[str, str]]] = {}
  660. async def _broadcast_unknown_tag(
  661. *,
  662. printer_id: int,
  663. ams_id: int,
  664. tray_id: int,
  665. tag_uid: str,
  666. tray_uuid: str,
  667. tray_type: str | None = None,
  668. tray_color: str | None = None,
  669. tray_sub_brands: str | None = None,
  670. tray_count: int | None = None,
  671. ) -> None:
  672. """Broadcast unknown_tag, deduped so repeated MQTT pushes for the same slot+tag don't spam the UI."""
  673. _logger = logging.getLogger(__name__)
  674. slot_key = (ams_id, tray_id)
  675. tag_key = (tag_uid or "", tray_uuid or "")
  676. per_printer = _unknown_tag_last_broadcast.setdefault(printer_id, {})
  677. if per_printer.get(slot_key) == tag_key:
  678. _logger.debug(
  679. "unknown_tag deduped for printer=%d AMS=%d slot=%d tag=%s",
  680. printer_id,
  681. ams_id,
  682. tray_id,
  683. tag_key[0][:8] or tag_key[1][:8] or "(none)",
  684. )
  685. return
  686. _logger.info(
  687. "unknown_tag broadcast: printer=%d AMS=%d slot=%d type=%r color=%r tag=%s",
  688. printer_id,
  689. ams_id,
  690. tray_id,
  691. tray_type,
  692. tray_color,
  693. tag_key[0][:8] or tag_key[1][:8] or "(none)",
  694. )
  695. # Broadcast first; only commit the dedup if the WS write succeeds.
  696. # If broadcast raises, the next MQTT push retries instead of being
  697. # permanently silenced by a poisoned dedup entry.
  698. await ws_manager.broadcast(
  699. {
  700. "type": "unknown_tag",
  701. "printer_id": printer_id,
  702. "ams_id": ams_id,
  703. "tray_id": tray_id,
  704. "tag_uid": tag_uid,
  705. "tray_uuid": tray_uuid,
  706. "tray_type": tray_type,
  707. "tray_color": tray_color,
  708. "tray_sub_brands": tray_sub_brands,
  709. "tray_count": tray_count,
  710. }
  711. )
  712. per_printer[slot_key] = tag_key
  713. def _clear_unknown_tag_dedup(printer_id: int, ams_id: int, tray_id: int) -> None:
  714. """Drop the cached last-broadcast tag for a slot (called when slot reports empty or gets matched)."""
  715. per_printer = _unknown_tag_last_broadcast.get(printer_id)
  716. if per_printer is None:
  717. return
  718. per_printer.pop((ams_id, tray_id), None)
  719. # TTL for expected-print entries: evict registrations older than this to prevent
  720. # unbounded growth when a print is registered but never starts (e.g. printer
  721. # disconnect, app restart, print started from the printer panel).
  722. _EXPECTED_PRINT_TTL_SECONDS: int = 2 * 60 * 60 # 2 hours
  723. # Registration timestamps used for TTL eviction: {(printer_id, filename): monotonic_time}
  724. _expected_print_registered_at: dict[tuple[int, str], float] = {}
  725. # Cleanup loop interval
  726. _EXPECTED_PRINT_CLEANUP_INTERVAL: int = 15 * 60 # 15 minutes
  727. _expected_prints_cleanup_task: asyncio.Task | None = None
  728. _ACTIVE_PRINT_STATES: set[str] = {"RUNNING", "PRINTING", "PAUSE"}
  729. def _build_status_print_keys(printer_id: int, state: PrinterState) -> list[tuple[int, str]]:
  730. """Build filename keys for matching a printer status update to Bambuddy-owned jobs."""
  731. possible_keys: list[tuple[int, str]] = []
  732. filename = (state.gcode_file or state.current_print or "").strip()
  733. subtask_name = (state.subtask_name or "").strip()
  734. if subtask_name:
  735. possible_keys.append((printer_id, subtask_name))
  736. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  737. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  738. if filename:
  739. base_name = filename.rsplit("/", 1)[-1]
  740. if base_name.endswith(".gcode.3mf"):
  741. root_name = base_name[: -len(".gcode.3mf")]
  742. possible_keys.append((printer_id, root_name))
  743. possible_keys.append((printer_id, base_name))
  744. possible_keys.append((printer_id, f"{root_name}.gcode"))
  745. possible_keys.append((printer_id, f"{root_name}.3mf"))
  746. elif base_name.endswith(".3mf"):
  747. root_name = base_name[: -len(".3mf")]
  748. possible_keys.append((printer_id, root_name))
  749. possible_keys.append((printer_id, base_name))
  750. elif base_name.endswith(".gcode"):
  751. root_name = base_name[: -len(".gcode")]
  752. possible_keys.append((printer_id, root_name))
  753. possible_keys.append((printer_id, f"{root_name}.3mf"))
  754. possible_keys.append((printer_id, base_name))
  755. else:
  756. possible_keys.append((printer_id, base_name))
  757. possible_keys.append((printer_id, f"{base_name}.3mf"))
  758. return possible_keys
  759. async def _print_user_id_if_still_there(db, print_user: dict | None) -> int | None:
  760. """The id of who started the print, unless that user has been deleted since.
  761. Deleting a user forgets them in printer_manager, but a print that ended a
  762. moment before still holds them.
  763. """
  764. if not print_user or print_user.get("user_id") is None:
  765. return None
  766. from backend.app.models.user import User
  767. return await db.scalar(select(User.id).where(User.id == print_user["user_id"]))
  768. def _is_bambuddy_authorized_print_in_memory(printer_id: int, state: PrinterState) -> bool:
  769. """Check the cheap, process-local print ownership signals."""
  770. if printer_manager.get_current_print_user(printer_id):
  771. return True
  772. return any(
  773. key in _expected_prints or _active_prints.get(key) in _owned_print_archive_ids
  774. for key in _build_status_print_keys(printer_id, state)
  775. )
  776. def _reported_print_names(printer_id: int, state: PrinterState) -> set[str]:
  777. """The names the printer reports for its print, as it echoes them (spaces become underscores)."""
  778. return {_normalise_subtask_name(name) for _pid, name in _build_status_print_keys(printer_id, state)} - {""}
  779. async def _printing_archive_without_job_id(
  780. db, printer_id: int, reported: set[str], *, dispatched_only: bool, live_only: bool = False
  781. ):
  782. """The newest printing archive on ``printer_id`` of a print that reported no job id, under a reported name.
  783. An archive with a real id of its own is a print that reported one: a "0"
  784. print is not it, even under the same name (one whose completion was missed
  785. would otherwise cover it). ``dispatched_only`` keeps to archives carrying
  786. Bambuddy's dispatch marker; ``live_only`` leaves out deleted ones.
  787. """
  788. from backend.app.models.archive import PrintArchive
  789. query = select(PrintArchive).where(
  790. PrintArchive.printer_id == printer_id,
  791. PrintArchive.status == "printing",
  792. or_(PrintArchive.subtask_id.is_(None), PrintArchive.subtask_id.in_(("", "0"))),
  793. )
  794. if dispatched_only:
  795. query = query.where(or_(PrintArchive.billing_run_id.is_not(None), PrintArchive.created_by_id.is_not(None)))
  796. if live_only:
  797. query = query.where(PrintArchive.deleted_at.is_(None))
  798. archives = await db.execute(query.order_by(PrintArchive.created_at.desc()))
  799. return next((a for a in archives.scalars() if _archive_name_forms(a) & reported), None)
  800. async def running_print_archive(db, printer_id: int, state: PrinterState, *, printing_only: bool, by_name: bool):
  801. """The archive of the print running on ``printer_id``, or None (#3327).
  802. By the print's subtask_id first, which the firmware mints per print; with
  803. ``printing_only`` only an archive that is still printing (a leftover row of
  804. a completion never seen must not stand in for another job). A print
  805. started outside the cloud (Bambu Studio over LAN, Handy, the printer's
  806. screen) reports subtask_id "0", which links it to nothing: the archive
  807. Bambuddy bound to the running print comes next, the one completion charges
  808. and photographs. A restart forgets that binding and rebinds only Bambuddy's
  809. own prints, so with ``by_name`` a "0" print last falls back to its printing
  810. archive under the name the printer reports. That can be an earlier run of
  811. the same name whose completion was missed while Bambuddy was down: good
  812. enough for a plate label or a cover, not for which objects skip-objects
  813. offers, so those callers leave it off. None until the archive exists, and
  814. never a deleted one.
  815. """
  816. from backend.app.models.archive import PrintArchive
  817. from backend.app.services.print_log import open_run_on_printer
  818. subtask_id = str(getattr(state, "subtask_id", "") or "").strip()
  819. if subtask_id not in ("", "0"):
  820. # The run open on the printer for this job: the archive row is shared
  821. # by every plate of it, its printer and job id the latest start's.
  822. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  823. if run is not None and run.archive_id is not None:
  824. archive = await db.scalar(
  825. select(PrintArchive).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  826. )
  827. if archive is not None:
  828. return archive
  829. query = select(PrintArchive).where(
  830. PrintArchive.printer_id == printer_id,
  831. PrintArchive.subtask_id == subtask_id,
  832. PrintArchive.deleted_at.is_(None),
  833. )
  834. if printing_only:
  835. query = query.where(PrintArchive.status == "printing")
  836. archive = await db.scalar(query.order_by(PrintArchive.created_at.desc()).limit(1))
  837. if archive is not None:
  838. return archive
  839. bound = next(
  840. (_active_prints[key] for key in _build_status_print_keys(printer_id, state) if key in _active_prints),
  841. None,
  842. )
  843. if bound is not None:
  844. return await db.scalar(select(PrintArchive).where(PrintArchive.id == bound, PrintArchive.deleted_at.is_(None)))
  845. if not by_name or subtask_id != "0":
  846. return None
  847. reported = _reported_print_names(printer_id, state)
  848. if not reported:
  849. return None
  850. return await _printing_archive_without_job_id(db, printer_id, reported, dispatched_only=False, live_only=True)
  851. def _bind_active_print(
  852. printer_id: int,
  853. archive_id: int,
  854. archive_filename: str | None,
  855. filename: str | None = None,
  856. subtask_name: str | None = None,
  857. ) -> None:
  858. """Track a running print under every name its completion may report.
  859. One set for every path that binds a print (new archive, expected print,
  860. restart resume, fallback): the archive's file name, the name the printer
  861. reported, and the subtask with and without ``.3mf``. Paths that registered
  862. fewer names lost the bound archive at completion, and with it the plate
  863. restore's height (#3240), whenever MQTT reported a name they had left out.
  864. """
  865. for name in {archive_filename, filename, f"{subtask_name}.3mf" if subtask_name else None, subtask_name}:
  866. if name:
  867. _active_prints[(printer_id, name)] = archive_id
  868. async def _archive_of_open_run(printer_id: int, subtask_id: str | None = None) -> int | None:
  869. """The archive of the run open on the printer, if it is still there.
  870. With ``subtask_id``, only a run of that job."""
  871. try:
  872. from backend.app.models.archive import PrintArchive
  873. from backend.app.services.print_log import open_run_on_printer
  874. async with async_session() as db:
  875. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  876. if run is None or run.archive_id is None:
  877. return None
  878. return await db.scalar(
  879. select(PrintArchive.id).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  880. )
  881. except Exception as e:
  882. logging.getLogger(__name__).warning("[RUN] Could not read the open run of printer %s: %s", printer_id, e)
  883. return None
  884. async def _announce_discarded_runs(discarded, printer_id: int) -> None:
  885. """Publish the runs ``discard_open_runs`` dropped (archive redesign):
  886. each announced by run/started, now gone without an end."""
  887. if not discarded:
  888. return
  889. try:
  890. from backend.app.models.printer import Printer
  891. async with async_session() as db:
  892. printer = (
  893. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  894. ).first()
  895. if printer is not None:
  896. for run in discarded:
  897. await mqtt_relay.on_run_discarded(run, printer.serial_number, printer.name)
  898. except Exception as e:
  899. logging.getLogger(__name__).debug("[RUN] Could not announce discarded runs on printer %s: %s", printer_id, e)
  900. async def _publish_run_completed(run_id: int | None, printer_id: int) -> None:
  901. """Publish the run a print's end just wrote on the MQTT relay (archive
  902. redesign); nothing without one. Never in the way of the completion."""
  903. if run_id is None:
  904. return
  905. try:
  906. from backend.app.models.archive import PrintArchive
  907. from backend.app.models.print_log import PrintLogEntry
  908. from backend.app.models.printer import Printer
  909. async with async_session() as db:
  910. run = await db.get(PrintLogEntry, run_id)
  911. printer = (
  912. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  913. ).first()
  914. archive = await db.get(PrintArchive, run.archive_id) if run is not None and run.archive_id else None
  915. if run is not None and printer is not None:
  916. await mqtt_relay.on_run_completed(run, archive, printer.serial_number, printer.name)
  917. except Exception as e:
  918. logging.getLogger(__name__).debug("[RUN] Could not publish the end of run %s: %s", run_id, e)
  919. async def _open_run_for(archive_id: int, printer_id: int, subtask_id: str | None = None) -> None:
  920. """Write the run of the print that just started on the archive (archive
  921. redesign). In a session of its own and never in the way of the print
  922. start: until its readers move over, the archive still carries the print.
  923. The archive's start must be committed before. ``subtask_id`` is the
  924. printer's job, which finds the run again at a resume."""
  925. try:
  926. from backend.app.models.archive import PrintArchive
  927. from backend.app.models.printer import Printer
  928. from backend.app.services.print_log import open_run, open_run_on_printer, run_snapshot
  929. async with async_session() as db:
  930. archive = await db.get(PrintArchive, archive_id)
  931. if archive is None:
  932. return
  933. printer = (
  934. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  935. ).first()
  936. printer_name = printer.name if printer is not None else None
  937. already = await open_run_on_printer(db, printer_id)
  938. before = run_snapshot(already) if already is not None else None
  939. run = await open_run(
  940. db,
  941. archive,
  942. printer_id=printer_id,
  943. printer_name=printer_name,
  944. plate_id=_print_plate_ids.get((printer_id, archive_id)),
  945. subtask_id=subtask_id,
  946. )
  947. await db.commit()
  948. # Once per print: a resume or a repeated start finds the run it opened.
  949. # A new one replaced the run still open there (open_run): that one's
  950. # print ended unseen.
  951. # Compared by start too: SQLite can give the new run the id of the one
  952. # it replaced.
  953. replaced = (
  954. run is not None and before is not None and (before.id != run.id or before.started_at != run.started_at)
  955. )
  956. if run is not None and printer is not None and (before is None or replaced):
  957. if replaced:
  958. await _announce_discarded_runs([before], printer_id)
  959. try:
  960. await mqtt_relay.on_run_started(run, archive, printer.serial_number, printer_name)
  961. except Exception:
  962. pass # Don't fail the print start if MQTT fails
  963. except Exception as e:
  964. logging.getLogger(__name__).warning("[RUN] Could not open the run of archive %s: %s", archive_id, e)
  965. async def _drop_replaced_plate_videos(
  966. db, archive_id: int, plate_id: int | None, stale_relpath: str | None, logger: logging.Logger
  967. ) -> None:
  968. """Delete the videos a reprint of a plate replaces: ``stale_relpath``, the
  969. one the archive showed (when print start judged it this plate's), and
  970. those of the archive's earlier runs of the same plate. A plate keeps one
  971. video, as when the archive held the only one; another plate's stays. The
  972. runs that held them lose them. Commits."""
  973. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, PrintLogEntry
  974. from backend.app.utils.safe_path import PathTraversalError, safe_join_under
  975. runs = select(PrintLogEntry).where(
  976. PrintLogEntry.archive_id == archive_id, PrintLogEntry.timelapse_path.is_not(None)
  977. )
  978. with_video = list(await db.scalars(runs.execution_options(**{INCLUDE_OPEN_RUNS: True})))
  979. paths = {stale_relpath} if stale_relpath else set()
  980. if plate_id is not None:
  981. paths |= {r.timelapse_path for r in with_video if r.plate_id == plate_id}
  982. # A video another plate's run holds is that plate's.
  983. paths -= {r.timelapse_path for r in with_video if r.plate_id is not None and r.plate_id != plate_id}
  984. if not paths:
  985. return
  986. for run in with_video:
  987. if run.timelapse_path in paths:
  988. run.timelapse_path = None
  989. await db.commit()
  990. for relpath in sorted(p for p in paths if p):
  991. try:
  992. stale_path = safe_join_under(app_settings.base_dir, relpath, http=False)
  993. if stale_path.is_file():
  994. stale_path.unlink()
  995. logger.info("Deleted stale timelapse %s on reprint of archive %s", relpath, archive_id)
  996. except (OSError, PathTraversalError) as e:
  997. logger.warning("Failed to delete stale timelapse %s on reprint: %s", relpath, e)
  998. def _mark_owned_if_dispatched(archive) -> None:
  999. """Remember a running print's archive as Bambuddy's when it carries a
  1000. dispatch marker Bambuddy writes itself: ``billing_run_id`` (minted per
  1001. dispatch in the scheduler) or ``created_by_id`` (from the queue item)."""
  1002. if archive.billing_run_id is not None or archive.created_by_id is not None:
  1003. _owned_print_archive_ids.add(archive.id)
  1004. async def _is_printer_kill_switch_enabled_cached() -> bool:
  1005. """Return the kill-switch setting without querying on every MQTT frame."""
  1006. global _kill_switch_setting_cache
  1007. now = time.monotonic()
  1008. if _kill_switch_setting_cache is not None:
  1009. enabled, expires_at = _kill_switch_setting_cache
  1010. if now < expires_at:
  1011. return enabled
  1012. async with async_session() as db:
  1013. from backend.app.services.finance_budget import is_printer_kill_switch_enabled
  1014. enabled = await is_printer_kill_switch_enabled(db)
  1015. _kill_switch_setting_cache = (enabled, now + _KILL_SWITCH_SETTING_CACHE_TTL_SECONDS)
  1016. return enabled
  1017. async def _is_bambuddy_authorized_print(printer_id: int, state: PrinterState, db) -> bool | None:
  1018. """Resolve whether the current print was started by Bambuddy.
  1019. ``None`` means identity is not yet safe to decide. The kill switch must
  1020. defer in that case: stopping a print is irreversible, and the first status
  1021. frames after a restart may arrive before all subtask fields are populated.
  1022. """
  1023. if _is_bambuddy_authorized_print_in_memory(printer_id, state):
  1024. return True
  1025. # The printer's built-in calibration runs (Bambuddy's Calibrate, the
  1026. # printer's screen) play gcode from its read-only system partition, where no
  1027. # user file can be. Only that path counts: a calibration *name* proves
  1028. # nothing, since Bambu Studio's pressure-advance print reports a user-style
  1029. # path, and any print can be given that name.
  1030. if str(getattr(state, "gcode_file", "") or "").startswith("/usr/"):
  1031. return True
  1032. # A1, A1 Mini, P1 and A2L report the same calibration as a bare file name,
  1033. # which nothing distinguishes from a user's file given that name, so it is
  1034. # not exempt: with the kill switch on, calibrate on those printers with the
  1035. # switch off.
  1036. possible_keys = _build_status_print_keys(printer_id, state)
  1037. # In-memory ownership is lost on every Bambuddy restart, so fall back to what
  1038. # is on disk. subtask_id is minted per print and pins the answer to the job
  1039. # actually running, rather than to an unrelated one that reuses a filename.
  1040. raw_subtask_id = getattr(state, "subtask_id", None)
  1041. subtask_id = str(raw_subtask_id).strip() if raw_subtask_id is not None else ""
  1042. if subtask_id == "":
  1043. # Not reported yet: the first frames after a restart can lack it.
  1044. return None
  1045. from backend.app.models.archive import PrintArchive
  1046. if subtask_id == "0":
  1047. # A print started outside the cloud from Bambu Studio, Handy or the
  1048. # printer's screen reports "0", so there is no id to match: one of
  1049. # Bambuddy's printing archives on this printer under the name the
  1050. # printer reports, compared as the printer echoes names (spaces become
  1051. # underscores). Nothing reported to compare yet: wait.
  1052. reported = _reported_print_names(printer_id, state)
  1053. if not reported:
  1054. return None
  1055. archive = await _printing_archive_without_job_id(db, printer_id, reported, dispatched_only=True)
  1056. else:
  1057. from backend.app.services.print_log import open_run_on_printer
  1058. # This job's run open on the printer: the archive is shared by every
  1059. # plate of it, and its status, printer and job id are the latest
  1060. # start's, which may be another printer's.
  1061. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  1062. archive = await db.get(PrintArchive, run.archive_id) if run is not None and run.archive_id else None
  1063. if archive is None:
  1064. result = await db.execute(
  1065. select(PrintArchive)
  1066. .where(
  1067. PrintArchive.printer_id == printer_id,
  1068. PrintArchive.status == "printing",
  1069. PrintArchive.subtask_id == subtask_id,
  1070. )
  1071. .order_by(PrintArchive.created_at.desc())
  1072. .limit(1)
  1073. )
  1074. archive = result.scalar_one_or_none()
  1075. # An archive row on its own proves nothing: `on_print_start` archives every
  1076. # print it observes, including ones started from Bambu Studio or Handy, and
  1077. # stamps them with the same status and subtask_id. Authorizing on its mere
  1078. # existence would disable the kill switch the moment the 3MF finishes
  1079. # downloading. Only a dispatch marker Bambuddy writes itself counts —
  1080. # `billing_run_id` (minted per dispatch in the scheduler) or `created_by_id`
  1081. # (carried over from the queue item that started it).
  1082. if archive is not None and (archive.billing_run_id is not None or archive.created_by_id is not None):
  1083. # Rehydrate the fast in-memory path for subsequent status frames. Include
  1084. # both the archive filename and every normalized key reported by MQTT.
  1085. _owned_print_archive_ids.add(archive.id)
  1086. _active_prints[(printer_id, archive.filename)] = archive.id
  1087. for key in possible_keys:
  1088. _active_prints[key] = archive.id
  1089. return True
  1090. # No dispatch marker. Before calling this someone else's print, check whether
  1091. # Bambuddy has a job of its own running on this printer: a library-file
  1092. # dispatch has no archive at send time, and an archive created seconds later
  1093. # by `on_print_start` carries neither marker. The queue row, which the
  1094. # scheduler commits to status="printing" before the MQTT send, is the one
  1095. # durable record every Bambuddy print has. It cannot be tied to this
  1096. # subtask_id, so it is grounds to defer, never to authorize — stopping a
  1097. # print is irreversible, and refusing to act costs nothing but a log line.
  1098. from backend.app.models.print_queue import PrintQueueItem
  1099. dispatched_here = await db.scalar(
  1100. select(PrintQueueItem.id)
  1101. .where(
  1102. PrintQueueItem.printer_id == printer_id,
  1103. PrintQueueItem.status == "printing",
  1104. )
  1105. .limit(1)
  1106. )
  1107. if dispatched_here is not None:
  1108. return None
  1109. return False
  1110. async def _send_kill_switch_provider_notification(
  1111. printer_id: int,
  1112. printer_name: str,
  1113. data: dict,
  1114. ) -> bool:
  1115. """Send the immediate print-stopped provider notification.
  1116. Returning a success flag lets the normal MQTT completion path retry when
  1117. this early notification could not be delivered.
  1118. """
  1119. logger = logging.getLogger(__name__)
  1120. try:
  1121. async with async_session() as db:
  1122. from backend.app.services.print_log import open_run_on_printer
  1123. # The print it stopped, for {archive_id} / {run_id}: the completion
  1124. # that would name them skips its notification after this one.
  1125. run = await open_run_on_printer(db, printer_id)
  1126. await notification_service.on_print_complete(
  1127. printer_id,
  1128. printer_name,
  1129. "stopped",
  1130. data,
  1131. db,
  1132. archive_data={"archive_id": run.archive_id, "run_id": run.id} if run is not None else None,
  1133. )
  1134. return True
  1135. except Exception as e:
  1136. logger.warning(
  1137. "[KILL SWITCH] Immediate provider notification failed for printer %s: %s",
  1138. printer_id,
  1139. e,
  1140. )
  1141. return False
  1142. async def _kill_switch_notification_already_sent(task: asyncio.Task[bool] | None) -> bool:
  1143. """Wait for an immediate kill-switch notification, if one was scheduled."""
  1144. if task is None:
  1145. return False
  1146. try:
  1147. return await task
  1148. except Exception as e:
  1149. logging.getLogger(__name__).warning("[KILL SWITCH] Notification task failed: %s", e)
  1150. return False
  1151. async def _run_needs_energy_start(archive, printer_id: int, db) -> bool:
  1152. """At a resume: whether the print's run still lacks its energy start
  1153. reading. The archive's may be an earlier print's. Commits a reading
  1154. found on the archive for this run onto it."""
  1155. try:
  1156. from backend.app.services.print_log import run_needs_energy_start
  1157. needs = await run_needs_energy_start(db, archive, printer_id=printer_id)
  1158. await db.commit()
  1159. return needs
  1160. except Exception as e:
  1161. logging.getLogger(__name__).warning("[RUN] Energy start of archive %s's run not checked: %s", archive.id, e)
  1162. return archive.energy_start_kwh is None
  1163. async def _record_energy_start(archive, printer_id: int, db, *, context: str = "") -> bool:
  1164. """Capture the smart plug lifetime counter on the archive at print start.
  1165. Persists `energy_start_kwh` on the archive row (#941) so per-print energy
  1166. tracking survives a backend restart mid-print. The print-end handler reads
  1167. this value back from the DB and computes the delta against the current
  1168. plug counter.
  1169. """
  1170. _logger = logging.getLogger(__name__)
  1171. try:
  1172. candidates = await energy_plug_candidates(db, printer_id)
  1173. if not candidates:
  1174. _logger.info("[ENERGY] No smart plug for printer %s (archive %s)", printer_id, archive.id)
  1175. return False
  1176. selected = await select_energy_reading(candidates, _get_plug_energy, db)
  1177. if selected is None:
  1178. # Naming the plugs matters here: with several linked to one printer
  1179. # this is the difference between "the meter is offline" and "you
  1180. # linked only accessories" (#2859).
  1181. _logger.warning(
  1182. "[ENERGY] No plug on printer %s reports a lifetime energy counter for archive %s (tried: %s)",
  1183. printer_id,
  1184. archive.id,
  1185. ", ".join(plug.name for plug in candidates),
  1186. )
  1187. return False
  1188. plug, energy = selected
  1189. archive.energy_start_kwh = float(energy["total"])
  1190. # With the plug's snapshots, what lets the cost follow the price over
  1191. # the print rather than take the price at its end (#1251). A price that
  1192. # can't be had must not cost the print its energy reading.
  1193. archive.energy_start_at = utcnow_naive()
  1194. archive.energy_start_plug_id = plug.id
  1195. try:
  1196. from backend.app.services.energy_price import current_price
  1197. archive.energy_start_price = await current_price(db)
  1198. except Exception as e:
  1199. _logger.warning("[ENERGY] Could not read the electricity price for archive %s: %s", archive.id, e)
  1200. archive.energy_start_price = None
  1201. try:
  1202. from backend.app.services.print_log import mirror_energy_start
  1203. await mirror_energy_start(db, archive, printer_id=printer_id)
  1204. except Exception as e:
  1205. _logger.warning("[RUN] Energy start not copied onto the run of archive %s: %s", archive.id, e)
  1206. await db.commit()
  1207. _logger.info(
  1208. "[ENERGY] Recorded starting energy%s for archive %s from plug '%s': %s kWh",
  1209. f" ({context})" if context else "",
  1210. archive.id,
  1211. plug.name,
  1212. energy["total"],
  1213. )
  1214. return True
  1215. except Exception as e:
  1216. _logger.warning("[ENERGY] Failed to record starting energy for archive %s: %s", archive.id, e)
  1217. return False
  1218. def forget_print_creator(user_id: int) -> None:
  1219. """The print-start emails of a deleted user's jobs go to nobody, not to
  1220. whoever is given the id next (SQLite reuses it)."""
  1221. for key in [key for key, creator in _expected_print_creators.items() if creator == user_id]:
  1222. del _expected_print_creators[key]
  1223. async def forget_printer(printer_id: int) -> None:
  1224. """Drop what is kept for a deleted printer's id, here and in the services.
  1225. SQLite used to give the id to the next printer created, which inherited it
  1226. all: the running print's owner (its prints passed the kill switch and were
  1227. billed to that user), the plate-clear gate, pending and suppressed
  1228. notifications, the camera feed, the plate references. Ids are no longer
  1229. reused (AUTOINCREMENT), but nothing kept for a printer that is gone should
  1230. stay. Called after the delete is committed, and again by work that was in
  1231. flight and wrote state back.
  1232. Every printer-keyed map of this module is listed here or, with the reason,
  1233. in tests/unit/test_forget_printer.py.
  1234. """
  1235. from backend.app.api.routes import camera
  1236. from backend.app.services import (
  1237. bambu_ftp,
  1238. kprofile_drift,
  1239. layer_timelapse,
  1240. plate_detection,
  1241. print_dispatch_context,
  1242. usage_tracker,
  1243. )
  1244. from backend.app.services.notify_live_activities import notify_live_activities
  1245. from backend.app.services.obico_detection import obico_detection_service
  1246. from backend.app.services.printer_file_cache import printer_file_cache
  1247. # Keyed (printer_id, filename). The archive-keyed maps of those prints go too.
  1248. archive_ids = set()
  1249. for tracked in (_active_prints, _expected_prints):
  1250. for key in [key for key in tracked if key[0] == printer_id]:
  1251. archive_ids.add(tracked.pop(key))
  1252. for archive_id in archive_ids:
  1253. _owned_print_archive_ids.discard(archive_id)
  1254. for registered in (_print_ams_mappings, _print_cost_center_ids, _print_plate_ids):
  1255. for key in [key for key in registered if key[0] == printer_id]:
  1256. del registered[key]
  1257. for tracked in (_expected_print_creators, _expected_print_registered_at):
  1258. for key in [key for key in tracked if key[0] == printer_id]:
  1259. del tracked[key]
  1260. for key in [key for key in _ams_index_only_logged if key[0] == printer_id]:
  1261. _ams_index_only_logged.discard(key)
  1262. for key in [key for key in _ams_alarm_cooldown if key.split(":", 1)[0] == str(printer_id)]:
  1263. del _ams_alarm_cooldown[key]
  1264. # Pending work that would act on the id: an offline notice, a 3MF retry.
  1265. # A kill-switch notice is about a stop that happened and is left to finish.
  1266. for tasks in (_printer_offline_notify_tasks, _fallback_3mf_retry_tasks):
  1267. task = tasks.pop(printer_id, None)
  1268. if task is not None:
  1269. task.cancel()
  1270. for keyed in (
  1271. _stage22_finish_frames,
  1272. _stage22_finish_in_flight,
  1273. _inprint_frame_bank,
  1274. _inprint_frame_bank_ts,
  1275. _printer_reconciled_since_connect,
  1276. _printer_kprofiles_primed_since_connect,
  1277. _last_progress_milestone,
  1278. _first_layer_notified,
  1279. _kill_switch_notification_tasks,
  1280. _notified_hms_errors,
  1281. _hms_print_state,
  1282. _hms_forget_when_gone,
  1283. _timelapse_baselines,
  1284. _bed_cool_waiters,
  1285. _printer_last_connected,
  1286. _unknown_tag_last_broadcast,
  1287. _last_status_broadcast,
  1288. _connection_watchdog_last_attempt,
  1289. ):
  1290. keyed.pop(printer_id, None)
  1291. for marked in (
  1292. _unauthorized_print_kill_sent,
  1293. _user_stopped_printers,
  1294. _kill_switch_stopped,
  1295. _printer_offline_published,
  1296. _nozzle_count_updated,
  1297. _runtime_seen_connected,
  1298. _timelapse_baselines_unread,
  1299. ):
  1300. marked.discard(printer_id)
  1301. printer_manager.forget_printer(printer_id)
  1302. printer_file_cache.forget_printer(printer_id)
  1303. print_dispatch_context.clear(printer_id)
  1304. kprofile_drift.forget_printer(printer_id, deleted=True)
  1305. layer_timelapse.cancel_session(printer_id)
  1306. usage_tracker.forget_printer(printer_id)
  1307. obico_detection_service.forget_printer(printer_id)
  1308. octoeverywhere_detection_service.forget_printer(printer_id)
  1309. bambu_ftp.clear_3mf_cache(printer_id)
  1310. await camera.forget_printer(printer_id)
  1311. printers.clear_cover_cache(printer_id)
  1312. print_scheduler.forget_printer(printer_id)
  1313. notify_live_activities.forget_printer(printer_id)
  1314. plate_detection.remove_printer_references(printer_id)
  1315. def register_expected_print(
  1316. printer_id: int,
  1317. filename: str,
  1318. archive_id: int,
  1319. ams_mapping: list[int] | None = None,
  1320. created_by_id: int | None = None,
  1321. cost_center_id: int | None = None,
  1322. plate_id: int | None = None,
  1323. ):
  1324. """Register an expected print from reprint/scheduled so we don't create duplicate archives."""
  1325. # Store with multiple filename variations to catch different naming patterns
  1326. _expected_prints[(printer_id, filename)] = archive_id
  1327. # Also store without .3mf extension if present
  1328. if filename.endswith(".3mf"):
  1329. base = filename[:-4]
  1330. _expected_prints[(printer_id, base)] = archive_id
  1331. _expected_prints[(printer_id, f"{base}.gcode")] = archive_id
  1332. # Store AMS mapping for usage tracking at print completion, and plate_id for
  1333. # a single-plate dispatch from a multi-plate 3MF — without it, the direct-Print
  1334. # path attributes the whole file's filament total to the spool instead of just
  1335. # the printed plate (#1697). They describe this dispatch: one that has none
  1336. # drops what an earlier dispatch of the archive left (sent, never started),
  1337. # unless a plate of the archive is printing now and still needs them.
  1338. dispatch = (printer_id, archive_id)
  1339. archive_printing = _printing_here(printer_id, archive_id)
  1340. if ams_mapping is not None:
  1341. _print_ams_mappings[dispatch] = ams_mapping
  1342. elif not archive_printing:
  1343. _print_ams_mappings.pop(dispatch, None)
  1344. if cost_center_id is not None:
  1345. _print_cost_center_ids[dispatch] = cost_center_id
  1346. elif not archive_printing:
  1347. _print_cost_center_ids.pop(dispatch, None)
  1348. if plate_id is not None:
  1349. _print_plate_ids[dispatch] = plate_id
  1350. elif not archive_printing:
  1351. _print_plate_ids.pop(dispatch, None)
  1352. # Store created_by_id so the user start email can be sent even when the archive
  1353. # itself has no created_by_id (e.g. library-file-based queue prints)
  1354. if created_by_id is not None:
  1355. _expected_print_creators[(printer_id, filename)] = created_by_id
  1356. if filename.endswith(".3mf"):
  1357. base = filename[:-4]
  1358. _expected_print_creators[(printer_id, base)] = created_by_id
  1359. _expected_print_creators[(printer_id, f"{base}.gcode")] = created_by_id
  1360. # Record registration time for TTL-based eviction
  1361. _registered_at = time.monotonic()
  1362. _expected_print_registered_at[(printer_id, filename)] = _registered_at
  1363. if filename.endswith(".3mf"):
  1364. base = filename[:-4]
  1365. _expected_print_registered_at[(printer_id, base)] = _registered_at
  1366. _expected_print_registered_at[(printer_id, f"{base}.gcode")] = _registered_at
  1367. logging.getLogger(__name__).info(
  1368. f"Registered expected print: printer={printer_id}, file={filename}, archive={archive_id}, ams_mapping={ams_mapping}, plate_id={plate_id}"
  1369. )
  1370. def unregister_expected_print(printer_id: int, filename: str, archive_id: int) -> None:
  1371. """Undo :func:`register_expected_print` when the print never went out.
  1372. Registration has to happen *before* the MQTT print command, because the
  1373. printer can report the print before the line after the send executes. So
  1374. every path that registers and then fails to send — a cancel winning the
  1375. #1853 CAS race, a ``start_print()`` that returns False, or any exception in
  1376. between — leaves an expectation for a print that will never arrive.
  1377. The TTL sweep evicts those after two hours, which is far longer than it
  1378. takes a user to react to a failed dispatch by pressing print again: that
  1379. reprint would be folded into the *old* archive and take the stale
  1380. ``ams_mapping`` / ``plate_id`` with it. Hence the explicit inverse.
  1381. Mirrors the sweep's rules, including the one that is easy to get wrong:
  1382. ``_print_ams_mappings`` / ``_print_plate_ids`` / ``_print_cost_center_ids``
  1383. are keyed by printer and archive, not by file, so they may only be dropped
  1384. once no live key of the printer still points at that archive.
  1385. """
  1386. keys = [(printer_id, filename)]
  1387. if filename.endswith(".3mf"):
  1388. base = filename[:-4]
  1389. keys.append((printer_id, base))
  1390. keys.append((printer_id, f"{base}.gcode"))
  1391. removed = False
  1392. for key in keys:
  1393. if _expected_prints.pop(key, None) is not None:
  1394. removed = True
  1395. _expected_print_creators.pop(key, None)
  1396. _expected_print_registered_at.pop(key, None)
  1397. # Still needed by another expected print of the archive on the printer,
  1398. # or a plate of it printing there, as in the sweep.
  1399. if not _expected_or_printing_here(printer_id, archive_id):
  1400. _drop_dispatch(printer_id, archive_id)
  1401. if removed:
  1402. logging.getLogger(__name__).info(
  1403. "Unregistered expected print: printer=%s, file=%s, archive=%s (print was never sent)",
  1404. printer_id,
  1405. filename,
  1406. archive_id,
  1407. )
  1408. def _compute_run_filament_grams(
  1409. status: str,
  1410. archive_filament_used_grams: float | None,
  1411. progress: float | int | None,
  1412. usage_results: list[dict] | None,
  1413. ) -> float | None:
  1414. """Per-run filament for PrintLogEntry, partial- and tracker-aware (#1378, #1390).
  1415. Priority for every status:
  1416. 1. Sum of tracked spool deltas in ``usage_results`` (AMS-measured
  1417. weight delta — same source that drives "Total Consumed" on the
  1418. Inventory page, so Stats and Inventory totals stay aligned).
  1419. 2. For ``completed``: the slicer estimate (no tracker available, fall
  1420. back to the canonical "this print used X" value).
  1421. 3. For partial statuses: ``estimate * progress%``.
  1422. 4. ``None`` if nothing is known.
  1423. """
  1424. tracked_grams = sum(r.get("weight_used") or 0 for r in (usage_results or []))
  1425. if tracked_grams > 0:
  1426. return round(tracked_grams, 1)
  1427. if status == "completed":
  1428. return archive_filament_used_grams
  1429. if archive_filament_used_grams:
  1430. scale = max(0.0, min(((progress or 0) / 100.0), 1.0))
  1431. if scale > 0:
  1432. return round(archive_filament_used_grams * scale, 1)
  1433. return None
  1434. def _printing_here(printer_id: int, archive_id: int) -> bool:
  1435. """Whether a plate of the archive is printing on the printer."""
  1436. return any(key[0] == printer_id and bound == archive_id for key, bound in _active_prints.items())
  1437. def _expected_or_printing_here(printer_id: int, archive_id: int) -> bool:
  1438. """Whether a print of the archive is expected or printing on the printer."""
  1439. return _printing_here(printer_id, archive_id) or any(
  1440. key[0] == printer_id and expected == archive_id for key, expected in _expected_prints.items()
  1441. )
  1442. def _drop_dispatch(printer_id: int, archive_id: int) -> None:
  1443. """Forget what the dispatch of the archive to the printer registered."""
  1444. dispatch = (printer_id, archive_id)
  1445. _print_ams_mappings.pop(dispatch, None)
  1446. _print_cost_center_ids.pop(dispatch, None)
  1447. _print_plate_ids.pop(dispatch, None)
  1448. def _get_start_ams_mapping(data: dict, printer_id: int, archive_id: int | None) -> list[int] | None:
  1449. """Resolve AMS mapping for print start without consuming stored queue/reprint state."""
  1450. stored_ams_mapping = data.get("ams_mapping")
  1451. if not stored_ams_mapping and archive_id:
  1452. stored_ams_mapping = _print_ams_mappings.get((printer_id, archive_id))
  1453. return stored_ams_mapping
  1454. def _get_start_plate_id(printer_id: int, archive_id: int | None) -> int | None:
  1455. """Resolve plate_id for print start without consuming stored direct-Print state.
  1456. Direct-Print of a single plate from a multi-plate 3MF registers plate_id in
  1457. ``_print_plate_ids`` at dispatch time; this lets the spoolman / usage tracker
  1458. read it back at print-start without popping (the entry is popped on print
  1459. completion or TTL eviction, mirroring ``_print_ams_mappings``).
  1460. """
  1461. if archive_id is None:
  1462. return None
  1463. return _print_plate_ids.get((printer_id, archive_id))
  1464. def _run_print_time_seconds(archive, printer_id: int) -> int | None:
  1465. """The slicer's time for the plate this print prints, for its start
  1466. notification (#3375): ``archive.print_time_seconds`` adds every plate of
  1467. the file up (#1593). The plate is the dispatch's, else the archive's. The
  1468. whole file's time when its plates are not known; None for a plate not
  1469. known of several, and the notification takes the printer's remaining
  1470. time."""
  1471. from backend.app.services.print_cost_estimate import run_estimate
  1472. plate_id = _print_plate_ids.get((printer_id, archive.id), archive.plate_id)
  1473. estimate = run_estimate(archive, plate_id)
  1474. return estimate.seconds if estimate is not None else None
  1475. _plate_estimates_task: asyncio.Task | None = None
  1476. async def _fill_plate_estimates() -> None:
  1477. from backend.app.services.print_cost_estimate import fill_plate_estimates
  1478. try:
  1479. filled = await fill_plate_estimates(async_session)
  1480. except asyncio.CancelledError:
  1481. raise
  1482. except Exception as exc:
  1483. logging.getLogger(__name__).warning("[ARCHIVE] Reading plate estimates failed: %s", exc)
  1484. return
  1485. if filled:
  1486. logging.getLogger(__name__).info("[ARCHIVE] Read the plate estimates of %d archive(s)", filled)
  1487. _gcode_hashes_task: asyncio.Task | None = None
  1488. async def _fill_gcode_hashes() -> None:
  1489. from backend.app.services.archive_gcode_hash import fill_gcode_hashes
  1490. try:
  1491. filled = await fill_gcode_hashes(async_session)
  1492. except asyncio.CancelledError:
  1493. raise
  1494. except Exception as exc:
  1495. logging.getLogger(__name__).warning("[ARCHIVE] Reading G-code fingerprints failed: %s", exc)
  1496. return
  1497. if filled:
  1498. logging.getLogger(__name__).info("[ARCHIVE] Read the G-code fingerprints of %d archive(s)", filled)
  1499. def _partial_progress_scale(progress: int | float | None) -> float:
  1500. """Clamp ``progress / 100`` into [0.0, 1.0] for partial-print scaling.
  1501. Used by every site that multiplies a "would-have-used" slicer estimate
  1502. down to "actually-used" for failed / cancelled / stopped prints. Centralised
  1503. so the three sites in ``_background_notifications`` (and the per-plate
  1504. override helper) can't drift apart on the coercion shape.
  1505. """
  1506. return max(0.0, min((progress or 0) / 100.0, 1.0))
  1507. def _scope_notification_archive_data_to_plate(
  1508. archive_data: dict,
  1509. archive_file_path: str | None,
  1510. plate_id: int | None,
  1511. print_status: str,
  1512. progress: int | float | None,
  1513. base_dir: Path,
  1514. ) -> dict:
  1515. """Override summed-across-plates totals in ``archive_data`` with the values
  1516. for ``plate_id`` so the completion notification reports what was actually
  1517. printed, not the whole project (#1785).
  1518. The 3MF parser at services/archive.py:200-264 sums ``prediction`` and
  1519. ``weight`` across every plate of a multi-plate file (#1593) — correct for
  1520. the archive card's "whole project" headline, wrong for the completion
  1521. notification of a single-plate print. The queue UI already re-reads the
  1522. 3MF per-plate at print_queue.py:272-285; this helper mirrors that for the
  1523. notification payload (filament grams, time estimate, per-slot breakdown).
  1524. No-ops when ``plate_id`` is None, the file is missing, or the 3MF carries
  1525. no per-plate values — in every fail case the original ``archive_data`` is
  1526. returned unchanged so the notification still sends.
  1527. """
  1528. if plate_id is None or not archive_file_path:
  1529. return archive_data
  1530. from backend.app.utils.threemf_tools import (
  1531. extract_filament_usage_from_3mf,
  1532. extract_print_time_from_3mf,
  1533. )
  1534. archive_path = base_dir / archive_file_path
  1535. if not archive_path.exists():
  1536. return archive_data
  1537. plate_slots = extract_filament_usage_from_3mf(archive_path, plate_id)
  1538. plate_grams = sum(f.get("used_g", 0) for f in plate_slots)
  1539. plate_time = extract_print_time_from_3mf(archive_path, plate_id)
  1540. scale = 1.0 if print_status == "completed" else _partial_progress_scale(progress)
  1541. if plate_time:
  1542. archive_data["print_time_seconds"] = plate_time
  1543. # Gate both the grams headline AND the per-slot breakdown on the same
  1544. # `plate_grams > 0` signal: if the 3MF carries per-plate filament rows but
  1545. # they all sum to zero (slicer bug / re-slice without estimate), drop back
  1546. # to the project-level grams the archive columns already provide rather
  1547. # than ship a project-level headline next to an all-zero per-plate
  1548. # breakdown.
  1549. if plate_grams > 0:
  1550. archive_data["actual_filament_grams"] = round(plate_grams * scale, 1)
  1551. archive_data["filament_slots"] = [
  1552. {
  1553. "slot_id": s.get("slot_id"),
  1554. "used_g": round((s.get("used_g") or 0) * scale, 1),
  1555. "type": s.get("type", ""),
  1556. "color": s.get("color", ""),
  1557. }
  1558. for s in plate_slots
  1559. ]
  1560. return archive_data
  1561. def _extract_filament_data_from_mqtt(data: dict, ams_mapping: list[int] | None = None) -> dict[str, str]:
  1562. """Best-effort filament metadata from the MQTT print-start snapshot.
  1563. Used when the 3MF can't be downloaded (P1S/A1/P2S firmwares lock the
  1564. file during print, see #1533) so the fallback PrintArchive still has
  1565. enough filament info to support the inventory views and AMS-expansion
  1566. planning the operator opens it for. Returns a dict with optional
  1567. ``filament_type`` and ``filament_color`` keys in the same
  1568. comma-separated format the 3MF extractor produces, so the rest of the
  1569. codebase treats the fallback archive identically to a normal one.
  1570. ``ams_mapping`` is the slicer's slot-per-print-filament list captured
  1571. from the MQTT print payload (global tray IDs, possibly -1 for VT-tray
  1572. entries). When supplied, only the slots actually consumed by this
  1573. print contribute. Without it the function falls back to every loaded
  1574. AMS slot — less accurate but still useful.
  1575. Accepts both the raw inner payload (``{"ams": {"ams": [...]}, ...}``)
  1576. that the unit tests pass directly, AND the on_print_start callback
  1577. shape (``{"raw_data": {"ams": {"ams": [...]}, ...}, ...}``) the
  1578. bambu_mqtt service hands to main.py at runtime. The original
  1579. ``_extract_filament_data_from_mqtt(data)`` shipped in #1533 only
  1580. handled the inner shape and silently returned ``{}`` for every real
  1581. print start, leaving fallback archives' filament fields NULL — the
  1582. exact regression the fix was meant to close. Reported with a log
  1583. proving the AMS state was right there at
  1584. ``data["raw_data"]["ams"]["ams"][0]["tray"][0]`` (#1533 follow-up).
  1585. """
  1586. result: dict[str, str] = {}
  1587. # Look at the on_print_start wrapper first, then the inner shape.
  1588. raw_data = (data or {}).get("raw_data")
  1589. ams_root = (raw_data or {}).get("ams") if isinstance(raw_data, dict) else None
  1590. if not isinstance(ams_root, dict):
  1591. ams_root = (data or {}).get("ams") or {}
  1592. ams_units = ams_root.get("ams") if isinstance(ams_root, dict) else None
  1593. if not isinstance(ams_units, list) or not ams_units:
  1594. return result
  1595. from backend.app.services.print_scheduler import _global_tray_id
  1596. # Map global tray id → (type, color), addressed as the print's ams_mapping
  1597. # is (utils.tray_ids).
  1598. layout = TrayLayout.from_units(ams_units)
  1599. loaded: dict[int, tuple[str, str]] = {}
  1600. for unit in ams_units:
  1601. if not isinstance(unit, dict):
  1602. continue
  1603. try:
  1604. unit_id = int(unit.get("id", 0))
  1605. except (TypeError, ValueError):
  1606. continue
  1607. for tray in unit.get("tray") or []:
  1608. if not isinstance(tray, dict):
  1609. continue
  1610. try:
  1611. tray_id = int(tray.get("id", 0))
  1612. except (TypeError, ValueError):
  1613. continue
  1614. ttype = (tray.get("tray_type") or "").strip()
  1615. # RRGGBBAA from the printer, stored as the 3MF path stores it:
  1616. # "#RRGGBB" (the Archives page's swatches and colour filter).
  1617. hex_color = (tray.get("tray_color") or "").strip().upper()[:6]
  1618. tcolor = f"#{hex_color}" if re.fullmatch(r"[0-9A-F]{6}", hex_color) else ""
  1619. if not ttype:
  1620. continue # Empty / unloaded slot.
  1621. loaded[_global_tray_id(unit_id, tray_id, layout)] = (ttype, tcolor)
  1622. if not loaded:
  1623. return result
  1624. if ams_mapping:
  1625. used_ids = [int(x) for x in ams_mapping if isinstance(x, (int, float)) and int(x) >= 0]
  1626. filaments = [loaded[g] for g in used_ids if g in loaded]
  1627. if not filaments:
  1628. return result # Mapping points entirely at slots we have no data for.
  1629. else:
  1630. filaments = [loaded[g] for g in sorted(loaded.keys())]
  1631. types_joined = ",".join(f[0] for f in filaments)
  1632. colors_joined = ",".join(f[1] for f in filaments if f[1])
  1633. # Column limits per backend/app/models/archive.py: filament_type=50,
  1634. # filament_color=200.
  1635. if types_joined:
  1636. result["filament_type"] = types_joined[:50]
  1637. if colors_joined:
  1638. result["filament_color"] = colors_joined[:200]
  1639. return result
  1640. def _maybe_start_layer_timelapse(printer, printer_id: int, archive_id: int) -> bool:
  1641. """Start a layer-timelapse session for *archive_id* when the printer has
  1642. an external camera configured. Returns True if a session was started.
  1643. Three call sites in on_print_start (expected-archive promotion, fallback
  1644. archive creation, fresh-archive creation) used to inline this same
  1645. if-block; the inline copies kept drifting (#1353 fixed only one of them
  1646. on the first pass). Centralising the conditional + call here makes the
  1647. contract testable in isolation and keeps the three sites locked in step.
  1648. """
  1649. if not (printer.external_camera_enabled and printer.external_camera_url):
  1650. return False
  1651. from backend.app.services.layer_timelapse import start_session
  1652. start_session(
  1653. printer_id,
  1654. archive_id,
  1655. printer.external_camera_url,
  1656. printer.external_camera_type or "mjpeg",
  1657. snapshot_url=printer.external_camera_snapshot_url,
  1658. rotation=getattr(printer, "camera_rotation", 0),
  1659. )
  1660. logging.getLogger(__name__).info("Started layer timelapse for printer %s, archive %s", printer_id, archive_id)
  1661. return True
  1662. def _format_hms_error_summary(hms_errors: list[dict]) -> str | None:
  1663. """Build a human-readable failure reason from MQTT hms_errors for PrintQueueItem.error_message.
  1664. Each entry has keys: code ('0x4038'), attr (32-bit int), module, severity, and
  1665. — since #2926 — the description the parser already resolved, which is preferred
  1666. when present so the queue's failure reason reads the same as the status
  1667. response. The short code still produces the bracketed label, and still
  1668. resolves the sentence for a caller whose entries predate the field. An entry
  1669. with a 16-char ``full_code`` is labelled with it instead, since the short code
  1670. of an ``hms[]`` fault is not a code anyone can look up. Falls back to the bare
  1671. label when no description is on file. Returns None for an empty list so
  1672. callers can leave error_message unset.
  1673. """
  1674. if not hms_errors:
  1675. return None
  1676. from backend.app.services.hms_errors import get_error_description
  1677. parts: list[str] = []
  1678. for err in hms_errors:
  1679. try:
  1680. # `_hms_short_code` rather than a local derivation: this one used to
  1681. # format the error without masking it to 16 bits, so an `hms[]` entry
  1682. # whose code carries an alert-level group produced a five-digit label
  1683. # like "0500_3000A" — not a code the user can look up, and never a
  1684. # catalogue key, so the sentence was lost with it.
  1685. short_code = _hms_short_code(err.get("attr", 0), err.get("code", 0))
  1686. except (TypeError, ValueError):
  1687. continue
  1688. description = err.get("description") or get_error_description(short_code)
  1689. # An `hms[]` fault's short code ("0500_000E") drops the part and level
  1690. # groups and is not a code anyone can look up; its full code in the
  1691. # printer screen's four groups is (#2728).
  1692. full_code = str(err.get("full_code") or "")
  1693. label = "-".join(full_code[i : i + 4] for i in range(0, 16, 4)) if len(full_code) == 16 else short_code
  1694. parts.append(f"[{label}] {description}" if description else f"[{label}]")
  1695. return "; ".join(parts) if parts else None
  1696. async def _bump_library_file_usage_if_completed(db, item, queue_status: str) -> None:
  1697. """Increment LibraryFile.print_count and stamp last_printed_at when a queued
  1698. print completes successfully. Gated to status=='completed': failed, cancelled
  1699. and aborted prints do not count as usage. Caller is responsible for committing
  1700. the session. No-op when the queue item has no linked library file (e.g. reprints
  1701. from an archive). See #1008."""
  1702. if queue_status != "completed" or item.library_file_id is None:
  1703. return
  1704. from backend.app.models.library import LibraryFile
  1705. lib_file = await db.scalar(select(LibraryFile).where(LibraryFile.id == item.library_file_id))
  1706. if lib_file is None:
  1707. return
  1708. lib_file.print_count = (lib_file.print_count or 0) + 1
  1709. lib_file.last_printed_at = datetime.now(timezone.utc)
  1710. def mark_printer_stopped_by_user(printer_id: int) -> None:
  1711. """Mark that the active print on this printer was stopped by the user from the queue UI.
  1712. When on_print_complete fires with status 'failed' for a printer in this set we
  1713. reclassify it as 'cancelled' so the correct 'print stopped' notification is sent
  1714. rather than a 'print failed' notification.
  1715. """
  1716. _user_stopped_printers.add(printer_id)
  1717. logging.getLogger(__name__).info("Marked printer %s as user-stopped from queue", printer_id)
  1718. _last_status_broadcast: dict[int, str] = {}
  1719. # Track printers where we've updated nozzle_count
  1720. _nozzle_count_updated: set[int] = set()
  1721. async def _maybe_notify_printer_offline(printer_id: int) -> None:
  1722. """Wait the debounce window then fire `on_printer_offline` if the printer
  1723. is still offline.
  1724. Scheduled by `on_printer_status_change` on the connected → disconnected
  1725. edge (#1752). Cancelled by the same handler if the printer reconnects
  1726. before the window elapses, so a single MQTT blip + recovery doesn't
  1727. notify. Both the staleness-detector path (`bambu_mqtt.py::check_staleness`)
  1728. and the smart-plug power-off path (`printer_manager.mark_printer_offline`)
  1729. route through the same status-change callback, so this covers both.
  1730. """
  1731. logger = logging.getLogger(__name__)
  1732. try:
  1733. await asyncio.sleep(_PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS)
  1734. still_offline = not printer_manager.is_connected(printer_id)
  1735. logger.info(
  1736. "[#1752] Printer %s offline debounce elapsed: still_offline=%s",
  1737. printer_id,
  1738. still_offline,
  1739. )
  1740. if not still_offline:
  1741. return
  1742. async with async_session() as db:
  1743. from backend.app.models.printer import Printer
  1744. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1745. printer = result.scalar_one_or_none()
  1746. if not printer:
  1747. logger.warning(
  1748. "[#1752] Printer %s missing from DB at offline-notify time; skipping",
  1749. printer_id,
  1750. )
  1751. return
  1752. # Recorded only when the relay can publish it, so the reconnect's
  1753. # online event always has an offline before it.
  1754. if mqtt_relay.enabled and mqtt_relay.connected:
  1755. _printer_offline_published.add(printer_id)
  1756. try:
  1757. await mqtt_relay.on_printer_offline(printer_id, printer.name, printer.serial_number)
  1758. except Exception as e:
  1759. logger.warning("MQTT relay offline event failed for printer %s: %s", printer_id, e)
  1760. logger.info(
  1761. "[#1752] Dispatching on_printer_offline for printer %s (%s)",
  1762. printer_id,
  1763. printer.name,
  1764. )
  1765. await notification_service.on_printer_offline(printer_id, printer.name, db)
  1766. except asyncio.CancelledError:
  1767. raise
  1768. except Exception as e:
  1769. logger.warning("Printer offline notification failed for printer %s: %s", printer_id, e)
  1770. finally:
  1771. _printer_offline_notify_tasks.pop(printer_id, None)
  1772. def _hms_notify_key(error) -> str:
  1773. """What identifies a fault for notification de-duplication.
  1774. The full code, which is unique per fault. ``attr`` alone used to be the key:
  1775. unique for a ``print_error``, but for an ``hms[]`` fault it is only the
  1776. module and part, so two faults on one part (#1840's H2C held 0500-0600-0002-0005
  1777. and -0006 together) shared a key and only the first was ever notified. An
  1778. entry without a full code falls back to attr and code together.
  1779. """
  1780. full_code = getattr(error, "full_code", "") or ""
  1781. if full_code:
  1782. return full_code.upper()
  1783. return f"{error.attr:08X}:{error.code}"
  1784. def _hms_errors_to_notify(errors: list, new_error_codes: set[str]) -> list:
  1785. """The new faults that count (see ``_hms_fault_counts``).
  1786. These go to the MQTT relay; the caller also needs a description before it
  1787. sends a notification. Level 0, Bambu's "invalid" level, never counts. This
  1788. used to read ``severity >= 2`` when severity held the part byte; on the real
  1789. level that would drop the task-stopping errors (#2728).
  1790. """
  1791. return [e for e in errors if _hms_notify_key(e) in new_error_codes and _hms_fault_counts(e)]
  1792. # What each HMS module byte is about, read off Bambu's own texts for its codes
  1793. # (hms_catalog.json). The table used to call 0x05 "Nozzle/Extruder", 0x0C
  1794. # "Motion Controller" and 0x12 "Chamber", which titled USB and MQTT errors as
  1795. # nozzle faults and an A1 mini's AMS Lite runouts as chamber errors (#3486).
  1796. _HMS_MODULE_NAMES = {
  1797. 0x03: "Print/Task", # bed, nozzle temperatures, motors, the print itself
  1798. 0x05: "System", # USB/SD, network, firmware, files, AMS communication
  1799. 0x07: "AMS/Filament",
  1800. 0x08: "Toolhead",
  1801. 0x0C: "Camera", # cameras, lidar, first-layer and AI inspection
  1802. 0x12: "AMS Lite",
  1803. 0x18: "AMS-HT",
  1804. 0x1A: "Tool Changer",
  1805. 0x29: "Air Filter",
  1806. 0x2A: "AMS 3H",
  1807. }
  1808. def _hms_error_title(module: int) -> str:
  1809. """The notification title for a fault of *module*."""
  1810. name = _HMS_MODULE_NAMES.get(module)
  1811. return f"{name} Error" if name else "Printer Error"
  1812. def _take_new_hms_faults(printer_id: int, errors: list, print_state: str | None = None) -> list:
  1813. """The faults on this printer not notified yet, and record them as notified.
  1814. Tracking is updated before anything is sent, so concurrent status callbacks
  1815. cannot notify the same fault twice. A notified fault is forgotten once it
  1816. has been gone for ``_HMS_CLEAR_GRACE_SECONDS``, so one that flickers off
  1817. and on is notified once (#3226), while one that clears and comes back much
  1818. later is notified again.
  1819. When ``print_state`` changes (a print resumed, started or finished), every
  1820. fault recorded so far is forgotten the first time it is gone, without the
  1821. wait: a runout fixed before resuming must be notified if it happens again a
  1822. few minutes later, while the printer sits paused. Not only the faults gone
  1823. at the change itself, because the update that flips the state can still
  1824. carry the old fault (a ``print_error`` entry stays until a payload brings
  1825. ``hms``).
  1826. """
  1827. now = time.time()
  1828. current = {_hms_notify_key(e) for e in errors}
  1829. seen = _notified_hms_errors.setdefault(printer_id, {})
  1830. forget_when_gone = _hms_forget_when_gone.setdefault(printer_id, set())
  1831. if print_state is not None:
  1832. print_state = print_state.upper()
  1833. if _hms_print_state.get(printer_id, print_state) != print_state:
  1834. forget_when_gone.update(seen)
  1835. _hms_print_state[printer_id] = print_state
  1836. for key, last_seen in list(seen.items()):
  1837. if key not in current and (key in forget_when_gone or now - last_seen >= _HMS_CLEAR_GRACE_SECONDS):
  1838. del seen[key]
  1839. forget_when_gone.discard(key)
  1840. new = current - seen.keys()
  1841. for key in current:
  1842. seen[key] = now
  1843. if not seen:
  1844. _notified_hms_errors.pop(printer_id, None)
  1845. if not forget_when_gone:
  1846. _hms_forget_when_gone.pop(printer_id, None)
  1847. return _hms_errors_to_notify(errors, new)
  1848. async def _take_new_hms_faults_and_save(printer_id: int, errors: list, print_state: str | None = None) -> list:
  1849. """``_take_new_hms_faults``, and store the notified set when it changed (#3486).
  1850. The set lives in memory, so without this every restart sent each fault the
  1851. printer still lists once more. A failed write only costs that protection
  1852. for the next restart, so it is logged and the notification goes ahead.
  1853. """
  1854. before = set(_notified_hms_errors.get(printer_id, ()))
  1855. new = _take_new_hms_faults(printer_id, errors, print_state)
  1856. if set(_notified_hms_errors.get(printer_id, ())) != before:
  1857. from backend.app.services.hms_notified import save_notified_hms_faults
  1858. try:
  1859. await save_notified_hms_faults(printer_id, lambda: _notified_hms_errors.get(printer_id, {}).keys())
  1860. except Exception as e:
  1861. logging.getLogger(__name__).warning(
  1862. "Could not store the notified HMS faults of printer %s: %s", printer_id, e
  1863. )
  1864. return new
  1865. async def _load_notified_hms_faults() -> None:
  1866. """Restore the faults notified before the restart (#3486).
  1867. Each counts as seen now: one still listed is not sent again, and one that
  1868. is gone is forgotten after the usual grace period.
  1869. """
  1870. from backend.app.services.hms_notified import load_notified_hms_faults
  1871. try:
  1872. loaded = await load_notified_hms_faults()
  1873. except Exception as e:
  1874. logging.getLogger(__name__).warning("Could not load the notified HMS faults: %s", e)
  1875. return
  1876. now = time.time()
  1877. for printer_id, keys in loaded.items():
  1878. seen = _notified_hms_errors.setdefault(printer_id, {})
  1879. for key in keys:
  1880. seen.setdefault(key, now)
  1881. # `stg_cur` values that mean the printer is not in a preparation stage: 0 is
  1882. # "Printing", -1 and 255 are "no stage" (#3211).
  1883. _NO_PREPARATION_STAGES = frozenset({0, -1, 255})
  1884. def _progress_milestone_to_notify(printer_id: int, state: PrinterState) -> int | None:
  1885. """The milestone (25, 50 or 75) this status update reaches for the first time, if any.
  1886. Records it as notified. Called only for a printing state with progress above 0.
  1887. Progress before the first layer is not progress through the print (#3211).
  1888. An A1 mini reports ``mc_percent`` 85 in the first frame of a print, while
  1889. still preheating the bed at layer 0, then 3, 7, 40 and 44 through its
  1890. calibration, and starts layer 1 at 45. Read as progress, that 85 sent the
  1891. 75% notification together with Print Started, and since 75 was then on
  1892. record, 25 and 50 could never fire. So nothing counts while the printer
  1893. reports a layer count but has not started layer 1.
  1894. Without a layer count -- a print whose first frame carried no
  1895. ``total_layer_num``, until the pushall that asks for it is answered
  1896. (#2702) -- the preparation stage says the same thing: before layer 1 the
  1897. A1 mini reported ``stg_cur`` 2, 4, 14 and 1 (bed preheating, calibration,
  1898. homing), and 0 from layer 1 on. -1 and 255 are "no stage". A printer that
  1899. reports neither keeps the old behaviour rather than never notifying.
  1900. """
  1901. if (state.layer_num or 0) < 1:
  1902. if (state.total_layers or 0) > 0:
  1903. return None
  1904. if state.stg_cur not in _NO_PREPARATION_STAGES:
  1905. return None
  1906. progress = state.progress or 0
  1907. current_milestone = 0
  1908. if progress >= 75:
  1909. current_milestone = 75
  1910. elif progress >= 50:
  1911. current_milestone = 50
  1912. elif progress >= 25:
  1913. current_milestone = 25
  1914. if current_milestone > _last_progress_milestone.get(printer_id, 0):
  1915. _last_progress_milestone[printer_id] = current_milestone
  1916. return current_milestone
  1917. return None
  1918. async def on_printer_status_change(printer_id: int, state: PrinterState):
  1919. """Handle printer status changes - broadcast via WebSocket."""
  1920. try:
  1921. notify_live_activities.observe(printer_id, state)
  1922. except Exception:
  1923. logging.getLogger(__name__).exception("Notify Live Activity status hook failed for printer %s", printer_id)
  1924. try:
  1925. pause_eta.observe(printer_id, state)
  1926. except Exception:
  1927. logging.getLogger(__name__).exception("Pause heads-up check failed for printer %s", printer_id)
  1928. wled_manager.handle_status(
  1929. printer_id,
  1930. state,
  1931. awaiting_plate_clear=printer_manager.is_awaiting_plate_clear(printer_id),
  1932. )
  1933. # Connected-edge reconciliation (#1542 follow-up). When the printer
  1934. # transitions disconnected → connected — which covers both Bambuddy
  1935. # startup (no prior connection) and a mid-session MQTT reconnect — fire
  1936. # `reconcile_stale_active_prints` exactly once for this connection so
  1937. # any archive still in `status="printing"` that can't actually be
  1938. # running anymore (printer IDLE / different subtask / empty subtask)
  1939. # gets a synthesised PRINT COMPLETE. Without this, a print that
  1940. # finished during a disconnect window + a smart-plug power cycle
  1941. # leaves the .3mf on the SD card and the firmware ghost-replays it on
  1942. # next boot. Reconciliation runs concurrently — it must not block the
  1943. # WebSocket dedup / broadcast logic below, and the connected edge is
  1944. # marked True BEFORE the await so concurrent status updates inside
  1945. # the same connection don't re-trigger reconciliation.
  1946. #
  1947. # Wait for a real push_status before reconciling (#1679): MQTT
  1948. # `_on_connect` broadcasts `state` IMMEDIATELY after the broker accepts
  1949. # the connection, BEFORE `_request_push_all` round-trips. At that
  1950. # instant the `PrinterState` is still on construction defaults — most
  1951. # importantly `state.state == "unknown"` and `state.subtask_name == ""`.
  1952. # If reconcile spawns here, every in-flight archive falls through to
  1953. # the empty-subtask_name trigger and gets synthesised `aborted`, which
  1954. # creates a duplicate archive on the real PRINT COMPLETE and
  1955. # double-counts filament. Gating on `state.state ∉ ("", "unknown")`
  1956. # keeps the #1542 mechanism intact: once the first real push_status
  1957. # updates `state.state` (RUNNING / IDLE / FINISH / …), this handler
  1958. # fires again with the flag still False — reconcile then runs against
  1959. # actual evidence.
  1960. state_known = bool(state.state) and state.state.upper() not in ("", "UNKNOWN")
  1961. if state.connected and state_known and not _printer_reconciled_since_connect.get(printer_id, False):
  1962. _printer_reconciled_since_connect[printer_id] = True
  1963. spawn_background_task(
  1964. reconcile_stale_active_prints(printer_id),
  1965. name=f"reconcile-stale-prints-{printer_id}",
  1966. )
  1967. elif not state.connected and _printer_reconciled_since_connect.get(printer_id, False):
  1968. # Re-arm so the next reconnect triggers reconciliation again.
  1969. _printer_reconciled_since_connect[printer_id] = False
  1970. # Same edge, for the calibration table the AMS card reads its K values from.
  1971. #
  1972. # Also gated on knowing a nozzle diameter, which is what decides *which*
  1973. # tables to ask for. A `state_known` gate alone is not enough: the first
  1974. # real push_status is what makes the state known, and the nozzle fields do
  1975. # not always arrive in it. Latching there would spend this connection's one
  1976. # attempt on a printer that could not yet say what was fitted.
  1977. nozzle_known = any(n.nozzle_diameter for n in (state.nozzles or []))
  1978. if (
  1979. state.connected
  1980. and state_known
  1981. and nozzle_known
  1982. and not _printer_kprofiles_primed_since_connect.get(printer_id, False)
  1983. ):
  1984. _printer_kprofiles_primed_since_connect[printer_id] = True
  1985. spawn_background_task(
  1986. prime_kprofile_table(printer_id),
  1987. name=f"prime-kprofiles-{printer_id}",
  1988. )
  1989. elif not state.connected and _printer_kprofiles_primed_since_connect.get(printer_id, False):
  1990. _printer_kprofiles_primed_since_connect[printer_id] = False
  1991. # A slot can lose its K-profile selection (cali_idx back to -1) with nothing
  1992. # else in the AMS report changing -- a power cycle does it to every slot --
  1993. # and on_ams_change never hears of it (#3219). Checked here, on every push
  1994. # while the printer is idle; needs_check is cheap and throttled per slot.
  1995. # Guarded: nothing here may stop the status broadcast below.
  1996. try:
  1997. if not state.connected:
  1998. kprofile_drift.forget_printer(printer_id)
  1999. elif kprofile_drift.needs_check(printer_id, state):
  2000. spawn_background_task(
  2001. kprofile_drift.reapply_lost_kprofiles(printer_id),
  2002. name=f"reapply-kprofiles-{printer_id}",
  2003. )
  2004. except Exception:
  2005. logging.getLogger(__name__).exception("[Printer %s] K-profile check failed", printer_id)
  2006. # Offline-notification edge (#1752): schedule `on_printer_offline` on
  2007. # connected → disconnected. The "back online" channel is already covered
  2008. # by the print-failure notification (firmware reports gcode_state=FAILED
  2009. # on reconnect of an interrupted print), so there is no online
  2010. # notification. The MQTT relay gets both: offline after the debounce, and
  2011. # online on the reconnect that follows a published offline.
  2012. prev_connected = _printer_last_connected.get(printer_id)
  2013. _printer_last_connected[printer_id] = state.connected
  2014. if prev_connected is True and not state.connected:
  2015. existing = _printer_offline_notify_tasks.get(printer_id)
  2016. if existing is None or existing.done():
  2017. logging.getLogger(__name__).info(
  2018. "[#1752] Printer %s connected→disconnected edge; scheduling offline notification in %.0fs",
  2019. printer_id,
  2020. _PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS,
  2021. )
  2022. _printer_offline_notify_tasks[printer_id] = asyncio.create_task(
  2023. _maybe_notify_printer_offline(printer_id),
  2024. name=f"printer-offline-notify-{printer_id}",
  2025. )
  2026. elif state.connected:
  2027. # Kept until the relay can actually send online: with its broker down
  2028. # now, the next status update after the reconnect sends it.
  2029. if printer_id in _printer_offline_published and mqtt_relay.enabled and mqtt_relay.connected:
  2030. _printer_offline_published.discard(printer_id)
  2031. printer_info = printer_manager.get_printer(printer_id)
  2032. if printer_info:
  2033. try:
  2034. await mqtt_relay.on_printer_online(printer_id, printer_info.name, printer_info.serial_number)
  2035. except Exception as e:
  2036. logging.getLogger(__name__).warning(
  2037. "MQTT relay online event failed for printer %s: %s", printer_id, e
  2038. )
  2039. pending = _printer_offline_notify_tasks.pop(printer_id, None)
  2040. if pending is not None and not pending.done():
  2041. logging.getLogger(__name__).info(
  2042. "[#1752] Printer %s reconnected before debounce; cancelling pending offline notification",
  2043. printer_id,
  2044. )
  2045. pending.cancel()
  2046. # Only broadcast if something meaningful changed (reduce WebSocket spam)
  2047. # Include rounded temperatures to detect meaningful temp changes (within 1 degree)
  2048. temps = state.temperatures or {}
  2049. nozzle_temp = round(temps.get("nozzle", 0))
  2050. bed_temp = round(temps.get("bed", 0))
  2051. nozzle_2_temp = round(temps.get("nozzle_2", 0)) if "nozzle_2" in temps else ""
  2052. chamber_temp = round(temps.get("chamber", 0)) if "chamber" in temps else ""
  2053. # Auto-detect dual-nozzle printers from MQTT temperature data
  2054. if "nozzle_2" in temps and printer_id not in _nozzle_count_updated:
  2055. _nozzle_count_updated.add(printer_id)
  2056. # Update nozzle_count in database
  2057. async with async_session() as db:
  2058. from backend.app.models.printer import Printer
  2059. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2060. printer = result.scalar_one_or_none()
  2061. if printer and printer.nozzle_count != 2:
  2062. printer.nozzle_count = 2
  2063. await db.commit()
  2064. logging.getLogger(__name__).info(
  2065. f"Auto-detected dual-nozzle printer {printer_id}, updated nozzle_count=2"
  2066. )
  2067. # Include target temps for heating phase detection
  2068. bed_target = round(temps.get("bed_target", 0))
  2069. nozzle_target = round(temps.get("nozzle_target", 0))
  2070. # Include tray_now and vt_tray hash so external spool changes trigger broadcasts
  2071. vt_tray_key = hash(str(state.raw_data.get("vt_tray", []))) if state.raw_data else 0
  2072. # Include AMS dry_time and tray state values so drying/slot changes trigger broadcasts.
  2073. #
  2074. # dry_countdown_stalled rides along because it is the one drying signal the
  2075. # countdown itself cannot carry: the MQTT layer raises it precisely BECAUSE
  2076. # dry_time stopped moving, so on the frame that flips it every other member
  2077. # of this key is identical and the push would be deduplicated away. Mid-print
  2078. # a temperature would eventually break the tie, but a parked command on an
  2079. # idle machine changes nothing else at all — AMS temp and humidity are not in
  2080. # the key — so the badge could sit unreachable indefinitely. The flag flips at
  2081. # most once per drying cycle, so it costs no mid-print broadcast traffic.
  2082. ams_dry_key = (
  2083. tuple((a.get("dry_time", 0), bool(a.get("dry_countdown_stalled"))) for a in (state.raw_data.get("ams") or []))
  2084. if state.raw_data
  2085. else ()
  2086. )
  2087. # Include tray states so load/unload transitions (state 11→10) trigger broadcasts (#784)
  2088. #
  2089. # The filament identity fields are here because Configure Slot writes
  2090. # exactly those and nothing else. Re-configuring a slot from PLA to another
  2091. # brand or colour of PLA leaves id/tray_type/state identical, so the key
  2092. # matched, this function returned before broadcasting, and the card kept
  2093. # showing the old filament until the 30s fallback poll or a page reload —
  2094. # even though the configure route asks the printer for a fresh pushall and
  2095. # that push does carry the new values. Reset always worked, because it
  2096. # clears tray_type.
  2097. #
  2098. # These fields only change when someone configures a slot or swaps a spool,
  2099. # so unlike temperature or progress they add no broadcast traffic mid-print.
  2100. ams_tray_key = (
  2101. tuple(
  2102. (
  2103. t.get("id"),
  2104. t.get("tray_type", ""),
  2105. t.get("state"),
  2106. t.get("tray_color", ""),
  2107. t.get("tray_info_idx", ""),
  2108. t.get("tray_sub_brands", ""),
  2109. t.get("cali_idx"),
  2110. )
  2111. for a in (state.raw_data.get("ams") or [])
  2112. for t in a.get("tray", [])
  2113. )
  2114. if state.raw_data
  2115. else ()
  2116. )
  2117. # Filament Track Switch: which inlet each AMS is bound to, and whether the
  2118. # accessory is fitted at all. Neither is in ams_tray_key (it is per-tray) nor
  2119. # in the AMS change-hash (tray fields only, and widening that would fire
  2120. # spurious Spoolman syncs), so without them a "Join IN-B" on the printer
  2121. # screen changed no key at all and the card's inlet badges sat stale until a
  2122. # reload. Like the filament-backup flag, these only move when someone
  2123. # reconfigures the machine, so they add no mid-print broadcast traffic.
  2124. fts_key = (
  2125. state.fila_switch.installed if state.fila_switch else False,
  2126. tuple(sorted(state.ams_switch_inlet.items())),
  2127. # Which hotend holds which slot. Unlike the two above this does move
  2128. # mid-print, on every filament change — but only between discrete slots,
  2129. # so it adds a push per toolchange, not a stream. The AMS slot menu needs
  2130. # it live: it decides which hotend the Load dialog may offer and whether
  2131. # Unload has anything to act on.
  2132. tuple(
  2133. sorted(
  2134. ((ext, slot.ams_id, slot.slot_id, slot.has_filament) for ext, slot in state.extruder_slots.items()),
  2135. # Sort on the extruder id alone: the other members are nullable
  2136. # and comparing None with an int raises.
  2137. key=lambda entry: entry[0],
  2138. )
  2139. ),
  2140. )
  2141. status_key = (
  2142. f"{state.connected}:{state.state}:{state.progress}:{state.layer_num}:"
  2143. f"{nozzle_temp}:{bed_temp}:{nozzle_2_temp}:{chamber_temp}:"
  2144. f"{state.stg_cur}:{bed_target}:{nozzle_target}:"
  2145. f"{state.cooling_fan_speed}:{state.big_fan1_speed}:{state.big_fan2_speed}:"
  2146. f"{state.chamber_light}:{state.active_extruder}:{state.tray_now}:{vt_tray_key}:"
  2147. f"{ams_dry_key}:{ams_tray_key}:{state.door_open}:{state.ams_filament_backup}:{fts_key}"
  2148. )
  2149. # Fields that change only when the machine does (a nozzle swap, a firmware
  2150. # update, developer mode) or once per print (its AMS mapping): no traffic,
  2151. # but on an idle printer nothing else moves the key when they change.
  2152. machine_key = (
  2153. tuple((n.nozzle_type, n.nozzle_diameter) for n in (state.nozzles or [])),
  2154. state.firmware_version,
  2155. state.developer_mode,
  2156. tuple(state.print_ams_mapping) if isinstance(state.print_ams_mapping, list) else None,
  2157. # Cooling/heating, switched on the printer's screen too: on an idle
  2158. # printer nothing else moves the key (PSF-4).
  2159. state.airduct_mode,
  2160. )
  2161. status_key = f"{status_key}:{machine_key}"
  2162. is_active_print = state.state in _ACTIVE_PRINT_STATES
  2163. if not is_active_print:
  2164. _unauthorized_print_kill_sent.discard(printer_id)
  2165. elif printer_id in _unauthorized_print_kill_sent:
  2166. # stop_print() was already sent for this print; avoid all further
  2167. # ownership and settings work until the printer leaves an active state.
  2168. pass
  2169. elif _is_bambuddy_authorized_print_in_memory(printer_id, state):
  2170. # Normal Bambuddy-started prints stay entirely on the in-memory path.
  2171. _unauthorized_print_kill_sent.discard(printer_id)
  2172. else:
  2173. kill_switch_enabled = False
  2174. authorization: bool | None = None
  2175. status_logger = logging.getLogger(__name__)
  2176. try:
  2177. kill_switch_enabled = await _is_printer_kill_switch_enabled_cached()
  2178. if kill_switch_enabled:
  2179. async with async_session() as db:
  2180. authorization = await _is_bambuddy_authorized_print(printer_id, state, db)
  2181. except Exception as e:
  2182. # Fail safe: a database/reconciliation error must never turn into an
  2183. # irreversible stop of a print whose ownership is still unknown.
  2184. authorization = None
  2185. status_logger.warning(
  2186. "[KILL SWITCH] Failed to reconcile print authorization for printer %s: %s", printer_id, e
  2187. )
  2188. if not kill_switch_enabled or authorization is True:
  2189. _unauthorized_print_kill_sent.discard(printer_id)
  2190. elif authorization is None:
  2191. _unauthorized_print_kill_sent.discard(printer_id)
  2192. status_logger.debug(
  2193. "[KILL SWITCH] Deferring authorization for printer %s until archive state is reconciled",
  2194. printer_id,
  2195. )
  2196. else:
  2197. try:
  2198. stopped = printer_manager.stop_print(printer_id)
  2199. if stopped:
  2200. _unauthorized_print_kill_sent.add(printer_id)
  2201. _kill_switch_stopped.add(printer_id)
  2202. printer_info = printer_manager.get_printer(printer_id)
  2203. printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  2204. filename = state.subtask_name or state.gcode_file or state.current_print or "Unknown"
  2205. notification_data = {
  2206. "status": "stopped",
  2207. "filename": state.gcode_file or state.current_print or "",
  2208. "subtask_name": state.subtask_name or "",
  2209. "progress": state.progress,
  2210. "reason": "unauthorized_print",
  2211. }
  2212. status_logger.warning(
  2213. "[KILL SWITCH] Stopped unauthorized print on printer %s (state=%s)",
  2214. printer_id,
  2215. state.state,
  2216. )
  2217. try:
  2218. await ws_manager.broadcast(
  2219. {
  2220. "type": "kill_switch_triggered",
  2221. "printer_id": printer_id,
  2222. "printer_name": printer_name,
  2223. "filename": filename,
  2224. "reason": "unauthorized_print",
  2225. }
  2226. )
  2227. except Exception as e:
  2228. status_logger.warning(
  2229. "[KILL SWITCH] WebSocket notification failed for printer %s: %s", printer_id, e
  2230. )
  2231. previous_task = _kill_switch_notification_tasks.pop(printer_id, None)
  2232. if previous_task is not None and not previous_task.done():
  2233. previous_task.cancel()
  2234. _kill_switch_notification_tasks[printer_id] = spawn_background_task(
  2235. _send_kill_switch_provider_notification(printer_id, printer_name, notification_data),
  2236. name=f"kill-switch-notification-{printer_id}",
  2237. )
  2238. else:
  2239. status_logger.warning(
  2240. "[KILL SWITCH] Could not stop unauthorized print on printer %s (state=%s)",
  2241. printer_id,
  2242. state.state,
  2243. )
  2244. except Exception as e:
  2245. status_logger.warning(
  2246. "[KILL SWITCH] Failed to stop unauthorized print on printer %s: %s", printer_id, e
  2247. )
  2248. # MQTT relay - publish status (before dedup check - always publish to MQTT)
  2249. try:
  2250. printer_info = printer_manager.get_printer(printer_id)
  2251. if printer_info:
  2252. await mqtt_relay.on_printer_status(
  2253. printer_id,
  2254. state,
  2255. printer_info.name,
  2256. printer_info.serial_number,
  2257. printer_manager.is_awaiting_plate_clear(printer_id),
  2258. )
  2259. except Exception:
  2260. pass # Don't fail status callback if MQTT fails
  2261. if _last_status_broadcast.get(printer_id) == status_key:
  2262. return # No change, skip WebSocket broadcast
  2263. _last_status_broadcast[printer_id] = status_key
  2264. # Check for progress milestone notifications (25%, 50%, 75%)
  2265. progress = state.progress or 0
  2266. is_printing = state.state in ("RUNNING", "PRINTING")
  2267. if is_printing and progress > 0:
  2268. current_milestone = _progress_milestone_to_notify(printer_id, state)
  2269. if current_milestone is not None:
  2270. try:
  2271. from backend.app.models.printer import Printer
  2272. # Read the printer in a short session and release the connection
  2273. # BEFORE the ~15s camera snapshot below — holding it across the grab
  2274. # pinned a pooled connection per milestone, per printer (issue #2572).
  2275. async with async_session() as db:
  2276. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2277. printer = result.scalar_one_or_none()
  2278. printer_name = printer.name if printer else f"Printer {printer_id}"
  2279. filename = state.subtask_name or state.gcode_file or "Unknown"
  2280. # remaining_time is in minutes, convert to seconds for notification
  2281. remaining_time_seconds = state.remaining_time * 60 if state.remaining_time else None
  2282. # Capture camera snapshot for notification image attachment (no DB held).
  2283. image_data = await _capture_snapshot_for_notification(printer_id, printer, logging.getLogger(__name__))
  2284. # Notification send needs a session (provider/template lookups).
  2285. async with async_session() as db:
  2286. await notification_service.on_print_progress(
  2287. printer_id,
  2288. printer_name,
  2289. filename,
  2290. current_milestone,
  2291. db,
  2292. remaining_time_seconds,
  2293. image_data=image_data,
  2294. )
  2295. except Exception as e:
  2296. logging.getLogger(__name__).warning(f"Progress milestone notification failed: {e}")
  2297. elif progress < 5:
  2298. # Reset milestone tracking when print restarts or new print begins
  2299. _last_progress_milestone[printer_id] = 0
  2300. _first_layer_notified[printer_id] = False
  2301. # HMS error codes that should not trigger notifications even though they
  2302. # have known descriptions (e.g. user-initiated actions, not real errors).
  2303. _HMS_NOTIFICATION_SUPPRESS = {
  2304. "0500_400E", # Printing was cancelled (user action, not an error)
  2305. }
  2306. # Check for new HMS errors and send notifications
  2307. current_hms_errors = getattr(state, "hms_errors", []) or []
  2308. if current_hms_errors:
  2309. new_errors = await _take_new_hms_faults_and_save(printer_id, current_hms_errors, state.state)
  2310. if new_errors:
  2311. try:
  2312. from backend.app.models.printer import Printer
  2313. # Read the printer in a short session and release the connection
  2314. # BEFORE the ~15s camera snapshot below (issue #2572).
  2315. async with async_session() as db:
  2316. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2317. printer = result.scalar_one_or_none()
  2318. printer_name = printer.name if printer else f"Printer {printer_id}"
  2319. # Capture camera snapshot once for all error notifications (no DB held).
  2320. error_image_data = await _capture_snapshot_for_notification(
  2321. printer_id, printer, logging.getLogger(__name__)
  2322. )
  2323. # Notification sends need a session (provider/template lookups).
  2324. async with async_session() as db:
  2325. sent_count = 0
  2326. for error in new_errors:
  2327. # Build short code like "0700_8010"
  2328. # Mask to 16 bits to handle printers that send larger values
  2329. error_code_int = int(error.code.replace("0x", ""), 16) if error.code else 0
  2330. error_code_masked = error_code_int & 0xFFFF
  2331. short_code = f"{(error.attr >> 16) & 0xFFFF:04X}_{error_code_masked:04X}"
  2332. # Only notify for errors Bambu publishes text for — printers
  2333. # send undocumented codes that aren't real errors, and Bambu
  2334. # lists some with empty text. Resolved at parse time (#2926);
  2335. # short_code is still needed for the suppression set below.
  2336. description = error.description
  2337. if not description or short_code in _HMS_NOTIFICATION_SUPPRESS:
  2338. continue
  2339. error_type = _hms_error_title(error.module)
  2340. error_detail = description
  2341. await notification_service.on_printer_error(
  2342. printer_id, printer_name, error_type, db, error_detail, image_data=error_image_data
  2343. )
  2344. sent_count += 1
  2345. if sent_count:
  2346. logging.getLogger(__name__).info(
  2347. f"[HMS] Sent notification for {sent_count} error(s) on printer {printer_id}"
  2348. )
  2349. # Also publish to MQTT relay (no DB): every new fault that
  2350. # counts, with or without text, the same set the UI counts.
  2351. printer_info = printer_manager.get_printer(printer_id)
  2352. if printer_info:
  2353. errors_data = [
  2354. {
  2355. "code": e.code,
  2356. "attr": e.attr,
  2357. "module": e.module,
  2358. "severity": e.severity,
  2359. }
  2360. for e in new_errors
  2361. ]
  2362. await mqtt_relay.on_printer_error(
  2363. printer_id, printer_info.name, printer_info.serial_number, errors_data
  2364. )
  2365. except Exception as e:
  2366. logging.getLogger(__name__).warning(f"HMS error notification failed: {e}")
  2367. else:
  2368. # No HMS errors: nothing to send, but faults gone long enough (or gone
  2369. # across a print state change) are forgotten. Some codes, e.g. chamber
  2370. # temperature regulation during PETG prints, toggle every few seconds.
  2371. await _take_new_hms_faults_and_save(printer_id, [], state.state)
  2372. await ws_manager.send_printer_status(
  2373. printer_id,
  2374. printer_state_to_dict(
  2375. state,
  2376. printer_id,
  2377. printer_manager.get_model(printer_id),
  2378. printer_manager.get_drying_targets(printer_id),
  2379. ),
  2380. )
  2381. def _is_bambu_uuid(tray_uuid: str) -> bool:
  2382. """Check if a tray UUID looks like a valid Bambu Lab RFID UUID (non-empty, non-zero)."""
  2383. return bool(tray_uuid) and tray_uuid not in ("", "0" * len(tray_uuid))
  2384. async def on_fts_inlet_change(printer_id: int, ams_id: int, inlet: str):
  2385. """Re-point a moved AMS's K-profiles at the nozzle it now feeds.
  2386. K-profiles are per-nozzle and the printer's calibration table is numbered
  2387. per-nozzle, but a tray holds exactly one ``cali_idx``. Moving an AMS to the
  2388. switch's other inlet therefore silently invalidates every configured slot in
  2389. it: the index stays put and now resolves against the other nozzle's table.
  2390. Measured on the maintainer's H2C — one spool calibrated 0.018 on the left
  2391. and 0.020 on the right kept the left profile after the move, and a manual
  2392. RFID re-read only re-asserted the same wrong one.
  2393. Configuring a slot is a deliberate preparation step, so this re-selects
  2394. rather than re-configures: only the calibration binding moves, and only for
  2395. slots whose spool already has a stored profile for the new nozzle. A slot
  2396. Bambuddy knows nothing about is left exactly as the operator left it.
  2397. """
  2398. logger = logging.getLogger(__name__)
  2399. target_extruder = extruder_for_inlet(inlet)
  2400. if target_extruder is None:
  2401. return
  2402. client = printer_manager.get_client(printer_id)
  2403. state = printer_manager.get_status(printer_id)
  2404. if not client or not state or not state.raw_data:
  2405. return
  2406. # The nozzle the AMS now feeds -- the diameter of the TARGET extruder, not
  2407. # of nozzle 0. On a machine with two sizes fitted, moving the inlet changes
  2408. # the nozzle width, which changes both the K profile to select and the
  2409. # preset the slot should carry.
  2410. nozzle_diameter = nozzle_diameter_for_extruder(state, target_extruder, printer_manager.get_model(printer_id))
  2411. ams_raw = state.raw_data.get("ams")
  2412. ams_list = ams_raw.get("ams", []) if isinstance(ams_raw, dict) else ams_raw if isinstance(ams_raw, list) else []
  2413. unit = next((u for u in ams_list if str(u.get("id")) == str(ams_id)), None)
  2414. if not unit:
  2415. return
  2416. try:
  2417. async with async_session() as db:
  2418. for tray in unit.get("tray", []):
  2419. tray_id = int(tray.get("id", -1))
  2420. if tray_id < 0 or not tray.get("tray_type"):
  2421. continue
  2422. current_idx = tray.get("cali_idx")
  2423. profile = await find_slot_kprofile_for_extruder(
  2424. db,
  2425. printer_id,
  2426. ams_id,
  2427. tray_id,
  2428. target_extruder,
  2429. nozzle_diameter,
  2430. printer_manager.get_model(printer_id),
  2431. nozzle_flow_for_extruder(state, target_extruder, printer_manager.get_model(printer_id)),
  2432. )
  2433. if profile is None or profile.cali_idx is None:
  2434. continue
  2435. if current_idx == profile.cali_idx:
  2436. continue # Already on the right one.
  2437. logger.info(
  2438. "[Printer %s] AMS %s slot %s moved to inlet %s (nozzle %s): "
  2439. "re-selecting K-profile %s (cali_idx %s -> %s, K=%s)",
  2440. printer_id,
  2441. ams_id,
  2442. tray_id,
  2443. inlet,
  2444. target_extruder,
  2445. profile.name,
  2446. current_idx,
  2447. profile.cali_idx,
  2448. profile.k_value,
  2449. )
  2450. client.extrusion_cali_sel(
  2451. ams_id=ams_id,
  2452. tray_id=tray_id,
  2453. cali_idx=profile.cali_idx,
  2454. filament_id=printer_safe_filament_id(profile.filament_id, tray.get("tray_info_idx", "")),
  2455. nozzle_diameter=nozzle_diameter,
  2456. )
  2457. except Exception as e:
  2458. logger.warning("[Printer %s] Could not re-apply K-profiles after inlet move: %s", printer_id, e)
  2459. def _slot_feeds_print(printer_id: int, ams_id: int, tray_id: int) -> bool:
  2460. """Is this slot the one feeding the printer right now?
  2461. ``tray_now`` names one slot for the whole printer, so on a dual-nozzle
  2462. machine each extruder's own fed slot is asked as well. Only meaningful
  2463. while a print runs; the caller checks that.
  2464. """
  2465. status = printer_manager.get_status(printer_id)
  2466. if status is None:
  2467. return False
  2468. global_id = printer_tray_layout(printer_id, printer_manager).tray_from_slot_key(ams_id, tray_id)
  2469. if getattr(status, "tray_now", None) == global_id:
  2470. return True
  2471. slots = getattr(status, "extruder_slots", None) or {}
  2472. return any(slot.has_filament and slot.holds(ams_id, tray_id) for slot in slots.values())
  2473. def _tray_reports_no_filament(color: str, tray_type: str) -> bool:
  2474. """Does this tray report carry no filament data at all?
  2475. No type, and a colour that is either empty or ``00000000``: firmware sends
  2476. both for a blank slot, and ``00000000`` is what Bambuddy's own slot reset
  2477. writes. A reconnect mid-print brought it in that form and got past every
  2478. guard that only knew the empty string (#3526). A clear spool also reports
  2479. ``00000000``, but always with a type, so it is not blank.
  2480. """
  2481. return not tray_type.strip() and color.strip().upper() in ("", "00000000")
  2482. async def _unlink_stale_assignments(printer_id: int, ams_data: list, printing_now: bool) -> None:
  2483. """Unlink built-in inventory assignments whose slot no longer holds their spool.
  2484. Runs from ``on_ams_change`` and, for removals held by ``slot_unlink_grace``,
  2485. from the delayed re-check -- which is why it takes the AMS data and print
  2486. state as arguments instead of reading them from the push (#3186).
  2487. """
  2488. logger = logging.getLogger(__name__)
  2489. from backend.app.utils.color_utils import colors_similar as _colors_similar
  2490. # Auto-unlink spool assignments with stale fingerprints. Under the
  2491. # per-printer assignment lock since #3186: the held-removal re-check runs
  2492. # this outside any MQTT push, so it can now overlap one.
  2493. try:
  2494. async with get_ams_assignment_lock(printer_id), async_session() as db:
  2495. from sqlalchemy.orm import selectinload
  2496. from backend.app.api.routes.inventory import _find_tray_in_ams_data
  2497. from backend.app.models.spool import Spool as _Spool
  2498. from backend.app.models.spool_assignment import SpoolAssignment as SA
  2499. from backend.app.services.ams_slot_presence import spool_present
  2500. from backend.app.services.inventory_mode import spoolman_owns_assignments
  2501. # Built-in assignments only. Since #2812 they survive a switch to
  2502. # Spoolman mode rather than being deleted by it, and this pass ends
  2503. # in ``db.delete`` — left ungated it would unlink them one slot at a
  2504. # time as the AMS contents changed under the other mode, undoing the
  2505. # preservation more slowly but just as completely.
  2506. assignments = []
  2507. if not await spoolman_owns_assignments(db):
  2508. result = await db.execute(
  2509. select(SA)
  2510. .where(SA.printer_id == printer_id)
  2511. .options(selectinload(SA.spool).selectinload(_Spool.k_profiles))
  2512. )
  2513. assignments = result.scalars().all()
  2514. # ``printing_now`` (top of this function) keeps a runout from
  2515. # unlinking the spool that fed the print — the next idle-time pass
  2516. # unlinks it if the user really did take it out.
  2517. stale = []
  2518. # Removals this pass is holding rather than unlinking (#3186).
  2519. held: set[tuple] = set()
  2520. for assignment in assignments:
  2521. # External spool assignments (ams_id=255) live in vt_tray, not AMS data
  2522. if assignment.ams_id == 255:
  2523. ps = printer_manager.get_status(printer_id)
  2524. vt_tray_raw = ps.raw_data.get("vt_tray", []) if ps else []
  2525. ext_id = external_tray_id(assignment.tray_id) # 0→254, 1→255, rack 2-8→247-253
  2526. current_tray = None
  2527. for vt in vt_tray_raw:
  2528. if isinstance(vt, dict) and int(vt.get("id", 254)) == ext_id:
  2529. current_tray = vt
  2530. break
  2531. if not current_tray:
  2532. # vt_tray data may not have arrived yet — keep assignment
  2533. continue
  2534. else:
  2535. current_tray = _find_tray_in_ams_data(ams_data, assignment.ams_id, assignment.tray_id)
  2536. if not current_tray:
  2537. if printing_now:
  2538. logger.info(
  2539. "Auto-unlink skipped: spool %d AMS%d-T%d — slot empty during a running print (runout?)",
  2540. assignment.spool_id,
  2541. assignment.ams_id,
  2542. assignment.tray_id,
  2543. )
  2544. continue
  2545. # A whole AMS unit can drop out of one push and come back in
  2546. # the next; only a slot that stays gone is a removal (#3186).
  2547. hold_key = ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id)
  2548. if not slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2549. held.add(hold_key)
  2550. logger.info(
  2551. "Auto-unlink held: spool %d AMS%d-T%d — tray not found in AMS data; "
  2552. "unlinking if it is still gone in %ds",
  2553. assignment.spool_id,
  2554. assignment.ams_id,
  2555. assignment.tray_id,
  2556. int(slot_unlink_grace.GRACE_SECONDS),
  2557. )
  2558. continue
  2559. logger.info(
  2560. "Auto-unlink: spool %d AMS%d-T%d — tray not found in AMS data (slot empty?)",
  2561. assignment.spool_id,
  2562. assignment.ams_id,
  2563. assignment.tray_id,
  2564. )
  2565. stale.append(assignment) # Slot empty
  2566. elif _is_bambu_uuid(current_tray.get("tray_uuid", "")):
  2567. # A Bambu Lab spool is in this slot — check if it's the same spool
  2568. # that's currently assigned. If yes, keep the assignment (avoids
  2569. # unnecessary unlink/re-assign/ams_filament_setting cycle that clears
  2570. # the printer's filament preset on every startup).
  2571. tray_uuid = current_tray.get("tray_uuid", "")
  2572. tag_uid = current_tray.get("tag_uid", "")
  2573. spool = assignment.spool
  2574. spool_matches = False
  2575. if spool:
  2576. if (spool.tray_uuid and spool.tray_uuid.upper() == tray_uuid.upper()) or (
  2577. spool.tag_uid
  2578. and tag_uid
  2579. and tag_uid != "0000000000000000"
  2580. and spool.tag_uid.upper() == tag_uid.upper()
  2581. ):
  2582. spool_matches = True
  2583. if spool_matches:
  2584. # Same BL spool still in slot — keep assignment, update fingerprint if needed
  2585. cur_color = current_tray.get("tray_color", "")
  2586. cur_type = current_tray.get("tray_type", "")
  2587. fp_color = assignment.fingerprint_color or ""
  2588. fp_type = assignment.fingerprint_type or ""
  2589. if cur_color.upper() != fp_color.upper() or cur_type.upper() != fp_type.upper():
  2590. assignment.fingerprint_color = cur_color
  2591. assignment.fingerprint_type = cur_type
  2592. logger.debug(
  2593. "Auto-unlink: spool %d AMS%d-T%d — same BL spool, updated fingerprint",
  2594. assignment.spool_id,
  2595. assignment.ams_id,
  2596. assignment.tray_id,
  2597. )
  2598. continue
  2599. # Different BL spool or unrecognized — unlink so auto-assign can match
  2600. logger.info(
  2601. "Auto-unlink: spool %d AMS%d-T%d — different Bambu Lab spool detected (uuid=%s)",
  2602. assignment.spool_id,
  2603. assignment.ams_id,
  2604. assignment.tray_id,
  2605. tray_uuid,
  2606. )
  2607. stale.append(assignment)
  2608. else:
  2609. cur_color = current_tray.get("tray_color", "")
  2610. cur_type = current_tray.get("tray_type", "")
  2611. cur_state = current_tray.get("state")
  2612. fp_color = assignment.fingerprint_color or ""
  2613. fp_type = assignment.fingerprint_type or ""
  2614. # SpoolBuddy pre-config replay: fingerprint_type empty means
  2615. # the slot was empty when the user pre-assigned via SpoolBuddy
  2616. # (the firmware drops ams_filament_setting on empty slots, so
  2617. # MQTT was deferred). The moment any filament gets inserted
  2618. # — Bambu RFID, 3rd-party, or even an existing-but-now-
  2619. # reconfigured spool — fire the deferred configuration.
  2620. # The "loaded" signal is state == 11 (Bambu's "filament fed to
  2621. # extruder" code) OR, on firmwares that don't use the state
  2622. # enum meaningfully, a non-empty tray_type when state is
  2623. # NOT one of the firmware's explicit empty signals (9, 10).
  2624. # state-only was wrong for firmwares that never set 11 — A1
  2625. # Mini BMCU 01.07.02.00 and P1S Standard AMS 00.00.06.75 both
  2626. # always report state=3 — so the replay never fired for them
  2627. # (#1322). The state ∉ {9,10} guard keeps the firmware's
  2628. # explicit "empty" signals authoritative over any stale
  2629. # tray_type that might survive the relay's auto-clearing.
  2630. #
  2631. # tray_exist_bits comes first because that guard cannot tell
  2632. # a firmware "empty" from Bambuddy's own: apply_tray_exist_bits
  2633. # writes state=9 when the bit is 0 and leaves it there when the
  2634. # bit returns. A non-RFID spool inserted into a pre-assigned
  2635. # slot brings no tray_type with it, so the stale 9 made this
  2636. # expression false forever and the deferred config never fired
  2637. # — the deadlock #1322 removed from the assign path, still in
  2638. # place here (#3084, #3100).
  2639. loaded = spool_present(current_tray) is True or (
  2640. cur_state == 11 or (cur_state not in (9, 10) and cur_type.strip())
  2641. )
  2642. # Not into the slot feeding a running print: the printer refuses
  2643. # the setting for it (``ams_filament_setting refused``), and a
  2644. # spool swapped in after a runout sat through one refused send
  2645. # per AMS change. Held, it fires on the first change once the
  2646. # print is over (#3434).
  2647. if (
  2648. not fp_type.strip()
  2649. and loaded
  2650. and assignment.spool
  2651. and printing_now
  2652. and _slot_feeds_print(printer_id, assignment.ams_id, assignment.tray_id)
  2653. ):
  2654. logger.debug(
  2655. "Pre-config held for spool %d AMS%d-T%d: the slot is feeding the print",
  2656. assignment.spool_id,
  2657. assignment.ams_id,
  2658. assignment.tray_id,
  2659. )
  2660. continue
  2661. if not fp_type.strip() and loaded and assignment.spool:
  2662. # The fingerprint ends this replay. Not recorded when nothing
  2663. # could be sent (no MQTT client while the printer reconnects),
  2664. # so the next AMS change tries again. An attempt that raised is
  2665. # recorded as before: it may have sent part of the commands and
  2666. # would raise again on every change, remain% steps included.
  2667. configured: bool | None = False
  2668. try:
  2669. from backend.app.api.routes.inventory import (
  2670. apply_spool_to_slot_via_mqtt,
  2671. load_preset_lookup_user,
  2672. )
  2673. from backend.app.core.auth import is_auth_enabled
  2674. # Whoever made the assignment: their cloud sign-in
  2675. # resolves a custom preset, which nobody's did here,
  2676. # so the slot got Generic PLA on insert (#3519). With
  2677. # login off the shared sign-in applies, as before.
  2678. replay_user = None
  2679. if assignment.assigned_by_user_id is not None and await is_auth_enabled(db):
  2680. replay_user = await load_preset_lookup_user(db, assignment.assigned_by_user_id)
  2681. configured = await apply_spool_to_slot_via_mqtt(
  2682. db=db,
  2683. current_user=replay_user,
  2684. spool=assignment.spool,
  2685. printer_id=printer_id,
  2686. ams_id=assignment.ams_id,
  2687. tray_id=assignment.tray_id,
  2688. current_tray_info_idx=current_tray.get("tray_info_idx", ""),
  2689. current_tray_type=cur_type,
  2690. current_tray_sub_brands=current_tray.get("tray_sub_brands", ""),
  2691. )
  2692. except Exception:
  2693. logger.exception(
  2694. "Pre-config apply failed for spool %d on printer %d AMS%d-T%d",
  2695. assignment.spool_id,
  2696. printer_id,
  2697. assignment.ams_id,
  2698. assignment.tray_id,
  2699. )
  2700. configured = None
  2701. if configured is False:
  2702. logger.warning(
  2703. "Pre-config not sent for spool %d on printer %d AMS%d-T%d (no MQTT client); "
  2704. "retrying on the next AMS change",
  2705. assignment.spool_id,
  2706. printer_id,
  2707. assignment.ams_id,
  2708. assignment.tray_id,
  2709. )
  2710. else:
  2711. if configured:
  2712. logger.info(
  2713. "SpoolBuddy pre-config applied on insert: spool %d → printer %d AMS%d-T%d",
  2714. assignment.spool_id,
  2715. printer_id,
  2716. assignment.ams_id,
  2717. assignment.tray_id,
  2718. )
  2719. assignment.fingerprint_color = cur_color
  2720. assignment.fingerprint_type = cur_type
  2721. continue
  2722. if not _colors_similar(cur_color, fp_color) or cur_type.upper() != fp_type.upper():
  2723. # Blank tray data mid-print is a runout, not a swap: the
  2724. # firmware clears colour and type when it unloads a spool
  2725. # it just emptied. Unlinking here would erase the record
  2726. # of which spool fed the print so far.
  2727. if printing_now and _tray_reports_no_filament(cur_color, cur_type):
  2728. logger.info(
  2729. "Auto-unlink skipped: spool %d AMS%d-T%d — tray data cleared during a running print "
  2730. "(runout?)",
  2731. assignment.spool_id,
  2732. assignment.ams_id,
  2733. assignment.tray_id,
  2734. )
  2735. continue
  2736. # Same reasoning off the print, on firmware's own say-so:
  2737. # a blank tray report from a slot whose tray_exist_bits
  2738. # bit is set describes a spool the AMS cannot identify —
  2739. # a non-RFID one, or one whose slot was reset — not a
  2740. # spool that was taken out. Deleting the assignment there
  2741. # threw away the identity the user had supplied, which is
  2742. # the only place it existed (#3100). A slot the bit calls
  2743. # empty, or one that carries no bit at all, still unlinks.
  2744. #
  2745. # Unless the slot was reported empty first: a removal
  2746. # already held means a spool came out and another went
  2747. # in, so the blank report keeps the hold running below
  2748. # rather than cancelling it (#3186).
  2749. if (
  2750. spool_present(current_tray) is True
  2751. and _tray_reports_no_filament(cur_color, cur_type)
  2752. and not slot_unlink_grace.is_held(
  2753. printer_id,
  2754. ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id),
  2755. )
  2756. ):
  2757. logger.info(
  2758. "Auto-unlink skipped: spool %d AMS%d-T%d — slot still occupied, "
  2759. "tray reports no filament data yet",
  2760. assignment.spool_id,
  2761. assignment.ams_id,
  2762. assignment.tray_id,
  2763. )
  2764. continue
  2765. # A blank report the presence bit does not vouch for is
  2766. # still only one push. An idle X1C cleared a whole AMS
  2767. # unit's bits, colour and type for a moment and lost
  2768. # four saved assignments that way (#3186); unlink only
  2769. # if the slot is still blank after the grace period.
  2770. if _tray_reports_no_filament(cur_color, cur_type):
  2771. hold_key = ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id)
  2772. if not slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2773. held.add(hold_key)
  2774. logger.info(
  2775. "Auto-unlink held: spool %d AMS%d-T%d — tray reports no filament data; "
  2776. "unlinking if it is still blank in %ds",
  2777. assignment.spool_id,
  2778. assignment.ams_id,
  2779. assignment.tray_id,
  2780. int(slot_unlink_grace.GRACE_SECONDS),
  2781. )
  2782. continue
  2783. # Fingerprint mismatch — but check if tray now matches the
  2784. # assigned spool (e.g. auto-configure changed the tray).
  2785. # Both sides are reduced to the type the slot can carry
  2786. # before comparing: the assign path writes that rather
  2787. # than the spool's raw material (#2902), so a spool whose
  2788. # material is a product line — "PLA+", "HTPLA" — reports
  2789. # back as "PLA" and would otherwise fail this check and
  2790. # be auto-unlinked from the slot it was just assigned to.
  2791. # Reducing the printer's side too keeps slots configured
  2792. # by an older Bambuddy, still reporting "PLA+", matching.
  2793. spool = assignment.spool
  2794. if spool:
  2795. spool_color = (spool.rgba or "FFFFFFFF").upper()
  2796. # Two ways the assign path can have arrived at the
  2797. # slot's type, so both count as "we wrote this".
  2798. # The material column is one; the spool's preset is
  2799. # the other, and it outranks the material when the
  2800. # spool has one -- a spool whose material says PLA
  2801. # and whose preset is "Bambu PLA Aero" puts
  2802. # PLA-AERO in the slot (#2902). Read from the stored
  2803. # preset name rather than resolving the preset,
  2804. # because this runs on every AMS push and a cloud
  2805. # lookup here would be both slow and unavailable on
  2806. # the unauthenticated replay path.
  2807. spool_types = {printer_filament_type(spool.material).upper()}
  2808. if spool.slicer_filament_name:
  2809. spool_types.add(printer_filament_type(spool.slicer_filament_name).upper())
  2810. # An imported local preset stores its type outright,
  2811. # which is what the assign path used -- and the name
  2812. # above may be unset. One keyed read, and only on a
  2813. # mismatch, which is rare.
  2814. #
  2815. # slicer_filament is free text up to fifty characters,
  2816. # so the digits have to be checked against the range
  2817. # of the integer primary key they are about to be
  2818. # compared with. Postgres raises on an out-of-range
  2819. # integer rather than simply not matching, and that
  2820. # would poison this session and abandon the rest of
  2821. # the cleanup pass.
  2822. lp_ref = (spool.slicer_filament or "").strip()
  2823. if lp_ref.isdigit() and int(lp_ref) <= 2147483647:
  2824. from backend.app.models.local_preset import LocalPreset as _LP
  2825. lp_type = await db.scalar(select(_LP.filament_type).where(_LP.id == int(lp_ref)))
  2826. if lp_type:
  2827. spool_types.add(printer_filament_type(lp_type).upper())
  2828. # A support spool went in with the slicer's suffix
  2829. # ("PLA-S") that none of the above carries (#3559).
  2830. # A spool made from a support RFID tray has its
  2831. # material as "Support For PLA", so that counts too.
  2832. support_name = (
  2833. f"{spool.slicer_filament_name or ''} {spool.material or ''} {spool.subtype or ''}"
  2834. )
  2835. spool_types |= {
  2836. support_filament_type(t, spool.slicer_filament, support_name).upper()
  2837. for t in spool_types
  2838. }
  2839. if (
  2840. _colors_similar(cur_color, spool_color)
  2841. and printer_filament_type(cur_type).upper() in spool_types
  2842. ):
  2843. logger.info(
  2844. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch but tray matches spool, updating fp",
  2845. assignment.spool_id,
  2846. assignment.ams_id,
  2847. assignment.tray_id,
  2848. )
  2849. assignment.fingerprint_color = cur_color
  2850. assignment.fingerprint_type = cur_type
  2851. continue
  2852. logger.info(
  2853. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch (cur=%s/%s fp=%s/%s spool=%s/%s)",
  2854. assignment.spool_id,
  2855. assignment.ams_id,
  2856. assignment.tray_id,
  2857. cur_color,
  2858. cur_type,
  2859. fp_color,
  2860. fp_type,
  2861. spool.rgba if spool else "?",
  2862. spool.material if spool else "?",
  2863. )
  2864. stale.append(assignment) # Spool changed
  2865. slot_unlink_grace.settle(printer_id, "inventory", held)
  2866. # Snapshot slots before delete — ORM attribute access after the
  2867. # commit would refresh against a deleted row.
  2868. unlinked_slots = [(a.ams_id, a.tray_id) for a in stale]
  2869. for a in stale:
  2870. await db.delete(a)
  2871. if stale:
  2872. logger.info("Auto-unlinked %d stale spool assignments for printer %d", len(stale), printer_id)
  2873. # Commit any changes (stale deletions and/or fingerprint updates)
  2874. await db.commit()
  2875. # Tell open browsers the assignment is gone (#2575). Only the manual
  2876. # REST assign/unassign endpoints broadcast this event; without it the
  2877. # frontend's spool-assignments cache keeps rendering the unlinked
  2878. # spool on the slot until an unrelated refetch — which reads exactly
  2879. # like "the fix didn't work" (reporter verified: a browser refresh
  2880. # after the swap showed the correct state all along).
  2881. for ams_id, tray_id in unlinked_slots:
  2882. await ws_manager.broadcast(
  2883. {
  2884. "type": "spool_assignment_changed",
  2885. "printer_id": printer_id,
  2886. "ams_id": ams_id,
  2887. "tray_id": tray_id,
  2888. }
  2889. )
  2890. except Exception as e:
  2891. logger.warning("Spool assignment cleanup failed: %s", e, exc_info=True)
  2892. async def _expire_spoolman_empty_slots(printer_id: int, ams_data: list, printing_now: bool) -> None:
  2893. """Delete Spoolman slot rows whose held removal has run its grace period.
  2894. The Spoolman half of the #3186 re-check. The full sync in ``on_ams_change``
  2895. talks to Spoolman for every tray; this only needs the local rows, so it
  2896. repeats that pass's empty-slot decision -- a tray with no type or no colour
  2897. (``parse_ams_tray`` returns None for exactly those), not during a print, and
  2898. not in a slot the presence bit calls occupied -- and nothing else.
  2899. """
  2900. logger = logging.getLogger(__name__)
  2901. try:
  2902. async with async_session() as db:
  2903. from backend.app.api.routes.settings import get_setting
  2904. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  2905. from backend.app.services.ams_slot_presence import spool_present
  2906. enabled = await get_setting(db, "spoolman_enabled")
  2907. if not enabled or enabled.lower() != "true":
  2908. return
  2909. sync_mode = await get_setting(db, "spoolman_sync_mode")
  2910. if sync_mode and sync_mode != "auto":
  2911. return
  2912. trays: dict[tuple[int, int], dict] = {}
  2913. for ams_unit in ams_data or []:
  2914. if not isinstance(ams_unit, dict):
  2915. continue
  2916. for tray in ams_unit.get("tray", []):
  2917. if isinstance(tray, dict):
  2918. trays[(int(ams_unit.get("id", 0)), int(tray.get("id", 0)))] = tray
  2919. rows = (
  2920. (
  2921. await db.execute(
  2922. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  2923. )
  2924. )
  2925. .scalars()
  2926. .all()
  2927. )
  2928. held: set[tuple] = set()
  2929. expired: list[tuple[int, int]] = []
  2930. for row in rows:
  2931. tray = trays.get((row.ams_id, row.tray_id))
  2932. if tray is None or printing_now:
  2933. continue
  2934. if (tray.get("tray_type") or "").strip() and (tray.get("tray_color") or "").strip():
  2935. continue
  2936. hold_key = ("spoolman", row.ams_id, row.tray_id, row.spoolman_spool_id)
  2937. if spool_present(tray) is True and not slot_unlink_grace.is_held(printer_id, hold_key):
  2938. continue
  2939. if slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2940. expired.append((row.ams_id, row.tray_id))
  2941. else:
  2942. held.add(hold_key)
  2943. slot_unlink_grace.settle(printer_id, "spoolman", held)
  2944. if not expired:
  2945. return
  2946. # A statement rather than ORM deletes, like the sync pass: the two
  2947. # can overlap, and a row the other already removed must not fail
  2948. # this commit.
  2949. for ams_id, tray_id in expired:
  2950. await db.execute(
  2951. delete(SpoolmanSlotAssignment).where(
  2952. SpoolmanSlotAssignment.printer_id == printer_id,
  2953. SpoolmanSlotAssignment.ams_id == ams_id,
  2954. SpoolmanSlotAssignment.tray_id == tray_id,
  2955. )
  2956. )
  2957. await db.commit()
  2958. logger.info("Unlinked %d Spoolman slot(s) that stayed empty for printer %d", len(expired), printer_id)
  2959. for ams_id, tray_id in expired:
  2960. await ws_manager.broadcast(
  2961. {
  2962. "type": "spool_assignment_changed",
  2963. "printer_id": printer_id,
  2964. "ams_id": ams_id,
  2965. "tray_id": tray_id,
  2966. }
  2967. )
  2968. except Exception as e:
  2969. logger.warning("Spoolman slot re-check failed for printer %s: %s", printer_id, e, exc_info=True)
  2970. async def _recheck_held_unlinks(printer_id: int) -> None:
  2971. """Re-run just the slot cleanup against the printer's current AMS state.
  2972. Scheduled by ``slot_unlink_grace`` when it holds a removal. The unlink
  2973. passes otherwise run only when the AMS hash changes, and a slot that went
  2974. empty and stayed empty may never change it again.
  2975. """
  2976. status = printer_manager.get_status(printer_id)
  2977. # A disconnected printer's state is its last report, not a new one: acting
  2978. # on it would "confirm" a removal nobody has seen for the whole grace
  2979. # period. Leave the holds; the first push after reconnecting decides.
  2980. if status is None or not status.connected:
  2981. return
  2982. ams_raw = status.raw_data.get("ams")
  2983. ams_data = ams_raw.get("ams", []) if isinstance(ams_raw, dict) else ams_raw if isinstance(ams_raw, list) else []
  2984. printing_now = (getattr(status, "state", "") or "").upper() in ("RUNNING", "PAUSE")
  2985. await _unlink_stale_assignments(printer_id, ams_data, printing_now)
  2986. await _expire_spoolman_empty_slots(printer_id, ams_data, printing_now)
  2987. slot_unlink_grace.set_recheck(_recheck_held_unlinks)
  2988. async def on_ams_change(printer_id: int, ams_data: list):
  2989. """Handle AMS data changes - sync to Spoolman if enabled and auto mode."""
  2990. logger = logging.getLogger(__name__)
  2991. # Snapshot BEFORE any await: if a print is active, or a finished one is
  2992. # still being charged, skip weight sync later. on_print_complete may pop
  2993. # _active_sessions during our awaits (#880).
  2994. from backend.app.services.usage_tracker import printer_is_charging
  2995. _print_active = printer_is_charging(printer_id)
  2996. # A slot that reports empty while a print is running is a filament runout,
  2997. # not a spool swap: the spool is still physically in the AMS, just
  2998. # consumed. Dropping either inventory backend's slot link there loses the
  2999. # only record of which spool fed the print, so the completion path can't
  3000. # charge the runout segment to anything. Both cleanup passes below consult
  3001. # this; computed once, up front, so neither depends on the other having run.
  3002. _unlink_state = printer_manager.get_status(printer_id)
  3003. printing_now = (getattr(_unlink_state, "state", "") or "").upper() in ("RUNNING", "PAUSE")
  3004. # The session alone is not enough: a restart that finds a stale persisted
  3005. # session registers none for the print that is running.
  3006. _print_active = _print_active or printing_now
  3007. # MQTT relay - publish AMS change
  3008. try:
  3009. printer_info = printer_manager.get_printer(printer_id)
  3010. if printer_info:
  3011. await mqtt_relay.on_ams_change(printer_id, printer_info.name, printer_info.serial_number, ams_data)
  3012. except Exception:
  3013. pass # Don't fail AMS callback if MQTT fails
  3014. # Broadcast AMS change via WebSocket (bypasses status_key deduplication)
  3015. # This ensures frontend gets immediate updates when AMS slots are configured
  3016. try:
  3017. state = printer_manager.get_status(printer_id)
  3018. if state:
  3019. logger.info("[Printer %s] Broadcasting AMS change via WebSocket", printer_id)
  3020. await ws_manager.send_printer_status(
  3021. printer_id,
  3022. printer_state_to_dict(
  3023. state,
  3024. printer_id,
  3025. printer_manager.get_model(printer_id),
  3026. printer_manager.get_drying_targets(printer_id),
  3027. ),
  3028. )
  3029. except Exception as e:
  3030. logger.warning("Failed to broadcast AMS change for printer %s: %s", printer_id, e)
  3031. await _unlink_stale_assignments(printer_id, ams_data, printing_now)
  3032. # Spools whose tag this pass assigned to a slot; their other slots are
  3033. # released below, once this printer's lock is let go (#3427).
  3034. rfid_assigned: list[tuple[int, tuple[int, int, int]]] = []
  3035. # Auto-manage inventory spools from AMS tray data (skip if Spoolman manages AMS).
  3036. # Serialised per-printer via services/ams_assignment_lock: MQTT bursts can deliver
  3037. # two AMS pushes ~30 ms apart, and without the lock both callbacks read
  3038. # "no existing assignment" for the same (printer, ams, tray) and race to
  3039. # INSERT, hitting the spool_assignment_printer_id_ams_id_tray_id_key
  3040. # unique constraint on Postgres. SQLite's WAL serialises writes so the
  3041. # bug stayed latent there. See services/ams_assignment_lock for details.
  3042. try:
  3043. async with get_ams_assignment_lock(printer_id), async_session() as db:
  3044. from sqlalchemy.orm import selectinload
  3045. from backend.app.api.routes.settings import get_setting
  3046. from backend.app.models.spool import Spool
  3047. from backend.app.models.spool_assignment import SpoolAssignment as SA
  3048. from backend.app.services.spool_tag_matcher import (
  3049. auto_assign_spool,
  3050. create_spool_from_tray,
  3051. find_matching_untagged_spool,
  3052. get_spool_by_tag,
  3053. is_bambu_tag,
  3054. is_valid_tag,
  3055. link_tag_to_inventory_spool,
  3056. tag_belongs_to_archived_spool,
  3057. )
  3058. _spoolman_on = await get_setting(db, "spoolman_enabled")
  3059. _auto_add_raw = await get_setting(db, "auto_add_unknown_rfid")
  3060. _auto_add_unknown = _auto_add_raw is None or _auto_add_raw.lower() == "true"
  3061. if not _spoolman_on or _spoolman_on.lower() != "true":
  3062. for ams_unit in ams_data:
  3063. if not isinstance(ams_unit, dict):
  3064. continue
  3065. ams_id = int(ams_unit.get("id", 0))
  3066. for tray in ams_unit.get("tray", []):
  3067. if not isinstance(tray, dict):
  3068. continue
  3069. tray_id = int(tray.get("id", 0))
  3070. tag_uid = tray.get("tag_uid", "")
  3071. tray_uuid = tray.get("tray_uuid", "")
  3072. tray_info_idx = tray.get("tray_info_idx", "")
  3073. if not tray.get("tray_type"):
  3074. # Slot reported empty — drop any cached unknown-tag
  3075. # broadcast so reinserting the same spool re-prompts.
  3076. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id)
  3077. continue # Empty slot
  3078. # Check if assignment already exists for this slot
  3079. existing = await db.execute(
  3080. select(SA)
  3081. .options(selectinload(SA.spool).selectinload(Spool.k_profiles))
  3082. .where(SA.printer_id == printer_id, SA.ams_id == ams_id, SA.tray_id == tray_id)
  3083. )
  3084. existing_assignment = existing.scalar_one_or_none()
  3085. if existing_assignment:
  3086. # Sync spool weight_used from AMS remain — only INCREASE, never decrease.
  3087. # The AMS remain% is low-resolution (integer %, i.e. 10g steps for 1kg spool)
  3088. # and must not overwrite precise values from the usage tracker (3MF/G-code).
  3089. # Skip during active prints: the usage tracker handles deduction
  3090. # precisely via 3MF data on print completion. Without this guard the
  3091. # AMS remain% SET and the usage tracker ADD both fire from the same
  3092. # MQTT message, doubling the deduction (#880).
  3093. if _print_active:
  3094. continue
  3095. remain_raw = tray.get("remain")
  3096. if (
  3097. remain_raw is not None
  3098. and existing_assignment.spool
  3099. and not existing_assignment.spool.weight_locked
  3100. ):
  3101. try:
  3102. remain_val = int(remain_raw)
  3103. except (TypeError, ValueError):
  3104. remain_val = -1
  3105. if 1 <= remain_val <= 100:
  3106. lw = existing_assignment.spool.label_weight or 1000
  3107. new_used = round(lw * (100 - remain_val) / 100.0, 1)
  3108. current_used = existing_assignment.spool.weight_used or 0
  3109. if new_used > current_used + 1:
  3110. # Rechecked in the statement: the spool was loaded
  3111. # before this, and a weight set by hand (or weighed)
  3112. # and locked since then must stay (WL-6).
  3113. result = cast(
  3114. CursorResult,
  3115. await db.execute(
  3116. update(Spool)
  3117. .where(
  3118. Spool.id == existing_assignment.spool_id,
  3119. Spool.weight_locked.is_not(True),
  3120. func.coalesce(Spool.weight_used, 0) < new_used - 1,
  3121. )
  3122. .values(weight_used=new_used)
  3123. ),
  3124. )
  3125. await db.commit()
  3126. if result.rowcount:
  3127. logger.info(
  3128. "Weight sync: spool %d weight_used %s -> %s (remain=%d)",
  3129. existing_assignment.spool_id,
  3130. current_used,
  3131. new_used,
  3132. remain_val,
  3133. )
  3134. await ws_manager.broadcast({"type": "inventory_changed"})
  3135. # A K-profile the slot lost (the default K after a reset or
  3136. # power cycle) is put back by kprofile_drift on the next idle
  3137. # push; a different real profile was picked on purpose and
  3138. # stays (#3219, item 44).
  3139. continue
  3140. if is_bambu_tag(tag_uid, tray_uuid, tray_info_idx):
  3141. # BL spool with RFID tag: auto-match → inventory match → auto-create
  3142. spool = await get_spool_by_tag(db, tag_uid, tray_uuid)
  3143. # Only a spool already known by this tag is proven to be
  3144. # here; a material/colour match or a new spool is not
  3145. # grounds to take it out of other slots (#3427).
  3146. known_by_tag = spool is not None
  3147. if not spool and await tag_belongs_to_archived_spool(db, tag_uid, tray_uuid):
  3148. # An archived spool still in the AMS: the slot
  3149. # stays free rather than adopting its tag.
  3150. continue
  3151. if not spool:
  3152. # Try matching an untagged inventory spool (same material/color)
  3153. spool = await find_matching_untagged_spool(db, tray)
  3154. if spool:
  3155. await link_tag_to_inventory_spool(db, spool, tray)
  3156. elif _auto_add_unknown:
  3157. spool = await create_spool_from_tray(db, tray)
  3158. else:
  3159. # Auto-add disabled: surface the slot so the
  3160. # user can add it manually via the UI.
  3161. await _broadcast_unknown_tag(
  3162. printer_id=printer_id,
  3163. ams_id=ams_id,
  3164. tray_id=tray_id,
  3165. tag_uid=tag_uid,
  3166. tray_uuid=tray_uuid,
  3167. tray_type=tray.get("tray_type"),
  3168. tray_color=tray.get("tray_color"),
  3169. tray_sub_brands=tray.get("tray_sub_brands"),
  3170. tray_count=len(ams_unit.get("tray", [])),
  3171. )
  3172. continue
  3173. # Slot matched (existing tag, untagged inventory
  3174. # match, or freshly auto-created spool) — drop any
  3175. # stale dedup so a future tag swap re-prompts.
  3176. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id)
  3177. await auto_assign_spool(
  3178. printer_id,
  3179. ams_id,
  3180. tray_id,
  3181. spool,
  3182. printer_manager,
  3183. db,
  3184. tray_info_idx=tray_info_idx,
  3185. )
  3186. await db.commit()
  3187. if known_by_tag:
  3188. rfid_assigned.append((spool.id, (printer_id, ams_id, tray_id)))
  3189. await ws_manager.broadcast(
  3190. {
  3191. "type": "spool_auto_assigned",
  3192. "printer_id": printer_id,
  3193. "ams_id": ams_id,
  3194. "tray_id": tray_id,
  3195. "spool_id": spool.id,
  3196. }
  3197. )
  3198. logger.info(
  3199. "RFID auto-assigned spool %d to printer %d AMS%d-T%d",
  3200. spool.id,
  3201. printer_id,
  3202. ams_id,
  3203. tray_id,
  3204. )
  3205. elif is_valid_tag(tag_uid, tray_uuid) and not await tag_belongs_to_archived_spool(
  3206. db, tag_uid, tray_uuid
  3207. ):
  3208. # Non-BL spool with some tag — let user choose (not
  3209. # an archived one's, whose slot stays free)
  3210. await _broadcast_unknown_tag(
  3211. printer_id=printer_id,
  3212. ams_id=ams_id,
  3213. tray_id=tray_id,
  3214. tag_uid=tag_uid,
  3215. tray_uuid=tray_uuid,
  3216. tray_type=tray.get("tray_type"),
  3217. tray_color=tray.get("tray_color"),
  3218. tray_sub_brands=tray.get("tray_sub_brands"),
  3219. tray_count=len(ams_unit.get("tray", [])),
  3220. )
  3221. # No-tag slots (generic non-RFID filament) are left alone:
  3222. # nothing to identify, prompting "+ Add" would just create
  3223. # ghost spools with empty tags on every confirm.
  3224. except Exception as e:
  3225. logger.warning("RFID spool auto-assign failed: %s", e, exc_info=True)
  3226. if rfid_assigned:
  3227. from backend.app.services.spool_relocation import INTERNAL, release_other_slots
  3228. for spool_id, slot in rfid_assigned:
  3229. try:
  3230. await release_other_slots(INTERNAL, spool_id, slot)
  3231. except Exception as e:
  3232. logger.warning("Releasing spool %s from its other slots failed: %s", spool_id, e, exc_info=True)
  3233. try:
  3234. async with async_session() as db:
  3235. from backend.app.api.routes.settings import get_setting
  3236. from backend.app.models.printer import Printer
  3237. # Check if Spoolman is enabled
  3238. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  3239. if not spoolman_enabled or spoolman_enabled.lower() != "true":
  3240. return
  3241. # Check sync mode
  3242. sync_mode = await get_setting(db, "spoolman_sync_mode")
  3243. if sync_mode and sync_mode != "auto":
  3244. return # Only sync on auto mode
  3245. _auto_add_raw_sm = await get_setting(db, "auto_add_unknown_rfid")
  3246. auto_add_unknown_rfid = _auto_add_raw_sm is None or _auto_add_raw_sm.lower() == "true"
  3247. # `spoolman_disable_weight_sync` is deprecated (#1119) — weight is now
  3248. # always owned by per-print tracking, never by AMS auto-sync. The
  3249. # setting is still read by the settings UI for backwards compat but
  3250. # has no effect on the sync path here.
  3251. # Get Spoolman URL
  3252. spoolman_url = await get_setting(db, "spoolman_url")
  3253. if not spoolman_url:
  3254. return
  3255. # Get or create Spoolman client
  3256. client = await get_spoolman_client()
  3257. if not client:
  3258. try:
  3259. client = await init_spoolman_client(spoolman_url)
  3260. except ValueError as exc:
  3261. logger.warning("Spoolman URL %r rejected by SSRF guard: %s", spoolman_url, exc)
  3262. return
  3263. # Check if Spoolman is reachable
  3264. if not await client.health_check():
  3265. logger.warning("Spoolman not reachable at %s", spoolman_url)
  3266. return
  3267. # Get printer name for location
  3268. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3269. printer = result.scalar_one_or_none()
  3270. printer_name = printer.name if printer else f"Printer {printer_id}"
  3271. # OPTIMIZATION: Fetch all spools once before processing trays
  3272. # This eliminates redundant API calls (one per tray) when syncing multiple trays
  3273. logger.debug("[Printer %s] Fetching spools cache for AMS sync...", printer_id)
  3274. try:
  3275. cached_spools = await client.get_spools()
  3276. logger.debug("[Printer %s] Cached %d spools for batch sync", printer_id, len(cached_spools))
  3277. except Exception as e:
  3278. logger.error(
  3279. "[Printer %s] Failed to fetch spools cache after retries, aborting AMS sync: %s",
  3280. printer_id,
  3281. e,
  3282. )
  3283. return
  3284. # Load inventory weights as fallback (when AMS MQTT data lacks remain values)
  3285. from sqlalchemy.orm import selectinload
  3286. from backend.app.models.spool_assignment import SpoolAssignment
  3287. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  3288. from backend.app.services.ams_slot_presence import spool_present
  3289. from backend.app.services.inventory_mode import spoolman_owns_assignments
  3290. # Built-in remaining weight, used by sync_ams_tray only when the
  3291. # firmware reports an unusable remain%/tray_weight for a slot.
  3292. #
  3293. # Left empty since #2812. This block runs in Spoolman mode only,
  3294. # and until then the built-in table was emptied on the switch, so
  3295. # there was never anything here to read and the fallback was inert.
  3296. # Preserving those rows makes it live again, and it is keyed by slot
  3297. # rather than by spool: after a mode switch the tray may well hold
  3298. # different filament, and ``create_spool`` writes ``remaining_weight``
  3299. # unconditionally, so a stale figure would be seeded into a brand new
  3300. # Spoolman spool. Deliberately kept inert rather than deleted, so the
  3301. # intent survives for whoever revisits the cross-mode fallback.
  3302. inventory_weights: dict[tuple[int, int], float] = {}
  3303. if not await spoolman_owns_assignments(db):
  3304. try:
  3305. assign_result = await db.execute(
  3306. select(SpoolAssignment)
  3307. .options(selectinload(SpoolAssignment.spool))
  3308. .where(SpoolAssignment.printer_id == printer_id)
  3309. )
  3310. for assignment in assign_result.scalars().all():
  3311. spool = assignment.spool
  3312. if spool and spool.label_weight > 0:
  3313. remaining = max(0.0, spool.label_weight - (spool.weight_used or 0))
  3314. inventory_weights[(assignment.ams_id, assignment.tray_id)] = remaining
  3315. except Exception as e:
  3316. logger.warning("Could not load inventory weights for printer %s: %s", printer_id, e)
  3317. # Load existing Spoolman slot assignments for the no-RFID fallback path
  3318. spoolman_slot_map: dict[tuple[int, int], int] = {}
  3319. slot_map_read = False
  3320. try:
  3321. slot_result = await db.execute(
  3322. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  3323. )
  3324. for slot in slot_result.scalars().all():
  3325. spoolman_slot_map[(slot.ams_id, slot.tray_id)] = slot.spoolman_spool_id
  3326. slot_map_read = True
  3327. except Exception as e:
  3328. logger.warning("Could not load Spoolman slot assignments for printer %s: %s", printer_id, e)
  3329. # Sync each AMS tray and collect slot changes for DB persistence
  3330. synced = 0
  3331. slot_changes: list[tuple[int, int, int]] = [] # (ams_id, tray_id, spoolman_spool_id) to upsert
  3332. empty_slots: list[tuple[int, int]] = [] # (ams_id, tray_id) whose tray is now empty
  3333. spoolman_held: set[tuple] = set() # removals held for the grace period (#3186)
  3334. for ams_unit in ams_data:
  3335. if not isinstance(ams_unit, dict):
  3336. continue
  3337. ams_id = int(ams_unit.get("id", 0))
  3338. trays = ams_unit.get("tray", [])
  3339. for tray_data in trays:
  3340. if not isinstance(tray_data, dict):
  3341. continue
  3342. tray_id_raw = int(tray_data.get("id", 0))
  3343. tray = client.parse_ams_tray(ams_id, tray_data)
  3344. if not tray:
  3345. # Empty tray slot — record for local assignment cleanup
  3346. # and drop any cached unknown-tag broadcast so a
  3347. # reinserted spool re-prompts.
  3348. #
  3349. # Not during a running print: a slot that empties there
  3350. # is a filament runout, and the spool is still in the
  3351. # AMS. `spoolman_slot_assignments` is how a tag-less
  3352. # spool assigned through the Bambuddy UI is resolved at
  3353. # completion (#1459), so deleting the row mid-print
  3354. # loses the runout segment's usage — the same failure
  3355. # the internal inventory's auto-unlink had.
  3356. #
  3357. # Nor when firmware's presence bit says the slot is
  3358. # occupied. parse_ams_tray calls a tray with no type or
  3359. # no colour empty, and a spool the AMS cannot read has
  3360. # neither until something configures it — so a tag-less
  3361. # spool assigned through the UI had its row deleted by
  3362. # the first idle push after it was inserted. Same
  3363. # deletion as the internal inventory's in #3100, same
  3364. # answer, so the two modes stay in step.
  3365. #
  3366. # And only once the slot has stayed empty for the grace
  3367. # period -- the internal inventory's #3186 answer, for
  3368. # the same one-push blank.
  3369. linked_spool = spoolman_slot_map.get((ams_id, tray_id_raw))
  3370. hold_key = ("spoolman", ams_id, tray_id_raw, linked_spool)
  3371. if not printing_now and (
  3372. spool_present(tray_data) is not True or slot_unlink_grace.is_held(printer_id, hold_key)
  3373. ):
  3374. if linked_spool is None or slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  3375. empty_slots.append((ams_id, tray_id_raw))
  3376. else:
  3377. spoolman_held.add(hold_key)
  3378. logger.info(
  3379. "Spoolman slot unlink held: AMS%d-T%d (spool %d) reports empty; "
  3380. "unlinking if it is still empty in %ds",
  3381. ams_id,
  3382. tray_id_raw,
  3383. linked_spool,
  3384. int(slot_unlink_grace.GRACE_SECONDS),
  3385. )
  3386. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id_raw)
  3387. continue
  3388. spool_tag = (
  3389. tray.tray_uuid
  3390. if tray.tray_uuid and tray.tray_uuid != "00000000000000000000000000000000"
  3391. else tray.tag_uid
  3392. )
  3393. # Provide the hint only when no RFID is available
  3394. hint = spoolman_slot_map.get((ams_id, tray.tray_id)) if not spool_tag else None
  3395. try:
  3396. inv_remaining = inventory_weights.get((ams_id, tray.tray_id))
  3397. result = await client.sync_ams_tray(
  3398. tray,
  3399. printer_name,
  3400. db,
  3401. # Per-print tracking is the only weight writer (#1119).
  3402. # AMS auto-sync still maintains spool metadata / slot
  3403. # assignments but no longer touches remaining_weight.
  3404. disable_weight_sync=True,
  3405. cached_spools=cached_spools,
  3406. inventory_remaining=inv_remaining,
  3407. spoolman_spool_id_hint=hint,
  3408. auto_add_unknown_rfid=auto_add_unknown_rfid,
  3409. )
  3410. if result is None and spool_tag and not auto_add_unknown_rfid:
  3411. # Spoolman skipped auto-create per user setting — surface
  3412. # the slot so the UI can offer "+ Add to inventory".
  3413. await _broadcast_unknown_tag(
  3414. printer_id=printer_id,
  3415. ams_id=ams_id,
  3416. tray_id=tray.tray_id,
  3417. tag_uid=tray.tag_uid or "",
  3418. tray_uuid=tray.tray_uuid or "",
  3419. tray_type=tray.tray_type,
  3420. tray_color=tray.tray_color,
  3421. tray_sub_brands=tray.tray_sub_brands,
  3422. tray_count=len(trays),
  3423. )
  3424. elif result:
  3425. _clear_unknown_tag_dedup(printer_id, ams_id, tray.tray_id)
  3426. if result:
  3427. synced += 1
  3428. if result.get("id"):
  3429. slot_changes.append((ams_id, tray.tray_id, result["id"]))
  3430. # If a new spool was created, add it to the cache
  3431. # so subsequent trays can find it if they reference the same tag
  3432. spool_exists = any(s.get("id") == result["id"] for s in cached_spools)
  3433. if not spool_exists:
  3434. cached_spools.append(result)
  3435. logger.debug(
  3436. "[Printer %s] Added newly created spool %s to cache",
  3437. printer_id,
  3438. result["id"],
  3439. )
  3440. # Reconcile slot_preset_mappings (the same row internal
  3441. # mode keeps in sync via inventory + spool_tag_matcher).
  3442. # Without this the slot card surfaces the previous spool's
  3443. # preset name — same bug shape, different inventory mode.
  3444. from backend.app.services.slot_preset_writer import (
  3445. upsert_slot_preset_for_spoolman_spool,
  3446. )
  3447. await upsert_slot_preset_for_spoolman_spool(
  3448. db=db,
  3449. spoolman_spool=result,
  3450. tray_info_idx=tray.tray_info_idx or "",
  3451. tray_sub_brands=tray.tray_sub_brands or "",
  3452. tray_type=tray.tray_type or "",
  3453. printer_id=printer_id,
  3454. ams_id=ams_id,
  3455. tray_id=tray.tray_id,
  3456. )
  3457. except Exception as e:
  3458. logger.error("Error syncing AMS %s tray %s: %s", ams_id, tray.tray_id, e)
  3459. slot_unlink_grace.settle(printer_id, "spoolman", spoolman_held)
  3460. if synced > 0:
  3461. logger.info("Auto-synced %s AMS trays to Spoolman for printer %s", synced, printer_id)
  3462. # Persist slot assignment changes to the local table: only slots
  3463. # whose spool this pass changed, and only while they still hold what
  3464. # it read before awaiting Spoolman, so a spool assigned by hand in
  3465. # between stays (SSA-3). The changed slots are announced: the slot
  3466. # card reads slot_preset_mappings (rewritten above) ahead of the live
  3467. # tray, so with no event it kept the previous spool's preset name.
  3468. if slot_changes or empty_slots:
  3469. from backend.app.services.spoolman_slot_sync import write_slot_results
  3470. try:
  3471. await write_slot_results(
  3472. db,
  3473. {(printer_id, a_id, t_id): spool_id for (a_id, t_id), spool_id in spoolman_slot_map.items()}
  3474. if slot_map_read
  3475. else None,
  3476. [(printer_id, a_id, t_id, spool_id) for a_id, t_id, spool_id in slot_changes],
  3477. [(printer_id, a_id, t_id) for a_id, t_id in empty_slots],
  3478. )
  3479. except Exception as e:
  3480. await db.rollback()
  3481. logger.error("Error persisting Spoolman slot assignments for printer %s: %s", printer_id, e)
  3482. except Exception as e:
  3483. logging.getLogger(__name__).error("Spoolman AMS sync failed for printer %s: %s", printer_id, e)
  3484. # Largest finish photo attached to a notification; a bigger one is still linked.
  3485. _FINISH_PHOTO_ATTACH_MAX_BYTES = 2_500_000
  3486. async def _finish_photo_for_notification(
  3487. db, archive, archive_id: int, filename: str
  3488. ) -> tuple[str | None, bytes | None]:
  3489. """The ``{finish_photo_url}`` link and the bytes to attach, for a print's finish photo.
  3490. With authentication off the link is the archive's own photo route, as it
  3491. always was: it opens without a login and doesn't expire. With
  3492. authentication on that route needs a media token, which nothing tapping a
  3493. link in Telegram, CallMeBot or a Home Assistant notification has, so the
  3494. link only ever answered 401. The photo is then saved as a notification
  3495. photo as well (utils/notification_photos.py) and the link points there: an
  3496. unguessable name that opens this one photo and nothing else, for 3 days.
  3497. The link is relative when no External URL is set. Bytes over
  3498. ``_FINISH_PHOTO_ATTACH_MAX_BYTES`` are linked but not attached. Returns
  3499. ``(None, None)`` when the photo can't be found.
  3500. """
  3501. from backend.app.api.routes.settings import get_setting
  3502. from backend.app.core.auth import is_auth_enabled
  3503. from backend.app.utils.archive_paths import find_archive_photo
  3504. from backend.app.utils.notification_photos import save_notification_photo
  3505. log = logging.getLogger(__name__)
  3506. base = ((await get_setting(db, "external_url")) or "").strip().rstrip("/")
  3507. url: str | None = f"{base}/api/v1/archives/{archive_id}/photos/{filename}"
  3508. photo_bytes: bytes | None = None
  3509. try:
  3510. photo_path = find_archive_photo(archive, filename)
  3511. if photo_path is not None:
  3512. photo_bytes = await asyncio.to_thread(photo_path.read_bytes)
  3513. except Exception as e:
  3514. log.warning("[NOTIFY-BG] Failed to read finish photo bytes: %s", e)
  3515. try:
  3516. auth_on = await is_auth_enabled(db)
  3517. except Exception:
  3518. auth_on = True # Unknown: the archive link may need a login, so don't rely on it.
  3519. if auth_on:
  3520. url = None
  3521. if photo_bytes:
  3522. try:
  3523. name = await asyncio.to_thread(save_notification_photo, photo_bytes, "print_complete")
  3524. url = f"{base}/api/v1/notifications/photos/{name}"
  3525. except Exception as e:
  3526. log.warning("[NOTIFY-BG] Failed to save finish photo for its link: %s", e)
  3527. if photo_bytes is not None and len(photo_bytes) > _FINISH_PHOTO_ATTACH_MAX_BYTES:
  3528. log.warning("[NOTIFY-BG] Finish photo too large for attachment: %s bytes", len(photo_bytes))
  3529. return url, None
  3530. if photo_bytes:
  3531. log.info("[NOTIFY-BG] Loaded finish photo bytes: %s bytes", len(photo_bytes))
  3532. return url, photo_bytes
  3533. async def _capture_snapshot_for_notification(printer_id: int, printer, logger, *, light: bool = True) -> bytes | None:
  3534. """Capture a camera snapshot for notification image attachment.
  3535. Returns JPEG bytes (max 2.5MB) or None if capture fails or is unavailable.
  3536. Uses: external camera > buffered frame > fresh capture.
  3537. Turns the chamber light on for the picture when the printer asks for it
  3538. (#1655). ``light=False`` is for callers that capture all through a print,
  3539. where the light would flash with every frame.
  3540. """
  3541. if not printer:
  3542. return None
  3543. async with camera_light(printer if light else None):
  3544. return await _capture_snapshot_frame(printer_id, printer, logger)
  3545. async def _capture_snapshot_frame(printer_id: int, printer, logger) -> bytes | None:
  3546. """The capture behind ``_capture_snapshot_for_notification``."""
  3547. try:
  3548. from backend.app.api.routes.settings import get_setting
  3549. async with async_session() as db:
  3550. capture_enabled = await get_setting(db, "capture_finish_photo")
  3551. if capture_enabled is not None and capture_enabled.lower() != "true":
  3552. return None
  3553. # Try external camera first
  3554. if printer.external_camera_enabled and printer.external_camera_url:
  3555. logger.info("[SNAPSHOT] Capturing from external camera for printer %s", printer_id)
  3556. from backend.app.api.routes.camera import live_frame_for_capture
  3557. from backend.app.services.external_camera import capture_frame
  3558. # An external camera allows one reader, so capturing while a viewer
  3559. # is attached fails (#2707). A None here falls through to the paths
  3560. # below exactly as a failed capture did.
  3561. defer, buffered = live_frame_for_capture(printer_id)
  3562. if defer:
  3563. frame_data = buffered
  3564. else:
  3565. frame_data = await capture_frame(
  3566. printer.external_camera_url,
  3567. printer.external_camera_type or "mjpeg",
  3568. snapshot_url=printer.external_camera_snapshot_url,
  3569. )
  3570. if frame_data and len(frame_data) <= 2_500_000:
  3571. logger.info("[SNAPSHOT] External camera frame: %s bytes", len(frame_data))
  3572. return _apply_camera_rotation(frame_data, printer, logger)
  3573. # Try buffered frame from active stream
  3574. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  3575. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  3576. active_chamber = [k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")]
  3577. buffered_frame = get_buffered_frame(printer_id)
  3578. if (active_for_printer or active_chamber) and buffered_frame:
  3579. logger.info("[SNAPSHOT] Using buffered frame for printer %s: %s bytes", printer_id, len(buffered_frame))
  3580. if len(buffered_frame) <= 2_500_000:
  3581. return _apply_camera_rotation(buffered_frame, printer, logger)
  3582. # Fresh capture from printer camera
  3583. logger.info("[SNAPSHOT] Capturing fresh frame for printer %s", printer_id)
  3584. from backend.app.services.camera import capture_camera_frame_bytes
  3585. frame_data = await capture_camera_frame_bytes(
  3586. printer.ip_address, printer.access_code, printer.model, timeout=15
  3587. )
  3588. if frame_data and len(frame_data) <= 2_500_000:
  3589. logger.info("[SNAPSHOT] Fresh camera frame: %s bytes", len(frame_data))
  3590. return _apply_camera_rotation(frame_data, printer, logger)
  3591. except Exception as e:
  3592. logger.warning("[SNAPSHOT] Failed to capture snapshot for printer %s: %s", printer_id, e)
  3593. return None
  3594. async def _maybe_bank_inprint_frame(printer_id: int, layer_num: int) -> None:
  3595. """#1867: bank a recent in-print camera frame for the finish photo.
  3596. Called on every layer change and (#2547) on every print-progress advance.
  3597. Grabs one frame (throttled) into ``_inprint_frame_bank`` so the finish-photo
  3598. path has a pre-End-G-code image for prints that end with a plate swap.
  3599. Both drivers are print telemetry that stops the instant printing ends: no
  3600. further layers, and progress freezes before the End G-code (e.g. SwapMod
  3601. plate swap) executes. So the last banked frame is always the finished print,
  3602. never the swapped plate — that property is what the #1867 path relies on and
  3603. it must survive any change to the throttle below.
  3604. Layer changes alone were not enough: they stop when the *final* layer
  3605. begins, which on a three-minute last layer left the bank stale by the whole
  3606. length of that layer (#2547). Progress keeps ticking through it.
  3607. Best-effort: any failure just leaves the previous banked frame.
  3608. """
  3609. logger = logging.getLogger(__name__)
  3610. client = printer_manager.get_client(printer_id)
  3611. state = client.state if client else None
  3612. if not state or state.state != "RUNNING":
  3613. return
  3614. # Only during actual extrusion — firmware ticks layer_num during the
  3615. # pre-print calibration sequence, whose sub-stages are non-zero.
  3616. if state.mc_print_sub_stage not in (None, 0):
  3617. return
  3618. # #2547: throttled uniformly, with no last-layer exemption. The old code
  3619. # bypassed the throttle on the final layer to guarantee a fresh frame there;
  3620. # now that progress advances also drive banking, that exemption would fire a
  3621. # camera grab on every percent tick of the last layer. Bambu printers accept
  3622. # one RTSP client at a time, so each grab contends with the live view.
  3623. now = time.monotonic()
  3624. last = _inprint_frame_bank_ts.get(printer_id, 0.0)
  3625. if (now - last) < _INPRINT_BANK_MIN_INTERVAL:
  3626. return
  3627. total = state.total_layers or 0
  3628. try:
  3629. async with async_session() as db:
  3630. from backend.app.models.printer import Printer
  3631. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3632. printer = result.scalar_one_or_none()
  3633. if not printer:
  3634. return
  3635. # Reuses the notification snapshot path, which honours the
  3636. # `capture_finish_photo` setting (returns None when disabled) so we
  3637. # don't bank frames the user never asked for. Without the chamber light
  3638. # (#1655): this runs every 25 seconds for the whole print.
  3639. frame = await _capture_snapshot_for_notification(printer_id, printer, logger, light=False)
  3640. if frame:
  3641. _inprint_frame_bank[printer_id] = frame
  3642. _inprint_frame_bank_ts[printer_id] = now
  3643. logger.debug(
  3644. "[FINISH-PHOTO-BANK] banked in-print frame for printer %s at layer %s/%s (%d bytes)",
  3645. printer_id,
  3646. layer_num,
  3647. total,
  3648. len(frame),
  3649. )
  3650. except Exception as e:
  3651. logger.debug("[FINISH-PHOTO-BANK] bank failed for printer %s: %s", printer_id, e)
  3652. def _apply_camera_rotation(image_data: bytes, printer, logger) -> bytes:
  3653. """Apply camera rotation to snapshot image if configured."""
  3654. from backend.app.services.camera import apply_camera_rotation
  3655. return apply_camera_rotation(image_data, getattr(printer, "camera_rotation", 0), logger)
  3656. async def _send_print_start_notification(
  3657. printer_id: int,
  3658. data: dict,
  3659. archive_data: dict | None = None,
  3660. logger=None,
  3661. ):
  3662. """Helper to send print start notification with optional archive data."""
  3663. if logger is None:
  3664. logger = logging.getLogger(__name__)
  3665. try:
  3666. async with async_session() as db:
  3667. from backend.app.models.printer import Printer
  3668. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3669. printer = result.scalar_one_or_none()
  3670. printer_name = printer.name if printer else f"Printer {printer_id}"
  3671. # Capture camera snapshot for notification image attachment
  3672. image_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  3673. if image_data:
  3674. if archive_data is None:
  3675. archive_data = {}
  3676. archive_data["image_data"] = image_data
  3677. await notification_service.on_print_start(printer_id, printer_name, data, db, archive_data=archive_data)
  3678. # Send user-specific email notification for print start
  3679. if archive_data and archive_data.get("created_by_id"):
  3680. await notification_service.send_user_print_email(
  3681. event_type="user_print_start",
  3682. created_by_id=archive_data["created_by_id"],
  3683. printer_name=printer_name,
  3684. filename=data.get("subtask_name") or data.get("filename", "Unknown"),
  3685. db=db,
  3686. )
  3687. except Exception as e:
  3688. logger.warning("Notification on_print_start failed: %s", e)
  3689. async def _dispatch_user_print_email(
  3690. status: str,
  3691. created_by_id: int | None,
  3692. printer_name: str,
  3693. filename: str,
  3694. db,
  3695. image_data: bytes | None = None,
  3696. finish_photo_url: str | None = None,
  3697. ) -> None:
  3698. """Send a user-specific print-completion email based on print status.
  3699. Maps the normalised print status to the correct event type and delegates
  3700. to :meth:`NotificationService.send_user_print_email`. A single helper
  3701. avoids duplicating the ``if status == "completed" / elif "failed" / elif
  3702. "stopped"`` dispatch block at every call site.
  3703. Does nothing if *created_by_id* is ``None``.
  3704. """
  3705. if created_by_id is None:
  3706. return
  3707. if status == "completed":
  3708. event_type = "user_print_complete"
  3709. elif status == "failed":
  3710. event_type = "user_print_failed"
  3711. elif status in ("stopped", "aborted", "cancelled"):
  3712. event_type = "user_print_stopped"
  3713. else:
  3714. return
  3715. await notification_service.send_user_print_email(
  3716. event_type=event_type,
  3717. created_by_id=created_by_id,
  3718. printer_name=printer_name,
  3719. filename=filename,
  3720. db=db,
  3721. image_data=image_data,
  3722. finish_photo_url=finish_photo_url,
  3723. )
  3724. def _load_objects_from_archive(archive, printer_id: int, logger) -> None:
  3725. """Extract printable objects from an archive's 3MF file and store in printer state."""
  3726. try:
  3727. from backend.app.services.archive import extract_printable_objects_from_archive
  3728. client = printer_manager.get_client(printer_id)
  3729. if not client:
  3730. return
  3731. # Extract with positions for UI overlay, scoped to the plate that
  3732. # is printing — resolve_plate_id is the same resolver /cover uses,
  3733. # so the object list can't disagree with the thumbnail it is drawn
  3734. # over (#2522).
  3735. printable_objects, bbox_all = extract_printable_objects_from_archive(
  3736. app_settings.base_dir / archive.file_path,
  3737. plate_number=resolve_plate_id(client.state),
  3738. )
  3739. if printable_objects:
  3740. client.state.printable_objects = printable_objects
  3741. client.state.printable_objects_bbox_all = bbox_all
  3742. client.state.skipped_objects = []
  3743. logger.info("Loaded %s printable objects for printer %s", len(printable_objects), printer_id)
  3744. except Exception as e:
  3745. logger.debug("Failed to extract printable objects from archive: %s", e)
  3746. def _load_pauses_from_3mf(threemf_path, printer_id: int, *, warned_layers=(), passed_through_layer: int = 0) -> None:
  3747. """Read the running print's slicer pauses from its 3MF, in the background (#2984).
  3748. In a thread: a plate's G-code runs to tens of MB, and the print-start
  3749. handler must not wait for it. Kept only while the printer is still on
  3750. the print they were read for: a new print replaces the list.
  3751. ``warned_layers``: pauses this print was already warned about, kept on its
  3752. archive, so a restart does not send the heads-up again.
  3753. ``passed_through_layer``: for a print resumed after a restart, the layer it
  3754. is on. Its pauses up to there are behind it, though the printer's layer
  3755. reads 0 until it reports the next one.
  3756. """
  3757. client = printer_manager.get_client(printer_id)
  3758. if not client:
  3759. return
  3760. state = client.state
  3761. read_for = state.pause_points
  3762. try:
  3763. plate_id = resolve_plate_id(state)
  3764. except Exception:
  3765. # Never at the cost of the print-start handler this runs inside.
  3766. logging.getLogger(__name__).warning("[Printer %s] Could not tell the plate to read pauses for", printer_id)
  3767. return
  3768. async def _load() -> None:
  3769. from backend.app.utils.threemf_tools import extract_pause_points_from_3mf
  3770. points = await asyncio.to_thread(extract_pause_points_from_3mf, Path(threemf_path), plate_id)
  3771. if state.pause_points is not read_for:
  3772. return
  3773. state.pause_points = [p for p in points if p.layer > passed_through_layer]
  3774. state.pause_warned_layers = set(state.pause_warned_layers) | set(warned_layers)
  3775. if points:
  3776. logging.getLogger(__name__).info(
  3777. "Loaded %s slicer pause(s) for printer %s at layer(s) %s",
  3778. len(points),
  3779. printer_id,
  3780. [p.layer for p in points],
  3781. )
  3782. spawn_background_task(_load(), name=f"load-pauses-{printer_id}")
  3783. def _load_pauses_from_resumed_archive(archive, printer_id: int) -> None:
  3784. """Pauses for a print start taken as the resumption of a running print.
  3785. A restart while the print was paused makes its resume look like a new
  3786. print (#2984): the pauses warned about and passed are kept from before.
  3787. """
  3788. client = printer_manager.get_client(printer_id)
  3789. state = client.state if client else None
  3790. _load_pauses_from_3mf(
  3791. app_settings.base_dir / archive.file_path,
  3792. printer_id,
  3793. warned_layers=pause_eta.warned_layers_of(archive, state) if state is not None else (),
  3794. passed_through_layer=getattr(client, "layer_before_new_print", 0) if client else 0,
  3795. )
  3796. async def _restore_printable_objects(printer_id: int, state, db, logger) -> None:
  3797. """Put the skip-objects list back after a restart mid-print.
  3798. ``PrinterState.printable_objects`` is in-memory only, and the only thing
  3799. that fills it is ``_load_objects_from_archive`` on the print-start paths —
  3800. which the #1304 guard suppresses on the first RUNNING push after startup.
  3801. Everything else this hook restores (the archive, the usage-tracking session,
  3802. the timelapse baseline) was already handled; the object list was not, so a
  3803. restart mid-print took skip-objects away for the rest of that print.
  3804. Nothing recovered it either: the printer card gates its Skip button on the
  3805. object count, and the one endpoint that can rebuild the list is reachable
  3806. only from the modal that button opens.
  3807. The archive is the running print's as ``running_print_archive`` finds it:
  3808. by subtask_id, or for a print that reports none by the archive bound to it
  3809. (#3327), never by name: a leftover ``status="printing"`` row from a
  3810. completion we never saw must not hand this print someone else's objects.
  3811. After a restart that binding exists only for Bambuddy's own prints, so a
  3812. print that reports no subtask_id loads nothing rather than a guess; the
  3813. reload path on ``GET /print/objects`` covers that case on demand.
  3814. """
  3815. client = printer_manager.get_client(printer_id)
  3816. if client is None or (client.state.printable_objects and client.state.pause_points):
  3817. return
  3818. archive = await running_print_archive(db, printer_id, state, printing_only=True, by_name=False)
  3819. if archive is not None and archive.file_path:
  3820. if not client.state.printable_objects:
  3821. _load_objects_from_archive(archive, printer_id, logger)
  3822. # The slicer's pauses are in-memory only as well (#2984).
  3823. if not client.state.pause_points:
  3824. _load_pauses_from_3mf(
  3825. app_settings.base_dir / archive.file_path,
  3826. printer_id,
  3827. warned_layers=pause_eta.warned_layers_of(archive, state),
  3828. )
  3829. # Retry ladder for a fallback archive created while the printer's FTPS cool-off
  3830. # was running (#2957). The cool-off is 300s, so the first attempt is placed just
  3831. # past it; the second covers a handshake that failed again on the way back and
  3832. # armed a fresh one. Module-level so tests can shrink them.
  3833. _FALLBACK_3MF_RETRY_DELAYS_SECONDS: tuple[float, ...] = (310.0, 620.0)
  3834. # Retry ladder for the other temporary give-up: the file service answered and
  3835. # the transfer still did not finish, which at print start is usually the printer
  3836. # serving MQTT, the camera and a job upload at the same time (#3063). Nothing has
  3837. # to expire here, so the first attempt comes early -- #3063's reporter had the
  3838. # same 19MB file complete 48 seconds after the download budget ran out. The later
  3839. # two cover a printer that stays busy well into the print.
  3840. _FALLBACK_3MF_TRANSFER_RETRY_DELAYS_SECONDS: tuple[float, ...] = (60.0, 240.0, 600.0)
  3841. # printer_id -> the in-flight retry task, so print completion can cancel it.
  3842. _fallback_3mf_retry_tasks: dict[int, asyncio.Task] = {}
  3843. # printer_id -> lock serialising recovery attempts for that printer. Three callers
  3844. # can reach one archive at once: the cover endpoint (whose single-flight coalesces
  3845. # by view, so two views race), the cool-off retry task, and print completion.
  3846. # Without this they each read file_path == "" and each run a full copy, so the row
  3847. # ends up pointing at one timestamped directory while the others sit orphaned.
  3848. #
  3849. # Keyed by printer rather than archive because a printer runs one print at a time,
  3850. # which makes the two equally strong here — and it bounds the dict by printer
  3851. # count instead of needing a cleanup pass. Popping a per-archive entry cannot be
  3852. # done safely: `Lock.locked()` reads False between release and the queued waiter
  3853. # resuming, so "no waiters" is not a question this API can answer.
  3854. _fallback_recovery_locks: dict[int, asyncio.Lock] = {}
  3855. # How many of a printer's newest archives a touchscreen reprint is compared
  3856. # against (#3307). A repeat is usually of a recent job; the bound keeps a
  3857. # print start from reading a long history's extra_data.
  3858. _REPEAT_LOOKBACK = 200
  3859. async def _archived_3mf_for_repeat(
  3860. db, printer_id: int, subtask_name: str | None, gcode_file: str | None, total_layers: int | None
  3861. ):
  3862. """The newest archive of the job this printer is running again, with its 3MF.
  3863. A "Print again" on the printer's screen never names the file on the card:
  3864. it reports its own history copy under a generic name
  3865. (``file:///userdata/project_file.gcode.3mf``), the 3MF's internal title as
  3866. ``subtask_name`` and the plate as ``gcode_file`` -- and the title is often
  3867. not the card file's name, so every lookup built from it misses (#3307).
  3868. An earlier run that was archived recorded that same title and plate in its
  3869. ``_print_data``, and holds the 3MF.
  3870. Same printer, same title, same plate, a 3MF still on disk. A layer count
  3871. known on both sides must agree: the same title and plate re-sliced is
  3872. another job. None when nothing matches.
  3873. """
  3874. from backend.app.models.archive import PrintArchive
  3875. if not subtask_name or not gcode_file:
  3876. return None
  3877. rows = (
  3878. (
  3879. await db.execute(
  3880. select(PrintArchive)
  3881. .where(
  3882. PrintArchive.printer_id == printer_id,
  3883. PrintArchive.deleted_at.is_(None),
  3884. PrintArchive.file_path.isnot(None),
  3885. PrintArchive.file_path != "",
  3886. )
  3887. .order_by(PrintArchive.created_at.desc(), PrintArchive.id.desc())
  3888. .limit(_REPEAT_LOOKBACK)
  3889. )
  3890. )
  3891. .scalars()
  3892. .all()
  3893. )
  3894. for archive in rows:
  3895. extra = archive.extra_data if isinstance(archive.extra_data, dict) else {}
  3896. run = extra.get("_print_data")
  3897. if not isinstance(run, dict):
  3898. continue
  3899. if run.get("subtask_name") != subtask_name or run.get("filename") != gcode_file:
  3900. continue
  3901. if total_layers and archive.total_layers and archive.total_layers != total_layers:
  3902. continue
  3903. if not (app_settings.base_dir / archive.file_path).is_file():
  3904. continue
  3905. return archive
  3906. return None
  3907. async def _recover_fallback_archive(archive_id: int, source_3mf: Path, printer_id: int, *, price: bool = True) -> bool:
  3908. """Fill in a no-3MF archive from a 3MF that turned up later.
  3909. Returns True when the row was upgraded. Safe to call speculatively: it
  3910. verifies the archive still exists, is still a fallback, and that the file
  3911. is a readable 3MF before touching anything.
  3912. Serialised per printer — see ``_fallback_recovery_locks``.
  3913. """
  3914. lock = _fallback_recovery_locks.setdefault(printer_id, asyncio.Lock())
  3915. async with lock:
  3916. return await _recover_fallback_archive_locked(archive_id, source_3mf, printer_id, price=price)
  3917. async def _recover_fallback_archive_locked(
  3918. archive_id: int, source_3mf: Path, printer_id: int, *, price: bool = True
  3919. ) -> bool:
  3920. """The body of :func:`_recover_fallback_archive`, under its per-printer lock.
  3921. ``price`` is False when the print's end is being handled: the archive
  3922. still reads "printing" then, and its end prices it.
  3923. """
  3924. import zipfile
  3925. from backend.app.models.archive import PrintArchive
  3926. from backend.app.services.archive import ArchiveService
  3927. logger = logging.getLogger(__name__)
  3928. if not source_3mf.exists() or source_3mf.stat().st_size == 0:
  3929. return False
  3930. if not await asyncio.to_thread(zipfile.is_zipfile, source_3mf):
  3931. # A truncated or half-written download is worse than no download: it
  3932. # would replace an honest empty archive with wrong metadata.
  3933. logger.warning("[RECOVER] %s is not a readable 3MF; leaving archive %s as-is", source_3mf, archive_id)
  3934. return False
  3935. async with async_session() as db:
  3936. archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  3937. if archive is None or archive.deleted_at is not None:
  3938. return False
  3939. if archive.file_path:
  3940. # Already recovered, or never was a fallback. Either way there is a
  3941. # real 3MF attached and overwriting it is not this function's job.
  3942. return False
  3943. print_data = (archive.extra_data or {}).get("_print_data") or {}
  3944. service = ArchiveService(db)
  3945. recovered = await service.archive_print(
  3946. printer_id=printer_id,
  3947. source_file=source_3mf,
  3948. print_data={**print_data, "status": archive.status or "printing"},
  3949. subtask_id=archive.subtask_id,
  3950. update_archive_id=archive.id,
  3951. )
  3952. if recovered is None:
  3953. return False
  3954. logger.info(
  3955. "[RECOVER] Archive %s filled in from %s (%s bytes) — it started as a no-3MF fallback",
  3956. archive_id,
  3957. source_3mf,
  3958. recovered.file_size,
  3959. )
  3960. # Spoolman's tracking stored no 3MF usage at print start and would
  3961. # charge the AMS remain% change; the built-in tracker reads the archive
  3962. # at completion and needs nothing. Also at the print's end (``price``
  3963. # False): that recovery runs inside the close-out, before the charge.
  3964. try:
  3965. from backend.app.services.spoolman_tracking import attach_recovered_3mf
  3966. from backend.app.services.usage_tracker import is_completing
  3967. await attach_recovered_3mf(
  3968. printer_id,
  3969. recovered.id,
  3970. recovered.file_path,
  3971. db,
  3972. printer_manager,
  3973. plate_id=_get_start_plate_id(printer_id, recovered.id) or recovered.plate_id,
  3974. may_start=not price or (recovered.status == "printing" and not is_completing(printer_id)),
  3975. )
  3976. except Exception:
  3977. logger.warning(
  3978. "[RECOVER] Archive %s: could not add the 3MF to Spoolman's usage tracking", archive_id, exc_info=True
  3979. )
  3980. if price and recovered.status == "printing" and recovered.file_path:
  3981. # Print start had no 3MF to read the slicer's pauses from (#2984).
  3982. _load_pauses_from_3mf(app_settings.base_dir / recovered.file_path, printer_id)
  3983. if price and recovered.status == "printing":
  3984. # Print start priced it from the default rate, having no 3MF to
  3985. # estimate from; now it has one. An ended print is left alone: its
  3986. # end prices it. A print Bambuddy didn't dispatch has no start
  3987. # plate; the one the archive just read from the 3MF stands in.
  3988. _schedule_archive_cost_estimate(
  3989. printer_id,
  3990. recovered.id,
  3991. printer_manager,
  3992. ams_mapping=_get_start_ams_mapping(print_data, printer_id, recovered.id),
  3993. plate_id=_get_start_plate_id(printer_id, recovered.id) or recovered.plate_id,
  3994. session_factory=async_session,
  3995. )
  3996. # `archive_updated`, not `archive_created` — the row was already on the
  3997. # Archives page as an empty card and is now filled in, not new.
  3998. await ws_manager.send_archive_updated(
  3999. {
  4000. "id": recovered.id,
  4001. "printer_id": recovered.printer_id,
  4002. "filename": recovered.filename,
  4003. "print_name": recovered.print_name,
  4004. "status": recovered.status,
  4005. }
  4006. )
  4007. return True
  4008. async def try_recover_fallback_archive(printer_id: int, name: str, path: Path, *, price: bool = True) -> bool:
  4009. """Offer a freshly-downloaded 3MF to this printer's running fallback archive.
  4010. Called from the paths that pull a 3MF for a print that is already under way
  4011. — chiefly the cover endpoint, which downloads the very file the archive flow
  4012. could not get and, before #2957, used it for a thumbnail and nothing else.
  4013. The bytes are already local, so this costs a parse and a row update.
  4014. No-op when the running print has a real archive, which is the common case.
  4015. """
  4016. from backend.app.models.archive import PrintArchive
  4017. logger = logging.getLogger(__name__)
  4018. # `_active_prints` is keyed on the raw names seen at print start — the
  4019. # dispatch filename, the subtask name, and the subtask name plus ".3mf".
  4020. # Callers here arrive with whichever variant their own path produced, so
  4021. # match on the same normalization the download cache uses rather than on an
  4022. # exact string; that is what makes "Desktop_Goose.gcode.3mf" from the cover
  4023. # endpoint find an archive registered under "Desktop_Goose".
  4024. wanted = normalize_3mf_name(name)
  4025. archive_id = None
  4026. for (key_printer_id, key_name), value in list(_active_prints.items()):
  4027. if key_printer_id == printer_id and normalize_3mf_name(key_name) == wanted:
  4028. archive_id = value
  4029. break
  4030. if archive_id is None:
  4031. return False
  4032. async with async_session() as db:
  4033. archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  4034. # Cheap pre-check so the common case (a normal archive) does no work.
  4035. if archive is None or archive.file_path or archive.deleted_at is not None:
  4036. return False
  4037. try:
  4038. return await _recover_fallback_archive(archive_id, path, printer_id, price=price)
  4039. except Exception as e:
  4040. # Recovery is opportunistic. A failure here must never take down the
  4041. # caller, which is usually just trying to render a thumbnail.
  4042. logger.warning("[RECOVER] Could not fill in archive %s from %s: %s", archive_id, path, e)
  4043. return False
  4044. def _schedule_fallback_3mf_retry(
  4045. printer_id: int,
  4046. archive_id: int,
  4047. filenames: list[str],
  4048. delays: tuple[float, ...] | None = None,
  4049. reason: str = REASON_FTPS_COOLOFF,
  4050. ) -> None:
  4051. """Re-attempt the 3MF download after a temporary give-up.
  4052. ``reason`` says which give-up this is, and picks the default ladder: an
  4053. FTPS cool-off has to be waited out, while a transfer that timed out under
  4054. contention is worth asking about again straight away (#3063). It is only
  4055. read for the ladder and the log line -- the retry itself is identical, since
  4056. in both cases the file is on the printer and the last attempt at it failed
  4057. for a reason that does not last.
  4058. """
  4059. logger = logging.getLogger(__name__)
  4060. if delays is None:
  4061. delays = (
  4062. _FALLBACK_3MF_TRANSFER_RETRY_DELAYS_SECONDS
  4063. if reason == REASON_FTP_TRANSFER_FAILED
  4064. else _FALLBACK_3MF_RETRY_DELAYS_SECONDS
  4065. )
  4066. async def _retry() -> None:
  4067. from backend.app.models.archive import PrintArchive
  4068. from backend.app.models.printer import Printer
  4069. for delay in delays:
  4070. await asyncio.sleep(delay)
  4071. async with async_session() as db:
  4072. archive = (
  4073. await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  4074. ).scalar_one_or_none()
  4075. if archive is None or archive.deleted_at is not None or archive.file_path:
  4076. return
  4077. printer = (await db.execute(select(Printer).where(Printer.id == printer_id))).scalar_one_or_none()
  4078. if printer is None:
  4079. return
  4080. # Read the fields while the session is open rather than touching
  4081. # a detached instance minutes later, mid-download.
  4082. printer_ip = printer.ip_address
  4083. printer_code = printer.access_code
  4084. printer_model = printer.model
  4085. # Someone else may have fetched it in the meantime — the cover
  4086. # endpoint routinely does, and its copy is the same bytes.
  4087. for name in filenames:
  4088. cached = get_cached_3mf(printer_id, name)
  4089. if cached and await _recover_fallback_archive(archive_id, cached, printer_id):
  4090. return
  4091. if ftps_handshake_blocked(printer_ip):
  4092. logger.info(
  4093. "[RECOVER] Printer %s is still in its FTPS cool-off; archive %s retry deferred",
  4094. printer_id,
  4095. archive_id,
  4096. )
  4097. continue
  4098. _, _, _, ftp_timeout = await get_ftp_retry_settings()
  4099. for candidate in filenames:
  4100. # Bare name only. These come from the print-start flow, which
  4101. # already strips the path, but the local temp write must not
  4102. # depend on that holding for every future caller — a name that
  4103. # is absolute or contains ".." would otherwise escape the data
  4104. # volume via the `/` operator.
  4105. name = Path(candidate).name
  4106. if not name or name in (".", ".."):
  4107. continue
  4108. if not name.endswith(".3mf"):
  4109. name = f"{name}.3mf"
  4110. temp_path = app_settings.archive_dir / "temp" / name
  4111. temp_path.parent.mkdir(parents=True, exist_ok=True)
  4112. try:
  4113. hit = await download_file_try_paths_async(
  4114. printer_ip,
  4115. printer_code,
  4116. ftp_probe_paths(name),
  4117. temp_path,
  4118. socket_timeout=ftp_timeout,
  4119. printer_model=printer_model,
  4120. max_total_timeout=RECOVERY_DOWNLOAD_MAX_TIMEOUT,
  4121. )
  4122. except Exception as e:
  4123. logger.debug("[RECOVER] Retry download of %s failed: %s", name, e)
  4124. continue
  4125. if not hit:
  4126. continue
  4127. cache_3mf_download(printer_id, name, temp_path)
  4128. if await _recover_fallback_archive(archive_id, temp_path, printer_id):
  4129. return
  4130. logger.info("[RECOVER] Archive %s still has no 3MF after a retry", archive_id)
  4131. async def _guarded() -> None:
  4132. try:
  4133. await _retry()
  4134. except asyncio.CancelledError:
  4135. raise
  4136. except Exception as e:
  4137. logger.warning("[RECOVER] Retry task for archive %s failed: %s", archive_id, e)
  4138. finally:
  4139. if _fallback_3mf_retry_tasks.get(printer_id) is asyncio.current_task():
  4140. _fallback_3mf_retry_tasks.pop(printer_id, None)
  4141. existing = _fallback_3mf_retry_tasks.pop(printer_id, None)
  4142. if existing and not existing.done():
  4143. existing.cancel()
  4144. task = asyncio.create_task(_guarded())
  4145. _fallback_3mf_retry_tasks[printer_id] = task
  4146. logger.info(
  4147. "[RECOVER] Archive %s has no 3MF (%s) and the file should still be on printer %s; will retry in %s",
  4148. archive_id,
  4149. reason,
  4150. printer_id,
  4151. ", ".join(f"{d:g}s" for d in delays),
  4152. )
  4153. async def _ask_outcome_for_external_print(db, printer_id: int, observed_name: str | None = None) -> bool:
  4154. """Whether an archive created here should ask for the print's outcome (#1898).
  4155. Only a print Bambuddy did not dispatch reaches the archive-*creating*
  4156. branches below: a queued job already has its archive and takes the
  4157. expected-print branch, where the queue item's own ``confirm_outcome``
  4158. decides. The queue is still consulted, because a restart mid-print empties
  4159. ``_expected_prints`` — without the check the setting could override a queue
  4160. item that deliberately has the flag off. Any failure answers "don't ask":
  4161. an unwanted prompt is worse than a missing one, and this must never be the
  4162. reason a print goes unarchived.
  4163. A ``printing`` row is not proof on its own, though: Bambuddy deliberately
  4164. leaves one behind when a completion cannot be matched to it
  4165. (``_completion_belongs_to_queue_item``), and the scheduler's stranded sweep
  4166. only takes it back once the printer has sat connected and terminal for
  4167. minutes. Until then every screen-started print on that printer would
  4168. silently lose its prompt, so the row is held against ``observed_name`` by
  4169. the same comparison a completion uses: a positive disagreement means the row
  4170. is about some other run and this print is external after all.
  4171. """
  4172. logger = logging.getLogger(__name__)
  4173. try:
  4174. # A savepoint rather than a rollback of the caller's transaction on
  4175. # failure: a failed statement leaves the transaction unusable, but a
  4176. # full rollback also expires every object the caller has loaded (the
  4177. # printer, the archive it just created), and on an async session the
  4178. # next attribute read then raises instead of reloading. Everything here
  4179. # is a read, so undoing the savepoint loses nothing.
  4180. async with db.begin_nested():
  4181. return await _decide_outcome_for_external_print(db, printer_id, observed_name, logger)
  4182. except Exception as e:
  4183. logger.warning("[CALLBACK] Could not decide the outcome prompt for printer %s: %s", printer_id, e)
  4184. return False
  4185. async def _decide_outcome_for_external_print(db, printer_id: int, observed_name: str | None, logger) -> bool:
  4186. """The reads behind ``_ask_outcome_for_external_print``; see there."""
  4187. from backend.app.api.routes.settings import get_setting, setting_is_true
  4188. if not setting_is_true(await get_setting(db, "confirm_outcome_external_prints")):
  4189. return False
  4190. from backend.app.models.print_queue import PrintQueueItem
  4191. dispatched_here = await db.scalar(
  4192. select(PrintQueueItem)
  4193. .where(
  4194. PrintQueueItem.printer_id == printer_id,
  4195. PrintQueueItem.status == "printing",
  4196. )
  4197. .limit(1)
  4198. )
  4199. if dispatched_here is None:
  4200. return True
  4201. expected = await _queue_item_dispatched_name(db, dispatched_here)
  4202. observed = (observed_name or "").strip()
  4203. if not expected or not observed or _dispatched_name_matches_echo(expected, observed):
  4204. logger.info(
  4205. "[CALLBACK] Not asking for the outcome on printer %s: queue item %s is still printing, so "
  4206. "Bambuddy dispatched this run and the item's own ask-for-outcome flag decides.",
  4207. printer_id,
  4208. dispatched_here.id,
  4209. )
  4210. return False
  4211. logger.info(
  4212. "[CALLBACK] Queue item %s is still marked printing on printer %s but was dispatched as %r, not "
  4213. "%r; treating this as an externally started print.",
  4214. dispatched_here.id,
  4215. printer_id,
  4216. expected,
  4217. observed,
  4218. )
  4219. return True
  4220. async def _dispatch_outcome_confirmation_safely(
  4221. db,
  4222. printer_id: int,
  4223. printer_name: str,
  4224. data: dict,
  4225. archive_id: int,
  4226. archive_data: dict | None = None,
  4227. run_id: int | None = None,
  4228. ) -> None:
  4229. """``dispatch_outcome_confirmation`` for a caller that has more to do on ``db``.
  4230. The completion task sends the per-user print email on the same session
  4231. right after the prompt. A failed statement in the prompt leaves that
  4232. session needing a rollback, and without one the email step fails with
  4233. PendingRollbackError; the prompt is the optional part, so it must not be
  4234. the reason the email never goes out.
  4235. """
  4236. try:
  4237. await dispatch_outcome_confirmation(db, printer_id, printer_name, data, archive_id, archive_data, run_id=run_id)
  4238. except Exception as e:
  4239. logging.getLogger(__name__).error("[NOTIFY-BG] Outcome-confirmation dispatch failed: %s", e, exc_info=True)
  4240. try:
  4241. await db.rollback()
  4242. except Exception:
  4243. pass
  4244. async def _outcome_answerers(db, owner_id: int | None) -> set[int] | None:
  4245. """Who may answer a print's outcome prompt in the app: its owner and anyone
  4246. who may change everyone's archives. None (everyone on the printer) with auth
  4247. off, or for a print nobody owns."""
  4248. from sqlalchemy.orm import selectinload
  4249. from backend.app.core.auth import is_auth_enabled
  4250. from backend.app.core.permissions import Permission
  4251. from backend.app.models.user import User
  4252. if owner_id is None or not await is_auth_enabled(db):
  4253. return None
  4254. users = (await db.execute(select(User).options(selectinload(User.groups)).where(User.is_active))).scalars().all()
  4255. return {owner_id} | {u.id for u in users if u.has_permission(Permission.ARCHIVES_UPDATE_ALL.value)}
  4256. async def dispatch_outcome_confirmation(
  4257. db,
  4258. printer_id: int,
  4259. printer_name: str,
  4260. data: dict,
  4261. archive_id: int,
  4262. archive_data: dict | None = None,
  4263. *,
  4264. run_id: int | None = None,
  4265. ) -> bool:
  4266. """Emit the post-print outcome prompt for a completed print (#1898).
  4267. Gated on the run that ended (``run_id``): ``confirm_requested`` is the
  4268. opt-in (from the queue item, or from ``confirm_outcome_external_prints``
  4269. for a print Bambuddy did not start, copied onto the run at its start) and
  4270. ``user_verdict`` being unset is what makes the question still open. Mints
  4271. the run's capability token the one-tap verdict links carry: plates of one
  4272. archive each have their own question. Without a run, the archive's, as
  4273. before runs.
  4274. Lives here rather than inline in ``on_print_complete``'s notification task
  4275. because that task swallows every exception: extracted, the gate and the two
  4276. emissions can be driven by a test, which is the only thing standing between
  4277. a regression here and a farm that quietly stops asking.
  4278. Returns whether a prompt was sent.
  4279. """
  4280. logger = logging.getLogger(__name__)
  4281. from backend.app.models.archive import PrintArchive
  4282. from backend.app.models.print_log import PrintLogEntry
  4283. confirm_archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  4284. if confirm_archive is None:
  4285. return False
  4286. run = await db.get(PrintLogEntry, run_id) if run_id is not None else None
  4287. if run is not None and run.archive_id != archive_id:
  4288. run = None
  4289. # The run carries the opt-in from its start; one logged before it did
  4290. # (None) asks as its archive does.
  4291. asks = confirm_archive if run is None or run.confirm_requested is None else run
  4292. holder = run if run is not None else confirm_archive
  4293. if not (asks.confirm_requested and holder.user_verdict is None):
  4294. return False
  4295. import secrets as _secrets
  4296. # A spent token stays on the row so the one-tap route can recognise it, so
  4297. # "has a token" no longer means "answerable". A prompt going out now needs a
  4298. # live one: mint a new token whenever the stored one is already spent, or
  4299. # every button in the new message would land on the already-answered page.
  4300. if not holder.confirm_token or holder.confirm_token_used_at is not None:
  4301. holder.confirm_token = _secrets.token_urlsafe(32)
  4302. holder.confirm_token_used_at = None
  4303. await db.commit()
  4304. from backend.app.api.routes.settings import get_external_base_url, get_setting
  4305. base = await get_external_base_url(db)
  4306. if not await get_setting(db, "external_url"):
  4307. # Both the Telegram inline keyboard and the ntfy action buttons need an
  4308. # absolute URL, so an unconfigured install used to get a message body
  4309. # with two unusable relative paths and no buttons at all. The shared
  4310. # fallback at least produces tappable links; say which setting makes
  4311. # them resolve from a phone.
  4312. logger.warning(
  4313. "[#1898] No external_url configured — the outcome prompt's Good/Reject links point at %s. "
  4314. "Set Settings → External URL so they resolve away from this host.",
  4315. base,
  4316. )
  4317. token = holder.confirm_token
  4318. good_url = f"{base}/api/v1/archives/confirm/{token}/good"
  4319. reject_url = f"{base}/api/v1/archives/confirm/{token}/reject"
  4320. confirm_url = f"{base}/archives?confirm={archive_id}" + (f"&run={run.id}" if run is not None else "")
  4321. await ws_manager.send_print_confirm_request(
  4322. printer_id,
  4323. {
  4324. "archive_id": archive_id,
  4325. "run_id": run.id if run is not None else None,
  4326. "print_name": confirm_archive.print_name or confirm_archive.filename,
  4327. },
  4328. user_ids=await _outcome_answerers(db, confirm_archive.created_by_id),
  4329. )
  4330. await notification_service.on_print_confirm_request(
  4331. printer_id,
  4332. printer_name,
  4333. data,
  4334. db,
  4335. archive_data=archive_data,
  4336. good_url=good_url,
  4337. reject_url=reject_url,
  4338. confirm_url=confirm_url,
  4339. archive_id=archive_id,
  4340. run_id=run.id if run is not None else None,
  4341. )
  4342. return True
  4343. async def _printer_gone(db, printer_id: int) -> bool:
  4344. """Whether the printer was deleted. A column read: the session's loaded
  4345. Printer would otherwise answer for the database."""
  4346. from backend.app.models.printer import Printer
  4347. return await db.scalar(select(Printer.id).where(Printer.id == printer_id)) is None
  4348. async def _drop_archive_of_deleted_printer(db, printer_id: int, archive_id: int) -> bool:
  4349. """Remove the archive just written if its printer was deleted meanwhile.
  4350. The delete route sweeps the printer's archives before it commits, so a row
  4351. committed after that sweep (the check before writing passed while the
  4352. delete was still uncommitted) is left for a printer that will never report
  4353. the print's end. A row the sweep reached is the route's: deleted with the
  4354. printer's archives, or detached (printer_id NULL) when the user kept them,
  4355. and then left as it is. Either way nothing is registered for it. Run after
  4356. the archive's commit, before anything is registered for it.
  4357. """
  4358. from backend.app.models.archive import PrintArchive
  4359. if not await _printer_gone(db, printer_id):
  4360. return False
  4361. owner = await db.scalar(select(PrintArchive.printer_id).where(PrintArchive.id == archive_id))
  4362. if owner == printer_id:
  4363. logging.getLogger(__name__).info(
  4364. "Printer %s was deleted while archive %s was written; removing it", printer_id, archive_id
  4365. )
  4366. await ArchiveService(db).delete_archive(archive_id)
  4367. await forget_printer(printer_id)
  4368. return True
  4369. async def _refused_for_deleted_printer(db, printer_id: int, what: str) -> bool:
  4370. """After an IntegrityError on a print start's archive write: whether it was
  4371. the printer being deleted, forgotten then (the caller returns).
  4372. PostgreSQL: the delete route locks the printer from its archive sweep to its
  4373. commit, so an archive written or adopted for it meanwhile (one that sets
  4374. printer_id) waits and the printer's foreign key then refuses it. That is
  4375. the printer being deleted, not an error; anything else is re-raised by the
  4376. caller.
  4377. """
  4378. await db.rollback()
  4379. if not await _printer_gone(db, printer_id):
  4380. return False
  4381. logging.getLogger(__name__).info("Printer %s was deleted while %s; refused", printer_id, what)
  4382. await forget_printer(printer_id)
  4383. return True
  4384. async def _forget_if_printer_gone(printer_id: int) -> None:
  4385. """At the end of a print start: the slow steps after the last check (smart
  4386. plug, notifications, the timelapse baseline's FTP listing) can outlast the
  4387. printer's delete and its forget_printer, and then wrote state back.
  4388. (An ActivePrintSpoolman row written then is the init_db orphan sweep's on
  4389. SQLite; PostgreSQL refuses it.)
  4390. Its own session: those steps can leave the handler's needing a rollback (a
  4391. write to the archive the delete removed). Never raises: it runs inside the
  4392. fallback branch's try, whose except reports the archive as failed.
  4393. """
  4394. try:
  4395. async with async_session() as db:
  4396. gone = await _printer_gone(db, printer_id)
  4397. if gone:
  4398. await forget_printer(printer_id)
  4399. except Exception:
  4400. logging.getLogger(__name__).warning(
  4401. "Could not check whether printer %s was deleted during its print start", printer_id, exc_info=True
  4402. )
  4403. async def _announce_plate_check_failed(printer_id: int, printer_name: str, reason: str) -> None:
  4404. """Tell the UI a plate check could not run. It does not block the print,
  4405. but an unchecked plate is not an empty one."""
  4406. try:
  4407. await ws_manager.broadcast(
  4408. {"type": "plate_check_failed", "printer_id": printer_id, "printer_name": printer_name, "reason": reason}
  4409. )
  4410. except Exception as e:
  4411. logging.getLogger(__name__).warning("[PLATE CHECK] Could not announce the failed check: %s", e)
  4412. def _expected_print_keys(printer_id: int, filename: str | None, subtask_name: str | None) -> list[tuple[int, str]]:
  4413. """The ``_expected_prints`` keys a print start may be registered under."""
  4414. keys: list[tuple[int, str]] = []
  4415. if subtask_name:
  4416. keys.append((printer_id, subtask_name))
  4417. keys.append((printer_id, f"{subtask_name}.3mf"))
  4418. keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  4419. if filename:
  4420. fname = filename.split("/")[-1] if "/" in filename else filename
  4421. keys.append((printer_id, fname))
  4422. # Strip extensions to match
  4423. base = fname.replace(".gcode", "").replace(".3mf", "")
  4424. keys.append((printer_id, base))
  4425. keys.append((printer_id, f"{base}.3mf"))
  4426. return keys
  4427. async def _dispatched_by_bambuddy(db, printer_id: int, filename: str | None, subtask_name: str | None) -> bool:
  4428. """Whether this print start is one Bambuddy sent to the printer."""
  4429. if any(key in _expected_prints for key in _expected_print_keys(printer_id, filename, subtask_name)):
  4430. return True
  4431. from backend.app.models.archive import PrintArchive
  4432. from backend.app.models.print_queue import PrintQueueItem
  4433. # The queue row the scheduler sets to printing before it sends -- but only
  4434. # when it was sent as this print: a row left printing by a run whose end
  4435. # was missed names another file.
  4436. dispatched_names = (
  4437. await db.execute(
  4438. select(PrintArchive.filename)
  4439. .join(PrintQueueItem, PrintQueueItem.archive_id == PrintArchive.id)
  4440. .where(PrintQueueItem.printer_id == printer_id, PrintQueueItem.status == "printing")
  4441. )
  4442. ).scalars()
  4443. observed = [name for name in (subtask_name, (filename or "").rsplit("/", 1)[-1]) if name]
  4444. return any(
  4445. _subtask_names_match(_subtask_name_from_filename(dispatched), _subtask_name_from_filename(seen))
  4446. for dispatched in dispatched_names
  4447. if dispatched
  4448. for seen in observed
  4449. )
  4450. async def _adopt_archive_for_run(
  4451. db: AsyncSession,
  4452. archive,
  4453. printer_id: int,
  4454. *,
  4455. starting_plate_id: int | None,
  4456. effective_subtask_id: str | None,
  4457. logger: logging.Logger,
  4458. started_at: datetime | None = None,
  4459. ) -> bool:
  4460. """Start a print's run on an archive that already exists: a reprint, a
  4461. queue print of a file printed before, or the same slice sent again from the
  4462. slicer (#1634). The archive says "printing" from now, without the previous
  4463. run's outcome, answer or timelapse, and on this printer with this print's
  4464. job id. False when the printer was deleted meanwhile: the adoption is
  4465. undone or was refused, the printer is forgotten, and the caller stops.
  4466. """
  4467. from backend.app.models.archive import PrintArchive
  4468. # As it was before this adoption: put back if the printer turns
  4469. # out to have been deleted while it was adopted (below).
  4470. before_adoption = {
  4471. attr.columns[0].name: getattr(archive, attr.key)
  4472. for attr in sa_inspect(PrintArchive).column_attrs
  4473. if attr.key != "id"
  4474. }
  4475. # Update archive status to printing
  4476. archive.status = "printing"
  4477. archive.started_at = started_at or datetime.now(timezone.utc)
  4478. # The previous run's outcome is not this one's: a reprint that
  4479. # completes must not keep showing why the last one failed.
  4480. archive.failure_reason = None
  4481. archive.completed_at = None
  4482. # The previous run's answer is still on this row and the
  4483. # completion prompt is gated on ``user_verdict is None`` (#1898),
  4484. # so without a reset the second run inherits the first run's
  4485. # verdict: no prompt at all, and a green "good" badge on a run
  4486. # nobody ever judged.
  4487. if archive.confirm_requested:
  4488. archive.user_verdict = None
  4489. archive.user_verdict_source = None
  4490. archive.user_verdict_at = None
  4491. archive.confirm_token = None
  4492. archive.confirm_token_used_at = None
  4493. # Reprint of an archive reuses the source row. Without resetting
  4494. # ``timelapse_path`` _scan_for_timelapse_with_retries early-returns
  4495. # ("already has timelapse") and _capture_finish_photo_from_timelapse
  4496. # extracts the *original* print's last frame, which then ships in
  4497. # the completion notification (#1707). Clear the path so the
  4498. # scanner runs fresh; also unlink the old video file so reprints
  4499. # don't accumulate orphans in the archive directory. Photos list
  4500. # is left alone — accumulating one finish photo per run is fine.
  4501. # The print-start baseline (#2704) is stale for the same reason:
  4502. # it describes the printer before the previous run. The capture
  4503. # below overwrites it, but clear it here too so an early failure
  4504. # can't leave the scan diffing against the wrong snapshot.
  4505. #
  4506. # Unlinked only when it is the same plate printing again. Every
  4507. # plate of a Send All shares this archive, and the video on it
  4508. # can be the only copy of another plate's run -- the printer's
  4509. # was deleted once it attached (#3275). That one is left in the
  4510. # archive directory, which goes with the archive. A video whose
  4511. # plate is not on record (attached by hand, or before the
  4512. # column existed) is kept too where the 3MF has other plates,
  4513. # or cannot be read: nothing tells it apart from another plate's,
  4514. # and keeping costs one file where deleting can cost the only
  4515. # copy. On an archive with exactly one plate it is that plate's.
  4516. archive.timelapse_baseline = None
  4517. stale_timelapse_relpath = archive.timelapse_path
  4518. stale_timelapse_plate_id = archive.timelapse_plate_id
  4519. if stale_timelapse_relpath:
  4520. archive.timelapse_path = None
  4521. archive.timelapse_plate_id = None
  4522. stale_is_this_plates = stale_timelapse_plate_id == starting_plate_id
  4523. if stale_timelapse_relpath and stale_timelapse_plate_id is None and archive.file_path:
  4524. from backend.app.services.archive import plate_indexes_in_3mf
  4525. plates = await asyncio.to_thread(
  4526. plate_indexes_in_3mf,
  4527. app_settings.base_dir / archive.file_path, # SEC-PATH-OK: stored relative archive path
  4528. )
  4529. stale_is_this_plates = len(plates) == 1
  4530. if stale_timelapse_relpath and not stale_is_this_plates:
  4531. logger.info(
  4532. "Kept timelapse %s of plate %s on archive %s: plate %s is starting",
  4533. stale_timelapse_relpath,
  4534. stale_timelapse_plate_id,
  4535. archive.id,
  4536. starting_plate_id,
  4537. )
  4538. # The video is deleted once the adoption stands (below).
  4539. delete_stale_video = bool(stale_timelapse_relpath) and stale_is_this_plates
  4540. # Update on first-set OR on reprint (the queue dispatcher mints
  4541. # a fresh subtask_id per dispatch in bambu_mqtt:3647). Skipping
  4542. # the rewrite for reprints leaves the archive holding the FIRST
  4543. # run's id; if MQTT then reconnects mid-print, the reconciler
  4544. # (#1542) compares the stale stored id against the printer's
  4545. # live id, sees a mismatch, and synthesises a bogus PRINT
  4546. # COMPLETE — exactly the false-positive "Print Stopped" reported
  4547. # in #1807. Inequality check preserves the noop-on-stable-push
  4548. # behaviour the earlier `not archive.subtask_id` guard provided.
  4549. if effective_subtask_id and archive.subtask_id != effective_subtask_id:
  4550. archive.subtask_id = effective_subtask_id
  4551. # #1403 follow-up: VP-queue archives are created with
  4552. # printer_id=None at queue-add time (we don't know which
  4553. # printer will run the job yet). When the print actually
  4554. # starts on a specific printer the expected-archive lookup
  4555. # used to skip this assignment, leaving printer_id=None
  4556. # forever — which then disables the "Scan for timelapse"
  4557. # button in ArchivesPage (gated on !archive.printer_id).
  4558. if archive.printer_id != printer_id:
  4559. archive.printer_id = printer_id
  4560. adopting_id = archive.id # a rollback expires the object
  4561. try:
  4562. await db.commit()
  4563. except (IntegrityError, StaleDataError):
  4564. # The archive stays unchanged, or is gone: "delete archives"
  4565. # removed the row under the adoption, which then matched no
  4566. # row (StaleDataError) rather than being refused.
  4567. if await _refused_for_deleted_printer(db, printer_id, f"archive {adopting_id} was adopted"):
  4568. return False
  4569. raise
  4570. # The printer can be deleted between the check above and this
  4571. # commit (the route's transaction lasts seconds), and the commit
  4572. # then adopted the archive anyway: status, verdict and timelapse
  4573. # changed, and the deleted printer's id written on a row the
  4574. # route had detached or never touched. Undo the adoption.
  4575. if await _printer_gone(db, printer_id):
  4576. logger.info(
  4577. "Printer %s was deleted while archive %s was adopted; undoing the adoption",
  4578. printer_id,
  4579. archive.id,
  4580. )
  4581. # Everything as before this adoption. The printer it had
  4582. # then stays, unless it was the deleted one: the route
  4583. # detached that row (kept) or deleted it (no row matches).
  4584. previous_printer = before_adoption["printer_id"]
  4585. await db.execute(
  4586. update(PrintArchive)
  4587. .where(PrintArchive.id == archive.id)
  4588. .values(
  4589. {
  4590. **before_adoption,
  4591. "printer_id": None if previous_printer == printer_id else previous_printer,
  4592. }
  4593. )
  4594. )
  4595. await db.commit()
  4596. await forget_printer(printer_id)
  4597. return False
  4598. await _drop_replaced_plate_videos(
  4599. db,
  4600. archive.id,
  4601. starting_plate_id,
  4602. stale_timelapse_relpath if delete_stale_video else None,
  4603. logger,
  4604. )
  4605. return True
  4606. async def _claim_slicer_resend_archive(db: AsyncSession, printer_id: int, source: Path):
  4607. """The archive a print sent from the slicer adds its run to, or None for a
  4608. new one (#1634): the same G-code, so the same slice sent again in a new 3MF.
  4609. As a queue print of a file printed before (``_claim_reusable_archive`` in
  4610. the scheduler): the same owner (who started the print, nobody for a print
  4611. from the slicer) and no project, and no File Manager file, whose own prints
  4612. find its archive by the file's bytes. Not one a merge is moving. Held in
  4613. ``archive_work`` from here, with no await between, until the caller
  4614. releases it once the run is open.
  4615. Returns the archive and whether something else holds it already: a queue
  4616. dispatch uploading or printing its file, which the file must stay for.
  4617. """
  4618. from backend.app.models.archive import PrintArchive
  4619. from backend.app.services import archive_work
  4620. from backend.app.services.archive_gcode_hash import gcode_fingerprint
  4621. # Unzipping a large 3MF takes seconds: off the event loop.
  4622. fingerprint = await asyncio.to_thread(gcode_fingerprint, source)
  4623. if not fingerprint:
  4624. return None
  4625. owner_id = await _print_user_id_if_still_there(db, printer_manager.get_current_print_user(printer_id))
  4626. owner = PrintArchive.created_by_id.is_(None) if owner_id is None else PrintArchive.created_by_id == owner_id
  4627. candidates = (
  4628. await db.scalars(
  4629. select(PrintArchive)
  4630. .where(
  4631. PrintArchive.gcode_hash == fingerprint,
  4632. owner,
  4633. PrintArchive.project_id.is_(None),
  4634. PrintArchive.library_file_id.is_(None),
  4635. PrintArchive.deleted_at.is_(None),
  4636. )
  4637. .order_by(PrintArchive.id.desc())
  4638. )
  4639. ).all()
  4640. for candidate in candidates:
  4641. if archive_work.is_merging(candidate.id):
  4642. continue
  4643. in_use = archive_work.is_held(candidate.id)
  4644. archive_work.hold(candidate.id)
  4645. return candidate, in_use
  4646. return None
  4647. async def _start_slicer_resend_run(
  4648. db: AsyncSession,
  4649. archive,
  4650. printer_id: int,
  4651. *,
  4652. source: Path,
  4653. print_data: dict,
  4654. subtask_id: str | None,
  4655. started_at: datetime | None,
  4656. original_filename: str | None,
  4657. logger: logging.Logger,
  4658. file_in_use: bool = False,
  4659. ):
  4660. """Start this print's run on the archive ``_claim_slicer_resend_archive``
  4661. found (#1634). The archive takes the new file in place of its own, unless
  4662. another print of it is running, or a queue dispatch holds it
  4663. (``file_in_use``): that one still reads the file (its upload, pauses,
  4664. objects and estimate), and the G-code is the same. None when the printer
  4665. was deleted meanwhile; the caller stops.
  4666. """
  4667. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, RUN_OPEN, PrintLogEntry
  4668. printing_elsewhere = file_in_use or (
  4669. await db.scalar(
  4670. select(PrintLogEntry.id)
  4671. .where(PrintLogEntry.archive_id == archive.id, PrintLogEntry.status == RUN_OPEN)
  4672. .limit(1)
  4673. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  4674. )
  4675. is not None
  4676. )
  4677. if not printing_elsewhere:
  4678. replaced = await ArchiveService(db).archive_print(
  4679. printer_id=printer_id,
  4680. source_file=source,
  4681. print_data=print_data,
  4682. subtask_id=subtask_id,
  4683. original_filename=original_filename,
  4684. replace_archive_id=archive.id,
  4685. )
  4686. if replaced is not None:
  4687. archive = replaced
  4688. else:
  4689. # The copy failed its check, or the printer is gone (the adoption
  4690. # below says so): the run still joins, on the file it has.
  4691. logger.warning("Archive %s keeps its file: the one sent again could not be taken", archive.id)
  4692. client = printer_manager.get_client(printer_id)
  4693. if not await _adopt_archive_for_run(
  4694. db,
  4695. archive,
  4696. printer_id,
  4697. starting_plate_id=resolve_plate_id(client.state) if client else None,
  4698. effective_subtask_id=subtask_id,
  4699. logger=logger,
  4700. started_at=started_at,
  4701. ):
  4702. return None
  4703. return archive
  4704. async def on_print_start(printer_id: int, data: dict):
  4705. """Handle print start - archive the 3MF file immediately."""
  4706. logger = logging.getLogger(__name__)
  4707. # The print's start, before the 3MF download that the archive waits for
  4708. # (#3327): a minute of it would otherwise be missing from the print time.
  4709. detected_at = datetime.now(timezone.utc)
  4710. logger.info("[CALLBACK] on_print_start called for printer %s, data keys: %s", printer_id, list(data.keys()))
  4711. octoeverywhere_detection_service.reset_printer(printer_id)
  4712. try:
  4713. notify_live_activities.print_started(printer_id, printer_manager.get_status(printer_id), data)
  4714. except Exception:
  4715. logger.exception("Notify Live Activity start hook failed for printer %s", printer_id)
  4716. # Clear any stale user-stopped flag from previous print cycles
  4717. _user_stopped_printers.discard(printer_id)
  4718. _kill_switch_stopped.discard(printer_id)
  4719. # A new print starts its milestones from zero (#3211). The status path only
  4720. # resets on progress below 5 while not printing, which a printer that goes
  4721. # from FINISH at 100% straight into a new print at a preparation-phase 85%
  4722. # never shows. This callback does not fire after a Bambuddy restart (#1304)
  4723. # or on resume from pause, so it cannot repeat a milestone mid-print.
  4724. _last_progress_milestone[printer_id] = 0
  4725. _kill_switch_notification_tasks.pop(printer_id, None)
  4726. # #1721: drop any leftover pre-captured finish frame from a prior print
  4727. # so a never-consumed cache entry can't bleed into the new print's photo.
  4728. _stage22_finish_frames.pop(printer_id, None)
  4729. # #1867: same for the in-print frame bank — a queued print must not reuse
  4730. # the previous job's banked frame.
  4731. _inprint_frame_bank.pop(printer_id, None)
  4732. _inprint_frame_bank_ts.pop(printer_id, None)
  4733. # #2547: bind (or clear) the "this print ends with injected End G-code" flag.
  4734. # Unconditional, so a print Bambuddy didn't dispatch drops the previous
  4735. # print's flag instead of inheriting it.
  4736. print_dispatch_context.adopt(printer_id)
  4737. # Cancel any active bed cooldown waiter for this printer
  4738. if _bed_cool_waiters.pop(printer_id, None):
  4739. logger.info("[BED-COOL] Cancelled bed cooldown waiter for printer %s (new print started)", printer_id)
  4740. # Clear cached cover images so the new print's thumbnail is fetched fresh
  4741. from backend.app.api.routes.printers import clear_cover_cache
  4742. clear_cover_cache(printer_id)
  4743. await ws_manager.send_print_start(printer_id, data)
  4744. # Notify when the print-start AMS mapping references tray slots without spool assignments.
  4745. await notify_missing_spool_assignments_on_print_start(printer_id, data, logger)
  4746. # MQTT relay - publish print start
  4747. try:
  4748. printer_info = printer_manager.get_printer(printer_id)
  4749. if printer_info:
  4750. await mqtt_relay.on_print_start(
  4751. printer_id,
  4752. printer_info.name,
  4753. printer_info.serial_number,
  4754. data.get("filename", ""),
  4755. data.get("subtask_name", ""),
  4756. )
  4757. except Exception:
  4758. pass # Don't fail print start callback if MQTT fails
  4759. # Capture AMS tray remain%, the assignment snapshot, the dispatched plate
  4760. # and mapping, and the seeded tray-change log.
  4761. #
  4762. # Unconditional, for both inventory backends. This only *captures* — the
  4763. # writing is still split, with the internal tracker skipped at completion
  4764. # when Spoolman owns usage. Spoolman's own durable row (#1820) already
  4765. # carries its plate-scoped 3MF figures and stored mapping, but not the
  4766. # tray-change log, and that log is the only record of which spool fed
  4767. # which layers when AMS Filament Backup swaps trays mid-print. Capturing
  4768. # it on one side only would leave Spoolman users with the mid-print
  4769. # restart bug this fixes for everyone else.
  4770. try:
  4771. async with async_session() as db:
  4772. from backend.app.api.routes.settings import get_setting
  4773. from backend.app.services.usage_tracker import on_print_start as usage_on_print_start
  4774. _spoolman_on = await get_setting(db, "spoolman_enabled")
  4775. await usage_on_print_start(
  4776. printer_id,
  4777. data,
  4778. printer_manager,
  4779. db=db,
  4780. spoolman_owns_usage=bool(_spoolman_on) and _spoolman_on.lower() == "true",
  4781. )
  4782. except Exception as e:
  4783. logger.warning("Usage tracker on_print_start failed: %s", e)
  4784. # Track if notification was sent (to avoid sending twice)
  4785. notification_sent = False
  4786. # Smart plug automation: turn on plug when print starts
  4787. try:
  4788. async with async_session() as db:
  4789. await smart_plug_manager.on_print_start(printer_id, db)
  4790. except Exception as e:
  4791. logger.warning("Smart plug on_print_start failed: %s", e)
  4792. async with async_session() as db:
  4793. from backend.app.models.printer import Printer
  4794. from backend.app.services.bambu_ftp import list_files_async
  4795. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  4796. printer = result.scalar_one_or_none()
  4797. # Plate detection check - pause if objects detected on build plate
  4798. logger.info(
  4799. f"[PLATE CHECK] printer_id={printer_id}, plate_detection_enabled={printer.plate_detection_enabled if printer else 'NO PRINTER'}"
  4800. )
  4801. if printer and printer.plate_detection_enabled:
  4802. logger.info("[PLATE CHECK] ENTERING plate detection code for printer %s", printer_id)
  4803. # Release the pooled DB connection before the plate-detection camera
  4804. # work (a 2.5s light-settle sleep + FTP/camera capture). Only the
  4805. # printer SELECT has run so far — nothing to persist — so this commit
  4806. # is a data-noop that ends the read transaction and returns the
  4807. # connection to the pool during the I/O (issue #2572). expire_on_commit
  4808. # =False keeps printer.* readable; on_plate_not_empty (rare) and the
  4809. # archive lookups below re-acquire a fresh connection on next execute.
  4810. await db.commit()
  4811. try:
  4812. from backend.app.services.plate_detection import check_plate_empty
  4813. # Build ROI tuple from printer settings if available
  4814. roi = None
  4815. if all(
  4816. [
  4817. printer.plate_detection_roi_x is not None,
  4818. printer.plate_detection_roi_y is not None,
  4819. printer.plate_detection_roi_w is not None,
  4820. printer.plate_detection_roi_h is not None,
  4821. ]
  4822. ):
  4823. roi = (
  4824. printer.plate_detection_roi_x,
  4825. printer.plate_detection_roi_y,
  4826. printer.plate_detection_roi_w,
  4827. printer.plate_detection_roi_h,
  4828. )
  4829. # The light is held, not switched, so a live view opened
  4830. # meanwhile keeps it when the check lets go (SP-C11). Every
  4831. # printer, whatever the camera light mode: detection needs it.
  4832. plate_photo_data = None
  4833. objects_detected = False
  4834. check_failure: str | None = None
  4835. async with camera_light(printer, any_mode=True, settle=2.5):
  4836. logger.info("[PLATE CHECK] Running plate detection for printer %s", printer_id)
  4837. plate_result = await check_plate_empty(
  4838. printer_id=printer_id,
  4839. ip_address=printer.ip_address,
  4840. access_code=printer.access_code,
  4841. model=printer.model,
  4842. include_debug_image=False,
  4843. external_camera_url=printer.external_camera_url,
  4844. external_camera_type=printer.external_camera_type,
  4845. use_external=printer.external_camera_enabled,
  4846. roi=roi,
  4847. external_camera_snapshot_url=printer.external_camera_snapshot_url,
  4848. )
  4849. # No reference to compare against is a check that did not
  4850. # run either: say so rather than call the plate empty.
  4851. check_failure = getattr(plate_result, "failure_reason", None) or (
  4852. "not_calibrated" if plate_result.needs_calibration else None
  4853. )
  4854. objects_detected = not check_failure and not plate_result.is_empty
  4855. if objects_detected:
  4856. # Objects detected - pause the print!
  4857. logger.warning(
  4858. f"[PLATE CHECK] Objects detected on plate for printer {printer_id}! "
  4859. f"Confidence: {plate_result.confidence:.0%}, Diff: {plate_result.difference_percent:.1f}%"
  4860. )
  4861. pause_client = printer_manager.get_client(printer_id)
  4862. if pause_client:
  4863. pause_client.pause_print()
  4864. logger.info("[PLATE CHECK] Print paused for printer %s", printer_id)
  4865. # Snapshot while the light's still on — restoring it first
  4866. # would leave the notification with a dark photo.
  4867. try:
  4868. plate_photo_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  4869. except Exception as snap_err:
  4870. logger.warning(
  4871. "[PLATE CHECK] Failed to capture snapshot for printer %s: %s", printer_id, snap_err
  4872. )
  4873. if objects_detected:
  4874. # Send notification about plate not empty
  4875. await ws_manager.broadcast(
  4876. {
  4877. "type": "plate_not_empty",
  4878. "printer_id": printer_id,
  4879. "printer_name": printer.name,
  4880. "message": f"Objects detected on build plate! Print paused. (Diff: {plate_result.difference_percent:.1f}%)",
  4881. }
  4882. )
  4883. # Also send push notification
  4884. try:
  4885. await notification_service.on_plate_not_empty(
  4886. printer_id=printer_id,
  4887. printer_name=printer.name,
  4888. db=db,
  4889. difference_percent=plate_result.difference_percent,
  4890. image_data=plate_photo_data,
  4891. )
  4892. except Exception as notif_err:
  4893. logger.warning("[PLATE CHECK] Failed to send notification: %s", notif_err)
  4894. elif check_failure:
  4895. logger.warning(
  4896. "[PLATE CHECK] Plate check could not run for printer %s (%s): %s; proceeding with print",
  4897. printer_id,
  4898. check_failure,
  4899. plate_result.message,
  4900. )
  4901. await _announce_plate_check_failed(printer_id, printer.name, check_failure)
  4902. else:
  4903. logger.info("[PLATE CHECK] Plate is empty for printer %s, proceeding with print", printer_id)
  4904. except Exception as plate_err:
  4905. # Don't block print on plate detection errors
  4906. logger.warning(
  4907. "[PLATE CHECK] Plate check could not run for printer %s: %s; proceeding with print",
  4908. printer_id,
  4909. plate_err,
  4910. )
  4911. await _announce_plate_check_failed(printer_id, printer.name, "error")
  4912. if not printer:
  4913. logger.info("[CALLBACK] Skipping archive - printer not found in database")
  4914. if not notification_sent:
  4915. await _send_print_start_notification(printer_id, data, logger=logger)
  4916. return
  4917. # Skip the printer's own jobs — a calibration run is not a user's print.
  4918. # See is_internal_printer_job for what counts and why both fields are
  4919. # tested; the pressure-advance line reports as a subtask name with no
  4920. # /usr/ path, which the old prefix-only test here missed entirely.
  4921. #
  4922. # No notification either. The event describes the printer calibrating
  4923. # itself, so "Print started" is as wrong as the archive was, and the
  4924. # matching completion is suppressed in on_print_complete for the same
  4925. # reason. Checked before the auto-archive switches, whose "off" path
  4926. # still notifies.
  4927. # A /usr/ path is the printer's own file. A name alone is not proof: a
  4928. # print Bambuddy dispatched is never one, whatever it is called.
  4929. # Dispatched = an expected print, or, after a restart that lost those,
  4930. # a queue job printing on the printer.
  4931. if is_internal_printer_job(data.get("filename", ""), data.get("subtask_name", "")) and (
  4932. (data.get("filename") or "").startswith("/usr/")
  4933. or not await _dispatched_by_bambuddy(db, printer_id, data.get("filename", ""), data.get("subtask_name", ""))
  4934. ):
  4935. logger.info(
  4936. "[CALLBACK] Skipping archive — internal printer job detected: filename=%s, subtask=%s",
  4937. data.get("filename", ""),
  4938. data.get("subtask_name", ""),
  4939. )
  4940. return
  4941. # The global switch is the master: off, no printer archives prints
  4942. # started outside Bambuddy; on, each printer's own switch decides.
  4943. from backend.app.api.routes.settings import get_setting
  4944. _global_auto_archive = await get_setting(db, "auto_archive")
  4945. _global_archives = _global_auto_archive is None or _global_auto_archive.lower() == "true"
  4946. archives_prints = printer.auto_archive and _global_archives
  4947. if not archives_prints:
  4948. # auto-archive disabled — check if there's an expected print (dispatched
  4949. # by BamBuddy via queue/reprint) that already has an archive to promote.
  4950. # If so, fall through to the expected-print handling below so the archive
  4951. # is tracked in _active_prints and usage tracking works at completion.
  4952. _fn = data.get("filename", "")
  4953. _sn = data.get("subtask_name", "")
  4954. _check_keys: list[tuple[int, str]] = []
  4955. if _sn:
  4956. _check_keys += [
  4957. (printer_id, _sn),
  4958. (printer_id, f"{_sn}.3mf"),
  4959. (printer_id, f"{_sn}.gcode.3mf"),
  4960. ]
  4961. if _fn:
  4962. _base_fn = _fn.split("/")[-1] if "/" in _fn else _fn
  4963. _check_keys.append((printer_id, _base_fn))
  4964. _no_archive_base = _base_fn.replace(".gcode", "").replace(".3mf", "")
  4965. _check_keys += [
  4966. (printer_id, _no_archive_base),
  4967. (printer_id, f"{_no_archive_base}.3mf"),
  4968. ]
  4969. _has_expected = any(k in _expected_prints for k in _check_keys)
  4970. if not _has_expected:
  4971. # No expected print — truly external print (started from slicer/touchscreen).
  4972. # Say which switch it was: before 1.3.0 the global one was saved
  4973. # and never read, so an install upgraded with it off stopped
  4974. # archiving with nothing to point at it (#3477).
  4975. if not _global_archives:
  4976. logger.info(
  4977. "[CALLBACK] Skipping archive: the global 'Auto-archive prints' switch "
  4978. "(Settings > General > Archive Settings) is off, and Bambuddy did not start this print"
  4979. )
  4980. else:
  4981. logger.info(
  4982. "[CALLBACK] Skipping archive: auto-archive is off for printer %s, "
  4983. "and Bambuddy did not start this print",
  4984. printer_id,
  4985. )
  4986. if not notification_sent:
  4987. _no_archive_creator: int | None = None
  4988. for _key in _check_keys:
  4989. _expected_prints.pop(_key, None)
  4990. _expected_print_registered_at.pop(_key, None)
  4991. popped_creator = _expected_print_creators.pop(_key, None)
  4992. if _no_archive_creator is None:
  4993. _no_archive_creator = popped_creator
  4994. _creator_data = {"created_by_id": _no_archive_creator} if _no_archive_creator else None
  4995. await _send_print_start_notification(printer_id, data, _creator_data, logger)
  4996. return
  4997. else:
  4998. logger.info("[CALLBACK] auto_archive disabled but expected print found — promoting archive")
  4999. # Get the filename and subtask_name
  5000. filename = data.get("filename", "")
  5001. subtask_name = data.get("subtask_name", "")
  5002. # MQTT subtask_id uniquely identifies a print job on the printer. When
  5003. # present, it lets us match an archive across a backend restart (#972):
  5004. # same id → same print → resume the existing row instead of cancelling
  5005. # it and recreating from scratch (which loses started_at). Treat "0"
  5006. # and "" as absent — Bambu reports "0" for non-cloud / local prints.
  5007. raw_mqtt = data.get("raw_data") or {}
  5008. subtask_id = raw_mqtt.get("subtask_id")
  5009. if subtask_id is not None:
  5010. subtask_id = str(subtask_id).strip()
  5011. if subtask_id in ("", "0"):
  5012. subtask_id = None
  5013. logger.info("[CALLBACK] Print start detected - filename: %s, subtask: %s", filename, subtask_name)
  5014. if not filename and not subtask_name:
  5015. # Send notification without archive data (no filename)
  5016. logger.info("[CALLBACK] Skipping archive - no filename or subtask_name")
  5017. if not notification_sent:
  5018. await _send_print_start_notification(printer_id, data, logger=logger)
  5019. return
  5020. # Check if this is an expected print from reprint/scheduled
  5021. expected_keys = _expected_print_keys(printer_id, filename, subtask_name)
  5022. expected_archive_id = None
  5023. # Who queued it, taken with the entries it was registered beside: left
  5024. # behind once those are gone (the archive deleted mid-dispatch, or the
  5025. # start notification already sent), the expiry never saw it, and an
  5026. # unrelated later print of that name was attributed to that user (EP-2).
  5027. fallback_creator = None
  5028. for key in expected_keys:
  5029. expected_archive_id = _expected_prints.pop(key, None)
  5030. _expected_print_registered_at.pop(key, None)
  5031. if expected_archive_id:
  5032. # Clean up other possible keys for this print
  5033. for other_key in expected_keys:
  5034. _expected_prints.pop(other_key, None)
  5035. _expected_print_registered_at.pop(other_key, None)
  5036. creator = _expected_print_creators.pop(other_key, None)
  5037. if fallback_creator is None:
  5038. fallback_creator = creator
  5039. break
  5040. if expected_archive_id:
  5041. # This is a reprint/scheduled print - use existing archive, don't create new one
  5042. logger.info("Using expected archive %s for print (skipping duplicate)", expected_archive_id)
  5043. # Bambuddy dispatched it: the kill switch's fast path keeps it as
  5044. # Bambuddy's once the expected entry is consumed here.
  5045. _owned_print_archive_ids.add(expected_archive_id)
  5046. from backend.app.models.archive import PrintArchive
  5047. result = await db.execute(select(PrintArchive).where(PrintArchive.id == expected_archive_id))
  5048. archive = result.scalar_one_or_none()
  5049. # Deleted after this handler loaded it (the plate check waits
  5050. # seconds) but before the route forgot the expected print: the
  5051. # archive the route kept or deleted is left as the route left it.
  5052. if archive and await _printer_gone(db, printer_id):
  5053. logger.info(
  5054. "Printer %s was deleted before its print started; not adopting archive %s", printer_id, archive.id
  5055. )
  5056. await forget_printer(printer_id)
  5057. return
  5058. if archive:
  5059. # Persist a restart-stable id so a later restart resumes this
  5060. # archive by subtask_id instead of name-matching + duplicating
  5061. # it (#1485). The printer often hasn't echoed subtask_id back
  5062. # this soon after dispatch, so fall back to the id Bambuddy
  5063. # minted when it sent the print command. Scoped to this
  5064. # expected-print branch on purpose: an expected match means
  5065. # Bambuddy dispatched this exact print in this process, so the
  5066. # client's last-dispatch id genuinely belongs to it — using it
  5067. # for an externally-started print could mis-tag the archive.
  5068. effective_subtask_id = subtask_id
  5069. if not effective_subtask_id:
  5070. _client = printer_manager.get_client(printer_id)
  5071. _dispatched = getattr(_client, "last_dispatch_subtask_id", None) if _client else None
  5072. if _dispatched:
  5073. effective_subtask_id = str(_dispatched).strip() or None
  5074. starting_plate_id = _print_plate_ids.get((printer_id, expected_archive_id))
  5075. if not await _adopt_archive_for_run(
  5076. db,
  5077. archive,
  5078. printer_id,
  5079. starting_plate_id=starting_plate_id,
  5080. effective_subtask_id=effective_subtask_id,
  5081. logger=logger,
  5082. ):
  5083. return
  5084. # Track as active print
  5085. _bind_active_print(printer_id, archive.id, archive.filename, filename, subtask_name)
  5086. await _open_run_for(archive.id, printer_id, subtask_id)
  5087. # Start timelapse session if external camera is enabled (#1353).
  5088. # Queue / VP-dispatched prints land here in the expected-archive
  5089. # branch and used to skip start_session entirely — frames were
  5090. # never captured and the post-print stitch silently returned None.
  5091. _maybe_start_layer_timelapse(printer, printer_id, archive.id)
  5092. # Inject ams_mapping into usage tracker session — the session was created
  5093. # before expected-print promotion, so it may have ams_mapping=None when
  5094. # the MQTT request topic subscription failed (common on P1S/A1).
  5095. _stored_map = _print_ams_mappings.get((printer_id, expected_archive_id))
  5096. _stored_plate_id = _print_plate_ids.get((printer_id, expected_archive_id))
  5097. if _stored_map or _stored_plate_id is not None:
  5098. try:
  5099. from backend.app.services.usage_tracker import _active_sessions
  5100. _ut_session = _active_sessions.get(printer_id)
  5101. if _ut_session and _stored_map and not _ut_session.ams_mapping:
  5102. _ut_session.ams_mapping = _stored_map
  5103. logger.info("[CALLBACK] Injected ams_mapping into usage tracker session: %s", _stored_map)
  5104. # plate_id injection covers direct-Print of plate N of a multi-plate
  5105. # 3MF — queue prints already capture it via the on_print_start queue
  5106. # lookup, but direct-Print never goes through the queue (#1697).
  5107. if _ut_session and _stored_plate_id is not None and _ut_session.plate_id is None:
  5108. _ut_session.plate_id = _stored_plate_id
  5109. logger.info("[CALLBACK] Injected plate_id into usage tracker session: %s", _stored_plate_id)
  5110. except Exception:
  5111. pass
  5112. # Set up energy tracking (#941: persist start on archive row)
  5113. await _record_energy_start(archive, printer_id, db, context="expected-print")
  5114. await ws_manager.send_archive_updated(
  5115. {
  5116. "id": archive.id,
  5117. "status": "printing",
  5118. }
  5119. )
  5120. # Send notification with archive data (reprint/scheduled)
  5121. if not notification_sent:
  5122. # Use archive's created_by_id; fall back to the creator registered via
  5123. # register_expected_print (handles library-file-based queue items where
  5124. # the freshly-created archive has no created_by_id yet), taken above.
  5125. archive_data = {
  5126. "print_time_seconds": _run_print_time_seconds(archive, printer_id),
  5127. "created_by_id": archive.created_by_id or fallback_creator,
  5128. }
  5129. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5130. # Extract printable objects from the archived 3MF file
  5131. _load_objects_from_archive(archive, printer_id, logger)
  5132. if archive.file_path:
  5133. _load_pauses_from_3mf(app_settings.base_dir / archive.file_path, printer_id)
  5134. # Store Spoolman tracking data for per-filament usage reporting
  5135. try:
  5136. await _store_spoolman_print_data(
  5137. printer_id,
  5138. archive.id,
  5139. archive.file_path,
  5140. db,
  5141. printer_manager,
  5142. ams_mapping=_get_start_ams_mapping(data, printer_id, archive.id),
  5143. plate_id=_get_start_plate_id(printer_id, archive.id),
  5144. )
  5145. except Exception as e:
  5146. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  5147. # Price the print from its spools now rather than only at completion (#3261)
  5148. _schedule_archive_cost_estimate(
  5149. printer_id,
  5150. archive.id,
  5151. printer_manager,
  5152. ams_mapping=_get_start_ams_mapping(data, printer_id, archive.id),
  5153. plate_id=_get_start_plate_id(printer_id, archive.id),
  5154. session_factory=async_session,
  5155. )
  5156. # Capture timelapse file baseline for snapshot-diff on completion
  5157. # (mirrors the new-archive branch). Queue / VP-dispatched prints
  5158. # hit this branch — without the baseline the completion-time scan
  5159. # falls into its "take baseline now" fallback, which snapshots
  5160. # AFTER the new MP4 already exists and never matches a diff
  5161. # (#1403 follow-up — see pwostran's 2026-05-18 support bundle).
  5162. await _capture_timelapse_baseline_at_start(printer, printer_id, logger, archive_id=archive.id)
  5163. await _forget_if_printer_gone(printer_id)
  5164. return # Skip creating a new archive
  5165. # The archive was deleted after the job was queued.
  5166. if not archives_prints:
  5167. # Only the expected print brought a printer that does not
  5168. # archive this far; with it gone, nothing is archived.
  5169. return
  5170. # Otherwise archive the print anew rather than drop it, so it still
  5171. # gets its notification, usage tracking and history.
  5172. logger.warning(
  5173. "Expected archive %s for printer %s no longer exists; archiving the print anew",
  5174. expected_archive_id,
  5175. printer_id,
  5176. )
  5177. # Check if there's already a "printing" archive for this printer/file
  5178. # This prevents duplicates when backend restarts during an active print
  5179. from backend.app.models.archive import PrintArchive
  5180. existing_archive: PrintArchive | None = None
  5181. resumed_by_run = False
  5182. # Best match: this job's run, open on the printer since the print
  5183. # started. The archive row is shared by every plate of it, and its
  5184. # printer, job id and status are the latest start's, which may be
  5185. # another printer's.
  5186. if subtask_id:
  5187. from backend.app.services.print_log import open_run_on_printer
  5188. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  5189. if run is not None and run.archive_id is not None:
  5190. existing_archive = await db.scalar(
  5191. select(PrintArchive).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  5192. )
  5193. resumed_by_run = existing_archive is not None
  5194. # Preferred match: subtask_id equality. MQTT reports the same subtask_id
  5195. # across a backend restart for the same print, so this is the most
  5196. # reliable way to reattach. Only a row still printing: a cancelled one
  5197. # has been through completion (usage, history, notifications) already,
  5198. # and reopening it would count the print twice.
  5199. if subtask_id and existing_archive is None:
  5200. by_id = await db.execute(
  5201. select(PrintArchive)
  5202. .where(PrintArchive.printer_id == printer_id)
  5203. .where(PrintArchive.subtask_id == subtask_id)
  5204. .where(PrintArchive.status == "printing")
  5205. .order_by(PrintArchive.created_at.desc())
  5206. .limit(1)
  5207. )
  5208. existing_archive = by_id.scalar_one_or_none()
  5209. # Fallback match: name-based lookup. Kept as-is for prints whose
  5210. # subtask_id is missing ("0" / local / non-cloud prints).
  5211. if existing_archive is None:
  5212. check_name = subtask_name or filename.split("/")[-1].replace(".gcode", "").replace(".3mf", "")
  5213. archive_filenames = [f"{check_name}.3mf", f"{check_name}.gcode.3mf"]
  5214. # A print started from the printer's screen names its file in full,
  5215. # so the forms above double its extension. The file itself is the
  5216. # one exact match, and nothing looser is added (#3009).
  5217. if _subtask_is_the_file(subtask_name, filename):
  5218. archive_filenames.append(subtask_name)
  5219. existing = await db.execute(
  5220. select(PrintArchive)
  5221. .where(PrintArchive.printer_id == printer_id)
  5222. .where(PrintArchive.status == "printing")
  5223. .where(
  5224. or_(
  5225. PrintArchive.print_name == check_name,
  5226. PrintArchive.filename.in_(archive_filenames),
  5227. )
  5228. )
  5229. .order_by(PrintArchive.created_at.desc())
  5230. .limit(1)
  5231. )
  5232. existing_archive = existing.scalar_one_or_none()
  5233. if existing_archive:
  5234. # subtask_id match → always resume, regardless of age. Same print,
  5235. # just a backend restart.
  5236. subtask_match = resumed_by_run or bool(subtask_id and existing_archive.subtask_id == subtask_id)
  5237. if subtask_match:
  5238. logger.info("Resuming archive %s on subtask_id match (%s)", existing_archive.id, subtask_id)
  5239. _bind_active_print(printer_id, existing_archive.id, existing_archive.filename, filename, subtask_name)
  5240. await _open_run_for(existing_archive.id, printer_id, subtask_id)
  5241. _mark_owned_if_dispatched(existing_archive)
  5242. if await _run_needs_energy_start(existing_archive, printer_id, db):
  5243. await _record_energy_start(existing_archive, printer_id, db, context="subtask-resume")
  5244. if not notification_sent:
  5245. archive_data = {
  5246. "print_time_seconds": _run_print_time_seconds(existing_archive, printer_id),
  5247. "created_by_id": existing_archive.created_by_id,
  5248. }
  5249. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5250. _load_objects_from_archive(existing_archive, printer_id, logger)
  5251. if existing_archive.file_path:
  5252. _load_pauses_from_resumed_archive(existing_archive, printer_id)
  5253. return
  5254. # Name-match only (no subtask_id to anchor on): decide resume vs.
  5255. # stale from the printer's *current* progress, not wall-clock age.
  5256. # A genuinely long print used to trip a blind 4h cutoff and have its
  5257. # live archive cancelled + duplicated on every backend restart
  5258. # (#1485). If the printer reports real progress, this name-matched
  5259. # 'printing' archive IS that ongoing print — resume it whatever its
  5260. # age. Only treat it as a stale leftover when the printer clearly
  5261. # shows a different, freshly-started print: near-0% progress on an
  5262. # archive far too old to still be at 0%. Unknown progress (printer
  5263. # not connected) never cancels — resuming is the safe default.
  5264. archive_age = datetime.now(timezone.utc) - existing_archive.created_at.replace(tzinfo=timezone.utc)
  5265. live_status = printer_manager.get_status(printer_id)
  5266. live_progress = getattr(live_status, "progress", None) if live_status else None
  5267. looks_stale = (
  5268. live_progress is not None and live_progress < 1.0 and archive_age.total_seconds() > 2 * 60 * 60
  5269. )
  5270. # Two subtask ids are two prints, however young the archive: one
  5271. # whose completion was missed must not swallow the next print of
  5272. # the same file. Except Bambuddy's own reprint, which reuses the
  5273. # row and mints a fresh id (#1807): the id it last sent says so,
  5274. # and after a restart, which forgets it, this printer's queue job
  5275. # running the archive.
  5276. if (
  5277. not looks_stale
  5278. and subtask_id
  5279. and existing_archive.subtask_id
  5280. and existing_archive.subtask_id != subtask_id
  5281. ):
  5282. _client = printer_manager.get_client(printer_id)
  5283. _dispatched = str(getattr(_client, "last_dispatch_subtask_id", None) or "").strip()
  5284. if _dispatched:
  5285. looks_stale = _dispatched != subtask_id
  5286. else:
  5287. from backend.app.models.print_queue import PrintQueueItem as _PQI
  5288. queue_run = await db.scalar(
  5289. select(_PQI.id)
  5290. .where(_PQI.archive_id == existing_archive.id)
  5291. .where(_PQI.printer_id == printer_id)
  5292. .where(_PQI.status == "printing")
  5293. .limit(1)
  5294. )
  5295. looks_stale = queue_run is None
  5296. if looks_stale:
  5297. logger.warning(
  5298. "Found stale 'printing' archive %s (age: %s, printer progress %s%%, subtask %s, printer reports %s)"
  5299. " — marking cancelled and creating new archive",
  5300. existing_archive.id,
  5301. archive_age,
  5302. live_progress,
  5303. existing_archive.subtask_id,
  5304. subtask_id,
  5305. )
  5306. existing_archive.status = "cancelled"
  5307. # Canonical key, not a sentence (issue #2974). "No status update
  5308. # received" is what both stale paths actually observed; which of
  5309. # the two it was is already carried by ``status`` -- cancelled
  5310. # here, the reconciled outcome at the reconnect site -- so one
  5311. # key loses no information and gives the Statistics breakdown a
  5312. # single bucket instead of two untranslatable prose strings.
  5313. existing_archive.failure_reason = "noStatusUpdate"
  5314. # And its run, as before runs were opened at the start: a new
  5315. # print starts on the printer, so whatever is open there ended
  5316. # unseen. Left open, the next reconnect would close it again.
  5317. from backend.app.services.print_log import discard_open_runs
  5318. discarded = await discard_open_runs(db, printer_id)
  5319. await db.commit()
  5320. await _announce_discarded_runs(discarded, printer_id)
  5321. # Its print keys go with it; the new archive registers its own.
  5322. for _key in [k for k, v in _active_prints.items() if v == existing_archive.id and k[0] == printer_id]:
  5323. del _active_prints[_key]
  5324. # Fall through to create new archive (don't return)
  5325. else:
  5326. logger.info(
  5327. f"Skipping duplicate - already have printing archive {existing_archive.id} for {check_name}"
  5328. )
  5329. # Track this as the active print
  5330. _bind_active_print(printer_id, existing_archive.id, existing_archive.filename, filename, subtask_name)
  5331. _mark_owned_if_dispatched(existing_archive)
  5332. # Attach subtask_id retroactively so future restarts can resume.
  5333. # Compare for inequality (not "is empty") to also pick up reprint
  5334. # dispatches that mint a fresh id — see #1807 for the bogus
  5335. # "Print Stopped" the strict-empty guard caused on reconnect.
  5336. if subtask_id and existing_archive.subtask_id != subtask_id:
  5337. existing_archive.subtask_id = subtask_id
  5338. await db.commit()
  5339. await _open_run_for(existing_archive.id, printer_id, subtask_id)
  5340. # Also set up energy tracking if not already tracked (#941: persisted column)
  5341. if await _run_needs_energy_start(existing_archive, printer_id, db):
  5342. await _record_energy_start(existing_archive, printer_id, db, context="existing-printing")
  5343. # Send notification with archive data (existing archive)
  5344. if not notification_sent:
  5345. archive_data = {
  5346. "print_time_seconds": _run_print_time_seconds(existing_archive, printer_id),
  5347. "created_by_id": existing_archive.created_by_id,
  5348. }
  5349. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5350. # Extract printable objects from the archived 3MF file
  5351. _load_objects_from_archive(existing_archive, printer_id, logger)
  5352. if existing_archive.file_path:
  5353. _load_pauses_from_resumed_archive(existing_archive, printer_id)
  5354. return
  5355. # Build list of possible 3MF filenames to try
  5356. possible_names = []
  5357. # Bambu printers typically store files as "Name.gcode.3mf"
  5358. # The subtask_name is usually the best source for the filename
  5359. if subtask_name:
  5360. if _subtask_is_the_file(subtask_name, filename):
  5361. # A print started from the printer's own screen reports the
  5362. # file's full name, extension included. That is the file, so
  5363. # no extension is appended to it (#3009).
  5364. possible_names.append(subtask_name)
  5365. else:
  5366. # Try common Bambu naming patterns
  5367. possible_names.append(f"{subtask_name}.gcode.3mf")
  5368. possible_names.append(f"{subtask_name}.3mf")
  5369. # Try original filename with .3mf extension
  5370. if filename:
  5371. # Extract just the filename part, not the full path
  5372. fname = filename.split("/")[-1] if "/" in filename else filename
  5373. if fname.endswith(".3mf"):
  5374. possible_names.append(fname)
  5375. elif fname.endswith(".gcode"):
  5376. base = fname.rsplit(".", 1)[0]
  5377. possible_names.append(f"{base}.gcode.3mf")
  5378. possible_names.append(f"{base}.3mf")
  5379. else:
  5380. possible_names.append(f"{fname}.gcode.3mf")
  5381. possible_names.append(f"{fname}.3mf")
  5382. # Also try with spaces converted to underscores (Bambu Studio may normalize filenames)
  5383. space_variants = []
  5384. for name in possible_names:
  5385. if " " in name:
  5386. space_variants.append(name.replace(" ", "_"))
  5387. possible_names.extend(space_variants)
  5388. # Remove duplicates while preserving order
  5389. seen = set()
  5390. possible_names = [x for x in possible_names if not (x in seen or seen.add(x))]
  5391. logger.info("Trying filenames: %s", possible_names)
  5392. # Release the pooled DB connection before the 3MF FTP download. Reaching
  5393. # here means none of the expected-/existing-archive write branches ran
  5394. # (they all return earlier) — only SELECTs have executed on this path, so
  5395. # this commit persists nothing; it ends the read transaction so the
  5396. # connection returns to the pool during the download. That download tries
  5397. # up to five remote paths per candidate filename with retry/backoff and
  5398. # can run for minutes under FTP contention; holding the session across it
  5399. # pinned one pooled connection idle-in-transaction (issue #2572). No DB
  5400. # work runs during the download — the new-archive writes below re-acquire
  5401. # a fresh connection, and expire_on_commit=False keeps printer.* readable.
  5402. await db.commit()
  5403. # Try to find and download the 3MF file
  5404. temp_path = None
  5405. downloaded_filename = None
  5406. # Cache check: cover endpoint may have already pulled this 3MF during
  5407. # the print (frontend opens the card and shows the thumbnail) — reuse
  5408. # that file instead of re-downloading 36MB over the same FTP link that
  5409. # just served it (#972). The cache keys on a normalized filename so
  5410. # variants like "X", "X.3mf", "X.gcode.3mf" all collapse to one entry.
  5411. for try_filename in possible_names:
  5412. if not try_filename.endswith(".3mf"):
  5413. continue
  5414. cached = get_cached_3mf(printer_id, try_filename)
  5415. if cached:
  5416. logger.info("Reusing cached 3MF from %s (avoided duplicate FTP)", cached)
  5417. temp_path = cached
  5418. downloaded_filename = try_filename
  5419. break
  5420. # Does this printer keep the sliced file somewhere FTPS can reach? On
  5421. # H2-series and P2S the answer is routinely no — the file stays on
  5422. # internal eMMC and port 990 only ever serves external storage — and
  5423. # then the whole sweep below (six filenames x five directories x four
  5424. # retries, then the directory walk) is ~110 connections that cannot
  5425. # succeed. Skip it and say why (#2780).
  5426. storage = print_file_reachable_over_ftp(printer_manager.get_status(printer_id))
  5427. # Set when a lookup is abandoned because the printer's FTPS cool-off is
  5428. # running rather than because the file is somewhere unreachable. The
  5429. # distinction is the whole of #2957: one is permanent, the other clears
  5430. # in minutes with the file still sitting on the printer.
  5431. blocked_by_ftps_cooloff = False
  5432. # Set when a probe reached the printer and still came back without the
  5433. # file -- a timeout mid-transfer, a refused connection, anything that is
  5434. # not a clean "not here". A 550 raises FileNotOnPrinterError and is
  5435. # caught by name below, so a file that genuinely is not on the card
  5436. # leaves this False and schedules nothing. Anything else means the
  5437. # transfer, not the file, is what failed, and that does not last (#3063).
  5438. ftp_transfer_failed = False
  5439. # The narrower case of the above: the directory walk *listed* a file
  5440. # for this print and could not fetch it, so the file is on the card
  5441. # under a name that matches. The only failure that rules out an
  5442. # earlier run's 3MF standing in for it (#3307).
  5443. walk_transfer_failed = False
  5444. # Paths the sweep reached and could not transfer -- a timeout, not a
  5445. # 550. The directory walk lists the same folders and would download
  5446. # the same file again: on a slow A1 that was a second 300 s on the
  5447. # printer's one FTPS slot, at print start, for a file the recovery
  5448. # retry fetches with a deadline twice as long (#3308).
  5449. sweep_failed_paths: set[str] = set()
  5450. # The print's name, for a fallback archive whose `subtask_name` the
  5451. # plate guard below had to disown. Display only, and deliberately kept
  5452. # apart from `subtask_name`: that variable is what every file lookup
  5453. # here is built from, and once a name has been shown to fetch another
  5454. # plate's 3MF it must not key `_active_prints` either, or the cover
  5455. # endpoint hands the same contradicted file to
  5456. # `_recover_fallback_archive` and fills the row in with it (#3126).
  5457. display_name_after_plate_reject: str | None = None
  5458. # Get FTP retry settings
  5459. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  5460. # ...but "the printer put it on eMMC" is where it went, not whether we
  5461. # can read it. An H2D with a card in mirrors the job to /cache and
  5462. # serves it happily, and skipping on the URL alone cost that reporter
  5463. # every archive for two days (#2856). So ask the printer instead of
  5464. # guessing: the dispatch named the exact file, which is one connection
  5465. # walking five paths rather than the sweep's ~110. Only when the probe
  5466. # comes back empty does the verdict's reason stand.
  5467. if not storage.reachable and not downloaded_filename and storage.probe_filename:
  5468. if ftps_handshake_blocked(printer.ip_address):
  5469. # Deliberately NOT recorded as a cool-off give-up. This branch
  5470. # only runs on an unreachable verdict, and that verdict is the
  5471. # honest, permanent reason the archive is empty — the probe was
  5472. # a long shot on top of it. Blaming the cool-off here would
  5473. # schedule a retry for a file sitting on internal eMMC, which is
  5474. # the sweep #2780 removed (#2957).
  5475. logger.debug(
  5476. "Not probing for %s on printer %s: its file service is not answering over TLS",
  5477. storage.probe_filename,
  5478. printer_id,
  5479. )
  5480. else:
  5481. probe_path = app_settings.archive_dir / "temp" / storage.probe_filename
  5482. probe_path.parent.mkdir(parents=True, exist_ok=True)
  5483. try:
  5484. probe_hit = await download_file_try_paths_async(
  5485. printer.ip_address,
  5486. printer.access_code,
  5487. ftp_probe_paths(storage.probe_filename),
  5488. probe_path,
  5489. socket_timeout=ftp_timeout,
  5490. printer_model=printer.model,
  5491. )
  5492. except Exception as e:
  5493. logger.debug("3MF probe for %s failed: %s", storage.probe_filename, e)
  5494. probe_hit = False
  5495. if probe_hit:
  5496. downloaded_filename = storage.probe_filename
  5497. temp_path = probe_path
  5498. cache_3mf_download(printer_id, downloaded_filename, probe_path)
  5499. # Naming the path, not just the file: a printer that keeps
  5500. # uploads around for weeks can serve a same-named copy of an
  5501. # earlier slice, and without the directory in the log that
  5502. # mismatch is invisible rather than merely rare (#1820).
  5503. logger.info(
  5504. "Found %s at %s over FTPS for printer %s even though the printer reported %s",
  5505. downloaded_filename,
  5506. probe_hit,
  5507. printer_id,
  5508. storage.reason,
  5509. )
  5510. if not storage.reachable and not downloaded_filename:
  5511. # Same opening words whether or not a probe ran, because that is
  5512. # the phrase support asks people to grep for — only the tail says
  5513. # which of the two happened.
  5514. logger.info(
  5515. "Skipping the 3MF lookup for printer %s: %s — %s",
  5516. printer_id,
  5517. storage.reason,
  5518. "no copy of it on external storage either"
  5519. if storage.probe_filename
  5520. else "the print file is not on storage Bambuddy can read over FTPS, so no path would find it",
  5521. )
  5522. for try_filename in possible_names if not downloaded_filename and storage.reachable else []:
  5523. if not try_filename.endswith(".3mf"):
  5524. continue
  5525. # Root (/) is where BambuStudio/OrcaSlicer uploads land on A1/P1-series
  5526. # printers, so try it first — deferring it to last cost #972's reporter
  5527. # ~48 minutes of retries on /cache//model//data//data/Metadata before
  5528. # landing on the path that actually had the file.
  5529. remote_paths = [
  5530. f"/{try_filename}",
  5531. f"/cache/{try_filename}",
  5532. f"/model/{try_filename}",
  5533. f"/data/{try_filename}",
  5534. f"/data/Metadata/{try_filename}",
  5535. ]
  5536. temp_path = app_settings.archive_dir / "temp" / try_filename
  5537. temp_path.parent.mkdir(parents=True, exist_ok=True)
  5538. for remote_path in remote_paths:
  5539. if ftps_handshake_blocked(printer.ip_address):
  5540. # The printer's FTPS service is not completing a TLS
  5541. # handshake, so it has no path we could reach — walking the
  5542. # remaining candidates only re-runs the same failure
  5543. # (#2780). Fall through to the no-3MF archive now.
  5544. #
  5545. # Remember *why*, though. This is the one give-up that is
  5546. # temporary: the cool-off clears in minutes and the file was
  5547. # on the printer the whole time. The fallback archive is
  5548. # stamped with it so a retry can be scheduled, and so the
  5549. # Archives banner stops blaming storage (#2957).
  5550. blocked_by_ftps_cooloff = True
  5551. logger.warning(
  5552. "Giving up on the 3MF for printer %s: its file service is not answering over TLS",
  5553. printer_id,
  5554. )
  5555. break
  5556. logger.debug("Trying FTP download: %s", remote_path)
  5557. try:
  5558. if ftp_retry_enabled:
  5559. downloaded = await with_ftp_retry(
  5560. download_file_async,
  5561. printer.ip_address,
  5562. printer.access_code,
  5563. remote_path,
  5564. temp_path,
  5565. timeout=ftp_timeout,
  5566. socket_timeout=ftp_timeout,
  5567. printer_model=printer.model,
  5568. max_retries=ftp_retry_count,
  5569. retry_delay=ftp_retry_delay,
  5570. operation_name=f"Download 3MF from {remote_path}",
  5571. cooloff_ip=printer.ip_address,
  5572. non_retry_exceptions=(FileNotOnPrinterError,),
  5573. )
  5574. else:
  5575. downloaded = await download_file_async(
  5576. printer.ip_address,
  5577. printer.access_code,
  5578. remote_path,
  5579. temp_path,
  5580. timeout=ftp_timeout,
  5581. socket_timeout=ftp_timeout,
  5582. printer_model=printer.model,
  5583. )
  5584. if downloaded:
  5585. downloaded_filename = try_filename
  5586. logger.info("Downloaded: %s", remote_path)
  5587. # Populate shared cache so the cover endpoint (if it
  5588. # runs next) doesn't refetch the same 36MB over FTP.
  5589. cache_3mf_download(printer_id, try_filename, temp_path)
  5590. break
  5591. # with_ftp_retry returns None once it has spent its budget,
  5592. # and download_file_async returns False on a timeout, so an
  5593. # exhausted transfer arrives here rather than as an
  5594. # exception (#3063).
  5595. ftp_transfer_failed = True
  5596. sweep_failed_paths.add(remote_path)
  5597. except FileNotOnPrinterError:
  5598. # 550 — file isn't at this path. Advance to next candidate
  5599. # without burning the retry budget.
  5600. logger.debug("3MF not at %s (550), trying next path", remote_path)
  5601. except Exception as e:
  5602. ftp_transfer_failed = True
  5603. sweep_failed_paths.add(remote_path)
  5604. logger.debug("FTP download failed for %s: %s", remote_path, e)
  5605. if downloaded_filename or ftps_handshake_blocked(printer.ip_address):
  5606. break
  5607. # If still not found, try listing directories to find matching file
  5608. # Different printer models use different directory structures. Skipped
  5609. # when the printer's FTPS handshake is failing — the directory walk is
  5610. # five more connections that cannot get further than the download did.
  5611. if (
  5612. not downloaded_filename
  5613. and storage.reachable
  5614. and (filename or subtask_name)
  5615. and not ftps_handshake_blocked(printer.ip_address)
  5616. ):
  5617. search_term = (subtask_name or filename).lower().replace(".gcode", "").replace(".3mf", "")
  5618. logger.info("Direct FTP download failed, searching directories for '%s'", search_term)
  5619. search_dirs = ["/cache", "/model", "/data", "/data/Metadata", "/"]
  5620. # Every folder is listed before one file is picked, so a file named
  5621. # for the print in a later folder wins over one that merely
  5622. # contains the name in an earlier folder.
  5623. listed_paths: list[str] = []
  5624. for search_dir in search_dirs:
  5625. try:
  5626. dir_files = await list_files_async(
  5627. printer.ip_address, printer.access_code, search_dir, printer_model=printer.model
  5628. )
  5629. except Exception as e:
  5630. logger.debug("Failed to list %s: %s", search_dir, e)
  5631. continue
  5632. threemf_files = [f.get("name") for f in dir_files if f.get("name", "").endswith(".3mf")]
  5633. if threemf_files:
  5634. logger.info(
  5635. f"Found {len(threemf_files)} 3MF files in {search_dir}: {threemf_files[:5]}{'...' if len(threemf_files) > 5 else ''}"
  5636. )
  5637. listed_paths += [
  5638. posixpath.join(search_dir, f.get("name", "")) for f in dir_files if not f.get("is_directory")
  5639. ]
  5640. for remote_full_path in _3mf_files_for_print(listed_paths, search_term):
  5641. if remote_full_path in sweep_failed_paths:
  5642. # Listed for this print, and its transfer already failed:
  5643. # the same as a failed walk download, without spending a
  5644. # second deadline on it. Skipping it cannot hand the walk a
  5645. # look-alike: while a file of the print's own name is
  5646. # listed, files that merely contain the name are not.
  5647. logger.info(
  5648. "Not downloading %s again: its transfer already failed; the recovery retry will fetch it",
  5649. remote_full_path,
  5650. )
  5651. walk_transfer_failed = True
  5652. continue
  5653. fname = posixpath.basename(remote_full_path)
  5654. logger.info("Found matching file: %s", remote_full_path)
  5655. temp_path = app_settings.archive_dir / "temp" / fname
  5656. temp_path.parent.mkdir(parents=True, exist_ok=True)
  5657. try:
  5658. if ftp_retry_enabled:
  5659. downloaded = await with_ftp_retry(
  5660. download_file_async,
  5661. printer.ip_address,
  5662. printer.access_code,
  5663. remote_full_path,
  5664. temp_path,
  5665. timeout=ftp_timeout,
  5666. socket_timeout=ftp_timeout,
  5667. printer_model=printer.model,
  5668. max_retries=ftp_retry_count,
  5669. retry_delay=ftp_retry_delay,
  5670. operation_name=f"Download 3MF from {remote_full_path}",
  5671. cooloff_ip=printer.ip_address,
  5672. )
  5673. else:
  5674. downloaded = await download_file_async(
  5675. printer.ip_address,
  5676. printer.access_code,
  5677. remote_full_path,
  5678. temp_path,
  5679. timeout=ftp_timeout,
  5680. socket_timeout=ftp_timeout,
  5681. printer_model=printer.model,
  5682. )
  5683. except Exception as e:
  5684. logger.debug("Failed to download %s: %s", remote_full_path, e)
  5685. continue
  5686. if downloaded:
  5687. downloaded_filename = fname
  5688. logger.info("Found and downloaded: %s", remote_full_path)
  5689. cache_3mf_download(printer_id, fname, temp_path)
  5690. break
  5691. # The listing named the file, so it is on the card;
  5692. # only the transfer failed (#3063).
  5693. ftp_transfer_failed = True
  5694. walk_transfer_failed = True
  5695. # A "Print again" on the printer's screen names no file Bambuddy can
  5696. # find: only its history copy under a generic name, and the 3MF's
  5697. # internal title (#3307). When an earlier run of the same job was
  5698. # archived, its 3MF is the one being printed again. Not when the
  5699. # directory walk listed a file for this print and only its transfer
  5700. # failed: that file is on the card, and the retry fetches it. A
  5701. # cool-off or a failed sweep path is no such evidence -- the retry
  5702. # tries the same names, which this print's file never had.
  5703. reused_3mf_from: int | None = None
  5704. if not downloaded_filename and not walk_transfer_failed:
  5705. reported_layers = raw_mqtt.get("total_layer_num")
  5706. earlier = await _archived_3mf_for_repeat(
  5707. db,
  5708. printer_id,
  5709. subtask_name,
  5710. filename,
  5711. reported_layers if isinstance(reported_layers, int) and reported_layers > 0 else None,
  5712. )
  5713. if earlier is not None:
  5714. reuse_name = Path(earlier.filename or "").name or f"archive_{earlier.id}.3mf"
  5715. # Per printer: two printers reprinting the same job at once
  5716. # must not write one temp file (archive_print names the
  5717. # archive after original_filename, not this path).
  5718. reuse_path = app_settings.archive_dir / "temp" / f"reprint_{printer_id}_{reuse_name}"
  5719. try:
  5720. reuse_path.parent.mkdir(parents=True, exist_ok=True)
  5721. await asyncio.to_thread(shutil.copyfile, app_settings.base_dir / earlier.file_path, reuse_path)
  5722. except OSError as e:
  5723. # A copy that fails leaves the print the fallback archive
  5724. # it would have had, not no archive at all.
  5725. logger.warning("Could not copy the 3MF of archive %s for printer %s: %s", earlier.id, printer_id, e)
  5726. else:
  5727. temp_path = reuse_path
  5728. downloaded_filename = reuse_name
  5729. reused_3mf_from = earlier.id
  5730. cache_3mf_download(printer_id, reuse_name, temp_path)
  5731. logger.info(
  5732. "No 3MF on printer %s for %s (%s); using the 3MF of archive %s, an earlier run of the same job",
  5733. printer_id,
  5734. subtask_name,
  5735. filename,
  5736. earlier.id,
  5737. )
  5738. # Validate the downloaded 3MF actually matches the plate that's running
  5739. # (#1204): subtask_name lags across consecutive plates of the same model,
  5740. # so the first FTP candidate (built from subtask_name) can land on the
  5741. # previous plate's still-resident upload. Cross-check the slice_info
  5742. # plate index against the plate parsed from gcode_file (always fresh —
  5743. # it's the field whose change triggered this callback).
  5744. if downloaded_filename and temp_path:
  5745. expected_plate = parse_plate_id(filename)
  5746. actual_plate = peek_plate_index_in_3mf(temp_path) if expected_plate is not None else None
  5747. if expected_plate is not None and actual_plate is not None and actual_plate != expected_plate:
  5748. logger.warning(
  5749. "[CALLBACK] 3MF plate mismatch: downloaded %s reports plate %s but printer is "
  5750. "running plate %s — subtask_name=%r appears stale, retrying with corrected name",
  5751. downloaded_filename,
  5752. actual_plate,
  5753. expected_plate,
  5754. subtask_name,
  5755. )
  5756. corrected_subtask = swap_plate_suffix(subtask_name, expected_plate)
  5757. retry_succeeded = False
  5758. if corrected_subtask and corrected_subtask != subtask_name:
  5759. for try_filename in (f"{corrected_subtask}.gcode.3mf", f"{corrected_subtask}.3mf"):
  5760. retry_temp_path = app_settings.archive_dir / "temp" / try_filename
  5761. retry_temp_path.parent.mkdir(parents=True, exist_ok=True)
  5762. for remote_path in (
  5763. f"/{try_filename}",
  5764. f"/cache/{try_filename}",
  5765. f"/model/{try_filename}",
  5766. f"/data/{try_filename}",
  5767. f"/data/Metadata/{try_filename}",
  5768. ):
  5769. try:
  5770. if ftp_retry_enabled:
  5771. downloaded = await with_ftp_retry(
  5772. download_file_async,
  5773. printer.ip_address,
  5774. printer.access_code,
  5775. remote_path,
  5776. retry_temp_path,
  5777. timeout=ftp_timeout,
  5778. socket_timeout=ftp_timeout,
  5779. printer_model=printer.model,
  5780. max_retries=ftp_retry_count,
  5781. retry_delay=ftp_retry_delay,
  5782. operation_name=f"Re-download 3MF from {remote_path}",
  5783. cooloff_ip=printer.ip_address,
  5784. non_retry_exceptions=(FileNotOnPrinterError,),
  5785. )
  5786. else:
  5787. downloaded = await download_file_async(
  5788. printer.ip_address,
  5789. printer.access_code,
  5790. remote_path,
  5791. retry_temp_path,
  5792. timeout=ftp_timeout,
  5793. socket_timeout=ftp_timeout,
  5794. printer_model=printer.model,
  5795. )
  5796. if downloaded and peek_plate_index_in_3mf(retry_temp_path) == expected_plate:
  5797. logger.info(
  5798. "[CALLBACK] Re-download succeeded with corrected name %s "
  5799. "(plate %s) — replacing wrong file",
  5800. try_filename,
  5801. expected_plate,
  5802. )
  5803. try:
  5804. temp_path.unlink(missing_ok=True)
  5805. except OSError:
  5806. pass
  5807. temp_path = retry_temp_path
  5808. downloaded_filename = try_filename
  5809. subtask_name = corrected_subtask
  5810. cache_3mf_download(printer_id, try_filename, temp_path)
  5811. retry_succeeded = True
  5812. break
  5813. elif downloaded:
  5814. # Wrong plate again — discard and keep trying
  5815. try:
  5816. retry_temp_path.unlink(missing_ok=True)
  5817. except OSError:
  5818. pass
  5819. except FileNotOnPrinterError:
  5820. continue
  5821. except Exception as e:
  5822. logger.debug("Re-download failed for %s: %s", remote_path, e)
  5823. if retry_succeeded:
  5824. break
  5825. # If the retry didn't find a matching file, drop the wrong 3MF
  5826. # so the no-3MF fallback below creates an archive whose name
  5827. # at least reflects the right plate.
  5828. if not retry_succeeded:
  5829. logger.warning(
  5830. "[CALLBACK] Could not re-download correct plate %s — falling back to no-3MF archive",
  5831. expected_plate,
  5832. )
  5833. try:
  5834. temp_path.unlink(missing_ok=True)
  5835. except OSError:
  5836. pass
  5837. temp_path = None
  5838. downloaded_filename = None
  5839. # Whatever the sweep's transport did earlier, it is not why
  5840. # this archive ends up empty: a 3MF downloaded fine, it was
  5841. # just the wrong plate. Retrying would re-fetch that same
  5842. # contradicted file under the same stale names and hand it
  5843. # to _recover_fallback_archive, which checks that a
  5844. # candidate is a readable 3MF but not which plate it is --
  5845. # so the row would be filled in with another plate's
  5846. # filament and cost, the exact swap #2957 removed (#3063).
  5847. ftp_transfer_failed = False
  5848. # Disown the name for *lookups*: it has just been shown to
  5849. # fetch another plate's 3MF, and it keys `_active_prints`
  5850. # below, where the cover endpoint's own download of that
  5851. # same name would find this archive and fill it in with the
  5852. # file we are discarding here.
  5853. #
  5854. # Keep it for the *title*, which is a separate question.
  5855. # ``swap_plate_suffix`` returns None both for a name that
  5856. # carries no "- Plate N" / "_plate_N" suffix and for no
  5857. # name at all, and those are not the same situation: a name
  5858. # without a suffix holds no stale plate number to be wrong
  5859. # about. Blanking both uses at once dropped the project
  5860. # name too, and the row fell through to the gcode_file path
  5861. # titled "plate_1" though the real name was in hand.
  5862. # #1204's own premise is consecutive plates *of the same
  5863. # model*, so the project part is right either way (#3126).
  5864. display_name_after_plate_reject = corrected_subtask or subtask_name or None
  5865. subtask_name = corrected_subtask or ""
  5866. # Deleted while the 3MF was being fetched (that can take minutes): no
  5867. # archive, timelapse or tracking for a printer that will never report
  5868. # the print's end. forget_printer again: the download registered its
  5869. # 3MF cache entry after the delete cleared the id. (A delete committing
  5870. # while the archive is written is caught after its commit.)
  5871. if await _printer_gone(db, printer_id):
  5872. logger.info("Printer %s was deleted during the print-start download; not archiving", printer_id)
  5873. await forget_printer(printer_id)
  5874. return
  5875. if not downloaded_filename or not temp_path:
  5876. logger.warning("Could not find 3MF file for print: %s", filename or subtask_name)
  5877. # Create a fallback archive without 3MF data so the print is still tracked
  5878. # This commonly happens with P1S/A1 printers where FTP has file size limitations
  5879. try:
  5880. from backend.app.models.archive import PrintArchive
  5881. # Why the card is empty. The two temporary causes outrank the
  5882. # storage verdict because they say the sweep never got a fair
  5883. # answer: a cool-off skipped it at the transport, and a failed
  5884. # transfer reached the printer but never finished. Either way
  5885. # the file is still on the card, so reporting where the printer
  5886. # files its jobs would describe a setting that is not the
  5887. # problem (#2957, #3063).
  5888. if blocked_by_ftps_cooloff:
  5889. no_3mf_reason = REASON_FTPS_COOLOFF
  5890. elif storage.reachable and ftp_transfer_failed:
  5891. no_3mf_reason = REASON_FTP_TRANSFER_FAILED
  5892. else:
  5893. no_3mf_reason = storage.reason
  5894. # Derive print name from subtask_name or filename. The
  5895. # plate guard's disowned name comes second: it is a real name
  5896. # for a real print, and only the gcode_file path is left
  5897. # otherwise -- which titles the row "plate_1" (#3126).
  5898. print_name = subtask_name or display_name_after_plate_reject or filename
  5899. if print_name:
  5900. # Clean up the name (remove extensions, path parts)
  5901. print_name = print_name.split("/")[-1]
  5902. print_name = print_name.replace(".gcode.3mf", "").replace(".gcode", "").replace(".3mf", "")
  5903. else:
  5904. print_name = "Unknown Print"
  5905. # Recover estimated print time from MQTT (best-effort for notifications)
  5906. fallback_print_time = None
  5907. mqtt_remaining = data.get("remaining_time")
  5908. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  5909. fallback_print_time = int(mqtt_remaining)
  5910. if fallback_print_time is None:
  5911. mc_remaining = (data.get("raw_data") or {}).get("mc_remaining_time")
  5912. if mc_remaining and isinstance(mc_remaining, (int, float)) and mc_remaining > 0:
  5913. fallback_print_time = int(mc_remaining * 60)
  5914. # Best-effort filament metadata from MQTT — see
  5915. # _extract_filament_data_from_mqtt. Without this the fallback
  5916. # archive's filament fields stayed NULL even though the AMS
  5917. # state at print start was sitting right there in `data`.
  5918. # The slicer's ams_mapping (when present) narrows the result
  5919. # to slots actually used by the print (#1533).
  5920. mqtt_filament_meta = _extract_filament_data_from_mqtt(
  5921. data, _get_start_ams_mapping(data, printer_id, None)
  5922. )
  5923. # Create minimal archive entry
  5924. fallback_archive = PrintArchive(
  5925. printer_id=printer_id,
  5926. filename=filename or f"{print_name}.3mf",
  5927. file_path="", # Empty - no 3MF file available
  5928. file_size=0,
  5929. print_name=print_name,
  5930. print_time_seconds=fallback_print_time,
  5931. status="printing",
  5932. started_at=detected_at,
  5933. subtask_id=subtask_id,
  5934. confirm_requested=await _ask_outcome_for_external_print(db, printer_id, subtask_name),
  5935. filament_type=mqtt_filament_meta.get("filament_type"),
  5936. filament_color=mqtt_filament_meta.get("filament_color"),
  5937. extra_data={
  5938. "no_3mf_available": True,
  5939. # Why the card is empty, when we know -- see above. The
  5940. # banner reads this to stop telling H2/P2 owners to
  5941. # switch on a setting that is already on and would not
  5942. # have helped (#2780).
  5943. "no_3mf_reason": no_3mf_reason,
  5944. "original_subtask": subtask_name or display_name_after_plate_reject or "",
  5945. "_print_data": data,
  5946. },
  5947. )
  5948. db.add(fallback_archive)
  5949. try:
  5950. await db.commit()
  5951. except IntegrityError:
  5952. # Not "Failed to create fallback archive" below, and no
  5953. # start notification for a printer that is gone.
  5954. if await _refused_for_deleted_printer(db, printer_id, "its fallback archive was written"):
  5955. return
  5956. raise
  5957. await db.refresh(fallback_archive)
  5958. logger.info("Created fallback archive %s for %s (no 3MF available)", fallback_archive.id, print_name)
  5959. if await _drop_archive_of_deleted_printer(db, printer_id, fallback_archive.id):
  5960. return
  5961. _maybe_start_layer_timelapse(printer, printer_id, fallback_archive.id)
  5962. # Track as active print
  5963. _bind_active_print(printer_id, fallback_archive.id, fallback_archive.filename, filename, subtask_name)
  5964. await _open_run_for(fallback_archive.id, printer_id, subtask_id)
  5965. # Record starting energy if smart plug available (#941: persisted column)
  5966. await _record_energy_start(fallback_archive, printer_id, db, context="fallback")
  5967. # Send WebSocket notification
  5968. await ws_manager.send_archive_created(
  5969. {
  5970. "id": fallback_archive.id,
  5971. "printer_id": fallback_archive.printer_id,
  5972. "filename": fallback_archive.filename,
  5973. "print_name": fallback_archive.print_name,
  5974. "status": fallback_archive.status,
  5975. }
  5976. )
  5977. # MQTT relay - publish archive created
  5978. try:
  5979. await mqtt_relay.on_archive_created(
  5980. archive_id=fallback_archive.id,
  5981. print_name=fallback_archive.print_name,
  5982. printer_name=printer.name,
  5983. status=fallback_archive.status,
  5984. )
  5985. except Exception:
  5986. pass # Don't fail if MQTT fails
  5987. # Store Spoolman tracking data (may not work for fallback since no 3MF)
  5988. try:
  5989. await _store_spoolman_print_data(
  5990. printer_id,
  5991. fallback_archive.id,
  5992. fallback_archive.file_path,
  5993. db,
  5994. printer_manager,
  5995. ams_mapping=_get_start_ams_mapping(data, printer_id, fallback_archive.id),
  5996. plate_id=_get_start_plate_id(printer_id, fallback_archive.id),
  5997. )
  5998. except Exception as e:
  5999. logger.debug("[SPOOLMAN] Could not store tracking for fallback archive: %s", e)
  6000. # Both temporary give-ups are worth coming back for, and for
  6001. # the same reason: the file is on the printer and the last look
  6002. # failed at the transport rather than finding nothing. One waits
  6003. # out the handshake block (#2957), the other waits for the
  6004. # printer to stop being busy (#3063). Deliberately not scheduled
  6005. # for a storage verdict: a file on internal eMMC will not appear
  6006. # at any FTPS path however long we wait, and retrying it is
  6007. # exactly the sweep #2780 removed.
  6008. if no_3mf_reason in (REASON_FTPS_COOLOFF, REASON_FTP_TRANSFER_FAILED) and possible_names:
  6009. # `possible_names`, not the raw MQTT strings: it is the exact
  6010. # list this flow just tried, already stripped of any path
  6011. # (`filename` arrives as "/data/Metadata/plate_1.gcode" on
  6012. # some firmware) and deduped.
  6013. _schedule_fallback_3mf_retry(
  6014. printer_id=printer_id,
  6015. archive_id=fallback_archive.id,
  6016. filenames=list(possible_names),
  6017. reason=no_3mf_reason,
  6018. )
  6019. # Send notification without archive data (file not found)
  6020. if not notification_sent:
  6021. await _send_print_start_notification(printer_id, data, logger=logger)
  6022. # The same baseline the other two on_print_start branches take
  6023. # (#2704), and last for the same reason they are: it lists the
  6024. # printer's timelapse directory, so a slow card must not delay
  6025. # the _active_prints registration, the energy reading, the
  6026. # archive-created event or the start notification above it.
  6027. #
  6028. # This branch never took one, so every no-3MF archive reached
  6029. # completion with no baseline in memory and none on the row, and
  6030. # the completion scan fell into its "snapshot now" fallback --
  6031. # which runs after the printer has written the video, so the new
  6032. # file landed inside the baseline and no diff ever matched
  6033. # (#2957 follow-up).
  6034. #
  6035. # Skipped when the FTPS cool-off is what produced this fallback:
  6036. # the listing needs the same connection that just failed, so it
  6037. # could only record that the card was unreadable. The scan
  6038. # handles that case by refusing to choose between candidates.
  6039. if not blocked_by_ftps_cooloff:
  6040. await _capture_timelapse_baseline_at_start(
  6041. printer, printer_id, logger, archive_id=fallback_archive.id
  6042. )
  6043. await _forget_if_printer_gone(printer_id)
  6044. return
  6045. except Exception as e:
  6046. logger.error("Failed to create fallback archive: %s", e)
  6047. # Send notification without archive data (file not found)
  6048. if not notification_sent:
  6049. await _send_print_start_notification(printer_id, data, logger=logger)
  6050. return
  6051. # The archive of the same slice sent before, which this print adds a
  6052. # run to instead of a new archive (#1634). Held until its run is open.
  6053. joined_archive_id: int | None = None
  6054. try:
  6055. # Archive the file with status "printing"
  6056. service = ArchiveService(db)
  6057. start_print_data = {
  6058. **data,
  6059. "status": "printing",
  6060. # Where the 3MF came from when the printer had none to
  6061. # give (#3307), for whoever reads this archive later.
  6062. **({"_3mf_from_archive": reused_3mf_from} if reused_3mf_from is not None else {}),
  6063. }
  6064. # The reused copy's temp name carries the printer id; a
  6065. # download's is the file's own name, as before.
  6066. start_filename = downloaded_filename if reused_3mf_from is not None else None
  6067. try:
  6068. claimed = await _claim_slicer_resend_archive(db, printer_id, temp_path)
  6069. if claimed is not None:
  6070. joined, file_in_use = claimed
  6071. joined_archive_id = joined.id
  6072. archive = await _start_slicer_resend_run(
  6073. db,
  6074. joined,
  6075. printer_id,
  6076. source=temp_path,
  6077. print_data=start_print_data,
  6078. subtask_id=subtask_id,
  6079. started_at=detected_at,
  6080. original_filename=start_filename,
  6081. logger=logger,
  6082. file_in_use=file_in_use,
  6083. )
  6084. if archive is None:
  6085. return
  6086. else:
  6087. archive = await service.archive_print(
  6088. printer_id=printer_id,
  6089. source_file=temp_path,
  6090. print_data=start_print_data,
  6091. subtask_id=subtask_id,
  6092. started_at=detected_at,
  6093. original_filename=start_filename,
  6094. )
  6095. except IntegrityError:
  6096. # archive_print removed the folder it had copied the 3MF into.
  6097. if await _refused_for_deleted_printer(db, printer_id, "its archive was written"):
  6098. return
  6099. raise
  6100. if archive:
  6101. # Ask-for-outcome for a print Bambuddy did not dispatch (#1898).
  6102. # Set on the row rather than passed to archive_print, which also
  6103. # serves the queue dispatcher — there the queue item decides.
  6104. # Guarded because this branch has no ``except``: an unhandled
  6105. # write error would take the _active_prints registration, the
  6106. # start notification, the energy reading and the timelapse
  6107. # baseline below it with it, and a missing prompt is by far the
  6108. # cheaper failure.
  6109. archive_id = archive.id
  6110. try:
  6111. if await _ask_outcome_for_external_print(db, printer_id, subtask_name):
  6112. archive.confirm_requested = True
  6113. await db.commit()
  6114. except Exception as e:
  6115. logger.warning("Could not flag archive %s for the outcome prompt: %s", archive_id, e)
  6116. # The rollback expires every loaded object, and on an async
  6117. # session reading one afterwards raises instead of
  6118. # reloading, so the two this branch goes on to use are
  6119. # fetched again. The archive itself was committed by
  6120. # archive_print; only the flag is lost.
  6121. try:
  6122. await db.rollback()
  6123. archive = await db.get(PrintArchive, archive_id)
  6124. printer = await db.get(Printer, printer_id)
  6125. except Exception as reload_error:
  6126. logger.warning(
  6127. "Could not reload archive %s after the failed flag write: %s", archive_id, reload_error
  6128. )
  6129. if archive is None:
  6130. # Deleted meanwhile: nothing to register the print for.
  6131. return
  6132. # Not a joined archive: it is not this print's to remove, and
  6133. # its adoption already checked the printer.
  6134. if joined_archive_id is None and await _drop_archive_of_deleted_printer(db, printer_id, archive_id):
  6135. return
  6136. # Track this active print (use both original filename and downloaded filename)
  6137. _bind_active_print(printer_id, archive_id, downloaded_filename, filename, subtask_name)
  6138. await _open_run_for(archive_id, printer_id, subtask_id)
  6139. if joined_archive_id is not None:
  6140. logger.info(
  6141. "Added the print of %s to archive %s: the same G-code was sent again",
  6142. downloaded_filename,
  6143. archive_id,
  6144. )
  6145. else:
  6146. logger.info("Created archive %s for %s", archive.id, downloaded_filename)
  6147. _maybe_start_layer_timelapse(printer, printer_id, archive.id)
  6148. # Record starting energy from smart plug if available (#941: persisted column)
  6149. await _record_energy_start(archive, printer_id, db, context="auto-archive")
  6150. archive_event = {
  6151. "id": archive.id,
  6152. "printer_id": archive.printer_id,
  6153. "filename": archive.filename,
  6154. "print_name": archive.print_name,
  6155. "status": archive.status,
  6156. }
  6157. if joined_archive_id is not None:
  6158. await ws_manager.send_archive_updated(archive_event)
  6159. else:
  6160. await ws_manager.send_archive_created(archive_event)
  6161. # MQTT relay - publish archive created
  6162. try:
  6163. await mqtt_relay.on_archive_created(
  6164. archive_id=archive.id,
  6165. print_name=archive.print_name or "",
  6166. printer_name=printer.name if printer else "",
  6167. status=archive.status,
  6168. )
  6169. except Exception:
  6170. pass # Don't fail if MQTT fails
  6171. # Send notification with archive data (new archive created)
  6172. if not notification_sent:
  6173. archive_data = {
  6174. "print_time_seconds": _run_print_time_seconds(archive, printer_id),
  6175. "created_by_id": archive.created_by_id,
  6176. }
  6177. await _send_print_start_notification(printer_id, data, archive_data, logger)
  6178. # Extract printable objects for skip object functionality
  6179. try:
  6180. from backend.app.services.archive import extract_printable_objects_from_3mf
  6181. client = printer_manager.get_client(printer_id)
  6182. if client:
  6183. with open(temp_path, "rb") as f:
  6184. threemf_data = f.read()
  6185. # Extract with positions for UI overlay, scoped to the
  6186. # plate that is printing — an all-plates 3MF carries
  6187. # every plate's objects (#2522).
  6188. printable_objects, bbox_all = extract_printable_objects_from_3mf(
  6189. threemf_data,
  6190. plate_number=resolve_plate_id(client.state),
  6191. include_positions=True,
  6192. )
  6193. if printable_objects:
  6194. # Store objects in printer state
  6195. client.state.printable_objects = printable_objects
  6196. client.state.printable_objects_bbox_all = bbox_all
  6197. client.state.skipped_objects = [] # Reset skipped objects for new print
  6198. logger.info(
  6199. "Loaded %s printable objects for printer %s", len(printable_objects), printer_id
  6200. )
  6201. except Exception as e:
  6202. logger.debug("Failed to extract printable objects: %s", e)
  6203. # From the archived copy: the downloaded one may be removed
  6204. # below while the pauses are still being read.
  6205. if archive is not None and archive.file_path:
  6206. _load_pauses_from_3mf(app_settings.base_dir / archive.file_path, printer_id)
  6207. # Store Spoolman tracking data for per-filament usage reporting
  6208. try:
  6209. await _store_spoolman_print_data(
  6210. printer_id,
  6211. archive.id,
  6212. archive.file_path,
  6213. db,
  6214. printer_manager,
  6215. ams_mapping=_get_start_ams_mapping(data, printer_id, archive_id),
  6216. plate_id=_get_start_plate_id(printer_id, archive_id),
  6217. )
  6218. except Exception as e:
  6219. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  6220. # Price the print from its spools now rather than only at completion (#3261)
  6221. _schedule_archive_cost_estimate(
  6222. printer_id,
  6223. archive.id,
  6224. printer_manager,
  6225. ams_mapping=_get_start_ams_mapping(data, printer_id, archive_id),
  6226. plate_id=_get_start_plate_id(printer_id, archive_id),
  6227. session_factory=async_session,
  6228. )
  6229. # Capture timelapse file baseline for snapshot-diff on completion
  6230. await _capture_timelapse_baseline_at_start(printer, printer_id, logger, archive_id=archive.id)
  6231. await _forget_if_printer_gone(printer_id)
  6232. finally:
  6233. if joined_archive_id is not None:
  6234. from backend.app.services import archive_work
  6235. archive_work.release(joined_archive_id)
  6236. # Keep temp_path around until print completes so the cover endpoint
  6237. # can reuse it (#972). Cache eviction in on_print_complete deletes
  6238. # the file. If the cache entry was evicted early (file vanished),
  6239. # clean up any stragglers here to avoid leaking disk on retries.
  6240. cached_now = get_cached_3mf(printer_id, downloaded_filename) if downloaded_filename else None
  6241. if temp_path and temp_path.exists() and cached_now != temp_path:
  6242. temp_path.unlink()
  6243. _TIMELAPSE_VIDEO_EXTENSIONS = (".mp4", ".avi")
  6244. _TIMELAPSE_DIRECTORIES = ("/timelapse", "/timelapse/video", "/record", "/recording")
  6245. # A baseline that could not be read at print start is read again on this
  6246. # schedule while the print runs (#3412). The printer writes the video only once
  6247. # the print has ended, so a reading taken mid-print is as good as one taken at
  6248. # the start. The interval is shorter than the FTPS cool-offs (300 s), which are
  6249. # what usually makes the card unreadable, so the first reading after one ends
  6250. # is not long behind; while a cool-off runs a round costs nothing, as no
  6251. # connection is opened. The cap bounds a print that never reports an end.
  6252. #
  6253. # A round that could not read the card doubles the wait before the next, up to
  6254. # the ceiling, and after that many failed rounds in a row the re-read gives up
  6255. # and leaves the baseline unread for the completion scan to handle (#3474). Each
  6256. # connection to a hung file service can leave a session behind on the printer.
  6257. _TIMELAPSE_BASELINE_REREAD_INTERVAL_SECONDS = 120.0
  6258. _TIMELAPSE_BASELINE_REREAD_MAX_INTERVAL_SECONDS = 1800.0
  6259. _TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS = 5
  6260. _TIMELAPSE_BASELINE_REREAD_MAX_ATTEMPTS = 720
  6261. # Poll schedule for the post-print timelapse scan (#2704). Module-level so
  6262. # tests can shrink them without waiting out real delays.
  6263. #
  6264. # This replaced a fixed [5, 10, 20, 30] retry ladder, i.e. roughly 65 s of
  6265. # looking. Across 247 support bundles the attempt that found the video was #1
  6266. # 272 times, then 17 / 13 / 13 — a flat tail against the cutoff rather than a
  6267. # decaying one, which is the signature of a budget that expires while files are
  6268. # still arriving. 457 scans were scheduled and only 262 ever attached. Big
  6269. # prints make big videos and the printer writes them after the print ends, so
  6270. # the poll now runs for minutes and costs one FTP LIST per round.
  6271. _TIMELAPSE_SCAN_FIRST_DELAY_SECONDS: float = 5.0
  6272. _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS: float = 30.0
  6273. _TIMELAPSE_SCAN_TIMEOUT_SECONDS: float = 900.0
  6274. def _timelapse_scan_max_attempts() -> int:
  6275. """Round cap for the poll, derived from the wall-clock budget.
  6276. The deadline alone is not a sufficient bound: it assumes each round really
  6277. waits, which stops being true the moment ``asyncio.sleep`` is patched out,
  6278. and an FTP list that fails immediately would otherwise spin against the
  6279. printer at full speed for the whole window. Whichever bound is reached
  6280. first ends the poll.
  6281. """
  6282. if _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS <= 0:
  6283. # A zero interval makes the wall-clock budget meaningless; fall back to
  6284. # the round count the production interval would have given.
  6285. return 32
  6286. return max(1, int(_TIMELAPSE_SCAN_TIMEOUT_SECONDS // _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS) + 1)
  6287. async def _read_timelapse_baseline(printer) -> tuple[set[str], bool]:
  6288. """The video names on *printer*'s card, and whether the card was read.
  6289. ``_list_timelapse_videos`` answers ``[]`` when its connect fails rather
  6290. than raising, so a card behind the FTPS handshake cool-off, or one whose
  6291. listing timed out, is indistinguishable from one holding no videos.
  6292. Everywhere that only wants to know "is there a video yet" the difference
  6293. does not matter -- both mean "not yet, retry".
  6294. It matters where an empty listing is recorded as a *baseline*. Recording
  6295. "the card held nothing" for a card that was never read means every video on
  6296. it counts as new, and the completion scan then attaches a stale video to
  6297. this print and deletes it from the printer (#2957 follow-up, #3412). So an
  6298. card counts as read only when every directory walked answered, with a
  6299. listing or with "no such directory". One that failed may be the one
  6300. holding the videos.
  6301. One walk, one connection per directory. It used to list the card with
  6302. ``_list_timelapse_videos`` first and walk the same directories again when
  6303. that came back empty, which against a hung file service was eight dead
  6304. connections a round instead of one (#3474). A directory that fails does
  6305. not end the walk unless the failure put the printer in a cool-off: a later
  6306. one holding videos still answers the question.
  6307. """
  6308. from backend.app.services.bambu_ftp import ftps_background_blocked, list_files_result_async
  6309. ip_address = getattr(printer, "ip_address", None)
  6310. printer_id = getattr(printer, "id", None)
  6311. if not ip_address:
  6312. return set(), True
  6313. # No card in the slot: nothing to read, and nothing a video could land on.
  6314. if printer_id is not None and not external_storage_present(printer_manager.get_status(printer_id)):
  6315. return set(), True
  6316. read = True
  6317. for timelapse_path in _TIMELAPSE_DIRECTORIES:
  6318. if ftps_background_blocked(ip_address):
  6319. # Below any path: every remaining one would fail the same way.
  6320. return set(), False
  6321. listing = await list_files_result_async(
  6322. ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  6323. )
  6324. if not (listing.available or listing.missing):
  6325. read = False
  6326. continue
  6327. videos = [
  6328. f
  6329. for f in listing.files
  6330. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  6331. ]
  6332. if videos:
  6333. return {f.get("name", "") for f in videos}, True
  6334. return set(), read
  6335. async def _list_timelapse_videos(printer) -> tuple[list[dict], str | None]:
  6336. """List video files from printer's timelapse directory.
  6337. Finds MP4 (X1/A1 series) and AVI (P1 series) timelapse files.
  6338. Returns (video_files, found_path) where video_files is a list of file dicts
  6339. and found_path is the directory where they were found, or ([], None).
  6340. An empty return does not distinguish "no videos" from "could not read the
  6341. card" — see :func:`_read_timelapse_baseline`, which the two baseline
  6342. callers use instead.
  6343. """
  6344. from backend.app.services.bambu_ftp import ftps_background_blocked, list_files_async
  6345. logger = logging.getLogger(__name__)
  6346. # No card in the slot means no /timelapse to walk — four connections that
  6347. # can only fail, on a path whose failures are swallowed and so would go on
  6348. # costing time silently forever (#2780).
  6349. #
  6350. # ``getattr`` rather than ``printer.id``: every dereference below happens
  6351. # inside the loop's own try/except, so a caller that passed something
  6352. # unexpected used to get an empty listing rather than an exception. Keep
  6353. # that, instead of making this gate the first thing that can raise here.
  6354. printer_id = getattr(printer, "id", None)
  6355. if printer_id is not None and not external_storage_present(printer_manager.get_status(printer_id)):
  6356. logger.debug("[TIMELAPSE] Skipping the scan for printer %s: it reports no external storage", printer_id)
  6357. return [], None
  6358. for timelapse_path in _TIMELAPSE_DIRECTORIES:
  6359. if ftps_background_blocked(getattr(printer, "ip_address", None) or ""):
  6360. # The printer's file service is refusing or hung: every remaining
  6361. # path is another connection it may keep a session for (#3474).
  6362. logger.debug("[TIMELAPSE] Skipping the scan for printer %s: its file service is cooling off", printer_id)
  6363. return [], None
  6364. try:
  6365. found_files = await list_files_async(
  6366. printer.ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  6367. )
  6368. if found_files:
  6369. video_files = [
  6370. f
  6371. for f in found_files
  6372. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  6373. ]
  6374. if video_files:
  6375. return video_files, timelapse_path
  6376. except Exception as e:
  6377. logger.debug("[TIMELAPSE] Path %s failed: %s", timelapse_path, e)
  6378. continue
  6379. return [], None
  6380. async def _capture_timelapse_baseline_at_start(
  6381. printer, printer_id: int, logger: logging.Logger, archive_id: int | None = None
  6382. ) -> None:
  6383. """Snapshot the printer's timelapse directory at print start so the
  6384. completion-time scan can pick the new file by set-difference.
  6385. Must be called from every on_print_start path that proceeds to a real
  6386. print — both the new-archive branch and the expected-archive branch (which
  6387. queue / VP-dispatched prints take). Without a baseline,
  6388. _scan_for_timelapse_with_retries falls into its "take baseline now"
  6389. fallback that runs AFTER the new MP4 has already landed on the SD card,
  6390. so the new file ends up in the "baseline" set and no diff ever matches.
  6391. Bambu printers in LAN-only mode don't sync NTP, so mtime ordering is
  6392. unreliable — the snapshot-diff approach sidesteps that entirely.
  6393. When ``archive_id`` is known the baseline is also written to the archive
  6394. row, so it survives a restart and the manual "Scan for Timelapse" button
  6395. can run the same diff instead of falling back to clock-based matching
  6396. (#2704). Only baselines taken at print start are persisted — one taken at
  6397. completion already contains the new video and would poison a later scan.
  6398. """
  6399. names: set[str] | None = None
  6400. unread = False
  6401. _timelapse_baselines_unread.discard(printer_id)
  6402. try:
  6403. names, read = await _read_timelapse_baseline(printer)
  6404. unread = not read
  6405. _timelapse_baselines[printer_id] = names
  6406. if unread:
  6407. # Recorded anyway, deliberately, and marked. An empty baseline taken
  6408. # off a card we could not read is not evidence, but it is still the
  6409. # right *default*: Bambuddy deletes each video from the printer once
  6410. # it is attached, so the usual card holds exactly one video at
  6411. # completion and an empty baseline resolves it. Persisting NULL
  6412. # instead would send completion to take its own snapshot, by which
  6413. # point this print's video is on the card and would be swallowed by
  6414. # it. The mark keeps the scan from choosing between several videos
  6415. # (#3412), and the card is read again while the print runs.
  6416. _timelapse_baselines_unread.add(printer_id)
  6417. logger.warning(
  6418. "[TIMELAPSE] Baseline for printer %s taken while its card could not be read — "
  6419. "treating it as empty for now and reading it again during the print",
  6420. printer_id,
  6421. )
  6422. logger.info(
  6423. "[TIMELAPSE] Baseline at print start: %s video files for printer %s",
  6424. len(names),
  6425. printer_id,
  6426. )
  6427. except Exception as e:
  6428. logger.warning("[TIMELAPSE] Failed to capture baseline at print start: %s", e)
  6429. run_started_at: datetime | None = None
  6430. run_id: int | None = None
  6431. if archive_id is not None:
  6432. try:
  6433. async with async_session() as db:
  6434. from backend.app.models.archive import PrintArchive
  6435. from backend.app.services.print_log import open_run_on_printer
  6436. archive = await db.get(PrintArchive, archive_id)
  6437. if archive is not None:
  6438. # Written even when the listing failed, and then as NULL. A
  6439. # reprint reuses the archive row, so leaving the previous
  6440. # run's baseline in place would have the scan diff this print
  6441. # against the state of the printer before the *last* one —
  6442. # and a stale baseline reads as authoritative, where NULL
  6443. # correctly falls back to a fresh snapshot.
  6444. baseline = sorted(names) if names is not None else None
  6445. archive.timelapse_baseline = baseline
  6446. archive.timelapse_baseline_unread = unread
  6447. run_started_at = archive.started_at
  6448. # And on the run this print opened, whose scan reads it:
  6449. # the archive's is its latest start's, which another plate
  6450. # of it on another printer can take over.
  6451. run = await open_run_on_printer(db, printer_id)
  6452. if run is not None and run.archive_id == archive_id:
  6453. run.timelapse_baseline = baseline
  6454. run.timelapse_baseline_unread = unread
  6455. run_id = run.id
  6456. await db.commit()
  6457. except Exception as e:
  6458. # In-memory baseline still covers the normal completion path.
  6459. logger.warning("[TIMELAPSE] Failed to persist baseline for archive %s: %s", archive_id, e)
  6460. if unread and names is not None:
  6461. spawn_background_task(
  6462. _reread_unread_timelapse_baseline(printer_id, names, archive_id, run_started_at, run_id),
  6463. name=f"reread-timelapse-baseline-{printer_id}",
  6464. )
  6465. # Before the print has ended there is no video yet: a long PREPARE (heating,
  6466. # levelling) counts as much as the print itself.
  6467. _BASELINE_REREAD_STATES = _ACTIVE_PRINT_STATES | {"PREPARE", "SLICING"}
  6468. def _schedule_timelapse_scan(
  6469. archive_id: int, printer_id: int, *, plate_id: int | None, run_id: int | None = None
  6470. ) -> None:
  6471. """Start the completion scan with the print-start baseline: the names, and
  6472. whether they were taken off a card that could not be read. Taking them
  6473. ends any re-read still running for it. ``run_id`` is the print that
  6474. ended, whose video it looks for."""
  6475. unread = printer_id in _timelapse_baselines_unread
  6476. _timelapse_baselines_unread.discard(printer_id)
  6477. baseline = _timelapse_baselines.pop(printer_id, None)
  6478. from backend.app.services import archive_work
  6479. # Held while it looks: it finds its print by the archive's id, which a
  6480. # merge would move away (archive redesign P2).
  6481. spawn_background_task(
  6482. archive_work.held_while(
  6483. archive_id,
  6484. _scan_for_timelapse_with_retries(
  6485. archive_id, baseline, plate_id=plate_id, baseline_unread=unread, run_id=run_id
  6486. ),
  6487. ),
  6488. name=f"scan-timelapse-{archive_id}",
  6489. )
  6490. def _unread_baseline_round(printer_id: int, names: set[str]) -> bool | None:
  6491. """Whether to read an unread baseline again now: True to read, None to
  6492. skip this round, False once it is over -- the print it was taken for is
  6493. no longer the one on the printer, has stopped printing, or the baseline
  6494. has been read.
  6495. ``names`` is the very set object recorded at print start: completion pops
  6496. it and the next print records a new one, so identity marks the run. A
  6497. printer that is momentarily disconnected says nothing about its print, so
  6498. that only skips the round.
  6499. So does a print that is not recording a timelapse: completion only looks
  6500. for a video when one was recorded, so reading the card for it is a
  6501. connection for nothing (#3474). Skipped rather than ended, as the flag can
  6502. arrive after the print has started.
  6503. """
  6504. if _timelapse_baselines.get(printer_id) is not names or printer_id not in _timelapse_baselines_unread:
  6505. return False
  6506. state = printer_manager.get_status(printer_id)
  6507. if state is None or not state.connected:
  6508. return None
  6509. if state.state not in _BASELINE_REREAD_STATES:
  6510. return False
  6511. return True if state.timelapse else None
  6512. async def _reread_unread_timelapse_baseline(
  6513. printer_id: int,
  6514. names: set[str],
  6515. archive_id: int | None,
  6516. run_started_at: datetime | None,
  6517. run_id: int | None = None,
  6518. ) -> None:
  6519. """Read again a print-start baseline the card could not give (#3412).
  6520. The printer writes the video only once the print has ended, so the card
  6521. read at any point during the print holds exactly what it held at the
  6522. start. Each round stops once the print is no longer the one the baseline
  6523. was taken for or has stopped printing, and checks that again after the
  6524. listing returns, before anything is recorded.
  6525. A round that cannot read the card doubles the wait before the next one,
  6526. and enough of them in a row end the re-read (#3474): see
  6527. ``_TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS``.
  6528. """
  6529. from backend.app.models.archive import PrintArchive
  6530. from backend.app.models.printer import Printer
  6531. from backend.app.services.print_log import run_of
  6532. logger = logging.getLogger(__name__)
  6533. interval = _TIMELAPSE_BASELINE_REREAD_INTERVAL_SECONDS
  6534. failed_rounds = 0
  6535. for _ in range(_TIMELAPSE_BASELINE_REREAD_MAX_ATTEMPTS):
  6536. await asyncio.sleep(interval)
  6537. now = _unread_baseline_round(printer_id, names)
  6538. if now is False:
  6539. return
  6540. if now is None:
  6541. continue
  6542. try:
  6543. async with async_session() as db:
  6544. printer = await db.get(Printer, printer_id)
  6545. if printer is None:
  6546. return
  6547. fresh, read = await _read_timelapse_baseline(printer)
  6548. except Exception as e:
  6549. logger.debug("[TIMELAPSE] Re-reading the baseline for printer %s failed: %s", printer_id, e)
  6550. fresh, read = set(), False
  6551. if not read:
  6552. failed_rounds += 1
  6553. if failed_rounds >= _TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS:
  6554. logger.warning(
  6555. "[TIMELAPSE] Card of printer %s still unreadable after %s tries; no longer reading "
  6556. "the baseline for this print, it stays unread",
  6557. printer_id,
  6558. failed_rounds,
  6559. )
  6560. return
  6561. interval = min(interval * 2, _TIMELAPSE_BASELINE_REREAD_MAX_INTERVAL_SECONDS)
  6562. continue
  6563. now = _unread_baseline_round(printer_id, names)
  6564. if now is False:
  6565. return
  6566. if now is None:
  6567. # The printer dropped off while the listing was out, so nothing
  6568. # says it did not end meanwhile: keep the reading for a round that
  6569. # can vouch for it.
  6570. continue
  6571. names.update(fresh)
  6572. _timelapse_baselines_unread.discard(printer_id)
  6573. logger.info(
  6574. "[TIMELAPSE] Baseline for printer %s read during the print: %s video files",
  6575. printer_id,
  6576. len(fresh),
  6577. )
  6578. if archive_id is None:
  6579. return
  6580. try:
  6581. async with async_session() as db:
  6582. archive = await db.get(PrintArchive, archive_id)
  6583. # Only the run it was taken for: by now the row can belong to a
  6584. # reprint, whose baseline is its own.
  6585. if archive is not None and archive.started_at == run_started_at and archive.timelapse_baseline_unread:
  6586. archive.timelapse_baseline = sorted(names)
  6587. archive.timelapse_baseline_unread = False
  6588. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6589. if run is not None and run.timelapse_baseline_unread:
  6590. run.timelapse_baseline = sorted(names)
  6591. run.timelapse_baseline_unread = False
  6592. await db.commit()
  6593. except Exception as e:
  6594. # The in-memory baseline is read now, which covers completion
  6595. # unless Bambuddy restarts first.
  6596. logger.warning("[TIMELAPSE] Failed to persist the re-read baseline for archive %s: %s", archive_id, e)
  6597. return
  6598. async def _timelapse_still_wanted(db, archive, run_id: int | None, run_started_at: datetime | None) -> bool:
  6599. """Whether the print a timelapse is for still has none: its run, by id,
  6600. exists without a video and no later print took over (``superseded``).
  6601. Without a run, the archive is still on that print's start (a newer run's
  6602. video is its own) and has none."""
  6603. from backend.app.services.print_log import run_of, superseded
  6604. if run_id is not None:
  6605. run = await run_of(db, archive.id, run_id)
  6606. return run is not None and run.timelapse_path is None and not await superseded(db, run)
  6607. return archive.timelapse_path is None and archive.started_at == run_started_at
  6608. async def _scan_for_timelapse_with_retries(
  6609. archive_id: int,
  6610. baseline_names: set[str] | None = None,
  6611. *,
  6612. plate_id: int | None = None,
  6613. baseline_unread: bool = False,
  6614. run_id: int | None = None,
  6615. ):
  6616. """Poll the printer for this print's timelapse and attach it.
  6617. Snapshot diff, not timestamp matching: a printer in LAN-only mode cannot
  6618. reach Bambu's NTP server, so the clock behind both the filename and the FTP
  6619. mtime is arbitrarily wrong — one reporter's P1S was six and a half days out
  6620. (#2704). Comparing the current listing against the set of filenames that
  6621. existed when the print started needs no clock at all, because the printer
  6622. writes the video only once the print has ended.
  6623. Baseline precedence: the caller's in-memory set, then the one persisted on
  6624. the archive at print start, then a snapshot taken now. The last of those is
  6625. a poor substitute — by completion the new video may already be on the card,
  6626. in which case it lands in the "baseline" and no diff can ever match — but it
  6627. is all that is available for a print that began before Bambuddy started.
  6628. On success the video is deleted from the printer, which keeps ``/timelapse``
  6629. down to the unclaimed files and makes the next diff unambiguous.
  6630. ``baseline_unread`` marks the caller's baseline as taken off a card that
  6631. could not be read; a stored one carries the same mark on the archive.
  6632. ``run_id`` is the print that ended (archive redesign): the video is that
  6633. run's, the stored baseline is the run's own, and the poll ends once the
  6634. run has a video or is gone. Without it (no run was written), the run is
  6635. whichever the archive is on when the scan starts.
  6636. """
  6637. from backend.app.services.print_log import run_of
  6638. logger = logging.getLogger(__name__)
  6639. # Cleared when the baseline was taken off a card we could not read, so the
  6640. # attach step refuses to choose between several candidates (#2957, #3412).
  6641. baseline_trusted = True
  6642. # --- Phase 1: establish the baseline -------------------------------------
  6643. try:
  6644. async with async_session() as db:
  6645. from backend.app.models.printer import Printer
  6646. service = ArchiveService(db)
  6647. archive = await service.get_archive(archive_id)
  6648. if not archive:
  6649. logger.warning("[TIMELAPSE] Archive %s not found, aborting", archive_id)
  6650. return
  6651. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6652. if run_id is not None and run is None:
  6653. logger.info("[TIMELAPSE] Run %s of archive %s is gone, aborting", run_id, archive_id)
  6654. return
  6655. if run.timelapse_path if run is not None else archive.timelapse_path:
  6656. logger.info("[TIMELAPSE] Archive %s already has timelapse attached", archive_id)
  6657. return
  6658. # The printer the print ran on: plates of one archive can print on
  6659. # two printers, and the archive's is the one that started last.
  6660. scan_printer_id = run.printer_id if run is not None and run.printer_id else archive.printer_id
  6661. if not scan_printer_id:
  6662. logger.warning("[TIMELAPSE] Archive %s has no printer, aborting", archive_id)
  6663. return
  6664. # The run this scan is for. The archive row passes to a newer run
  6665. # when it prints again (the next plate, a reprint), and that run's
  6666. # video is its own.
  6667. run_started_at = run.started_at if run is not None else archive.started_at
  6668. # The baseline stored for it: the run's own, else the archive's
  6669. # while that is still this run's start's.
  6670. stored_baseline, stored_unread = None, False
  6671. if run is not None and run.timelapse_baseline is not None:
  6672. stored_baseline, stored_unread = run.timelapse_baseline, bool(run.timelapse_baseline_unread)
  6673. elif archive.started_at == run_started_at:
  6674. stored_baseline, stored_unread = archive.timelapse_baseline, archive.timelapse_baseline_unread
  6675. if baseline_names is not None and baseline_unread and stored_baseline is not None and not stored_unread:
  6676. # The card was read at print start, and the in-memory copy is
  6677. # one Bambuddy took after a restart off a card it could not
  6678. # read: the stored one is this run's and is evidence.
  6679. baseline_names = set(stored_baseline)
  6680. logger.info(
  6681. "[TIMELAPSE] Using stored baseline over an unread one: %s existing video files for archive %s",
  6682. len(baseline_names),
  6683. archive_id,
  6684. )
  6685. elif baseline_names is not None:
  6686. baseline_trusted = not baseline_unread
  6687. logger.info(
  6688. "[TIMELAPSE] Using print-start baseline: %s existing video files for archive %s",
  6689. len(baseline_names),
  6690. archive_id,
  6691. )
  6692. elif stored_baseline is not None:
  6693. # Persisted at print start — survives a restart mid-print.
  6694. baseline_names = set(stored_baseline)
  6695. baseline_trusted = not stored_unread
  6696. logger.info(
  6697. "[TIMELAPSE] Using stored baseline: %s existing video files for archive %s",
  6698. len(baseline_names),
  6699. archive_id,
  6700. )
  6701. else:
  6702. result = await db.execute(select(Printer).where(Printer.id == scan_printer_id))
  6703. printer = result.scalar_one_or_none()
  6704. if not printer:
  6705. logger.warning("[TIMELAPSE] Printer not found for archive %s, aborting", archive_id)
  6706. return
  6707. baseline_names, baseline_trusted = await _read_timelapse_baseline(printer)
  6708. if not baseline_trusted:
  6709. # The card is unreadable at the one moment a baseline has to
  6710. # be taken, so the empty listing means "we never looked",
  6711. # not "these are all new". Carry on with it anyway — the
  6712. # usual card holds exactly one video, which resolves
  6713. # correctly — but stop the poll from *choosing* between
  6714. # several, which is how a stale video got attached to this
  6715. # print and then deleted off the printer (#2957 follow-up).
  6716. logger.warning(
  6717. "[TIMELAPSE] Baseline for archive %s taken while printer %s's card could not be "
  6718. "read. A single new video still resolves; several will not be guessed between — "
  6719. "use Scan for Timelapse to pick one by hand",
  6720. archive_id,
  6721. scan_printer_id,
  6722. )
  6723. logger.info(
  6724. "[TIMELAPSE] Baseline snapshot (fallback): %s existing video files for archive %s",
  6725. len(baseline_names),
  6726. archive_id,
  6727. )
  6728. except Exception as e:
  6729. logger.warning("[TIMELAPSE] Failed to take baseline snapshot for archive %s: %s", archive_id, e)
  6730. return
  6731. # --- Phase 2: poll for a file that was not there when the print began -----
  6732. deadline = time.monotonic() + _TIMELAPSE_SCAN_TIMEOUT_SECONDS
  6733. max_attempts = _timelapse_scan_max_attempts()
  6734. seen_names: set[str] = set()
  6735. delay = _TIMELAPSE_SCAN_FIRST_DELAY_SECONDS
  6736. attempt = 0
  6737. while True:
  6738. await asyncio.sleep(delay)
  6739. delay = _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS
  6740. attempt += 1
  6741. try:
  6742. from backend.app.models.printer import Printer
  6743. # Read phase: fetch archive + printer in a short session and release
  6744. # the pooled connection BEFORE the FTP list/download below. Holding it
  6745. # across the FTP round-trips left one connection idle-in-transaction per
  6746. # in-flight scan (issue #2572).
  6747. async with async_session() as db:
  6748. from backend.app.services.print_log import claimed_timelapse_stems
  6749. service = ArchiveService(db)
  6750. archive = await service.get_archive(archive_id)
  6751. if not archive:
  6752. logger.warning("[TIMELAPSE] Archive %s not found, stopping poll", archive_id)
  6753. return
  6754. if not await _timelapse_still_wanted(db, archive, run_id, run_started_at):
  6755. logger.info(
  6756. "[TIMELAPSE] Archive %s has its video or is printing again, stopping the run's poll",
  6757. archive_id,
  6758. )
  6759. return
  6760. result = await db.execute(select(Printer).where(Printer.id == scan_printer_id))
  6761. printer = result.scalar_one_or_none()
  6762. if not printer:
  6763. logger.warning("[TIMELAPSE] Printer not found for archive %s, stopping poll", archive_id)
  6764. return
  6765. claimed = await claimed_timelapse_stems(db, scan_printer_id, archive_id=archive_id, run_id=run_id)
  6766. # I/O phase (no DB connection held): FTP list + download.
  6767. video_files, found_path = await _list_timelapse_videos(printer)
  6768. # The poll can run for dozens of rounds, so only narrate a round
  6769. # that saw something change. Repeating the whole listing every 30 s
  6770. # would bury the one interesting line in the support bundle.
  6771. names_now = {f.get("name", "") for f in video_files}
  6772. changed = attempt == 1 or names_now != seen_names
  6773. seen_names = names_now
  6774. speak = logger.info if changed else logger.debug
  6775. if video_files:
  6776. speak("[TIMELAPSE] Attempt %s: Found %s video files in %s", attempt, len(video_files), found_path)
  6777. if changed:
  6778. for f in video_files[:5]:
  6779. logger.info("[TIMELAPSE] - %s", f.get("name"))
  6780. attached = await _attach_first_unclaimed_timelapse(
  6781. archive_id,
  6782. printer,
  6783. video_files,
  6784. baseline_names,
  6785. claimed,
  6786. attempt,
  6787. logger,
  6788. quiet=not changed,
  6789. require_unambiguous=not baseline_trusted,
  6790. plate_id=plate_id,
  6791. run_started_at=run_started_at,
  6792. run_id=run_id,
  6793. )
  6794. if attached:
  6795. return
  6796. else:
  6797. speak("[TIMELAPSE] Attempt %s: No video files found, will retry", attempt)
  6798. except Exception as e:
  6799. logger.warning("[TIMELAPSE] Attempt %s failed with error: %s", attempt, e)
  6800. if attempt >= max_attempts or time.monotonic() >= deadline:
  6801. break
  6802. # No name-match fallback: it compared the print name against the filename,
  6803. # and Bambu firmware only ever writes "video_<timestamp>". Across 247 support
  6804. # bundles it fired 159 times and matched zero times, so all it added was a
  6805. # misleading log line before giving up.
  6806. logger.warning(
  6807. "[TIMELAPSE] No new video appeared for archive %s within %ss, giving up",
  6808. archive_id,
  6809. int(_TIMELAPSE_SCAN_TIMEOUT_SECONDS),
  6810. )
  6811. async def _attach_layer_timelapse(
  6812. archive_id: int,
  6813. timelapse_path: Path,
  6814. plate_id: int | None,
  6815. run_started_at: datetime | None,
  6816. run_id: int | None = None,
  6817. ) -> None:
  6818. """Attach the external camera's stitched timelapse to the archive.
  6819. The camera's video is the one the user set up in Bambuddy, so it wins
  6820. over the printer's own when both are on: it replaces the printer's video
  6821. (usually there first; stitching takes minutes) and that file is deleted,
  6822. leaving nothing in the archive folder unreferenced. ``run_started_at`` is
  6823. the run the frames were taken for: when the archive is printing again by
  6824. the time the stitch is done, the video is not the new run's and is
  6825. dropped. The stitched file is a temporary either way.
  6826. ``run_id`` is the print that ended (archive redesign): the video is that
  6827. run's and replaces that run's printer video; the poll by start above is
  6828. for a print without a run.
  6829. """
  6830. from backend.app.services.print_log import run_of, superseded
  6831. from backend.app.utils.safe_path import PathTraversalError, safe_join_under
  6832. logger = logging.getLogger(__name__)
  6833. try:
  6834. async with async_session() as db:
  6835. service = ArchiveService(db)
  6836. archive = await service.get_archive(archive_id)
  6837. if archive is None:
  6838. return
  6839. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6840. if run_id is not None and run is None:
  6841. logger.info("[LAYER-TL] Run %s of archive %s is gone; dropping its layer timelapse", run_id, archive_id)
  6842. return
  6843. if run is not None and await superseded(db, run):
  6844. logger.info(
  6845. "[LAYER-TL] A later print took over run %s of archive %s; dropping its layer timelapse",
  6846. run_id,
  6847. archive_id,
  6848. )
  6849. return
  6850. if run is None and archive.started_at != run_started_at:
  6851. logger.info(
  6852. "[LAYER-TL] Archive %s is printing again; dropping the previous run's layer timelapse", archive_id
  6853. )
  6854. return
  6855. holder = run if run is not None else archive
  6856. replaced = holder.timelapse_path
  6857. logger.info("[LAYER-TL] Attaching timelapse %s to archive %s", timelapse_path, archive_id)
  6858. timelapse_data = await asyncio.to_thread(timelapse_path.read_bytes)
  6859. if not await service.attach_timelapse(
  6860. archive_id, timelapse_data, "layer_timelapse.mp4", plate_id=plate_id, run_id=run_id
  6861. ):
  6862. return
  6863. attached = holder.timelapse_path
  6864. logger.info("[LAYER-TL] Layer timelapse attached successfully")
  6865. if replaced and replaced != attached:
  6866. try:
  6867. old = safe_join_under(app_settings.base_dir, replaced, http=False)
  6868. except PathTraversalError:
  6869. return
  6870. await asyncio.to_thread(old.unlink, missing_ok=True)
  6871. logger.info("[LAYER-TL] Replaced the printer's timelapse %s on archive %s", replaced, archive_id)
  6872. finally:
  6873. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  6874. # ``run_started_at`` not given: attach whichever run the archive is on.
  6875. _ANY_RUN = object()
  6876. async def _attach_first_unclaimed_timelapse(
  6877. archive_id: int,
  6878. printer,
  6879. video_files: list[dict],
  6880. baseline_names: set[str],
  6881. claimed: set[str],
  6882. attempt: int,
  6883. logger: logging.Logger,
  6884. *,
  6885. quiet: bool = False,
  6886. require_unambiguous: bool = False,
  6887. plate_id: int | None = None,
  6888. run_started_at: datetime | None | object = _ANY_RUN,
  6889. run_id: int | None = None,
  6890. ) -> bool:
  6891. """Download and attach the one video that belongs to this print.
  6892. A candidate is any file absent from the print-start baseline. More than one
  6893. can qualify when a previous print's video landed late, after this print's
  6894. baseline was taken — those are filtered out by name, because they are
  6895. already attached to another archive. Sorting the candidates instead would
  6896. mean sorting on mtime or on the filename timestamp, both of which come from
  6897. the printer's unsynced clock.
  6898. Returns True once a video is attached. The printer's copy is deleted only
  6899. after the attach succeeds on bytes whose length matched the listing.
  6900. ``quiet`` downgrades the "nothing yet" lines to DEBUG when the caller has
  6901. already seen this exact listing — the poll runs for many rounds and only the
  6902. rounds where something changed are worth an INFO line.
  6903. """
  6904. from backend.app.services.bambu_ftp import (
  6905. DownloadDeadlineExceeded,
  6906. delete_archived_timelapse,
  6907. download_file_bytes_async,
  6908. remote_file_settled,
  6909. )
  6910. speak = logger.debug if quiet else logger.info
  6911. new_files = [f for f in video_files if f.get("name", "") not in baseline_names]
  6912. if not new_files:
  6913. speak("[TIMELAPSE] Attempt %s: No new files since baseline, will retry", attempt)
  6914. return False
  6915. candidates = [f for f in new_files if Path(f.get("name", "")).stem not in claimed]
  6916. if not candidates:
  6917. speak(
  6918. "[TIMELAPSE] Attempt %s: %s new file(s), all already attached to other archives, will retry",
  6919. attempt,
  6920. len(new_files),
  6921. )
  6922. return False
  6923. if len(candidates) > 1:
  6924. if require_unambiguous:
  6925. # The baseline is not evidence -- it was taken off a card that could
  6926. # not be read -- so "new since the baseline" does not narrow these
  6927. # down at all. Taking the first would attach an arbitrary video to
  6928. # this print and then delete it from the printer.
  6929. logger.warning(
  6930. "[TIMELAPSE] Attempt %s: %s unclaimed videos (%s) and no baseline to tell them apart — "
  6931. "leaving all of them on the printer for manual selection",
  6932. attempt,
  6933. len(candidates),
  6934. ", ".join(str(f.get("name")) for f in candidates),
  6935. )
  6936. return False
  6937. logger.warning(
  6938. "[TIMELAPSE] Attempt %s: %s unclaimed new files (%s) — taking the first; "
  6939. "the rest stay on the printer for manual selection",
  6940. attempt,
  6941. len(candidates),
  6942. ", ".join(str(f.get("name")) for f in candidates),
  6943. )
  6944. target = candidates[0]
  6945. file_name = str(target.get("name") or "") # every listed entry has one
  6946. remote_path = target.get("path") or f"/timelapse/{file_name}"
  6947. logger.info(
  6948. "[TIMELAPSE] Attempt %s: New file detected: %s (downloading for archive %s)",
  6949. attempt,
  6950. file_name,
  6951. archive_id,
  6952. )
  6953. # The listing always carries a size (`list_files` skips entries it can't
  6954. # parse), but read it explicitly: the delete below is destructive and must
  6955. # depend on a size we actually had, not on one we hoped was there.
  6956. expected_size = target.get("size")
  6957. try:
  6958. timelapse_data = await download_file_bytes_async(
  6959. printer.ip_address,
  6960. printer.access_code,
  6961. remote_path,
  6962. printer_model=printer.model,
  6963. expected_size=expected_size,
  6964. )
  6965. except DownloadDeadlineExceeded:
  6966. # Too slow for its size this round; the printer keeps its copy.
  6967. timelapse_data = None
  6968. if not timelapse_data:
  6969. # Short or failed transfer. The printer keeps its copy, so the next
  6970. # round can try again — which is exactly why the delete below is
  6971. # gated on a verified download.
  6972. logger.warning("[TIMELAPSE] Attempt %s: Failed to download new file, will retry", attempt)
  6973. return False
  6974. # The length check above proves we got what the listing said, not that the
  6975. # printer had finished writing. A video still being written can be listed
  6976. # short, served short, and pass — so confirm it has stopped growing before
  6977. # committing to it and deleting the original (#2704).
  6978. if not await remote_file_settled(
  6979. printer.ip_address,
  6980. printer.access_code,
  6981. remote_path,
  6982. len(timelapse_data),
  6983. printer_model=printer.model,
  6984. ):
  6985. return False
  6986. # Write phase: attach in a fresh short-lived session. The download took a
  6987. # while: by now the archive may have its video, or be printing again for a
  6988. # newer run, whose video this is not. Then nothing is attached or deleted,
  6989. # and the poll ends (True).
  6990. async with async_session() as db:
  6991. service = ArchiveService(db)
  6992. if run_id is not None or run_started_at is not _ANY_RUN:
  6993. current = await service.get_archive(archive_id)
  6994. started = None if run_started_at is _ANY_RUN else cast("datetime | None", run_started_at)
  6995. if current is None or not await _timelapse_still_wanted(db, current, run_id, started):
  6996. logger.info(
  6997. "[TIMELAPSE] Archive %s moved on during the download (video attached or printing again); "
  6998. "%s stays on the printer",
  6999. archive_id,
  7000. file_name,
  7001. )
  7002. return True
  7003. success = await service.attach_timelapse(
  7004. archive_id, timelapse_data, file_name, plate_id=plate_id, run_id=run_id
  7005. )
  7006. if not success:
  7007. logger.warning("[TIMELAPSE] Failed to attach timelapse to archive %s", archive_id)
  7008. return False
  7009. logger.info("[TIMELAPSE] Successfully attached timelapse to archive %s", archive_id)
  7010. await delete_archived_timelapse(
  7011. printer.ip_address,
  7012. printer.access_code,
  7013. remote_path,
  7014. verified=expected_size is not None,
  7015. printer_model=printer.model,
  7016. printer_name=printer.name,
  7017. )
  7018. return True
  7019. # Defaults for the finish-photo-from-timelapse polling loop (#1397). These are
  7020. # module-level so tests can monkeypatch them down to ~0 without timing out.
  7021. _FINISH_PHOTO_TIMELAPSE_POLL_INTERVAL_SECONDS: float = 3.0
  7022. _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS: float = 60.0
  7023. # How long the *background* upgrade keeps waiting after the notification has
  7024. # already gone out (#2704 follow-up). The short bound above exists so a slow
  7025. # printer can't hold up the print-complete notification; this one exists so the
  7026. # archive still ends up with the better frame afterwards.
  7027. #
  7028. # Measured across 261 attaches in the support bundles, the video lands a median
  7029. # 13s after the print ends — but the P1 series writes MJPEG AVI rather than
  7030. # H.264 MP4 and serves it slowly, so its p90 is 167s and the worst observed case
  7031. # was 546s. Every other model was inside 26s. The long budget is therefore
  7032. # almost entirely for P1-series users; on everything else the short wait already
  7033. # wins and this task never runs.
  7034. _FINISH_PHOTO_UPGRADE_TIMEOUT_SECONDS: float = 900.0
  7035. async def _capture_finish_photo_from_timelapse(
  7036. archive_id: int,
  7037. archive_dir: Path,
  7038. timeout: float | None = None,
  7039. rotation: int = 0,
  7040. run_id: int | None = None,
  7041. ) -> tuple[str | None, bool]:
  7042. """Wait for the per-print timelapse to land on the archive and extract its
  7043. last frame as the finish photo (#1397).
  7044. Bambu firmware stops timelapse recording after the toolhead parks but
  7045. before the bed-drop end-gcode runs, so the last frame frames the finished
  7046. print correctly. A live camera grab at gcode_state=FINISH captures the
  7047. bed already lowered.
  7048. ``_scan_for_timelapse_with_retries`` runs in parallel and writes
  7049. ``archive.timelapse_path`` when the file lands. This function polls for
  7050. that field.
  7051. Returns ``(filename, still_pending)``. ``still_pending`` is True only when
  7052. the wait ran out with no video on the archive yet — i.e. the video may
  7053. still be coming and a later attempt could succeed. It is False when the
  7054. video landed (whether or not extraction worked), because in that case
  7055. waiting longer changes nothing. The caller uses that to decide between
  7056. falling back permanently and scheduling a background upgrade.
  7057. ``rotation`` is the printer's camera_rotation, applied to the extracted
  7058. still (#2708) so this source agrees with every other finish-photo source.
  7059. The archived video itself is the printer's own file and is left alone —
  7060. rotating it would mean re-encoding it.
  7061. ``run_id`` is the print that ended: its own video is the one waited for,
  7062. not the archive's, which can be another plate's.
  7063. """
  7064. import uuid
  7065. from backend.app.models.archive import PrintArchive
  7066. from backend.app.services.camera import apply_camera_rotation_to_file, extract_video_last_frame
  7067. from backend.app.services.print_log import run_of
  7068. logger = logging.getLogger(__name__)
  7069. budget = _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS if timeout is None else timeout
  7070. deadline = asyncio.get_event_loop().time() + budget
  7071. poll_interval = _FINISH_PHOTO_TIMELAPSE_POLL_INTERVAL_SECONDS
  7072. while True:
  7073. async with async_session() as db:
  7074. if run_id is not None:
  7075. run = await run_of(db, archive_id, run_id)
  7076. timelapse_relpath = run.timelapse_path if run else None
  7077. else:
  7078. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  7079. archive = result.scalar_one_or_none()
  7080. timelapse_relpath = archive.timelapse_path if archive else None
  7081. if timelapse_relpath:
  7082. video_path = app_settings.base_dir / timelapse_relpath
  7083. if video_path.exists() and video_path.stat().st_size > 0:
  7084. photos_dir = archive_dir / "photos"
  7085. photos_dir.mkdir(parents=True, exist_ok=True)
  7086. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  7087. filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  7088. output_path = photos_dir / filename
  7089. if await extract_video_last_frame(video_path, output_path):
  7090. await apply_camera_rotation_to_file(output_path, rotation, logger)
  7091. logger.info(
  7092. "[PHOTO-BG] Extracted finish photo from timelapse %s for archive %s",
  7093. video_path.name,
  7094. archive_id,
  7095. )
  7096. return filename, False
  7097. logger.warning(
  7098. "[PHOTO-BG] Timelapse %s landed but last-frame extraction failed for archive %s; falling back",
  7099. video_path.name,
  7100. archive_id,
  7101. )
  7102. return None, False
  7103. if asyncio.get_event_loop().time() >= deadline:
  7104. logger.info(
  7105. "[PHOTO-BG] Timelapse for archive %s didn't land within %.0fs; falling back to live camera",
  7106. archive_id,
  7107. budget,
  7108. )
  7109. return None, True
  7110. await asyncio.sleep(poll_interval)
  7111. async def _upgrade_finish_photo_from_timelapse(
  7112. archive_id: int,
  7113. archive_dir: Path,
  7114. rotation: int = 0,
  7115. run_id: int | None = None,
  7116. replace: str | None = None,
  7117. ) -> None:
  7118. """Put the timelapse's last frame on an archive after the fact (#2704).
  7119. The print-complete notification waits only ~60s for the video, because
  7120. holding a notification for minutes is worse than sending it with a live
  7121. camera grab. On a P1-series printer the video often lands well after that,
  7122. so the archive used to be stuck with the live grab — which is taken at
  7123. ``gcode_state=FINISH``, after the end G-code has dropped the bed, and is
  7124. the worse photo of the two.
  7125. This keeps waiting in the background and, when the video arrives, extracts
  7126. the frame and puts it *first* in the archive's photo list, so opening the
  7127. gallery shows it. ``replace`` is the live grab: the frame is written over
  7128. that file, under its name, rather than added next to it. Two finish photos
  7129. of the same moment is one too many, and the name has to survive: with
  7130. authentication off, the notification that already went out links to
  7131. ``/archives/<id>/photos/<name>``, which now shows the better photo. (With
  7132. it on, the link is to a copy in notification_photos, and attachments are
  7133. bytes, so neither cares.) Without ``replace``, or once that photo has been
  7134. deleted, the frame is added under its own name. ``run_id`` is the print it
  7135. is for, whose list leads with it too.
  7136. """
  7137. from backend.app.services.print_log import add_run_photo, run_of
  7138. logger = logging.getLogger(__name__)
  7139. filename, _ = await _capture_finish_photo_from_timelapse(
  7140. archive_id, archive_dir, timeout=_FINISH_PHOTO_UPGRADE_TIMEOUT_SECONDS, rotation=rotation, run_id=run_id
  7141. )
  7142. if not filename:
  7143. logger.info("[PHOTO-UPGRADE] No timelapse frame for archive %s; keeping the live grab", archive_id)
  7144. return
  7145. try:
  7146. async with async_session() as db:
  7147. from backend.app.models.archive import PrintArchive
  7148. archive = await db.get(PrintArchive, archive_id)
  7149. if archive is None:
  7150. return
  7151. photos = list(archive.photos or [])
  7152. if filename in photos:
  7153. return
  7154. photos_dir = archive_dir / "photos"
  7155. if replace and replace in photos and (photos_dir / replace).is_file():
  7156. # Atomic, so the gallery never reads half a JPEG.
  7157. await asyncio.to_thread(os.replace, photos_dir / filename, photos_dir / replace)
  7158. filename = replace
  7159. photos.remove(replace)
  7160. # Front of the list: PhotoGalleryModal opens at index 0.
  7161. archive.photos = [filename, *photos]
  7162. add_run_photo(await run_of(db, archive_id, run_id) if run_id is not None else None, filename, first=True)
  7163. await db.commit()
  7164. except Exception as e:
  7165. logger.warning("[PHOTO-UPGRADE] Failed to attach upgraded photo to archive %s: %s", archive_id, e)
  7166. return
  7167. logger.info(
  7168. "[PHOTO-UPGRADE] Archive %s now leads with the timelapse frame %s%s",
  7169. archive_id,
  7170. filename,
  7171. " (replacing the live grab)" if filename == replace else "",
  7172. )
  7173. await ws_manager.send_archive_updated({"id": archive_id, "photo_added": filename})
  7174. async def _restore_usage_tracking_session(printer_id: int, state, db, logger) -> None:
  7175. """Put the filament-attribution context back after a restart mid-print.
  7176. ``usage_tracker._active_sessions`` and ``PrinterState.tray_change_log``
  7177. both die with the process. The print keeps running, so at completion the
  7178. tracker would fall back to whatever the printer reports *now* — and AMS
  7179. filament backup makes "now" the substitute tray, charging the whole print
  7180. to the spool that only finished it.
  7181. The persisted row is only trusted when its print name still matches what
  7182. the printer says it is running: a row left behind by a completion we never
  7183. saw must not attach itself to the next print.
  7184. """
  7185. try:
  7186. from backend.app.api.routes.settings import get_setting
  7187. from backend.app.services.usage_tracker import (
  7188. clear_persisted_session,
  7189. get_persisted_ams_mapping,
  7190. get_persisted_print_name,
  7191. get_persisted_spool_change_log,
  7192. restore_session,
  7193. )
  7194. persisted_name = await get_persisted_print_name(db, printer_id)
  7195. current_name = (state.subtask_name or "").strip()
  7196. if persisted_name and current_name and persisted_name.strip() != current_name:
  7197. logger.info(
  7198. "[RESTART] Discarding stale print session for printer %s (%r != running %r)",
  7199. printer_id,
  7200. persisted_name,
  7201. current_name,
  7202. )
  7203. await clear_persisted_session(db, printer_id)
  7204. # Fall through to seeding: the print on the printer is real, it just
  7205. # isn't the one the row described.
  7206. persisted_log = None
  7207. else:
  7208. # Spoolman users get the tray-change log back but no in-memory
  7209. # session — see ``on_print_start`` on why that dict is load-bearing
  7210. # for the remain%-sync guard.
  7211. _spoolman_on = await get_setting(db, "spoolman_enabled")
  7212. persisted_log = await restore_session(
  7213. db,
  7214. printer_id,
  7215. register_active=not (bool(_spoolman_on) and _spoolman_on.lower() == "true"),
  7216. )
  7217. # The slot pills (#1762) read the mapping captured from the print
  7218. # command, which dies with the process as well. One this process
  7219. # has captured since is newer.
  7220. if state.print_ams_mapping is None:
  7221. state.print_ams_mapping = await get_persisted_ams_mapping(db, printer_id)
  7222. # Spools swapped on a tray before the restart (#3325); anything this
  7223. # process has recorded since goes after them.
  7224. persisted_swaps = await get_persisted_spool_change_log(db, printer_id)
  7225. if persisted_swaps:
  7226. seen = getattr(state, "spool_change_log", None) or []
  7227. state.spool_change_log = persisted_swaps + [item for item in seen if item not in persisted_swaps]
  7228. if persisted_log:
  7229. restored = [tuple(entry) for entry in persisted_log if isinstance(entry, (list, tuple)) and len(entry) == 2]
  7230. # Anything this process already observed goes after the persisted
  7231. # history — the log is ordered by layer, and a fresh process can
  7232. # only have seen changes from later in the print.
  7233. for entry in state.tray_change_log or []:
  7234. if tuple(entry) not in restored:
  7235. restored.append(tuple(entry))
  7236. state.tray_change_log = restored
  7237. tray_now = state.tray_now
  7238. if 0 <= tray_now <= 254:
  7239. if not state.tray_change_log:
  7240. # No persisted history — a print that started before this build,
  7241. # or before the row existed. Seed with the tray feeding right
  7242. # now so the remainder of the print is at least attributable to
  7243. # the right spool.
  7244. state.tray_change_log = [(tray_now, state.layer_num)]
  7245. logger.info(
  7246. "[RESTART] Seeded tray change log for printer %s: tray=%d at layer=%d",
  7247. printer_id,
  7248. tray_now,
  7249. state.layer_num,
  7250. )
  7251. # The tray handler updates ``last_loaded_tray`` on every push
  7252. # regardless of whether it logged a change, so re-align it to avoid
  7253. # a duplicate entry on the next push. Only ever with a real tray:
  7254. # ``last_loaded_tray`` is the "survives the end-of-print retract to
  7255. # 255" fallback, and writing 255 into it would defeat that.
  7256. state.last_loaded_tray = tray_now
  7257. except Exception:
  7258. # Never let attribution recovery cost the caller its timelapse
  7259. # baseline — that capture has to happen before the printer uploads
  7260. # the in-flight MP4 and there is no second chance at it.
  7261. logger.exception("[RESTART] Failed to restore usage-tracking session for printer %s", printer_id)
  7262. async def on_print_running_observed(printer_id: int, data: dict):
  7263. """Restart-recovery for a print that started before Bambuddy came up.
  7264. bambu_mqtt.py suppresses ``on_print_start`` on the first RUNNING push
  7265. after Bambuddy startup (#1304 guard, prevents duplicate archive
  7266. creation). This hook restores the persisted archive into ``_active_prints``
  7267. and captures the timelapse baseline that normally hangs off print start.
  7268. Fires once per session, in lieu of on_print_start when restart-recovery
  7269. kicks in. The printer doesn't upload the timelapse until after PRINT
  7270. COMPLETE, so a baseline captured any time during the print is still
  7271. pre-upload.
  7272. """
  7273. logger = logging.getLogger(__name__)
  7274. async with async_session() as db:
  7275. from backend.app.models.printer import Printer
  7276. state = printer_manager.get_status(printer_id)
  7277. if state is not None:
  7278. authorization = await _is_bambuddy_authorized_print(printer_id, state, db)
  7279. if authorization is True:
  7280. logger.info("[RESTART] Restored active Bambuddy print for printer %s", printer_id)
  7281. await _restore_usage_tracking_session(printer_id, state, db, logger)
  7282. await _restore_printable_objects(printer_id, state, db, logger)
  7283. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  7284. printer = result.scalar_one_or_none()
  7285. if not printer:
  7286. logger.warning(
  7287. "[TIMELAPSE] on_print_running_observed: printer %s not found in DB, skipping baseline",
  7288. printer_id,
  7289. )
  7290. return
  7291. # Avoid double-capture: ownership reconciliation above must still run when
  7292. # a baseline already exists, but the camera work itself is one-shot.
  7293. if printer_id in _timelapse_baselines:
  7294. logger.debug(
  7295. "[TIMELAPSE] on_print_running_observed: baseline already present for printer %s, skipping",
  7296. printer_id,
  7297. )
  7298. return
  7299. await _capture_timelapse_baseline_at_start(printer, printer_id, logger)
  7300. class _RunOfArchive:
  7301. """An archive as one of its runs printed it: the run's job id, start and
  7302. plate, the rest the archive's. What the reconcile judges and closes."""
  7303. def __init__(self, archive, run):
  7304. self._archive = archive
  7305. self.subtask_id = run.subtask_id
  7306. self.started_at = run.started_at
  7307. self.plate_id = run.plate_id if run.plate_id is not None else archive.plate_id
  7308. def __getattr__(self, name):
  7309. return getattr(self._archive, name)
  7310. def _is_active_archive_stale(archive, state) -> tuple[bool, str]:
  7311. """Return ``(is_stale, reason)`` for an archive in ``status="printing"``
  7312. against the printer's current MQTT state.
  7313. Reconciliation triggers (#1542 follow-up — recovers from missed PRINT
  7314. COMPLETE events, typically a print finishing during an MQTT disconnect
  7315. window followed by a smart-plug power cycle):
  7316. 1. Printer state is terminal (IDLE / FINISH / FAILED). The print is
  7317. provably not running anymore — only branch that should fire under
  7318. normal disconnect-then-reconnect timing.
  7319. 2. Printer has a different ``subtask_id`` than the archive. Bambu
  7320. firmware mints a fresh ``subtask_id`` for each print, including the
  7321. ghost replay it runs after a power cycle from a leftover SD file —
  7322. so a mismatch unambiguously means the in-DB archive is no longer
  7323. the print on the printer.
  7324. 3. Printer is running but ``subtask_name`` is empty. The printer
  7325. doesn't know what it's running; the archive's reference to it is
  7326. already broken.
  7327. Conservative on purpose: PAUSE / PREPARE / SLICING and any RUNNING state
  7328. with matching subtask_id+subtask_name is left alone. The cost of a false
  7329. positive is a print closed while it still runs: the reconcile takes its
  7330. ``_active_prints`` entry and the archive is no longer ``printing``, so
  7331. the real PRINT COMPLETE later finds no archive to close (#1679). The
  7332. cost of a false negative is the ghost-print loop in #1542. What the
  7333. archive is closed as is ``_reconciled_outcome``'s call.
  7334. Pre-push guard (#1679): when ``state.state`` is empty or ``"unknown"``,
  7335. MQTT has connected but the first ``push_status`` response hasn't been
  7336. applied yet — ``PrinterState`` is sitting on its construction defaults.
  7337. The reconcile caller in ``on_printer_status_change`` is already gated
  7338. on a real ``state.state``, so in normal operation this branch is
  7339. unreachable; it's kept as belt-and-braces for future callers and for
  7340. the narrow window where a partial state update could arrive
  7341. (``state.state`` set but ``subtask_name`` not yet populated). Returning
  7342. ``not stale`` on degenerate input is strictly conservative: a real
  7343. stale archive will still be caught by the next push_status arriving
  7344. with terminal state.
  7345. """
  7346. current_state = (state.state or "").upper()
  7347. if current_state in ("", "UNKNOWN"):
  7348. # No real push_status yet — PrinterState defaults are not evidence.
  7349. return False, ""
  7350. if current_state in ("IDLE", "FINISH", "FAILED"):
  7351. return True, f"printer state {current_state}"
  7352. # Below here the printer is in a running / pre-running state (RUNNING /
  7353. # PAUSE / PREPARE / SLICING / etc.) — decide based on subtask identity.
  7354. current_subtask_id = (state.subtask_id or "").strip()
  7355. if archive.subtask_id and current_subtask_id and archive.subtask_id != current_subtask_id:
  7356. return True, f"subtask_id changed ({archive.subtask_id!r} → {current_subtask_id!r})"
  7357. current_subtask_name = (state.subtask_name or "").strip()
  7358. if not current_subtask_name:
  7359. return True, "printer subtask_name empty"
  7360. return False, ""
  7361. async def prime_kprofile_table(printer_id: int) -> int:
  7362. """Read the printer's calibration table once per connection.
  7363. The AMS slot card shows a K value per slot (#2854). On the printers whose
  7364. trays carry no ``k`` field of their own -- the whole H2 series, whose trays
  7365. report ``cali_idx`` and nothing else -- that number can only come from
  7366. ``state.kprofiles``, and nothing used to fill it on connect. It arrived by
  7367. luck: someone opening the Profiles page or Configure Slot, a nightly GitHub
  7368. backup, or the printer answering a query BambuStudio made on the report
  7369. topic we share. A Bambuddy that nobody visited showed a card with no K
  7370. values at all.
  7371. Only the diameters actually fitted are asked for, which is one request on a
  7372. single-nozzle printer and two on a dual. Probing the four sizes blind is
  7373. what the backup does, and it is both wasteful and the thing that used to
  7374. blank the table.
  7375. Returns the number of nozzles whose table was read.
  7376. """
  7377. client = printer_manager.get_client(printer_id)
  7378. state = printer_manager.get_status(printer_id)
  7379. if client is None or state is None or not state.connected:
  7380. return 0
  7381. # Deduplicated, order preserved: a dual-nozzle printer with two 0.4s should
  7382. # ask once, and both entries are empty until the first push_status lands.
  7383. diameters = list(dict.fromkeys(n.nozzle_diameter for n in (state.nozzles or []) if n.nozzle_diameter))
  7384. if not diameters:
  7385. logging.getLogger(__name__).debug(
  7386. "[Printer %s] No nozzle diameter reported yet; leaving the K-profile table to the next reader",
  7387. printer_id,
  7388. )
  7389. return 0
  7390. primed = 0
  7391. for diameter in diameters:
  7392. try:
  7393. profiles = await client.get_kprofiles(nozzle_diameter=diameter, max_retries=2)
  7394. except Exception as exc: # noqa: BLE001
  7395. # A printer that won't answer costs the card its K values, nothing
  7396. # more — never the connection this runs on the back of.
  7397. logging.getLogger(__name__).warning(
  7398. "[Printer %s] Could not read the K-profile table for nozzle %s: %s", printer_id, diameter, exc
  7399. )
  7400. continue
  7401. primed += 1
  7402. logging.getLogger(__name__).info(
  7403. "[Printer %s] Primed K-profile table for nozzle %s: %d profiles", printer_id, diameter, len(profiles)
  7404. )
  7405. return primed
  7406. def _sd_files_in_use(state) -> set[str]:
  7407. """Names of the SD-card files the printer may be printing right now (#3009).
  7408. The post-print cleanup works out what to delete from the finished archive,
  7409. not from the printer. When reconciliation closes an old archive because the
  7410. printer has moved on to a new job, and that job is the same file reprinted
  7411. from the printer's screen, the cleanup would delete the file mid-print.
  7412. Empty unless the printer is busy: on an ordinary completion the state is
  7413. FINISH or FAILED and nothing is held back. ``gcode_file`` arrives as a bare
  7414. name, a path or a URL depending on firmware, and some report only the
  7415. plate's G-code, so the job name is matched as well, in the same spellings
  7416. the cleanup tries. Names are lower-cased; the card's FAT filesystem
  7417. ignores case.
  7418. """
  7419. def _text(value) -> str:
  7420. return value.strip() if isinstance(value, str) else ""
  7421. if state is None:
  7422. return set()
  7423. current_state = _text(getattr(state, "state", None)).upper()
  7424. if current_state in ("", "UNKNOWN", "IDLE", "FINISH", "FAILED"):
  7425. return set()
  7426. in_use: set[str] = set()
  7427. gcode_file = _text(getattr(state, "gcode_file", None))
  7428. if gcode_file:
  7429. # Drop a scheme by hand: urlparse would cut a name at "#" or "?",
  7430. # both legal in a file name, and raises on some inputs.
  7431. name = PurePosixPath(gcode_file.split("://", 1)[-1]).name
  7432. if name:
  7433. in_use.add(name.lower())
  7434. subtask_name = _text(getattr(state, "subtask_name", None))
  7435. if subtask_name:
  7436. for name in (subtask_name, subtask_name.replace(" ", "_")):
  7437. for ext in (".3mf", ".gcode"):
  7438. in_use.add(f"{name}{ext}".lower())
  7439. return in_use
  7440. def _reconciled_outcome(archive, state) -> str:
  7441. """What a stale archive is closed as at reconnect (#3310).
  7442. A print that ends while Bambuddy restarts is never seen to end: the new
  7443. MQTT client's first status is already FINISH, which is not a transition.
  7444. The printer keeps reporting the finished job's ``subtask_id`` until the
  7445. next print, so a matching id proves the FINISH or FAILED is this
  7446. archive's. Anything else -- IDLE, another job, an id of "0" (a print
  7447. started outside the cloud) or none -- says only that the print is over,
  7448. not how, and stays ``aborted``.
  7449. """
  7450. current_state = (state.state or "").upper()
  7451. if current_state not in ("FINISH", "FAILED"):
  7452. return "aborted"
  7453. archive_id = (archive.subtask_id or "").strip()
  7454. current_id = str(state.subtask_id or "").strip()
  7455. if archive_id in ("", "0") or archive_id != current_id:
  7456. return "aborted"
  7457. return "completed" if current_state == "FINISH" else "failed"
  7458. def _reconciled_end_time(archive, now: datetime, plate_id: int | None = None) -> datetime | None:
  7459. """When a print reconciled as completed most likely ended, or None (#3310).
  7460. The printer does not say when it finished, and the reconnect can be hours
  7461. later. Its start plus the slicer's estimate for the plate it printed,
  7462. never later than now, is the best guess; None when either is missing,
  7463. and the run then keeps no duration, as an aborted one does.
  7464. ``plate_id`` is the plate this run printed, from its queue job: a reprint
  7465. of another plate reuses the archive without changing ``archive.plate_id``.
  7466. """
  7467. started = archive.started_at
  7468. if not started:
  7469. return None
  7470. if started.tzinfo is None:
  7471. started = started.replace(tzinfo=timezone.utc)
  7472. plate = plate_id if plate_id is not None else archive.plate_id
  7473. estimate = None
  7474. if archive.file_path:
  7475. from backend.app.utils.threemf_tools import extract_print_time_from_3mf
  7476. full_path = (
  7477. app_settings.base_dir / archive.file_path
  7478. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  7479. try:
  7480. estimate = extract_print_time_from_3mf(full_path, plate)
  7481. except Exception as exc: # noqa: BLE001
  7482. logging.getLogger(__name__).debug(
  7483. "Plate estimate for reconciled archive %s unreadable: %s", archive.id, exc
  7484. )
  7485. if plate is None and estimate != archive.print_time_seconds:
  7486. # Without a plate the first plate's estimate is read. The archive
  7487. # carries the sum over all plates (#1593), so a difference means
  7488. # several plates and no telling which one printed.
  7489. estimate = None
  7490. elif plate is None:
  7491. # Nothing to read the plate from; the archive's own estimate, as for
  7492. # a single-plate file.
  7493. estimate = archive.print_time_seconds
  7494. if not estimate or estimate <= 0:
  7495. return None
  7496. return min(started + timedelta(seconds=int(estimate)), now)
  7497. async def reconcile_stale_active_prints(printer_id: int) -> int:
  7498. """Synthesise ``on_print_complete`` for archives whose print can't be
  7499. running on the printer anymore.
  7500. Called once per MQTT (re)connection (from on_printer_status_change when
  7501. the connected edge flips False → True) and at Bambuddy startup (from
  7502. the FastAPI lifespan). Without this, a print that completes during a
  7503. disconnect window — followed by a smart-plug-driven power cycle — leaves
  7504. the ``.3mf`` on the SD card, the firmware auto-replays it on next boot,
  7505. and Bambuddy fires a fresh PRINT START for the ghost rather than the
  7506. SD cleanup that PRINT COMPLETE was supposed to run. Repeats every
  7507. power cycle until the operator notices (#1542 follow-up). Reconciliation
  7508. closes the loop by faking the missed PRINT COMPLETE — the existing
  7509. cleanup chain handles SD-file deletion, status updates, usage tracking,
  7510. and notifications.
  7511. The status is ``_reconciled_outcome``'s: ``completed`` or ``failed``
  7512. when the printer still reports the archive's own job as finished or
  7513. failed, ``aborted`` otherwise. Nothing corrects it later: the archive is
  7514. no longer ``printing``, so a later PRINT COMPLETE finds no archive, and
  7515. after a restart the printer's FINISH is not one (#3310).
  7516. Returns the number of archives reconciled.
  7517. """
  7518. state = printer_manager.get_status(printer_id)
  7519. if not state:
  7520. return 0
  7521. # Don't reconcile while disconnected — we'd be making a decision against
  7522. # stale cached state. The connected → reconcile edge handles this.
  7523. if not state.connected:
  7524. return 0
  7525. from backend.app.models.archive import PrintArchive
  7526. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, RUN_OPEN, PrintLogEntry
  7527. reconciled = 0
  7528. async with async_session() as db:
  7529. # The prints on this printer Bambuddy saw start: their runs open here.
  7530. # The archive row is shared by every plate of a file, its printer, job
  7531. # and start the latest start's, so each is judged as its run.
  7532. open_runs = (
  7533. await db.scalars(
  7534. select(PrintLogEntry)
  7535. .where(PrintLogEntry.printer_id == printer_id, PrintLogEntry.status == RUN_OPEN)
  7536. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  7537. )
  7538. ).all()
  7539. by_run = {
  7540. archive.id: archive
  7541. for archive in (
  7542. await db.scalars(
  7543. select(PrintArchive).where(
  7544. PrintArchive.id.in_([run.archive_id for run in open_runs if run.archive_id is not None])
  7545. )
  7546. )
  7547. ).all()
  7548. }
  7549. active: list = [_RunOfArchive(by_run[run.archive_id], run) for run in open_runs if run.archive_id in by_run]
  7550. # And as before runs were opened at the start: a printing archive of
  7551. # this printer with no run open anywhere (printing since before).
  7552. archives_with_open_runs = select(PrintLogEntry.archive_id).where(
  7553. PrintLogEntry.status == RUN_OPEN, PrintLogEntry.archive_id.is_not(None)
  7554. )
  7555. result = await db.execute(
  7556. select(PrintArchive)
  7557. .where(
  7558. PrintArchive.printer_id == printer_id,
  7559. PrintArchive.status == "printing",
  7560. PrintArchive.id.not_in(archives_with_open_runs),
  7561. )
  7562. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  7563. )
  7564. active.extend(result.scalars().all())
  7565. if not active:
  7566. return 0
  7567. # The plate each run printed, from its queue job (#3310).
  7568. from backend.app.models.print_queue import PrintQueueItem
  7569. async with async_session() as db:
  7570. plate_rows = await db.execute(
  7571. select(PrintQueueItem.archive_id, PrintQueueItem.plate_id).where(
  7572. PrintQueueItem.printer_id == printer_id,
  7573. PrintQueueItem.status == "printing",
  7574. PrintQueueItem.archive_id.in_([a.id for a in active]),
  7575. )
  7576. )
  7577. run_plates = {archive_id: plate for archive_id, plate in plate_rows.all() if plate is not None}
  7578. logger = logging.getLogger(__name__)
  7579. for archive in active:
  7580. is_stale, reason = _is_active_archive_stale(archive, state)
  7581. if not is_stale:
  7582. continue
  7583. logger.info(
  7584. "[RECONCILE] Printer %s: synthesising missed PRINT COMPLETE for archive %s (%s) — %s",
  7585. printer_id,
  7586. archive.id,
  7587. archive.filename,
  7588. reason,
  7589. )
  7590. # Synthesised payload: minimal fields the on_print_complete chain
  7591. # needs. `_reconciled` marker lets downstream code distinguish this
  7592. # from a real MQTT-driven completion if it ever needs to (e.g. for
  7593. # metrics / debug logging). raw_data is the live printer state so
  7594. # the usage tracker can compare end-of-print remain% against the
  7595. # captured start values.
  7596. outcome = _reconciled_outcome(archive, state)
  7597. payload = {
  7598. "status": outcome,
  7599. "filename": archive.filename,
  7600. "subtask_name": archive.print_name or "",
  7601. "subtask_id": archive.subtask_id or "",
  7602. "raw_data": state.raw_data or {},
  7603. "_reconciled": True,
  7604. }
  7605. if isinstance(archive, _RunOfArchive):
  7606. # The archive it closes, which its names and printer may not find:
  7607. # another printer's plate of it can have started since.
  7608. payload["_reconciled_archive_id"] = archive.id
  7609. if outcome != "aborted":
  7610. # The printer's progress and layer are this job's, so a partial
  7611. # run is charged what it printed rather than nothing.
  7612. from backend.app.utils.printer_models import usage_progress
  7613. run_progress = usage_progress(
  7614. printer_manager.get_model(printer_id),
  7615. state.progress,
  7616. getattr(state, "layer_num", None),
  7617. getattr(state, "total_layers", None),
  7618. )
  7619. if run_progress:
  7620. payload["last_progress"] = run_progress
  7621. if state.layer_num:
  7622. payload["last_layer_num"] = state.layer_num
  7623. if outcome == "failed":
  7624. payload["hms_errors"] = [e.completion_payload() for e in state.hms_errors or []]
  7625. if outcome == "completed":
  7626. ended = _reconciled_end_time(archive, datetime.now(timezone.utc), run_plates.get(archive.id))
  7627. if ended:
  7628. payload["_reconciled_completed_at"] = ended
  7629. try:
  7630. await on_print_complete(printer_id, payload)
  7631. reconciled += 1
  7632. except Exception as e:
  7633. # Catch-all: a reconciliation failure must not block the
  7634. # printer's normal status flow. The archive stays in
  7635. # ``status="printing"`` and the next reconnect retries.
  7636. logger.warning(
  7637. "[RECONCILE] on_print_complete synthesis failed for archive %s: %s",
  7638. archive.id,
  7639. e,
  7640. )
  7641. return reconciled
  7642. # #2547: clearance left between the nozzle and the top of the print when the
  7643. # plate is commanded back into camera framing. The nozzle is parked away from
  7644. # the part by then, so this is belt-and-braces against a max_z_height that
  7645. # under-reports (e.g. a slicer that excludes a final Z hop).
  7646. _PLATE_RESTORE_CLEARANCE_MM = 10.0
  7647. # Feedrate for both moves. F600 is exactly what Bambu's own end G-code uses on
  7648. # this axis, so it is a proven-safe speed for the full travel.
  7649. _PLATE_RESTORE_FEEDRATE = 600
  7650. # Time allowed for the plate to reach the restored position before the camera
  7651. # grab. Sized for the ~100 mm the stock end G-code drops on X1/P1 at F600
  7652. # (10 mm/s); most H2 and P2 end G-code drops less, or not at all.
  7653. _PLATE_RESTORE_SETTLE_SECONDS = 12.0
  7654. # How long `_background_finish_photo` waits for this producer. Must cover the
  7655. # settle window plus a worst-case RTSP grab (15s), and stay below the
  7656. # notification path's own photo wait so a slow producer degrades to a
  7657. # photo-less notification rather than a missed one. The chamber light's
  7658. # snapshot delay (#1655) spends from the spare 8 seconds, which is why
  7659. # camera_light.MAX_DELAY_SECONDS stays below it.
  7660. _FINISH_PHOTO_PRODUCER_WAIT_SECONDS = _PLATE_RESTORE_SETTLE_SECONDS + 23.0
  7661. def _completion_print_keys(printer_id: int, filename: str, subtask_name: str) -> list[tuple[int, str]]:
  7662. """The ``_active_prints`` keys a finished print may be registered under.
  7663. Matching how they were registered in on_print_start. Shared by
  7664. ``on_print_complete`` and the finish-photo plate restore (#3240), which has
  7665. to find the same binding.
  7666. """
  7667. possible_keys = []
  7668. # Try subtask_name variations first (most reliable for matching)
  7669. if subtask_name:
  7670. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  7671. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  7672. possible_keys.append((printer_id, subtask_name))
  7673. # Try filename variations
  7674. if filename:
  7675. # Extract just the filename if it's a path
  7676. fname = filename.split("/")[-1] if "/" in filename else filename
  7677. if fname.endswith(".3mf"):
  7678. possible_keys.append((printer_id, fname))
  7679. elif fname.endswith(".gcode"):
  7680. base_name = fname.rsplit(".", 1)[0]
  7681. possible_keys.append((printer_id, f"{base_name}.gcode.3mf"))
  7682. possible_keys.append((printer_id, f"{base_name}.3mf"))
  7683. possible_keys.append((printer_id, fname))
  7684. else:
  7685. possible_keys.append((printer_id, f"{fname}.gcode.3mf"))
  7686. possible_keys.append((printer_id, f"{fname}.3mf"))
  7687. possible_keys.append((printer_id, fname))
  7688. # Also try full path versions
  7689. if filename.endswith(".3mf"):
  7690. possible_keys.append((printer_id, filename))
  7691. elif filename.endswith(".gcode"):
  7692. base_name = filename.rsplit(".", 1)[0]
  7693. possible_keys.append((printer_id, f"{base_name}.3mf"))
  7694. possible_keys.append((printer_id, filename))
  7695. else:
  7696. possible_keys.append((printer_id, f"{filename}.3mf"))
  7697. possible_keys.append((printer_id, filename))
  7698. return possible_keys
  7699. def _archive_name_forms(archive) -> set[str]:
  7700. """The names an archive could be echoed back as, in normalised form (#3240).
  7701. Its print name, and its filename with and without the 3MF extension.
  7702. """
  7703. forms = set()
  7704. if archive.print_name:
  7705. forms.add(_normalise_subtask_name(archive.print_name))
  7706. if archive.filename:
  7707. forms.add(_normalise_subtask_name(archive.filename))
  7708. for suffix in (".gcode.3mf", ".3mf"):
  7709. if archive.filename.lower().endswith(suffix):
  7710. forms.add(_normalise_subtask_name(archive.filename[: -len(suffix)]))
  7711. forms.discard("")
  7712. return forms
  7713. def _bound_print(printer_id: int, data: dict) -> tuple[int | None, int | None]:
  7714. """The archive and plate Bambuddy bound to the print ending on ``printer_id`` (#3240).
  7715. Read from ``_active_prints`` with the keys ``on_print_complete`` tries, and
  7716. from the plate recorded when the job was dispatched. Neither is touched:
  7717. ``on_print_complete`` pops both, which is why the finish-photo producer
  7718. calls this before its first await, while they are still there.
  7719. ``(None, None)`` for a print Bambuddy did not bind -- one it failed to
  7720. archive, or that it only knows by a name -- and on any error: the producer
  7721. calls this before the ``try`` that always releases the photo consumer, so
  7722. it must not raise.
  7723. """
  7724. try:
  7725. keys = _completion_print_keys(printer_id, str(data.get("filename") or ""), str(data.get("subtask_name") or ""))
  7726. archive_id = next((_active_prints[key] for key in keys if key in _active_prints), None)
  7727. if archive_id is None:
  7728. return None, None
  7729. return archive_id, _print_plate_ids.get((printer_id, archive_id))
  7730. except Exception:
  7731. logging.getLogger(__name__).debug("Could not read the print binding for printer %s", printer_id, exc_info=True)
  7732. return None, None
  7733. class PlateHeights(NamedTuple):
  7734. """Where the finished print is, and where its end G-code left the plate (#2547)."""
  7735. print_top: float
  7736. """The plate's ``max_z_height``: the Z the toolhead sat at for the last layer."""
  7737. end_z: float
  7738. """The last Z the end G-code moved to: where the plate is now, and where it is parked back to."""
  7739. async def _plate_heights_for_current_print(
  7740. printer_id: int,
  7741. data: dict,
  7742. logger,
  7743. archive_id: int | None,
  7744. plate_id: int | None = None,
  7745. ) -> PlateHeights | None:
  7746. """Heights of the print that just finished on ``printer_id``, or None (#2547).
  7747. These numbers become the targets of real Z moves, so every step here refuses
  7748. rather than guesses. Both have to be known: without the end Z there is no
  7749. position to park the plate back to that the printer's own G-code chose, and
  7750. the firmware won't stop a made-up one at the travel limit (#2579). A height belonging to some *other* print is the one
  7751. failure that could drive the nozzle into the model: 20 mm carried onto a
  7752. 200 mm print would command the plate up through the part.
  7753. Three independent things therefore have to agree before a height is
  7754. returned:
  7755. 1. **Binding.** ``archive_id`` is the archive Bambuddy bound to this print
  7756. when it started (see ``_bound_print``), never one found by name. A name
  7757. is not an identity: every plate of a multi-plate file shares one, and a
  7758. reprint reuses its archive row, so the newest archive with the right
  7759. name can be another plate of the same file. Support bundles show the
  7760. old name lookup doing exactly that (archive 288 for a print bound to
  7761. 169, 313 layers against 125), stopped only by the layer check below.
  7762. No binding, no move.
  7763. 2. **Identity.** The bound archive's name still has to be the finished
  7764. print's ``subtask_name``, compared in ``_normalise_subtask_name`` form
  7765. because the printer echoes spaces as underscores (#3240). Exact: not a
  7766. substring, and not the completion check's tolerance for a truncated
  7767. echo.
  7768. 3. **Corroboration.** The archive's layer count (parsed from the 3MF) has to
  7769. match the layer count the printer itself reported over MQTT for the print
  7770. that just ended. These come from genuinely different sources, so a
  7771. mismatch means the row is not this print, whatever its name says.
  7772. ``plate_id`` is the plate the job was dispatched with; a reprint of one
  7773. plate of a multi-plate file reuses the archive row without updating its
  7774. ``plate_id``. ``completed`` is accepted alongside ``printing`` only because
  7775. ``on_print_complete`` may already have flipped the status by the time we
  7776. run.
  7777. """
  7778. subtask_name = (data.get("subtask_name") or "").strip()
  7779. if not subtask_name:
  7780. # Nothing to identify the print by — refuse rather than fall back to
  7781. # "whatever ran last on this printer".
  7782. logger.info("[PLATE-RESTORE] printer %s: print has no name to match on — skipping", printer_id)
  7783. return None
  7784. if archive_id is None:
  7785. logger.info(
  7786. "[PLATE-RESTORE] printer %s: %r is not bound to an archive — skipping",
  7787. printer_id,
  7788. subtask_name,
  7789. )
  7790. return None
  7791. try:
  7792. from backend.app.models.archive import PrintArchive
  7793. from backend.app.utils.threemf_tools import extract_end_z_from_3mf, extract_max_z_height_from_3mf
  7794. async with async_session() as db:
  7795. result = await db.execute(
  7796. select(PrintArchive).where(
  7797. PrintArchive.id == archive_id,
  7798. PrintArchive.printer_id == printer_id,
  7799. PrintArchive.status.in_(("printing", "completed")),
  7800. PrintArchive.deleted_at.is_(None),
  7801. )
  7802. )
  7803. archive = result.scalar_one_or_none()
  7804. if archive is None or not archive.file_path:
  7805. logger.info("[PLATE-RESTORE] printer %s: archive %s has no usable file — skipping", printer_id, archive_id)
  7806. return None
  7807. if _normalise_subtask_name(subtask_name) not in _archive_name_forms(archive):
  7808. logger.warning(
  7809. "[PLATE-RESTORE] printer %s: archive %s is not named %r — refusing to move the plate",
  7810. printer_id,
  7811. archive_id,
  7812. subtask_name,
  7813. )
  7814. return None
  7815. client = printer_manager.get_client(printer_id)
  7816. reported_layers = getattr(getattr(client, "state", None), "total_layers", None)
  7817. if reported_layers and archive.total_layers and reported_layers != archive.total_layers:
  7818. logger.warning(
  7819. "[PLATE-RESTORE] printer %s: archive %s says %s layers but the printer reported %s "
  7820. "— refusing to move the plate on a height that may not be this print's",
  7821. printer_id,
  7822. archive.id,
  7823. archive.total_layers,
  7824. reported_layers,
  7825. )
  7826. return None
  7827. # Stored relative to the data directory, like every other archive path
  7828. # (#3240: this read a `data_dir` setting that does not exist).
  7829. path = app_settings.base_dir / archive.file_path
  7830. plate = plate_id or archive.plate_id or 1
  7831. print_top = await asyncio.to_thread(extract_max_z_height_from_3mf, path, plate)
  7832. if print_top is None:
  7833. return None
  7834. end_z = await asyncio.to_thread(extract_end_z_from_3mf, path, plate)
  7835. if end_z is None:
  7836. logger.info(
  7837. "[PLATE-RESTORE] printer %s: archive %s has no end G-code position to park at — skipping",
  7838. printer_id,
  7839. archive_id,
  7840. )
  7841. return None
  7842. return PlateHeights(print_top, end_z)
  7843. except Exception as e:
  7844. logger.debug("[PLATE-RESTORE] printer %s: no usable print height: %s", printer_id, e)
  7845. return None
  7846. async def _restore_plate_for_finish_photo(printer_id: int, max_z_height: float, logger) -> bool:
  7847. """Raise the plate back into camera framing before the finish photo (#2547).
  7848. On X1/P1/A1 Bambu's end G-code drops the plate ~100 mm as the last thing it
  7849. does, so by the time ``gcode_state`` reaches FINISH the finished print sits
  7850. far below the camera's natural framing — the complaint behind #1145, #1397
  7851. and #1565. This commands an absolute ``G1 Z`` back to just above the last
  7852. printed layer. Whether that is needed at all is ``_raise_plate_for_finish_photo``'s
  7853. call: plenty of end G-code (H2C, tall prints anywhere) leaves the plate in
  7854. view already.
  7855. Absolute, not relative, is the whole safety argument. ``max_z_height +
  7856. clearance`` is a height the toolhead was physically at seconds earlier, so
  7857. it is inside the travel limits by construction and leaves the nozzle above
  7858. the part. It is also unambiguous across model families: Z is the
  7859. nozzle-to-bed gap whether the bed moves (X1/P1/H2) or the toolhead does
  7860. (A1), so unlike the relative bed-jog path (#1334) there is no sign to get
  7861. wrong. ``M211`` is never touched — see the bed-jog docstring for why
  7862. (#2579).
  7863. Returns True if the move was sent and waited out, False if it was skipped.
  7864. """
  7865. client = printer_manager.get_client(printer_id)
  7866. if client is None:
  7867. return False
  7868. # Re-read state immediately before commanding motion. If the queue has
  7869. # already started the next print, the printer is no longer ours to move.
  7870. state = getattr(client, "state", None)
  7871. if state is None or state.state != "FINISH":
  7872. logger.info(
  7873. "[PLATE-RESTORE] printer %s is in state %s, not FINISH — skipping",
  7874. printer_id,
  7875. getattr(state, "state", "unknown"),
  7876. )
  7877. return False
  7878. target_z = max_z_height + _PLATE_RESTORE_CLEARANCE_MM
  7879. if not client.send_gcode(f"G90\nG1 Z{target_z:.2f} F{_PLATE_RESTORE_FEEDRATE}"):
  7880. logger.warning("[PLATE-RESTORE] printer %s: send failed — capturing where it is", printer_id)
  7881. return False
  7882. logger.info(
  7883. "[PLATE-RESTORE] printer %s: plate to Z%.2f (print top %.2f + %.1f clearance), settling %.0fs",
  7884. printer_id,
  7885. target_z,
  7886. max_z_height,
  7887. _PLATE_RESTORE_CLEARANCE_MM,
  7888. _PLATE_RESTORE_SETTLE_SECONDS,
  7889. )
  7890. await asyncio.sleep(_PLATE_RESTORE_SETTLE_SECONDS)
  7891. return True
  7892. def _park_plate_after_finish_photo(printer_id: int, end_z: float, logger) -> None:
  7893. """Put the plate back where the end G-code left it, after the finish photo (#2547).
  7894. ``end_z`` is the last Z the print's own end G-code moved to, never a height
  7895. worked out here. That is the safety argument: Bambu's templates keep their
  7896. end moves inside each model's travel, and the firmware does not clamp
  7897. G-code sent over MQTT (#2579). Parking at print top + 100 mm instead drove
  7898. the plate of a 238 mm H2C print to the bottom of its travel, a move Bambu's
  7899. own template had declined to make.
  7900. Fire-and-forget: if it doesn't land, the plate is merely high, and the next
  7901. print homes anyway.
  7902. """
  7903. client = printer_manager.get_client(printer_id)
  7904. state = getattr(client, "state", None) if client else None
  7905. if client is None or state is None or state.state != "FINISH":
  7906. return
  7907. client.send_gcode(f"G90\nG1 Z{end_z:.2f} F{_PLATE_RESTORE_FEEDRATE}")
  7908. logger.info("[PLATE-RESTORE] printer %s: plate back to the end G-code's Z%.2f", printer_id, end_z)
  7909. async def _raise_plate_for_finish_photo(
  7910. printer_id: int, data: dict, logger, archive_id: int | None, plate_id: int | None
  7911. ) -> float | None:
  7912. """Raise the plate into view for the finish photo if it needs it (#2547).
  7913. Returns the Z to park the plate back to once the photo is taken, or None
  7914. when the plate wasn't moved. It needs moving only when the end G-code
  7915. left it lower than the restore height; when it is already at or above it
  7916. (the H2C end G-code puts it back 2 mm under the nozzle), raising would be
  7917. a move away from the camera.
  7918. """
  7919. heights = await _plate_heights_for_current_print(printer_id, data, logger, archive_id, plate_id)
  7920. if heights is None:
  7921. logger.info("[PLATE-RESTORE] printer %s: print height unknown — capturing without restore", printer_id)
  7922. return None
  7923. if heights.end_z <= heights.print_top + _PLATE_RESTORE_CLEARANCE_MM:
  7924. logger.info(
  7925. "[PLATE-RESTORE] printer %s: end G-code left the plate at Z%.2f (print top %.2f) — already in view",
  7926. printer_id,
  7927. heights.end_z,
  7928. heights.print_top,
  7929. )
  7930. return None
  7931. if await _plate_restore_is_blocked_by_queue(printer_id):
  7932. logger.info("[PLATE-RESTORE] printer %s has queued work — skipping plate restore", printer_id)
  7933. return None
  7934. if await _restore_plate_for_finish_photo(printer_id, heights.print_top, logger):
  7935. return heights.end_z
  7936. return None
  7937. async def _plate_restore_is_blocked_by_queue(printer_id: int) -> bool:
  7938. """True if a queue item is about to take this printer (#2547).
  7939. The scheduler dispatches the next job the moment a print completes, and a
  7940. plate move interleaved with a print start is not a race worth having. The
  7941. state re-check in ``_restore_plate_for_finish_photo`` closes the tail of
  7942. this window; this closes the head of it.
  7943. """
  7944. try:
  7945. from backend.app.models.print_queue import PrintQueueItem
  7946. async with async_session() as db:
  7947. result = await db.execute(
  7948. select(PrintQueueItem.id)
  7949. .where(
  7950. PrintQueueItem.printer_id == printer_id,
  7951. PrintQueueItem.status.in_(("pending", "printing")),
  7952. )
  7953. .limit(1)
  7954. )
  7955. return result.scalar_one_or_none() is not None
  7956. except Exception as e:
  7957. # Fail closed: if we can't tell, don't move the plate.
  7958. logging.getLogger(__name__).debug(
  7959. "[PLATE-RESTORE] queue check failed for printer %s: %s — skipping restore", printer_id, e
  7960. )
  7961. return True
  7962. async def on_finish_photo_moment(printer_id: int, data: dict):
  7963. """Pre-capture a finish photo when the printer enters stage 22 / FINISH (#1721).
  7964. Fires either at the stage-22 ("Filament unloading") edge — toolhead
  7965. parked, bed not yet dropped, optimal framing — or as a FINISH-state
  7966. fallback for prints that skip stage 22 (cancel, external-spool-only,
  7967. HMS halt, firmware variants). Grabs one frame via the same
  7968. external-camera / RTSP path the post-completion fallback uses, stores
  7969. the JPEG bytes in ``_stage22_finish_frames[printer_id]``, and lets
  7970. ``_background_finish_photo`` consume the cached bytes when it runs.
  7971. Replaces the #1397 "force timelapse on at dispatch" mechanism, which
  7972. caused per-layer nozzle parking on slicer profiles with Timelapse Type
  7973. set to Smooth (#1721). No force-on now means the user's explicit
  7974. timelapse=off in the slicer send dialog is respected.
  7975. """
  7976. logger = logging.getLogger(__name__)
  7977. trigger = data.get("trigger", "unknown")
  7978. timelapse_was_active = bool(data.get("timelapse_was_active"))
  7979. logger.info(
  7980. "[FINISH-PHOTO-MOMENT] printer=%s trigger=%s timelapse_active=%s",
  7981. printer_id,
  7982. trigger,
  7983. timelapse_was_active,
  7984. )
  7985. # If a timelapse is actively recording, skip the pre-capture — the
  7986. # post-completion path will extract the last frame from the recorded
  7987. # video, which still provides the best framing (toolhead parked,
  7988. # before bed drop) without the per-layer parking side effects.
  7989. if timelapse_was_active:
  7990. logger.info(
  7991. "[FINISH-PHOTO-MOMENT] timelapse active for printer %s — skipping pre-capture (last-frame extraction will run post-completion)",
  7992. printer_id,
  7993. )
  7994. return
  7995. # #1790: register the producer-done event BEFORE the first await so the
  7996. # consumer in `_background_finish_photo` — which is dispatched back-to-back
  7997. # with us on the FINISH-state fallback path — sees it as soon as it polls.
  7998. # The `finally` below guarantees `set()` runs on every exit, including
  7999. # early returns and exceptions, so the consumer's bounded wait can't hang.
  8000. producer_done = asyncio.Event()
  8001. _stage22_finish_in_flight[printer_id] = producer_done
  8002. # #3240: also before the first await. On the FINISH-state path
  8003. # `on_print_complete` is dispatched right behind us and pops both the
  8004. # print's archive binding and its plate; read here, they are still there.
  8005. bound_archive_id, bound_plate_id = _bound_print(printer_id, data)
  8006. # #2547: the Z to park the plate back to, set once the plate has actually
  8007. # been raised, and read by the `finally` below. Declared out here so a
  8008. # failure anywhere after the move — a camera timeout, a DB error — still
  8009. # lowers the plate again.
  8010. park_z: float | None = None
  8011. try:
  8012. async with async_session() as db:
  8013. from backend.app.api.routes.settings import get_setting
  8014. from backend.app.models.printer import Printer
  8015. capture_setting = await get_setting(db, "capture_finish_photo")
  8016. if capture_setting is not None and capture_setting.lower() != "true":
  8017. logger.info("[FINISH-PHOTO-MOMENT] capture_finish_photo disabled — skipping pre-capture")
  8018. return
  8019. restore_setting = await get_setting(db, "finish_photo_restore_plate")
  8020. restore_plate_enabled = restore_setting is None or restore_setting.lower() == "true"
  8021. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8022. printer = result.scalar_one_or_none()
  8023. if printer is None:
  8024. logger.warning(
  8025. "[FINISH-PHOTO-MOMENT] printer %s not found in DB",
  8026. printer_id,
  8027. )
  8028. return
  8029. frame_bytes: bytes | None = None
  8030. # #2708: the banked frame arrives already rotated — it comes from
  8031. # `_capture_snapshot_for_notification`, which rotates before returning.
  8032. # Every other source below is a raw grab. Tracking which lets us store
  8033. # exactly one rotation in `_stage22_finish_frames` either way.
  8034. frame_already_rotated = False
  8035. # On the FINISH-state path the End G-code has already run, and two very
  8036. # different situations arrive here needing opposite answers.
  8037. #
  8038. # #1867: if Bambuddy injected End G-code into this print, a SwapMod
  8039. # snippet may have ejected the plate — the scene in front of the camera
  8040. # is no longer the finished print, and no amount of moving the plate
  8041. # brings it back. Use the banked in-print frame instead.
  8042. #
  8043. # #2547: otherwise the print is still sitting there — on X1/P1/A1 ~100 mm
  8044. # lower than the camera frames well — and the toolhead is parked out of
  8045. # the way. That is the *best* moment available on firmware that never emits
  8046. # stage 22 (H2C, A1 Mini) — so capture live, after putting the plate
  8047. # back. Preferring the bank here unconditionally, as this code used to,
  8048. # is what shipped a mid-print photo with the toolhead over the part.
  8049. if trigger == "finish_state" and print_dispatch_context.end_gcode_injected(printer_id):
  8050. banked = _inprint_frame_bank.get(printer_id)
  8051. if banked:
  8052. frame_bytes = banked
  8053. frame_already_rotated = True
  8054. logger.info(
  8055. "[FINISH-PHOTO-MOMENT] End G-code was injected — using banked in-print "
  8056. "frame (%d bytes) instead of a post-swap live grab",
  8057. len(banked),
  8058. )
  8059. else:
  8060. logger.warning(
  8061. "[FINISH-PHOTO-MOMENT] End G-code was injected for printer %s but the "
  8062. "in-print bank is empty — falling back to a live grab, which may show a "
  8063. "swapped or empty plate",
  8064. printer_id,
  8065. )
  8066. # `park_z` is set only once the plate is actually up, because the
  8067. # `finally` reads it to decide whether it owes a move back down.
  8068. #
  8069. # Never on a print whose End G-code Bambuddy injected, even when the bank
  8070. # came up empty above: that machine may have just ejected its plate, and
  8071. # driving Z into whatever a swap mechanism is doing is not a risk worth
  8072. # taking for a photo of a bed we already know may be bare.
  8073. if (
  8074. frame_bytes is None
  8075. and trigger == "finish_state"
  8076. and restore_plate_enabled
  8077. and not print_dispatch_context.end_gcode_injected(printer_id)
  8078. ):
  8079. park_z = await _raise_plate_for_finish_photo(printer_id, data, logger, bound_archive_id, bound_plate_id)
  8080. # Chamber light for the live capture below (#1655). The banked frame
  8081. # above was taken without it, like every in-print bank.
  8082. async with camera_light(printer if frame_bytes is None else None):
  8083. if frame_bytes is None and printer.external_camera_enabled and printer.external_camera_url:
  8084. from backend.app.api.routes.camera import live_frame_for_capture
  8085. from backend.app.services.external_camera import capture_frame
  8086. # #2707: this used to collide with the live view and fail, which is
  8087. # how finish-photo notifications went out with no image attached.
  8088. # Leaving frame_bytes None keeps the rest of the fallback chain.
  8089. defer, buffered = live_frame_for_capture(printer_id)
  8090. if defer:
  8091. frame_bytes = buffered
  8092. else:
  8093. frame_bytes = await capture_frame(
  8094. printer.external_camera_url,
  8095. printer.external_camera_type or "mjpeg",
  8096. snapshot_url=printer.external_camera_snapshot_url,
  8097. )
  8098. if frame_bytes:
  8099. logger.info(
  8100. "[FINISH-PHOTO-MOMENT] captured external-camera frame (%d bytes)",
  8101. len(frame_bytes),
  8102. )
  8103. elif frame_bytes is None:
  8104. from backend.app.api.routes.camera import get_buffered_frame
  8105. buffered = get_buffered_frame(printer_id)
  8106. if buffered:
  8107. frame_bytes = buffered
  8108. logger.info(
  8109. "[FINISH-PHOTO-MOMENT] used buffered RTSP frame (%d bytes)",
  8110. len(frame_bytes),
  8111. )
  8112. else:
  8113. from backend.app.services.camera import capture_camera_frame_bytes
  8114. frame_bytes = await capture_camera_frame_bytes(
  8115. ip_address=printer.ip_address,
  8116. access_code=printer.access_code,
  8117. model=printer.model,
  8118. timeout=15,
  8119. )
  8120. if frame_bytes:
  8121. logger.info(
  8122. "[FINISH-PHOTO-MOMENT] captured RTSP frame (%d bytes)",
  8123. len(frame_bytes),
  8124. )
  8125. if frame_bytes:
  8126. if not frame_already_rotated:
  8127. frame_bytes = _apply_camera_rotation(frame_bytes, printer, logger)
  8128. _stage22_finish_frames[printer_id] = frame_bytes
  8129. else:
  8130. logger.warning(
  8131. "[FINISH-PHOTO-MOMENT] no frame captured for printer %s — post-completion fallback will retry",
  8132. printer_id,
  8133. )
  8134. except Exception as e:
  8135. logger.warning(
  8136. "[FINISH-PHOTO-MOMENT] pre-capture failed for printer %s: %s",
  8137. printer_id,
  8138. e,
  8139. )
  8140. finally:
  8141. # #2547: we raised the plate, so we own lowering it — including when the
  8142. # capture above failed or threw partway through.
  8143. if park_z is not None:
  8144. try:
  8145. _park_plate_after_finish_photo(printer_id, park_z, logger)
  8146. except Exception as e:
  8147. logger.warning("[PLATE-RESTORE] printer %s: could not lower plate: %s", printer_id, e)
  8148. # #1790: always unblock the consumer's bounded wait — whether we stored
  8149. # a frame, gave up, or hit an exception. Local ref means cleanup of the
  8150. # dict entry by the consumer doesn't affect signalling.
  8151. producer_done.set()
  8152. def _subtask_name_from_filename(filename: str) -> str:
  8153. """Recover the subtask name a print command would have carried for *filename*.
  8154. The dispatcher derives the printer-facing subtask name from the archive's
  8155. file name, so stripping the extensions back off gives the value MQTT echoes
  8156. on completion. Only the two extensions Bambuddy actually stores are removed,
  8157. and in the order they nest (``.gcode.3mf``), so a model whose own name
  8158. contains a dot -- ``My.Model.3mf`` -- keeps it.
  8159. """
  8160. name = PurePosixPath(filename).name
  8161. for suffix in (".3mf", ".gcode"):
  8162. if name.lower().endswith(suffix):
  8163. name = name[: -len(suffix)]
  8164. return name
  8165. def _subtask_is_the_file(subtask_name: str, filename: str) -> bool:
  8166. """Whether the subtask name is the printed 3MF's own file name.
  8167. A print started from the printer's screen reports the same full name, with
  8168. its extension, as both subtask and file (#3009). A dispatched print reports
  8169. a bare subtask name; one that merely ends in ".3mf" can be a model named
  8170. that, whose file is ``Foo.3mf.gcode.3mf``, so it does not count.
  8171. """
  8172. if not subtask_name or not subtask_name.lower().endswith(".3mf"):
  8173. return False
  8174. return PurePosixPath((filename or "").split("://", 1)[-1]).name == subtask_name
  8175. # How the printer marks a subtask name it had to cut short. Observed on real
  8176. # hardware at ~100 characters, but the cut-off is not a fixed character count
  8177. # (a name with multibyte characters came back at 98), so match the marker
  8178. # rather than a length.
  8179. _SUBTASK_TRUNCATION_MARKER = "..."
  8180. def _normalise_subtask_name(name: str) -> str:
  8181. """Canonical form for comparing a dispatched name against MQTT's echo.
  8182. The printer does not echo the name back verbatim: it substitutes
  8183. underscores for spaces. ``H2D_Carbon_Filter_(V2)_Body & Solid Lid`` is
  8184. dispatched and ``H2D_Carbon_Filter_(V2)_Body_&_Solid_Lid`` comes back.
  8185. The 3MF lookup in this module has always known that -- it builds
  8186. space-to-underscore variants of every candidate filename, and its
  8187. directory search normalises both sides before comparing. This exists so
  8188. the completion check reads the same rule from the same place instead of
  8189. growing its own, which is exactly how it came to disagree (#2829).
  8190. A space at either end is substituted too: ``Part .gcode.3mf`` is
  8191. dispatched as ``Part `` and echoed as ``Part_``. Underscores at the ends
  8192. are dropped after the substitution so both sides agree (#3241).
  8193. """
  8194. return name.strip().replace(" ", "_").strip("_").casefold()
  8195. def _3mf_files_for_print(paths: list[str], search_term: str) -> list[str]:
  8196. """The 3MF files of the printer's folder listings that are this print's.
  8197. A file of the print's own name wins, in whichever folder. Only without one
  8198. is a file that merely contains the name taken, and then only when it is
  8199. the one such file: "Cube_large.3mf" is not "Cube"'s file, and of several
  8200. candidates none can be told apart. Paths keep the listings' order.
  8201. """
  8202. threemf = [path for path in paths if path.lower().endswith(".3mf")]
  8203. exact = [path for path in threemf if _subtask_names_match(_subtask_name_from_filename(path), search_term)]
  8204. if exact:
  8205. return exact
  8206. term = _normalise_subtask_name(search_term)
  8207. partial = [path for path in threemf if term in _normalise_subtask_name(PurePosixPath(path).name)]
  8208. if len(partial) > 1:
  8209. logging.getLogger(__name__).info(
  8210. "Several 3MF files contain %r and none is named for it: %s", search_term, partial
  8211. )
  8212. return []
  8213. return partial
  8214. def _subtask_names_match(expected: str, observed: str) -> bool:
  8215. """Whether two subtask names describe the same print.
  8216. Beyond the space/underscore substitution, the printer truncates long names
  8217. and marks the cut with ``...``. A truncated echo has to count as a match or
  8218. every print with a long name strands its queue item the same way.
  8219. """
  8220. expected_n = _normalise_subtask_name(expected)
  8221. observed_n = _normalise_subtask_name(observed)
  8222. if expected_n == observed_n:
  8223. return True
  8224. # Either side can be the truncated one: the printer truncates what it
  8225. # echoes, and an archive whose own filename was recorded from a previous
  8226. # truncated echo carries the marker too.
  8227. for full, cut in ((expected_n, observed_n), (observed_n, expected_n)):
  8228. if not cut.endswith(_SUBTASK_TRUNCATION_MARKER):
  8229. continue
  8230. # The full side has lost its edge underscores; a cut that lands right
  8231. # after one keeps it, so drop it here too (#3241). Nothing left before
  8232. # the marker is no evidence of anything, and must not match every name.
  8233. kept = cut[: -len(_SUBTASK_TRUNCATION_MARKER)].rstrip("_")
  8234. if kept and full.startswith(kept):
  8235. return True
  8236. return False
  8237. def _dispatched_name_matches_echo(expected: str, observed: str) -> bool:
  8238. """Whether the printer's subtask echo is the name a print was dispatched as.
  8239. A dispatched print is echoed under its bare name at start, but a printer
  8240. that picks a print back up -- Bambuddy reconnecting to a print already
  8241. running, or an A1 mini resuming after power-loss recovery -- can report the
  8242. file name with its extension instead (#3485). The echo is tried both as it
  8243. came and with the extensions stripped; the raw form still has to count, so
  8244. a model named ``Foo.3mf`` (file ``Foo.3mf.gcode.3mf``) keeps matching.
  8245. """
  8246. return _subtask_names_match(expected, observed) or _subtask_names_match(
  8247. expected, _subtask_name_from_filename(observed)
  8248. )
  8249. async def _queue_item_dispatched_name(db, item) -> str:
  8250. """The subtask name *item* was dispatched under, or "" when unknowable.
  8251. A row with no archive, or an archive with no file name, is unverifiable
  8252. rather than wrong; every caller answers that with its permissive branch.
  8253. """
  8254. if item.archive_id is None:
  8255. return ""
  8256. from backend.app.models.archive import PrintArchive
  8257. archive = await db.get(PrintArchive, item.archive_id)
  8258. if archive is None or not archive.filename:
  8259. return ""
  8260. return _subtask_name_from_filename(archive.filename)
  8261. async def _completion_belongs_to_queue_item(db, item, data: dict) -> bool:
  8262. """Whether this completion event is plausibly about *item*'s print.
  8263. The caller finds its queue row by printer and ``status='printing'`` alone,
  8264. which is all a completion event gives it -- there is no run identifier in
  8265. the MQTT payload to match on. That makes the lookup indiscriminate: any
  8266. completion delivered for this printer closes whichever row happens to be
  8267. printing, however unrelated. Comparing the subtask name against the archive
  8268. the row was dispatched with costs one primary-key load and rules that out.
  8269. Deliberately permissive: it answers False only on a positive disagreement
  8270. between two names we actually have. A row with no archive, an archive with
  8271. no file name, or an event with no subtask name is unverifiable rather than
  8272. wrong, and refusing those would strand the item in ``printing`` and wedge
  8273. the printer's queue -- a worse failure than the one being prevented.
  8274. """
  8275. observed = (data.get("subtask_name") or "").strip()
  8276. if not observed or item.archive_id is None:
  8277. return True
  8278. from backend.app.models.archive import PrintArchive
  8279. archive = await db.get(PrintArchive, item.archive_id)
  8280. if archive is None or not archive.filename:
  8281. return True
  8282. expected = _subtask_name_from_filename(archive.filename)
  8283. if not expected or _dispatched_name_matches_echo(expected, observed):
  8284. return True
  8285. logging.getLogger(__name__).warning(
  8286. "Ignoring print completion for queue item %s: it was dispatched as %r "
  8287. "(archive %s, %s) but the completion reports subtask %r. Leaving the item "
  8288. "printing rather than closing a run this event is not about.",
  8289. item.id,
  8290. expected,
  8291. archive.id,
  8292. archive.filename,
  8293. observed,
  8294. )
  8295. return False
  8296. async def _recover_fallback_from_cache_before_eviction(printer_id: int, data: dict) -> None:
  8297. """Spend the 3MF download cache on a still-empty fallback archive.
  8298. ``on_print_complete`` drops the cache as its first act, which deletes the
  8299. file. If the cover endpoint (or anything else) pulled the 3MF while the
  8300. print ran and the archive never got one, this is the last moment those bytes
  8301. exist (#2957).
  8302. """
  8303. logger = logging.getLogger(__name__)
  8304. names = [
  8305. n
  8306. for n in (data.get("filename"), data.get("subtask_name"), (data.get("raw_data") or {}).get("subtask_name"))
  8307. if n
  8308. ]
  8309. for name in names:
  8310. try:
  8311. cached = get_cached_3mf(printer_id, name)
  8312. # The print has ended: its completion prices the archive, so no
  8313. # print-start estimate is scheduled to race it.
  8314. if cached and await try_recover_fallback_archive(printer_id, name, cached, price=False):
  8315. return
  8316. except Exception as e:
  8317. logger.debug("[RECOVER] Pre-eviction recovery for %s failed: %s", name, e)
  8318. # Longest the print's end waits for its file to be deleted from the SD card
  8319. _SD_CLEANUP_BUDGET_SECONDS = 45.0
  8320. async def _cleanup_sd_card_after_print(
  8321. printer_id: int, subtask_name: str | None, archive_id: int | None, logger
  8322. ) -> None:
  8323. """Delete a finished print's file from the printer's SD card."""
  8324. # Cleanup: delete uploaded file from printer SD card to prevent phantom prints (Issue #374, #1542)
  8325. # The print scheduler uploads files to the SD card root (/). Some printers (e.g. P1S, A1)
  8326. # auto-start files found in root on power cycle, causing ghost prints.
  8327. # Must run before the archive_id early-return so it executes even when archiving is disabled.
  8328. try:
  8329. if subtask_name:
  8330. archive_filename: str | None = None
  8331. async with async_session() as db:
  8332. from backend.app.models.archive import PrintArchive
  8333. from backend.app.models.printer import Printer
  8334. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8335. printer = result.scalar_one_or_none()
  8336. if archive_id:
  8337. archive_row = await db.execute(select(PrintArchive.filename).where(PrintArchive.id == archive_id))
  8338. archive_filename = archive_row.scalar_one_or_none()
  8339. if printer:
  8340. from backend.app.services.bambu_ftp import DeleteResult, delete_file_async
  8341. from backend.app.utils.filename import derive_remote_filename
  8342. # Primary candidate: the exact path the dispatcher uploaded to
  8343. # (derived from archive.filename via the same rule as upload).
  8344. # Without it, a library row that ended up with a doubled
  8345. # .gcode.3mf (#1542) leaves the real file behind because the
  8346. # subtask_name + ext fallbacks below don't match what's on the
  8347. # SD card. Fallbacks remain for archive-less prints (subtask
  8348. # never resolved to an archive) and for older naming variants.
  8349. candidate_paths: list[str] = []
  8350. if archive_filename:
  8351. candidate_paths.append(f"/{derive_remote_filename(archive_filename)}")
  8352. for ext in (".3mf", ".gcode"):
  8353. fallback = f"/{subtask_name}{ext}"
  8354. if fallback not in candidate_paths:
  8355. candidate_paths.append(fallback)
  8356. # Three outcomes track across all candidates so the final log
  8357. # line reflects what actually happened. The A1 in #1721 always
  8358. # ends here with ``any_not_found=True`` and the others False
  8359. # — its firmware auto-cleans the SD card before our cleanup
  8360. # runs, every candidate FTP-DELE returns 550, and the old
  8361. # code burned 3 retries × 2 s × 3 candidates per print
  8362. # logging a misleading "may linger" WARNING on a successful
  8363. # print.
  8364. any_deleted = False
  8365. any_real_failure = False
  8366. any_not_found = False
  8367. files_in_use = _sd_files_in_use(printer_manager.get_status(printer_id))
  8368. for remote_path in candidate_paths:
  8369. if PurePosixPath(remote_path).name.lower() in files_in_use:
  8370. logger.info(
  8371. "SD card cleanup: keeping %s on printer %s, which is printing it now",
  8372. remote_path,
  8373. printer.name,
  8374. )
  8375. continue
  8376. # Retry only the FAILED case — 550 NOT_FOUND will never
  8377. # recover by waiting, so a "file isn't here" answer
  8378. # advances immediately to the next candidate without
  8379. # consuming the retry budget.
  8380. for attempt in range(1, 4):
  8381. try:
  8382. delete_result = await delete_file_async(
  8383. printer.ip_address,
  8384. printer.access_code,
  8385. remote_path,
  8386. printer_model=printer.model,
  8387. )
  8388. except Exception as e:
  8389. delete_result = DeleteResult.FAILED
  8390. logger.warning(
  8391. "SD card cleanup attempt %d/3 raised for %s: %s",
  8392. attempt,
  8393. remote_path,
  8394. e,
  8395. )
  8396. if delete_result == DeleteResult.DELETED:
  8397. any_deleted = True
  8398. logger.info("Deleted %s from printer %s SD card", remote_path, printer.name)
  8399. break
  8400. if delete_result == DeleteResult.NOT_FOUND:
  8401. any_not_found = True
  8402. break # 550 will not recover; try next candidate
  8403. # FAILED: real error — retry with backoff, then give up
  8404. if attempt < 3:
  8405. await asyncio.sleep(2)
  8406. else:
  8407. any_real_failure = True
  8408. logger.warning(
  8409. "SD card cleanup failed after 3 attempts for %s "
  8410. "(network/auth/transient error — file may linger on SD card)",
  8411. remote_path,
  8412. )
  8413. if not any_deleted and not any_real_failure and any_not_found:
  8414. # Every candidate said "not here." Either the printer
  8415. # firmware swept the SD card itself (common on A1) or the
  8416. # dispatcher's upload path doesn't match our candidate
  8417. # rule. Either way: nothing to clean up, no warning.
  8418. logger.debug(
  8419. "SD card cleanup: nothing to delete on %s — every candidate returned 550 "
  8420. "(printer likely self-cleaned)",
  8421. printer.name,
  8422. )
  8423. except Exception as e:
  8424. logger.warning("SD card file cleanup failed for printer %s: %s", printer_id, e)
  8425. async def on_print_complete(printer_id: int, data: dict):
  8426. """Handle print completion - update the archive status.
  8427. The printer counts as printing until the print's usage is booked (or this
  8428. ends), so the AMS remain% weight sync can't charge it a second time while
  8429. the usage tracker works (WU-3). Marked before the first await.
  8430. """
  8431. from backend.app.services.usage_tracker import done_completing, mark_completing
  8432. mark_completing(printer_id)
  8433. try:
  8434. await _close_out_print(printer_id, data)
  8435. finally:
  8436. done_completing(printer_id)
  8437. async def _close_out_print(printer_id: int, data: dict):
  8438. import time
  8439. logger = logging.getLogger(__name__)
  8440. start_time = time.time()
  8441. def log_timing(section: str):
  8442. elapsed = time.time() - start_time
  8443. logger.info("[TIMING] %s: %.3fs elapsed", section, elapsed)
  8444. logger.info("[CALLBACK] on_print_complete started for printer %s", printer_id)
  8445. octoeverywhere_detection_service.reset_printer(printer_id)
  8446. # A kill-switch stop sends its provider notification immediately. Keep the
  8447. # task so the later notification path can await it and avoid a duplicate;
  8448. # if that immediate attempt failed, the regular completion path retries.
  8449. kill_switch_notification_task = _kill_switch_notification_tasks.pop(printer_id, None)
  8450. # Last chance before the bytes go: if this print's archive is still an empty
  8451. # fallback and something downloaded the 3MF while it ran, fill the archive in
  8452. # now. The cover endpoint's copy lives in exactly this cache, and clearing it
  8453. # below deletes the file (#2957).
  8454. await _recover_fallback_from_cache_before_eviction(printer_id, data)
  8455. # A pending cool-off retry has nothing left to recover for — the cache is
  8456. # about to be dropped and the print is over.
  8457. retry_task = _fallback_3mf_retry_tasks.pop(printer_id, None)
  8458. if retry_task and not retry_task.done():
  8459. retry_task.cancel()
  8460. # Drop the 3MF download cache for this printer (#972). The print is over,
  8461. # nothing else legitimately needs the bytes; keeping them would only risk
  8462. # handing a stale file to the next print if it reuses the same name.
  8463. clear_3mf_cache(printer_id)
  8464. try:
  8465. ws_data = {
  8466. "status": data.get("status"),
  8467. "filename": data.get("filename"),
  8468. "subtask_name": data.get("subtask_name"),
  8469. "timelapse_was_active": data.get("timelapse_was_active"),
  8470. }
  8471. await ws_manager.send_print_complete(printer_id, ws_data)
  8472. log_timing("WebSocket send_print_complete")
  8473. except Exception as e:
  8474. logger.warning("[CALLBACK] WebSocket send_print_complete failed: %s", e)
  8475. # Capture user info before clearing (needed for print log entry)
  8476. _print_user_info = printer_manager.get_current_print_user(printer_id)
  8477. # Clear current print user tracking (Issue #206)
  8478. printer_manager.clear_current_print_user(printer_id)
  8479. # If the user explicitly stopped this print from the queue UI the printer will
  8480. # report "failed" or "aborted" via MQTT. Override that to "cancelled" so the
  8481. # correct "print stopped" notification/email is sent instead of a failure alert.
  8482. _raw_status = data.get("status", "completed")
  8483. if printer_id in _user_stopped_printers and _raw_status in ("failed", "aborted"):
  8484. logger.info(
  8485. "[CALLBACK] Overriding status '%s' -> 'cancelled' for printer %s (print was stopped from queue by user)",
  8486. _raw_status,
  8487. printer_id,
  8488. )
  8489. data = {**data, "status": "cancelled"}
  8490. elif printer_id in _kill_switch_stopped and _raw_status in ("failed", "aborted"):
  8491. logger.info(
  8492. "[CALLBACK] Overriding status '%s' -> 'cancelled' for printer %s (stopped by the printer kill switch)",
  8493. _raw_status,
  8494. printer_id,
  8495. )
  8496. data = {**data, "status": "cancelled", "_kill_switch": True}
  8497. _user_stopped_printers.discard(printer_id)
  8498. _kill_switch_stopped.discard(printer_id)
  8499. # Raise the plate-clear gate for queued dispatch (#961). Any terminal status
  8500. # may have left material on the bed: a user can cancel ten hours into a
  8501. # twelve-hour print, a printer can self-abort mid-job after a clog, and a
  8502. # touchscreen-stop reports `aborted` rather than `cancelled` because
  8503. # `_user_stopped_printers` is only populated when the user stops via the
  8504. # Bambuddy queue UI. Earlier code raised the flag only for completed/failed,
  8505. # which auto-dispatched the next queued print onto a fouled bed two seconds
  8506. # after a touchscreen-abort (#1171). Persisted to DB so the gate survives
  8507. # Auto Off power cycles and Bambuddy restarts.
  8508. try:
  8509. notify_live_activities.print_finished(printer_id, data)
  8510. except Exception:
  8511. logger.exception("Notify Live Activity completion hook failed for printer %s", printer_id)
  8512. _final_status = data.get("status", "completed")
  8513. if _final_status in ("completed", "failed", "aborted", "cancelled"):
  8514. printer_manager.set_awaiting_plate_clear(printer_id, True)
  8515. # MQTT relay - publish print complete
  8516. try:
  8517. printer_info = printer_manager.get_printer(printer_id)
  8518. if printer_info:
  8519. await mqtt_relay.on_print_complete(
  8520. printer_id,
  8521. printer_info.name,
  8522. printer_info.serial_number,
  8523. data.get("filename", ""),
  8524. data.get("subtask_name", ""),
  8525. data.get("status", "completed"),
  8526. )
  8527. except Exception:
  8528. pass # Don't fail print complete callback if MQTT fails
  8529. filename = data.get("filename", "")
  8530. subtask_name = data.get("subtask_name", "")
  8531. if not filename and not subtask_name:
  8532. # Which print ended can't be told: the printer reported no name since
  8533. # Bambuddy started (a print that failed while preparing, a calibration).
  8534. # Charge nothing, but close the print session, which otherwise kept the
  8535. # AMS weight sync off until the next print (AS-5). A print archived as
  8536. # running is closed and charged by the reconnect reconcile.
  8537. logger.warning("Print complete without filename or subtask_name")
  8538. try:
  8539. from backend.app.services.usage_tracker import discard_session
  8540. async with async_session() as db:
  8541. await discard_session(db, printer_id)
  8542. except Exception as e:
  8543. logger.warning("Failed to clear print session for printer %s: %s", printer_id, e)
  8544. return
  8545. logger.info("Print complete - filename: %s, subtask: %s, status: %s", filename, subtask_name, data.get("status"))
  8546. # Build list of possible keys to try (matching how they were registered in on_print_start)
  8547. possible_keys = _completion_print_keys(printer_id, filename, subtask_name)
  8548. # Find the archive for this print
  8549. logger.info("Looking for archive in _active_prints, keys to try: %s...", possible_keys[:5])
  8550. logger.info("Current _active_prints: %s", list(_active_prints.keys()))
  8551. archive_id = None
  8552. for key in possible_keys:
  8553. archive_id = _active_prints.pop(key, None)
  8554. if archive_id:
  8555. _owned_print_archive_ids.discard(archive_id)
  8556. logger.info("Found archive %s with key %s", archive_id, key)
  8557. # Also clean up the printer's other keys pointing to this archive;
  8558. # another printer's are a plate of it still printing there.
  8559. keys_to_remove = [k for k, v in _active_prints.items() if v == archive_id and k[0] == printer_id]
  8560. for k in keys_to_remove:
  8561. _active_prints.pop(k, None)
  8562. break
  8563. # The archive bound when the print started, before the name search below.
  8564. # Only this one may give the finish-photo plate restore a height (#3240):
  8565. # the search matches names by substring.
  8566. bound_archive_id = archive_id
  8567. if not archive_id and data.get("_reconciled") and data.get("_reconciled_archive_id"):
  8568. # The reconcile names the archive whose run on this printer it
  8569. # closes. Only while that run is open: the print's real end may have
  8570. # closed it meanwhile, or closed the print without managing to write
  8571. # it, and either way there is nothing left to end twice.
  8572. if await _archive_of_open_run(printer_id) != data["_reconciled_archive_id"]:
  8573. logger.info(
  8574. "[RECONCILE] Archive %s has no run open on printer %s any more; nothing to close",
  8575. data["_reconciled_archive_id"],
  8576. printer_id,
  8577. )
  8578. return
  8579. archive_id = data["_reconciled_archive_id"]
  8580. ending_job = str(data.get("subtask_id") or "").strip()
  8581. if not archive_id and ending_job not in ("", "0"):
  8582. # Not bound (a restart since the start): this job's run, open on the
  8583. # printer. Only by its job id: a run open there can be of the print
  8584. # running now, while the end is of an earlier one closed late (the
  8585. # restart's reconcile of a missed end).
  8586. archive_id = await _archive_of_open_run(printer_id, subtask_id=ending_job)
  8587. if archive_id:
  8588. logger.info("Found archive %s by the run of job %s open on printer %s", archive_id, ending_job, printer_id)
  8589. if not archive_id:
  8590. # Try to find by filename or subtask_name if not tracked (for prints started before app)
  8591. async with async_session() as db:
  8592. from backend.app.models.archive import PrintArchive
  8593. # Try matching by subtask_name (stored as print_name) first. The
  8594. # same name rule as the queue's completion check: a name inside
  8595. # another ("Clip" in "Cable Clip") is a different print, and the
  8596. # printer's underscores for spaces and truncation still match.
  8597. if subtask_name:
  8598. printing = await db.scalars(
  8599. select(PrintArchive)
  8600. .where(PrintArchive.printer_id == printer_id)
  8601. .where(PrintArchive.status == "printing")
  8602. .order_by(PrintArchive.created_at.desc())
  8603. )
  8604. # A print started from the printer's screen reports its file
  8605. # name, extensions included, as the subtask (#3009).
  8606. reported = {subtask_name, _subtask_name_from_filename(subtask_name)}
  8607. archive = next(
  8608. (
  8609. candidate
  8610. for candidate in printing
  8611. if any(
  8612. (candidate.print_name and _subtask_names_match(candidate.print_name, name))
  8613. or (
  8614. candidate.filename
  8615. and _subtask_names_match(_subtask_name_from_filename(candidate.filename), name)
  8616. )
  8617. for name in reported
  8618. )
  8619. ),
  8620. None,
  8621. )
  8622. if archive:
  8623. archive_id = archive.id
  8624. logger.info("Found archive %s by subtask_name match: %s", archive_id, subtask_name)
  8625. # Also try by filename
  8626. if not archive_id and filename:
  8627. result = await db.execute(
  8628. select(PrintArchive)
  8629. .where(PrintArchive.printer_id == printer_id)
  8630. .where(PrintArchive.filename == filename)
  8631. .where(PrintArchive.status == "printing")
  8632. .order_by(PrintArchive.created_at.desc())
  8633. .limit(1)
  8634. )
  8635. archive = result.scalar_one_or_none()
  8636. if archive:
  8637. archive_id = archive.id
  8638. # Bounded: a printer whose FTP hangs on DELE took up to 3 candidates x 3
  8639. # attempts x a 60 s timeout, about nine minutes, before the queue item was
  8640. # closed below. The stranded-job closer gives up after five and closed it
  8641. # first, so the job's auto-off and its billing never ran. Inline rather
  8642. # than in the background, so the next job of the same file isn't released
  8643. # (and uploaded under the name being deleted) while the cleanup still
  8644. # works on it. A delete already sent may still finish in its FTP thread
  8645. # after the cutoff, as one past delete_file_async's own timeout always did.
  8646. try:
  8647. await asyncio.wait_for(
  8648. _cleanup_sd_card_after_print(printer_id, subtask_name, archive_id, logger),
  8649. timeout=_SD_CLEANUP_BUDGET_SECONDS,
  8650. )
  8651. except TimeoutError:
  8652. logger.warning(
  8653. "SD card cleanup on printer %s took over %ss; leaving the file there and closing the print",
  8654. printer_id,
  8655. _SD_CLEANUP_BUDGET_SECONDS,
  8656. )
  8657. log_timing("SD card cleanup")
  8658. # Update queue item status early — must run before the archive_id early-return
  8659. # so queue items don't get stuck in "printing" when archive lookup fails.
  8660. # Uses run_with_retry to handle SQLite "database is locked" errors (#897).
  8661. queue_item_id = None
  8662. billing_run_id: str | None = None
  8663. billing_user_id: int | None = None
  8664. billing_cost_center_id: int | None = None
  8665. billing_plate_id: int | None = None
  8666. queue_status = None
  8667. queue_auto_off = False
  8668. try:
  8669. from backend.app.core.database import run_with_retry
  8670. from backend.app.models.print_queue import PrintQueueItem
  8671. async def _update_queue_status(db):
  8672. nonlocal billing_run_id, billing_user_id, billing_cost_center_id, billing_plate_id
  8673. nonlocal queue_item_id, queue_status, queue_auto_off
  8674. result = await db.execute(
  8675. select(PrintQueueItem)
  8676. .where(PrintQueueItem.printer_id == printer_id)
  8677. .where(PrintQueueItem.status == "printing")
  8678. )
  8679. printing_items = list(result.scalars().all())
  8680. if len(printing_items) > 1:
  8681. logger.warning(
  8682. "BUG: Multiple queue items in 'printing' status for printer %s: %s",
  8683. printer_id,
  8684. [(i.id, i.archive_id, i.library_file_id) for i in printing_items],
  8685. )
  8686. item = printing_items[0] if printing_items else None
  8687. if item is not None and not await _completion_belongs_to_queue_item(db, item, data):
  8688. return
  8689. if item:
  8690. queue_status = data.get("status", "completed")
  8691. # MQTT sends "aborted" for cancelled prints; normalise to
  8692. # "cancelled" so it matches the queue schema Literal.
  8693. if queue_status == "aborted":
  8694. queue_status = "cancelled"
  8695. item.status = queue_status
  8696. item.completed_at = datetime.now(timezone.utc)
  8697. if queue_status == "failed" and not item.error_message:
  8698. item.error_message = _format_hms_error_summary(data.get("hms_errors") or [])
  8699. # Bump usage counters on the source library file so admins can
  8700. # sort by "last printed" and (eventually) auto-purge stale
  8701. # files — #1008.
  8702. await _bump_library_file_usage_if_completed(db, item, queue_status)
  8703. await db.commit()
  8704. queue_item_id = item.id
  8705. billing_run_id = item.billing_run_id
  8706. billing_user_id = item.created_by_id
  8707. billing_cost_center_id = item.cost_center_id
  8708. billing_plate_id = item.plate_id
  8709. queue_auto_off = item.auto_off_after
  8710. logger.info("Updated queue item %s status to %s", item.id, queue_status)
  8711. await run_with_retry(_update_queue_status, label="queue status update")
  8712. # Post-commit side effects (notifications, MQTT relay, auto-off) use
  8713. # their own sessions and have their own error handling — no retry needed.
  8714. if queue_item_id is not None:
  8715. # Batch orders (#342): this run may have been the last one an order
  8716. # owed. Re-evaluate here rather than lazily on read, so a finished
  8717. # order reports itself complete without someone opening the page.
  8718. try:
  8719. from backend.app.services.print_batch import refresh_batch_status_for_item
  8720. async with async_session() as db:
  8721. await refresh_batch_status_for_item(db, queue_item_id)
  8722. await db.commit()
  8723. except Exception as e:
  8724. logger.warning("[BATCH] Failed to refresh batch status for queue item %s: %s", queue_item_id, e)
  8725. # MQTT relay - publish queue job completed
  8726. try:
  8727. printer_info = printer_manager.get_printer(printer_id)
  8728. await mqtt_relay.on_queue_job_completed(
  8729. job_id=queue_item_id,
  8730. filename=filename or subtask_name,
  8731. printer_id=printer_id,
  8732. printer_name=printer_info.name if printer_info else "Unknown",
  8733. status=queue_status,
  8734. )
  8735. except Exception:
  8736. pass # Don't fail if MQTT fails
  8737. # Check if queue is now empty and send notification
  8738. try:
  8739. from sqlalchemy import func as sa_func
  8740. async with async_session() as db:
  8741. count_result = await db.execute(
  8742. select(sa_func.count(PrintQueueItem.id)).where(PrintQueueItem.status == "pending")
  8743. )
  8744. pending_count = count_result.scalar() or 0
  8745. if pending_count == 0:
  8746. today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
  8747. completed_result = await db.execute(
  8748. select(sa_func.count(PrintQueueItem.id)).where(
  8749. PrintQueueItem.status.in_(["completed", "failed", "skipped"]),
  8750. PrintQueueItem.completed_at >= today_start,
  8751. )
  8752. )
  8753. completed_count = completed_result.scalar() or 1
  8754. await notification_service.on_queue_completed(
  8755. completed_count=completed_count,
  8756. db=db,
  8757. )
  8758. except Exception:
  8759. pass # Don't fail if notification fails
  8760. # Handle auto_off_after - power off printer if the queue item opted
  8761. # in. Delegates to the smart-plug manager so the off honours each
  8762. # plug's configured strategy (time delay or temperature threshold),
  8763. # is cancelled if the printer starts printing again, and never cuts
  8764. # power on a loaded print (#1890). Previously an inline block here
  8765. # hardcoded a 50°C / 600s cooldown wait and powered off on the
  8766. # timeout regardless of print state — cutting a touchscreen reprint.
  8767. if queue_auto_off:
  8768. try:
  8769. async with async_session() as db:
  8770. await smart_plug_manager.schedule_off_after_queue_job(printer_id, db)
  8771. except Exception as e:
  8772. logger.warning("Failed to schedule queue auto-off for printer %s: %s", printer_id, e)
  8773. except Exception as e:
  8774. logging.getLogger(__name__).warning(f"Queue item update failed: {e}")
  8775. log_timing("Queue item update")
  8776. # Register bed cooldown waiter (event-driven via on_bed_temp_update callback).
  8777. # Must run before archive_id early-return so it fires for all prints (including
  8778. # prints started from BambuStudio/touchscreen that have no archive).
  8779. if data.get("status") == "completed":
  8780. try:
  8781. from backend.app.api.routes.settings import get_setting
  8782. async with async_session() as db:
  8783. threshold_str = await get_setting(db, "bed_cooled_threshold")
  8784. threshold = float(threshold_str) if threshold_str else 35.0
  8785. # Check if any provider has on_bed_cooled enabled (skip registration if none)
  8786. async with async_session() as db:
  8787. providers = await notification_service._get_providers_for_event(db, "on_bed_cooled", printer_id)
  8788. if providers:
  8789. _bed_cool_waiters[printer_id] = {
  8790. "threshold": threshold,
  8791. "filename": filename or subtask_name or "",
  8792. "registered_at": time.time(),
  8793. }
  8794. logger.info(
  8795. "[BED-COOL] Registered waiter for printer %s (threshold: %.0f°C)",
  8796. printer_id,
  8797. threshold,
  8798. )
  8799. else:
  8800. logger.debug("[BED-COOL] No providers enabled for bed_cooled on printer %s", printer_id)
  8801. except Exception as e:
  8802. logger.warning("[BED-COOL] Failed to register waiter: %s", e)
  8803. # Capture the slicer estimate before usage tracking runs. The tracker may
  8804. # update archive.cost with this run's measured cost; billing partial runs
  8805. # against that already-partial value would discount the charge twice.
  8806. billing_planned_grams: float | None = None
  8807. billing_base_cost: float | None = None
  8808. if archive_id:
  8809. try:
  8810. async with async_session() as db:
  8811. from backend.app.models.archive import PrintArchive
  8812. billing_archive = await db.get(PrintArchive, archive_id)
  8813. if billing_archive:
  8814. billing_path = (
  8815. app_settings.base_dir / billing_archive.file_path if billing_archive.file_path else None
  8816. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  8817. billing_planned_grams, billing_base_cost = _plate_scoped_run_estimate(
  8818. billing_archive,
  8819. billing_path,
  8820. billing_plate_id
  8821. if billing_plate_id is not None
  8822. else _get_start_plate_id(printer_id, archive_id),
  8823. )
  8824. except Exception as e:
  8825. logger.warning("[FINANCE] Failed to capture planned usage for archive %s: %s", archive_id, e)
  8826. # --- Track filament consumption (must run before archive_id early-return so usage
  8827. # is recorded even when auto-archive is disabled) ---
  8828. usage_results: list[dict] = []
  8829. # Prefer ams_mapping captured from MQTT request topic (works for all print sources)
  8830. stored_ams_mapping = data.get("ams_mapping")
  8831. # Fallback to _print_ams_mappings for queue/reprint (set before print starts).
  8832. # Drained either way, like the plate below: left over, it would be read by
  8833. # the archive's next run, dispatched without a mapping.
  8834. registered_ams_mapping = _print_ams_mappings.pop((printer_id, archive_id), None) if archive_id else None
  8835. if not stored_ams_mapping:
  8836. stored_ams_mapping = registered_ams_mapping
  8837. # Always drain the plate_id register on completion — the session already
  8838. # consumed it at print-start injection; leaving it would leak into the next
  8839. # print on the same archive_id (rare but possible with reprints) (#1697).
  8840. # Capture the popped value so the completion notification can scope the
  8841. # archive-level (summed-across-plates per #1593) filament + time totals
  8842. # down to the single plate that was actually printed (#1785).
  8843. notify_plate_id: int | None = None
  8844. if archive_id:
  8845. notify_plate_id = _print_plate_ids.pop((printer_id, archive_id), None)
  8846. # The start estimate must not outlive the real cost written below (COST-4).
  8847. if archive_id:
  8848. await _settle_archive_cost_estimate(archive_id)
  8849. # Internal inventory: track AMS remain% deltas (skip if Spoolman handles usage)
  8850. try:
  8851. async with async_session() as db:
  8852. from backend.app.api.routes.settings import get_setting
  8853. _spoolman_on = await get_setting(db, "spoolman_enabled")
  8854. if not _spoolman_on or _spoolman_on.lower() != "true":
  8855. from backend.app.services.usage_tracker import on_print_complete as usage_on_print_complete
  8856. async with async_session() as db:
  8857. usage_results = await usage_on_print_complete(
  8858. printer_id,
  8859. data,
  8860. printer_manager,
  8861. db,
  8862. archive_id=archive_id,
  8863. ams_mapping=stored_ams_mapping,
  8864. )
  8865. if usage_results:
  8866. await ws_manager.broadcast(
  8867. {
  8868. "type": "spool_usage_logged",
  8869. "printer_id": printer_id,
  8870. "usage": usage_results,
  8871. }
  8872. )
  8873. log_timing("Usage tracker")
  8874. except Exception as e:
  8875. logger.warning("Usage tracker on_print_complete failed: %s", e)
  8876. # Drop the print-start context unconditionally — the Spoolman branch above
  8877. # skips the internal tracker entirely, so nothing else would clear what
  8878. # print start captured, and a row surviving its print would be restored
  8879. # onto the next one after a restart.
  8880. try:
  8881. from backend.app.services.usage_tracker import discard_session
  8882. async with async_session() as db:
  8883. await discard_session(db, printer_id)
  8884. except Exception as e:
  8885. logger.warning("Failed to clear persisted print session for printer %s: %s", printer_id, e)
  8886. # Spoolman: report filament usage (requires archive_id for tracking data lookup)
  8887. # A partial run's cost from the Spoolman spools it was charged to; the
  8888. # built-in inventory's comes in usage_results.
  8889. spoolman_run_cost: float | None = None
  8890. if archive_id:
  8891. if data.get("status") == "completed":
  8892. try:
  8893. await _report_spoolman_usage(printer_id, archive_id)
  8894. log_timing("Spoolman usage report")
  8895. except Exception as e:
  8896. logger.warning("Spoolman usage reporting failed: %s", e)
  8897. else:
  8898. # Report partial usage if tracking data exists (only stored when weight sync is disabled)
  8899. try:
  8900. async with async_session() as db:
  8901. spoolman_run_cost = await _cleanup_spoolman_tracking(
  8902. printer_id,
  8903. archive_id,
  8904. db,
  8905. last_layer_num=data.get("last_layer_num"),
  8906. last_progress=data.get("last_progress"),
  8907. status=data.get("status") or "failed",
  8908. )
  8909. except Exception as e:
  8910. logger.debug("[SPOOLMAN] Cleanup failed: %s", e)
  8911. await _release_archived_spoolman_slots(printer_id)
  8912. log_timing("Filament usage tracking")
  8913. if not archive_id:
  8914. # The printer's own calibration run has no archive by design, so this
  8915. # arrives here every time one finishes. Returning before the no-archive
  8916. # notification keeps a calibration from being announced as a finished
  8917. # print. Everything above this point has already run — the plate-clear
  8918. # gate, the queue reconciliation, the SD-card cleanup — so only the
  8919. # notification is skipped.
  8920. if is_internal_printer_job(filename, subtask_name):
  8921. logger.info(
  8922. "[CALLBACK] Internal printer job completed, no notification: filename=%s, subtask=%s",
  8923. filename,
  8924. subtask_name,
  8925. )
  8926. return
  8927. logger.warning("Could not find archive for print complete: filename=%s, subtask=%s", filename, subtask_name)
  8928. # Still send print-complete/failed/stopped notifications even without an archive.
  8929. # Try to enrich with queue/library-file data so user-specific emails work too.
  8930. async def _notify_no_archive():
  8931. try:
  8932. async with async_session() as db:
  8933. from backend.app.models.library import LibraryFile
  8934. from backend.app.models.print_queue import PrintQueueItem
  8935. from backend.app.models.printer import Printer
  8936. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8937. printer_obj = result.scalar_one_or_none()
  8938. p_name = printer_obj.name if printer_obj else f"Printer {printer_id}"
  8939. # Only the queue item this completion itself closed (the
  8940. # reconciliation above, which checks the run belongs to it).
  8941. # A print started outside Bambuddy closed none, and must not
  8942. # borrow an earlier job's owner, email or times.
  8943. no_archive_data: dict | None = None
  8944. try:
  8945. queue_item = await db.get(PrintQueueItem, queue_item_id) if queue_item_id else None
  8946. if queue_item:
  8947. no_archive_data = {"created_by_id": queue_item.created_by_id}
  8948. # How long it really ran, as the archive path reports it (#1198).
  8949. started = to_naive_utc(queue_item.started_at)
  8950. ended = to_naive_utc(queue_item.completed_at)
  8951. # Not for a completion reconciled at reconnect: it
  8952. # ended at an unknown time while the printer was
  8953. # offline, and "now" is the reconnect (DB-C9).
  8954. if (
  8955. not data.get("_reconciled")
  8956. and started
  8957. and ended
  8958. and (ended - started).total_seconds() > 0
  8959. ):
  8960. no_archive_data["actual_time_seconds"] = int((ended - started).total_seconds())
  8961. # Estimated time: the queue item caches it from its
  8962. # archive or library file; otherwise read the library
  8963. # file's metadata (LibraryFile has no column for it).
  8964. if queue_item.print_time_seconds:
  8965. no_archive_data["print_time_seconds"] = queue_item.print_time_seconds
  8966. elif queue_item.library_file_id:
  8967. lib_result = await db.execute(
  8968. select(LibraryFile).where(LibraryFile.id == queue_item.library_file_id)
  8969. )
  8970. lib_file = lib_result.scalar_one_or_none()
  8971. estimate = (
  8972. (lib_file.file_metadata or {}).get("print_time_seconds") if lib_file else None
  8973. )
  8974. if estimate:
  8975. no_archive_data["print_time_seconds"] = estimate
  8976. except Exception as lookup_err:
  8977. logger.debug(
  8978. "[NOTIFY-BG] Could not look up queue item for no-archive notification: %s", lookup_err
  8979. )
  8980. # Enrich with usage tracker results (captured in enclosing scope)
  8981. if usage_results:
  8982. if no_archive_data is None:
  8983. no_archive_data = {}
  8984. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  8985. if total_from_usage > 0:
  8986. no_archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  8987. no_archive_data["usage_results"] = usage_results
  8988. # Try MQTT remaining_time for print duration when no queue/library data
  8989. if no_archive_data and not no_archive_data.get("print_time_seconds"):
  8990. mqtt_remaining = data.get("remaining_time")
  8991. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  8992. no_archive_data["print_time_seconds"] = int(mqtt_remaining)
  8993. ps = data.get("status", "completed")
  8994. logger.info(
  8995. "[NOTIFY-BG] Sending notification without archive: printer=%s, status=%s", printer_id, ps
  8996. )
  8997. if not await _kill_switch_notification_already_sent(kill_switch_notification_task):
  8998. await notification_service.on_print_complete(
  8999. printer_id, p_name, ps, data, db, archive_data=no_archive_data
  9000. )
  9001. else:
  9002. logger.info("[NOTIFY-BG] Skipped duplicate kill-switch provider notification")
  9003. # Send user-specific email if we have a created_by_id
  9004. if no_archive_data and no_archive_data.get("created_by_id"):
  9005. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  9006. await _dispatch_user_print_email(
  9007. ps,
  9008. no_archive_data["created_by_id"],
  9009. p_name,
  9010. raw_filename,
  9011. db,
  9012. )
  9013. logger.info("[NOTIFY-BG] Completed (no-archive path)")
  9014. except Exception as e:
  9015. logger.warning("[NOTIFY-BG] Failed to send notification without archive: %s", e, exc_info=True)
  9016. spawn_background_task(_notify_no_archive(), name="notify-no-archive")
  9017. return
  9018. log_timing("Archive lookup")
  9019. # Update archive status
  9020. logger.info("[ARCHIVE] Updating archive %s status...", archive_id)
  9021. try:
  9022. async with async_session() as db:
  9023. service = ArchiveService(db)
  9024. status = data.get("status", "completed")
  9025. hms_errors = data.get("hms_errors", []) if status == "failed" else None
  9026. if hms_errors:
  9027. logger.info("[ARCHIVE] HMS errors at failure: %s", hms_errors)
  9028. failure_reason = derive_failure_reason(status, hms_errors)
  9029. if data.get("_reconciled") and (status == "aborted" or (status == "failed" and not failure_reason)):
  9030. # A reconciled completion closes out a stale archive at
  9031. # reconnect — it is not a user action, so don't mislabel it
  9032. # "userCancelled". It shares the stale-cleanup path's key
  9033. # (issue #2974) and records that the real end time is unknown,
  9034. # which is also why its logged duration is 0 (#2592). A failed
  9035. # one the printer's errors explain keeps their reason (#3310).
  9036. failure_reason = "noStatusUpdate"
  9037. elif data.get("_kill_switch"):
  9038. # Not the user's cancel: the kill switch stopped a print that did
  9039. # not come from Bambuddy.
  9040. failure_reason = "killSwitch"
  9041. if failure_reason:
  9042. logger.info("[ARCHIVE] failure_reason=%r (status=%s)", failure_reason, status)
  9043. elif status == "failed" and hms_errors:
  9044. logger.info("[ARCHIVE] HMS errors present but none matched a known failure-reason short code")
  9045. await service.update_archive_status(
  9046. archive_id,
  9047. status=status,
  9048. completed_at=(
  9049. # A reconciled completed print ended at an estimated time,
  9050. # not at the reconnect (#3310).
  9051. data.get("_reconciled_completed_at") or datetime.now(timezone.utc)
  9052. if status in ("completed", "failed", "aborted", "cancelled")
  9053. else None
  9054. ),
  9055. failure_reason=failure_reason,
  9056. )
  9057. logger.info(
  9058. "[ARCHIVE] Archive %s status updated to %s, failure_reason=%s", archive_id, status, failure_reason
  9059. )
  9060. await ws_manager.send_archive_updated(
  9061. {
  9062. "id": archive_id,
  9063. "status": status,
  9064. }
  9065. )
  9066. logger.info("[ARCHIVE] WebSocket notification sent for archive %s", archive_id)
  9067. # MQTT relay - publish archive updated
  9068. try:
  9069. await mqtt_relay.on_archive_updated(
  9070. archive_id=archive_id,
  9071. print_name=filename or subtask_name,
  9072. status=status,
  9073. )
  9074. except Exception:
  9075. pass # Don't fail if MQTT fails
  9076. except Exception as e:
  9077. logger.error("[ARCHIVE] Failed to update archive %s status: %s", archive_id, e, exc_info=True)
  9078. # Continue with other operations even if archive update fails
  9079. log_timing("Archive status update")
  9080. # Apply finance wallet charge or release reservations once. For all partial
  9081. # terminal states (failed, aborted at the printer display, or cancelled via
  9082. # Bambuddy) use this run's measured spool delta, falling back to the last
  9083. # valid printer progress. PrintArchive.filament_used_grams is the slicer
  9084. # estimate and therefore cannot represent an interrupted run.
  9085. try:
  9086. if data.get("status") in ("completed", "failed", "aborted", "cancelled"):
  9087. async with async_session() as db:
  9088. from backend.app.models.archive import PrintArchive
  9089. from backend.app.services.finance_billing import apply_print_charge_for_archive
  9090. from backend.app.services.print_log import run_being_ended
  9091. archive = await db.get(PrintArchive, archive_id)
  9092. # The run this end fills in below: charged as its own, not as
  9093. # the archive's latest dispatch (another plate of it, maybe).
  9094. charged_run = await run_being_ended(db, archive_id, printer_id, archive.started_at) if archive else None
  9095. if archive and billing_run_id is None and charged_run is None:
  9096. billing_run_id = getattr(archive, "billing_run_id", None)
  9097. if archive and archive.created_by_id is None and _print_user_info:
  9098. archive.created_by_id = await _print_user_id_if_still_there(db, _print_user_info)
  9099. await db.flush()
  9100. run_status = data.get("status", "completed")
  9101. last_progress = data.get("last_progress")
  9102. if last_progress is None:
  9103. last_progress = data.get("progress")
  9104. actual_run_grams = _compute_run_filament_grams(
  9105. run_status,
  9106. billing_planned_grams,
  9107. last_progress,
  9108. usage_results,
  9109. )
  9110. filament_usage = (actual_run_grams, billing_planned_grams) if run_status != "completed" else None
  9111. in_memory_cost_center_id = _print_cost_center_ids.pop((printer_id, archive_id), None)
  9112. charged = await apply_print_charge_for_archive(
  9113. db,
  9114. archive_id,
  9115. charged_user_id=billing_user_id,
  9116. cost_center_id=(
  9117. billing_cost_center_id if billing_cost_center_id is not None else in_memory_cost_center_id
  9118. ),
  9119. print_queue_id=queue_item_id,
  9120. print_run_id=billing_run_id,
  9121. base_cost_override=billing_base_cost,
  9122. filament_usage=filament_usage,
  9123. run_entry_id=charged_run.id if charged_run is not None else None,
  9124. run_status=run_status,
  9125. )
  9126. await db.commit()
  9127. if charged:
  9128. logger.info("[FINANCE] Applied print charge for archive %s", archive_id)
  9129. except Exception as e:
  9130. logger.warning("[FINANCE] Failed to apply print charge for archive %s: %s", archive_id, e)
  9131. printer_info = printer_manager.get_printer(printer_id)
  9132. billing_printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  9133. billing_filename = filename or subtask_name or "Unknown"
  9134. billing_error = str(e)
  9135. try:
  9136. await ws_manager.broadcast(
  9137. {
  9138. "type": "billing_charge_failed",
  9139. "printer_id": printer_id,
  9140. "printer_name": billing_printer_name,
  9141. "filename": billing_filename,
  9142. "archive_id": archive_id,
  9143. }
  9144. )
  9145. except Exception as notification_error:
  9146. logger.error(
  9147. "[FINANCE] Failed to broadcast billing error for archive %s: %s",
  9148. archive_id,
  9149. notification_error,
  9150. )
  9151. async def _notify_billing_charge_failed() -> None:
  9152. try:
  9153. async with async_session() as notification_db:
  9154. await notification_service.on_billing_charge_failed(
  9155. printer_id,
  9156. billing_printer_name,
  9157. billing_filename,
  9158. archive_id,
  9159. billing_error,
  9160. notification_db,
  9161. )
  9162. except Exception as provider_error:
  9163. logger.error(
  9164. "[FINANCE] Failed to send provider billing alert for archive %s: %s",
  9165. archive_id,
  9166. provider_error,
  9167. exc_info=True,
  9168. )
  9169. spawn_background_task(
  9170. _notify_billing_charge_failed(),
  9171. name=f"billing-charge-failed-{archive_id}",
  9172. )
  9173. log_timing("Finance charge update")
  9174. # Write independent print log entry (separate table, never touches archives)
  9175. # The run's id, for the energy work below: the archive's newest run is
  9176. # the one started last, which with plates of it on two printers need not
  9177. # be the one that just ended.
  9178. completed_run_id: int | None = None
  9179. async def _write_run(db):
  9180. # Retried on a locked SQLite database: a run left open here would be
  9181. # closed again by the next reconnect's reconcile.
  9182. nonlocal completed_run_id
  9183. from backend.app.models.archive import PrintArchive
  9184. from backend.app.services.print_log import discard_open_runs, record_archive_wear, write_log_entry
  9185. archive = await db.get(PrintArchive, archive_id)
  9186. if archive is None:
  9187. # Deleted while it printed: its run has nothing left to end on.
  9188. discarded = await discard_open_runs(db, printer_id)
  9189. await db.commit()
  9190. await _announce_discarded_runs(discarded, printer_id)
  9191. if archive:
  9192. # Back-fill created_by_id on reprint (#730): reprint reuses the
  9193. # source archive row rather than creating a new one, so an
  9194. # archive that was auto-created from a printer-initiated
  9195. # print (created_by_id=NULL) would otherwise stay unattributed
  9196. # forever. When we have a print-session user AND the archive
  9197. # has no attribution yet, credit the current user. Never
  9198. # overwrite an existing attribution — the original uploader
  9199. # keeps ownership.
  9200. _print_user_id = await _print_user_id_if_still_there(db, _print_user_info)
  9201. if archive.created_by_id is None and _print_user_id is not None:
  9202. archive.created_by_id = _print_user_id
  9203. p_info = printer_manager.get_printer(printer_id)
  9204. # Per-run actuals — written to PrintLogEntry so stats reflect
  9205. # what THIS print actually used, not the source archive's
  9206. # first-run values (#1378). Helper handles the partial-print
  9207. # math (failed / cancelled / stopped get scaled to progress
  9208. # or to tracked spool deltas).
  9209. _run_status = data.get("status", "completed")
  9210. # #2614: scope the per-run estimate to the printed plate. For a
  9211. # multi-plate 3MF dispatched one plate at a time, the archive's
  9212. # filament/cost are the whole-file totals; the PrintLogEntry must
  9213. # reflect only this plate. No effect on single-plate archives (the
  9214. # plate estimate equals the whole-file value) or on the tracker
  9215. # path (measured spool deltas win in _compute_run_filament_grams).
  9216. _est_full_path = (
  9217. app_settings.base_dir / archive.file_path if archive.file_path else None
  9218. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  9219. # The plate that printed, as billing reads it: a reprint of
  9220. # another plate must not log the archive's first plate.
  9221. _run_plate_id = billing_plate_id if billing_plate_id is not None else notify_plate_id
  9222. _est_grams, _est_cost = _plate_scoped_run_estimate(archive, _est_full_path, _run_plate_id)
  9223. if _run_plate_id is None:
  9224. _run_plate_id = archive.plate_id
  9225. _run_grams = _compute_run_filament_grams(
  9226. _run_status,
  9227. _est_grams,
  9228. data.get("last_progress", data.get("progress")),
  9229. usage_results,
  9230. )
  9231. # Per-run cost — prefer usage_results sum. For partial prints
  9232. # we deliberately skip the topup-to-estimate logic in
  9233. # usage_tracker (which assumes the print completed); the raw
  9234. # tracked-spool sum is closer to what THIS run actually cost.
  9235. _run_cost: float | None = None
  9236. if usage_results:
  9237. _run_cost = sum(r.get("cost") or 0 for r in usage_results) or None
  9238. if _run_cost is None and _run_status != "completed":
  9239. _run_cost = spoolman_run_cost
  9240. if _run_cost is None and _run_status == "completed":
  9241. _run_cost = _est_cost
  9242. from backend.app.models.printer import Printer as _Printer
  9243. _wear_rate = await db.scalar(select(_Printer.wear_cost_per_hour).where(_Printer.id == printer_id))
  9244. run_entry = await write_log_entry(
  9245. db,
  9246. archive_id=archive.id,
  9247. # Captured by _update_queue_status above; None for
  9248. # printer-initiated prints with no queue row. Batch
  9249. # cost/energy roll-up joins on it (#342).
  9250. queue_item_id=queue_item_id,
  9251. status=_run_status,
  9252. print_name=archive.print_name,
  9253. # The plate this run printed, as the estimate above takes
  9254. # it; the archive's own is the latest run's.
  9255. plate_id=_run_plate_id,
  9256. printer_name=p_info.name if p_info else None,
  9257. printer_id=printer_id,
  9258. started_at=archive.started_at,
  9259. completed_at=archive.completed_at,
  9260. filament_type=archive.filament_type,
  9261. filament_color=archive.filament_color,
  9262. filament_used_grams=_run_grams,
  9263. cost=_run_cost,
  9264. wear_cost_per_hour=_wear_rate,
  9265. failure_reason=archive.failure_reason,
  9266. thumbnail_path=archive.thumbnail_path,
  9267. created_by_id=archive.created_by_id,
  9268. created_by_username=_print_user_info.get("username") if _print_user_info else None,
  9269. # Reconciled completions have an unknown real end time —
  9270. # log 0 duration instead of the whole disconnect gap (#2592),
  9271. # unless it was estimated (#3310).
  9272. reconciled=bool(data.get("_reconciled")) and not data.get("_reconciled_completed_at"),
  9273. # Its time is the estimate or nothing, never measured (#3310).
  9274. duration_unmeasured=bool(data.get("_reconciled")),
  9275. # The real end of a print stopped while its printer was offline.
  9276. replaces_offline_stop=True,
  9277. )
  9278. await record_archive_wear(db, archive, run_entry)
  9279. await db.commit()
  9280. completed_run_id = run_entry.id
  9281. logger.info("[PRINT_LOG] Log entry written for archive %s", archive_id)
  9282. try:
  9283. from backend.app.core.database import run_with_retry
  9284. await run_with_retry(_write_run, label="print log entry")
  9285. except Exception as e:
  9286. logger.warning("[PRINT_LOG] Failed to write log entry for archive %s: %s", archive_id, e)
  9287. await _publish_run_completed(completed_run_id, printer_id)
  9288. log_timing("Print log entry")
  9289. # Run slow operations as background tasks to avoid blocking the event loop
  9290. # These operations can take 5-10+ seconds and would freeze the UI if awaited
  9291. async def _background_energy_calculation():
  9292. """Calculate and save energy usage in background.
  9293. Reads the starting kWh from the archive row (#941: persisted so a mid-print
  9294. backend restart no longer loses per-print energy data).
  9295. """
  9296. try:
  9297. logger.info("[ENERGY-BG] Starting energy calculation for archive %s", archive_id)
  9298. async with async_session() as db:
  9299. from backend.app.models.archive import PrintArchive
  9300. from backend.app.models.print_log import PrintLogEntry
  9301. from backend.app.services.print_log import energy_start_of
  9302. archive = await db.get(PrintArchive, archive_id)
  9303. if archive is None:
  9304. logger.warning("[ENERGY-BG] Archive %s no longer exists", archive_id)
  9305. return
  9306. # The ended run's own start reading: the archive's is its
  9307. # latest start's, maybe another plate's on another printer.
  9308. run_row = await db.get(PrintLogEntry, completed_run_id) if completed_run_id is not None else None
  9309. start = energy_start_of(run_row, archive)
  9310. if start is None:
  9311. logger.info("[ENERGY-BG] No start kWh recorded for archive %s's run", archive_id)
  9312. return
  9313. starting_kwh = start.kwh
  9314. candidates = await energy_plug_candidates(db, printer_id)
  9315. if not candidates:
  9316. logger.info("[ENERGY-BG] No smart plug for printer %s", printer_id)
  9317. return
  9318. # Same ordering as the start reading, so the delta below is
  9319. # against the counter that produced `starting_kwh` (#2859).
  9320. selected = await select_energy_reading(candidates, _get_plug_energy, db)
  9321. if selected is None:
  9322. logger.warning(
  9323. "[ENERGY-BG] No plug on printer %s reports a lifetime energy counter (tried: %s)",
  9324. printer_id,
  9325. ", ".join(plug.name for plug in candidates),
  9326. )
  9327. return
  9328. plug, energy = selected
  9329. logger.info("[ENERGY-BG] Energy response from plug '%s': %s", plug.name, energy)
  9330. energy_used = round(energy["total"] - starting_kwh, 4)
  9331. logger.info("[ENERGY-BG] Per-print energy: %s kWh", energy_used)
  9332. if energy_used < 0:
  9333. logger.warning(
  9334. "[ENERGY-BG] Negative energy delta for archive %s (start=%s, end=%s) — counter reset?",
  9335. archive_id,
  9336. starting_kwh,
  9337. energy["total"],
  9338. )
  9339. return
  9340. from backend.app.services.energy_price import current_price, print_energy_cost, stored_price
  9341. # Each hour of the print at the price of that hour (#1251). If
  9342. # that fails, the last price known, as before, rather than no
  9343. # energy at all. Read first: the session may be unusable after.
  9344. last_price = await stored_price(db)
  9345. try:
  9346. energy_cost_value = round(
  9347. await print_energy_cost(
  9348. db,
  9349. plug_id=start.plug_id,
  9350. start_at=start.at,
  9351. start_kwh=starting_kwh,
  9352. start_price=start.price,
  9353. end_kwh=energy["total"],
  9354. end_plug_id=plug.id,
  9355. price_now=await current_price(db),
  9356. ),
  9357. 3,
  9358. )
  9359. except Exception as e:
  9360. logger.warning("[ENERGY-BG] Hour-by-hour cost failed for archive %s: %s", archive_id, e)
  9361. energy_cost_value = round(energy_used * last_price, 3)
  9362. # First-run-only overwrite of archive.energy_kwh / energy_cost so a
  9363. # reprint doesn't visually clobber the source archive's energy data
  9364. # (#1378). Reprint energy lives in the matching PrintLogEntry below.
  9365. from sqlalchemy import func
  9366. existing_runs = await db.scalar(
  9367. select(func.count(PrintLogEntry.id)).where(PrintLogEntry.archive_id == archive_id)
  9368. )
  9369. if (existing_runs or 0) <= 1:
  9370. # 0 = legacy archive that pre-dates per-run logging; 1 = the row
  9371. # we just wrote for THIS print. Either way it's the first run.
  9372. archive.energy_kwh = energy_used
  9373. archive.energy_cost = energy_cost_value
  9374. # The ended run's energy (write_log_entry above ran before this
  9375. # background task, so its energy fields are still NULL). With
  9376. # no run written, none: the archive's newest is another print's.
  9377. if run_row is not None:
  9378. run_row.energy_kwh = energy_used
  9379. run_row.energy_cost = energy_cost_value
  9380. await db.commit()
  9381. logger.info("[ENERGY-BG] Saved: %s kWh, cost=%s", energy_used, energy_cost_value)
  9382. except Exception as e:
  9383. logger.warning("[ENERGY-BG] Failed: %s", e)
  9384. async def _background_finish_photo() -> str | None:
  9385. """Capture finish photo in background. Returns photo filename if captured."""
  9386. # #2547: the Z to park the plate back to, set once this function has
  9387. # raised the plate itself (the timelapse path, where the moment producer
  9388. # returned without doing it). Declared out here so the `finally` can
  9389. # lower it again no matter where the capture below fails.
  9390. park_z: float | None = None
  9391. try:
  9392. logger.info("[PHOTO-BG] Starting finish photo capture for archive %s", archive_id)
  9393. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  9394. # Read phase: settings + printer + archive in a short session, released
  9395. # BEFORE the capture pipeline below. The capture (timelapse last-frame,
  9396. # stage-22 wait, external-camera grab, or a fresh RTSP shot) can take
  9397. # tens of seconds; holding this session across it pinned one pooled
  9398. # connection idle-in-transaction per finishing print (issue #2572).
  9399. async with async_session() as db:
  9400. from backend.app.api.routes.settings import get_setting
  9401. from backend.app.models.archive import PrintArchive
  9402. from backend.app.models.printer import Printer
  9403. capture_enabled = await get_setting(db, "capture_finish_photo")
  9404. if capture_enabled is not None and capture_enabled.lower() != "true":
  9405. return None
  9406. if not archive_id:
  9407. return None
  9408. printer = (await db.execute(select(Printer).where(Printer.id == printer_id))).scalar_one_or_none()
  9409. archive = (
  9410. await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  9411. ).scalar_one_or_none()
  9412. if not printer or not archive:
  9413. return None
  9414. import uuid
  9415. from datetime import datetime
  9416. from backend.app.utils.archive_paths import archive_dir as resolve_archive_dir
  9417. if not archive.file_path:
  9418. logger.warning("[PHOTO-BG] Archive %s has no file_path, using fallback dir", archive_id)
  9419. archive_dir = resolve_archive_dir(archive)
  9420. photo_filename = None
  9421. # Prefer the timelapse last-frame source when a timelapse was
  9422. # recording — it captures the moment after the toolhead parks
  9423. # but before the bed drops, which the live-camera grab below
  9424. # would miss (#1397). Skipped for external cameras (those have
  9425. # their own framing and don't see a Bambu timelapse). Only
  9426. # runs when the USER explicitly enabled timelapse for this
  9427. # print — #1721 removed Bambuddy's force-on at dispatch
  9428. # because it caused per-layer nozzle parking on Smooth-mode
  9429. # slicer profiles.
  9430. prefer_timelapse_source = bool(data.get("timelapse_was_active")) and not (
  9431. printer.external_camera_enabled and printer.external_camera_url
  9432. )
  9433. timelapse_still_pending = False
  9434. if prefer_timelapse_source:
  9435. photo_filename, timelapse_still_pending = await _capture_finish_photo_from_timelapse(
  9436. archive_id=archive_id,
  9437. archive_dir=archive_dir,
  9438. rotation=getattr(printer, "camera_rotation", 0),
  9439. run_id=completed_run_id,
  9440. )
  9441. # #1721: replacement framing path — on_finish_photo_moment
  9442. # pre-captured a frame at the stage-22 / FINISH edge (toolhead
  9443. # parked, bed not yet dropped) and cached the JPEG bytes in
  9444. # _stage22_finish_frames. Consume them now so the saved photo
  9445. # has the better framing instead of the post-bed-drop angle
  9446. # the live-camera fallback below would give.
  9447. if not photo_filename:
  9448. # #1790: on the FINISH-state fallback path the producer
  9449. # task is dispatched back-to-back with this consumer, so
  9450. # a bare pop would race past with an empty result and
  9451. # the RTSP fallback below would collide with the
  9452. # producer's still-in-flight grab (single-client RTSP
  9453. # on Bambu printers). Wait for the producer to finish
  9454. # or give up before touching the cache.
  9455. #
  9456. # #2547: 20s was enough when the producer only ever grabbed a
  9457. # frame. It now also raises the plate first, which costs the
  9458. # settle window before the grab even starts — so the budget has
  9459. # to cover settle + a worst-case 15s RTSP timeout, and still sit
  9460. # under the notification's own photo wait below.
  9461. in_flight = _stage22_finish_in_flight.pop(printer_id, None)
  9462. if in_flight is not None:
  9463. try:
  9464. await asyncio.wait_for(in_flight.wait(), timeout=_FINISH_PHOTO_PRODUCER_WAIT_SECONDS)
  9465. except TimeoutError:
  9466. logger.warning(
  9467. "[PHOTO-BG] timed out waiting for stage-22 producer for printer %s — proceeding to fallback",
  9468. printer_id,
  9469. )
  9470. cached_frame = _stage22_finish_frames.pop(printer_id, None)
  9471. if cached_frame:
  9472. # Already rotated by the producer (#2708) — rotating again
  9473. # here would undo the fix on the banked-frame path, whose
  9474. # bytes reach the cache having been rotated once already.
  9475. photos_dir = archive_dir / "photos"
  9476. photos_dir.mkdir(parents=True, exist_ok=True)
  9477. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9478. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9479. photo_path = photos_dir / photo_filename
  9480. await asyncio.to_thread(photo_path.write_bytes, cached_frame)
  9481. logger.info(
  9482. "[PHOTO-BG] Saved stage-22 pre-captured frame: %s (%d bytes)",
  9483. photo_filename,
  9484. len(cached_frame),
  9485. )
  9486. # #2547: the timelapse path reaches the live grab below whenever the
  9487. # video hasn't landed in time — the documented usual outcome on
  9488. # P1-series, where transfers are slowest. `on_finish_photo_moment`
  9489. # returned early for those prints without raising the plate, so
  9490. # without this the photo that actually ships in the notification is
  9491. # of an already-dropped plate: exactly the framing #1145/#1397/#1565
  9492. # asked us to fix. The archive still gets the better video frame
  9493. # later; this is about the image the user is sent.
  9494. #
  9495. # Gated on `timelapse_was_active` precisely because that is the
  9496. # condition under which the producer skipped. On every other path it
  9497. # has already raised and lowered the plate, and repeating that here
  9498. # would be a second pointless round trip.
  9499. if (
  9500. not photo_filename
  9501. and data.get("timelapse_was_active")
  9502. and not print_dispatch_context.end_gcode_injected(printer_id)
  9503. ):
  9504. try:
  9505. async with async_session() as db:
  9506. from backend.app.api.routes.settings import get_setting
  9507. restore_setting = await get_setting(db, "finish_photo_restore_plate")
  9508. if restore_setting is None or restore_setting.lower() == "true":
  9509. park_z = await _raise_plate_for_finish_photo(
  9510. printer_id, data, logger, bound_archive_id, notify_plate_id
  9511. )
  9512. except Exception as e:
  9513. logger.warning("[PLATE-RESTORE] printer %s: restore failed: %s", printer_id, e)
  9514. # Fallback chain: external camera → buffered live frame →
  9515. # fresh RTSP capture. Only runs if the timelapse path above
  9516. # didn't already produce a photo.
  9517. # Chamber light for the live capture below (#1655).
  9518. async with camera_light(printer if not photo_filename else None):
  9519. if not photo_filename:
  9520. if printer.external_camera_enabled and printer.external_camera_url:
  9521. logger.info("[PHOTO-BG] Using external camera")
  9522. from backend.app.api.routes.camera import live_frame_for_capture
  9523. from backend.app.services.external_camera import capture_frame
  9524. # #2707: the second half of the finish-photo failure — the
  9525. # pre-capture and this fallback both collided with the live
  9526. # view. None here continues down the fallback chain.
  9527. defer, buffered = live_frame_for_capture(printer_id)
  9528. if defer:
  9529. frame_data = buffered
  9530. else:
  9531. frame_data = await capture_frame(
  9532. printer.external_camera_url,
  9533. printer.external_camera_type or "mjpeg",
  9534. snapshot_url=printer.external_camera_snapshot_url,
  9535. )
  9536. if frame_data:
  9537. frame_data = _apply_camera_rotation(frame_data, printer, logger)
  9538. photos_dir = archive_dir / "photos"
  9539. photos_dir.mkdir(parents=True, exist_ok=True)
  9540. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9541. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9542. photo_path = photos_dir / photo_filename
  9543. await asyncio.to_thread(photo_path.write_bytes, frame_data)
  9544. logger.info("[PHOTO-BG] Saved external camera frame: %s", photo_filename)
  9545. else:
  9546. # Check if camera stream is active - use buffered frame to avoid freeze
  9547. # Check both RTSP streams (_active_streams) and chamber image streams (_active_chamber_streams)
  9548. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  9549. active_chamber_for_printer = [
  9550. k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")
  9551. ]
  9552. buffered_frame = get_buffered_frame(printer_id)
  9553. if (active_for_printer or active_chamber_for_printer) and buffered_frame:
  9554. # Use frame from active stream
  9555. logger.info("[PHOTO-BG] Using buffered frame from active stream")
  9556. buffered_frame = _apply_camera_rotation(buffered_frame, printer, logger)
  9557. photos_dir = archive_dir / "photos"
  9558. photos_dir.mkdir(parents=True, exist_ok=True)
  9559. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9560. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9561. photo_path = photos_dir / photo_filename
  9562. await asyncio.to_thread(photo_path.write_bytes, buffered_frame)
  9563. logger.info("[PHOTO-BG] Saved buffered frame: %s", photo_filename)
  9564. else:
  9565. # No active stream - capture new frame
  9566. from backend.app.services.camera import capture_finish_photo
  9567. photo_filename = await capture_finish_photo(
  9568. printer_id=printer_id,
  9569. ip_address=printer.ip_address,
  9570. access_code=printer.access_code,
  9571. model=printer.model,
  9572. archive_dir=archive_dir,
  9573. rotation=getattr(printer, "camera_rotation", 0),
  9574. )
  9575. # Write phase: attach the photo in a fresh short-lived session.
  9576. if photo_filename:
  9577. async with async_session() as db:
  9578. from backend.app.models.archive import PrintArchive
  9579. from backend.app.services.print_log import add_run_photo, run_of
  9580. arch = await db.get(PrintArchive, archive_id)
  9581. # The ended print's own list takes it too: the archive's
  9582. # holds every print's.
  9583. run = await run_of(db, archive_id, completed_run_id) if completed_run_id is not None else None
  9584. if arch is not None:
  9585. # A new list: appended in place, the ORM sees no
  9586. # change and a second print's photo is never saved.
  9587. photos = list(arch.photos or [])
  9588. photos.append(photo_filename)
  9589. arch.photos = photos
  9590. add_run_photo(run, photo_filename)
  9591. await db.commit()
  9592. logger.info("[PHOTO-BG] Saved: %s", photo_filename)
  9593. # The short wait above is bounded so a slow printer can't hold up
  9594. # the print-complete notification, which is what the caller is
  9595. # blocking on. When it ran out with the video still on its way,
  9596. # keep waiting off to the side and swap the better frame into the
  9597. # archive once it arrives (#2704 follow-up) — otherwise P1-series
  9598. # users, whose videos routinely take minutes to transfer, never get
  9599. # the pre-bed-drop framing this path exists to provide.
  9600. #
  9601. # Spawned here rather than at the point the wait gave up: both this
  9602. # function and the upgrade do a read-modify-write on `photos`, and
  9603. # the live-camera fallback above can take tens of seconds. Starting
  9604. # the upgrade before that write means the two can interleave and one
  9605. # silently drops the other's entry, leaving a JPEG on disk that the
  9606. # gallery never lists.
  9607. if timelapse_still_pending:
  9608. from backend.app.services import archive_work
  9609. spawn_background_task(
  9610. archive_work.held_while(
  9611. archive_id,
  9612. _upgrade_finish_photo_from_timelapse(
  9613. archive_id,
  9614. archive_dir,
  9615. rotation=getattr(printer, "camera_rotation", 0),
  9616. run_id=completed_run_id,
  9617. replace=photo_filename,
  9618. ),
  9619. ),
  9620. name=f"finish-photo-upgrade-{archive_id}",
  9621. )
  9622. return photo_filename
  9623. except Exception as e:
  9624. logger.warning("[PHOTO-BG] Failed: %s", e)
  9625. return None
  9626. finally:
  9627. # #2547: we raised the plate, so we owe the move back down — even if
  9628. # the capture in between threw. Otherwise the user finds the print
  9629. # pinned under the nozzle.
  9630. if park_z is not None:
  9631. try:
  9632. _park_plate_after_finish_photo(printer_id, park_z, logger)
  9633. except Exception as e:
  9634. logger.warning("[PLATE-RESTORE] printer %s: could not lower plate: %s", printer_id, e)
  9635. spawn_background_task(_background_energy_calculation(), name="background-energy-calc")
  9636. # Photo capture task - result will be used by notifications
  9637. photo_task = spawn_background_task(_background_finish_photo(), name="background-finish-photo")
  9638. log_timing("Background tasks scheduled (energy, photo)")
  9639. # Also run smart plug, notifications, and maintenance as background tasks
  9640. print_status = data.get("status", "completed")
  9641. async def _background_smart_plug():
  9642. """Handle smart plug automation in background."""
  9643. try:
  9644. logger.info("[AUTO-OFF-BG] Starting smart plug automation for printer %s", printer_id)
  9645. async with async_session() as db:
  9646. await smart_plug_manager.on_print_complete(printer_id, print_status, db)
  9647. logger.info("[AUTO-OFF-BG] Completed")
  9648. except Exception as e:
  9649. logger.warning("[AUTO-OFF-BG] Failed: %s", e)
  9650. async def _background_notifications(finish_photo_filename: str | None = None):
  9651. """Send print complete notifications in background."""
  9652. try:
  9653. logger.info(
  9654. "[NOTIFY-BG] Starting notifications for printer %s, photo=%s", printer_id, finish_photo_filename
  9655. )
  9656. async with async_session() as db:
  9657. from backend.app.models.archive import PrintArchive
  9658. from backend.app.models.printer import Printer
  9659. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  9660. printer = result.scalar_one_or_none()
  9661. printer_name = printer.name if printer else f"Printer {printer_id}"
  9662. archive_data = None
  9663. if archive_id:
  9664. from backend.app.models.print_log import PrintLogEntry
  9665. archive_result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  9666. archive = archive_result.scalar_one_or_none()
  9667. # The run that ended: the archive shows its latest, which
  9668. # with another plate of it printing elsewhere is not this.
  9669. run = await db.get(PrintLogEntry, completed_run_id) if completed_run_id is not None else None
  9670. run_started = run.started_at if run is not None else archive.started_at if archive else None
  9671. run_completed = run.completed_at if run is not None else archive.completed_at if archive else None
  9672. if archive:
  9673. # Actual elapsed time from started_at/completed_at when both are
  9674. # populated (every terminal status sets completed_at after #1198).
  9675. # Falls back to None so the notification path can decide whether to
  9676. # render the slicer estimate as a last resort.
  9677. actual_time_seconds = None
  9678. # Unknown for a completion reconciled at reconnect: the
  9679. # print ended while the printer was offline and
  9680. # completed_at is the reconnect, so the gap would read
  9681. # as the duration -- 14 h for a night offline (DB-C9).
  9682. # An estimated end (#3310) gives the estimate.
  9683. if (
  9684. run_started
  9685. and run_completed
  9686. and (not data.get("_reconciled") or data.get("_reconciled_completed_at"))
  9687. ):
  9688. elapsed = (run_completed - run_started).total_seconds()
  9689. if elapsed > 0:
  9690. actual_time_seconds = int(elapsed)
  9691. archive_data = {
  9692. "print_time_seconds": archive.print_time_seconds,
  9693. "actual_time_seconds": actual_time_seconds,
  9694. "actual_filament_grams": archive.filament_used_grams,
  9695. "failure_reason": run.failure_reason if run is not None else archive.failure_reason,
  9696. "created_by_id": archive.created_by_id,
  9697. }
  9698. # Scale filament usage for partial prints
  9699. if print_status != "completed" and archive.filament_used_grams:
  9700. progress = data.get("progress") or 0
  9701. scale = _partial_progress_scale(progress)
  9702. archive_data["actual_filament_grams"] = round(archive.filament_used_grams * scale, 1)
  9703. archive_data["progress"] = progress
  9704. # Pass per-slot data from archive.extra_data
  9705. if archive.extra_data and archive.extra_data.get("filament_slots"):
  9706. slots = archive.extra_data["filament_slots"]
  9707. if print_status != "completed":
  9708. scale = _partial_progress_scale(data.get("progress"))
  9709. slots = [{**s, "used_g": round(s["used_g"] * scale, 1)} for s in slots]
  9710. archive_data["filament_slots"] = slots
  9711. # Scope project-summed totals down to the plate that was
  9712. # actually printed — see _scope_notification_archive_data_to_plate
  9713. # for the why (#1785).
  9714. archive_data = _scope_notification_archive_data_to_plate(
  9715. archive_data,
  9716. archive.file_path,
  9717. notify_plate_id,
  9718. print_status,
  9719. data.get("progress"),
  9720. app_settings.base_dir,
  9721. )
  9722. # Enrich filament_grams from usage_results when archive has no 3MF data
  9723. if not archive_data.get("actual_filament_grams") and usage_results:
  9724. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  9725. if total_from_usage > 0:
  9726. archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  9727. # Pass usage tracker results for AMS slot info in notifications
  9728. if usage_results:
  9729. archive_data["usage_results"] = usage_results
  9730. # Which archive and print it was (archive redesign).
  9731. archive_data["archive_id"] = archive_id
  9732. archive_data["run_id"] = completed_run_id
  9733. # Add finish photo URL and image bytes if available
  9734. if finish_photo_filename:
  9735. photo_url, photo_bytes = await _finish_photo_for_notification(
  9736. db, archive, archive_id, finish_photo_filename
  9737. )
  9738. if photo_url:
  9739. archive_data["finish_photo_url"] = photo_url
  9740. if photo_bytes:
  9741. archive_data["image_data"] = photo_bytes
  9742. if not await _kill_switch_notification_already_sent(kill_switch_notification_task):
  9743. await notification_service.on_print_complete(
  9744. printer_id, printer_name, print_status, data, db, archive_data=archive_data
  9745. )
  9746. else:
  9747. logger.info("[NOTIFY-BG] Skipped duplicate kill-switch provider notification")
  9748. # Post-print outcome confirmation (#1898). Runs in this
  9749. # background task so the finish photo fetched above rides
  9750. # along with the prompt.
  9751. if print_status == "completed" and archive_id:
  9752. await _dispatch_outcome_confirmation_safely(
  9753. db, printer_id, printer_name, data, archive_id, archive_data, run_id=completed_run_id
  9754. )
  9755. # Send user-specific email notification
  9756. if archive_data:
  9757. created_by_id = archive_data.get("created_by_id")
  9758. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  9759. await _dispatch_user_print_email(
  9760. print_status,
  9761. created_by_id,
  9762. printer_name,
  9763. raw_filename,
  9764. db,
  9765. image_data=archive_data.get("image_data"),
  9766. finish_photo_url=archive_data.get("finish_photo_url"),
  9767. )
  9768. logger.info("[NOTIFY-BG] Completed")
  9769. except Exception as e:
  9770. logger.error("[NOTIFY-BG] Failed: %s", e, exc_info=True)
  9771. async def _background_maintenance_check():
  9772. """Check for maintenance due in background."""
  9773. if print_status != "completed":
  9774. return
  9775. try:
  9776. logger.info("[MAINT-BG] Starting maintenance check for printer %s", printer_id)
  9777. async with async_session() as db:
  9778. from backend.app.models.printer import Printer
  9779. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  9780. printer = result.scalar_one_or_none()
  9781. printer_name = printer.name if printer else f"Printer {printer_id}"
  9782. await ensure_default_types(db)
  9783. overview = await _get_printer_maintenance_internal(printer_id, db, commit=True)
  9784. attention = [
  9785. item for item in overview.maintenance_items if item.enabled and (item.is_due or item.is_warning)
  9786. ]
  9787. items_needing_attention = [
  9788. {"name": item.maintenance_type_name, "is_due": item.is_due, "is_warning": item.is_warning}
  9789. for item in attention
  9790. ]
  9791. if items_needing_attention:
  9792. await notification_service.on_maintenance_due(printer_id, printer_name, items_needing_attention, db)
  9793. logger.info("[MAINT-BG] Sent notification: %s items need attention", len(items_needing_attention))
  9794. # MQTT relay - publish maintenance alerts
  9795. for item in attention:
  9796. current_value, threshold, unit = maintenance_alert_values(item)
  9797. try:
  9798. await mqtt_relay.on_maintenance_alert(
  9799. printer_id=printer_id,
  9800. printer_name=printer_name,
  9801. maintenance_type=item.maintenance_type_name,
  9802. current_value=current_value,
  9803. threshold=threshold,
  9804. unit=unit,
  9805. )
  9806. except Exception:
  9807. pass # Don't fail if MQTT fails
  9808. else:
  9809. logger.info("[MAINT-BG] Completed (no items need attention)")
  9810. except Exception as e:
  9811. logger.warning("[MAINT-BG] Failed: %s", e)
  9812. spawn_background_task(_background_smart_plug(), name="background-smart-plug")
  9813. spawn_background_task(_background_maintenance_check(), name="background-maintenance-check")
  9814. # Notification task waits for photo capture to complete first (with timeout).
  9815. # When a timelapse was recording, photo sourcing polls the per-print
  9816. # timelapse for up to 60s (#1397) — extend the budget so the notification
  9817. # carries the correct bed-up photo instead of falling through to the
  9818. # live-cam grab. Adds ~30s of notification latency at worst on slow links.
  9819. #
  9820. # #2547: both budgets now have to cover a plate restore as well.
  9821. #
  9822. # Without timelapse, the wait is on the moment producer, which raises the
  9823. # plate before its grab — so this has to outlast that producer's own budget.
  9824. #
  9825. # With timelapse, the capture polls up to
  9826. # `_FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS` for the video and only then
  9827. # falls back to a live grab, which is the case that raises the plate. At the
  9828. # old flat 75s that fallback was guaranteed to be cut off mid-settle, so the
  9829. # restore would have moved the plate for a photo nobody waited for.
  9830. photo_wait_timeout = (
  9831. _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS + _FINISH_PHOTO_PRODUCER_WAIT_SECONDS
  9832. if data.get("timelapse_was_active")
  9833. else _FINISH_PHOTO_PRODUCER_WAIT_SECONDS + 15
  9834. )
  9835. async def _photo_then_notify():
  9836. """Wait for photo capture, then send notification with photo URL."""
  9837. finish_photo = None
  9838. try:
  9839. finish_photo = await asyncio.wait_for(photo_task, timeout=photo_wait_timeout)
  9840. logger.info("[PHOTO-NOTIFY] Photo task returned: %s", finish_photo)
  9841. except TimeoutError:
  9842. logger.warning(
  9843. "[PHOTO-NOTIFY] Photo capture timed out after %ss, sending notification without photo",
  9844. photo_wait_timeout,
  9845. )
  9846. except Exception as e:
  9847. logger.warning("[PHOTO-NOTIFY] Photo task failed: %s", e)
  9848. try:
  9849. await _background_notifications(finish_photo)
  9850. except Exception as e:
  9851. logger.error("[PHOTO-NOTIFY] Notification sending failed: %s", e, exc_info=True)
  9852. spawn_background_task(_photo_then_notify(), name="photo-then-notify")
  9853. # Stitch external camera layer timelapse if session was active
  9854. print_status = data.get("status", "completed")
  9855. async def _background_layer_timelapse():
  9856. """Stitch layer timelapse and attach to archive."""
  9857. from backend.app.services.layer_timelapse import cancel_session, on_print_complete as tl_complete
  9858. try:
  9859. if print_status == "completed":
  9860. logger.info("[LAYER-TL] Stitching layer timelapse for printer %s", printer_id)
  9861. # The run the frames belong to, read before the stitch: it can
  9862. # take long enough for the archive to start printing again.
  9863. run_started_at = None
  9864. if archive_id:
  9865. async with async_session() as db:
  9866. run_archive = await ArchiveService(db).get_archive(archive_id)
  9867. run_started_at = run_archive.started_at if run_archive else None
  9868. timelapse_path = await tl_complete(printer_id)
  9869. if timelapse_path and archive_id:
  9870. await _attach_layer_timelapse(
  9871. archive_id, timelapse_path, notify_plate_id, run_started_at, run_id=completed_run_id
  9872. )
  9873. elif timelapse_path:
  9874. # Timelapse created but no archive - just clean up
  9875. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  9876. else:
  9877. # Print failed or cancelled - cancel timelapse session
  9878. cancel_session(printer_id)
  9879. logger.info(
  9880. "[LAYER-TL] Cancelled layer timelapse for printer %s (status: %s)", printer_id, print_status
  9881. )
  9882. except Exception as e:
  9883. logger.warning("[LAYER-TL] Failed: %s", e)
  9884. # A completed print's session left _active_sessions when its stitch
  9885. # began: the one there now belongs to the next print, which may have
  9886. # started during the stitch, and must not lose its frames.
  9887. if print_status != "completed":
  9888. try:
  9889. cancel_session(printer_id)
  9890. except Exception:
  9891. pass # Best-effort timelapse session cancellation on error
  9892. from backend.app.services import archive_work
  9893. # Held through the stitch, which can take longer than a merge waits after
  9894. # a print (archive redesign P2).
  9895. layer_timelapse = _background_layer_timelapse()
  9896. if archive_id:
  9897. layer_timelapse = archive_work.held_while(archive_id, layer_timelapse)
  9898. spawn_background_task(layer_timelapse, name="background-layer-timelapse")
  9899. log_timing("All background tasks scheduled")
  9900. # Auto-scan for timelapse if recording was active during the print. A
  9901. # failed or cancelled print gets one too: the printer writes the video of
  9902. # the run so far when it stops (measured on an H2C, #3481), and that video
  9903. # is how the user sees what went wrong. A model that writes none costs one
  9904. # poll that finds nothing new.
  9905. if archive_id and data.get("timelapse_was_active"):
  9906. logger.info("[TIMELAPSE] Timelapse was active during print, scheduling auto-scan for archive %s", archive_id)
  9907. # Schedule timelapse scan as background task with retries
  9908. # The printer needs time to encode the video after print completion
  9909. _schedule_timelapse_scan(archive_id, printer_id, plate_id=notify_plate_id, run_id=completed_run_id)
  9910. log_timing("Timelapse scan scheduled")
  9911. logger.info("[CALLBACK] on_print_complete finished for printer %s, archive %s", printer_id, archive_id)
  9912. # AMS sensor history recording
  9913. _ams_history_task: asyncio.Task | None = None
  9914. AMS_HISTORY_INTERVAL = 300 # Record every 5 minutes
  9915. AMS_HISTORY_RETENTION_DAYS = 30 # Keep data for 30 days
  9916. def _retention_days(stored: str | None, default: int) -> int:
  9917. """A history retention setting as days to keep: at least 1, the default
  9918. when unset or unreadable. A stored 0 or less (the API took one before it
  9919. had a minimum) would put the cutoff at now and delete all history."""
  9920. try:
  9921. return max(1, int(stored)) if stored is not None else default
  9922. except (TypeError, ValueError):
  9923. return default
  9924. _ams_cleanup_counter = 0 # Track recordings to trigger periodic cleanup
  9925. # Track alarm cooldowns (printer_id:ams_id:type -> last_alarm_time)
  9926. _ams_alarm_cooldown: dict[str, datetime] = {}
  9927. AMS_ALARM_COOLDOWN_MINUTES = 60 # Don't send same alarm more than once per hour
  9928. # (printer_id, ams_id) already reported as sending the drop index and no
  9929. # percentage. Logged once each so a supported printer that turns out to do this
  9930. # shows up in a support bundle rather than as a user wondering where the
  9931. # humidity reading went -- see the note at the read site below (#3140).
  9932. _ams_index_only_logged: set[tuple[int, int]] = set()
  9933. def _resolve_temp_alarm_threshold(fair_threshold: float, raw_alarm_value: str | None) -> float:
  9934. """Temperature at which the AMS alarm fires, falling back to the display band.
  9935. ``ams_temp_fair`` decides when the AMS card turns amber. It used to decide
  9936. when a notification was sent as well, which is why a room above it made the
  9937. alarm fire once an hour for as long as the weather lasted -- and the only way
  9938. to stop that was to raise the display band and lose the colour that says the
  9939. unit is warm (#2905).
  9940. Unset resolves to the fair threshold, so an install that never sets one is
  9941. unchanged. Settings storage stringifies ``None`` to the literal ``"None"``,
  9942. so that arrives here as a string and is handled by the same branch as any
  9943. other unparseable value -- there is no separate sentinel to keep in sync.
  9944. A non-positive value is refused rather than honoured: zero would alarm
  9945. permanently, and it is far more likely to be a cleared field than a
  9946. deliberate choice.
  9947. """
  9948. if raw_alarm_value is None:
  9949. return fair_threshold
  9950. try:
  9951. value = float(raw_alarm_value)
  9952. except (TypeError, ValueError):
  9953. return fair_threshold
  9954. if not math.isfinite(value) or value <= 0:
  9955. return fair_threshold
  9956. return value
  9957. # Per-AMS "drying was live at" latch that suppresses the high-temperature alarm
  9958. # through a cycle and the cool-down after it (#1802). Stored in the settings
  9959. # table rather than alongside _ams_alarm_cooldown above, because a restart
  9960. # partway through a cool-down would otherwise resume alarming about heat the
  9961. # user asked for — the same internal-timestamp-row pattern as
  9962. # support.py's debug_logging_enabled_at.
  9963. AMS_DRYING_LATCH_KEY = "ams_drying_alarm_latch"
  9964. # Upper bound on that suppression. The latch normally clears as soon as the unit
  9965. # reads at or below the threshold; see utils.ams_drying for why this cap only
  9966. # matters when it never does.
  9967. AMS_DRYING_GRACE_MINUTES = 120
  9968. async def _load_ams_drying_latch(db) -> dict[str, datetime]:
  9969. """Read the persisted per-AMS drying latch, dropping entries out of window.
  9970. Anything older than the grace cap would expire on its next visit anyway, so
  9971. discarding it here costs nothing and stops rows for deleted printers from
  9972. accumulating.
  9973. Stamps ahead of now get two defences, because a box whose clock jumps
  9974. backwards (a Pi with no RTC coming up before NTP) writes them: wildly future
  9975. ones are discarded outright, and the rest are clamped to now. Without the
  9976. clamp the cap would measure from a moment that has not happened yet and hold
  9977. the alarm quiet for the skew on top of the cap. One unnecessary notification
  9978. after a clock jump is a far better failure than an alarm silently disabled
  9979. for hours.
  9980. """
  9981. from backend.app.models.settings import Settings
  9982. result = await db.execute(select(Settings).where(Settings.key == AMS_DRYING_LATCH_KEY))
  9983. setting = result.scalar_one_or_none()
  9984. if not setting or not setting.value:
  9985. return {}
  9986. try:
  9987. raw = json.loads(setting.value)
  9988. except (ValueError, TypeError):
  9989. return {} # Corrupted row → no latch, alarms behave as they did before
  9990. if not isinstance(raw, dict):
  9991. return {}
  9992. now = datetime.now(timezone.utc)
  9993. window = timedelta(minutes=AMS_DRYING_GRACE_MINUTES)
  9994. latch: dict[str, datetime] = {}
  9995. for key, value in raw.items():
  9996. try:
  9997. stamp = datetime.fromisoformat(str(value))
  9998. except (ValueError, TypeError):
  9999. continue
  10000. if stamp.tzinfo is None:
  10001. stamp = stamp.replace(tzinfo=timezone.utc)
  10002. if not (now - window <= stamp <= now + window):
  10003. continue
  10004. # Nothing may sit in the future: suppression is measured as now minus
  10005. # the stamp, so a stamp ahead of now would extend it by the skew on top
  10006. # of the cap. Clamping the survivors keeps the cap an actual cap.
  10007. latch[str(key)] = min(stamp, now)
  10008. return latch
  10009. async def _save_ams_drying_latch(db, latch: dict[str, datetime]) -> None:
  10010. """Persist the latch, writing only when it actually changed.
  10011. Adds the session change but does not commit — the caller's own commit
  10012. carries it, so the latch lands in the same transaction as the sensor rows
  10013. that produced it.
  10014. """
  10015. from backend.app.models.settings import Settings
  10016. payload = json.dumps({key: stamp.isoformat() for key, stamp in sorted(latch.items())})
  10017. result = await db.execute(select(Settings).where(Settings.key == AMS_DRYING_LATCH_KEY))
  10018. setting = result.scalar_one_or_none()
  10019. if setting is None:
  10020. # Don't create the row on installs that never dry anything.
  10021. if payload != "{}":
  10022. db.add(Settings(key=AMS_DRYING_LATCH_KEY, value=payload))
  10023. elif setting.value != payload:
  10024. setting.value = payload
  10025. def _ams_has_filament(ams_data: dict) -> bool:
  10026. """True if this AMS unit has at least one tray slot holding filament.
  10027. Bambu firmware reports loaded slots via `tray_exist_bits`, a per-AMS hex
  10028. bitmap (one bit per tray slot — bit set = spool present). Empty AMS units
  10029. still report sensor readings, but those readings are ambient and not
  10030. actionable: no filament to dry, no humidity to push down. #1619 — gate
  10031. humidity/temperature alarms on this check so empty units don't generate
  10032. hourly noise. Sensor history still records regardless so the UI charts
  10033. stay continuous.
  10034. Fallback path inspects the `tray` array's `tray_type` fields for setups
  10035. where `tray_exist_bits` is missing (some early-connection pushall shapes).
  10036. """
  10037. bits = ams_data.get("tray_exist_bits")
  10038. if isinstance(bits, str) and bits.strip():
  10039. try:
  10040. return int(bits, 16) > 0
  10041. except ValueError:
  10042. pass
  10043. trays = ams_data.get("tray")
  10044. if isinstance(trays, list):
  10045. return any(
  10046. isinstance(t, dict) and isinstance(t.get("tray_type"), str) and t["tray_type"].strip() for t in trays
  10047. )
  10048. return False
  10049. async def record_ams_history():
  10050. """Background task to record AMS humidity and temperature data."""
  10051. logger = logging.getLogger(__name__)
  10052. # Wait a short time for MQTT connections to establish on startup
  10053. await asyncio.sleep(10)
  10054. while True:
  10055. try:
  10056. from backend.app.models.ams_history import AMSSensorHistory
  10057. from backend.app.models.printer import Printer
  10058. from backend.app.models.settings import Settings
  10059. async with async_session() as db:
  10060. # Get all active printers
  10061. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  10062. printers = result.scalars().all()
  10063. # Get alarm thresholds from settings
  10064. humidity_threshold = 60.0 # Default: fair threshold
  10065. temp_fair_threshold = 35.0 # Display band default (ams_temp_fair)
  10066. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  10067. setting = result.scalar_one_or_none()
  10068. if setting:
  10069. try:
  10070. humidity_threshold = float(setting.value)
  10071. except (ValueError, TypeError):
  10072. pass # Keep default threshold if stored value is invalid
  10073. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_fair"))
  10074. setting = result.scalar_one_or_none()
  10075. if setting:
  10076. try:
  10077. temp_fair_threshold = float(setting.value)
  10078. except (ValueError, TypeError):
  10079. pass # Keep default threshold if stored value is invalid
  10080. # The alarm gets its own threshold, seeded from the resolved fair
  10081. # value so an install that has never set one behaves exactly as
  10082. # it did before (#2905). ams_temp_fair decides when the card turns
  10083. # amber; 35 C is a reasonable place to change a colour and not a
  10084. # reasonable place to page someone. A room above it makes the
  10085. # alarm fire once an hour for as long as the weather lasts, and
  10086. # the only way to stop it was to raise the display band and lose
  10087. # the colour that says the unit is warm.
  10088. #
  10089. # An unset value is stored as the literal "None", which the except
  10090. # below swallows the same way it swallows garbage -- so the
  10091. # fallback costs nothing and needs no sentinel of its own.
  10092. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_alarm"))
  10093. setting = result.scalar_one_or_none()
  10094. temp_alarm_threshold = _resolve_temp_alarm_threshold(
  10095. temp_fair_threshold, setting.value if setting else None
  10096. )
  10097. # Per-filament humidity threshold overrides (#1605) — resolved
  10098. # per-AMS below from the loaded tray types. Reuses the same
  10099. # resolver as the auto-drying scheduler so behavior stays in
  10100. # lockstep across both consumers.
  10101. from backend.app.services.print_scheduler import PrintScheduler
  10102. per_type_humidity_thresholds: dict[str, int] = {}
  10103. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_thresholds"))
  10104. setting = result.scalar_one_or_none()
  10105. if setting and setting.value:
  10106. try:
  10107. raw = json.loads(setting.value)
  10108. if isinstance(raw, dict):
  10109. for k, v in raw.items():
  10110. try:
  10111. per_type_humidity_thresholds[str(k).upper() if k != "default" else "default"] = int(
  10112. v
  10113. )
  10114. except (TypeError, ValueError):
  10115. continue
  10116. except (ValueError, TypeError):
  10117. pass # Invalid JSON → no overrides, fall through to global threshold
  10118. # Per-AMS drying latch (#1802), loaded once per pass and written
  10119. # back below only if a unit changed it.
  10120. drying_latch = await _load_ams_drying_latch(db)
  10121. drying_latch_before = dict(drying_latch)
  10122. recorded_count = 0
  10123. for printer in printers:
  10124. # Get current state from printer manager
  10125. state = printer_manager.get_status(printer.id)
  10126. if not state or not state.connected or not state.raw_data:
  10127. continue # Skip disconnected printers - don't use stale data
  10128. raw_data = state.raw_data
  10129. if "ams" not in raw_data or not isinstance(raw_data["ams"], list):
  10130. continue
  10131. # Record data for each AMS unit
  10132. for ams_data in raw_data["ams"]:
  10133. ams_id = int(ams_data.get("id", 0))
  10134. # Percentage only. The 1-5 index is inverted, so
  10135. # charting it as a percentage drew the wettest units as
  10136. # the driest (#3140); a unit that reports no percentage
  10137. # leaves a gap in the chart instead. See
  10138. # utils/ams_humidity.
  10139. humidity = ams_humidity_percent(ams_data)
  10140. # No supported printer is known to send the index
  10141. # alone -- the report came from unsupported firmware,
  10142. # and no install has been seen using the old fallback.
  10143. # "Known" is doing work there, so say so once per unit:
  10144. # the alternative is a silent blank card.
  10145. if humidity is None and ams_data.get("humidity") is not None:
  10146. unit_key = (printer.id, ams_id)
  10147. if unit_key not in _ams_index_only_logged:
  10148. _ams_index_only_logged.add(unit_key)
  10149. logger.info(
  10150. "[%s] AMS %d reports the 1-5 humidity index but no usable humidity_raw "
  10151. "percentage. The index is inverted and is not shown as a percentage "
  10152. "(#3140), so this unit has no humidity reading, chart or alarm. "
  10153. "Please report this with the printer and AMS firmware versions.",
  10154. printer.name,
  10155. ams_id,
  10156. )
  10157. # Get temperature
  10158. temperature = None
  10159. temp_str = ams_data.get("temp")
  10160. if temp_str is not None:
  10161. try:
  10162. temperature = float(temp_str)
  10163. except (ValueError, TypeError):
  10164. pass # Skip unparseable temperature value
  10165. # Skip if no data
  10166. if humidity is None and temperature is None:
  10167. continue
  10168. # Record the data point
  10169. history = AMSSensorHistory(
  10170. printer_id=printer.id,
  10171. ams_id=ams_id,
  10172. humidity=humidity,
  10173. # Both columns hold the same reading now that the
  10174. # index can no longer reach ``humidity``. Writing it
  10175. # through the same value also stops a genuine 0%
  10176. # from being stored as NULL, which the old truthiness
  10177. # test did.
  10178. humidity_raw=humidity,
  10179. temperature=temperature,
  10180. )
  10181. db.add(history)
  10182. recorded_count += 1
  10183. # Generate AMS label and determine if it's AMS-HT (A, B, C, D or HT-A for AMS-Lite/Hub)
  10184. ams_type = unit_ams_type(ams_id, ams_data.get("info"))
  10185. is_ams_ht = ams_type == AmsType.N3S
  10186. ams_label = unit_label(ams_id, ams_type)
  10187. # Skip alarm dispatch for empty AMS units — humidity /
  10188. # temperature readings are ambient with no filament to
  10189. # protect, and the hourly notification just becomes
  10190. # noise. Sensor history was already recorded above so
  10191. # the UI charts stay continuous (#1619). Per-AMS check
  10192. # so a multi-AMS setup with one loaded + one empty
  10193. # still alarms on the loaded unit.
  10194. if not _ams_has_filament(ams_data):
  10195. continue
  10196. # Resolve per-filament humidity threshold for this AMS
  10197. # unit (#1605). Falls back to the global ams_humidity_fair
  10198. # when no per-type overrides are configured.
  10199. trays = ams_data.get("tray", []) or []
  10200. effective_humidity_threshold = float(
  10201. PrintScheduler.resolve_humidity_threshold(
  10202. trays, per_type_humidity_thresholds, int(humidity_threshold)
  10203. )
  10204. )
  10205. # Check humidity alarm (only if above threshold)
  10206. if humidity is not None and humidity > effective_humidity_threshold:
  10207. cooldown_key = f"{printer.id}:{ams_id}:humidity"
  10208. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  10209. now = datetime.now(timezone.utc)
  10210. if (
  10211. last_alarm is None
  10212. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  10213. ):
  10214. _ams_alarm_cooldown[cooldown_key] = now
  10215. logger.info(
  10216. f"Sending humidity alarm for {printer.name} {ams_label}: {humidity}% > {effective_humidity_threshold}%"
  10217. )
  10218. try:
  10219. # Call different notification method based on AMS type
  10220. if is_ams_ht:
  10221. await notification_service.on_ams_ht_humidity_high(
  10222. printer.id,
  10223. printer.name,
  10224. ams_label,
  10225. humidity,
  10226. effective_humidity_threshold,
  10227. db,
  10228. )
  10229. else:
  10230. await notification_service.on_ams_humidity_high(
  10231. printer.id,
  10232. printer.name,
  10233. ams_label,
  10234. humidity,
  10235. effective_humidity_threshold,
  10236. db,
  10237. )
  10238. except Exception as e:
  10239. logger.warning("Failed to send humidity alarm: %s", e)
  10240. # A drying cycle heats the unit far past ams_temp_fair on
  10241. # purpose — 45 C for PLA, 65 C for PETG, 85 C on an
  10242. # AMS-HT, against a 35 C default — so the alarm fired
  10243. # once an hour for the whole cycle and kept firing while
  10244. # the unit cooled back down (#1802). Latch on the
  10245. # firmware's own drying state and hold until the reading
  10246. # returns to normal. Humidity is deliberately left alone:
  10247. # it falls during drying, which is the whole point.
  10248. latch_key = f"{printer.id}:{ams_id}"
  10249. # The latch releases at `threshold`, so it takes the alarm
  10250. # number too. Handing it the display band would strand the
  10251. # latch on any unit that settles back above it -- a room
  10252. # where the AMS rests at 37.7 C never returns under a 35 C
  10253. # band, so the latch could only expire on the grace cap
  10254. # rather than releasing when the unit had actually cooled.
  10255. suppress_temp_alarm, new_latch = temperature_alarm_suppressed(
  10256. drying_active=is_drying_active(ams_data),
  10257. temperature=temperature,
  10258. threshold=temp_alarm_threshold,
  10259. latched_at=drying_latch.get(latch_key),
  10260. now=datetime.now(timezone.utc),
  10261. grace_minutes=AMS_DRYING_GRACE_MINUTES,
  10262. )
  10263. if new_latch is None:
  10264. drying_latch.pop(latch_key, None)
  10265. else:
  10266. drying_latch[latch_key] = new_latch
  10267. # Check temperature alarm (only if above threshold)
  10268. if temperature is not None and temperature > temp_alarm_threshold and not suppress_temp_alarm:
  10269. cooldown_key = f"{printer.id}:{ams_id}:temperature"
  10270. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  10271. now = datetime.now(timezone.utc)
  10272. if (
  10273. last_alarm is None
  10274. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  10275. ):
  10276. _ams_alarm_cooldown[cooldown_key] = now
  10277. logger.info(
  10278. f"Sending temperature alarm for {printer.name} {ams_label}: "
  10279. f"{temperature}°C > {temp_alarm_threshold}°C"
  10280. )
  10281. try:
  10282. # Call different notification method based on AMS type
  10283. if is_ams_ht:
  10284. # The reported threshold has to be the one
  10285. # that fired, or the message says "> 35 °C"
  10286. # while firing at 45.
  10287. await notification_service.on_ams_ht_temperature_high(
  10288. printer.id, printer.name, ams_label, temperature, temp_alarm_threshold, db
  10289. )
  10290. else:
  10291. await notification_service.on_ams_temperature_high(
  10292. printer.id, printer.name, ams_label, temperature, temp_alarm_threshold, db
  10293. )
  10294. except Exception as e:
  10295. logger.warning("Failed to send temperature alarm: %s", e)
  10296. if drying_latch != drying_latch_before:
  10297. await _save_ams_drying_latch(db, drying_latch)
  10298. await db.commit()
  10299. if recorded_count > 0:
  10300. logger.info("Recorded %s AMS sensor history entries", recorded_count)
  10301. # Periodic cleanup of old data (every ~288 recordings = ~24 hours at 5min interval)
  10302. global _ams_cleanup_counter
  10303. _ams_cleanup_counter += 1
  10304. if _ams_cleanup_counter >= 288:
  10305. _ams_cleanup_counter = 0
  10306. # Get retention days from settings
  10307. from backend.app.models.settings import Settings
  10308. result = await db.execute(select(Settings).where(Settings.key == "ams_history_retention_days"))
  10309. setting = result.scalar_one_or_none()
  10310. retention_days = _retention_days(setting.value if setting else None, AMS_HISTORY_RETENTION_DAYS)
  10311. cutoff = utcnow_naive() - timedelta(days=retention_days)
  10312. result = await db.execute(delete(AMSSensorHistory).where(AMSSensorHistory.recorded_at < cutoff))
  10313. await db.commit()
  10314. if result.rowcount > 0:
  10315. logger.info(
  10316. f"Cleaned up {result.rowcount} old AMS sensor history entries (older than {retention_days} days)"
  10317. )
  10318. # Wait until next recording interval
  10319. await asyncio.sleep(AMS_HISTORY_INTERVAL)
  10320. except asyncio.CancelledError:
  10321. break
  10322. except Exception as e:
  10323. logger.warning("AMS history recording failed: %s", e)
  10324. await asyncio.sleep(60) # Wait a bit before retrying
  10325. def start_ams_history_recording():
  10326. """Start the AMS history recording background task."""
  10327. global _ams_history_task
  10328. if _ams_history_task is None:
  10329. _ams_history_task = asyncio.create_task(record_ams_history())
  10330. logging.getLogger(__name__).info("AMS history recording started")
  10331. def stop_ams_history_recording():
  10332. """Stop the AMS history recording background task."""
  10333. global _ams_history_task
  10334. if _ams_history_task:
  10335. _ams_history_task.cancel()
  10336. _ams_history_task = None
  10337. logging.getLogger(__name__).info("AMS history recording stopped")
  10338. # Printer sensor history recording (nozzle / bed / chamber)
  10339. _printer_sensor_history_task: asyncio.Task | None = None
  10340. PRINTER_SENSOR_HISTORY_INTERVAL = 60 # Record every minute — heaters move faster than AMS humidity
  10341. PRINTER_SENSOR_HISTORY_RETENTION_DAYS = 30
  10342. _printer_sensor_cleanup_counter = 0
  10343. # Sensor kinds tracked in state.temperatures — these are the normalised keys the
  10344. # MQTT parser writes, so we don't need to handle per-model field aliases here
  10345. # (nozzle_temper / left_nozzle_temper / right_nozzle_temper / chamber_temper
  10346. # are all collapsed by services/bambu_mqtt.py before they reach this loop).
  10347. _SENSOR_KINDS = ("nozzle", "nozzle_2", "bed", "chamber")
  10348. _SENSOR_TARGET_KEYS = {
  10349. "nozzle": "nozzle_target",
  10350. "nozzle_2": "nozzle_2_target",
  10351. "bed": "bed_target",
  10352. "chamber": "chamber_target",
  10353. }
  10354. async def record_printer_sensor_history():
  10355. """Background task to record nozzle / bed / chamber readings.
  10356. Pulls from `state.temperatures` (already normalised across all printer
  10357. models by the MQTT parser) rather than re-parsing raw_data, so we get
  10358. free coverage of dual-nozzle H2D, sensor-only X1C chamber, etc.
  10359. """
  10360. logger = logging.getLogger(__name__)
  10361. await asyncio.sleep(10)
  10362. while True:
  10363. try:
  10364. from backend.app.models.printer import Printer
  10365. from backend.app.models.printer_sensor_history import PrinterSensorHistory
  10366. from backend.app.models.settings import Settings
  10367. async with async_session() as db:
  10368. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  10369. printers = result.scalars().all()
  10370. recorded_count = 0
  10371. for printer in printers:
  10372. state = printer_manager.get_status(printer.id)
  10373. if not state or not state.connected:
  10374. continue
  10375. temps = getattr(state, "temperatures", None) or {}
  10376. if not isinstance(temps, dict):
  10377. continue
  10378. for kind in _SENSOR_KINDS:
  10379. if kind not in temps:
  10380. continue
  10381. try:
  10382. value = float(temps[kind])
  10383. except (ValueError, TypeError):
  10384. continue
  10385. target_raw = temps.get(_SENSOR_TARGET_KEYS[kind])
  10386. target_val: float | None = None
  10387. if target_raw is not None:
  10388. try:
  10389. target_val = float(target_raw)
  10390. except (ValueError, TypeError):
  10391. target_val = None
  10392. db.add(
  10393. PrinterSensorHistory(
  10394. printer_id=printer.id,
  10395. sensor_kind=kind,
  10396. value=value,
  10397. target=target_val,
  10398. )
  10399. )
  10400. recorded_count += 1
  10401. await db.commit()
  10402. if recorded_count > 0:
  10403. logger.debug("Recorded %s printer sensor history entries", recorded_count)
  10404. # Periodic cleanup — once every ~24h at this interval.
  10405. global _printer_sensor_cleanup_counter
  10406. _printer_sensor_cleanup_counter += 1
  10407. cleanup_every = max(1, (24 * 60 * 60) // PRINTER_SENSOR_HISTORY_INTERVAL)
  10408. if _printer_sensor_cleanup_counter >= cleanup_every:
  10409. _printer_sensor_cleanup_counter = 0
  10410. result = await db.execute(
  10411. select(Settings).where(Settings.key == "printer_sensor_history_retention_days")
  10412. )
  10413. setting = result.scalar_one_or_none()
  10414. retention_days = _retention_days(
  10415. setting.value if setting else None, PRINTER_SENSOR_HISTORY_RETENTION_DAYS
  10416. )
  10417. cutoff = utcnow_naive() - timedelta(days=retention_days)
  10418. cleanup = await db.execute(
  10419. delete(PrinterSensorHistory).where(PrinterSensorHistory.recorded_at < cutoff)
  10420. )
  10421. await db.commit()
  10422. if cleanup.rowcount > 0:
  10423. logger.info(
  10424. "Cleaned up %s old printer sensor history entries (older than %s days)",
  10425. cleanup.rowcount,
  10426. retention_days,
  10427. )
  10428. await asyncio.sleep(PRINTER_SENSOR_HISTORY_INTERVAL)
  10429. except asyncio.CancelledError:
  10430. break
  10431. except Exception as e:
  10432. logger.warning("Printer sensor history recording failed: %s", e)
  10433. await asyncio.sleep(60)
  10434. def start_printer_sensor_history_recording():
  10435. global _printer_sensor_history_task
  10436. if _printer_sensor_history_task is None:
  10437. _printer_sensor_history_task = asyncio.create_task(record_printer_sensor_history())
  10438. logging.getLogger(__name__).info("Printer sensor history recording started")
  10439. def stop_printer_sensor_history_recording():
  10440. global _printer_sensor_history_task
  10441. if _printer_sensor_history_task:
  10442. _printer_sensor_history_task.cancel()
  10443. _printer_sensor_history_task = None
  10444. logging.getLogger(__name__).info("Printer sensor history recording stopped")
  10445. # Printer runtime tracking
  10446. _runtime_tracking_task: asyncio.Task | None = None
  10447. RUNTIME_TRACKING_INTERVAL = 30 # Update every 30 seconds
  10448. # Printers the runtime tracker has seen connected since startup. One of them
  10449. # going offline clears its timestamp, so the time it was away is not counted
  10450. # when it comes back printing (DB-C5). A printer not yet seen -- still
  10451. # connecting after a restart -- keeps it for a grace period: a print that ran
  10452. # on through a Bambuddy restart is counted from the stored timestamp, as before.
  10453. _runtime_seen_connected: set[int] = set()
  10454. # A printer never seen connected this long after the tracker started is
  10455. # offline, not still connecting: its stored timestamp predates a gap too.
  10456. RUNTIME_STARTUP_GRACE_SECONDS = 120
  10457. async def track_printer_runtime():
  10458. """Background task to track printer active runtime (RUNNING state only).
  10459. PAUSE is intentionally excluded — the runtime counter feeds hours-based
  10460. maintenance intervals (rod lubrication, belt checks, nozzle cleaning)
  10461. which track mechanical wear. Pause time has no motion and no wear, so
  10462. counting it inflates maintenance warnings (#1521).
  10463. """
  10464. logger = logging.getLogger(__name__)
  10465. # Wait for MQTT connections to establish on startup
  10466. await asyncio.sleep(15)
  10467. started = time.monotonic()
  10468. while True:
  10469. try:
  10470. from backend.app.models.printer import Printer
  10471. # Fetch printer IDs in a short-lived read-only session
  10472. async with async_session() as db:
  10473. result = await db.execute(
  10474. select(Printer.id, Printer.name, Printer.runtime_seconds, Printer.last_runtime_update).where(
  10475. Printer.is_active.is_(True)
  10476. )
  10477. )
  10478. printer_rows = result.all()
  10479. now = datetime.now(timezone.utc)
  10480. updated_count = 0
  10481. # Update each printer in its own short session to minimise write-lock
  10482. # hold time and avoid blocking critical commits like queue status
  10483. # updates (#897).
  10484. for pid, pname, runtime_secs, last_update in printer_rows:
  10485. state = printer_manager.get_status(pid)
  10486. if not state:
  10487. logger.debug("[%s] Runtime tracking: no state available", pname)
  10488. continue
  10489. if not state.connected:
  10490. logger.debug("[%s] Runtime tracking: not connected", pname)
  10491. offline = pid in _runtime_seen_connected or (
  10492. time.monotonic() - started > RUNTIME_STARTUP_GRACE_SECONDS
  10493. )
  10494. if offline and last_update is not None:
  10495. try:
  10496. async with async_session() as db:
  10497. printer = await db.get(Printer, pid)
  10498. if printer:
  10499. printer.last_runtime_update = None
  10500. await db.commit()
  10501. except Exception as e:
  10502. # Still marked: the next offline tick tries again.
  10503. logger.warning("[%s] Runtime tracking commit failed: %s", pname, e)
  10504. continue
  10505. _runtime_seen_connected.discard(pid)
  10506. continue
  10507. _runtime_seen_connected.add(pid)
  10508. needs_commit = False
  10509. new_runtime = runtime_secs
  10510. new_last_update = last_update
  10511. if state.state == "RUNNING":
  10512. if last_update:
  10513. lu = last_update if last_update.tzinfo else last_update.replace(tzinfo=timezone.utc)
  10514. elapsed = (now - lu).total_seconds()
  10515. if elapsed > 0:
  10516. new_runtime = runtime_secs + int(elapsed)
  10517. updated_count += 1
  10518. needs_commit = True
  10519. logger.debug(
  10520. f"[{pname}] Runtime tracking: added {int(elapsed)}s, "
  10521. f"total={new_runtime}s ({new_runtime / 3600:.2f}h)"
  10522. )
  10523. else:
  10524. needs_commit = True
  10525. logger.debug("[%s] Runtime tracking: first active detection", pname)
  10526. new_last_update = now
  10527. else:
  10528. if last_update is not None:
  10529. logger.debug(f"[{pname}] Runtime tracking: state={state.state}, clearing last_runtime_update")
  10530. new_last_update = None
  10531. needs_commit = True
  10532. if needs_commit:
  10533. try:
  10534. async with async_session() as db:
  10535. result = await db.execute(select(Printer).where(Printer.id == pid))
  10536. printer = result.scalar_one_or_none()
  10537. if printer:
  10538. printer.runtime_seconds = new_runtime
  10539. printer.last_runtime_update = new_last_update
  10540. await db.commit()
  10541. except Exception as e:
  10542. logger.warning("[%s] Runtime tracking commit failed: %s", pname, e)
  10543. if updated_count > 0:
  10544. logger.debug("Updated runtime for %s printer(s)", updated_count)
  10545. except asyncio.CancelledError:
  10546. logger.info("Runtime tracking cancelled")
  10547. break
  10548. except Exception as e:
  10549. logger.warning("Runtime tracking failed: %s", e)
  10550. await asyncio.sleep(RUNTIME_TRACKING_INTERVAL)
  10551. def start_runtime_tracking():
  10552. """Start the printer runtime tracking background task."""
  10553. global _runtime_tracking_task
  10554. if _runtime_tracking_task is None:
  10555. _runtime_tracking_task = asyncio.create_task(track_printer_runtime())
  10556. logging.getLogger(__name__).info("Printer runtime tracking started")
  10557. def stop_runtime_tracking():
  10558. """Stop the printer runtime tracking background task."""
  10559. global _runtime_tracking_task
  10560. if _runtime_tracking_task:
  10561. _runtime_tracking_task.cancel()
  10562. _runtime_tracking_task = None
  10563. logging.getLogger(__name__).info("Printer runtime tracking stopped")
  10564. # SpoolBuddy device watchdog
  10565. _spoolbuddy_watchdog_task: asyncio.Task | None = None
  10566. SPOOLBUDDY_WATCHDOG_INTERVAL = 15
  10567. async def _spoolbuddy_watchdog_loop():
  10568. """Periodic check for SpoolBuddy devices that have gone offline."""
  10569. from backend.app.api.routes.spoolbuddy import spoolbuddy_watchdog
  10570. while True:
  10571. try:
  10572. await spoolbuddy_watchdog()
  10573. except asyncio.CancelledError:
  10574. break
  10575. except Exception as e:
  10576. logging.getLogger(__name__).warning("SpoolBuddy watchdog failed: %s", e)
  10577. await asyncio.sleep(SPOOLBUDDY_WATCHDOG_INTERVAL)
  10578. def start_spoolbuddy_watchdog():
  10579. global _spoolbuddy_watchdog_task
  10580. if _spoolbuddy_watchdog_task is None:
  10581. _spoolbuddy_watchdog_task = asyncio.create_task(_spoolbuddy_watchdog_loop())
  10582. logging.getLogger(__name__).info("SpoolBuddy watchdog started")
  10583. def stop_spoolbuddy_watchdog():
  10584. global _spoolbuddy_watchdog_task
  10585. if _spoolbuddy_watchdog_task:
  10586. _spoolbuddy_watchdog_task.cancel()
  10587. _spoolbuddy_watchdog_task = None
  10588. logging.getLogger(__name__).info("SpoolBuddy watchdog stopped")
  10589. # Dead-MQTT-session recovery
  10590. #
  10591. # check_staleness() covers the "connected but silent" half-broken session. It
  10592. # does nothing once ``state.connected`` is False, and paho's own auto-reconnect
  10593. # is the only thing left watching at that point. When paho stops making
  10594. # progress there is no backstop at all: the #2732 bundle has a P1S drop on a
  10595. # keep-alive timeout at 02:19 and not reconnect until 11:24 — nine hours
  10596. # offline with the UI open the whole time, recovered only when something
  10597. # happened to nudge it.
  10598. #
  10599. # This loop is that backstop. It only touches printers that had a working
  10600. # session and lost it, and only when the MQTT port still answers — a printer
  10601. # that is simply switched off is left to paho, since rebuilding a client
  10602. # against an unreachable host achieves nothing and would fill the log every
  10603. # night.
  10604. _connection_watchdog_task: asyncio.Task | None = None
  10605. CONNECTION_WATCHDOG_INTERVAL = 60
  10606. # How long a printer must have been silent before we stop trusting paho.
  10607. # Comfortably above STALE_TIMEOUT (60 s) and the max reconnect backoff (30 s),
  10608. # so a session that is recovering on its own is never interrupted.
  10609. CONNECTION_WATCHDOG_OFFLINE_GRACE = 300
  10610. # Per-printer floor between rebuild attempts.
  10611. CONNECTION_WATCHDOG_RETRY_INTERVAL = 300
  10612. _connection_watchdog_last_attempt: dict[int, float] = {}
  10613. async def _recover_dead_printer_sessions() -> int:
  10614. """Rebuild MQTT clients that have been offline too long to still be trying.
  10615. Returns the number of printers a rebuild was attempted for (for tests and
  10616. for the caller's logging). Never raises: one unreachable printer must not
  10617. stop the sweep for the rest of the farm.
  10618. """
  10619. logger = logging.getLogger(__name__)
  10620. from backend.app.services.printer_diagnostic import PORT_MQTT, check_port
  10621. now = time.monotonic()
  10622. recovered = 0
  10623. for printer_id, client in list(printer_manager._clients.items()):
  10624. try:
  10625. if client.state.connected:
  10626. _connection_watchdog_last_attempt.pop(printer_id, None)
  10627. continue
  10628. # Time since the last inbound message is the age of the last known
  10629. # good session — no extra bookkeeping needed, and it is the same
  10630. # clock is_stale() reads. 0 means this client has never had one:
  10631. # that is the initial-connect path, where paho retrying is the
  10632. # correct and only behaviour, so leave it be.
  10633. last_msg = client._last_message_time
  10634. if not last_msg:
  10635. continue
  10636. offline_for = time.time() - last_msg
  10637. if offline_for < CONNECTION_WATCHDOG_OFFLINE_GRACE:
  10638. continue
  10639. last_attempt = _connection_watchdog_last_attempt.get(printer_id)
  10640. if last_attempt is not None and now - last_attempt < CONNECTION_WATCHDOG_RETRY_INTERVAL:
  10641. continue
  10642. if not await check_port(client.ip_address, PORT_MQTT):
  10643. # Switched off, unplugged, or off the network. Paho's retry is
  10644. # the right handler; say so at debug level and move on.
  10645. logger.debug(
  10646. "[#2732] Printer %s offline for %.0fs and its MQTT port is not answering "
  10647. "— leaving the reconnect to paho",
  10648. printer_id,
  10649. offline_for,
  10650. )
  10651. _connection_watchdog_last_attempt[printer_id] = now
  10652. continue
  10653. _connection_watchdog_last_attempt[printer_id] = now
  10654. recovered += 1
  10655. logger.warning(
  10656. "[#2732] Printer %s has been offline for %.0fs but answers on MQTT port %d — "
  10657. "rebuilding the client with a fresh session (last connect error: %s)",
  10658. printer_id,
  10659. offline_for,
  10660. PORT_MQTT,
  10661. client.last_connect_error or "none recorded",
  10662. )
  10663. # Async context, so this takes the hard-reset path: fresh client_id,
  10664. # paho's QoS 1 queue dropped. That matters — a project_file left
  10665. # unacked on the dead session would otherwise replay into the new
  10666. # one and trip 0500_4003 on the printer (#1136).
  10667. client.force_reconnect_stale_session(f"offline for {offline_for:.0f}s, port still answering")
  10668. except Exception as e:
  10669. logger.warning("[#2732] Connection watchdog failed for printer %s: %s", printer_id, e)
  10670. return recovered
  10671. async def _connection_watchdog_loop():
  10672. logger = logging.getLogger(__name__)
  10673. # Let the initial connects settle before judging anyone offline.
  10674. await asyncio.sleep(CONNECTION_WATCHDOG_OFFLINE_GRACE)
  10675. while True:
  10676. try:
  10677. await _recover_dead_printer_sessions()
  10678. except asyncio.CancelledError:
  10679. break
  10680. except Exception as e:
  10681. logger.warning("Connection watchdog sweep failed: %s", e)
  10682. await asyncio.sleep(CONNECTION_WATCHDOG_INTERVAL)
  10683. def start_connection_watchdog():
  10684. global _connection_watchdog_task
  10685. if _connection_watchdog_task is None:
  10686. _connection_watchdog_task = asyncio.create_task(_connection_watchdog_loop())
  10687. logging.getLogger(__name__).info("Printer connection watchdog started")
  10688. def stop_connection_watchdog():
  10689. global _connection_watchdog_task
  10690. if _connection_watchdog_task:
  10691. _connection_watchdog_task.cancel()
  10692. _connection_watchdog_task = None
  10693. _connection_watchdog_last_attempt.clear()
  10694. logging.getLogger(__name__).info("Printer connection watchdog stopped")
  10695. # Camera stream orphan cleanup
  10696. _camera_cleanup_task: asyncio.Task | None = None
  10697. CAMERA_CLEANUP_INTERVAL = 60
  10698. # Every this many camera cleanups (about an hour), the timelapse janitor too:
  10699. # kept frames of a failed stitch go after a day, not at the next restart.
  10700. TIMELAPSE_CLEANUP_EVERY = 60
  10701. async def _camera_cleanup_loop():
  10702. """Periodically clean up orphaned ffmpeg processes and timelapse leftovers."""
  10703. from backend.app.api.routes.camera import cleanup_orphaned_streams
  10704. from backend.app.services.layer_timelapse import cleanup_orphaned_timelapse_sessions
  10705. rounds = 0
  10706. while True:
  10707. try:
  10708. rounds += 1
  10709. await cleanup_orphaned_streams()
  10710. except asyncio.CancelledError:
  10711. break
  10712. except Exception as e:
  10713. logging.getLogger(__name__).warning("Camera stream cleanup failed: %s", e)
  10714. try:
  10715. if rounds % TIMELAPSE_CLEANUP_EVERY == 0:
  10716. await asyncio.to_thread(cleanup_orphaned_timelapse_sessions)
  10717. except asyncio.CancelledError:
  10718. break
  10719. except Exception as e:
  10720. logging.getLogger(__name__).warning("Timelapse frames cleanup failed: %s", e)
  10721. await asyncio.sleep(CAMERA_CLEANUP_INTERVAL)
  10722. def start_camera_cleanup():
  10723. global _camera_cleanup_task
  10724. if _camera_cleanup_task is None:
  10725. _camera_cleanup_task = asyncio.create_task(_camera_cleanup_loop())
  10726. logging.getLogger(__name__).info("Camera stream cleanup started")
  10727. def stop_camera_cleanup():
  10728. global _camera_cleanup_task
  10729. if _camera_cleanup_task:
  10730. _camera_cleanup_task.cancel()
  10731. _camera_cleanup_task = None
  10732. logging.getLogger(__name__).info("Camera stream cleanup stopped")
  10733. # ---------------------------------------------------------------------------
  10734. # Expected-print TTL eviction
  10735. # ---------------------------------------------------------------------------
  10736. def _evict_stale_expected_prints() -> None:
  10737. """Remove entries from _expected_prints / _expected_print_creators that are
  10738. older than _EXPECTED_PRINT_TTL_SECONDS.
  10739. This prevents unbounded growth when a print is registered (via
  10740. register_expected_print) but on_print_start never fires — e.g. because the
  10741. printer disconnects, the app restarts, or the print is started directly from
  10742. the printer panel without going through the queue.
  10743. """
  10744. # Use monotonic time so the TTL is unaffected by system clock adjustments
  10745. # (e.g. NTP sync, DST changes).
  10746. # A print that ended without its archive being found at completion leaves
  10747. # its id here; only running prints are owned.
  10748. _owned_print_archive_ids.intersection_update(_active_prints.values())
  10749. cutoff = time.monotonic() - _EXPECTED_PRINT_TTL_SECONDS
  10750. stale_keys = [k for k, t in _expected_print_registered_at.items() if t < cutoff]
  10751. if not stale_keys:
  10752. return
  10753. evicted: set[tuple[int, int]] = set()
  10754. for key in stale_keys:
  10755. archive_id = _expected_prints.pop(key, None)
  10756. if archive_id is not None:
  10757. evicted.add((key[0], archive_id))
  10758. _expected_print_creators.pop(key, None)
  10759. _expected_print_registered_at.pop(key, None)
  10760. # Also clean up _print_ams_mappings and _print_plate_ids for archive_ids
  10761. # that have no remaining live keys in _expected_prints (all variants
  10762. # were just evicted) and are not printing right now. Print start pops
  10763. # only the names the printer reported, so the "<base>.gcode" variant
  10764. # outlives it; without the printing check its eviction two hours after
  10765. # dispatch threw away a running print's plate, AMS mapping and cost
  10766. # centre, which completion still reads (#3275).
  10767. for printer_id, archive_id in evicted:
  10768. if not _expected_or_printing_here(printer_id, archive_id):
  10769. _drop_dispatch(printer_id, archive_id)
  10770. logging.getLogger(__name__).info(
  10771. "Evicted %d stale expected-print entries (TTL=%ds)", len(stale_keys), _EXPECTED_PRINT_TTL_SECONDS
  10772. )
  10773. async def _expected_prints_cleanup_loop() -> None:
  10774. """Background task: periodically evict stale expected-print entries."""
  10775. while True:
  10776. try:
  10777. _evict_stale_expected_prints()
  10778. except asyncio.CancelledError:
  10779. raise
  10780. except Exception as e:
  10781. logging.getLogger(__name__).warning("Expected prints cleanup failed: %s", e)
  10782. await asyncio.sleep(_EXPECTED_PRINT_CLEANUP_INTERVAL)
  10783. def start_expected_prints_cleanup() -> None:
  10784. global _expected_prints_cleanup_task
  10785. if _expected_prints_cleanup_task is None:
  10786. _expected_prints_cleanup_task = asyncio.create_task(_expected_prints_cleanup_loop())
  10787. logging.getLogger(__name__).info("Expected prints cleanup started")
  10788. def stop_expected_prints_cleanup() -> None:
  10789. global _expected_prints_cleanup_task
  10790. if _expected_prints_cleanup_task:
  10791. _expected_prints_cleanup_task.cancel()
  10792. _expected_prints_cleanup_task = None
  10793. logging.getLogger(__name__).info("Expected prints cleanup stopped")
  10794. # ---------------------------------------------------------------------------
  10795. # L-2: Periodic auth-token cleanup (stale TOTP + expired revoked JTIs)
  10796. # ---------------------------------------------------------------------------
  10797. _auth_cleanup_task: asyncio.Task | None = None
  10798. _AUTH_CLEANUP_INTERVAL = 3600 # seconds (hourly)
  10799. async def _run_auth_cleanup() -> None:
  10800. """Single cleanup pass: remove stale TOTP records, expired revoked JTIs, and old rate-limit events."""
  10801. from backend.app.core.database import async_session
  10802. from backend.app.models.auth_ephemeral import AuthEphemeralToken, AuthRateLimitEvent
  10803. from backend.app.models.user_totp import UserTOTP
  10804. now = datetime.now(timezone.utc)
  10805. # Remove unconfirmed (is_enabled=False) TOTP records older than 1 hour.
  10806. try:
  10807. async with async_session() as db:
  10808. stale_cutoff = now - timedelta(hours=1)
  10809. result = await db.execute(
  10810. select(UserTOTP).where(
  10811. UserTOTP.is_enabled.is_(False),
  10812. UserTOTP.created_at < stale_cutoff,
  10813. )
  10814. )
  10815. stale_records = result.scalars().all()
  10816. if stale_records:
  10817. for rec in stale_records:
  10818. await db.delete(rec)
  10819. await db.commit()
  10820. logging.info("Auth cleanup: removed %d stale unconfirmed TOTP record(s)", len(stale_records))
  10821. except Exception as e:
  10822. logging.warning("Auth cleanup: failed to purge stale TOTP records: %s", e)
  10823. # Remove expired revoked-JTI entries (they are no longer needed once the
  10824. # original token's exp has passed — the token would be rejected by JWT
  10825. # signature verification regardless).
  10826. try:
  10827. async with async_session() as db:
  10828. await db.execute(
  10829. delete(AuthEphemeralToken).where(
  10830. AuthEphemeralToken.token_type == "revoked_jti",
  10831. AuthEphemeralToken.expires_at < now,
  10832. )
  10833. )
  10834. await db.commit()
  10835. except Exception as e:
  10836. logging.warning("Auth cleanup: failed to purge expired revoked JTIs: %s", e)
  10837. # L-R6-B: Purge AuthRateLimitEvent rows older than the lockout window (15 min).
  10838. # Events outside this window can never affect rate-limit decisions — they only
  10839. # consume DB space. Use the same window constant as the rate limiter so the
  10840. # two are always in sync.
  10841. try:
  10842. from backend.app.api.routes.mfa import LOCKOUT_WINDOW
  10843. async with async_session() as db:
  10844. await db.execute(
  10845. delete(AuthRateLimitEvent).where(
  10846. AuthRateLimitEvent.occurred_at < now - LOCKOUT_WINDOW,
  10847. )
  10848. )
  10849. await db.commit()
  10850. except Exception as e:
  10851. logging.warning("Auth cleanup: failed to purge stale rate-limit events: %s", e)
  10852. async def _auth_cleanup_loop() -> None:
  10853. """Periodic background task: run auth cleanup every hour."""
  10854. while True:
  10855. try:
  10856. await _run_auth_cleanup()
  10857. except asyncio.CancelledError:
  10858. break
  10859. except Exception as e:
  10860. logging.warning("Auth cleanup loop error: %s", e)
  10861. await asyncio.sleep(_AUTH_CLEANUP_INTERVAL)
  10862. def start_auth_cleanup() -> None:
  10863. global _auth_cleanup_task
  10864. if _auth_cleanup_task is None:
  10865. _auth_cleanup_task = asyncio.create_task(_auth_cleanup_loop())
  10866. logging.getLogger(__name__).info("Auth periodic cleanup started")
  10867. def stop_auth_cleanup() -> None:
  10868. global _auth_cleanup_task
  10869. if _auth_cleanup_task:
  10870. _auth_cleanup_task.cancel()
  10871. _auth_cleanup_task = None
  10872. logging.getLogger(__name__).info("Auth periodic cleanup stopped")
  10873. async def initialize_wled_printers(db: AsyncSession) -> None:
  10874. """Seed WLED settings and apply offline presets before MQTT connections start."""
  10875. from backend.app.api.routes.settings import get_setting, setting_is_true
  10876. from backend.app.models.printer import Printer
  10877. wled_manager.require_plate_clear = setting_is_true(await get_setting(db, "require_plate_clear"))
  10878. result = await db.execute(select(Printer.id, Printer.wled_config, Printer.is_active))
  10879. for printer_id, config, is_active in result.all():
  10880. wled_manager.configure_printer(printer_id, config)
  10881. if not is_active and config:
  10882. wled_manager.handle_offline(printer_id)
  10883. @asynccontextmanager
  10884. async def lifespan(app: FastAPI):
  10885. # Startup
  10886. # Install Windows-only asyncio Proactor cleanup-RST filter (#1113) before
  10887. # anything else can spawn tasks that might trip it.
  10888. from backend.app.core.asyncio_handlers import install_proactor_reset_filter, warn_if_running_on_uvloop
  10889. install_proactor_reset_filter()
  10890. # Before anything opens files or sockets in bulk: a soft limit of 1024 is
  10891. # what turned descriptor exhaustion into a corrupted database (#2883).
  10892. from backend.app.core.fd_limit import raise_open_file_limit
  10893. raise_open_file_limit()
  10894. # Before init_db, so the warning is near the top of the log rather than
  10895. # below a migration run. See warn_if_running_on_uvloop for what is at stake.
  10896. warn_if_running_on_uvloop()
  10897. await init_db()
  10898. # Browser download tokens expire after five minutes. Remove abandoned
  10899. # prepared ZIPs at startup as well as before each new preparation so a
  10900. # quiet appliance cannot retain an unusable bundle indefinitely.
  10901. try:
  10902. from backend.app.services.printer_media import prune_stale_printer_file_bundles
  10903. await prune_stale_printer_file_bundles()
  10904. except Exception as exc:
  10905. logging.warning("Failed to prune stale printer download bundles: %s", exc)
  10906. # After migrations, so the is_env_managed column exists. Never raises --
  10907. # a bad BAMBUDDY_OIDC_* value is logged and skipped rather than blocking
  10908. # startup (see apply_env_oidc_provider).
  10909. from backend.app.core.oidc_env import apply_env_oidc_provider
  10910. async with async_session() as oidc_db:
  10911. await apply_env_oidc_provider(oidc_db)
  10912. # Close out batches that finished before `completed` was a reachable status
  10913. # (#342). Without this the Batches tab opens on every batch created since
  10914. # the feature shipped, all still marked active. Never blocks startup.
  10915. try:
  10916. from backend.app.services.print_batch import backfill_batch_statuses
  10917. async with async_session() as batch_db:
  10918. await backfill_batch_statuses(batch_db)
  10919. except Exception as exc:
  10920. logging.warning("[BATCH] Startup status backfill failed: %s", exc)
  10921. # Each plate's estimate of archives made before they were kept (#3375),
  10922. # read from their 3MFs in the background: the readers use the whole
  10923. # file's figures until then.
  10924. global _plate_estimates_task
  10925. _plate_estimates_task = asyncio.create_task(_fill_plate_estimates())
  10926. # The G-code fingerprint of archives made before it was kept (#1634), read
  10927. # from their files in the background: until then a slice sent again from
  10928. # the slicer archives anew, as before.
  10929. global _gcode_hashes_task
  10930. _gcode_hashes_task = asyncio.create_task(_fill_gcode_hashes())
  10931. # Register an app-scoped httpx client for Bambu Cloud services so
  10932. # per-request BambuCloudService instances reuse the same connection pool
  10933. # (important for routes like /cloud/filament-info that chain many
  10934. # get_setting_detail calls). The shared client stores no region/token
  10935. # state, so the per-request ownership pattern that fixed the region-bleed
  10936. # bug is preserved.
  10937. import httpx as _httpx
  10938. from backend.app.services.bambu_cloud import set_shared_http_client
  10939. from backend.app.services.model_providers.makerworld.service import (
  10940. set_shared_http_client as set_shared_makerworld_http_client,
  10941. )
  10942. from backend.app.services.orca_cloud import (
  10943. set_shared_http_client as set_shared_orca_http_client,
  10944. )
  10945. _shared_cloud_http_client = _httpx.AsyncClient(timeout=30.0)
  10946. set_shared_http_client(_shared_cloud_http_client)
  10947. # Reuse the same connection pool for MakerWorld — different host, same
  10948. # keep-alive pool saves a TLS handshake per request.
  10949. set_shared_makerworld_http_client(_shared_cloud_http_client)
  10950. # Same for Orca Cloud — without this the per-request OrcaCloudService()
  10951. # each spun up (and never closed) its own client, leaking sockets.
  10952. set_shared_orca_http_client(_shared_cloud_http_client)
  10953. # Fix queue items stuck with invalid "aborted" status (should be "cancelled").
  10954. # This can happen when a print was cancelled mid-print on versions before this fix.
  10955. try:
  10956. async with async_session() as db:
  10957. from backend.app.models.print_queue import PrintQueueItem
  10958. result = await db.execute(select(PrintQueueItem).where(PrintQueueItem.status == "aborted"))
  10959. aborted_items = result.scalars().all()
  10960. if aborted_items:
  10961. for item in aborted_items:
  10962. item.status = "cancelled"
  10963. await db.commit()
  10964. logging.info("Fixed %d queue item(s) with invalid 'aborted' status → 'cancelled'", len(aborted_items))
  10965. except Exception as e:
  10966. logging.warning("Failed to fix aborted queue items: %s", e)
  10967. # Restore debug logging state from previous session
  10968. await init_debug_logging()
  10969. # Set up printer manager callbacks
  10970. loop = asyncio.get_event_loop()
  10971. printer_manager.set_event_loop(loop)
  10972. printer_manager.set_status_change_callback(on_printer_status_change)
  10973. printer_manager.set_print_start_callback(on_print_start)
  10974. printer_manager.set_print_complete_callback(on_print_complete)
  10975. printer_manager.set_print_running_observed_callback(on_print_running_observed)
  10976. printer_manager.set_finish_photo_moment_callback(on_finish_photo_moment)
  10977. printer_manager.set_ams_change_callback(on_ams_change)
  10978. printer_manager.set_fts_inlet_change_callback(on_fts_inlet_change)
  10979. # Rehydrate persisted awaiting-plate-clear gate (#961) so prompts survive restarts
  10980. await printer_manager.load_awaiting_plate_clear_from_db()
  10981. # Before any printer connects, or its first report re-sends every fault
  10982. # still listed (#3486).
  10983. await _load_notified_hms_faults()
  10984. # WLED is entirely optional. Seed its in-memory per-printer configuration
  10985. # before MQTT connections begin emitting status callbacks.
  10986. async with async_session() as db:
  10987. await initialize_wled_printers(db)
  10988. # Layer change callback for external camera timelapse
  10989. async def on_layer_change(printer_id: int, layer_num: int):
  10990. """Capture timelapse frame on layer change + first layer notification."""
  10991. from backend.app.services.layer_timelapse import on_layer_change as tl_layer_change
  10992. await tl_layer_change(printer_id, layer_num)
  10993. # #1867: bank a recent in-print frame so the finish-photo path has a
  10994. # pre-End-G-code image to use instead of a live grab of a swapped plate.
  10995. # #2547 added `on_print_progress` as a second driver — this one alone
  10996. # stops firing once the final layer begins.
  10997. await _maybe_bank_inprint_frame(printer_id, layer_num)
  10998. # First layer complete notification (layer_num >= 2 means layer 1 is done).
  10999. # Gate on actual printing state — Bambu firmware ticks layer_num during
  11000. # the pre-print calibration sequence (homing / mesh-level / bed scan /
  11001. # nozzle clean), so a bare layer_num check can fire minutes before the
  11002. # first real extrusion. We require gcode_state == RUNNING and
  11003. # mc_print_sub_stage in (0 = "Printing", None) so calibration sub-stages
  11004. # (1, 9, 14, ...) are excluded. The window widens to [2, 10] because if
  11005. # the layer counter advanced past 2 during PREPARE, the next on_layer_change
  11006. # edge fires later; _first_layer_notified stays clear until we actually send
  11007. # so a deferred re-evaluation can win. See issue #1837.
  11008. if 2 <= layer_num <= 10 and not _first_layer_notified.get(printer_id, False):
  11009. client = printer_manager.get_client(printer_id)
  11010. state = client.state if client else None
  11011. if not state or state.state != "RUNNING":
  11012. return
  11013. if state.mc_print_sub_stage not in (None, 0):
  11014. return
  11015. _first_layer_notified[printer_id] = True
  11016. try:
  11017. async with async_session() as db:
  11018. from backend.app.models.printer import Printer
  11019. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  11020. printer = result.scalar_one_or_none()
  11021. if not printer:
  11022. return
  11023. printer_name = printer.name
  11024. filename = (state.subtask_name or state.gcode_file or "Unknown") if state else "Unknown"
  11025. total_layers = state.total_layers if state else 0
  11026. image_data = await _capture_snapshot_for_notification(
  11027. printer_id, printer, logging.getLogger(__name__)
  11028. )
  11029. await notification_service.on_first_layer_complete(
  11030. printer_id, printer_name, filename, total_layers, db, image_data=image_data
  11031. )
  11032. except Exception as e:
  11033. logging.getLogger(__name__).warning("First layer notification failed: %s", e)
  11034. printer_manager.set_layer_change_callback(on_layer_change)
  11035. async def on_print_progress(printer_id: int, percent: int):
  11036. """#2547: keep the in-print frame bank fresh through the final layer.
  11037. `on_layer_change` stops the moment the last layer starts, which on the
  11038. H2C capture that closed #2547 left the bank stale for the three minutes
  11039. that layer took. Progress is the only field that keeps advancing there,
  11040. and it freezes before the End G-code runs — so banking on it stays
  11041. inside the print and never sees a swapped plate.
  11042. """
  11043. client = printer_manager.get_client(printer_id)
  11044. state = client.state if client else None
  11045. await _maybe_bank_inprint_frame(printer_id, state.layer_num if state else 0)
  11046. printer_manager.set_print_progress_callback(on_print_progress)
  11047. # Event-driven bed cooldown: fires whenever bed_temper arrives via MQTT
  11048. async def on_bed_temp_update(printer_id: int, bed_temp: float):
  11049. waiter = _bed_cool_waiters.get(printer_id)
  11050. if not waiter:
  11051. return
  11052. threshold = waiter["threshold"]
  11053. if bed_temp > threshold:
  11054. return
  11055. # Bed is at or below threshold — fire notification and remove waiter
  11056. waiter_info = _bed_cool_waiters.pop(printer_id, None)
  11057. if not waiter_info:
  11058. return # Another callback already handled it
  11059. bed_cool_logger = logging.getLogger(__name__)
  11060. bed_cool_logger.info(
  11061. "[BED-COOL] Bed cooled to %.1f°C on printer %s (threshold: %.0f°C)",
  11062. bed_temp,
  11063. printer_id,
  11064. threshold,
  11065. )
  11066. try:
  11067. printer_info = printer_manager.get_printer(printer_id)
  11068. p_name = printer_info.name if printer_info else "Unknown"
  11069. async with async_session() as db:
  11070. await notification_service.on_bed_cooled(
  11071. printer_id=printer_id,
  11072. printer_name=p_name,
  11073. bed_temp=bed_temp,
  11074. threshold=threshold,
  11075. filename=waiter_info["filename"],
  11076. db=db,
  11077. )
  11078. except Exception as e:
  11079. bed_cool_logger.warning("[BED-COOL] Failed to send notification: %s", e)
  11080. printer_manager.set_bed_temp_update_callback(on_bed_temp_update)
  11081. async def on_drying_complete(printer_id: int, ams_id: int):
  11082. """Smart-plug auto-off-after-drying trigger (#1349).
  11083. Fires once per AMS unit when ``dry_time`` falls from >0 to 0. The
  11084. manager walks all plugs linked to this printer and turns off only
  11085. the ones with ``auto_off_after_drying`` enabled, after their
  11086. per-plug delay. Multiple AMS units finishing close together (e.g. a
  11087. dual-AMS dry that ends within the same MQTT push) call this once
  11088. per unit — the manager's ``_cancel_pending_off`` collapses
  11089. repeated scheduling on the same plug to one timer, so duplicate
  11090. fires are safe.
  11091. """
  11092. try:
  11093. async with async_session() as db:
  11094. await smart_plug_manager.on_drying_complete(printer_id, db)
  11095. except Exception as e:
  11096. logging.getLogger(__name__).warning(
  11097. "Failed to schedule auto-off-after-drying for printer %d (AMS %d): %s",
  11098. printer_id,
  11099. ams_id,
  11100. e,
  11101. )
  11102. printer_manager.set_drying_complete_callback(on_drying_complete)
  11103. async def on_drying_cycle_end(printer_id: int, cycle):
  11104. """Mark the spools in an AMS that just finished drying as dried (#2863)."""
  11105. from backend.app.services.spool_drying import record_drying_cycle
  11106. try:
  11107. async with async_session() as db:
  11108. await record_drying_cycle(db, printer_id, cycle)
  11109. except Exception as e:
  11110. logging.getLogger(__name__).warning(
  11111. "Failed to record drying for printer %d (AMS %d): %s",
  11112. printer_id,
  11113. cycle.ams_id,
  11114. e,
  11115. )
  11116. printer_manager.set_drying_cycle_end_callback(on_drying_cycle_end)
  11117. async def on_assignment_verified(printer_id: int, ams_id: int, tray_id: int, verified: bool, detail: dict):
  11118. """Surface the read-back result of a spool assignment to the UI (#2582).
  11119. The MQTT client confirms (or fails to confirm) that the tray telemetry
  11120. echoed back the filament id we pushed. We relay that as a websocket
  11121. event so the frontend can toast "loaded" / "assignment didn't take"
  11122. instead of the historic silent fire-and-forget, which made the
  11123. AMS→Studio hand-off feel random to users.
  11124. """
  11125. try:
  11126. from backend.app.services.spool_assignment_notifications import (
  11127. _slot_label_from_global_tray,
  11128. )
  11129. layout = printer_tray_layout(printer_id, printer_manager)
  11130. global_id = layout.tray_from_slot_key(ams_id, tray_id)
  11131. slot_label = _slot_label_from_global_tray(global_id, layout)
  11132. printer_info = printer_manager.get_printer(printer_id)
  11133. printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  11134. await ws_manager.broadcast(
  11135. {
  11136. "type": "spool_assignment_verified",
  11137. "printer_id": printer_id,
  11138. "printer_name": printer_name,
  11139. "ams_id": ams_id,
  11140. "tray_id": tray_id,
  11141. "slot": slot_label,
  11142. "verified": verified,
  11143. # Present on success: False means the filament setting landed
  11144. # but the K-profile (cali_idx) did not — the reporter's exact
  11145. # "loaded but no flow profile" symptom.
  11146. "kprofile_applied": detail.get("kprofile_applied", True),
  11147. }
  11148. )
  11149. except Exception as e:
  11150. logging.getLogger(__name__).warning(
  11151. "Failed to broadcast assignment verification for printer %d AMS%d-T%d: %s",
  11152. printer_id,
  11153. ams_id,
  11154. tray_id,
  11155. e,
  11156. )
  11157. printer_manager.set_assignment_verified_callback(on_assignment_verified)
  11158. async def on_tray_change(printer_id: int, tray_global: int, layer_num: int):
  11159. """Persist a mid-print tray change for completion-time attribution.
  11160. AMS filament backup switches trays without telling the slicer, so the
  11161. tray-change log is the only record of which spool fed which layers.
  11162. Keeping it only in memory meant a restart mid-print charged everything
  11163. to the tray that finished the job.
  11164. """
  11165. try:
  11166. from backend.app.services.usage_tracker import record_tray_change
  11167. async with async_session() as db:
  11168. await record_tray_change(db, printer_id, tray_global, layer_num)
  11169. except Exception as e:
  11170. logging.getLogger(__name__).warning(
  11171. "Failed to persist tray change for printer %d (tray=%d, layer=%d): %s",
  11172. printer_id,
  11173. tray_global,
  11174. layer_num,
  11175. e,
  11176. )
  11177. printer_manager.set_tray_change_callback(on_tray_change)
  11178. # Initialize MQTT relay from settings
  11179. async with async_session() as db:
  11180. from backend.app.api.routes.settings import get_setting
  11181. mqtt_settings = {
  11182. "mqtt_enabled": (await get_setting(db, "mqtt_enabled") or "false") == "true",
  11183. "mqtt_broker": await get_setting(db, "mqtt_broker") or "",
  11184. "mqtt_port": int(await get_setting(db, "mqtt_port") or "1883"),
  11185. "mqtt_username": await get_setting(db, "mqtt_username") or "",
  11186. "mqtt_password": await get_setting(db, "mqtt_password") or "",
  11187. "mqtt_topic_prefix": await get_setting(db, "mqtt_topic_prefix") or "bambuddy",
  11188. "mqtt_use_tls": (await get_setting(db, "mqtt_use_tls") or "false") == "true",
  11189. }
  11190. await mqtt_relay.configure(mqtt_settings)
  11191. # Restore MQTT smart plug subscriptions
  11192. if mqtt_settings.get("mqtt_enabled"):
  11193. from backend.app.models.smart_plug import SmartPlug
  11194. from backend.app.services.mqtt_smart_plug import subscribe_plug_to_mqtt
  11195. result = await db.execute(select(SmartPlug).where(SmartPlug.plug_type == "mqtt"))
  11196. mqtt_plugs = result.scalars().all()
  11197. restored = 0
  11198. for plug in mqtt_plugs:
  11199. if subscribe_plug_to_mqtt(mqtt_relay.smart_plug_service, plug):
  11200. restored += 1
  11201. if restored:
  11202. logging.info("Restored %s MQTT smart plug subscriptions", restored)
  11203. # Connect to all active printers
  11204. async with async_session() as db:
  11205. await init_printer_connections(db)
  11206. # Auto-connect to Spoolman if enabled
  11207. async with async_session() as db:
  11208. from backend.app.api.routes.settings import get_setting
  11209. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  11210. spoolman_url = await get_setting(db, "spoolman_url")
  11211. if spoolman_enabled and spoolman_enabled.lower() == "true" and spoolman_url:
  11212. try:
  11213. client = await init_spoolman_client(spoolman_url)
  11214. if await client.health_check():
  11215. logging.info("Auto-connected to Spoolman at %s", spoolman_url)
  11216. # Ensure the 'tag' extra field exists for RFID/UUID storage
  11217. field_ok = await client.ensure_tag_extra_field()
  11218. if not field_ok:
  11219. logging.error("Spoolman tag extra field registration failed — NFC tag links may not persist")
  11220. # Register the BambuStudio slicer-preset fields used by the
  11221. # spool-edit / assign flow. Spoolman rejects PATCHes with
  11222. # unknown extra keys, so these must exist before any update
  11223. # that touches them.
  11224. for field_name in ("bambu_slicer_filament", "bambu_slicer_filament_name"):
  11225. if not await client.ensure_extra_field(field_name):
  11226. logging.warning(
  11227. "Spoolman extra field %r registration failed — "
  11228. "spool slicer-preset edits will return 502",
  11229. field_name,
  11230. )
  11231. else:
  11232. logging.warning("Spoolman at %s is not reachable", spoolman_url)
  11233. except Exception as e:
  11234. logging.warning("Failed to auto-connect to Spoolman: %s", e)
  11235. # Start the print scheduler
  11236. spawn_background_task(print_scheduler.run(), name="print-scheduler")
  11237. # Start the smart plug scheduler for time-based on/off
  11238. smart_plug_manager.start_scheduler()
  11239. # Start the Home Assistant sensor poller (#1148)
  11240. ha_sensor_manager.start()
  11241. location_ha_sensor_manager.start()
  11242. # Resume any pending auto-offs that were interrupted by restart
  11243. await smart_plug_manager.resume_pending_auto_offs()
  11244. # Start the notification digest scheduler
  11245. notification_service.start_digest_scheduler()
  11246. notify_live_activities.start()
  11247. notify_widgets.start()
  11248. # Start the Telegram reaction pollers (#3046), one per bot token used by a
  11249. # provider in reactions/both mode; the notification routes resync them.
  11250. # Never fatal: a bad provider row or a DB hiccup here costs reactions
  11251. # until the next provider save, not the whole startup.
  11252. try:
  11253. await telegram_reaction_poller.start()
  11254. except Exception as e:
  11255. logging.warning("Telegram reaction poller did not start: %s", e)
  11256. # Start the GitHub backup scheduler
  11257. await github_backup_service.start_scheduler()
  11258. # Start the local backup scheduler
  11259. await local_backup_service.start_scheduler()
  11260. await obico_detection_service.start()
  11261. await octoeverywhere_detection_service.start()
  11262. # Start the library trash sweeper (#1008)
  11263. await library_trash_service.start_scheduler()
  11264. # Start the archive auto-purge sweeper (#1008 follow-up)
  11265. await archive_purge_service.start_scheduler()
  11266. # Start AMS history recording
  11267. start_ams_history_recording()
  11268. # Start printer sensor (nozzle / bed / chamber) history recording
  11269. start_printer_sensor_history_recording()
  11270. # Start printer runtime tracking
  11271. start_runtime_tracking()
  11272. # Start SpoolBuddy device watchdog
  11273. start_spoolbuddy_watchdog()
  11274. # Start camera stream orphan cleanup
  11275. start_camera_cleanup()
  11276. # Start the backstop for MQTT sessions paho has stopped recovering (#2732)
  11277. start_connection_watchdog()
  11278. # One-shot sweep for timelapse session directories orphaned by a crash
  11279. # or restart that happened mid-print (in-memory session tracking can't
  11280. # survive that, and nothing else reaps the leftover frames/output file)
  11281. try:
  11282. from backend.app.services.layer_timelapse import cleanup_orphaned_timelapse_sessions
  11283. removed = cleanup_orphaned_timelapse_sessions()
  11284. if removed:
  11285. logging.getLogger(__name__).info("Removed %d orphaned timelapse session artifact(s)", removed)
  11286. except Exception as e:
  11287. logging.getLogger(__name__).warning("Orphaned timelapse session cleanup failed: %s", e)
  11288. # Start expected-print TTL eviction (prevents memory leak when prints are
  11289. # registered but on_print_start never fires)
  11290. start_expected_prints_cleanup()
  11291. # L-2: Start periodic auth cleanup (stale TOTP + expired revoked JTIs)
  11292. start_auth_cleanup()
  11293. # Maintainer announcements: a signed feed fetched from GitHub every few hours.
  11294. from backend.app.services import announcements as announcements_service
  11295. announcements_service.start()
  11296. from backend.app.services.printer_media import start_printer_download_cleanup
  11297. start_printer_download_cleanup()
  11298. # Event-loop stall watchdog: dumps all thread stacks to stderr if the loop
  11299. # freezes (#1486 — silent "container hangs after adding a printer" reports).
  11300. from backend.app.services.loop_watchdog import start_loop_watchdog
  11301. start_loop_watchdog()
  11302. # Initialize virtual printer manager and sync from DB
  11303. from backend.app.services.virtual_printer import virtual_printer_manager
  11304. virtual_printer_manager.set_session_factory(async_session)
  11305. virtual_printer_manager.set_printer_manager(printer_manager)
  11306. try:
  11307. await virtual_printer_manager.sync_from_db()
  11308. logging.info("Virtual printer manager synced from database")
  11309. except Exception as e:
  11310. logging.warning("Failed to sync virtual printers: %s", e)
  11311. yield
  11312. # Shutdown
  11313. print_scheduler.stop()
  11314. smart_plug_manager.stop_scheduler()
  11315. ha_sensor_manager.stop()
  11316. location_ha_sensor_manager.stop()
  11317. notification_service.stop_digest_scheduler()
  11318. await notify_live_activities.close()
  11319. await notify_widgets.close()
  11320. await telegram_reaction_poller.aclose()
  11321. github_backup_service.stop_scheduler()
  11322. local_backup_service.stop_scheduler()
  11323. library_trash_service.stop_scheduler()
  11324. archive_purge_service.stop_scheduler()
  11325. obico_detection_service.stop()
  11326. octoeverywhere_detection_service.stop()
  11327. stop_ams_history_recording()
  11328. stop_printer_sensor_history_recording()
  11329. stop_runtime_tracking()
  11330. stop_spoolbuddy_watchdog()
  11331. stop_camera_cleanup()
  11332. stop_connection_watchdog()
  11333. from backend.app.services.loop_watchdog import stop_loop_watchdog
  11334. stop_loop_watchdog()
  11335. # Tear down all camera fan-out broadcasters (#1089) so subscribers exit
  11336. # cleanly rather than waiting on a queue that nothing will ever fill.
  11337. try:
  11338. from backend.app.services.camera_fanout import shutdown_all_broadcasters
  11339. await shutdown_all_broadcasters()
  11340. except Exception as e:
  11341. logging.warning("Failed to shut down camera broadcasters: %s", e)
  11342. stop_expected_prints_cleanup()
  11343. stop_auth_cleanup()
  11344. if _plate_estimates_task is not None and not _plate_estimates_task.done():
  11345. _plate_estimates_task.cancel()
  11346. if _gcode_hashes_task is not None and not _gcode_hashes_task.done():
  11347. _gcode_hashes_task.cancel()
  11348. from backend.app.services import announcements as announcements_service
  11349. announcements_service.stop()
  11350. from backend.app.services.printer_media import stop_printer_download_cleanup
  11351. await stop_printer_download_cleanup()
  11352. # Stops a timelapse editor's ffmpeg rather than leaving it to the OS.
  11353. from backend.app.services import timelapse_jobs
  11354. await timelapse_jobs.cancel_all()
  11355. printer_manager.disconnect_all()
  11356. slot_unlink_grace.reset()
  11357. await close_spoolman_client()
  11358. # Stop all virtual printer services
  11359. await virtual_printer_manager.stop_all()
  11360. await mqtt_smart_plug_service.disconnect(timeout=2)
  11361. await mqtt_relay.disconnect(timeout=2)
  11362. await wled_manager.shutdown()
  11363. # Drop the shared Bambu Cloud HTTP client we registered at startup.
  11364. set_shared_http_client(None)
  11365. set_shared_makerworld_http_client(None)
  11366. set_shared_orca_http_client(None)
  11367. await _shared_cloud_http_client.aclose()
  11368. # Checkpoint WAL (SQLite only) and close all database connections
  11369. from backend.app.core.db_dialect import is_sqlite
  11370. if is_sqlite():
  11371. try:
  11372. async with engine.begin() as conn:
  11373. await conn.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
  11374. logging.info("WAL checkpoint completed")
  11375. except Exception as e:
  11376. logging.warning("WAL checkpoint failed: %s", e)
  11377. await engine.dispose()
  11378. app = FastAPI(
  11379. title=app_settings.app_name,
  11380. description="Archive and manage Bambu Lab 3MF files",
  11381. version=APP_VERSION,
  11382. lifespan=lifespan,
  11383. )
  11384. # Added first so it sits innermost, next to the routes: a refused header's value
  11385. # (often a stored token) masked in every JSON body. See core.response_redaction.
  11386. app.add_middleware(RefusedHeaderResponseMask)
  11387. def _json_safe(value):
  11388. """The validation errors with every non-finite number written as text."""
  11389. if isinstance(value, float) and not math.isfinite(value):
  11390. return str(value)
  11391. if isinstance(value, dict):
  11392. return {key: _json_safe(item) for key, item in value.items()}
  11393. if isinstance(value, list | tuple):
  11394. return [_json_safe(item) for item in value]
  11395. return value
  11396. @app.exception_handler(RequestValidationError)
  11397. async def _validation_error(_request, exc: RequestValidationError) -> JSONResponse:
  11398. """FastAPI's own 422, except that it survives a non-finite number.
  11399. A body may carry ``Infinity`` or ``NaN`` (Python's JSON reader accepts
  11400. them), and the default handler echoes the input back in its errors,
  11401. which JSON cannot hold: the request ended in an error instead of a 422.
  11402. """
  11403. return JSONResponse(status_code=422, content={"detail": _json_safe(jsonable_encoder(exc.errors()))})
  11404. # =============================================================================
  11405. # Authentication Middleware - Secures ALL API routes by default
  11406. # =============================================================================
  11407. # Public routes that don't require authentication even when auth is enabled
  11408. PUBLIC_API_ROUTES = {
  11409. # Auth routes needed before/during login
  11410. "/api/v1/auth/status",
  11411. "/api/v1/auth/login",
  11412. "/api/v1/auth/setup", # Needed for initial setup and recovery
  11413. # Advanced auth status needed for login page
  11414. "/api/v1/auth/advanced-auth/status",
  11415. "/api/v1/auth/forgot-password", # Password reset for advanced auth
  11416. "/api/v1/auth/forgot-password/confirm", # Complete password reset with token (H-6)
  11417. # 2FA routes that are called BEFORE a JWT is issued (pre-auth flow)
  11418. "/api/v1/auth/2fa/verify", # Exchange pre_auth_token + 2FA code for JWT
  11419. "/api/v1/auth/2fa/email/send", # Send OTP email (pre_auth_token based)
  11420. # OIDC routes that must be reachable without a JWT
  11421. "/api/v1/auth/oidc/providers", # Public list of enabled providers
  11422. "/api/v1/auth/oidc/callback", # Redirect target from OIDC provider
  11423. "/api/v1/auth/oidc/exchange", # Exchange short-lived OIDC token for JWT
  11424. # Connected apps: the app's server swaps a code for the user's identity,
  11425. # authenticated by its client secret rather than a login token.
  11426. "/api/v1/connect/token",
  11427. # Version check for updates (no sensitive data)
  11428. "/api/v1/updates/version",
  11429. # Metrics endpoint handles its own prometheus_token authentication
  11430. "/api/v1/metrics",
  11431. # Appliance bootstrap (#1589 follow-up): the SPA's i18n setup polls
  11432. # this BEFORE a JWT is available to pick up the firstboot wizard's
  11433. # hostname / timezone / locale and the chrony NTP-gate state. The
  11434. # response contains user-set defaults and a public sync flag — no
  11435. # secrets. Without this entry the global auth middleware returns 401
  11436. # before the route handler runs, regardless of the route's own
  11437. # "no auth required" intent.
  11438. "/api/v1/system/appliance",
  11439. # Cam Wall kiosk feed (#2531): a TV or Pi in kiosk mode has no login, so it
  11440. # authenticates with a long-lived ``camwall``-scoped token in the query
  11441. # string — exactly like the camera streams two lists below, and for the same
  11442. # reason (no header to put a JWT in). "Public" here only means the middleware
  11443. # steps aside; the route still runs RequireCamWallTokenIfAuthEnabled, which
  11444. # rejects an absent, expired, revoked, or wrong-scoped token. In particular a
  11445. # plain ``camera_stream`` token does NOT open this door.
  11446. "/api/v1/camwall/printers",
  11447. # Overlay branding: the route enforces overlay-scoped token authentication.
  11448. "/api/v1/overlay-branding/logo",
  11449. }
  11450. # Route prefixes that are public (for routes with dynamic segments)
  11451. PUBLIC_API_PREFIXES = [
  11452. # WebSocket connections handle their own auth
  11453. "/api/v1/ws",
  11454. # OIDC authorize redirects — include provider_id in path
  11455. "/api/v1/auth/oidc/authorize/",
  11456. # One-tap outcome-verdict links from push notifications (#1898). Tapped on
  11457. # a phone with no session, so no header can carry a JWT — the single-use
  11458. # capability token in the path IS the credential (same reasoning as the
  11459. # /dl/ slicer downloads below). The route grants nothing beyond writing
  11460. # good/reject on the one archive the token was minted for.
  11461. "/api/v1/archives/confirm/",
  11462. ]
  11463. # Route patterns that are public (read-only display data)
  11464. # These are checked with "in path" - needed because browsers load images/videos
  11465. # via <img src> and <video src> which don't include Authorization headers
  11466. PUBLIC_API_PATTERNS = [
  11467. # Thumbnails
  11468. "/thumbnail", # /archives/{id}/thumbnail, /library/files/{id}/thumbnail
  11469. "/plate-thumbnail/", # /archives/{id}/plate-thumbnail/{plate_id}
  11470. # Images and media
  11471. "/photos/", # /archives/{id}/photos/{filename}
  11472. "/project-image/", # /archives/{id}/project-image/{path}
  11473. "/qrcode", # /archives/{id}/qrcode
  11474. "/timelapse", # /archives/{id}/timelapse (video)
  11475. "/cover", # /printers/{id}/cover
  11476. "/icon", # /external-links/{id}/icon
  11477. # Camera (streams loaded via <img> tag)
  11478. "/camera/stream", # /printers/{id}/camera/stream
  11479. "/camera/snapshot", # /printers/{id}/camera/snapshot
  11480. # Streaming-overlay status feed (#2613): OBS loads /overlay/{id} with no login
  11481. # and this backs it, authenticated by an ``overlay``-scoped token in the query
  11482. # string (same reasoning as the camera streams above — no header to carry a
  11483. # JWT). "Public" only means the middleware steps aside; the route still runs
  11484. # RequireOverlayTokenIfAuthEnabled, which rejects an absent, expired, revoked,
  11485. # or wrong-scoped token — a camwall or camera_stream token does NOT open it.
  11486. "/overlay-status", # /printers/{id}/overlay-status
  11487. # Slicer token-authenticated downloads — protocol handlers (bambustudioopen://,
  11488. # orcaslicer://) cannot send auth headers. These endpoints validate a short-lived
  11489. # download token in the URL path instead.
  11490. "/dl/", # /archives/{id}/dl/{token}/{filename}, /library/files/{id}/dl/{token}/{filename}
  11491. # Same family, but the segment is "source-dl" — which does NOT contain "/dl/",
  11492. # and these patterns match by substring. Without its own entry the middleware
  11493. # 401s the slicer's header-less request before the route's token check runs,
  11494. # so "Open source 3MF in slicer" failed whenever auth was enabled (#3029).
  11495. "/source-dl/", # /archives/{id}/source-dl/{token}/{filename}
  11496. # Obico ML API fetches JPEG frames by one-shot nonce (issue #172 follow-up).
  11497. # The nonce itself is the credential: 32-byte random, single-use, ~30s TTL.
  11498. "/obico/cached-frame/", # /obico/cached-frame/{nonce}
  11499. ]
  11500. _security_headers_logger = logging.getLogger("backend.app.main.security_headers")
  11501. def _parse_trusted_frame_origins() -> tuple[str, ...]:
  11502. """Parse TRUSTED_FRAME_ORIGINS env var into a validated allowlist (#1191).
  11503. Format: comma-separated list of ``scheme://host[:port]`` origins.
  11504. Used by ``security_headers_middleware`` to relax ``frame-ancestors`` for
  11505. trusted same-LAN deployments (e.g. Home Assistant Webpage panel embedding
  11506. Bambuddy from a different port). Defaults to empty — strict ``'none'``.
  11507. Invalid entries are dropped with a warning rather than failing startup, so
  11508. a typo in one origin doesn't take the whole deployment down.
  11509. """
  11510. raw = os.environ.get("TRUSTED_FRAME_ORIGINS", "").strip()
  11511. if not raw:
  11512. return ()
  11513. valid: list[str] = []
  11514. for item in raw.split(","):
  11515. candidate = item.strip()
  11516. if not candidate:
  11517. continue
  11518. try:
  11519. parsed = urlparse(candidate)
  11520. except ValueError as e:
  11521. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — %s", candidate, e)
  11522. continue
  11523. if parsed.scheme not in ("http", "https"):
  11524. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — must be http(s)", candidate)
  11525. continue
  11526. if not parsed.netloc:
  11527. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — missing host", candidate)
  11528. continue
  11529. if parsed.path and parsed.path != "/":
  11530. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — paths not allowed", candidate)
  11531. continue
  11532. if parsed.query or parsed.fragment:
  11533. _security_headers_logger.warning(
  11534. "TRUSTED_FRAME_ORIGINS: dropping %r — query/fragment not allowed", candidate
  11535. )
  11536. continue
  11537. if "*" in parsed.netloc:
  11538. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — wildcards not allowed", candidate)
  11539. continue
  11540. valid.append(f"{parsed.scheme}://{parsed.netloc}")
  11541. if valid:
  11542. _security_headers_logger.info("TRUSTED_FRAME_ORIGINS: %s", ", ".join(valid))
  11543. return tuple(valid)
  11544. _TRUSTED_FRAME_ORIGINS: tuple[str, ...] = _parse_trusted_frame_origins()
  11545. def _frame_ancestors(default_value: str) -> str:
  11546. """Compose the ``frame-ancestors`` CSP directive (#1191).
  11547. ``default_value`` is the strict directive used when the operator has not
  11548. configured ``TRUSTED_FRAME_ORIGINS`` — typically ``'none'`` (catch-all and
  11549. docs) or ``'self'`` (the streaming overlay, embedded same-origin by the
  11550. Settings URL builder's preview). When trusted origins
  11551. are configured, ``'self'`` is always included so same-origin embedding never
  11552. breaks even if an operator forgets to add their own origin to the list.
  11553. """
  11554. if _TRUSTED_FRAME_ORIGINS:
  11555. return "frame-ancestors 'self' " + " ".join(_TRUSTED_FRAME_ORIGINS) + ";"
  11556. return f"frame-ancestors {default_value};"
  11557. # The two Vite-emitted worker assets that compile WebAssembly (#2976). Both
  11558. # patterns are anchored on the exact emitted name so the relaxed policies
  11559. # below can never apply to any other asset.
  11560. # src/workers/stepPreview.worker.ts -> /assets/stepPreview.worker-<hash>.js
  11561. # pdfjs-dist/legacy/build/pdf.worker.min.mjs?worker&url
  11562. # -> /assets/pdf.worker.min-<hash>.js
  11563. # Vite also emits a one-line chunk under the second name that only exports the
  11564. # worker's URL. The page imports it as a module, and a module script is run
  11565. # under the importing document's policy, never its own response's, so it
  11566. # matching as well changes nothing.
  11567. _STEP_WORKER_ASSET_RE = re.compile(r"^/assets/stepPreview\.worker-[\w-]+\.js$")
  11568. _PDF_WORKER_ASSET_RE = re.compile(r"^/assets/pdf\.worker\.min-[\w-]+\.js$")
  11569. @app.middleware("http")
  11570. async def security_headers_middleware(request, call_next):
  11571. """Add standard HTTP security headers to every response."""
  11572. # Per-request nonce stamped into `script-src` (#1460). On its own this
  11573. # changes nothing for Bambuddy's own pages — index.html has no inline
  11574. # scripts since the SW registration moved to /sw-register.js. The reason
  11575. # it's here is Cloudflare: a CF-fronted deployment has the bot-detection
  11576. # script injected into the HTML on the edge, with a fresh hash on every
  11577. # load (so hashes can't be allowlisted). When CF sees a nonce in our CSP,
  11578. # it clones the same nonce onto its injected <script>, and the inline
  11579. # script passes the policy without us needing 'unsafe-inline'. See
  11580. # https://developers.cloudflare.com/cloudflare-challenges/challenge-types/javascript-detections/#if-you-have-a-content-security-policy-csp
  11581. csp_nonce = secrets.token_urlsafe(16)
  11582. # Routes that render their own HTML need it too, or their inline script is
  11583. # blocked by the policy below. The outcome-confirmation page (#1898) is the
  11584. # one that does: it submits its own form so a verdict still costs one tap.
  11585. request.state.csp_nonce = csp_nonce
  11586. response = await call_next(request)
  11587. response.headers["X-Content-Type-Options"] = "nosniff"
  11588. # X-Frame-Options is the legacy cross-origin embedding control. Modern
  11589. # browsers honour CSP frame-ancestors instead, and the legacy
  11590. # `ALLOW-FROM <url>` syntax is deprecated and inconsistent across vendors.
  11591. # When operators have explicitly allowlisted trusted frame origins (#1191
  11592. # — typically Home Assistant on a different port), drop X-Frame-Options
  11593. # and let the CSP-side frame-ancestors directive govern embedding.
  11594. if not _TRUSTED_FRAME_ORIGINS:
  11595. response.headers["X-Frame-Options"] = "SAMEORIGIN"
  11596. response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
  11597. # Content-Security-Policy for the React SPA.
  11598. # Notes:
  11599. # - 'unsafe-inline' for style-src: React and UI libs inject inline styles at runtime.
  11600. # - connect-src ws:/wss:: MQTT/printer WebSocket connections.
  11601. # - img-src data: / blob:: base64 thumbnails and Blob-URL timelapse previews.
  11602. # - media-src blob:: timelapse video player uses Blob URLs.
  11603. # - font-src data:: some icon fonts are embedded as data URIs.
  11604. if request.url.path in ("/docs", "/redoc", "/docs/oauth2-redirect"):
  11605. # FastAPI's built-in Swagger UI / ReDoc pages load assets from
  11606. # cdn.jsdelivr.net and bootstrap with an inline <script>, so the
  11607. # default CSP would render a blank page.
  11608. response.headers["Content-Security-Policy"] = (
  11609. "default-src 'self'; "
  11610. "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
  11611. "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com; "
  11612. "img-src 'self' data: blob: https://fastapi.tiangolo.com https://cdn.redoc.ly; "
  11613. "connect-src 'self'; "
  11614. "font-src 'self' data: https://fonts.gstatic.com; "
  11615. "worker-src 'self' blob:; "
  11616. "object-src 'none'; "
  11617. "base-uri 'self'; " + _frame_ancestors("'none'")
  11618. )
  11619. elif _STEP_WORKER_ASSET_RE.match(request.url.path):
  11620. # The STEP preview worker (#2976) runs OpenCascade compiled to WASM;
  11621. # its emscripten/embind glue generates invoker functions with `new
  11622. # Function(...)`, which needs 'unsafe-eval'. Per CSP3 a dedicated
  11623. # worker is governed by the policy delivered with the WORKER SCRIPT's
  11624. # own response — not the document's — so relaxing it here confines
  11625. # eval to that DOM-less worker context. The document policy below
  11626. # stays nonce-strict, and this response header has no effect when the
  11627. # file is merely fetched (a fetch's CSP is enforced against the
  11628. # requesting document, not the resource's own headers).
  11629. response.headers["Content-Security-Policy"] = (
  11630. "default-src 'self'; "
  11631. "script-src 'self' 'wasm-unsafe-eval' 'unsafe-eval'; "
  11632. "connect-src 'self'; "
  11633. "object-src 'none'; "
  11634. "base-uri 'self'; " + _frame_ancestors("'none'")
  11635. )
  11636. elif _PDF_WORKER_ASSET_RE.match(request.url.path):
  11637. # pdf.js decodes JPEG2000/JBIG2 images and ICC colour with WebAssembly
  11638. # and fetches those modules from /assets/pdfjs/wasm/ (#2976). Same CSP3
  11639. # rule as the STEP worker above: the policy that governs a dedicated
  11640. # worker is the one delivered with its own script, so the wasm compile
  11641. # has to be permitted here rather than on the document. Unlike the STEP
  11642. # worker this one needs no JS eval, so it gets 'wasm-unsafe-eval' only.
  11643. response.headers["Content-Security-Policy"] = (
  11644. "default-src 'self'; "
  11645. "script-src 'self' 'wasm-unsafe-eval'; "
  11646. "connect-src 'self'; "
  11647. "object-src 'none'; "
  11648. "base-uri 'self'; " + _frame_ancestors("'none'")
  11649. )
  11650. else:
  11651. # The streaming overlay is embedded same-origin by the URL builder's
  11652. # preview in Settings (#1422), so this branch allows 'self'.
  11653. # Embedding from anywhere else is still refused: 'self'
  11654. # only permits a framer on this origin, which is Bambuddy's own UI, so
  11655. # a clickjacking page on another host is blocked exactly as before.
  11656. # (The overlay draws status over a camera feed and its only interactive
  11657. # element is the logo link, so there is nothing to bait a click into
  11658. # even from a same-origin framer.) Cross-origin embedding of the
  11659. # overlay — Home Assistant on another port — remains what
  11660. # TRUSTED_FRAME_ORIGINS is for, and _frame_ancestors already folds that
  11661. # allowlist in.
  11662. #
  11663. # The connected-app consent page (/connect/authorize) gets the same
  11664. # 'self': an app opened from Bambuddy's sidebar runs in an iframe, and
  11665. # its "Sign in with Bambuddy" navigates that iframe to this page. With
  11666. # 'none' the browser refuses to show it even inside Bambuddy. 'self'
  11667. # requires every ancestor to be this origin, so a foreign page -- a
  11668. # sidebar link's site included -- still cannot frame the consent
  11669. # screen to bait a click.
  11670. embeddable_same_origin = request.url.path.startswith("/overlay/") or request.url.path == "/connect/authorize"
  11671. # No 'wasm-unsafe-eval' here: nothing compiles WebAssembly on the main
  11672. # thread. Both wasm consumers — the STEP preview and pdf.js's image
  11673. # decoders (#2976) — run in dedicated workers, which CSP3 governs by
  11674. # the policy served with their own script, so each gets it in its own
  11675. # branch above and the document policy stays as strict as it was.
  11676. response.headers["Content-Security-Policy"] = (
  11677. "default-src 'self'; "
  11678. f"script-src 'self' 'nonce-{csp_nonce}'; "
  11679. "style-src 'self' 'unsafe-inline'; "
  11680. "img-src 'self' data: blob:; "
  11681. "media-src 'self' blob:; "
  11682. "connect-src 'self' ws: wss:; "
  11683. "font-src 'self' data:; "
  11684. "object-src 'none'; "
  11685. "base-uri 'self'; "
  11686. "frame-src 'self' http: https:; " + _frame_ancestors("'self'" if embeddable_same_origin else "'none'")
  11687. )
  11688. if request.url.scheme == "https":
  11689. response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
  11690. return response
  11691. @app.middleware("http")
  11692. async def auth_middleware(request, call_next):
  11693. """Enforce authentication on all API routes when auth is enabled.
  11694. This middleware provides defense-in-depth by checking auth at the API gateway level,
  11695. regardless of whether individual routes have auth dependencies.
  11696. """
  11697. from starlette.responses import JSONResponse
  11698. path = request.url.path
  11699. # Only apply to API routes
  11700. if not path.startswith("/api/"):
  11701. return await call_next(request)
  11702. # Allow public routes
  11703. if path in PUBLIC_API_ROUTES:
  11704. return await call_next(request)
  11705. # Allow public prefixes
  11706. for prefix in PUBLIC_API_PREFIXES:
  11707. if path.startswith(prefix):
  11708. return await call_next(request)
  11709. # Allow public patterns (read-only display data like thumbnails)
  11710. for pattern in PUBLIC_API_PATTERNS:
  11711. if pattern in path:
  11712. return await call_next(request)
  11713. # Check if auth is enabled. Fail CLOSED on any exception during the
  11714. # probe — GHSA-6mf4-q26m-47pv: the previous fail-open path here let
  11715. # an attacker who could force a DB exception (e.g. file-descriptor
  11716. # exhaustion via login flood) bypass auth on every protected endpoint.
  11717. # Only the probe is guarded: the request itself runs outside the try, so an
  11718. # error in a route surfaces as the 500 it is rather than as an auth outage.
  11719. try:
  11720. async with async_session() as db:
  11721. from backend.app.core.auth import is_auth_enabled
  11722. auth_enabled = await is_auth_enabled(db)
  11723. except Exception:
  11724. logging.getLogger(__name__).exception("auth_middleware: failing closed on auth-probe error from %s", path)
  11725. return JSONResponse(
  11726. status_code=503,
  11727. content={"detail": "Authentication service temporarily unavailable"},
  11728. )
  11729. if not auth_enabled:
  11730. # Auth disabled, allow all requests
  11731. return await call_next(request)
  11732. # Auth is enabled - require valid token
  11733. auth_header = request.headers.get("Authorization")
  11734. x_api_key = request.headers.get("X-API-Key")
  11735. # Check for API key auth first
  11736. if x_api_key or (auth_header and auth_header.startswith("Bearer bb_")):
  11737. # API key authentication - let the request through to be validated by route handler
  11738. # API keys are validated per-route since they have different permission levels
  11739. return await call_next(request)
  11740. # Check for JWT auth
  11741. if not auth_header or not auth_header.startswith("Bearer "):
  11742. return JSONResponse(
  11743. status_code=401,
  11744. content={"detail": "Authentication required"},
  11745. headers={"WWW-Authenticate": "Bearer"},
  11746. )
  11747. # Validate JWT token
  11748. import jwt
  11749. try:
  11750. from backend.app.core.auth import (
  11751. ALGORITHM,
  11752. SECRET_KEY,
  11753. _is_token_fresh,
  11754. get_user_by_username,
  11755. is_jti_revoked,
  11756. )
  11757. token = auth_header.replace("Bearer ", "")
  11758. payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
  11759. username = payload.get("sub")
  11760. if not username:
  11761. raise ValueError("No username in token")
  11762. jti = payload.get("jti")
  11763. if not jti:
  11764. raise ValueError("No jti in token")
  11765. iat = payload.get("iat")
  11766. # Verify user exists, is active, and token is still fresh (L-R8-A).
  11767. # Reject revoked tokens first (defense-in-depth gateway check), reusing
  11768. # this session so the gateway adds a single pooled checkout, not two (#2572).
  11769. async with async_session() as db:
  11770. if await is_jti_revoked(jti, db):
  11771. return JSONResponse(
  11772. status_code=401,
  11773. content={"detail": "Token has been revoked"},
  11774. headers={"WWW-Authenticate": "Bearer"},
  11775. )
  11776. user = await get_user_by_username(db, username)
  11777. if not user or not user.is_active:
  11778. return JSONResponse(
  11779. status_code=401,
  11780. content={"detail": "User not found or inactive"},
  11781. headers={"WWW-Authenticate": "Bearer"},
  11782. )
  11783. if not _is_token_fresh(iat, user):
  11784. return JSONResponse(
  11785. status_code=401,
  11786. content={"detail": "Token no longer valid"},
  11787. headers={"WWW-Authenticate": "Bearer"},
  11788. )
  11789. except jwt.ExpiredSignatureError:
  11790. return JSONResponse(
  11791. status_code=401,
  11792. content={"detail": "Token has expired"},
  11793. headers={"WWW-Authenticate": "Bearer"},
  11794. )
  11795. except (jwt.InvalidTokenError, ValueError):
  11796. return JSONResponse(
  11797. status_code=401,
  11798. content={"detail": "Invalid token"},
  11799. headers={"WWW-Authenticate": "Bearer"},
  11800. )
  11801. except Exception:
  11802. # Not a bad token but a failure checking it (database, pool): refuse,
  11803. # as for the probe above, rather than answer 401 -- the frontend reads
  11804. # a 401 from /auth/me as final and signs the user out.
  11805. logging.getLogger(__name__).exception("auth_middleware: failing closed on token-check error from %s", path)
  11806. return JSONResponse(
  11807. status_code=503,
  11808. content={"detail": "Authentication service temporarily unavailable"},
  11809. )
  11810. return await call_next(request)
  11811. @app.middleware("http")
  11812. async def trace_id_middleware(request, call_next):
  11813. """Stamp every HTTP request with a trace ID and echo it back.
  11814. Decorated AFTER auth_middleware on purpose: Starlette stacks
  11815. @app.middleware decorators LIFO, so the last-decorated runs first
  11816. inbound. Putting the trace stamp last makes it the OUTERMOST layer,
  11817. which means auth-middleware log lines (and every line emitted on the
  11818. way down to and back from the route handler) all carry the same
  11819. trace ID. If we put it before auth, auth's logs would be stamped
  11820. with the *previous* request's ID — useless for correlation.
  11821. Honours an inbound ``X-Trace-Id`` header so callers running their
  11822. own tracing can correlate their span IDs with our log lines, but
  11823. only if the value passes the whitelist gate in
  11824. ``backend.app.core.trace.normalise_inbound_trace_id`` — anything
  11825. rejected (too long, contains control chars, etc.) silently triggers
  11826. a freshly minted server-side ID rather than failing the request.
  11827. The minted (or echoed) ID is set on a ContextVar so that every log
  11828. record emitted during the request — application logs *and* uvicorn's
  11829. access log — carries it via TraceIDFilter, and is also written to
  11830. the ``X-Trace-Id`` response header so clients can pin a server-side
  11831. log search to the exact request they made.
  11832. """
  11833. from backend.app.core.trace import (
  11834. generate_trace_id,
  11835. normalise_inbound_trace_id,
  11836. trace_id_var,
  11837. )
  11838. inbound = normalise_inbound_trace_id(request.headers.get("X-Trace-Id"))
  11839. trace_id = inbound if inbound is not None else generate_trace_id()
  11840. token = trace_id_var.set(trace_id)
  11841. try:
  11842. response = await call_next(request)
  11843. finally:
  11844. # Reset the ContextVar so a record emitted in a totally
  11845. # unrelated background task that just happens to inherit this
  11846. # context doesn't keep referencing this request's ID forever.
  11847. # In practice ContextVar.reset is best-effort under asyncio
  11848. # task-spawn semantics, but the cost is one attribute write so
  11849. # we may as well do it.
  11850. trace_id_var.reset(token)
  11851. response.headers["X-Trace-Id"] = trace_id
  11852. return response
  11853. # API routes
  11854. app.include_router(auth.router, prefix=app_settings.api_prefix)
  11855. app.include_router(mfa.router, prefix=app_settings.api_prefix)
  11856. app.include_router(bug_report.router, prefix=app_settings.api_prefix)
  11857. app.include_router(users.router, prefix=app_settings.api_prefix)
  11858. app.include_router(groups.router, prefix=app_settings.api_prefix)
  11859. app.include_router(printers.router, prefix=app_settings.api_prefix)
  11860. app.include_router(printer_locations.router, prefix=app_settings.api_prefix)
  11861. app.include_router(archives.router, prefix=app_settings.api_prefix)
  11862. app.include_router(filaments.router, prefix=app_settings.api_prefix)
  11863. app.include_router(finance.router, prefix=app_settings.api_prefix)
  11864. app.include_router(inventory.router, prefix=app_settings.api_prefix)
  11865. app.include_router(labels.router, prefix=app_settings.api_prefix)
  11866. app.include_router(label_templates.router, prefix=app_settings.api_prefix)
  11867. app.include_router(settings_routes.router, prefix=app_settings.api_prefix)
  11868. app.include_router(overlay_branding.router, prefix=app_settings.api_prefix)
  11869. app.include_router(cloud.router, prefix=app_settings.api_prefix)
  11870. app.include_router(orca_cloud.router, prefix=app_settings.api_prefix)
  11871. app.include_router(local_presets.router, prefix=app_settings.api_prefix)
  11872. app.include_router(smart_plugs.router, prefix=app_settings.api_prefix)
  11873. app.include_router(ha_sensors.router, prefix=app_settings.api_prefix)
  11874. app.include_router(location_ha_sensors.router, prefix=app_settings.api_prefix)
  11875. app.include_router(print_log.router, prefix=app_settings.api_prefix)
  11876. app.include_router(print_queue.router, prefix=app_settings.api_prefix)
  11877. app.include_router(scheduled_dryings.router, prefix=app_settings.api_prefix)
  11878. app.include_router(kprofiles.router, prefix=app_settings.api_prefix)
  11879. app.include_router(notifications.router, prefix=app_settings.api_prefix)
  11880. app.include_router(notification_templates.router, prefix=app_settings.api_prefix)
  11881. app.include_router(user_notifications.router, prefix=app_settings.api_prefix)
  11882. app.include_router(spoolman.router, prefix=app_settings.api_prefix)
  11883. app.include_router(spoolman_inventory.router, prefix=app_settings.api_prefix)
  11884. app.include_router(updates.router, prefix=app_settings.api_prefix)
  11885. app.include_router(sponsor_prompt.router, prefix=app_settings.api_prefix)
  11886. app.include_router(announcements.router, prefix=app_settings.api_prefix)
  11887. app.include_router(maintenance.router, prefix=app_settings.api_prefix)
  11888. app.include_router(camera.router, prefix=app_settings.api_prefix)
  11889. app.include_router(camwall.router, prefix=app_settings.api_prefix)
  11890. app.include_router(external_links.router, prefix=app_settings.api_prefix)
  11891. app.include_router(projects.router, prefix=app_settings.api_prefix)
  11892. app.include_router(library.router, prefix=app_settings.api_prefix)
  11893. app.include_router(library_tags.router, prefix=app_settings.api_prefix)
  11894. app.include_router(library_trash.router, prefix=app_settings.api_prefix)
  11895. app.include_router(library_variants.router, prefix=app_settings.api_prefix)
  11896. app.include_router(slice_jobs.router, prefix=app_settings.api_prefix)
  11897. app.include_router(slicer_pipelines.router, prefix=app_settings.api_prefix)
  11898. app.include_router(pipeline_runs.pipeline_run_create_router, prefix=app_settings.api_prefix)
  11899. app.include_router(pipeline_runs.pipeline_run_router, prefix=app_settings.api_prefix)
  11900. app.include_router(slicer_presets.router, prefix=app_settings.api_prefix)
  11901. app.include_router(archive_purge.router, prefix=app_settings.api_prefix)
  11902. app.include_router(makerworld.router, prefix=app_settings.api_prefix)
  11903. app.include_router(manyfold.router, prefix=app_settings.api_prefix)
  11904. app.include_router(api_keys.router, prefix=app_settings.api_prefix)
  11905. app.include_router(connected_apps.router, prefix=app_settings.api_prefix)
  11906. app.include_router(webhook.router, prefix=app_settings.api_prefix)
  11907. app.include_router(ams_history.router, prefix=app_settings.api_prefix)
  11908. app.include_router(printer_sensor_history.router, prefix=app_settings.api_prefix)
  11909. app.include_router(system.router, prefix=app_settings.api_prefix)
  11910. app.include_router(support.router, prefix=app_settings.api_prefix)
  11911. app.include_router(websocket.router, prefix=app_settings.api_prefix)
  11912. app.include_router(discovery.router, prefix=app_settings.api_prefix)
  11913. app.include_router(pending_uploads.router, prefix=app_settings.api_prefix)
  11914. app.include_router(firmware.router, prefix=app_settings.api_prefix)
  11915. app.include_router(bulk.router, prefix=app_settings.api_prefix)
  11916. app.include_router(github_backup.router, prefix=app_settings.api_prefix)
  11917. app.include_router(local_backup.router, prefix=app_settings.api_prefix)
  11918. app.include_router(obico.router, prefix=app_settings.api_prefix)
  11919. app.include_router(octoeverywhere.router, prefix=app_settings.api_prefix)
  11920. app.include_router(metrics.router, prefix=app_settings.api_prefix)
  11921. app.include_router(virtual_printers.router, prefix=app_settings.api_prefix)
  11922. app.include_router(spoolbuddy.router, prefix=app_settings.api_prefix)
  11923. # Serve static files (React build)
  11924. if app_settings.static_dir.exists() and any(app_settings.static_dir.iterdir()):
  11925. # Compressed, and cached for good where Vite hashed the name (#3175).
  11926. app.mount(
  11927. "/assets",
  11928. AssetStaticFiles(directory=app_settings.static_dir / "assets"),
  11929. name="assets",
  11930. )
  11931. if (app_settings.static_dir / "img").exists():
  11932. app.mount(
  11933. "/img",
  11934. StaticFiles(directory=app_settings.static_dir / "img"),
  11935. name="img",
  11936. )
  11937. if (app_settings.static_dir / "icons").exists():
  11938. app.mount(
  11939. "/icons",
  11940. StaticFiles(directory=app_settings.static_dir / "icons"),
  11941. name="icons",
  11942. )
  11943. # Self-hosted Inter woff2 files (#1460). Without this mount /fonts/*.woff2
  11944. # falls through to the SPA catch-all and returns index.html, which the
  11945. # browser's font sanitizer rejects ("downloadable font: rejected by
  11946. # sanitizer").
  11947. if (app_settings.static_dir / "fonts").exists():
  11948. app.mount(
  11949. "/fonts",
  11950. StaticFiles(directory=app_settings.static_dir / "fonts"),
  11951. name="fonts",
  11952. )
  11953. @app.get("/")
  11954. async def serve_frontend():
  11955. """Serve the React frontend."""
  11956. index_file = app_settings.static_dir / "index.html"
  11957. if index_file.exists():
  11958. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  11959. return {
  11960. "message": "Bambuddy API",
  11961. "docs": "/docs",
  11962. "frontend": "Build and place React app in /static directory",
  11963. }
  11964. # index.html must always be revalidated — Vite emits content-hashed JS/CSS
  11965. # bundles (e.g. `index-JRaF_JhW.js`), so the JS itself is safe to cache
  11966. # forever, but the HTML wrapping it is the only file that knows which hash
  11967. # is current. Without explicit cache-control headers Chromium decides
  11968. # heuristically (typically 10% of the time since Last-Modified) and on
  11969. # long-running kiosks happily serves stale HTML across browser restarts.
  11970. # That stale HTML references an old bundle hash, the old bundle is also
  11971. # in the disk cache, and the user ends up running pre-update JS forever
  11972. # without ever knowing why. ``no-cache`` (revalidate every time, but a
  11973. # 304 is cheap) is the correct setting for an SPA's entry HTML.
  11974. _HTML_CACHE_HEADERS = {"Cache-Control": "no-cache, must-revalidate"}
  11975. @app.get("/health")
  11976. async def health_check():
  11977. """Health check endpoint."""
  11978. return {"status": "healthy"}
  11979. # GET + HEAD on the three PWA bootstrap routes (#1460). Scanners and a plain
  11980. # `curl -I` use HEAD; FastAPI's @app.get only registers GET, so HEAD answers
  11981. # with 405 Method Not Allowed and shows up as a "broken manifest" red herring
  11982. # in deployment debugging. One decorator per method, not api_route(methods=
  11983. # [...]): api_route gives both methods the same OpenAPI operation id.
  11984. @app.get("/manifest.json")
  11985. @app.head("/manifest.json")
  11986. async def serve_manifest():
  11987. """Serve PWA manifest."""
  11988. manifest_file = app_settings.static_dir / "manifest.json"
  11989. if manifest_file.exists():
  11990. return FileResponse(manifest_file, media_type="application/manifest+json")
  11991. return {"error": "Manifest not found"}
  11992. @app.get("/sw.js")
  11993. @app.head("/sw.js")
  11994. async def serve_service_worker():
  11995. """Serve service worker."""
  11996. sw_file = app_settings.static_dir / "sw.js"
  11997. if sw_file.exists():
  11998. return FileResponse(
  11999. sw_file,
  12000. media_type="application/javascript",
  12001. headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
  12002. )
  12003. return {"error": "Service worker not found"}
  12004. @app.get("/sw-register.js")
  12005. @app.head("/sw-register.js")
  12006. async def serve_sw_register():
  12007. """Serve the service-worker registration bootstrap script.
  12008. Served as a real JS file so the strict `script-src 'self'` CSP covers it
  12009. without needing 'unsafe-inline' or per-build hashes on the inline tag.
  12010. """
  12011. reg_file = app_settings.static_dir / "sw-register.js"
  12012. if reg_file.exists():
  12013. return FileResponse(reg_file, media_type="application/javascript")
  12014. return {"error": "sw-register.js not found"}
  12015. # ── GCode viewer static files ────────────────────────────────────────────────
  12016. # Catch-all route for React Router (must be last)
  12017. @app.get("/{full_path:path}")
  12018. async def serve_spa(full_path: str):
  12019. """Serve React app for client-side routing."""
  12020. # Don't intercept API routes - raise proper 404 so FastAPI can handle redirects
  12021. if full_path.startswith("api/"):
  12022. from fastapi import HTTPException
  12023. raise HTTPException(status_code=404, detail="Not found")
  12024. index_file = app_settings.static_dir / "index.html"
  12025. if index_file.exists():
  12026. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  12027. return {"error": "Frontend not built"}