main.py 362 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593
  1. import asyncio
  2. import json
  3. import logging
  4. import mimetypes as _mimetypes
  5. import os
  6. import posixpath
  7. import secrets
  8. import time
  9. from contextlib import asynccontextmanager
  10. from datetime import datetime, timedelta, timezone
  11. from logging.handlers import RotatingFileHandler
  12. from pathlib import Path
  13. from urllib.parse import urlparse
  14. from fastapi import FastAPI
  15. from fastapi.responses import FileResponse
  16. from fastapi.staticfiles import StaticFiles
  17. from sqlalchemy import delete, or_, select, text
  18. from backend.app.api.routes import (
  19. ams_history,
  20. api_keys,
  21. archive_purge,
  22. archives,
  23. auth,
  24. bug_report,
  25. camera,
  26. camwall,
  27. cloud,
  28. discovery,
  29. external_links,
  30. filaments,
  31. firmware,
  32. github_backup,
  33. groups,
  34. inventory,
  35. kprofiles,
  36. labels,
  37. library,
  38. library_tags,
  39. library_trash,
  40. local_backup,
  41. local_presets,
  42. maintenance,
  43. makerworld,
  44. metrics,
  45. mfa,
  46. notification_templates,
  47. notifications,
  48. obico,
  49. orca_cloud,
  50. pending_uploads,
  51. pipeline_runs,
  52. print_log,
  53. print_queue,
  54. printer_sensor_history,
  55. printers,
  56. projects,
  57. settings as settings_routes,
  58. slice_jobs,
  59. slicer_pipelines,
  60. slicer_presets,
  61. smart_plugs,
  62. sponsor_prompt,
  63. spoolbuddy,
  64. spoolman,
  65. spoolman_inventory,
  66. support,
  67. system,
  68. updates,
  69. user_notifications,
  70. users,
  71. virtual_printers,
  72. webhook,
  73. websocket,
  74. )
  75. from backend.app.api.routes.maintenance import _get_printer_maintenance_internal, ensure_default_types
  76. from backend.app.api.routes.support import init_debug_logging
  77. from backend.app.core.config import APP_VERSION, settings as app_settings
  78. from backend.app.core.database import async_session, engine, init_db
  79. from backend.app.core.tasks import spawn_background_task
  80. from backend.app.core.websocket import ws_manager
  81. from backend.app.models.smart_plug import SmartPlug
  82. from backend.app.services.archive import ArchiveService, peek_plate_index_in_3mf, swap_plate_suffix
  83. from backend.app.services.archive_purge import archive_purge_service
  84. from backend.app.services.bambu_ftp import (
  85. FileNotOnPrinterError,
  86. cache_3mf_download,
  87. clear_3mf_cache,
  88. download_file_async,
  89. get_cached_3mf,
  90. get_ftp_retry_settings,
  91. with_ftp_retry,
  92. )
  93. from backend.app.services.bambu_mqtt import PrinterState
  94. from backend.app.services.github_backup import github_backup_service
  95. from backend.app.services.homeassistant import homeassistant_service
  96. from backend.app.services.library_trash import library_trash_service
  97. from backend.app.services.local_backup import local_backup_service
  98. from backend.app.services.mqtt_relay import mqtt_relay
  99. from backend.app.services.mqtt_smart_plug import mqtt_smart_plug_service
  100. from backend.app.services.notification_service import notification_service
  101. from backend.app.services.obico_detection import obico_detection_service
  102. from backend.app.services.print_scheduler import scheduler as print_scheduler
  103. from backend.app.services.printer_manager import (
  104. init_printer_connections,
  105. parse_plate_id,
  106. printer_manager,
  107. printer_state_to_dict,
  108. resolve_plate_id,
  109. )
  110. from backend.app.services.smart_plug_manager import smart_plug_manager
  111. from backend.app.services.spool_assignment_notifications import (
  112. notify_missing_spool_assignments_on_print_start,
  113. )
  114. from backend.app.services.spoolman import close_spoolman_client, get_spoolman_client, init_spoolman_client
  115. from backend.app.services.spoolman_tracking import (
  116. cleanup_tracking as _cleanup_spoolman_tracking,
  117. report_usage as _report_spoolman_usage,
  118. store_print_data as _store_spoolman_print_data,
  119. )
  120. from backend.app.services.tasmota import tasmota_service
  121. # =============================================================================
  122. # Dependency Check - runs before other imports to give helpful error messages
  123. # =============================================================================
  124. def _start_error_server(missing_packages: list):
  125. """Start a minimal HTTP server to display dependency errors in browser."""
  126. import os
  127. import signal
  128. from http.server import BaseHTTPRequestHandler, HTTPServer
  129. packages_html = "".join(f"<li><code>{p}</code></li>" for p in missing_packages)
  130. html = f"""<!DOCTYPE html>
  131. <html>
  132. <head>
  133. <title>Bambuddy - Setup Required</title>
  134. <style>
  135. body {{
  136. font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
  137. background: #0f172a; color: #e2e8f0;
  138. display: flex; justify-content: center; align-items: center;
  139. min-height: 100vh; margin: 0; padding: 20px; box-sizing: border-box;
  140. }}
  141. .container {{
  142. background: #1e293b; border-radius: 12px; padding: 40px;
  143. max-width: 600px; text-align: center; box-shadow: 0 4px 20px rgba(0,0,0,0.3);
  144. }}
  145. h1 {{ color: #f87171; margin-bottom: 10px; }}
  146. h2 {{ color: #94a3b8; font-weight: normal; margin-top: 0; }}
  147. .packages {{
  148. background: #0f172a; border-radius: 8px; padding: 20px;
  149. margin: 20px 0; text-align: left;
  150. }}
  151. .packages ul {{ margin: 0; padding-left: 20px; }}
  152. .packages li {{ color: #fbbf24; margin: 8px 0; }}
  153. .command {{
  154. background: #0f172a; border-radius: 8px; padding: 15px 20px;
  155. margin: 15px 0; font-family: monospace; color: #4ade80;
  156. text-align: left; overflow-x: auto;
  157. }}
  158. .note {{ color: #94a3b8; font-size: 14px; margin-top: 20px; }}
  159. </style>
  160. </head>
  161. <body>
  162. <div class="container">
  163. <h1>Setup Required</h1>
  164. <h2>Missing Python packages</h2>
  165. <div class="packages"><ul>{packages_html}</ul></div>
  166. <p>To fix, run this command on your server:</p>
  167. <div class="command">pip install -r requirements.txt</div>
  168. <p>Or if using a virtual environment:</p>
  169. <div class="command">./venv/bin/pip install -r requirements.txt</div>
  170. <p class="note">After installing, restart Bambuddy:<br>
  171. <code>sudo systemctl restart bambuddy</code></p>
  172. </div>
  173. </body>
  174. </html>"""
  175. class ErrorHandler(BaseHTTPRequestHandler):
  176. def do_GET(self):
  177. self.send_response(503)
  178. self.send_header("Content-type", "text/html")
  179. self.end_headers()
  180. self.wfile.write(html.encode())
  181. def log_message(self, format, *args):
  182. print(f"[Error Server] {args[0]}")
  183. port = int(os.environ.get("PORT", 8000))
  184. print(f"\nStarting error server on http://0.0.0.0:{port}")
  185. print("Visit this URL in your browser to see the error details.\n")
  186. server = HTTPServer(("0.0.0.0", port), ErrorHandler) # nosec B104
  187. def shutdown(signum, frame):
  188. print("\nShutting down error server...")
  189. raise SystemExit(0)
  190. signal.signal(signal.SIGTERM, shutdown)
  191. signal.signal(signal.SIGINT, shutdown)
  192. server.serve_forever()
  193. def check_dependencies():
  194. """Check that all required packages are installed."""
  195. missing = []
  196. # Map of import name -> package name (for pip install)
  197. required = {
  198. "jwt": "PyJWT",
  199. "fastapi": "fastapi",
  200. "uvicorn": "uvicorn",
  201. "sqlalchemy": "sqlalchemy",
  202. "aiosqlite": "aiosqlite",
  203. "pydantic": "pydantic",
  204. "paho.mqtt": "paho-mqtt",
  205. }
  206. for module, package in required.items():
  207. try:
  208. __import__(module)
  209. except ImportError:
  210. missing.append(package)
  211. if missing:
  212. print("\n" + "=" * 60)
  213. print("ERROR: Missing required Python packages!")
  214. print("=" * 60)
  215. print(f"\nMissing packages: {', '.join(missing)}")
  216. print("\nTo fix, run:")
  217. print(" pip install -r requirements.txt")
  218. print("\nOr if using a virtual environment:")
  219. print(" ./venv/bin/pip install -r requirements.txt")
  220. print("=" * 60 + "\n")
  221. _start_error_server(missing)
  222. check_dependencies()
  223. # =============================================================================
  224. # Import settings first for logging configuration
  225. # Configure logging based on settings
  226. # DEBUG=true -> DEBUG level, else use LOG_LEVEL setting
  227. log_level_str = "DEBUG" if app_settings.debug else app_settings.log_level.upper()
  228. log_level = getattr(logging, log_level_str, logging.INFO)
  229. # Trace ID column ([-] when no request scope is active — startup, MQTT
  230. # callbacks, scheduled tasks not chained from a request — so the column
  231. # stays visually aligned and missing values are obvious in grep). See
  232. # backend/app/core/trace.py for the ContextVar that feeds this slot.
  233. log_format = "%(asctime)s %(levelname)s [%(name)s] [%(trace_id)s] %(message)s"
  234. # Create root logger
  235. root_logger = logging.getLogger()
  236. root_logger.setLevel(log_level)
  237. # Trace-ID injection: this filter populates record.trace_id from the
  238. # per-request ContextVar so the format string above can reference it.
  239. # Attached to each HANDLER (not the root logger) because Python's
  240. # logging semantics only invoke a logger's filters on records that
  241. # *originated* at that logger — records propagated up from child
  242. # loggers (every named logger in the app) never trigger root's filter.
  243. # Putting it on the handlers means every record any handler emits gets
  244. # trace_id injected just before the formatter runs, regardless of which
  245. # logger created the record. Without this, the formatter raises
  246. # KeyError on every child-logger record and the record is silently
  247. # dropped — which is exactly the "logs/bambuddy.log only shows logs
  248. # partially" bug we hit. See backend/app/core/trace.py for the
  249. # ContextVar the filter reads.
  250. from backend.app.core.trace import TraceIDFilter
  251. _trace_id_filter = TraceIDFilter()
  252. # Console handler - always enabled
  253. console_handler = logging.StreamHandler()
  254. console_handler.setLevel(log_level)
  255. console_handler.setFormatter(logging.Formatter(log_format))
  256. console_handler.addFilter(_trace_id_filter)
  257. root_logger.addHandler(console_handler)
  258. # File handler - only in production or if explicitly enabled
  259. if app_settings.log_to_file:
  260. log_file = app_settings.log_dir / "bambuddy.log"
  261. file_handler = RotatingFileHandler(
  262. log_file,
  263. maxBytes=app_settings.log_max_bytes,
  264. backupCount=app_settings.log_backup_count,
  265. encoding="utf-8",
  266. )
  267. file_handler.setLevel(log_level)
  268. file_handler.setFormatter(logging.Formatter(log_format))
  269. file_handler.addFilter(_trace_id_filter)
  270. root_logger.addHandler(file_handler)
  271. logging.info("Logging to file: %s", log_file)
  272. # Pipe uvicorn's HTTP access log to bambuddy.log too. Uvicorn ships its
  273. # access logger with propagate=False by default, so without this attach
  274. # there is no on-disk record of which endpoint triggered a server-state
  275. # change — the rogue stop_print mystery on 2026-04-26 was untraceable
  276. # for exactly this reason. Filtered to write methods only
  277. # (POST/PUT/PATCH/DELETE) so the high-volume status-poll GETs from the
  278. # frontend don't churn the rotation window faster than it's useful.
  279. from backend.app.core.logging_filters import (
  280. CancelledPoolNoiseFilter,
  281. WriteRequestsOnlyFilter,
  282. )
  283. uvicorn_access_logger = logging.getLogger("uvicorn.access")
  284. uvicorn_access_logger.addHandler(file_handler)
  285. uvicorn_access_logger.addFilter(WriteRequestsOnlyFilter())
  286. # Uvicorn's access logger has propagate=False (its own default), so the
  287. # root-attached TraceIDFilter never sees these records. Attach a
  288. # second instance directly so HTTP access lines carry the same trace
  289. # ID column as the application logs they correlate with.
  290. uvicorn_access_logger.addFilter(TraceIDFilter())
  291. # Drop SQLAlchemy connection-pool log noise that's caused by Starlette's
  292. # BaseHTTPMiddleware cancelling the inner task scope on client
  293. # disconnect (#1112). The cancel-safe `get_db` already prevents the
  294. # underlying transaction leak; this filter only suppresses the residual
  295. # log records that pre-existing pools still emit during their cleanup.
  296. logging.getLogger("sqlalchemy.pool").addFilter(CancelledPoolNoiseFilter())
  297. # Reduce noise from third-party libraries in production
  298. if not app_settings.debug:
  299. logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
  300. logging.getLogger("httpcore").setLevel(logging.WARNING)
  301. logging.getLogger("httpx").setLevel(logging.WARNING)
  302. logging.getLogger("paho.mqtt").setLevel(logging.WARNING)
  303. logging.info("Bambuddy starting - debug=%s, log_level=%s", app_settings.debug, log_level_str)
  304. # Track active prints: {(printer_id, filename): archive_id}
  305. _active_prints: dict[tuple[int, str], int] = {}
  306. # #1721: stage-22 pre-captured finish photo bytes per printer. on_finish_photo_moment
  307. # fires when stg_cur enters 22 ("Filament unloading") at end-of-print — toolhead
  308. # parked, bed not yet dropped — and grabs a single camera frame into this cache.
  309. # `_background_finish_photo` (inside on_print_complete) consumes the cached bytes
  310. # instead of running its own grab-now chain when present, so the finish photo
  311. # captures the better-framed pre-bed-drop moment without us having to force
  312. # timelapse on at dispatch (the #1397 mechanism that caused #1721's per-layer
  313. # nozzle parking on slicer profiles with Timelapse Type = Smooth).
  314. #
  315. # #2708: the bytes in here are ALWAYS already rotated by the printer's
  316. # camera_rotation. `on_finish_photo_moment` owns that, because one of its
  317. # sources (the #1867 in-print bank) is rotated before it ever reaches the
  318. # bank and the others are not — so the consumer can't tell them apart and
  319. # must not rotate again.
  320. _stage22_finish_frames: dict[int, bytes] = {}
  321. # #1790: per-printer producer-done event. Set by `on_finish_photo_moment` in its
  322. # `finally` block (whether it captured a frame or not). The consumer in
  323. # `_background_finish_photo` waits on it before reading `_stage22_finish_frames`
  324. # so the FINISH-state fallback path — where moment and completion are dispatched
  325. # back-to-back — doesn't race past the producer with an empty pop, and the
  326. # consumer's RTSP fallback can't collide with the producer's still-in-flight RTSP
  327. # grab (Bambu printers allow only one RTSP client at a time).
  328. _stage22_finish_in_flight: dict[int, asyncio.Event] = {}
  329. # #1867: rolling "last in-print camera frame" per printer. Refreshed on
  330. # layer-change while the model is still printing, then consumed by the
  331. # FINISH-state finish-photo path. Firmware that never emits `stg_cur=22`
  332. # (A1 Mini, confirmed) only reaches `on_finish_photo_moment` at the
  333. # gcode_state=FINISH transition — which Bambu reports AFTER the user End
  334. # G-code (e.g. SwapMod plate-swap) has run, so a live grab there captures the
  335. # swapped/empty plate. Banking is layer-driven, so it naturally freezes at the
  336. # final object layer: the End G-code emits no further layer_num increases, so
  337. # the last banked frame is always the finished print before the swap.
  338. _inprint_frame_bank: dict[int, bytes] = {}
  339. # Monotonic timestamp of the last banked frame per printer — throttles banking
  340. # so tall prints don't add a camera grab on every layer.
  341. _inprint_frame_bank_ts: dict[int, float] = {}
  342. # Minimum seconds between banked frames, except the final object layer which
  343. # always refreshes for the best framing.
  344. _INPRINT_BANK_MIN_INTERVAL = 25.0
  345. # Per-printer "connected" edge tracker. Used by `on_printer_status_change`
  346. # to fire `reconcile_stale_active_prints` exactly once per (re)connection
  347. # (#1542 follow-up — power-cycle ghost prints). The value is True after
  348. # the first connected status update for that connection; transitions back
  349. # to False whenever we observe `state.connected = False` so the next
  350. # reconnect re-arms reconciliation. Keyed by printer_id.
  351. _printer_reconciled_since_connect: dict[int, bool] = {}
  352. # Track expected prints from reprint/scheduled (skip auto-archiving for these)
  353. # {(printer_id, filename): archive_id}
  354. _expected_prints: dict[tuple[int, str], int] = {}
  355. # Track AMS mapping for prints: {archive_id: [global_tray_id_per_slot]}
  356. # Used by usage tracker to map 3MF slots to physical AMS trays
  357. _print_ams_mappings: dict[int, list[int]] = {}
  358. # Track plate_id for prints from multi-plate 3MFs: {archive_id: plate_id}
  359. # Used by usage tracker to scope 3MF parsing to the dispatched plate (#1697).
  360. # Populated by direct-Print and queue dispatch paths; queue prints also have a
  361. # redundant queue-item lookup in on_print_start so this dict isn't load-bearing
  362. # for the queue path. Cleared on print completion or TTL eviction.
  363. _print_plate_ids: dict[int, int] = {}
  364. # Track progress milestones for notifications: {printer_id: last_milestone_notified}
  365. # Milestones are 25, 50, 75. Value of 0 means no milestone notified yet for current print.
  366. _last_progress_milestone: dict[int, int] = {}
  367. # Track whether first layer complete notification has been sent for current print
  368. _first_layer_notified: dict[int, bool] = {}
  369. # Track HMS errors that have been notified: {printer_id: set of error codes}
  370. # This prevents sending duplicate notifications for the same error
  371. _notified_hms_errors: dict[int, set[str]] = {}
  372. # Track when HMS errors were last seen: {printer_id: timestamp}
  373. # Used to debounce clearing — prevents flapping errors from re-triggering notifications
  374. _hms_last_seen: dict[int, float] = {}
  375. _HMS_CLEAR_GRACE_SECONDS = 30.0
  376. # Track timelapse file baselines at print start: {printer_id: set of video filenames}
  377. # Used for snapshot-diff detection at print completion
  378. _timelapse_baselines: dict[int, set[str]] = {}
  379. # Track printers waiting for bed to cool after print completion.
  380. # Event-driven: fires when bed_temper arrives via MQTT below threshold.
  381. # {printer_id: {"threshold": float, "filename": str, "registered_at": float}}
  382. _bed_cool_waiters: dict[int, dict] = {}
  383. # Track printers where the user explicitly stopped the print from the queue UI.
  384. # When on_print_complete fires with status "failed" for these printers we treat it
  385. # as "cancelled" (stopped by user) so the correct notification email is sent.
  386. _user_stopped_printers: set[int] = set()
  387. # Offline-notification edge state (#1752): fire `on_printer_offline` exactly
  388. # once when a printer transitions connected → disconnected. `_printer_last_connected`
  389. # holds the previous observation so we only fire on the True → False edge (a
  390. # False → False repeat doesn't notify; an initial False at startup doesn't
  391. # notify either, since there's no prior True). `_printer_offline_notify_tasks`
  392. # holds the per-printer pending asyncio task that fires the notification
  393. # after a debounce window — cancelled if the printer reconnects before the
  394. # window elapses, so transient MQTT blips don't flood the user.
  395. _printer_last_connected: dict[int, bool] = {}
  396. _printer_offline_notify_tasks: dict[int, asyncio.Task] = {}
  397. # Debounce: a printer must stay offline this long before we notify. Sized
  398. # against the staleness path (`bambu_mqtt.py::STALE_RECONNECT_COOLDOWN = 30s`)
  399. # so a single stale-trigger cooldown isn't enough to fire — only a real
  400. # offline that survives one reconnect attempt notifies.
  401. _PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS = 60.0
  402. # HMS short-code → human-readable failure reason. Used by _dispatch_archive_update
  403. # when status="failed" to label the print's failure_reason in archives.
  404. #
  405. # Earlier code matched on `module` alone (e.g. "any module 0x0C HMS → Layer shift"),
  406. # which is wrong on two counts:
  407. # 1. Real layer-shift codes live in module 0x03 (see Bambu wiki), not 0x0C.
  408. # 2. Module 0x0C is "Motion Controller" — broad category that also covers cameras
  409. # and visual markers, AND the H2D firmware emits a 0x0C HMS (0C00_001B, not in
  410. # the public wiki) as part of its user-cancel sequence. Matching on the module
  411. # alone caused user-cancellations to be archived as "Layer shift" failures.
  412. # We now match by full short code only — anything not in this map leaves
  413. # failure_reason=None rather than guessing.
  414. _HMS_FAILURE_REASONS: dict[str, str] = {
  415. # Layer shift / step loss
  416. "0300_4057": "Layer shift",
  417. "0300_4068": "Layer shift",
  418. "0300_800C": "Layer shift",
  419. # Filament runout (printer-side & per-AMS-slot)
  420. "0300_8004": "Filament runout",
  421. "0700_8011": "Filament runout",
  422. "0701_8011": "Filament runout",
  423. "0702_8011": "Filament runout",
  424. "0703_8011": "Filament runout",
  425. "0704_8011": "Filament runout",
  426. "0705_8011": "Filament runout",
  427. "0706_8011": "Filament runout",
  428. "0707_8011": "Filament runout",
  429. "07FF_8011": "Filament runout",
  430. # Clogged nozzle / extruder
  431. "0300_4006": "Clogged nozzle",
  432. "0300_8016": "Clogged nozzle",
  433. "0300_801C": "Clogged nozzle",
  434. "0700_8003": "Clogged nozzle",
  435. "0700_8007": "Clogged nozzle",
  436. "0700_8013": "Clogged nozzle",
  437. "0701_8003": "Clogged nozzle",
  438. "0701_8007": "Clogged nozzle",
  439. "0701_8013": "Clogged nozzle",
  440. "0702_8003": "Clogged nozzle",
  441. }
  442. def _hms_short_code(attr: int, code: int | str) -> str:
  443. """Build the canonical "MMMM_CCCC" HMS short code from raw attr/code values."""
  444. if isinstance(code, str):
  445. code_int = int(code.replace("0x", ""), 16) if code else 0
  446. else:
  447. code_int = int(code or 0)
  448. attr_int = int(attr or 0)
  449. return f"{(attr_int >> 16) & 0xFFFF:04X}_{code_int & 0xFFFF:04X}"
  450. def derive_failure_reason(status: str, hms_errors: list[dict] | None) -> str | None:
  451. """Derive a human-readable failure_reason for an archived print.
  452. Returns "User cancelled" for cancelled/aborted prints; for failed prints,
  453. returns the first matching reason from _HMS_FAILURE_REASONS, or None when
  454. no HMS code matches (don't guess — null is honest).
  455. """
  456. if status in ("aborted", "cancelled"):
  457. return "User cancelled"
  458. if status != "failed":
  459. return None
  460. for err in hms_errors or []:
  461. short_code = _hms_short_code(err.get("attr", 0), err.get("code", 0))
  462. if short_code in _HMS_FAILURE_REASONS:
  463. return _HMS_FAILURE_REASONS[short_code]
  464. return None
  465. # Track created_by_id for expected prints so the user email can be sent even when
  466. # the archive itself doesn't have created_by_id set (e.g. library-file-based prints).
  467. # {(printer_id, filename): created_by_id}
  468. _expected_print_creators: dict[tuple[int, str], int] = {}
  469. # Per-printer lock that serialises the spool-assignment side of on_ams_change
  470. # (auto-unlink stale + auto-assign new) when MQTT bursts deliver multiple AMS
  471. # updates for the same printer in quick succession (~30 ms apart, observed in
  472. # the wild on H2D + dual AMS).
  473. #
  474. # Without this serialisation, two concurrent on_ams_change callbacks each read
  475. # "no assignment for (printer, ams, tray)", each call auto_assign_spool, and
  476. # the second commit hits
  477. # IntegrityError: duplicate key value violates unique constraint
  478. # "spool_assignment_printer_id_ams_id_tray_id_key"
  479. # SQLite's WAL serial-write semantics had been silently swallowing the race
  480. # until optional Postgres support landed (asyncpg allows true concurrent
  481. # transactions and surfaces the constraint violation).
  482. #
  483. # Scope is intentionally narrow: only the two DB-mutating blocks (unlink +
  484. # assign) are inside the lock. The Spoolman sync block further down stays
  485. # concurrent because it's network-bound and idempotent.
  486. _ams_assignment_locks: dict[int, asyncio.Lock] = {}
  487. def _get_ams_assignment_lock(printer_id: int) -> asyncio.Lock:
  488. """Return the per-printer assignment lock, creating it on first use."""
  489. lock = _ams_assignment_locks.get(printer_id)
  490. if lock is None:
  491. lock = asyncio.Lock()
  492. _ams_assignment_locks[printer_id] = lock
  493. return lock
  494. # Per-printer dedup for unknown_tag WS broadcasts. Keyed by
  495. # (ams_id, tray_id) -> (tag_uid, tray_uuid); we only re-broadcast when the
  496. # tag tuple changes for the slot. Cleared when the slot is reported empty
  497. # so remove + reinsert reliably re-prompts the UI.
  498. _unknown_tag_last_broadcast: dict[int, dict[tuple[int, int], tuple[str, str]]] = {}
  499. async def _broadcast_unknown_tag(
  500. *,
  501. printer_id: int,
  502. ams_id: int,
  503. tray_id: int,
  504. tag_uid: str,
  505. tray_uuid: str,
  506. tray_type: str | None = None,
  507. tray_color: str | None = None,
  508. tray_sub_brands: str | None = None,
  509. tray_count: int | None = None,
  510. ) -> None:
  511. """Broadcast unknown_tag, deduped so repeated MQTT pushes for the same slot+tag don't spam the UI."""
  512. _logger = logging.getLogger(__name__)
  513. slot_key = (ams_id, tray_id)
  514. tag_key = (tag_uid or "", tray_uuid or "")
  515. per_printer = _unknown_tag_last_broadcast.setdefault(printer_id, {})
  516. if per_printer.get(slot_key) == tag_key:
  517. _logger.debug(
  518. "unknown_tag deduped for printer=%d AMS=%d slot=%d tag=%s",
  519. printer_id,
  520. ams_id,
  521. tray_id,
  522. tag_key[0][:8] or tag_key[1][:8] or "(none)",
  523. )
  524. return
  525. _logger.info(
  526. "unknown_tag broadcast: printer=%d AMS=%d slot=%d type=%r color=%r tag=%s",
  527. printer_id,
  528. ams_id,
  529. tray_id,
  530. tray_type,
  531. tray_color,
  532. tag_key[0][:8] or tag_key[1][:8] or "(none)",
  533. )
  534. # Broadcast first; only commit the dedup if the WS write succeeds.
  535. # If broadcast raises, the next MQTT push retries instead of being
  536. # permanently silenced by a poisoned dedup entry.
  537. await ws_manager.broadcast(
  538. {
  539. "type": "unknown_tag",
  540. "printer_id": printer_id,
  541. "ams_id": ams_id,
  542. "tray_id": tray_id,
  543. "tag_uid": tag_uid,
  544. "tray_uuid": tray_uuid,
  545. "tray_type": tray_type,
  546. "tray_color": tray_color,
  547. "tray_sub_brands": tray_sub_brands,
  548. "tray_count": tray_count,
  549. }
  550. )
  551. per_printer[slot_key] = tag_key
  552. def _clear_unknown_tag_dedup(printer_id: int, ams_id: int, tray_id: int) -> None:
  553. """Drop the cached last-broadcast tag for a slot (called when slot reports empty or gets matched)."""
  554. per_printer = _unknown_tag_last_broadcast.get(printer_id)
  555. if per_printer is None:
  556. return
  557. per_printer.pop((ams_id, tray_id), None)
  558. # TTL for expected-print entries: evict registrations older than this to prevent
  559. # unbounded growth when a print is registered but never starts (e.g. printer
  560. # disconnect, app restart, print started from the printer panel).
  561. _EXPECTED_PRINT_TTL_SECONDS: int = 2 * 60 * 60 # 2 hours
  562. # Registration timestamps used for TTL eviction: {(printer_id, filename): monotonic_time}
  563. _expected_print_registered_at: dict[tuple[int, str], float] = {}
  564. # Cleanup loop interval
  565. _EXPECTED_PRINT_CLEANUP_INTERVAL: int = 15 * 60 # 15 minutes
  566. _expected_prints_cleanup_task: asyncio.Task | None = None
  567. async def _get_plug_energy(plug, db) -> dict | None:
  568. """Get energy from plug regardless of type (Tasmota, Home Assistant, MQTT, or REST).
  569. For HA plugs, configures the service with current settings from DB.
  570. For MQTT plugs, returns data from the subscription service.
  571. For REST plugs, polls the status URL with JSON path extraction.
  572. """
  573. if plug.plug_type == "homeassistant":
  574. from backend.app.api.routes.settings import get_homeassistant_settings
  575. ha_settings = await get_homeassistant_settings(db)
  576. homeassistant_service.configure(ha_settings["ha_url"], ha_settings["ha_token"])
  577. return await homeassistant_service.get_energy(plug)
  578. elif plug.plug_type == "mqtt":
  579. # MQTT plugs report "today" energy, not lifetime total
  580. # For per-print tracking, we use "today" as the counter (resets at midnight)
  581. mqtt_data = mqtt_relay.smart_plug_service.get_plug_data(plug.id)
  582. if mqtt_data:
  583. return {
  584. "power": mqtt_data.power,
  585. "today": mqtt_data.energy,
  586. "total": mqtt_data.energy, # Use today as total for per-print calculations
  587. }
  588. return None
  589. elif plug.plug_type == "rest":
  590. from backend.app.services.rest_smart_plug import rest_smart_plug_service
  591. return await rest_smart_plug_service.get_energy(plug)
  592. else:
  593. return await tasmota_service.get_energy(plug)
  594. async def _record_energy_start(archive, printer_id: int, db, *, context: str = "") -> bool:
  595. """Capture the smart plug lifetime counter on the archive at print start.
  596. Persists `energy_start_kwh` on the archive row (#941) so per-print energy
  597. tracking survives a backend restart mid-print. The print-end handler reads
  598. this value back from the DB and computes the delta against the current
  599. plug counter.
  600. """
  601. _logger = logging.getLogger(__name__)
  602. try:
  603. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  604. plug = plug_result.scalar_one_or_none()
  605. if not plug:
  606. _logger.info("[ENERGY] No smart plug for printer %s (archive %s)", printer_id, archive.id)
  607. return False
  608. energy = await _get_plug_energy(plug, db)
  609. if not energy or energy.get("total") is None:
  610. _logger.warning("[ENERGY] No 'total' in energy response for archive %s", archive.id)
  611. return False
  612. archive.energy_start_kwh = float(energy["total"])
  613. await db.commit()
  614. _logger.info(
  615. "[ENERGY] Recorded starting energy%s for archive %s: %s kWh",
  616. f" ({context})" if context else "",
  617. archive.id,
  618. energy["total"],
  619. )
  620. return True
  621. except Exception as e:
  622. _logger.warning("[ENERGY] Failed to record starting energy for archive %s: %s", archive.id, e)
  623. return False
  624. def register_expected_print(
  625. printer_id: int,
  626. filename: str,
  627. archive_id: int,
  628. ams_mapping: list[int] | None = None,
  629. created_by_id: int | None = None,
  630. plate_id: int | None = None,
  631. ):
  632. """Register an expected print from reprint/scheduled so we don't create duplicate archives."""
  633. # Store with multiple filename variations to catch different naming patterns
  634. _expected_prints[(printer_id, filename)] = archive_id
  635. # Also store without .3mf extension if present
  636. if filename.endswith(".3mf"):
  637. base = filename[:-4]
  638. _expected_prints[(printer_id, base)] = archive_id
  639. _expected_prints[(printer_id, f"{base}.gcode")] = archive_id
  640. # Store AMS mapping for usage tracking at print completion
  641. if ams_mapping is not None:
  642. _print_ams_mappings[archive_id] = ams_mapping
  643. # Store plate_id for usage tracking when this is a single-plate dispatch from
  644. # a multi-plate 3MF — without this, the direct-Print path attributes the whole
  645. # file's filament total to the spool instead of just the printed plate (#1697).
  646. if plate_id is not None:
  647. _print_plate_ids[archive_id] = plate_id
  648. # Store created_by_id so the user start email can be sent even when the archive
  649. # itself has no created_by_id (e.g. library-file-based queue prints)
  650. if created_by_id is not None:
  651. _expected_print_creators[(printer_id, filename)] = created_by_id
  652. if filename.endswith(".3mf"):
  653. base = filename[:-4]
  654. _expected_print_creators[(printer_id, base)] = created_by_id
  655. _expected_print_creators[(printer_id, f"{base}.gcode")] = created_by_id
  656. # Record registration time for TTL-based eviction
  657. _registered_at = time.monotonic()
  658. _expected_print_registered_at[(printer_id, filename)] = _registered_at
  659. if filename.endswith(".3mf"):
  660. base = filename[:-4]
  661. _expected_print_registered_at[(printer_id, base)] = _registered_at
  662. _expected_print_registered_at[(printer_id, f"{base}.gcode")] = _registered_at
  663. logging.getLogger(__name__).info(
  664. f"Registered expected print: printer={printer_id}, file={filename}, archive={archive_id}, ams_mapping={ams_mapping}, plate_id={plate_id}"
  665. )
  666. def unregister_expected_print(printer_id: int, filename: str, archive_id: int) -> None:
  667. """Undo :func:`register_expected_print` when the print never went out.
  668. Registration has to happen *before* the MQTT print command, because the
  669. printer can report the print before the line after the send executes. So
  670. every path that registers and then fails to send — a cancel winning the
  671. #1853 CAS race, a ``start_print()`` that returns False, or any exception in
  672. between — leaves an expectation for a print that will never arrive.
  673. The TTL sweep evicts those after two hours, which is far longer than it
  674. takes a user to react to a failed dispatch by pressing print again: that
  675. reprint would be folded into the *old* archive and take the stale
  676. ``ams_mapping`` / ``plate_id`` with it. Hence the explicit inverse.
  677. Mirrors the sweep's rules, including the one that is easy to get wrong:
  678. ``_print_ams_mappings`` / ``_print_plate_ids`` are keyed by archive, not by
  679. file, so they may only be dropped once no live key still points at that
  680. archive.
  681. """
  682. keys = [(printer_id, filename)]
  683. if filename.endswith(".3mf"):
  684. base = filename[:-4]
  685. keys.append((printer_id, base))
  686. keys.append((printer_id, f"{base}.gcode"))
  687. removed = False
  688. for key in keys:
  689. if _expected_prints.pop(key, None) is not None:
  690. removed = True
  691. _expected_print_creators.pop(key, None)
  692. _expected_print_registered_at.pop(key, None)
  693. if archive_id not in set(_expected_prints.values()):
  694. _print_ams_mappings.pop(archive_id, None)
  695. _print_plate_ids.pop(archive_id, None)
  696. if removed:
  697. logging.getLogger(__name__).info(
  698. "Unregistered expected print: printer=%s, file=%s, archive=%s (print was never sent)",
  699. printer_id,
  700. filename,
  701. archive_id,
  702. )
  703. def _compute_run_filament_grams(
  704. status: str,
  705. archive_filament_used_grams: float | None,
  706. progress: float | int | None,
  707. usage_results: list[dict] | None,
  708. ) -> float | None:
  709. """Per-run filament for PrintLogEntry, partial- and tracker-aware (#1378, #1390).
  710. Priority for every status:
  711. 1. Sum of tracked spool deltas in ``usage_results`` (AMS-measured
  712. weight delta — same source that drives "Total Consumed" on the
  713. Inventory page, so Stats and Inventory totals stay aligned).
  714. 2. For ``completed``: the slicer estimate (no tracker available, fall
  715. back to the canonical "this print used X" value).
  716. 3. For partial statuses: ``estimate * progress%``.
  717. 4. ``None`` if nothing is known.
  718. """
  719. tracked_grams = sum(r.get("weight_used") or 0 for r in (usage_results or []))
  720. if tracked_grams > 0:
  721. return round(tracked_grams, 1)
  722. if status == "completed":
  723. return archive_filament_used_grams
  724. if archive_filament_used_grams:
  725. scale = max(0.0, min(((progress or 0) / 100.0), 1.0))
  726. if scale > 0:
  727. return round(archive_filament_used_grams * scale, 1)
  728. return None
  729. def _plate_scoped_run_estimate(archive, full_path) -> tuple[float | None, float | None]:
  730. """Per-run (grams, cost) scoped to the plate this run actually printed (#2614).
  731. ``PrintArchive.filament_used_grams`` / ``.cost`` are the sum over EVERY plate of
  732. the source 3MF — correct for the archive card and project rollup, but wrong for a
  733. single plate dispatched from a multi-plate file: without scoping, each printed
  734. plate of a 22-plate file logs the whole ~12 kg and inflates every statistic. When
  735. the archive carries a ``plate_id`` and its 3MF is on disk, return that plate's
  736. slicer estimate instead; cost is scaled by the plate's share of the whole so it
  737. stays consistent with the scoped grams without re-doing the filament price lookup.
  738. Falls back to the archive's whole-file values when there's no plate to scope to.
  739. """
  740. whole_grams = archive.filament_used_grams
  741. if archive.plate_id is None or full_path is None or not full_path.exists():
  742. return whole_grams, archive.cost
  743. try:
  744. from backend.app.utils.threemf_tools import extract_plate_metadata_from_3mf
  745. plate_grams = extract_plate_metadata_from_3mf(full_path, archive.plate_id).filament_used_grams
  746. except Exception as exc:
  747. logging.getLogger(__name__).debug(
  748. "[#2614] plate-scoped estimate failed for archive %s (plate %s): %s",
  749. archive.id,
  750. archive.plate_id,
  751. exc,
  752. )
  753. return whole_grams, archive.cost
  754. if not plate_grams or plate_grams <= 0:
  755. return whole_grams, archive.cost
  756. plate_cost = archive.cost
  757. if archive.cost and whole_grams and whole_grams > 0:
  758. plate_cost = round(archive.cost * (plate_grams / whole_grams), 2)
  759. return round(plate_grams, 2), plate_cost
  760. def _get_start_ams_mapping(data: dict, archive_id: int | None) -> list[int] | None:
  761. """Resolve AMS mapping for print start without consuming stored queue/reprint state."""
  762. stored_ams_mapping = data.get("ams_mapping")
  763. if not stored_ams_mapping and archive_id:
  764. stored_ams_mapping = _print_ams_mappings.get(archive_id)
  765. return stored_ams_mapping
  766. def _get_start_plate_id(archive_id: int | None) -> int | None:
  767. """Resolve plate_id for print start without consuming stored direct-Print state.
  768. Direct-Print of a single plate from a multi-plate 3MF registers plate_id in
  769. ``_print_plate_ids`` at dispatch time; this lets the spoolman / usage tracker
  770. read it back at print-start without popping (the entry is popped on print
  771. completion or TTL eviction, mirroring ``_print_ams_mappings``).
  772. """
  773. if archive_id is None:
  774. return None
  775. return _print_plate_ids.get(archive_id)
  776. def _partial_progress_scale(progress: int | float | None) -> float:
  777. """Clamp ``progress / 100`` into [0.0, 1.0] for partial-print scaling.
  778. Used by every site that multiplies a "would-have-used" slicer estimate
  779. down to "actually-used" for failed / cancelled / stopped prints. Centralised
  780. so the three sites in ``_background_notifications`` (and the per-plate
  781. override helper) can't drift apart on the coercion shape.
  782. """
  783. return max(0.0, min((progress or 0) / 100.0, 1.0))
  784. def _scope_notification_archive_data_to_plate(
  785. archive_data: dict,
  786. archive_file_path: str | None,
  787. plate_id: int | None,
  788. print_status: str,
  789. progress: int | float | None,
  790. base_dir: Path,
  791. ) -> dict:
  792. """Override summed-across-plates totals in ``archive_data`` with the values
  793. for ``plate_id`` so the completion notification reports what was actually
  794. printed, not the whole project (#1785).
  795. The 3MF parser at services/archive.py:200-264 sums ``prediction`` and
  796. ``weight`` across every plate of a multi-plate file (#1593) — correct for
  797. the archive card's "whole project" headline, wrong for the completion
  798. notification of a single-plate print. The queue UI already re-reads the
  799. 3MF per-plate at print_queue.py:272-285; this helper mirrors that for the
  800. notification payload (filament grams, time estimate, per-slot breakdown).
  801. No-ops when ``plate_id`` is None, the file is missing, or the 3MF carries
  802. no per-plate values — in every fail case the original ``archive_data`` is
  803. returned unchanged so the notification still sends.
  804. """
  805. if plate_id is None or not archive_file_path:
  806. return archive_data
  807. from backend.app.utils.threemf_tools import (
  808. extract_filament_usage_from_3mf,
  809. extract_print_time_from_3mf,
  810. )
  811. archive_path = base_dir / archive_file_path
  812. if not archive_path.exists():
  813. return archive_data
  814. plate_slots = extract_filament_usage_from_3mf(archive_path, plate_id)
  815. plate_grams = sum(f.get("used_g", 0) for f in plate_slots)
  816. plate_time = extract_print_time_from_3mf(archive_path, plate_id)
  817. scale = 1.0 if print_status == "completed" else _partial_progress_scale(progress)
  818. if plate_time:
  819. archive_data["print_time_seconds"] = plate_time
  820. # Gate both the grams headline AND the per-slot breakdown on the same
  821. # `plate_grams > 0` signal: if the 3MF carries per-plate filament rows but
  822. # they all sum to zero (slicer bug / re-slice without estimate), drop back
  823. # to the project-level grams the archive columns already provide rather
  824. # than ship a project-level headline next to an all-zero per-plate
  825. # breakdown.
  826. if plate_grams > 0:
  827. archive_data["actual_filament_grams"] = round(plate_grams * scale, 1)
  828. archive_data["filament_slots"] = [
  829. {
  830. "slot_id": s.get("slot_id"),
  831. "used_g": round((s.get("used_g") or 0) * scale, 1),
  832. "type": s.get("type", ""),
  833. "color": s.get("color", ""),
  834. }
  835. for s in plate_slots
  836. ]
  837. return archive_data
  838. def _extract_filament_data_from_mqtt(data: dict, ams_mapping: list[int] | None = None) -> dict[str, str]:
  839. """Best-effort filament metadata from the MQTT print-start snapshot.
  840. Used when the 3MF can't be downloaded (P1S/A1/P2S firmwares lock the
  841. file during print, see #1533) so the fallback PrintArchive still has
  842. enough filament info to support the inventory views and AMS-expansion
  843. planning the operator opens it for. Returns a dict with optional
  844. ``filament_type`` and ``filament_color`` keys in the same
  845. comma-separated format the 3MF extractor produces, so the rest of the
  846. codebase treats the fallback archive identically to a normal one.
  847. ``ams_mapping`` is the slicer's slot-per-print-filament list captured
  848. from the MQTT print payload (global tray IDs, possibly -1 for VT-tray
  849. entries). When supplied, only the slots actually consumed by this
  850. print contribute. Without it the function falls back to every loaded
  851. AMS slot — less accurate but still useful.
  852. Accepts both the raw inner payload (``{"ams": {"ams": [...]}, ...}``)
  853. that the unit tests pass directly, AND the on_print_start callback
  854. shape (``{"raw_data": {"ams": {"ams": [...]}, ...}, ...}``) the
  855. bambu_mqtt service hands to main.py at runtime. The original
  856. ``_extract_filament_data_from_mqtt(data)`` shipped in #1533 only
  857. handled the inner shape and silently returned ``{}`` for every real
  858. print start, leaving fallback archives' filament fields NULL — the
  859. exact regression the fix was meant to close. Reported with a log
  860. proving the AMS state was right there at
  861. ``data["raw_data"]["ams"]["ams"][0]["tray"][0]`` (#1533 follow-up).
  862. """
  863. result: dict[str, str] = {}
  864. # Look at the on_print_start wrapper first, then the inner shape.
  865. raw_data = (data or {}).get("raw_data")
  866. ams_root = (raw_data or {}).get("ams") if isinstance(raw_data, dict) else None
  867. if not isinstance(ams_root, dict):
  868. ams_root = (data or {}).get("ams") or {}
  869. ams_units = ams_root.get("ams") if isinstance(ams_root, dict) else None
  870. if not isinstance(ams_units, list) or not ams_units:
  871. return result
  872. # Map global tray id (unit * 4 + tray) → (type, color).
  873. loaded: dict[int, tuple[str, str]] = {}
  874. for unit in ams_units:
  875. if not isinstance(unit, dict):
  876. continue
  877. try:
  878. unit_id = int(unit.get("id", 0))
  879. except (TypeError, ValueError):
  880. continue
  881. for tray in unit.get("tray") or []:
  882. if not isinstance(tray, dict):
  883. continue
  884. try:
  885. tray_id = int(tray.get("id", 0))
  886. except (TypeError, ValueError):
  887. continue
  888. ttype = (tray.get("tray_type") or "").strip()
  889. tcolor = (tray.get("tray_color") or "").strip().upper()
  890. if not ttype:
  891. continue # Empty / unloaded slot.
  892. loaded[unit_id * 4 + tray_id] = (ttype, tcolor)
  893. if not loaded:
  894. return result
  895. if ams_mapping:
  896. used_ids = [int(x) for x in ams_mapping if isinstance(x, (int, float)) and int(x) >= 0]
  897. filaments = [loaded[g] for g in used_ids if g in loaded]
  898. if not filaments:
  899. return result # Mapping points entirely at slots we have no data for.
  900. else:
  901. filaments = [loaded[g] for g in sorted(loaded.keys())]
  902. types_joined = ",".join(f[0] for f in filaments)
  903. colors_joined = ",".join(f[1] for f in filaments if f[1])
  904. # Column limits per backend/app/models/archive.py: filament_type=50,
  905. # filament_color=200.
  906. if types_joined:
  907. result["filament_type"] = types_joined[:50]
  908. if colors_joined:
  909. result["filament_color"] = colors_joined[:200]
  910. return result
  911. def _maybe_start_layer_timelapse(printer, printer_id: int, archive_id: int) -> bool:
  912. """Start a layer-timelapse session for *archive_id* when the printer has
  913. an external camera configured. Returns True if a session was started.
  914. Three call sites in on_print_start (expected-archive promotion, fallback
  915. archive creation, fresh-archive creation) used to inline this same
  916. if-block; the inline copies kept drifting (#1353 fixed only one of them
  917. on the first pass). Centralising the conditional + call here makes the
  918. contract testable in isolation and keeps the three sites locked in step.
  919. """
  920. if not (printer.external_camera_enabled and printer.external_camera_url):
  921. return False
  922. from backend.app.services.layer_timelapse import start_session
  923. start_session(
  924. printer_id,
  925. archive_id,
  926. printer.external_camera_url,
  927. printer.external_camera_type or "mjpeg",
  928. snapshot_url=printer.external_camera_snapshot_url,
  929. rotation=getattr(printer, "camera_rotation", 0),
  930. )
  931. logging.getLogger(__name__).info("Started layer timelapse for printer %s, archive %s", printer_id, archive_id)
  932. return True
  933. def _format_hms_error_summary(hms_errors: list[dict]) -> str | None:
  934. """Build a human-readable failure reason from MQTT hms_errors for PrintQueueItem.error_message.
  935. Each entry has keys: code ('0x4038'), attr (32-bit int), module, severity.
  936. The short code used for the hms_errors.py lookup table is 'MMMM_EEEE' — module
  937. from attr bits 16-31, error from the numeric part of code. Falls back to the raw
  938. short code when no description is on file. Returns None for an empty list so
  939. callers can leave error_message unset.
  940. """
  941. if not hms_errors:
  942. return None
  943. from backend.app.services.hms_errors import get_error_description
  944. parts: list[str] = []
  945. for err in hms_errors:
  946. try:
  947. code_str = str(err.get("code", "")).replace("0x", "")
  948. error_num = int(code_str, 16) if code_str else 0
  949. module_num = (int(err.get("attr", 0)) >> 16) & 0xFFFF
  950. short_code = f"{module_num:04X}_{error_num:04X}"
  951. except (TypeError, ValueError):
  952. continue
  953. description = get_error_description(short_code)
  954. parts.append(f"[{short_code}] {description}" if description else f"[{short_code}]")
  955. return "; ".join(parts) if parts else None
  956. async def _bump_library_file_usage_if_completed(db, item, queue_status: str) -> None:
  957. """Increment LibraryFile.print_count and stamp last_printed_at when a queued
  958. print completes successfully. Gated to status=='completed': failed, cancelled
  959. and aborted prints do not count as usage. Caller is responsible for committing
  960. the session. No-op when the queue item has no linked library file (e.g. reprints
  961. from an archive). See #1008."""
  962. if queue_status != "completed" or item.library_file_id is None:
  963. return
  964. from backend.app.models.library import LibraryFile
  965. lib_file = await db.scalar(select(LibraryFile).where(LibraryFile.id == item.library_file_id))
  966. if lib_file is None:
  967. return
  968. lib_file.print_count = (lib_file.print_count or 0) + 1
  969. lib_file.last_printed_at = datetime.now(timezone.utc)
  970. def mark_printer_stopped_by_user(printer_id: int) -> None:
  971. """Mark that the active print on this printer was stopped by the user from the queue UI.
  972. When on_print_complete fires with status 'failed' for a printer in this set we
  973. reclassify it as 'cancelled' so the correct 'print stopped' notification is sent
  974. rather than a 'print failed' notification.
  975. """
  976. _user_stopped_printers.add(printer_id)
  977. logging.getLogger(__name__).info("Marked printer %s as user-stopped from queue", printer_id)
  978. _last_status_broadcast: dict[int, str] = {}
  979. # Track printers where we've updated nozzle_count
  980. _nozzle_count_updated: set[int] = set()
  981. async def _maybe_notify_printer_offline(printer_id: int) -> None:
  982. """Wait the debounce window then fire `on_printer_offline` if the printer
  983. is still offline.
  984. Scheduled by `on_printer_status_change` on the connected → disconnected
  985. edge (#1752). Cancelled by the same handler if the printer reconnects
  986. before the window elapses, so a single MQTT blip + recovery doesn't
  987. notify. Both the staleness-detector path (`bambu_mqtt.py::check_staleness`)
  988. and the smart-plug power-off path (`printer_manager.mark_printer_offline`)
  989. route through the same status-change callback, so this covers both.
  990. """
  991. logger = logging.getLogger(__name__)
  992. try:
  993. await asyncio.sleep(_PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS)
  994. still_offline = not printer_manager.is_connected(printer_id)
  995. logger.info(
  996. "[#1752] Printer %s offline debounce elapsed: still_offline=%s",
  997. printer_id,
  998. still_offline,
  999. )
  1000. if not still_offline:
  1001. return
  1002. async with async_session() as db:
  1003. from backend.app.models.printer import Printer
  1004. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1005. printer = result.scalar_one_or_none()
  1006. if not printer:
  1007. logger.warning(
  1008. "[#1752] Printer %s missing from DB at offline-notify time; skipping",
  1009. printer_id,
  1010. )
  1011. return
  1012. logger.info(
  1013. "[#1752] Dispatching on_printer_offline for printer %s (%s)",
  1014. printer_id,
  1015. printer.name,
  1016. )
  1017. await notification_service.on_printer_offline(printer_id, printer.name, db)
  1018. except asyncio.CancelledError:
  1019. raise
  1020. except Exception as e:
  1021. logger.warning("Printer offline notification failed for printer %s: %s", printer_id, e)
  1022. finally:
  1023. _printer_offline_notify_tasks.pop(printer_id, None)
  1024. async def on_printer_status_change(printer_id: int, state: PrinterState):
  1025. """Handle printer status changes - broadcast via WebSocket."""
  1026. # Connected-edge reconciliation (#1542 follow-up). When the printer
  1027. # transitions disconnected → connected — which covers both Bambuddy
  1028. # startup (no prior connection) and a mid-session MQTT reconnect — fire
  1029. # `reconcile_stale_active_prints` exactly once for this connection so
  1030. # any archive still in `status="printing"` that can't actually be
  1031. # running anymore (printer IDLE / different subtask / empty subtask)
  1032. # gets a synthesised PRINT COMPLETE. Without this, a print that
  1033. # finished during a disconnect window + a smart-plug power cycle
  1034. # leaves the .3mf on the SD card and the firmware ghost-replays it on
  1035. # next boot. Reconciliation runs concurrently — it must not block the
  1036. # WebSocket dedup / broadcast logic below, and the connected edge is
  1037. # marked True BEFORE the await so concurrent status updates inside
  1038. # the same connection don't re-trigger reconciliation.
  1039. #
  1040. # Wait for a real push_status before reconciling (#1679): MQTT
  1041. # `_on_connect` broadcasts `state` IMMEDIATELY after the broker accepts
  1042. # the connection, BEFORE `_request_push_all` round-trips. At that
  1043. # instant the `PrinterState` is still on construction defaults — most
  1044. # importantly `state.state == "unknown"` and `state.subtask_name == ""`.
  1045. # If reconcile spawns here, every in-flight archive falls through to
  1046. # the empty-subtask_name trigger and gets synthesised `aborted`, which
  1047. # creates a duplicate archive on the real PRINT COMPLETE and
  1048. # double-counts filament. Gating on `state.state ∉ ("", "unknown")`
  1049. # keeps the #1542 mechanism intact: once the first real push_status
  1050. # updates `state.state` (RUNNING / IDLE / FINISH / …), this handler
  1051. # fires again with the flag still False — reconcile then runs against
  1052. # actual evidence.
  1053. state_known = bool(state.state) and state.state.upper() not in ("", "UNKNOWN")
  1054. if state.connected and state_known and not _printer_reconciled_since_connect.get(printer_id, False):
  1055. _printer_reconciled_since_connect[printer_id] = True
  1056. spawn_background_task(
  1057. reconcile_stale_active_prints(printer_id),
  1058. name=f"reconcile-stale-prints-{printer_id}",
  1059. )
  1060. elif not state.connected and _printer_reconciled_since_connect.get(printer_id, False):
  1061. # Re-arm so the next reconnect triggers reconciliation again.
  1062. _printer_reconciled_since_connect[printer_id] = False
  1063. # Offline-notification edge (#1752): schedule `on_printer_offline` on
  1064. # connected → disconnected. The "back online" channel is already covered
  1065. # by the print-failure notification (firmware reports gcode_state=FAILED
  1066. # on reconnect of an interrupted print), so we don't add a symmetric
  1067. # online event here.
  1068. prev_connected = _printer_last_connected.get(printer_id)
  1069. _printer_last_connected[printer_id] = state.connected
  1070. if prev_connected is True and not state.connected:
  1071. existing = _printer_offline_notify_tasks.get(printer_id)
  1072. if existing is None or existing.done():
  1073. logging.getLogger(__name__).info(
  1074. "[#1752] Printer %s connected→disconnected edge; scheduling offline notification in %.0fs",
  1075. printer_id,
  1076. _PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS,
  1077. )
  1078. _printer_offline_notify_tasks[printer_id] = asyncio.create_task(
  1079. _maybe_notify_printer_offline(printer_id),
  1080. name=f"printer-offline-notify-{printer_id}",
  1081. )
  1082. elif state.connected:
  1083. pending = _printer_offline_notify_tasks.pop(printer_id, None)
  1084. if pending is not None and not pending.done():
  1085. logging.getLogger(__name__).info(
  1086. "[#1752] Printer %s reconnected before debounce; cancelling pending offline notification",
  1087. printer_id,
  1088. )
  1089. pending.cancel()
  1090. # Only broadcast if something meaningful changed (reduce WebSocket spam)
  1091. # Include rounded temperatures to detect meaningful temp changes (within 1 degree)
  1092. temps = state.temperatures or {}
  1093. nozzle_temp = round(temps.get("nozzle", 0))
  1094. bed_temp = round(temps.get("bed", 0))
  1095. nozzle_2_temp = round(temps.get("nozzle_2", 0)) if "nozzle_2" in temps else ""
  1096. chamber_temp = round(temps.get("chamber", 0)) if "chamber" in temps else ""
  1097. # Auto-detect dual-nozzle printers from MQTT temperature data
  1098. if "nozzle_2" in temps and printer_id not in _nozzle_count_updated:
  1099. _nozzle_count_updated.add(printer_id)
  1100. # Update nozzle_count in database
  1101. async with async_session() as db:
  1102. from backend.app.models.printer import Printer
  1103. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1104. printer = result.scalar_one_or_none()
  1105. if printer and printer.nozzle_count != 2:
  1106. printer.nozzle_count = 2
  1107. await db.commit()
  1108. logging.getLogger(__name__).info(
  1109. f"Auto-detected dual-nozzle printer {printer_id}, updated nozzle_count=2"
  1110. )
  1111. # Include target temps for heating phase detection
  1112. bed_target = round(temps.get("bed_target", 0))
  1113. nozzle_target = round(temps.get("nozzle_target", 0))
  1114. # Include tray_now and vt_tray hash so external spool changes trigger broadcasts
  1115. vt_tray_key = hash(str(state.raw_data.get("vt_tray", []))) if state.raw_data else 0
  1116. # Include AMS dry_time and tray state values so drying/slot changes trigger broadcasts
  1117. ams_dry_key = tuple(a.get("dry_time", 0) for a in (state.raw_data.get("ams") or [])) if state.raw_data else ()
  1118. # Include tray states so load/unload transitions (state 11→10) trigger broadcasts (#784)
  1119. ams_tray_key = (
  1120. tuple(
  1121. (t.get("id"), t.get("tray_type", ""), t.get("state"))
  1122. for a in (state.raw_data.get("ams") or [])
  1123. for t in a.get("tray", [])
  1124. )
  1125. if state.raw_data
  1126. else ()
  1127. )
  1128. status_key = (
  1129. f"{state.connected}:{state.state}:{state.progress}:{state.layer_num}:"
  1130. f"{nozzle_temp}:{bed_temp}:{nozzle_2_temp}:{chamber_temp}:"
  1131. f"{state.stg_cur}:{bed_target}:{nozzle_target}:"
  1132. f"{state.cooling_fan_speed}:{state.big_fan1_speed}:{state.big_fan2_speed}:"
  1133. f"{state.chamber_light}:{state.active_extruder}:{state.tray_now}:{vt_tray_key}:"
  1134. f"{ams_dry_key}:{ams_tray_key}:{state.door_open}:{state.ams_filament_backup}"
  1135. )
  1136. # MQTT relay - publish status (before dedup check - always publish to MQTT)
  1137. try:
  1138. printer_info = printer_manager.get_printer(printer_id)
  1139. if printer_info:
  1140. await mqtt_relay.on_printer_status(
  1141. printer_id,
  1142. state,
  1143. printer_info.name,
  1144. printer_info.serial_number,
  1145. printer_manager.is_awaiting_plate_clear(printer_id),
  1146. )
  1147. except Exception:
  1148. pass # Don't fail status callback if MQTT fails
  1149. if _last_status_broadcast.get(printer_id) == status_key:
  1150. return # No change, skip WebSocket broadcast
  1151. _last_status_broadcast[printer_id] = status_key
  1152. # Check for progress milestone notifications (25%, 50%, 75%)
  1153. progress = state.progress or 0
  1154. is_printing = state.state in ("RUNNING", "PRINTING")
  1155. if is_printing and progress > 0:
  1156. # Determine which milestone we've reached
  1157. current_milestone = 0
  1158. if progress >= 75:
  1159. current_milestone = 75
  1160. elif progress >= 50:
  1161. current_milestone = 50
  1162. elif progress >= 25:
  1163. current_milestone = 25
  1164. last_milestone = _last_progress_milestone.get(printer_id, 0)
  1165. # If we've crossed a new milestone, send notification
  1166. if current_milestone > last_milestone:
  1167. _last_progress_milestone[printer_id] = current_milestone
  1168. try:
  1169. from backend.app.models.printer import Printer
  1170. # Read the printer in a short session and release the connection
  1171. # BEFORE the ~15s camera snapshot below — holding it across the grab
  1172. # pinned a pooled connection per milestone, per printer (issue #2572).
  1173. async with async_session() as db:
  1174. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1175. printer = result.scalar_one_or_none()
  1176. printer_name = printer.name if printer else f"Printer {printer_id}"
  1177. filename = state.subtask_name or state.gcode_file or "Unknown"
  1178. # remaining_time is in minutes, convert to seconds for notification
  1179. remaining_time_seconds = state.remaining_time * 60 if state.remaining_time else None
  1180. # Capture camera snapshot for notification image attachment (no DB held).
  1181. image_data = await _capture_snapshot_for_notification(printer_id, printer, logging.getLogger(__name__))
  1182. # Notification send needs a session (provider/template lookups).
  1183. async with async_session() as db:
  1184. await notification_service.on_print_progress(
  1185. printer_id,
  1186. printer_name,
  1187. filename,
  1188. current_milestone,
  1189. db,
  1190. remaining_time_seconds,
  1191. image_data=image_data,
  1192. )
  1193. except Exception as e:
  1194. logging.getLogger(__name__).warning(f"Progress milestone notification failed: {e}")
  1195. elif progress < 5:
  1196. # Reset milestone tracking when print restarts or new print begins
  1197. _last_progress_milestone[printer_id] = 0
  1198. _first_layer_notified[printer_id] = False
  1199. # HMS error codes that should not trigger notifications even though they
  1200. # have known descriptions (e.g. user-initiated actions, not real errors).
  1201. _HMS_NOTIFICATION_SUPPRESS = {
  1202. "0500_400E", # Printing was cancelled (user action, not an error)
  1203. }
  1204. # Check for new HMS errors and send notifications
  1205. current_hms_errors = getattr(state, "hms_errors", []) or []
  1206. if current_hms_errors:
  1207. # Build set of current error codes (using attr for uniqueness)
  1208. current_error_codes = {f"{e.attr:08x}" for e in current_hms_errors}
  1209. previously_notified = _notified_hms_errors.get(printer_id, set())
  1210. # Find new errors that haven't been notified yet
  1211. new_error_codes = current_error_codes - previously_notified
  1212. # Update tracking immediately to prevent duplicate notifications from concurrent callbacks
  1213. _notified_hms_errors[printer_id] = current_error_codes
  1214. _hms_last_seen[printer_id] = time.time()
  1215. if new_error_codes:
  1216. # Get the actual new errors for the notification
  1217. # Filter to severity >= 2 (skip informational/status messages like H2D sends)
  1218. new_errors = [e for e in current_hms_errors if f"{e.attr:08x}" in new_error_codes and e.severity >= 2]
  1219. try:
  1220. from backend.app.models.printer import Printer
  1221. # Read the printer in a short session and release the connection
  1222. # BEFORE the ~15s camera snapshot below (issue #2572).
  1223. async with async_session() as db:
  1224. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1225. printer = result.scalar_one_or_none()
  1226. printer_name = printer.name if printer else f"Printer {printer_id}"
  1227. # Format error details for notification
  1228. # Module 0x07 = AMS/Filament, 0x05 = Nozzle, 0x0C = Motion Controller, etc.
  1229. module_names = {
  1230. 0x03: "Print/Task",
  1231. 0x05: "Nozzle/Extruder",
  1232. 0x07: "AMS/Filament",
  1233. 0x0C: "Motion Controller",
  1234. 0x12: "Chamber",
  1235. }
  1236. from backend.app.services.hms_errors import get_error_description
  1237. # Capture camera snapshot once for all error notifications (no DB held).
  1238. error_image_data = await _capture_snapshot_for_notification(
  1239. printer_id, printer, logging.getLogger(__name__)
  1240. )
  1241. # Notification sends need a session (provider/template lookups).
  1242. async with async_session() as db:
  1243. sent_count = 0
  1244. for error in new_errors:
  1245. module_name = module_names.get(error.module, f"Module 0x{error.module:02X}")
  1246. # Build short code like "0700_8010"
  1247. # Mask to 16 bits to handle printers that send larger values
  1248. error_code_int = int(error.code.replace("0x", ""), 16) if error.code else 0
  1249. error_code_masked = error_code_int & 0xFFFF
  1250. short_code = f"{(error.attr >> 16) & 0xFFFF:04X}_{error_code_masked:04X}"
  1251. # Only notify for errors with known descriptions — printers
  1252. # send many undocumented/phantom codes that aren't real errors.
  1253. description = get_error_description(short_code)
  1254. if not description or short_code in _HMS_NOTIFICATION_SUPPRESS:
  1255. continue
  1256. error_type = f"{module_name} Error"
  1257. error_detail = description
  1258. await notification_service.on_printer_error(
  1259. printer_id, printer_name, error_type, db, error_detail, image_data=error_image_data
  1260. )
  1261. sent_count += 1
  1262. if sent_count:
  1263. logging.getLogger(__name__).info(
  1264. f"[HMS] Sent notification for {sent_count} error(s) on printer {printer_id}"
  1265. )
  1266. # Also publish to MQTT relay (no DB).
  1267. printer_info = printer_manager.get_printer(printer_id)
  1268. if printer_info:
  1269. errors_data = [
  1270. {
  1271. "code": e.code,
  1272. "attr": e.attr,
  1273. "module": e.module,
  1274. "severity": e.severity,
  1275. }
  1276. for e in new_errors
  1277. ]
  1278. await mqtt_relay.on_printer_error(
  1279. printer_id, printer_info.name, printer_info.serial_number, errors_data
  1280. )
  1281. except Exception as e:
  1282. logging.getLogger(__name__).warning(f"HMS error notification failed: {e}")
  1283. else:
  1284. # No HMS errors — only clear tracking after a grace period to prevent
  1285. # flapping errors (brief hms:[] gaps) from re-triggering notifications.
  1286. # Some HMS codes (e.g. chamber temp regulation during PETG prints) toggle
  1287. # on/off every few seconds as conditions fluctuate around thresholds.
  1288. if printer_id in _notified_hms_errors:
  1289. last_seen = _hms_last_seen.get(printer_id, 0)
  1290. if time.time() - last_seen >= _HMS_CLEAR_GRACE_SECONDS:
  1291. _notified_hms_errors.pop(printer_id, None)
  1292. _hms_last_seen.pop(printer_id, None)
  1293. await ws_manager.send_printer_status(
  1294. printer_id,
  1295. printer_state_to_dict(
  1296. state,
  1297. printer_id,
  1298. printer_manager.get_model(printer_id),
  1299. printer_manager.get_drying_targets(printer_id),
  1300. ),
  1301. )
  1302. def _is_bambu_uuid(tray_uuid: str) -> bool:
  1303. """Check if a tray UUID looks like a valid Bambu Lab RFID UUID (non-empty, non-zero)."""
  1304. return bool(tray_uuid) and tray_uuid not in ("", "0" * len(tray_uuid))
  1305. async def on_ams_change(printer_id: int, ams_data: list):
  1306. """Handle AMS data changes - sync to Spoolman if enabled and auto mode."""
  1307. logger = logging.getLogger(__name__)
  1308. # Snapshot BEFORE any await: if a print is active, skip weight sync later.
  1309. # on_print_complete may pop _active_sessions during our awaits (#880).
  1310. from backend.app.services.usage_tracker import _active_sessions
  1311. _print_active = printer_id in _active_sessions
  1312. # MQTT relay - publish AMS change
  1313. try:
  1314. printer_info = printer_manager.get_printer(printer_id)
  1315. if printer_info:
  1316. await mqtt_relay.on_ams_change(printer_id, printer_info.name, printer_info.serial_number, ams_data)
  1317. except Exception:
  1318. pass # Don't fail AMS callback if MQTT fails
  1319. # Broadcast AMS change via WebSocket (bypasses status_key deduplication)
  1320. # This ensures frontend gets immediate updates when AMS slots are configured
  1321. try:
  1322. state = printer_manager.get_status(printer_id)
  1323. if state:
  1324. logger.info("[Printer %s] Broadcasting AMS change via WebSocket", printer_id)
  1325. await ws_manager.send_printer_status(
  1326. printer_id,
  1327. printer_state_to_dict(
  1328. state,
  1329. printer_id,
  1330. printer_manager.get_model(printer_id),
  1331. printer_manager.get_drying_targets(printer_id),
  1332. ),
  1333. )
  1334. except Exception as e:
  1335. logger.warning("Failed to broadcast AMS change for printer %s: %s", printer_id, e)
  1336. from backend.app.utils.color_utils import colors_similar as _colors_similar
  1337. # Auto-unlink spool assignments with stale fingerprints
  1338. try:
  1339. async with async_session() as db:
  1340. from sqlalchemy.orm import selectinload
  1341. from backend.app.api.routes.inventory import _find_tray_in_ams_data
  1342. from backend.app.models.spool import Spool as _Spool
  1343. from backend.app.models.spool_assignment import SpoolAssignment as SA
  1344. result = await db.execute(
  1345. select(SA)
  1346. .where(SA.printer_id == printer_id)
  1347. .options(selectinload(SA.spool).selectinload(_Spool.k_profiles))
  1348. )
  1349. stale = []
  1350. for assignment in result.scalars().all():
  1351. # External spool assignments (ams_id=255) live in vt_tray, not AMS data
  1352. if assignment.ams_id == 255:
  1353. ps = printer_manager.get_status(printer_id)
  1354. vt_tray_raw = ps.raw_data.get("vt_tray", []) if ps else []
  1355. ext_id = assignment.tray_id + 254 # 0→254, 1→255
  1356. current_tray = None
  1357. for vt in vt_tray_raw:
  1358. if isinstance(vt, dict) and int(vt.get("id", 254)) == ext_id:
  1359. current_tray = vt
  1360. break
  1361. if not current_tray:
  1362. # vt_tray data may not have arrived yet — keep assignment
  1363. continue
  1364. else:
  1365. current_tray = _find_tray_in_ams_data(ams_data, assignment.ams_id, assignment.tray_id)
  1366. if not current_tray:
  1367. logger.info(
  1368. "Auto-unlink: spool %d AMS%d-T%d — tray not found in AMS data (slot empty?)",
  1369. assignment.spool_id,
  1370. assignment.ams_id,
  1371. assignment.tray_id,
  1372. )
  1373. stale.append(assignment) # Slot empty
  1374. elif _is_bambu_uuid(current_tray.get("tray_uuid", "")):
  1375. # A Bambu Lab spool is in this slot — check if it's the same spool
  1376. # that's currently assigned. If yes, keep the assignment (avoids
  1377. # unnecessary unlink/re-assign/ams_filament_setting cycle that clears
  1378. # the printer's filament preset on every startup).
  1379. tray_uuid = current_tray.get("tray_uuid", "")
  1380. tag_uid = current_tray.get("tag_uid", "")
  1381. spool = assignment.spool
  1382. spool_matches = False
  1383. if spool:
  1384. if (spool.tray_uuid and spool.tray_uuid.upper() == tray_uuid.upper()) or (
  1385. spool.tag_uid
  1386. and tag_uid
  1387. and tag_uid != "0000000000000000"
  1388. and spool.tag_uid.upper() == tag_uid.upper()
  1389. ):
  1390. spool_matches = True
  1391. if spool_matches:
  1392. # Same BL spool still in slot — keep assignment, update fingerprint if needed
  1393. cur_color = current_tray.get("tray_color", "")
  1394. cur_type = current_tray.get("tray_type", "")
  1395. fp_color = assignment.fingerprint_color or ""
  1396. fp_type = assignment.fingerprint_type or ""
  1397. if cur_color.upper() != fp_color.upper() or cur_type.upper() != fp_type.upper():
  1398. assignment.fingerprint_color = cur_color
  1399. assignment.fingerprint_type = cur_type
  1400. logger.debug(
  1401. "Auto-unlink: spool %d AMS%d-T%d — same BL spool, updated fingerprint",
  1402. assignment.spool_id,
  1403. assignment.ams_id,
  1404. assignment.tray_id,
  1405. )
  1406. continue
  1407. # Different BL spool or unrecognized — unlink so auto-assign can match
  1408. logger.info(
  1409. "Auto-unlink: spool %d AMS%d-T%d — different Bambu Lab spool detected (uuid=%s)",
  1410. assignment.spool_id,
  1411. assignment.ams_id,
  1412. assignment.tray_id,
  1413. tray_uuid,
  1414. )
  1415. stale.append(assignment)
  1416. else:
  1417. cur_color = current_tray.get("tray_color", "")
  1418. cur_type = current_tray.get("tray_type", "")
  1419. cur_state = current_tray.get("state")
  1420. fp_color = assignment.fingerprint_color or ""
  1421. fp_type = assignment.fingerprint_type or ""
  1422. # SpoolBuddy pre-config replay: fingerprint_type empty means
  1423. # the slot was empty when the user pre-assigned via SpoolBuddy
  1424. # (the firmware drops ams_filament_setting on empty slots, so
  1425. # MQTT was deferred). The moment any filament gets inserted
  1426. # — Bambu RFID, 3rd-party, or even an existing-but-now-
  1427. # reconfigured spool — fire the deferred configuration.
  1428. # The "loaded" signal is state == 11 (Bambu's "filament fed to
  1429. # extruder" code) OR, on firmwares that don't use the state
  1430. # enum meaningfully, a non-empty tray_type when state is
  1431. # NOT one of the firmware's explicit empty signals (9, 10).
  1432. # state-only was wrong for firmwares that never set 11 — A1
  1433. # Mini BMCU 01.07.02.00 and P1S Standard AMS 00.00.06.75 both
  1434. # always report state=3 — so the replay never fired for them
  1435. # (#1322). The state ∉ {9,10} guard keeps the firmware's
  1436. # explicit "empty" signals authoritative over any stale
  1437. # tray_type that might survive the relay's auto-clearing.
  1438. loaded = cur_state == 11 or (cur_state not in (9, 10) and cur_type.strip())
  1439. if not fp_type.strip() and loaded and assignment.spool:
  1440. try:
  1441. from backend.app.api.routes.inventory import (
  1442. apply_spool_to_slot_via_mqtt,
  1443. )
  1444. await apply_spool_to_slot_via_mqtt(
  1445. db=db,
  1446. current_user=None,
  1447. spool=assignment.spool,
  1448. printer_id=printer_id,
  1449. ams_id=assignment.ams_id,
  1450. tray_id=assignment.tray_id,
  1451. current_tray_info_idx=current_tray.get("tray_info_idx", ""),
  1452. current_tray_type=cur_type,
  1453. )
  1454. logger.info(
  1455. "SpoolBuddy pre-config applied on insert: spool %d → printer %d AMS%d-T%d",
  1456. assignment.spool_id,
  1457. printer_id,
  1458. assignment.ams_id,
  1459. assignment.tray_id,
  1460. )
  1461. except Exception:
  1462. logger.exception(
  1463. "Pre-config apply failed for spool %d on printer %d AMS%d-T%d",
  1464. assignment.spool_id,
  1465. printer_id,
  1466. assignment.ams_id,
  1467. assignment.tray_id,
  1468. )
  1469. assignment.fingerprint_color = cur_color
  1470. assignment.fingerprint_type = cur_type
  1471. continue
  1472. if not _colors_similar(cur_color, fp_color) or cur_type.upper() != fp_type.upper():
  1473. # Fingerprint mismatch — but check if tray now matches the
  1474. # assigned spool (e.g. auto-configure changed the tray).
  1475. spool = assignment.spool
  1476. if spool:
  1477. spool_color = (spool.rgba or "FFFFFFFF").upper()
  1478. spool_type = (spool.material or "").upper()
  1479. if _colors_similar(cur_color, spool_color) and cur_type.upper() == spool_type:
  1480. logger.info(
  1481. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch but tray matches spool, updating fp",
  1482. assignment.spool_id,
  1483. assignment.ams_id,
  1484. assignment.tray_id,
  1485. )
  1486. assignment.fingerprint_color = cur_color
  1487. assignment.fingerprint_type = cur_type
  1488. continue
  1489. logger.info(
  1490. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch (cur=%s/%s fp=%s/%s spool=%s/%s)",
  1491. assignment.spool_id,
  1492. assignment.ams_id,
  1493. assignment.tray_id,
  1494. cur_color,
  1495. cur_type,
  1496. fp_color,
  1497. fp_type,
  1498. spool.rgba if spool else "?",
  1499. spool.material if spool else "?",
  1500. )
  1501. stale.append(assignment) # Spool changed
  1502. # Snapshot slots before delete — ORM attribute access after the
  1503. # commit would refresh against a deleted row.
  1504. unlinked_slots = [(a.ams_id, a.tray_id) for a in stale]
  1505. for a in stale:
  1506. await db.delete(a)
  1507. if stale:
  1508. logger.info("Auto-unlinked %d stale spool assignments for printer %d", len(stale), printer_id)
  1509. # Commit any changes (stale deletions and/or fingerprint updates)
  1510. await db.commit()
  1511. # Tell open browsers the assignment is gone (#2575). Only the manual
  1512. # REST assign/unassign endpoints broadcast this event; without it the
  1513. # frontend's spool-assignments cache keeps rendering the unlinked
  1514. # spool on the slot until an unrelated refetch — which reads exactly
  1515. # like "the fix didn't work" (reporter verified: a browser refresh
  1516. # after the swap showed the correct state all along).
  1517. for ams_id, tray_id in unlinked_slots:
  1518. await ws_manager.broadcast(
  1519. {
  1520. "type": "spool_assignment_changed",
  1521. "printer_id": printer_id,
  1522. "ams_id": ams_id,
  1523. "tray_id": tray_id,
  1524. }
  1525. )
  1526. except Exception as e:
  1527. logger.warning("Spool assignment cleanup failed: %s", e, exc_info=True)
  1528. # Auto-manage inventory spools from AMS tray data (skip if Spoolman manages AMS).
  1529. # Serialised per-printer via _ams_assignment_locks: MQTT bursts can deliver
  1530. # two AMS pushes ~30 ms apart, and without the lock both callbacks read
  1531. # "no existing assignment" for the same (printer, ams, tray) and race to
  1532. # INSERT, hitting the spool_assignment_printer_id_ams_id_tray_id_key
  1533. # unique constraint on Postgres. SQLite's WAL serialises writes so the
  1534. # bug stayed latent there. See _ams_assignment_locks comment for details.
  1535. try:
  1536. async with _get_ams_assignment_lock(printer_id), async_session() as db:
  1537. from backend.app.api.routes.settings import get_setting
  1538. from backend.app.models.spool import Spool
  1539. from backend.app.models.spool_assignment import SpoolAssignment as SA
  1540. from backend.app.services.spool_tag_matcher import (
  1541. auto_assign_spool,
  1542. create_spool_from_tray,
  1543. find_matching_untagged_spool,
  1544. get_spool_by_tag,
  1545. is_bambu_tag,
  1546. is_valid_tag,
  1547. link_tag_to_inventory_spool,
  1548. )
  1549. _spoolman_on = await get_setting(db, "spoolman_enabled")
  1550. _auto_add_raw = await get_setting(db, "auto_add_unknown_rfid")
  1551. _auto_add_unknown = _auto_add_raw is None or _auto_add_raw.lower() == "true"
  1552. if not _spoolman_on or _spoolman_on.lower() != "true":
  1553. for ams_unit in ams_data:
  1554. if not isinstance(ams_unit, dict):
  1555. continue
  1556. ams_id = int(ams_unit.get("id", 0))
  1557. for tray in ams_unit.get("tray", []):
  1558. if not isinstance(tray, dict):
  1559. continue
  1560. tray_id = int(tray.get("id", 0))
  1561. tag_uid = tray.get("tag_uid", "")
  1562. tray_uuid = tray.get("tray_uuid", "")
  1563. tray_info_idx = tray.get("tray_info_idx", "")
  1564. if not tray.get("tray_type"):
  1565. # Slot reported empty — drop any cached unknown-tag
  1566. # broadcast so reinserting the same spool re-prompts.
  1567. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id)
  1568. continue # Empty slot
  1569. # Check if assignment already exists for this slot
  1570. existing = await db.execute(
  1571. select(SA)
  1572. .options(selectinload(SA.spool).selectinload(Spool.k_profiles))
  1573. .where(SA.printer_id == printer_id, SA.ams_id == ams_id, SA.tray_id == tray_id)
  1574. )
  1575. existing_assignment = existing.scalar_one_or_none()
  1576. if existing_assignment:
  1577. # Sync spool weight_used from AMS remain — only INCREASE, never decrease.
  1578. # The AMS remain% is low-resolution (integer %, i.e. 10g steps for 1kg spool)
  1579. # and must not overwrite precise values from the usage tracker (3MF/G-code).
  1580. # Skip during active prints: the usage tracker handles deduction
  1581. # precisely via 3MF data on print completion. Without this guard the
  1582. # AMS remain% SET and the usage tracker ADD both fire from the same
  1583. # MQTT message, doubling the deduction (#880).
  1584. if _print_active:
  1585. continue
  1586. remain_raw = tray.get("remain")
  1587. if (
  1588. remain_raw is not None
  1589. and existing_assignment.spool
  1590. and not existing_assignment.spool.weight_locked
  1591. ):
  1592. try:
  1593. remain_val = int(remain_raw)
  1594. except (TypeError, ValueError):
  1595. remain_val = -1
  1596. if 1 <= remain_val <= 100:
  1597. lw = existing_assignment.spool.label_weight or 1000
  1598. new_used = round(lw * (100 - remain_val) / 100.0, 1)
  1599. current_used = existing_assignment.spool.weight_used or 0
  1600. if new_used > current_used + 1:
  1601. logger.info(
  1602. "Weight sync: spool %d weight_used %s -> %s (remain=%d)",
  1603. existing_assignment.spool_id,
  1604. current_used,
  1605. new_used,
  1606. remain_val,
  1607. )
  1608. existing_assignment.spool.weight_used = new_used
  1609. await db.commit()
  1610. # Re-apply stored K-profile when the live tray's
  1611. # cali_idx drifted from the spool's stored profile.
  1612. # This catches "reset slot → re-read" and any other
  1613. # path where the firmware loses the user's K-profile
  1614. # selection while the SpoolAssignment row persists.
  1615. # Per the maintainer's rule: any time a spool tag is
  1616. # identified and matches inventory, the slot must be
  1617. # configured with the spool's stored settings. Without
  1618. # this block the existing-assignment branch only ran
  1619. # weight-sync and let the firmware-default cali_idx win.
  1620. try:
  1621. spool = existing_assignment.spool
  1622. if (
  1623. spool is not None
  1624. and is_bambu_tag(tag_uid, tray_uuid, tray_info_idx)
  1625. and spool.k_profiles
  1626. ):
  1627. state = printer_manager.get_status(printer_id)
  1628. nozzle_diameter = "0.4"
  1629. if state and state.nozzles:
  1630. nd = state.nozzles[0].nozzle_diameter
  1631. if nd:
  1632. nozzle_diameter = nd
  1633. slot_extruder: int | None = None
  1634. if state and state.ams_extruder_map:
  1635. if ams_id == 255:
  1636. slot_extruder = 1 - tray_id
  1637. else:
  1638. slot_extruder = state.ams_extruder_map.get(str(ams_id))
  1639. # Prefer exact extruder match, fall back to
  1640. # extruder-agnostic kp for the same printer +
  1641. # nozzle. Avoids hard-skipping when the AMS is
  1642. # mapped differently than at calibration time.
  1643. matching_kp = None
  1644. fallback_kp = None
  1645. for kp in spool.k_profiles:
  1646. if (
  1647. kp.printer_id != printer_id
  1648. or kp.nozzle_diameter != nozzle_diameter
  1649. or kp.cali_idx is None
  1650. ):
  1651. continue
  1652. if (
  1653. slot_extruder is not None
  1654. and kp.extruder is not None
  1655. and kp.extruder == slot_extruder
  1656. ):
  1657. matching_kp = kp
  1658. break
  1659. if fallback_kp is None:
  1660. fallback_kp = kp
  1661. chosen_kp = matching_kp or fallback_kp
  1662. if chosen_kp is not None:
  1663. live_cali_idx = tray.get("cali_idx")
  1664. # Only fire MQTT when the printer's live
  1665. # cali_idx differs from the stored value.
  1666. # Avoids spamming the broker on every
  1667. # MQTT push during steady-state operation.
  1668. if live_cali_idx != chosen_kp.cali_idx:
  1669. client = printer_manager.get_client(printer_id)
  1670. if client:
  1671. cali_filament_id = spool.slicer_filament or tray_info_idx or ""
  1672. client.extrusion_cali_sel(
  1673. ams_id=ams_id,
  1674. tray_id=tray_id,
  1675. cali_idx=chosen_kp.cali_idx,
  1676. filament_id=cali_filament_id,
  1677. nozzle_diameter=nozzle_diameter,
  1678. )
  1679. logger.info(
  1680. "Re-applied K-profile cali_idx=%d for spool %d "
  1681. "on printer %d AMS%d-T%d (live=%s drift detected)",
  1682. chosen_kp.cali_idx,
  1683. spool.id,
  1684. printer_id,
  1685. ams_id,
  1686. tray_id,
  1687. live_cali_idx,
  1688. )
  1689. except Exception:
  1690. logger.exception(
  1691. "K-profile re-apply failed for printer %d AMS%d-T%d",
  1692. printer_id,
  1693. ams_id,
  1694. tray_id,
  1695. )
  1696. continue
  1697. if is_bambu_tag(tag_uid, tray_uuid, tray_info_idx):
  1698. # BL spool with RFID tag: auto-match → inventory match → auto-create
  1699. spool = await get_spool_by_tag(db, tag_uid, tray_uuid)
  1700. if not spool:
  1701. # Try matching an untagged inventory spool (same material/color)
  1702. spool = await find_matching_untagged_spool(db, tray)
  1703. if spool:
  1704. await link_tag_to_inventory_spool(db, spool, tray)
  1705. elif _auto_add_unknown:
  1706. spool = await create_spool_from_tray(db, tray)
  1707. else:
  1708. # Auto-add disabled: surface the slot so the
  1709. # user can add it manually via the UI.
  1710. await _broadcast_unknown_tag(
  1711. printer_id=printer_id,
  1712. ams_id=ams_id,
  1713. tray_id=tray_id,
  1714. tag_uid=tag_uid,
  1715. tray_uuid=tray_uuid,
  1716. tray_type=tray.get("tray_type"),
  1717. tray_color=tray.get("tray_color"),
  1718. tray_sub_brands=tray.get("tray_sub_brands"),
  1719. tray_count=len(ams_unit.get("tray", [])),
  1720. )
  1721. continue
  1722. # Slot matched (existing tag, untagged inventory
  1723. # match, or freshly auto-created spool) — drop any
  1724. # stale dedup so a future tag swap re-prompts.
  1725. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id)
  1726. await auto_assign_spool(
  1727. printer_id,
  1728. ams_id,
  1729. tray_id,
  1730. spool,
  1731. printer_manager,
  1732. db,
  1733. tray_info_idx=tray_info_idx,
  1734. )
  1735. await db.commit()
  1736. await ws_manager.broadcast(
  1737. {
  1738. "type": "spool_auto_assigned",
  1739. "printer_id": printer_id,
  1740. "ams_id": ams_id,
  1741. "tray_id": tray_id,
  1742. "spool_id": spool.id,
  1743. }
  1744. )
  1745. logger.info(
  1746. "RFID auto-assigned spool %d to printer %d AMS%d-T%d",
  1747. spool.id,
  1748. printer_id,
  1749. ams_id,
  1750. tray_id,
  1751. )
  1752. elif is_valid_tag(tag_uid, tray_uuid):
  1753. # Non-BL spool with some tag — let user choose
  1754. await _broadcast_unknown_tag(
  1755. printer_id=printer_id,
  1756. ams_id=ams_id,
  1757. tray_id=tray_id,
  1758. tag_uid=tag_uid,
  1759. tray_uuid=tray_uuid,
  1760. tray_type=tray.get("tray_type"),
  1761. tray_color=tray.get("tray_color"),
  1762. tray_sub_brands=tray.get("tray_sub_brands"),
  1763. tray_count=len(ams_unit.get("tray", [])),
  1764. )
  1765. # No-tag slots (generic non-RFID filament) are left alone:
  1766. # nothing to identify, prompting "+ Add" would just create
  1767. # ghost spools with empty tags on every confirm.
  1768. except Exception as e:
  1769. logger.warning("RFID spool auto-assign failed: %s", e, exc_info=True)
  1770. try:
  1771. async with async_session() as db:
  1772. from backend.app.api.routes.settings import get_setting
  1773. from backend.app.models.printer import Printer
  1774. # Check if Spoolman is enabled
  1775. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  1776. if not spoolman_enabled or spoolman_enabled.lower() != "true":
  1777. return
  1778. # Check sync mode
  1779. sync_mode = await get_setting(db, "spoolman_sync_mode")
  1780. if sync_mode and sync_mode != "auto":
  1781. return # Only sync on auto mode
  1782. _auto_add_raw_sm = await get_setting(db, "auto_add_unknown_rfid")
  1783. auto_add_unknown_rfid = _auto_add_raw_sm is None or _auto_add_raw_sm.lower() == "true"
  1784. # `spoolman_disable_weight_sync` is deprecated (#1119) — weight is now
  1785. # always owned by per-print tracking, never by AMS auto-sync. The
  1786. # setting is still read by the settings UI for backwards compat but
  1787. # has no effect on the sync path here.
  1788. # Get Spoolman URL
  1789. spoolman_url = await get_setting(db, "spoolman_url")
  1790. if not spoolman_url:
  1791. return
  1792. # Get or create Spoolman client
  1793. client = await get_spoolman_client()
  1794. if not client:
  1795. try:
  1796. client = await init_spoolman_client(spoolman_url)
  1797. except ValueError as exc:
  1798. logger.warning("Spoolman URL %r rejected by SSRF guard: %s", spoolman_url, exc)
  1799. return
  1800. # Check if Spoolman is reachable
  1801. if not await client.health_check():
  1802. logger.warning("Spoolman not reachable at %s", spoolman_url)
  1803. return
  1804. # Get printer name for location
  1805. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1806. printer = result.scalar_one_or_none()
  1807. printer_name = printer.name if printer else f"Printer {printer_id}"
  1808. # OPTIMIZATION: Fetch all spools once before processing trays
  1809. # This eliminates redundant API calls (one per tray) when syncing multiple trays
  1810. logger.debug("[Printer %s] Fetching spools cache for AMS sync...", printer_id)
  1811. try:
  1812. cached_spools = await client.get_spools()
  1813. logger.debug("[Printer %s] Cached %d spools for batch sync", printer_id, len(cached_spools))
  1814. except Exception as e:
  1815. logger.error(
  1816. "[Printer %s] Failed to fetch spools cache after retries, aborting AMS sync: %s",
  1817. printer_id,
  1818. e,
  1819. )
  1820. return
  1821. # Load inventory weights as fallback (when AMS MQTT data lacks remain values)
  1822. from sqlalchemy.orm import selectinload
  1823. from backend.app.models.spool_assignment import SpoolAssignment
  1824. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  1825. inventory_weights: dict[tuple[int, int], float] = {}
  1826. try:
  1827. assign_result = await db.execute(
  1828. select(SpoolAssignment)
  1829. .options(selectinload(SpoolAssignment.spool))
  1830. .where(SpoolAssignment.printer_id == printer_id)
  1831. )
  1832. for assignment in assign_result.scalars().all():
  1833. spool = assignment.spool
  1834. if spool and spool.label_weight > 0:
  1835. remaining = max(0.0, spool.label_weight - (spool.weight_used or 0))
  1836. inventory_weights[(assignment.ams_id, assignment.tray_id)] = remaining
  1837. except Exception as e:
  1838. logger.warning("Could not load inventory weights for printer %s: %s", printer_id, e)
  1839. # Load existing Spoolman slot assignments for the no-RFID fallback path
  1840. spoolman_slot_map: dict[tuple[int, int], int] = {}
  1841. try:
  1842. slot_result = await db.execute(
  1843. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  1844. )
  1845. for slot in slot_result.scalars().all():
  1846. spoolman_slot_map[(slot.ams_id, slot.tray_id)] = slot.spoolman_spool_id
  1847. except Exception as e:
  1848. logger.warning("Could not load Spoolman slot assignments for printer %s: %s", printer_id, e)
  1849. # Sync each AMS tray and collect slot changes for DB persistence
  1850. synced = 0
  1851. slot_changes: list[tuple[int, int, int]] = [] # (ams_id, tray_id, spoolman_spool_id) to upsert
  1852. empty_slots: list[tuple[int, int]] = [] # (ams_id, tray_id) whose tray is now empty
  1853. for ams_unit in ams_data:
  1854. if not isinstance(ams_unit, dict):
  1855. continue
  1856. ams_id = int(ams_unit.get("id", 0))
  1857. trays = ams_unit.get("tray", [])
  1858. for tray_data in trays:
  1859. if not isinstance(tray_data, dict):
  1860. continue
  1861. tray_id_raw = int(tray_data.get("id", 0))
  1862. tray = client.parse_ams_tray(ams_id, tray_data)
  1863. if not tray:
  1864. # Empty tray slot — record for local assignment cleanup
  1865. # and drop any cached unknown-tag broadcast so a
  1866. # reinserted spool re-prompts.
  1867. empty_slots.append((ams_id, tray_id_raw))
  1868. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id_raw)
  1869. continue
  1870. spool_tag = (
  1871. tray.tray_uuid
  1872. if tray.tray_uuid and tray.tray_uuid != "00000000000000000000000000000000"
  1873. else tray.tag_uid
  1874. )
  1875. # Provide the hint only when no RFID is available
  1876. hint = spoolman_slot_map.get((ams_id, tray.tray_id)) if not spool_tag else None
  1877. try:
  1878. inv_remaining = inventory_weights.get((ams_id, tray.tray_id))
  1879. result = await client.sync_ams_tray(
  1880. tray,
  1881. printer_name,
  1882. # Per-print tracking is the only weight writer (#1119).
  1883. # AMS auto-sync still maintains spool metadata / slot
  1884. # assignments but no longer touches remaining_weight.
  1885. disable_weight_sync=True,
  1886. cached_spools=cached_spools,
  1887. inventory_remaining=inv_remaining,
  1888. spoolman_spool_id_hint=hint,
  1889. auto_add_unknown_rfid=auto_add_unknown_rfid,
  1890. )
  1891. if result is None and spool_tag and not auto_add_unknown_rfid:
  1892. # Spoolman skipped auto-create per user setting — surface
  1893. # the slot so the UI can offer "+ Add to inventory".
  1894. await _broadcast_unknown_tag(
  1895. printer_id=printer_id,
  1896. ams_id=ams_id,
  1897. tray_id=tray.tray_id,
  1898. tag_uid=tray.tag_uid or "",
  1899. tray_uuid=tray.tray_uuid or "",
  1900. tray_type=tray.tray_type,
  1901. tray_color=tray.tray_color,
  1902. tray_sub_brands=tray.tray_sub_brands,
  1903. tray_count=len(trays),
  1904. )
  1905. elif result:
  1906. _clear_unknown_tag_dedup(printer_id, ams_id, tray.tray_id)
  1907. if result:
  1908. synced += 1
  1909. if result.get("id"):
  1910. slot_changes.append((ams_id, tray.tray_id, result["id"]))
  1911. # If a new spool was created, add it to the cache
  1912. # so subsequent trays can find it if they reference the same tag
  1913. spool_exists = any(s.get("id") == result["id"] for s in cached_spools)
  1914. if not spool_exists:
  1915. cached_spools.append(result)
  1916. logger.debug(
  1917. "[Printer %s] Added newly created spool %s to cache",
  1918. printer_id,
  1919. result["id"],
  1920. )
  1921. # Reconcile slot_preset_mappings (the same row internal
  1922. # mode keeps in sync via inventory + spool_tag_matcher).
  1923. # Without this the slot card surfaces the previous spool's
  1924. # preset name — same bug shape, different inventory mode.
  1925. from backend.app.services.slot_preset_writer import (
  1926. upsert_slot_preset_for_spoolman_spool,
  1927. )
  1928. await upsert_slot_preset_for_spoolman_spool(
  1929. db=db,
  1930. spoolman_spool=result,
  1931. tray_info_idx=tray.tray_info_idx or "",
  1932. tray_sub_brands=tray.tray_sub_brands or "",
  1933. tray_type=tray.tray_type or "",
  1934. printer_id=printer_id,
  1935. ams_id=ams_id,
  1936. tray_id=tray.tray_id,
  1937. )
  1938. except Exception as e:
  1939. logger.error("Error syncing AMS %s tray %s: %s", ams_id, tray.tray_id, e)
  1940. if synced > 0:
  1941. logger.info("Auto-synced %s AMS trays to Spoolman for printer %s", synced, printer_id)
  1942. # Persist slot assignment changes to the local table
  1943. if slot_changes or empty_slots:
  1944. try:
  1945. for ams_id, tray_id, spool_id in slot_changes:
  1946. await db.execute(
  1947. text(
  1948. "INSERT INTO spoolman_slot_assignments"
  1949. " (printer_id, ams_id, tray_id, spoolman_spool_id)"
  1950. " VALUES (:printer_id, :ams_id, :tray_id, :spool_id)"
  1951. " ON CONFLICT(printer_id, ams_id, tray_id)"
  1952. " DO UPDATE SET spoolman_spool_id = excluded.spoolman_spool_id"
  1953. ),
  1954. {
  1955. "printer_id": printer_id,
  1956. "ams_id": ams_id,
  1957. "tray_id": tray_id,
  1958. "spool_id": spool_id,
  1959. },
  1960. )
  1961. for ams_id, tray_id in empty_slots:
  1962. await db.execute(
  1963. delete(SpoolmanSlotAssignment).where(
  1964. SpoolmanSlotAssignment.printer_id == printer_id,
  1965. SpoolmanSlotAssignment.ams_id == ams_id,
  1966. SpoolmanSlotAssignment.tray_id == tray_id,
  1967. )
  1968. )
  1969. await db.commit()
  1970. except Exception as e:
  1971. await db.rollback()
  1972. logger.error("Error persisting Spoolman slot assignments for printer %s: %s", printer_id, e)
  1973. except Exception as e:
  1974. logging.getLogger(__name__).error("Spoolman AMS sync failed for printer %s: %s", printer_id, e)
  1975. async def _capture_snapshot_for_notification(printer_id: int, printer, logger) -> bytes | None:
  1976. """Capture a camera snapshot for notification image attachment.
  1977. Returns JPEG bytes (max 2.5MB) or None if capture fails or is unavailable.
  1978. Uses: external camera > buffered frame > fresh capture.
  1979. """
  1980. if not printer:
  1981. return None
  1982. try:
  1983. from backend.app.api.routes.settings import get_setting
  1984. async with async_session() as db:
  1985. capture_enabled = await get_setting(db, "capture_finish_photo")
  1986. if capture_enabled is not None and capture_enabled.lower() != "true":
  1987. return None
  1988. # Try external camera first
  1989. if printer.external_camera_enabled and printer.external_camera_url:
  1990. logger.info("[SNAPSHOT] Capturing from external camera for printer %s", printer_id)
  1991. from backend.app.api.routes.camera import live_frame_for_capture
  1992. from backend.app.services.external_camera import capture_frame
  1993. # An external camera allows one reader, so capturing while a viewer
  1994. # is attached fails (#2707). A None here falls through to the paths
  1995. # below exactly as a failed capture did.
  1996. defer, buffered = live_frame_for_capture(printer_id)
  1997. if defer:
  1998. frame_data = buffered
  1999. else:
  2000. frame_data = await capture_frame(
  2001. printer.external_camera_url,
  2002. printer.external_camera_type or "mjpeg",
  2003. snapshot_url=printer.external_camera_snapshot_url,
  2004. )
  2005. if frame_data and len(frame_data) <= 2_500_000:
  2006. logger.info("[SNAPSHOT] External camera frame: %s bytes", len(frame_data))
  2007. return _apply_camera_rotation(frame_data, printer, logger)
  2008. # Try buffered frame from active stream
  2009. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  2010. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  2011. active_chamber = [k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")]
  2012. buffered_frame = get_buffered_frame(printer_id)
  2013. if (active_for_printer or active_chamber) and buffered_frame:
  2014. logger.info("[SNAPSHOT] Using buffered frame for printer %s: %s bytes", printer_id, len(buffered_frame))
  2015. if len(buffered_frame) <= 2_500_000:
  2016. return _apply_camera_rotation(buffered_frame, printer, logger)
  2017. # Fresh capture from printer camera
  2018. logger.info("[SNAPSHOT] Capturing fresh frame for printer %s", printer_id)
  2019. from backend.app.services.camera import capture_camera_frame_bytes
  2020. frame_data = await capture_camera_frame_bytes(
  2021. printer.ip_address, printer.access_code, printer.model, timeout=15
  2022. )
  2023. if frame_data and len(frame_data) <= 2_500_000:
  2024. logger.info("[SNAPSHOT] Fresh camera frame: %s bytes", len(frame_data))
  2025. return _apply_camera_rotation(frame_data, printer, logger)
  2026. except Exception as e:
  2027. logger.warning("[SNAPSHOT] Failed to capture snapshot for printer %s: %s", printer_id, e)
  2028. return None
  2029. async def _maybe_bank_inprint_frame(printer_id: int, layer_num: int) -> None:
  2030. """#1867: bank a recent in-print camera frame for the finish photo.
  2031. Called on every layer change. Grabs one frame (throttled) into
  2032. ``_inprint_frame_bank`` so the FINISH-state finish-photo path has a
  2033. pre-swap image on firmware that never emits ``stg_cur=22``. Because it is
  2034. driven by layer_num increases, banking stops the instant printing ends and
  2035. the End G-code (e.g. SwapMod plate swap) runs — no further layer changes
  2036. arrive — so the last banked frame is the finished print, not the swapped
  2037. plate. Best-effort: any failure just leaves the previous banked frame.
  2038. """
  2039. logger = logging.getLogger(__name__)
  2040. client = printer_manager.get_client(printer_id)
  2041. state = client.state if client else None
  2042. if not state or state.state != "RUNNING":
  2043. return
  2044. # Only during actual extrusion — firmware ticks layer_num during the
  2045. # pre-print calibration sequence, whose sub-stages are non-zero.
  2046. if state.mc_print_sub_stage not in (None, 0):
  2047. return
  2048. total = state.total_layers or 0
  2049. is_last_layer = total > 0 and layer_num >= total
  2050. now = time.monotonic()
  2051. last = _inprint_frame_bank_ts.get(printer_id, 0.0)
  2052. if not is_last_layer and (now - last) < _INPRINT_BANK_MIN_INTERVAL:
  2053. return
  2054. try:
  2055. async with async_session() as db:
  2056. from backend.app.models.printer import Printer
  2057. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2058. printer = result.scalar_one_or_none()
  2059. if not printer:
  2060. return
  2061. # Reuses the notification snapshot path, which honours the
  2062. # `capture_finish_photo` setting (returns None when disabled) so we
  2063. # don't bank frames the user never asked for.
  2064. frame = await _capture_snapshot_for_notification(printer_id, printer, logger)
  2065. if frame:
  2066. _inprint_frame_bank[printer_id] = frame
  2067. _inprint_frame_bank_ts[printer_id] = now
  2068. logger.debug(
  2069. "[FINISH-PHOTO-BANK] banked in-print frame for printer %s at layer %s/%s (%d bytes)",
  2070. printer_id,
  2071. layer_num,
  2072. total,
  2073. len(frame),
  2074. )
  2075. except Exception as e:
  2076. logger.debug("[FINISH-PHOTO-BANK] bank failed for printer %s: %s", printer_id, e)
  2077. def _apply_camera_rotation(image_data: bytes, printer, logger) -> bytes:
  2078. """Apply camera rotation to snapshot image if configured."""
  2079. from backend.app.services.camera import apply_camera_rotation
  2080. return apply_camera_rotation(image_data, getattr(printer, "camera_rotation", 0), logger)
  2081. async def _send_print_start_notification(
  2082. printer_id: int,
  2083. data: dict,
  2084. archive_data: dict | None = None,
  2085. logger=None,
  2086. ):
  2087. """Helper to send print start notification with optional archive data."""
  2088. if logger is None:
  2089. logger = logging.getLogger(__name__)
  2090. try:
  2091. async with async_session() as db:
  2092. from backend.app.models.printer import Printer
  2093. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2094. printer = result.scalar_one_or_none()
  2095. printer_name = printer.name if printer else f"Printer {printer_id}"
  2096. # Capture camera snapshot for notification image attachment
  2097. image_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  2098. if image_data:
  2099. if archive_data is None:
  2100. archive_data = {}
  2101. archive_data["image_data"] = image_data
  2102. await notification_service.on_print_start(printer_id, printer_name, data, db, archive_data=archive_data)
  2103. # Send user-specific email notification for print start
  2104. if archive_data and archive_data.get("created_by_id"):
  2105. await notification_service.send_user_print_email(
  2106. event_type="user_print_start",
  2107. created_by_id=archive_data["created_by_id"],
  2108. printer_name=printer_name,
  2109. filename=data.get("subtask_name") or data.get("filename", "Unknown"),
  2110. db=db,
  2111. )
  2112. except Exception as e:
  2113. logger.warning("Notification on_print_start failed: %s", e)
  2114. async def _dispatch_user_print_email(
  2115. status: str,
  2116. created_by_id: int | None,
  2117. printer_name: str,
  2118. filename: str,
  2119. db,
  2120. ) -> None:
  2121. """Send a user-specific print-completion email based on print status.
  2122. Maps the normalised print status to the correct event type and delegates
  2123. to :meth:`NotificationService.send_user_print_email`. A single helper
  2124. avoids duplicating the ``if status == "completed" / elif "failed" / elif
  2125. "stopped"`` dispatch block at every call site.
  2126. Does nothing if *created_by_id* is ``None``.
  2127. """
  2128. if created_by_id is None:
  2129. return
  2130. if status == "completed":
  2131. event_type = "user_print_complete"
  2132. elif status == "failed":
  2133. event_type = "user_print_failed"
  2134. elif status in ("stopped", "aborted", "cancelled"):
  2135. event_type = "user_print_stopped"
  2136. else:
  2137. return
  2138. await notification_service.send_user_print_email(
  2139. event_type=event_type,
  2140. created_by_id=created_by_id,
  2141. printer_name=printer_name,
  2142. filename=filename,
  2143. db=db,
  2144. )
  2145. def _load_objects_from_archive(archive, printer_id: int, logger) -> None:
  2146. """Extract printable objects from an archive's 3MF file and store in printer state."""
  2147. try:
  2148. from backend.app.services.archive import extract_printable_objects_from_3mf
  2149. client = printer_manager.get_client(printer_id)
  2150. if not client:
  2151. return
  2152. file_path = app_settings.base_dir / archive.file_path
  2153. if file_path.is_file() and str(file_path).endswith(".3mf"):
  2154. with open(file_path, "rb") as f:
  2155. threemf_data = f.read()
  2156. # Extract with positions for UI overlay, scoped to the plate that
  2157. # is printing — resolve_plate_id is the same resolver /cover uses,
  2158. # so the object list can't disagree with the thumbnail it is drawn
  2159. # over (#2522).
  2160. printable_objects, bbox_all = extract_printable_objects_from_3mf(
  2161. threemf_data,
  2162. plate_number=resolve_plate_id(client.state),
  2163. include_positions=True,
  2164. )
  2165. if printable_objects:
  2166. client.state.printable_objects = printable_objects
  2167. client.state.printable_objects_bbox_all = bbox_all
  2168. client.state.skipped_objects = []
  2169. logger.info("Loaded %s printable objects for printer %s", len(printable_objects), printer_id)
  2170. except Exception as e:
  2171. logger.debug("Failed to extract printable objects from archive: %s", e)
  2172. async def on_print_start(printer_id: int, data: dict):
  2173. """Handle print start - archive the 3MF file immediately."""
  2174. logger = logging.getLogger(__name__)
  2175. logger.info("[CALLBACK] on_print_start called for printer %s, data keys: %s", printer_id, list(data.keys()))
  2176. # Clear any stale user-stopped flag from previous print cycles
  2177. _user_stopped_printers.discard(printer_id)
  2178. # #1721: drop any leftover pre-captured finish frame from a prior print
  2179. # so a never-consumed cache entry can't bleed into the new print's photo.
  2180. _stage22_finish_frames.pop(printer_id, None)
  2181. # #1867: same for the in-print frame bank — a queued print must not reuse
  2182. # the previous job's banked frame.
  2183. _inprint_frame_bank.pop(printer_id, None)
  2184. _inprint_frame_bank_ts.pop(printer_id, None)
  2185. # Cancel any active bed cooldown waiter for this printer
  2186. if _bed_cool_waiters.pop(printer_id, None):
  2187. logger.info("[BED-COOL] Cancelled bed cooldown waiter for printer %s (new print started)", printer_id)
  2188. # Clear cached cover images so the new print's thumbnail is fetched fresh
  2189. from backend.app.api.routes.printers import clear_cover_cache
  2190. clear_cover_cache(printer_id)
  2191. await ws_manager.send_print_start(printer_id, data)
  2192. # Notify when the print-start AMS mapping references tray slots without spool assignments.
  2193. await notify_missing_spool_assignments_on_print_start(printer_id, data, logger)
  2194. # MQTT relay - publish print start
  2195. try:
  2196. printer_info = printer_manager.get_printer(printer_id)
  2197. if printer_info:
  2198. await mqtt_relay.on_print_start(
  2199. printer_id,
  2200. printer_info.name,
  2201. printer_info.serial_number,
  2202. data.get("filename", ""),
  2203. data.get("subtask_name", ""),
  2204. )
  2205. except Exception:
  2206. pass # Don't fail print start callback if MQTT fails
  2207. # Capture AMS tray remain% for filament consumption tracking (skip if Spoolman handles usage)
  2208. try:
  2209. async with async_session() as db:
  2210. from backend.app.api.routes.settings import get_setting
  2211. _spoolman_on = await get_setting(db, "spoolman_enabled")
  2212. if not _spoolman_on or _spoolman_on.lower() != "true":
  2213. from backend.app.services.usage_tracker import on_print_start as usage_on_print_start
  2214. await usage_on_print_start(printer_id, data, printer_manager, db=db)
  2215. except Exception as e:
  2216. logger.warning("Usage tracker on_print_start failed: %s", e)
  2217. # Track if notification was sent (to avoid sending twice)
  2218. notification_sent = False
  2219. # Smart plug automation: turn on plug when print starts
  2220. try:
  2221. async with async_session() as db:
  2222. await smart_plug_manager.on_print_start(printer_id, db)
  2223. except Exception as e:
  2224. logger.warning("Smart plug on_print_start failed: %s", e)
  2225. async with async_session() as db:
  2226. from backend.app.models.printer import Printer
  2227. from backend.app.services.bambu_ftp import list_files_async
  2228. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2229. printer = result.scalar_one_or_none()
  2230. # Plate detection check - pause if objects detected on build plate
  2231. logger.info(
  2232. f"[PLATE CHECK] printer_id={printer_id}, plate_detection_enabled={printer.plate_detection_enabled if printer else 'NO PRINTER'}"
  2233. )
  2234. if printer and printer.plate_detection_enabled:
  2235. logger.info("[PLATE CHECK] ENTERING plate detection code for printer %s", printer_id)
  2236. # Release the pooled DB connection before the plate-detection camera
  2237. # work (a 2.5s light-settle sleep + FTP/camera capture). Only the
  2238. # printer SELECT has run so far — nothing to persist — so this commit
  2239. # is a data-noop that ends the read transaction and returns the
  2240. # connection to the pool during the I/O (issue #2572). expire_on_commit
  2241. # =False keeps printer.* readable; on_plate_not_empty (rare) and the
  2242. # archive lookups below re-acquire a fresh connection on next execute.
  2243. await db.commit()
  2244. try:
  2245. from backend.app.services.plate_detection import check_plate_empty
  2246. # Build ROI tuple from printer settings if available
  2247. roi = None
  2248. if all(
  2249. [
  2250. printer.plate_detection_roi_x is not None,
  2251. printer.plate_detection_roi_y is not None,
  2252. printer.plate_detection_roi_w is not None,
  2253. printer.plate_detection_roi_h is not None,
  2254. ]
  2255. ):
  2256. roi = (
  2257. printer.plate_detection_roi_x,
  2258. printer.plate_detection_roi_y,
  2259. printer.plate_detection_roi_w,
  2260. printer.plate_detection_roi_h,
  2261. )
  2262. # Auto-turn on chamber light if it's off for better detection
  2263. light_was_off = False
  2264. client = printer_manager.get_client(printer_id)
  2265. if client and client.state:
  2266. light_was_off = not client.state.chamber_light
  2267. if light_was_off:
  2268. logger.info("[PLATE CHECK] Turning on chamber light for printer %s", printer_id)
  2269. client.set_chamber_light(True)
  2270. # Wait for light to physically turn on and camera to adjust exposure
  2271. await asyncio.sleep(2.5)
  2272. logger.info("[PLATE CHECK] Running plate detection for printer %s", printer_id)
  2273. plate_result = await check_plate_empty(
  2274. printer_id=printer_id,
  2275. ip_address=printer.ip_address,
  2276. access_code=printer.access_code,
  2277. model=printer.model,
  2278. include_debug_image=False,
  2279. external_camera_url=printer.external_camera_url,
  2280. external_camera_type=printer.external_camera_type,
  2281. use_external=printer.external_camera_enabled,
  2282. roi=roi,
  2283. external_camera_snapshot_url=printer.external_camera_snapshot_url,
  2284. )
  2285. # Restore chamber light to original state
  2286. if light_was_off and client:
  2287. logger.info("[PLATE CHECK] Restoring chamber light to off for printer %s", printer_id)
  2288. client.set_chamber_light(False)
  2289. if not plate_result.needs_calibration and not plate_result.is_empty:
  2290. # Objects detected - pause the print!
  2291. logger.warning(
  2292. f"[PLATE CHECK] Objects detected on plate for printer {printer_id}! "
  2293. f"Confidence: {plate_result.confidence:.0%}, Diff: {plate_result.difference_percent:.1f}%"
  2294. )
  2295. client = printer_manager.get_client(printer_id)
  2296. if client:
  2297. client.pause_print()
  2298. logger.info("[PLATE CHECK] Print paused for printer %s", printer_id)
  2299. # Send notification about plate not empty
  2300. await ws_manager.broadcast(
  2301. {
  2302. "type": "plate_not_empty",
  2303. "printer_id": printer_id,
  2304. "printer_name": printer.name,
  2305. "message": f"Objects detected on build plate! Print paused. (Diff: {plate_result.difference_percent:.1f}%)",
  2306. }
  2307. )
  2308. # Also send push notification
  2309. try:
  2310. await notification_service.on_plate_not_empty(
  2311. printer_id=printer_id,
  2312. printer_name=printer.name,
  2313. db=db,
  2314. difference_percent=plate_result.difference_percent,
  2315. )
  2316. except Exception as notif_err:
  2317. logger.warning("[PLATE CHECK] Failed to send notification: %s", notif_err)
  2318. else:
  2319. logger.info("[PLATE CHECK] Plate is empty for printer %s, proceeding with print", printer_id)
  2320. except Exception as plate_err:
  2321. # Don't block print on plate detection errors
  2322. logger.warning("[PLATE CHECK] Plate detection failed for printer %s: %s", printer_id, plate_err)
  2323. if not printer:
  2324. logger.info("[CALLBACK] Skipping archive - printer not found in database")
  2325. if not notification_sent:
  2326. await _send_print_start_notification(printer_id, data, logger=logger)
  2327. return
  2328. if not printer.auto_archive:
  2329. # auto-archive disabled — check if there's an expected print (dispatched
  2330. # by BamBuddy via queue/reprint) that already has an archive to promote.
  2331. # If so, fall through to the expected-print handling below so the archive
  2332. # is tracked in _active_prints and usage tracking works at completion.
  2333. _fn = data.get("filename", "")
  2334. _sn = data.get("subtask_name", "")
  2335. _check_keys: list[tuple[int, str]] = []
  2336. if _sn:
  2337. _check_keys += [
  2338. (printer_id, _sn),
  2339. (printer_id, f"{_sn}.3mf"),
  2340. (printer_id, f"{_sn}.gcode.3mf"),
  2341. ]
  2342. if _fn:
  2343. _base_fn = _fn.split("/")[-1] if "/" in _fn else _fn
  2344. _check_keys.append((printer_id, _base_fn))
  2345. _no_archive_base = _base_fn.replace(".gcode", "").replace(".3mf", "")
  2346. _check_keys += [
  2347. (printer_id, _no_archive_base),
  2348. (printer_id, f"{_no_archive_base}.3mf"),
  2349. ]
  2350. _has_expected = any(k in _expected_prints for k in _check_keys)
  2351. if not _has_expected:
  2352. # No expected print — truly external print (started from slicer/touchscreen)
  2353. logger.info("[CALLBACK] Skipping archive - auto_archive: False, no expected print")
  2354. if not notification_sent:
  2355. _no_archive_creator: int | None = None
  2356. for _key in _check_keys:
  2357. _expected_prints.pop(_key, None)
  2358. _expected_print_registered_at.pop(_key, None)
  2359. popped_creator = _expected_print_creators.pop(_key, None)
  2360. if _no_archive_creator is None:
  2361. _no_archive_creator = popped_creator
  2362. _creator_data = {"created_by_id": _no_archive_creator} if _no_archive_creator else None
  2363. await _send_print_start_notification(printer_id, data, _creator_data, logger)
  2364. return
  2365. else:
  2366. logger.info("[CALLBACK] auto_archive disabled but expected print found — promoting archive")
  2367. # Get the filename and subtask_name
  2368. filename = data.get("filename", "")
  2369. subtask_name = data.get("subtask_name", "")
  2370. # MQTT subtask_id uniquely identifies a print job on the printer. When
  2371. # present, it lets us match an archive across a backend restart (#972):
  2372. # same id → same print → resume the existing row instead of cancelling
  2373. # it and recreating from scratch (which loses started_at). Treat "0"
  2374. # and "" as absent — Bambu reports "0" for non-cloud / local prints.
  2375. raw_mqtt = data.get("raw_data") or {}
  2376. subtask_id = raw_mqtt.get("subtask_id")
  2377. if subtask_id is not None:
  2378. subtask_id = str(subtask_id).strip()
  2379. if subtask_id in ("", "0"):
  2380. subtask_id = None
  2381. logger.info("[CALLBACK] Print start detected - filename: %s, subtask: %s", filename, subtask_name)
  2382. # Skip calibration prints — internal printer files should not be archived
  2383. # Bambu calibration gcode lives under /usr/ (e.g. /usr/etc/print/auto_cali_for_user.gcode)
  2384. if filename and filename.startswith("/usr/"):
  2385. logger.info("[CALLBACK] Skipping archive — internal printer file detected: %s", filename)
  2386. if not notification_sent:
  2387. await _send_print_start_notification(printer_id, data, logger=logger)
  2388. return
  2389. if not filename and not subtask_name:
  2390. # Send notification without archive data (no filename)
  2391. logger.info("[CALLBACK] Skipping archive - no filename or subtask_name")
  2392. if not notification_sent:
  2393. await _send_print_start_notification(printer_id, data, logger=logger)
  2394. return
  2395. # Check if this is an expected print from reprint/scheduled
  2396. # Build list of possible keys to check
  2397. expected_keys = []
  2398. if subtask_name:
  2399. expected_keys.append((printer_id, subtask_name))
  2400. expected_keys.append((printer_id, f"{subtask_name}.3mf"))
  2401. expected_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  2402. if filename:
  2403. fname = filename.split("/")[-1] if "/" in filename else filename
  2404. expected_keys.append((printer_id, fname))
  2405. # Strip extensions to match
  2406. base = fname.replace(".gcode", "").replace(".3mf", "")
  2407. expected_keys.append((printer_id, base))
  2408. expected_keys.append((printer_id, f"{base}.3mf"))
  2409. expected_archive_id = None
  2410. for key in expected_keys:
  2411. expected_archive_id = _expected_prints.pop(key, None)
  2412. _expected_print_registered_at.pop(key, None)
  2413. if expected_archive_id:
  2414. # Clean up other possible keys for this print
  2415. for other_key in expected_keys:
  2416. _expected_prints.pop(other_key, None)
  2417. _expected_print_registered_at.pop(other_key, None)
  2418. break
  2419. if expected_archive_id:
  2420. # This is a reprint/scheduled print - use existing archive, don't create new one
  2421. logger.info("Using expected archive %s for print (skipping duplicate)", expected_archive_id)
  2422. from backend.app.models.archive import PrintArchive
  2423. result = await db.execute(select(PrintArchive).where(PrintArchive.id == expected_archive_id))
  2424. archive = result.scalar_one_or_none()
  2425. if archive:
  2426. # Update archive status to printing
  2427. archive.status = "printing"
  2428. archive.started_at = datetime.now(timezone.utc)
  2429. # Reprint of an archive reuses the source row. Without resetting
  2430. # ``timelapse_path`` _scan_for_timelapse_with_retries early-returns
  2431. # ("already has timelapse") and _capture_finish_photo_from_timelapse
  2432. # extracts the *original* print's last frame, which then ships in
  2433. # the completion notification (#1707). Clear the path so the
  2434. # scanner runs fresh; also unlink the old video file so reprints
  2435. # don't accumulate orphans in the archive directory. Photos list
  2436. # is left alone — accumulating one finish photo per run is fine.
  2437. # The print-start baseline (#2704) is stale for the same reason:
  2438. # it describes the printer before the previous run. The capture
  2439. # below overwrites it, but clear it here too so an early failure
  2440. # can't leave the scan diffing against the wrong snapshot.
  2441. archive.timelapse_baseline = None
  2442. stale_timelapse_relpath = archive.timelapse_path
  2443. if stale_timelapse_relpath:
  2444. archive.timelapse_path = None
  2445. try:
  2446. stale_path = app_settings.base_dir / stale_timelapse_relpath
  2447. if stale_path.is_file():
  2448. stale_path.unlink()
  2449. logger.info(
  2450. "Deleted stale timelapse %s on reprint of archive %s",
  2451. stale_timelapse_relpath,
  2452. expected_archive_id,
  2453. )
  2454. except OSError as e:
  2455. logger.warning(
  2456. "Failed to delete stale timelapse %s on reprint: %s",
  2457. stale_timelapse_relpath,
  2458. e,
  2459. )
  2460. # Persist a restart-stable id so a later restart resumes this
  2461. # archive by subtask_id instead of name-matching + duplicating
  2462. # it (#1485). The printer often hasn't echoed subtask_id back
  2463. # this soon after dispatch, so fall back to the id Bambuddy
  2464. # minted when it sent the print command. Scoped to this
  2465. # expected-print branch on purpose: an expected match means
  2466. # Bambuddy dispatched this exact print in this process, so the
  2467. # client's last-dispatch id genuinely belongs to it — using it
  2468. # for an externally-started print could mis-tag the archive.
  2469. effective_subtask_id = subtask_id
  2470. if not effective_subtask_id:
  2471. _client = printer_manager.get_client(printer_id)
  2472. _dispatched = getattr(_client, "last_dispatch_subtask_id", None) if _client else None
  2473. if _dispatched:
  2474. effective_subtask_id = str(_dispatched).strip() or None
  2475. # Update on first-set OR on reprint (the queue dispatcher mints
  2476. # a fresh subtask_id per dispatch in bambu_mqtt:3647). Skipping
  2477. # the rewrite for reprints leaves the archive holding the FIRST
  2478. # run's id; if MQTT then reconnects mid-print, the reconciler
  2479. # (#1542) compares the stale stored id against the printer's
  2480. # live id, sees a mismatch, and synthesises a bogus PRINT
  2481. # COMPLETE — exactly the false-positive "Print Stopped" reported
  2482. # in #1807. Inequality check preserves the noop-on-stable-push
  2483. # behaviour the earlier `not archive.subtask_id` guard provided.
  2484. if effective_subtask_id and archive.subtask_id != effective_subtask_id:
  2485. archive.subtask_id = effective_subtask_id
  2486. # #1403 follow-up: VP-queue archives are created with
  2487. # printer_id=None at queue-add time (we don't know which
  2488. # printer will run the job yet). When the print actually
  2489. # starts on a specific printer the expected-archive lookup
  2490. # used to skip this assignment, leaving printer_id=None
  2491. # forever — which then disables the "Scan for timelapse"
  2492. # button in ArchivesPage (gated on !archive.printer_id).
  2493. if archive.printer_id != printer_id:
  2494. archive.printer_id = printer_id
  2495. await db.commit()
  2496. # Track as active print
  2497. _active_prints[(printer_id, archive.filename)] = archive.id
  2498. if subtask_name:
  2499. _active_prints[(printer_id, f"{subtask_name}.3mf")] = archive.id
  2500. # Start timelapse session if external camera is enabled (#1353).
  2501. # Queue / VP-dispatched prints land here in the expected-archive
  2502. # branch and used to skip start_session entirely — frames were
  2503. # never captured and the post-print stitch silently returned None.
  2504. _maybe_start_layer_timelapse(printer, printer_id, archive.id)
  2505. # Inject ams_mapping into usage tracker session — the session was created
  2506. # before expected-print promotion, so it may have ams_mapping=None when
  2507. # the MQTT request topic subscription failed (common on P1S/A1).
  2508. _stored_map = _print_ams_mappings.get(expected_archive_id)
  2509. _stored_plate_id = _print_plate_ids.get(expected_archive_id)
  2510. if _stored_map or _stored_plate_id is not None:
  2511. try:
  2512. from backend.app.services.usage_tracker import _active_sessions
  2513. _ut_session = _active_sessions.get(printer_id)
  2514. if _ut_session and _stored_map and not _ut_session.ams_mapping:
  2515. _ut_session.ams_mapping = _stored_map
  2516. logger.info("[CALLBACK] Injected ams_mapping into usage tracker session: %s", _stored_map)
  2517. # plate_id injection covers direct-Print of plate N of a multi-plate
  2518. # 3MF — queue prints already capture it via the on_print_start queue
  2519. # lookup, but direct-Print never goes through the queue (#1697).
  2520. if _ut_session and _stored_plate_id is not None and _ut_session.plate_id is None:
  2521. _ut_session.plate_id = _stored_plate_id
  2522. logger.info("[CALLBACK] Injected plate_id into usage tracker session: %s", _stored_plate_id)
  2523. except Exception:
  2524. pass
  2525. # Set up energy tracking (#941: persist start on archive row)
  2526. await _record_energy_start(archive, printer_id, db, context="expected-print")
  2527. await ws_manager.send_archive_updated(
  2528. {
  2529. "id": archive.id,
  2530. "status": "printing",
  2531. }
  2532. )
  2533. # Send notification with archive data (reprint/scheduled)
  2534. if not notification_sent:
  2535. # Use archive's created_by_id; fall back to the creator registered via
  2536. # register_expected_print (handles library-file-based queue items where
  2537. # the freshly-created archive has no created_by_id yet).
  2538. # Pop ALL matching keys so no stale entries remain in the dict.
  2539. fallback_creator = None
  2540. for key in expected_keys:
  2541. popped = _expected_print_creators.pop(key, None)
  2542. if fallback_creator is None:
  2543. fallback_creator = popped
  2544. archive_data = {
  2545. "print_time_seconds": archive.print_time_seconds,
  2546. "created_by_id": archive.created_by_id or fallback_creator,
  2547. }
  2548. await _send_print_start_notification(printer_id, data, archive_data, logger)
  2549. # Extract printable objects from the archived 3MF file
  2550. _load_objects_from_archive(archive, printer_id, logger)
  2551. # Store Spoolman tracking data for per-filament usage reporting
  2552. try:
  2553. await _store_spoolman_print_data(
  2554. printer_id,
  2555. archive.id,
  2556. archive.file_path,
  2557. db,
  2558. printer_manager,
  2559. ams_mapping=_get_start_ams_mapping(data, archive.id),
  2560. plate_id=_get_start_plate_id(archive.id),
  2561. )
  2562. except Exception as e:
  2563. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  2564. # Capture timelapse file baseline for snapshot-diff on completion
  2565. # (mirrors the new-archive branch). Queue / VP-dispatched prints
  2566. # hit this branch — without the baseline the completion-time scan
  2567. # falls into its "take baseline now" fallback, which snapshots
  2568. # AFTER the new MP4 already exists and never matches a diff
  2569. # (#1403 follow-up — see pwostran's 2026-05-18 support bundle).
  2570. await _capture_timelapse_baseline_at_start(printer, printer_id, logger, archive_id=archive.id)
  2571. return # Skip creating a new archive
  2572. # Check if there's already a "printing" archive for this printer/file
  2573. # This prevents duplicates when backend restarts during an active print
  2574. from backend.app.models.archive import PrintArchive
  2575. existing_archive: PrintArchive | None = None
  2576. # Preferred match: subtask_id equality. MQTT reports the same subtask_id
  2577. # across a backend restart for the same print, so this is the most
  2578. # reliable way to reattach. We also accept a previously stale-cancelled
  2579. # archive here so users upgrading mid-print get revived when the row
  2580. # their earlier Bambuddy version wrongly cancelled reappears (#972).
  2581. if subtask_id:
  2582. by_id = await db.execute(
  2583. select(PrintArchive)
  2584. .where(PrintArchive.printer_id == printer_id)
  2585. .where(PrintArchive.subtask_id == subtask_id)
  2586. .where(PrintArchive.status.in_(["printing", "cancelled"]))
  2587. .order_by(PrintArchive.created_at.desc())
  2588. .limit(1)
  2589. )
  2590. candidate = by_id.scalar_one_or_none()
  2591. if candidate and (candidate.status == "printing" or (candidate.failure_reason or "").startswith("Stale")):
  2592. existing_archive = candidate
  2593. # Fallback match: name-based lookup. Kept as-is for prints whose
  2594. # subtask_id is missing ("0" / local / non-cloud prints).
  2595. if existing_archive is None:
  2596. check_name = subtask_name or filename.split("/")[-1].replace(".gcode", "").replace(".3mf", "")
  2597. existing = await db.execute(
  2598. select(PrintArchive)
  2599. .where(PrintArchive.printer_id == printer_id)
  2600. .where(PrintArchive.status == "printing")
  2601. .where(
  2602. or_(
  2603. PrintArchive.print_name == check_name,
  2604. PrintArchive.filename.in_(
  2605. [
  2606. f"{check_name}.3mf",
  2607. f"{check_name}.gcode.3mf",
  2608. ]
  2609. ),
  2610. )
  2611. )
  2612. .order_by(PrintArchive.created_at.desc())
  2613. .limit(1)
  2614. )
  2615. existing_archive = existing.scalar_one_or_none()
  2616. if existing_archive:
  2617. # subtask_id match → always resume, regardless of age. Same print,
  2618. # just a backend restart. Revive if it was previously stale-cancelled.
  2619. subtask_match = bool(subtask_id and existing_archive.subtask_id == subtask_id)
  2620. if subtask_match:
  2621. if existing_archive.status == "cancelled":
  2622. logger.warning(
  2623. "Reviving stale-cancelled archive %s — matching subtask_id %s confirms same print (#972)",
  2624. existing_archive.id,
  2625. subtask_id,
  2626. )
  2627. existing_archive.status = "printing"
  2628. existing_archive.failure_reason = None
  2629. await db.commit()
  2630. else:
  2631. logger.info("Resuming archive %s on subtask_id match (%s)", existing_archive.id, subtask_id)
  2632. _active_prints[(printer_id, existing_archive.filename)] = existing_archive.id
  2633. if existing_archive.energy_start_kwh is None:
  2634. await _record_energy_start(existing_archive, printer_id, db, context="subtask-resume")
  2635. if not notification_sent:
  2636. archive_data = {
  2637. "print_time_seconds": existing_archive.print_time_seconds,
  2638. "created_by_id": existing_archive.created_by_id,
  2639. }
  2640. await _send_print_start_notification(printer_id, data, archive_data, logger)
  2641. _load_objects_from_archive(existing_archive, printer_id, logger)
  2642. return
  2643. # Name-match only (no subtask_id to anchor on): decide resume vs.
  2644. # stale from the printer's *current* progress, not wall-clock age.
  2645. # A genuinely long print used to trip a blind 4h cutoff and have its
  2646. # live archive cancelled + duplicated on every backend restart
  2647. # (#1485). If the printer reports real progress, this name-matched
  2648. # 'printing' archive IS that ongoing print — resume it whatever its
  2649. # age. Only treat it as a stale leftover when the printer clearly
  2650. # shows a different, freshly-started print: near-0% progress on an
  2651. # archive far too old to still be at 0%. Unknown progress (printer
  2652. # not connected) never cancels — resuming is the safe default.
  2653. archive_age = datetime.now(timezone.utc) - existing_archive.created_at.replace(tzinfo=timezone.utc)
  2654. live_status = printer_manager.get_status(printer_id)
  2655. live_progress = getattr(live_status, "progress", None) if live_status else None
  2656. looks_stale = (
  2657. live_progress is not None and live_progress < 1.0 and archive_age.total_seconds() > 2 * 60 * 60
  2658. )
  2659. if looks_stale:
  2660. logger.warning(
  2661. f"Found stale 'printing' archive {existing_archive.id} (age: {archive_age}, "
  2662. f"printer progress {live_progress:.0f}%) — marking cancelled and creating new archive"
  2663. )
  2664. existing_archive.status = "cancelled"
  2665. existing_archive.failure_reason = "Stale - print likely cancelled or failed without status update"
  2666. await db.commit()
  2667. # Fall through to create new archive (don't return)
  2668. else:
  2669. logger.info(
  2670. f"Skipping duplicate - already have printing archive {existing_archive.id} for {check_name}"
  2671. )
  2672. # Track this as the active print
  2673. _active_prints[(printer_id, existing_archive.filename)] = existing_archive.id
  2674. # Attach subtask_id retroactively so future restarts can resume.
  2675. # Compare for inequality (not "is empty") to also pick up reprint
  2676. # dispatches that mint a fresh id — see #1807 for the bogus
  2677. # "Print Stopped" the strict-empty guard caused on reconnect.
  2678. if subtask_id and existing_archive.subtask_id != subtask_id:
  2679. existing_archive.subtask_id = subtask_id
  2680. await db.commit()
  2681. # Also set up energy tracking if not already tracked (#941: persisted column)
  2682. if existing_archive.energy_start_kwh is None:
  2683. await _record_energy_start(existing_archive, printer_id, db, context="existing-printing")
  2684. # Send notification with archive data (existing archive)
  2685. if not notification_sent:
  2686. archive_data = {
  2687. "print_time_seconds": existing_archive.print_time_seconds,
  2688. "created_by_id": existing_archive.created_by_id,
  2689. }
  2690. await _send_print_start_notification(printer_id, data, archive_data, logger)
  2691. # Extract printable objects from the archived 3MF file
  2692. _load_objects_from_archive(existing_archive, printer_id, logger)
  2693. return
  2694. # Build list of possible 3MF filenames to try
  2695. possible_names = []
  2696. # Bambu printers typically store files as "Name.gcode.3mf"
  2697. # The subtask_name is usually the best source for the filename
  2698. if subtask_name:
  2699. # Try common Bambu naming patterns
  2700. possible_names.append(f"{subtask_name}.gcode.3mf")
  2701. possible_names.append(f"{subtask_name}.3mf")
  2702. # Try original filename with .3mf extension
  2703. if filename:
  2704. # Extract just the filename part, not the full path
  2705. fname = filename.split("/")[-1] if "/" in filename else filename
  2706. if fname.endswith(".3mf"):
  2707. possible_names.append(fname)
  2708. elif fname.endswith(".gcode"):
  2709. base = fname.rsplit(".", 1)[0]
  2710. possible_names.append(f"{base}.gcode.3mf")
  2711. possible_names.append(f"{base}.3mf")
  2712. else:
  2713. possible_names.append(f"{fname}.gcode.3mf")
  2714. possible_names.append(f"{fname}.3mf")
  2715. # Also try with spaces converted to underscores (Bambu Studio may normalize filenames)
  2716. space_variants = []
  2717. for name in possible_names:
  2718. if " " in name:
  2719. space_variants.append(name.replace(" ", "_"))
  2720. possible_names.extend(space_variants)
  2721. # Remove duplicates while preserving order
  2722. seen = set()
  2723. possible_names = [x for x in possible_names if not (x in seen or seen.add(x))]
  2724. logger.info("Trying filenames: %s", possible_names)
  2725. # Release the pooled DB connection before the 3MF FTP download. Reaching
  2726. # here means none of the expected-/existing-archive write branches ran
  2727. # (they all return earlier) — only SELECTs have executed on this path, so
  2728. # this commit persists nothing; it ends the read transaction so the
  2729. # connection returns to the pool during the download. That download tries
  2730. # up to five remote paths per candidate filename with retry/backoff and
  2731. # can run for minutes under FTP contention; holding the session across it
  2732. # pinned one pooled connection idle-in-transaction (issue #2572). No DB
  2733. # work runs during the download — the new-archive writes below re-acquire
  2734. # a fresh connection, and expire_on_commit=False keeps printer.* readable.
  2735. await db.commit()
  2736. # Try to find and download the 3MF file
  2737. temp_path = None
  2738. downloaded_filename = None
  2739. # Cache check: cover endpoint may have already pulled this 3MF during
  2740. # the print (frontend opens the card and shows the thumbnail) — reuse
  2741. # that file instead of re-downloading 36MB over the same FTP link that
  2742. # just served it (#972). The cache keys on a normalized filename so
  2743. # variants like "X", "X.3mf", "X.gcode.3mf" all collapse to one entry.
  2744. for try_filename in possible_names:
  2745. if not try_filename.endswith(".3mf"):
  2746. continue
  2747. cached = get_cached_3mf(printer_id, try_filename)
  2748. if cached:
  2749. logger.info("Reusing cached 3MF from %s (avoided duplicate FTP)", cached)
  2750. temp_path = cached
  2751. downloaded_filename = try_filename
  2752. break
  2753. # Get FTP retry settings
  2754. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  2755. for try_filename in possible_names if not downloaded_filename else []:
  2756. if not try_filename.endswith(".3mf"):
  2757. continue
  2758. # Root (/) is where BambuStudio/OrcaSlicer uploads land on A1/P1-series
  2759. # printers, so try it first — deferring it to last cost #972's reporter
  2760. # ~48 minutes of retries on /cache//model//data//data/Metadata before
  2761. # landing on the path that actually had the file.
  2762. remote_paths = [
  2763. f"/{try_filename}",
  2764. f"/cache/{try_filename}",
  2765. f"/model/{try_filename}",
  2766. f"/data/{try_filename}",
  2767. f"/data/Metadata/{try_filename}",
  2768. ]
  2769. temp_path = app_settings.archive_dir / "temp" / try_filename
  2770. temp_path.parent.mkdir(parents=True, exist_ok=True)
  2771. for remote_path in remote_paths:
  2772. logger.debug("Trying FTP download: %s", remote_path)
  2773. try:
  2774. if ftp_retry_enabled:
  2775. downloaded = await with_ftp_retry(
  2776. download_file_async,
  2777. printer.ip_address,
  2778. printer.access_code,
  2779. remote_path,
  2780. temp_path,
  2781. timeout=ftp_timeout,
  2782. socket_timeout=ftp_timeout,
  2783. printer_model=printer.model,
  2784. max_retries=ftp_retry_count,
  2785. retry_delay=ftp_retry_delay,
  2786. operation_name=f"Download 3MF from {remote_path}",
  2787. non_retry_exceptions=(FileNotOnPrinterError,),
  2788. )
  2789. else:
  2790. downloaded = await download_file_async(
  2791. printer.ip_address,
  2792. printer.access_code,
  2793. remote_path,
  2794. temp_path,
  2795. timeout=ftp_timeout,
  2796. socket_timeout=ftp_timeout,
  2797. printer_model=printer.model,
  2798. )
  2799. if downloaded:
  2800. downloaded_filename = try_filename
  2801. logger.info("Downloaded: %s", remote_path)
  2802. # Populate shared cache so the cover endpoint (if it
  2803. # runs next) doesn't refetch the same 36MB over FTP.
  2804. cache_3mf_download(printer_id, try_filename, temp_path)
  2805. break
  2806. except FileNotOnPrinterError:
  2807. # 550 — file isn't at this path. Advance to next candidate
  2808. # without burning the retry budget.
  2809. logger.debug("3MF not at %s (550), trying next path", remote_path)
  2810. except Exception as e:
  2811. logger.debug("FTP download failed for %s: %s", remote_path, e)
  2812. if downloaded_filename:
  2813. break
  2814. # If still not found, try listing directories to find matching file
  2815. # Different printer models use different directory structures
  2816. if not downloaded_filename and (filename or subtask_name):
  2817. search_term = (subtask_name or filename).lower().replace(".gcode", "").replace(".3mf", "")
  2818. logger.info("Direct FTP download failed, searching directories for '%s'", search_term)
  2819. search_dirs = ["/cache", "/model", "/data", "/data/Metadata", "/"]
  2820. for search_dir in search_dirs:
  2821. if downloaded_filename:
  2822. break
  2823. try:
  2824. dir_files = await list_files_async(
  2825. printer.ip_address, printer.access_code, search_dir, printer_model=printer.model
  2826. )
  2827. threemf_files = [f.get("name") for f in dir_files if f.get("name", "").endswith(".3mf")]
  2828. if threemf_files:
  2829. logger.info(
  2830. f"Found {len(threemf_files)} 3MF files in {search_dir}: {threemf_files[:5]}{'...' if len(threemf_files) > 5 else ''}"
  2831. )
  2832. for f in dir_files:
  2833. if f.get("is_directory"):
  2834. continue
  2835. fname = f.get("name", "")
  2836. # Normalize both for comparison (spaces and underscores are equivalent)
  2837. fname_normalized = fname.lower().replace(" ", "_")
  2838. search_normalized = search_term.replace(" ", "_")
  2839. if fname.endswith(".3mf") and search_normalized in fname_normalized:
  2840. logger.info("Found matching file in %s: %s", search_dir, fname)
  2841. temp_path = app_settings.archive_dir / "temp" / fname
  2842. temp_path.parent.mkdir(parents=True, exist_ok=True)
  2843. remote_full_path = posixpath.join(search_dir, fname)
  2844. if ftp_retry_enabled:
  2845. downloaded = await with_ftp_retry(
  2846. download_file_async,
  2847. printer.ip_address,
  2848. printer.access_code,
  2849. remote_full_path,
  2850. temp_path,
  2851. timeout=ftp_timeout,
  2852. socket_timeout=ftp_timeout,
  2853. printer_model=printer.model,
  2854. max_retries=ftp_retry_count,
  2855. retry_delay=ftp_retry_delay,
  2856. operation_name=f"Download 3MF from {remote_full_path}",
  2857. )
  2858. else:
  2859. downloaded = await download_file_async(
  2860. printer.ip_address,
  2861. printer.access_code,
  2862. remote_full_path,
  2863. temp_path,
  2864. timeout=ftp_timeout,
  2865. socket_timeout=ftp_timeout,
  2866. printer_model=printer.model,
  2867. )
  2868. if downloaded:
  2869. downloaded_filename = fname
  2870. logger.info("Found and downloaded from %s: %s", search_dir, fname)
  2871. cache_3mf_download(printer_id, fname, temp_path)
  2872. break
  2873. except Exception as e:
  2874. logger.debug("Failed to list %s: %s", search_dir, e)
  2875. # Validate the downloaded 3MF actually matches the plate that's running
  2876. # (#1204): subtask_name lags across consecutive plates of the same model,
  2877. # so the first FTP candidate (built from subtask_name) can land on the
  2878. # previous plate's still-resident upload. Cross-check the slice_info
  2879. # plate index against the plate parsed from gcode_file (always fresh —
  2880. # it's the field whose change triggered this callback).
  2881. if downloaded_filename and temp_path:
  2882. expected_plate = parse_plate_id(filename)
  2883. actual_plate = peek_plate_index_in_3mf(temp_path) if expected_plate is not None else None
  2884. if expected_plate is not None and actual_plate is not None and actual_plate != expected_plate:
  2885. logger.warning(
  2886. "[CALLBACK] 3MF plate mismatch: downloaded %s reports plate %s but printer is "
  2887. "running plate %s — subtask_name=%r appears stale, retrying with corrected name",
  2888. downloaded_filename,
  2889. actual_plate,
  2890. expected_plate,
  2891. subtask_name,
  2892. )
  2893. corrected_subtask = swap_plate_suffix(subtask_name, expected_plate)
  2894. retry_succeeded = False
  2895. if corrected_subtask and corrected_subtask != subtask_name:
  2896. for try_filename in (f"{corrected_subtask}.gcode.3mf", f"{corrected_subtask}.3mf"):
  2897. retry_temp_path = app_settings.archive_dir / "temp" / try_filename
  2898. retry_temp_path.parent.mkdir(parents=True, exist_ok=True)
  2899. for remote_path in (
  2900. f"/{try_filename}",
  2901. f"/cache/{try_filename}",
  2902. f"/model/{try_filename}",
  2903. f"/data/{try_filename}",
  2904. f"/data/Metadata/{try_filename}",
  2905. ):
  2906. try:
  2907. if ftp_retry_enabled:
  2908. downloaded = await with_ftp_retry(
  2909. download_file_async,
  2910. printer.ip_address,
  2911. printer.access_code,
  2912. remote_path,
  2913. retry_temp_path,
  2914. timeout=ftp_timeout,
  2915. socket_timeout=ftp_timeout,
  2916. printer_model=printer.model,
  2917. max_retries=ftp_retry_count,
  2918. retry_delay=ftp_retry_delay,
  2919. operation_name=f"Re-download 3MF from {remote_path}",
  2920. non_retry_exceptions=(FileNotOnPrinterError,),
  2921. )
  2922. else:
  2923. downloaded = await download_file_async(
  2924. printer.ip_address,
  2925. printer.access_code,
  2926. remote_path,
  2927. retry_temp_path,
  2928. timeout=ftp_timeout,
  2929. socket_timeout=ftp_timeout,
  2930. printer_model=printer.model,
  2931. )
  2932. if downloaded and peek_plate_index_in_3mf(retry_temp_path) == expected_plate:
  2933. logger.info(
  2934. "[CALLBACK] Re-download succeeded with corrected name %s "
  2935. "(plate %s) — replacing wrong file",
  2936. try_filename,
  2937. expected_plate,
  2938. )
  2939. try:
  2940. temp_path.unlink(missing_ok=True)
  2941. except OSError:
  2942. pass
  2943. temp_path = retry_temp_path
  2944. downloaded_filename = try_filename
  2945. subtask_name = corrected_subtask
  2946. cache_3mf_download(printer_id, try_filename, temp_path)
  2947. retry_succeeded = True
  2948. break
  2949. elif downloaded:
  2950. # Wrong plate again — discard and keep trying
  2951. try:
  2952. retry_temp_path.unlink(missing_ok=True)
  2953. except OSError:
  2954. pass
  2955. except FileNotOnPrinterError:
  2956. continue
  2957. except Exception as e:
  2958. logger.debug("Re-download failed for %s: %s", remote_path, e)
  2959. if retry_succeeded:
  2960. break
  2961. # If the retry didn't find a matching file, drop the wrong 3MF
  2962. # so the no-3MF fallback below creates an archive whose name
  2963. # at least reflects the right plate.
  2964. if not retry_succeeded:
  2965. logger.warning(
  2966. "[CALLBACK] Could not re-download correct plate %s — falling back to no-3MF archive",
  2967. expected_plate,
  2968. )
  2969. try:
  2970. temp_path.unlink(missing_ok=True)
  2971. except OSError:
  2972. pass
  2973. temp_path = None
  2974. downloaded_filename = None
  2975. # Override the stale subtask_name so the fallback archive's
  2976. # print_name reflects the correct plate. Prefer the swapped
  2977. # name when we have one; otherwise let filename win.
  2978. if corrected_subtask:
  2979. subtask_name = corrected_subtask
  2980. else:
  2981. subtask_name = ""
  2982. if not downloaded_filename or not temp_path:
  2983. logger.warning("Could not find 3MF file for print: %s", filename or subtask_name)
  2984. # Create a fallback archive without 3MF data so the print is still tracked
  2985. # This commonly happens with P1S/A1 printers where FTP has file size limitations
  2986. try:
  2987. from backend.app.models.archive import PrintArchive
  2988. # Derive print name from subtask_name or filename
  2989. print_name = subtask_name or filename
  2990. if print_name:
  2991. # Clean up the name (remove extensions, path parts)
  2992. print_name = print_name.split("/")[-1]
  2993. print_name = print_name.replace(".gcode.3mf", "").replace(".gcode", "").replace(".3mf", "")
  2994. else:
  2995. print_name = "Unknown Print"
  2996. # Recover estimated print time from MQTT (best-effort for notifications)
  2997. fallback_print_time = None
  2998. mqtt_remaining = data.get("remaining_time")
  2999. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  3000. fallback_print_time = int(mqtt_remaining)
  3001. if fallback_print_time is None:
  3002. mc_remaining = (data.get("raw_data") or {}).get("mc_remaining_time")
  3003. if mc_remaining and isinstance(mc_remaining, (int, float)) and mc_remaining > 0:
  3004. fallback_print_time = int(mc_remaining * 60)
  3005. # Best-effort filament metadata from MQTT — see
  3006. # _extract_filament_data_from_mqtt. Without this the fallback
  3007. # archive's filament fields stayed NULL even though the AMS
  3008. # state at print start was sitting right there in `data`.
  3009. # The slicer's ams_mapping (when present) narrows the result
  3010. # to slots actually used by the print (#1533).
  3011. mqtt_filament_meta = _extract_filament_data_from_mqtt(data, _get_start_ams_mapping(data, None))
  3012. # Create minimal archive entry
  3013. fallback_archive = PrintArchive(
  3014. printer_id=printer_id,
  3015. filename=filename or f"{print_name}.3mf",
  3016. file_path="", # Empty - no 3MF file available
  3017. file_size=0,
  3018. print_name=print_name,
  3019. print_time_seconds=fallback_print_time,
  3020. status="printing",
  3021. started_at=datetime.now(timezone.utc),
  3022. subtask_id=subtask_id,
  3023. filament_type=mqtt_filament_meta.get("filament_type"),
  3024. filament_color=mqtt_filament_meta.get("filament_color"),
  3025. extra_data={"no_3mf_available": True, "original_subtask": subtask_name, "_print_data": data},
  3026. )
  3027. db.add(fallback_archive)
  3028. await db.commit()
  3029. await db.refresh(fallback_archive)
  3030. logger.info("Created fallback archive %s for %s (no 3MF available)", fallback_archive.id, print_name)
  3031. _maybe_start_layer_timelapse(printer, printer_id, fallback_archive.id)
  3032. # Track as active print
  3033. _active_prints[(printer_id, fallback_archive.filename)] = fallback_archive.id
  3034. if filename:
  3035. _active_prints[(printer_id, filename)] = fallback_archive.id
  3036. if subtask_name:
  3037. _active_prints[(printer_id, f"{subtask_name}.3mf")] = fallback_archive.id
  3038. _active_prints[(printer_id, subtask_name)] = fallback_archive.id
  3039. # Record starting energy if smart plug available (#941: persisted column)
  3040. await _record_energy_start(fallback_archive, printer_id, db, context="fallback")
  3041. # Send WebSocket notification
  3042. await ws_manager.send_archive_created(
  3043. {
  3044. "id": fallback_archive.id,
  3045. "printer_id": fallback_archive.printer_id,
  3046. "filename": fallback_archive.filename,
  3047. "print_name": fallback_archive.print_name,
  3048. "status": fallback_archive.status,
  3049. }
  3050. )
  3051. # MQTT relay - publish archive created
  3052. try:
  3053. await mqtt_relay.on_archive_created(
  3054. archive_id=fallback_archive.id,
  3055. print_name=fallback_archive.print_name,
  3056. printer_name=printer.name,
  3057. status=fallback_archive.status,
  3058. )
  3059. except Exception:
  3060. pass # Don't fail if MQTT fails
  3061. # Store Spoolman tracking data (may not work for fallback since no 3MF)
  3062. try:
  3063. await _store_spoolman_print_data(
  3064. printer_id,
  3065. fallback_archive.id,
  3066. fallback_archive.file_path,
  3067. db,
  3068. printer_manager,
  3069. ams_mapping=_get_start_ams_mapping(data, fallback_archive.id),
  3070. plate_id=_get_start_plate_id(fallback_archive.id),
  3071. )
  3072. except Exception as e:
  3073. logger.debug("[SPOOLMAN] Could not store tracking for fallback archive: %s", e)
  3074. # Send notification without archive data (file not found)
  3075. if not notification_sent:
  3076. await _send_print_start_notification(printer_id, data, logger=logger)
  3077. return
  3078. except Exception as e:
  3079. logger.error("Failed to create fallback archive: %s", e)
  3080. # Send notification without archive data (file not found)
  3081. if not notification_sent:
  3082. await _send_print_start_notification(printer_id, data, logger=logger)
  3083. return
  3084. try:
  3085. # Archive the file with status "printing"
  3086. service = ArchiveService(db)
  3087. archive = await service.archive_print(
  3088. printer_id=printer_id,
  3089. source_file=temp_path,
  3090. print_data={**data, "status": "printing"},
  3091. subtask_id=subtask_id,
  3092. )
  3093. if archive:
  3094. # Track this active print (use both original filename and downloaded filename)
  3095. _active_prints[(printer_id, downloaded_filename)] = archive.id
  3096. if filename and filename != downloaded_filename:
  3097. _active_prints[(printer_id, filename)] = archive.id
  3098. if subtask_name:
  3099. _active_prints[(printer_id, f"{subtask_name}.3mf")] = archive.id
  3100. logger.info("Created archive %s for %s", archive.id, downloaded_filename)
  3101. _maybe_start_layer_timelapse(printer, printer_id, archive.id)
  3102. # Record starting energy from smart plug if available (#941: persisted column)
  3103. await _record_energy_start(archive, printer_id, db, context="auto-archive")
  3104. await ws_manager.send_archive_created(
  3105. {
  3106. "id": archive.id,
  3107. "printer_id": archive.printer_id,
  3108. "filename": archive.filename,
  3109. "print_name": archive.print_name,
  3110. "status": archive.status,
  3111. }
  3112. )
  3113. # MQTT relay - publish archive created
  3114. try:
  3115. await mqtt_relay.on_archive_created(
  3116. archive_id=archive.id,
  3117. print_name=archive.print_name,
  3118. printer_name=printer.name,
  3119. status=archive.status,
  3120. )
  3121. except Exception:
  3122. pass # Don't fail if MQTT fails
  3123. # Send notification with archive data (new archive created)
  3124. if not notification_sent:
  3125. archive_data = {
  3126. "print_time_seconds": archive.print_time_seconds,
  3127. "created_by_id": archive.created_by_id,
  3128. }
  3129. await _send_print_start_notification(printer_id, data, archive_data, logger)
  3130. # Extract printable objects for skip object functionality
  3131. try:
  3132. from backend.app.services.archive import extract_printable_objects_from_3mf
  3133. client = printer_manager.get_client(printer_id)
  3134. if client:
  3135. with open(temp_path, "rb") as f:
  3136. threemf_data = f.read()
  3137. # Extract with positions for UI overlay, scoped to the
  3138. # plate that is printing — an all-plates 3MF carries
  3139. # every plate's objects (#2522).
  3140. printable_objects, bbox_all = extract_printable_objects_from_3mf(
  3141. threemf_data,
  3142. plate_number=resolve_plate_id(client.state),
  3143. include_positions=True,
  3144. )
  3145. if printable_objects:
  3146. # Store objects in printer state
  3147. client.state.printable_objects = printable_objects
  3148. client.state.printable_objects_bbox_all = bbox_all
  3149. client.state.skipped_objects = [] # Reset skipped objects for new print
  3150. logger.info(
  3151. "Loaded %s printable objects for printer %s", len(printable_objects), printer_id
  3152. )
  3153. except Exception as e:
  3154. logger.debug("Failed to extract printable objects: %s", e)
  3155. # Store Spoolman tracking data for per-filament usage reporting
  3156. try:
  3157. await _store_spoolman_print_data(
  3158. printer_id,
  3159. archive.id,
  3160. archive.file_path,
  3161. db,
  3162. printer_manager,
  3163. ams_mapping=_get_start_ams_mapping(data, archive.id),
  3164. plate_id=_get_start_plate_id(archive.id),
  3165. )
  3166. except Exception as e:
  3167. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  3168. # Capture timelapse file baseline for snapshot-diff on completion
  3169. await _capture_timelapse_baseline_at_start(printer, printer_id, logger, archive_id=archive.id)
  3170. finally:
  3171. # Keep temp_path around until print completes so the cover endpoint
  3172. # can reuse it (#972). Cache eviction in on_print_complete deletes
  3173. # the file. If the cache entry was evicted early (file vanished),
  3174. # clean up any stragglers here to avoid leaking disk on retries.
  3175. cached_now = get_cached_3mf(printer_id, downloaded_filename) if downloaded_filename else None
  3176. if temp_path and temp_path.exists() and cached_now != temp_path:
  3177. temp_path.unlink()
  3178. _TIMELAPSE_VIDEO_EXTENSIONS = (".mp4", ".avi")
  3179. # Poll schedule for the post-print timelapse scan (#2704). Module-level so
  3180. # tests can shrink them without waiting out real delays.
  3181. #
  3182. # This replaced a fixed [5, 10, 20, 30] retry ladder, i.e. roughly 65 s of
  3183. # looking. Across 247 support bundles the attempt that found the video was #1
  3184. # 272 times, then 17 / 13 / 13 — a flat tail against the cutoff rather than a
  3185. # decaying one, which is the signature of a budget that expires while files are
  3186. # still arriving. 457 scans were scheduled and only 262 ever attached. Big
  3187. # prints make big videos and the printer writes them after the print ends, so
  3188. # the poll now runs for minutes and costs one FTP LIST per round.
  3189. _TIMELAPSE_SCAN_FIRST_DELAY_SECONDS: float = 5.0
  3190. _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS: float = 30.0
  3191. _TIMELAPSE_SCAN_TIMEOUT_SECONDS: float = 900.0
  3192. def _timelapse_scan_max_attempts() -> int:
  3193. """Round cap for the poll, derived from the wall-clock budget.
  3194. The deadline alone is not a sufficient bound: it assumes each round really
  3195. waits, which stops being true the moment ``asyncio.sleep`` is patched out,
  3196. and an FTP list that fails immediately would otherwise spin against the
  3197. printer at full speed for the whole window. Whichever bound is reached
  3198. first ends the poll.
  3199. """
  3200. if _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS <= 0:
  3201. # A zero interval makes the wall-clock budget meaningless; fall back to
  3202. # the round count the production interval would have given.
  3203. return 32
  3204. return max(1, int(_TIMELAPSE_SCAN_TIMEOUT_SECONDS // _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS) + 1)
  3205. async def _claimed_timelapse_names(db, printer_id: int, exclude_archive_id: int) -> set[str]:
  3206. """Video filenames already attached to some other archive of this printer.
  3207. Used to disambiguate when more than one file is new since the baseline —
  3208. which happens when a previous print's video landed after this print's
  3209. baseline was taken. Ordering the candidates would be the obvious fix and is
  3210. the wrong one: it can only be done on mtime or on the filename timestamp,
  3211. both of which come from the printer's own clock, and a LAN-only printer
  3212. can't reach Bambu's NTP server. Exclusion needs no clock at all.
  3213. ``attach_timelapse`` saves the video into the archive directory under the
  3214. printer's original filename, and the later MP4 conversion keeps the stem,
  3215. so the stem of ``timelapse_path`` recovers what was claimed.
  3216. """
  3217. from backend.app.models.archive import PrintArchive
  3218. rows = await db.execute(
  3219. select(PrintArchive.timelapse_path).where(
  3220. PrintArchive.printer_id == printer_id,
  3221. PrintArchive.id != exclude_archive_id,
  3222. PrintArchive.timelapse_path.is_not(None),
  3223. )
  3224. )
  3225. return {Path(p).stem for p in rows.scalars().all() if p}
  3226. async def _list_timelapse_videos(printer) -> tuple[list[dict], str | None]:
  3227. """List video files from printer's timelapse directory.
  3228. Finds MP4 (X1/A1 series) and AVI (P1 series) timelapse files.
  3229. Returns (video_files, found_path) where video_files is a list of file dicts
  3230. and found_path is the directory where they were found, or ([], None).
  3231. """
  3232. from backend.app.services.bambu_ftp import list_files_async
  3233. logger = logging.getLogger(__name__)
  3234. for timelapse_path in ["/timelapse", "/timelapse/video", "/record", "/recording"]:
  3235. try:
  3236. found_files = await list_files_async(
  3237. printer.ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  3238. )
  3239. if found_files:
  3240. video_files = [
  3241. f
  3242. for f in found_files
  3243. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  3244. ]
  3245. if video_files:
  3246. return video_files, timelapse_path
  3247. except Exception as e:
  3248. logger.debug("[TIMELAPSE] Path %s failed: %s", timelapse_path, e)
  3249. continue
  3250. return [], None
  3251. async def _capture_timelapse_baseline_at_start(
  3252. printer, printer_id: int, logger: logging.Logger, archive_id: int | None = None
  3253. ) -> None:
  3254. """Snapshot the printer's timelapse directory at print start so the
  3255. completion-time scan can pick the new file by set-difference.
  3256. Must be called from every on_print_start path that proceeds to a real
  3257. print — both the new-archive branch and the expected-archive branch (which
  3258. queue / VP-dispatched prints take). Without a baseline,
  3259. _scan_for_timelapse_with_retries falls into its "take baseline now"
  3260. fallback that runs AFTER the new MP4 has already landed on the SD card,
  3261. so the new file ends up in the "baseline" set and no diff ever matches.
  3262. Bambu printers in LAN-only mode don't sync NTP, so mtime ordering is
  3263. unreliable — the snapshot-diff approach sidesteps that entirely.
  3264. When ``archive_id`` is known the baseline is also written to the archive
  3265. row, so it survives a restart and the manual "Scan for Timelapse" button
  3266. can run the same diff instead of falling back to clock-based matching
  3267. (#2704). Only baselines taken at print start are persisted — one taken at
  3268. completion already contains the new video and would poison a later scan.
  3269. """
  3270. names: set[str] | None = None
  3271. try:
  3272. baseline_files, _ = await _list_timelapse_videos(printer)
  3273. names = {f.get("name", "") for f in baseline_files}
  3274. _timelapse_baselines[printer_id] = names
  3275. logger.info(
  3276. "[TIMELAPSE] Baseline at print start: %s video files for printer %s",
  3277. len(names),
  3278. printer_id,
  3279. )
  3280. except Exception as e:
  3281. logger.warning("[TIMELAPSE] Failed to capture baseline at print start: %s", e)
  3282. if archive_id is None:
  3283. return
  3284. try:
  3285. async with async_session() as db:
  3286. from backend.app.models.archive import PrintArchive
  3287. archive = await db.get(PrintArchive, archive_id)
  3288. if archive is not None:
  3289. # Written even when the listing failed, and then as NULL. A
  3290. # reprint reuses the archive row, so leaving the previous run's
  3291. # baseline in place would have the scan diff this print against
  3292. # the state of the printer before the *last* one — and a stale
  3293. # baseline reads as authoritative, where NULL correctly falls
  3294. # back to a fresh snapshot.
  3295. archive.timelapse_baseline = sorted(names) if names is not None else None
  3296. await db.commit()
  3297. except Exception as e:
  3298. # In-memory baseline still covers the normal completion path.
  3299. logger.warning("[TIMELAPSE] Failed to persist baseline for archive %s: %s", archive_id, e)
  3300. async def _scan_for_timelapse_with_retries(archive_id: int, baseline_names: set[str] | None = None):
  3301. """Poll the printer for this print's timelapse and attach it.
  3302. Snapshot diff, not timestamp matching: a printer in LAN-only mode cannot
  3303. reach Bambu's NTP server, so the clock behind both the filename and the FTP
  3304. mtime is arbitrarily wrong — one reporter's P1S was six and a half days out
  3305. (#2704). Comparing the current listing against the set of filenames that
  3306. existed when the print started needs no clock at all, because the printer
  3307. writes the video only once the print has ended.
  3308. Baseline precedence: the caller's in-memory set, then the one persisted on
  3309. the archive at print start, then a snapshot taken now. The last of those is
  3310. a poor substitute — by completion the new video may already be on the card,
  3311. in which case it lands in the "baseline" and no diff can ever match — but it
  3312. is all that is available for a print that began before Bambuddy started.
  3313. On success the video is deleted from the printer, which keeps ``/timelapse``
  3314. down to the unclaimed files and makes the next diff unambiguous.
  3315. """
  3316. logger = logging.getLogger(__name__)
  3317. # --- Phase 1: establish the baseline -------------------------------------
  3318. try:
  3319. async with async_session() as db:
  3320. from backend.app.models.printer import Printer
  3321. service = ArchiveService(db)
  3322. archive = await service.get_archive(archive_id)
  3323. if not archive:
  3324. logger.warning("[TIMELAPSE] Archive %s not found, aborting", archive_id)
  3325. return
  3326. if archive.timelapse_path:
  3327. logger.info("[TIMELAPSE] Archive %s already has timelapse attached", archive_id)
  3328. return
  3329. if not archive.printer_id:
  3330. logger.warning("[TIMELAPSE] Archive %s has no printer, aborting", archive_id)
  3331. return
  3332. if baseline_names is not None:
  3333. logger.info(
  3334. "[TIMELAPSE] Using print-start baseline: %s existing video files for archive %s",
  3335. len(baseline_names),
  3336. archive_id,
  3337. )
  3338. elif archive.timelapse_baseline is not None:
  3339. # Persisted at print start — survives a restart mid-print.
  3340. baseline_names = set(archive.timelapse_baseline)
  3341. logger.info(
  3342. "[TIMELAPSE] Using stored baseline: %s existing video files for archive %s",
  3343. len(baseline_names),
  3344. archive_id,
  3345. )
  3346. else:
  3347. result = await db.execute(select(Printer).where(Printer.id == archive.printer_id))
  3348. printer = result.scalar_one_or_none()
  3349. if not printer:
  3350. logger.warning("[TIMELAPSE] Printer not found for archive %s, aborting", archive_id)
  3351. return
  3352. baseline_files, _ = await _list_timelapse_videos(printer)
  3353. baseline_names = {f.get("name", "") for f in baseline_files}
  3354. logger.info(
  3355. "[TIMELAPSE] Baseline snapshot (fallback): %s existing video files for archive %s",
  3356. len(baseline_names),
  3357. archive_id,
  3358. )
  3359. except Exception as e:
  3360. logger.warning("[TIMELAPSE] Failed to take baseline snapshot for archive %s: %s", archive_id, e)
  3361. return
  3362. # --- Phase 2: poll for a file that was not there when the print began -----
  3363. deadline = time.monotonic() + _TIMELAPSE_SCAN_TIMEOUT_SECONDS
  3364. max_attempts = _timelapse_scan_max_attempts()
  3365. seen_names: set[str] = set()
  3366. delay = _TIMELAPSE_SCAN_FIRST_DELAY_SECONDS
  3367. attempt = 0
  3368. while True:
  3369. await asyncio.sleep(delay)
  3370. delay = _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS
  3371. attempt += 1
  3372. try:
  3373. from backend.app.models.printer import Printer
  3374. # Read phase: fetch archive + printer in a short session and release
  3375. # the pooled connection BEFORE the FTP list/download below. Holding it
  3376. # across the FTP round-trips left one connection idle-in-transaction per
  3377. # in-flight scan (issue #2572).
  3378. async with async_session() as db:
  3379. service = ArchiveService(db)
  3380. archive = await service.get_archive(archive_id)
  3381. if not archive:
  3382. logger.warning("[TIMELAPSE] Archive %s not found, stopping poll", archive_id)
  3383. return
  3384. if archive.timelapse_path:
  3385. logger.info("[TIMELAPSE] Archive %s already has timelapse attached, stopping poll", archive_id)
  3386. return
  3387. result = await db.execute(select(Printer).where(Printer.id == archive.printer_id))
  3388. printer = result.scalar_one_or_none()
  3389. if not printer:
  3390. logger.warning("[TIMELAPSE] Printer not found for archive %s, stopping poll", archive_id)
  3391. return
  3392. claimed = await _claimed_timelapse_names(db, archive.printer_id, archive_id)
  3393. # I/O phase (no DB connection held): FTP list + download.
  3394. video_files, found_path = await _list_timelapse_videos(printer)
  3395. # The poll can run for dozens of rounds, so only narrate a round
  3396. # that saw something change. Repeating the whole listing every 30 s
  3397. # would bury the one interesting line in the support bundle.
  3398. names_now = {f.get("name", "") for f in video_files}
  3399. changed = attempt == 1 or names_now != seen_names
  3400. seen_names = names_now
  3401. speak = logger.info if changed else logger.debug
  3402. if video_files:
  3403. speak("[TIMELAPSE] Attempt %s: Found %s video files in %s", attempt, len(video_files), found_path)
  3404. if changed:
  3405. for f in video_files[:5]:
  3406. logger.info("[TIMELAPSE] - %s", f.get("name"))
  3407. attached = await _attach_first_unclaimed_timelapse(
  3408. archive_id, printer, video_files, baseline_names, claimed, attempt, logger, quiet=not changed
  3409. )
  3410. if attached:
  3411. return
  3412. else:
  3413. speak("[TIMELAPSE] Attempt %s: No video files found, will retry", attempt)
  3414. except Exception as e:
  3415. logger.warning("[TIMELAPSE] Attempt %s failed with error: %s", attempt, e)
  3416. if attempt >= max_attempts or time.monotonic() >= deadline:
  3417. break
  3418. # No name-match fallback: it compared the print name against the filename,
  3419. # and Bambu firmware only ever writes "video_<timestamp>". Across 247 support
  3420. # bundles it fired 159 times and matched zero times, so all it added was a
  3421. # misleading log line before giving up.
  3422. logger.warning(
  3423. "[TIMELAPSE] No new video appeared for archive %s within %ss, giving up",
  3424. archive_id,
  3425. int(_TIMELAPSE_SCAN_TIMEOUT_SECONDS),
  3426. )
  3427. async def _attach_first_unclaimed_timelapse(
  3428. archive_id: int,
  3429. printer,
  3430. video_files: list[dict],
  3431. baseline_names: set[str],
  3432. claimed: set[str],
  3433. attempt: int,
  3434. logger: logging.Logger,
  3435. *,
  3436. quiet: bool = False,
  3437. ) -> bool:
  3438. """Download and attach the one video that belongs to this print.
  3439. A candidate is any file absent from the print-start baseline. More than one
  3440. can qualify when a previous print's video landed late, after this print's
  3441. baseline was taken — those are filtered out by name, because they are
  3442. already attached to another archive. Sorting the candidates instead would
  3443. mean sorting on mtime or on the filename timestamp, both of which come from
  3444. the printer's unsynced clock.
  3445. Returns True once a video is attached. The printer's copy is deleted only
  3446. after the attach succeeds on bytes whose length matched the listing.
  3447. ``quiet`` downgrades the "nothing yet" lines to DEBUG when the caller has
  3448. already seen this exact listing — the poll runs for many rounds and only the
  3449. rounds where something changed are worth an INFO line.
  3450. """
  3451. from backend.app.services.bambu_ftp import (
  3452. delete_archived_timelapse,
  3453. download_file_bytes_async,
  3454. remote_file_settled,
  3455. )
  3456. speak = logger.debug if quiet else logger.info
  3457. new_files = [f for f in video_files if f.get("name", "") not in baseline_names]
  3458. if not new_files:
  3459. speak("[TIMELAPSE] Attempt %s: No new files since baseline, will retry", attempt)
  3460. return False
  3461. candidates = [f for f in new_files if Path(f.get("name", "")).stem not in claimed]
  3462. if not candidates:
  3463. speak(
  3464. "[TIMELAPSE] Attempt %s: %s new file(s), all already attached to other archives, will retry",
  3465. attempt,
  3466. len(new_files),
  3467. )
  3468. return False
  3469. if len(candidates) > 1:
  3470. logger.warning(
  3471. "[TIMELAPSE] Attempt %s: %s unclaimed new files (%s) — taking the first; "
  3472. "the rest stay on the printer for manual selection",
  3473. attempt,
  3474. len(candidates),
  3475. ", ".join(str(f.get("name")) for f in candidates),
  3476. )
  3477. target = candidates[0]
  3478. file_name = target.get("name")
  3479. remote_path = target.get("path") or f"/timelapse/{file_name}"
  3480. logger.info(
  3481. "[TIMELAPSE] Attempt %s: New file detected: %s (downloading for archive %s)",
  3482. attempt,
  3483. file_name,
  3484. archive_id,
  3485. )
  3486. # The listing always carries a size (`list_files` skips entries it can't
  3487. # parse), but read it explicitly: the delete below is destructive and must
  3488. # depend on a size we actually had, not on one we hoped was there.
  3489. expected_size = target.get("size")
  3490. timelapse_data = await download_file_bytes_async(
  3491. printer.ip_address,
  3492. printer.access_code,
  3493. remote_path,
  3494. printer_model=printer.model,
  3495. expected_size=expected_size,
  3496. )
  3497. if not timelapse_data:
  3498. # Short or failed transfer. The printer keeps its copy, so the next
  3499. # round can try again — which is exactly why the delete below is
  3500. # gated on a verified download.
  3501. logger.warning("[TIMELAPSE] Attempt %s: Failed to download new file, will retry", attempt)
  3502. return False
  3503. # The length check above proves we got what the listing said, not that the
  3504. # printer had finished writing. A video still being written can be listed
  3505. # short, served short, and pass — so confirm it has stopped growing before
  3506. # committing to it and deleting the original (#2704).
  3507. if not await remote_file_settled(
  3508. printer.ip_address,
  3509. printer.access_code,
  3510. remote_path,
  3511. len(timelapse_data),
  3512. printer_model=printer.model,
  3513. ):
  3514. return False
  3515. # Write phase: attach in a fresh short-lived session.
  3516. async with async_session() as db:
  3517. success = await ArchiveService(db).attach_timelapse(archive_id, timelapse_data, file_name)
  3518. if not success:
  3519. logger.warning("[TIMELAPSE] Failed to attach timelapse to archive %s", archive_id)
  3520. return False
  3521. logger.info("[TIMELAPSE] Successfully attached timelapse to archive %s", archive_id)
  3522. await ws_manager.send_archive_updated({"id": archive_id, "timelapse_attached": True})
  3523. await delete_archived_timelapse(
  3524. printer.ip_address,
  3525. printer.access_code,
  3526. remote_path,
  3527. verified=expected_size is not None,
  3528. printer_model=printer.model,
  3529. printer_name=printer.name,
  3530. )
  3531. return True
  3532. # Defaults for the finish-photo-from-timelapse polling loop (#1397). These are
  3533. # module-level so tests can monkeypatch them down to ~0 without timing out.
  3534. _FINISH_PHOTO_TIMELAPSE_POLL_INTERVAL_SECONDS: float = 3.0
  3535. _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS: float = 60.0
  3536. # How long the *background* upgrade keeps waiting after the notification has
  3537. # already gone out (#2704 follow-up). The short bound above exists so a slow
  3538. # printer can't hold up the print-complete notification; this one exists so the
  3539. # archive still ends up with the better frame afterwards.
  3540. #
  3541. # Measured across 261 attaches in the support bundles, the video lands a median
  3542. # 13s after the print ends — but the P1 series writes MJPEG AVI rather than
  3543. # H.264 MP4 and serves it slowly, so its p90 is 167s and the worst observed case
  3544. # was 546s. Every other model was inside 26s. The long budget is therefore
  3545. # almost entirely for P1-series users; on everything else the short wait already
  3546. # wins and this task never runs.
  3547. _FINISH_PHOTO_UPGRADE_TIMEOUT_SECONDS: float = 900.0
  3548. async def _capture_finish_photo_from_timelapse(
  3549. archive_id: int,
  3550. archive_dir: Path,
  3551. timeout: float | None = None,
  3552. rotation: int = 0,
  3553. ) -> tuple[str | None, bool]:
  3554. """Wait for the per-print timelapse to land on the archive and extract its
  3555. last frame as the finish photo (#1397).
  3556. Bambu firmware stops timelapse recording after the toolhead parks but
  3557. before the bed-drop end-gcode runs, so the last frame frames the finished
  3558. print correctly. A live camera grab at gcode_state=FINISH captures the
  3559. bed already lowered.
  3560. ``_scan_for_timelapse_with_retries`` runs in parallel and writes
  3561. ``archive.timelapse_path`` when the file lands. This function polls for
  3562. that field.
  3563. Returns ``(filename, still_pending)``. ``still_pending`` is True only when
  3564. the wait ran out with no video on the archive yet — i.e. the video may
  3565. still be coming and a later attempt could succeed. It is False when the
  3566. video landed (whether or not extraction worked), because in that case
  3567. waiting longer changes nothing. The caller uses that to decide between
  3568. falling back permanently and scheduling a background upgrade.
  3569. ``rotation`` is the printer's camera_rotation, applied to the extracted
  3570. still (#2708) so this source agrees with every other finish-photo source.
  3571. The archived video itself is the printer's own file and is left alone —
  3572. rotating it would mean re-encoding it.
  3573. """
  3574. import uuid
  3575. from backend.app.models.archive import PrintArchive
  3576. from backend.app.services.camera import apply_camera_rotation_to_file, extract_video_last_frame
  3577. logger = logging.getLogger(__name__)
  3578. budget = _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS if timeout is None else timeout
  3579. deadline = asyncio.get_event_loop().time() + budget
  3580. poll_interval = _FINISH_PHOTO_TIMELAPSE_POLL_INTERVAL_SECONDS
  3581. while True:
  3582. async with async_session() as db:
  3583. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  3584. archive = result.scalar_one_or_none()
  3585. timelapse_relpath = archive.timelapse_path if archive else None
  3586. if timelapse_relpath:
  3587. video_path = app_settings.base_dir / timelapse_relpath
  3588. if video_path.exists() and video_path.stat().st_size > 0:
  3589. photos_dir = archive_dir / "photos"
  3590. photos_dir.mkdir(parents=True, exist_ok=True)
  3591. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  3592. filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  3593. output_path = photos_dir / filename
  3594. if await extract_video_last_frame(video_path, output_path):
  3595. await apply_camera_rotation_to_file(output_path, rotation, logger)
  3596. logger.info(
  3597. "[PHOTO-BG] Extracted finish photo from timelapse %s for archive %s",
  3598. video_path.name,
  3599. archive_id,
  3600. )
  3601. return filename, False
  3602. logger.warning(
  3603. "[PHOTO-BG] Timelapse %s landed but last-frame extraction failed for archive %s; falling back",
  3604. video_path.name,
  3605. archive_id,
  3606. )
  3607. return None, False
  3608. if asyncio.get_event_loop().time() >= deadline:
  3609. logger.info(
  3610. "[PHOTO-BG] Timelapse for archive %s didn't land within %.0fs; falling back to live camera",
  3611. archive_id,
  3612. budget,
  3613. )
  3614. return None, True
  3615. await asyncio.sleep(poll_interval)
  3616. async def _upgrade_finish_photo_from_timelapse(archive_id: int, archive_dir: Path, rotation: int = 0) -> None:
  3617. """Add the timelapse's last frame to an archive after the fact (#2704).
  3618. The print-complete notification waits only ~60s for the video, because
  3619. holding a notification for minutes is worse than sending it with a live
  3620. camera grab. On a P1-series printer the video often lands well after that,
  3621. so the archive used to be stuck with the live grab — which is taken at
  3622. ``gcode_state=FINISH``, after the end G-code has dropped the bed, and is
  3623. the worse photo of the two.
  3624. This keeps waiting in the background and, when the video arrives, extracts
  3625. the frame and puts it *first* in the archive's photo list, so opening the
  3626. gallery shows it. The live grab is deliberately kept: the notification that
  3627. already went out links to that exact file, and deleting it would leave a
  3628. broken image in Discord or Telegram.
  3629. """
  3630. logger = logging.getLogger(__name__)
  3631. filename, _ = await _capture_finish_photo_from_timelapse(
  3632. archive_id, archive_dir, timeout=_FINISH_PHOTO_UPGRADE_TIMEOUT_SECONDS, rotation=rotation
  3633. )
  3634. if not filename:
  3635. logger.info("[PHOTO-UPGRADE] No timelapse frame for archive %s; keeping the live grab", archive_id)
  3636. return
  3637. try:
  3638. async with async_session() as db:
  3639. from backend.app.models.archive import PrintArchive
  3640. archive = await db.get(PrintArchive, archive_id)
  3641. if archive is None:
  3642. return
  3643. photos = list(archive.photos or [])
  3644. if filename in photos:
  3645. return
  3646. # Front of the list: PhotoGalleryModal opens at index 0.
  3647. archive.photos = [filename, *photos]
  3648. await db.commit()
  3649. except Exception as e:
  3650. logger.warning("[PHOTO-UPGRADE] Failed to attach upgraded photo to archive %s: %s", archive_id, e)
  3651. return
  3652. logger.info("[PHOTO-UPGRADE] Archive %s now leads with the timelapse frame %s", archive_id, filename)
  3653. await ws_manager.send_archive_updated({"id": archive_id, "photo_added": filename})
  3654. async def on_print_running_observed(printer_id: int, data: dict):
  3655. """Restart-recovery: capture a fresh timelapse baseline for a print that
  3656. started before Bambuddy came up.
  3657. bambu_mqtt.py suppresses ``on_print_start`` on the first RUNNING push
  3658. after Bambuddy startup (#1304 guard, prevents duplicate archive
  3659. creation). Without that path, ``_capture_timelapse_baseline_at_start``
  3660. never runs and ``_scan_for_timelapse_with_retries`` falls into its
  3661. "take baseline now" fallback at completion time — but by then the
  3662. printer has already uploaded the in-flight MP4, so the baseline
  3663. includes it and no diff ever matches (#1485 follow-up).
  3664. Fires once per session, in lieu of on_print_start when restart-recovery
  3665. kicks in. The printer doesn't upload the timelapse until after PRINT
  3666. COMPLETE, so a baseline captured any time during the print is still
  3667. pre-upload.
  3668. """
  3669. logger = logging.getLogger(__name__)
  3670. # Avoid double-capture: on_print_start may have run earlier in this
  3671. # Bambuddy process if the print started AFTER startup and we crashed
  3672. # later in the same session. (Realistically this can't happen — the
  3673. # MQTT client object would have been recreated — but the cheap guard
  3674. # is correct regardless.)
  3675. if printer_id in _timelapse_baselines:
  3676. logger.debug(
  3677. "[TIMELAPSE] on_print_running_observed: baseline already present for printer %s, skipping",
  3678. printer_id,
  3679. )
  3680. return
  3681. async with async_session() as db:
  3682. from backend.app.models.printer import Printer
  3683. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3684. printer = result.scalar_one_or_none()
  3685. if not printer:
  3686. logger.warning(
  3687. "[TIMELAPSE] on_print_running_observed: printer %s not found in DB, skipping baseline",
  3688. printer_id,
  3689. )
  3690. return
  3691. await _capture_timelapse_baseline_at_start(printer, printer_id, logger)
  3692. def _is_active_archive_stale(archive, state) -> tuple[bool, str]:
  3693. """Return ``(is_stale, reason)`` for an archive in ``status="printing"``
  3694. against the printer's current MQTT state.
  3695. Reconciliation triggers (#1542 follow-up — recovers from missed PRINT
  3696. COMPLETE events, typically a print finishing during an MQTT disconnect
  3697. window followed by a smart-plug power cycle):
  3698. 1. Printer state is terminal (IDLE / FINISH / FAILED). The print is
  3699. provably not running anymore — only branch that should fire under
  3700. normal disconnect-then-reconnect timing.
  3701. 2. Printer has a different ``subtask_id`` than the archive. Bambu
  3702. firmware mints a fresh ``subtask_id`` for each print, including the
  3703. ghost replay it runs after a power cycle from a leftover SD file —
  3704. so a mismatch unambiguously means the in-DB archive is no longer
  3705. the print on the printer.
  3706. 3. Printer is running but ``subtask_name`` is empty. The printer
  3707. doesn't know what it's running; the archive's reference to it is
  3708. already broken.
  3709. Conservative on purpose: PAUSE / PREPARE / SLICING and any RUNNING state
  3710. with matching subtask_id+subtask_name is left alone. The cost of a false
  3711. positive is a duplicate archive on the next real PRINT COMPLETE — the
  3712. reactive handler uses ``_active_prints`` for lookup, which the reconcile
  3713. clears on synthesis, so the real completion creates a fresh row instead
  3714. of overwriting the synthesised one (#1679). The cost of a false negative
  3715. is the ghost-print loop in #1542.
  3716. Pre-push guard (#1679): when ``state.state`` is empty or ``"unknown"``,
  3717. MQTT has connected but the first ``push_status`` response hasn't been
  3718. applied yet — ``PrinterState`` is sitting on its construction defaults.
  3719. The reconcile caller in ``on_printer_status_change`` is already gated
  3720. on a real ``state.state``, so in normal operation this branch is
  3721. unreachable; it's kept as belt-and-braces for future callers and for
  3722. the narrow window where a partial state update could arrive
  3723. (``state.state`` set but ``subtask_name`` not yet populated). Returning
  3724. ``not stale`` on degenerate input is strictly conservative: a real
  3725. stale archive will still be caught by the next push_status arriving
  3726. with terminal state.
  3727. """
  3728. current_state = (state.state or "").upper()
  3729. if current_state in ("", "UNKNOWN"):
  3730. # No real push_status yet — PrinterState defaults are not evidence.
  3731. return False, ""
  3732. if current_state in ("IDLE", "FINISH", "FAILED"):
  3733. return True, f"printer state {current_state}"
  3734. # Below here the printer is in a running / pre-running state (RUNNING /
  3735. # PAUSE / PREPARE / SLICING / etc.) — decide based on subtask identity.
  3736. current_subtask_id = (state.subtask_id or "").strip()
  3737. if archive.subtask_id and current_subtask_id and archive.subtask_id != current_subtask_id:
  3738. return True, f"subtask_id changed ({archive.subtask_id!r} → {current_subtask_id!r})"
  3739. current_subtask_name = (state.subtask_name or "").strip()
  3740. if not current_subtask_name:
  3741. return True, "printer subtask_name empty"
  3742. return False, ""
  3743. async def reconcile_stale_active_prints(printer_id: int) -> int:
  3744. """Synthesise ``on_print_complete`` for archives whose print can't be
  3745. running on the printer anymore.
  3746. Called once per MQTT (re)connection (from on_printer_status_change when
  3747. the connected edge flips False → True) and at Bambuddy startup (from
  3748. the FastAPI lifespan). Without this, a print that completes during a
  3749. disconnect window — followed by a smart-plug-driven power cycle — leaves
  3750. the ``.3mf`` on the SD card, the firmware auto-replays it on next boot,
  3751. and Bambuddy fires a fresh PRINT START for the ghost rather than the
  3752. SD cleanup that PRINT COMPLETE was supposed to run. Repeats every
  3753. power cycle until the operator notices (#1542 follow-up). Reconciliation
  3754. closes the loop by faking the missed PRINT COMPLETE — the existing
  3755. cleanup chain handles SD-file deletion, status updates, usage tracking,
  3756. and notifications.
  3757. Synthesised ``status="aborted"`` is the conservative label: we have no
  3758. proof the print finished successfully (and no progress evidence to
  3759. promote to ``"completed"``). The real PRINT COMPLETE callback, if it
  3760. fires later, overwrites the status with the correct value.
  3761. Returns the number of archives reconciled.
  3762. """
  3763. state = printer_manager.get_status(printer_id)
  3764. if not state:
  3765. return 0
  3766. # Don't reconcile while disconnected — we'd be making a decision against
  3767. # stale cached state. The connected → reconcile edge handles this.
  3768. if not state.connected:
  3769. return 0
  3770. from backend.app.models.archive import PrintArchive
  3771. reconciled = 0
  3772. async with async_session() as db:
  3773. result = await db.execute(
  3774. select(PrintArchive).where(
  3775. PrintArchive.printer_id == printer_id,
  3776. PrintArchive.status == "printing",
  3777. )
  3778. )
  3779. active = list(result.scalars().all())
  3780. if not active:
  3781. return 0
  3782. logger = logging.getLogger(__name__)
  3783. for archive in active:
  3784. is_stale, reason = _is_active_archive_stale(archive, state)
  3785. if not is_stale:
  3786. continue
  3787. logger.info(
  3788. "[RECONCILE] Printer %s: synthesising missed PRINT COMPLETE for archive %s (%s) — %s",
  3789. printer_id,
  3790. archive.id,
  3791. archive.filename,
  3792. reason,
  3793. )
  3794. # Synthesised payload: minimal fields the on_print_complete chain
  3795. # needs. `_reconciled` marker lets downstream code distinguish this
  3796. # from a real MQTT-driven completion if it ever needs to (e.g. for
  3797. # metrics / debug logging). raw_data is the live printer state so
  3798. # the usage tracker can compare end-of-print remain% against the
  3799. # captured start values.
  3800. try:
  3801. await on_print_complete(
  3802. printer_id,
  3803. {
  3804. "status": "aborted",
  3805. "filename": archive.filename,
  3806. "subtask_name": archive.print_name or "",
  3807. "subtask_id": archive.subtask_id or "",
  3808. "raw_data": state.raw_data or {},
  3809. "_reconciled": True,
  3810. },
  3811. )
  3812. reconciled += 1
  3813. except Exception as e:
  3814. # Catch-all: a reconciliation failure must not block the
  3815. # printer's normal status flow. The archive stays in
  3816. # ``status="printing"`` and the next reconnect retries.
  3817. logger.warning(
  3818. "[RECONCILE] on_print_complete synthesis failed for archive %s: %s",
  3819. archive.id,
  3820. e,
  3821. )
  3822. return reconciled
  3823. async def on_finish_photo_moment(printer_id: int, data: dict):
  3824. """Pre-capture a finish photo when the printer enters stage 22 / FINISH (#1721).
  3825. Fires either at the stage-22 ("Filament unloading") edge — toolhead
  3826. parked, bed not yet dropped, optimal framing — or as a FINISH-state
  3827. fallback for prints that skip stage 22 (cancel, external-spool-only,
  3828. HMS halt, firmware variants). Grabs one frame via the same
  3829. external-camera / RTSP path the post-completion fallback uses, stores
  3830. the JPEG bytes in ``_stage22_finish_frames[printer_id]``, and lets
  3831. ``_background_finish_photo`` consume the cached bytes when it runs.
  3832. Replaces the #1397 "force timelapse on at dispatch" mechanism, which
  3833. caused per-layer nozzle parking on slicer profiles with Timelapse Type
  3834. set to Smooth (#1721). No force-on now means the user's explicit
  3835. timelapse=off in the slicer send dialog is respected.
  3836. """
  3837. logger = logging.getLogger(__name__)
  3838. trigger = data.get("trigger", "unknown")
  3839. timelapse_was_active = bool(data.get("timelapse_was_active"))
  3840. logger.info(
  3841. "[FINISH-PHOTO-MOMENT] printer=%s trigger=%s timelapse_active=%s",
  3842. printer_id,
  3843. trigger,
  3844. timelapse_was_active,
  3845. )
  3846. # If a timelapse is actively recording, skip the pre-capture — the
  3847. # post-completion path will extract the last frame from the recorded
  3848. # video, which still provides the best framing (toolhead parked,
  3849. # before bed drop) without the per-layer parking side effects.
  3850. if timelapse_was_active:
  3851. logger.info(
  3852. "[FINISH-PHOTO-MOMENT] timelapse active for printer %s — skipping pre-capture (last-frame extraction will run post-completion)",
  3853. printer_id,
  3854. )
  3855. return
  3856. # #1790: register the producer-done event BEFORE the first await so the
  3857. # consumer in `_background_finish_photo` — which is dispatched back-to-back
  3858. # with us on the FINISH-state fallback path — sees it as soon as it polls.
  3859. # The `finally` below guarantees `set()` runs on every exit, including
  3860. # early returns and exceptions, so the consumer's bounded wait can't hang.
  3861. producer_done = asyncio.Event()
  3862. _stage22_finish_in_flight[printer_id] = producer_done
  3863. try:
  3864. async with async_session() as db:
  3865. from backend.app.api.routes.settings import get_setting
  3866. from backend.app.models.printer import Printer
  3867. capture_setting = await get_setting(db, "capture_finish_photo")
  3868. if capture_setting is not None and capture_setting.lower() != "true":
  3869. logger.info("[FINISH-PHOTO-MOMENT] capture_finish_photo disabled — skipping pre-capture")
  3870. return
  3871. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3872. printer = result.scalar_one_or_none()
  3873. if printer is None:
  3874. logger.warning(
  3875. "[FINISH-PHOTO-MOMENT] printer %s not found in DB",
  3876. printer_id,
  3877. )
  3878. return
  3879. frame_bytes: bytes | None = None
  3880. # #2708: the banked frame arrives already rotated — it comes from
  3881. # `_capture_snapshot_for_notification`, which rotates before returning.
  3882. # Every other source below is a raw grab. Tracking which lets us store
  3883. # exactly one rotation in `_stage22_finish_frames` either way.
  3884. frame_already_rotated = False
  3885. # #1867: on the FINISH-state fallback the End G-code (e.g. SwapMod
  3886. # plate-swap) has already run, so a live grab now captures the swapped
  3887. # or empty plate. Prefer the banked in-print frame — the finished
  3888. # print from the last object layer, before the swap. Only for
  3889. # `finish_state`: the `stage_22` and `last_layer` triggers fire before
  3890. # the swap and give cleaner (parked-toolhead) framing via a live grab.
  3891. if trigger == "finish_state":
  3892. banked = _inprint_frame_bank.get(printer_id)
  3893. if banked:
  3894. frame_bytes = banked
  3895. frame_already_rotated = True
  3896. logger.info(
  3897. "[FINISH-PHOTO-MOMENT] using banked in-print frame (%d bytes) — "
  3898. "avoids post-swap live grab on stage-22-less firmware",
  3899. len(banked),
  3900. )
  3901. if frame_bytes is None and printer.external_camera_enabled and printer.external_camera_url:
  3902. from backend.app.api.routes.camera import live_frame_for_capture
  3903. from backend.app.services.external_camera import capture_frame
  3904. # #2707: this used to collide with the live view and fail, which is
  3905. # how finish-photo notifications went out with no image attached.
  3906. # Leaving frame_bytes None keeps the rest of the fallback chain.
  3907. defer, buffered = live_frame_for_capture(printer_id)
  3908. if defer:
  3909. frame_bytes = buffered
  3910. else:
  3911. frame_bytes = await capture_frame(
  3912. printer.external_camera_url,
  3913. printer.external_camera_type or "mjpeg",
  3914. snapshot_url=printer.external_camera_snapshot_url,
  3915. )
  3916. if frame_bytes:
  3917. logger.info(
  3918. "[FINISH-PHOTO-MOMENT] captured external-camera frame (%d bytes)",
  3919. len(frame_bytes),
  3920. )
  3921. elif frame_bytes is None:
  3922. from backend.app.api.routes.camera import get_buffered_frame
  3923. buffered = get_buffered_frame(printer_id)
  3924. if buffered:
  3925. frame_bytes = buffered
  3926. logger.info(
  3927. "[FINISH-PHOTO-MOMENT] used buffered RTSP frame (%d bytes)",
  3928. len(frame_bytes),
  3929. )
  3930. else:
  3931. from backend.app.services.camera import capture_camera_frame_bytes
  3932. frame_bytes = await capture_camera_frame_bytes(
  3933. ip_address=printer.ip_address,
  3934. access_code=printer.access_code,
  3935. model=printer.model,
  3936. timeout=15,
  3937. )
  3938. if frame_bytes:
  3939. logger.info(
  3940. "[FINISH-PHOTO-MOMENT] captured RTSP frame (%d bytes)",
  3941. len(frame_bytes),
  3942. )
  3943. if frame_bytes:
  3944. if not frame_already_rotated:
  3945. frame_bytes = _apply_camera_rotation(frame_bytes, printer, logger)
  3946. _stage22_finish_frames[printer_id] = frame_bytes
  3947. else:
  3948. logger.warning(
  3949. "[FINISH-PHOTO-MOMENT] no frame captured for printer %s — post-completion fallback will retry",
  3950. printer_id,
  3951. )
  3952. except Exception as e:
  3953. logger.warning(
  3954. "[FINISH-PHOTO-MOMENT] pre-capture failed for printer %s: %s",
  3955. printer_id,
  3956. e,
  3957. )
  3958. finally:
  3959. # #1790: always unblock the consumer's bounded wait — whether we stored
  3960. # a frame, gave up, or hit an exception. Local ref means cleanup of the
  3961. # dict entry by the consumer doesn't affect signalling.
  3962. producer_done.set()
  3963. async def on_print_complete(printer_id: int, data: dict):
  3964. """Handle print completion - update the archive status."""
  3965. import time
  3966. logger = logging.getLogger(__name__)
  3967. start_time = time.time()
  3968. def log_timing(section: str):
  3969. elapsed = time.time() - start_time
  3970. logger.info("[TIMING] %s: %.3fs elapsed", section, elapsed)
  3971. logger.info("[CALLBACK] on_print_complete started for printer %s", printer_id)
  3972. # Drop the 3MF download cache for this printer (#972). The print is over,
  3973. # nothing else legitimately needs the bytes; keeping them would only risk
  3974. # handing a stale file to the next print if it reuses the same name.
  3975. clear_3mf_cache(printer_id)
  3976. try:
  3977. ws_data = {
  3978. "status": data.get("status"),
  3979. "filename": data.get("filename"),
  3980. "subtask_name": data.get("subtask_name"),
  3981. "timelapse_was_active": data.get("timelapse_was_active"),
  3982. }
  3983. await ws_manager.send_print_complete(printer_id, ws_data)
  3984. log_timing("WebSocket send_print_complete")
  3985. except Exception as e:
  3986. logger.warning("[CALLBACK] WebSocket send_print_complete failed: %s", e)
  3987. # Capture user info before clearing (needed for print log entry)
  3988. _print_user_info = printer_manager.get_current_print_user(printer_id)
  3989. # Clear current print user tracking (Issue #206)
  3990. printer_manager.clear_current_print_user(printer_id)
  3991. # If the user explicitly stopped this print from the queue UI the printer will
  3992. # report "failed" or "aborted" via MQTT. Override that to "cancelled" so the
  3993. # correct "print stopped" notification/email is sent instead of a failure alert.
  3994. _raw_status = data.get("status", "completed")
  3995. if printer_id in _user_stopped_printers and _raw_status in ("failed", "aborted"):
  3996. logger.info(
  3997. "[CALLBACK] Overriding status '%s' -> 'cancelled' for printer %s (print was stopped from queue by user)",
  3998. _raw_status,
  3999. printer_id,
  4000. )
  4001. data = {**data, "status": "cancelled"}
  4002. _user_stopped_printers.discard(printer_id)
  4003. # Raise the plate-clear gate for queued dispatch (#961). Any terminal status
  4004. # may have left material on the bed: a user can cancel ten hours into a
  4005. # twelve-hour print, a printer can self-abort mid-job after a clog, and a
  4006. # touchscreen-stop reports `aborted` rather than `cancelled` because
  4007. # `_user_stopped_printers` is only populated when the user stops via the
  4008. # Bambuddy queue UI. Earlier code raised the flag only for completed/failed,
  4009. # which auto-dispatched the next queued print onto a fouled bed two seconds
  4010. # after a touchscreen-abort (#1171). Persisted to DB so the gate survives
  4011. # Auto Off power cycles and Bambuddy restarts.
  4012. _final_status = data.get("status", "completed")
  4013. if _final_status in ("completed", "failed", "aborted", "cancelled"):
  4014. printer_manager.set_awaiting_plate_clear(printer_id, True)
  4015. # MQTT relay - publish print complete
  4016. try:
  4017. printer_info = printer_manager.get_printer(printer_id)
  4018. if printer_info:
  4019. await mqtt_relay.on_print_complete(
  4020. printer_id,
  4021. printer_info.name,
  4022. printer_info.serial_number,
  4023. data.get("filename", ""),
  4024. data.get("subtask_name", ""),
  4025. data.get("status", "completed"),
  4026. )
  4027. except Exception:
  4028. pass # Don't fail print complete callback if MQTT fails
  4029. filename = data.get("filename", "")
  4030. subtask_name = data.get("subtask_name", "")
  4031. if not filename and not subtask_name:
  4032. logger.warning("Print complete without filename or subtask_name")
  4033. return
  4034. logger.info("Print complete - filename: %s, subtask: %s, status: %s", filename, subtask_name, data.get("status"))
  4035. # Build list of possible keys to try (matching how they were registered in on_print_start)
  4036. possible_keys = []
  4037. # Try subtask_name variations first (most reliable for matching)
  4038. if subtask_name:
  4039. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  4040. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  4041. possible_keys.append((printer_id, subtask_name))
  4042. # Try filename variations
  4043. if filename:
  4044. # Extract just the filename if it's a path
  4045. fname = filename.split("/")[-1] if "/" in filename else filename
  4046. if fname.endswith(".3mf"):
  4047. possible_keys.append((printer_id, fname))
  4048. elif fname.endswith(".gcode"):
  4049. base_name = fname.rsplit(".", 1)[0]
  4050. possible_keys.append((printer_id, f"{base_name}.gcode.3mf"))
  4051. possible_keys.append((printer_id, f"{base_name}.3mf"))
  4052. possible_keys.append((printer_id, fname))
  4053. else:
  4054. possible_keys.append((printer_id, f"{fname}.gcode.3mf"))
  4055. possible_keys.append((printer_id, f"{fname}.3mf"))
  4056. possible_keys.append((printer_id, fname))
  4057. # Also try full path versions
  4058. if filename.endswith(".3mf"):
  4059. possible_keys.append((printer_id, filename))
  4060. elif filename.endswith(".gcode"):
  4061. base_name = filename.rsplit(".", 1)[0]
  4062. possible_keys.append((printer_id, f"{base_name}.3mf"))
  4063. possible_keys.append((printer_id, filename))
  4064. else:
  4065. possible_keys.append((printer_id, f"{filename}.3mf"))
  4066. possible_keys.append((printer_id, filename))
  4067. # Find the archive for this print
  4068. logger.info("Looking for archive in _active_prints, keys to try: %s...", possible_keys[:5])
  4069. logger.info("Current _active_prints: %s", list(_active_prints.keys()))
  4070. archive_id = None
  4071. for key in possible_keys:
  4072. archive_id = _active_prints.pop(key, None)
  4073. if archive_id:
  4074. logger.info("Found archive %s with key %s", archive_id, key)
  4075. # Also clean up any other keys pointing to this archive
  4076. keys_to_remove = [k for k, v in _active_prints.items() if v == archive_id]
  4077. for k in keys_to_remove:
  4078. _active_prints.pop(k, None)
  4079. break
  4080. if not archive_id:
  4081. # Try to find by filename or subtask_name if not tracked (for prints started before app)
  4082. async with async_session() as db:
  4083. from backend.app.models.archive import PrintArchive
  4084. # Try matching by subtask_name (stored as print_name) first
  4085. if subtask_name:
  4086. result = await db.execute(
  4087. select(PrintArchive)
  4088. .where(PrintArchive.printer_id == printer_id)
  4089. .where(PrintArchive.status == "printing")
  4090. .where(
  4091. or_(
  4092. PrintArchive.print_name.ilike(f"%{subtask_name}%"),
  4093. PrintArchive.filename.ilike(f"%{subtask_name}%"),
  4094. )
  4095. )
  4096. .order_by(PrintArchive.created_at.desc())
  4097. .limit(1)
  4098. )
  4099. archive = result.scalar_one_or_none()
  4100. if archive:
  4101. archive_id = archive.id
  4102. logger.info("Found archive %s by subtask_name match: %s", archive_id, subtask_name)
  4103. # Also try by filename
  4104. if not archive_id and filename:
  4105. result = await db.execute(
  4106. select(PrintArchive)
  4107. .where(PrintArchive.printer_id == printer_id)
  4108. .where(PrintArchive.filename == filename)
  4109. .where(PrintArchive.status == "printing")
  4110. .order_by(PrintArchive.created_at.desc())
  4111. .limit(1)
  4112. )
  4113. archive = result.scalar_one_or_none()
  4114. if archive:
  4115. archive_id = archive.id
  4116. # Cleanup: delete uploaded file from printer SD card to prevent phantom prints (Issue #374, #1542)
  4117. # The print scheduler uploads files to the SD card root (/). Some printers (e.g. P1S, A1)
  4118. # auto-start files found in root on power cycle, causing ghost prints.
  4119. # Must run before the archive_id early-return so it executes even when archiving is disabled.
  4120. try:
  4121. if subtask_name:
  4122. archive_filename: str | None = None
  4123. async with async_session() as db:
  4124. from backend.app.models.archive import PrintArchive
  4125. from backend.app.models.printer import Printer
  4126. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  4127. printer = result.scalar_one_or_none()
  4128. if archive_id:
  4129. archive_row = await db.execute(select(PrintArchive.filename).where(PrintArchive.id == archive_id))
  4130. archive_filename = archive_row.scalar_one_or_none()
  4131. if printer:
  4132. from backend.app.services.bambu_ftp import DeleteResult, delete_file_async
  4133. from backend.app.utils.filename import derive_remote_filename
  4134. # Primary candidate: the exact path the dispatcher uploaded to
  4135. # (derived from archive.filename via the same rule as upload).
  4136. # Without it, a library row that ended up with a doubled
  4137. # .gcode.3mf (#1542) leaves the real file behind because the
  4138. # subtask_name + ext fallbacks below don't match what's on the
  4139. # SD card. Fallbacks remain for archive-less prints (subtask
  4140. # never resolved to an archive) and for older naming variants.
  4141. candidate_paths: list[str] = []
  4142. if archive_filename:
  4143. candidate_paths.append(f"/{derive_remote_filename(archive_filename)}")
  4144. for ext in (".3mf", ".gcode"):
  4145. fallback = f"/{subtask_name}{ext}"
  4146. if fallback not in candidate_paths:
  4147. candidate_paths.append(fallback)
  4148. # Three outcomes track across all candidates so the final log
  4149. # line reflects what actually happened. The A1 in #1721 always
  4150. # ends here with ``any_not_found=True`` and the others False
  4151. # — its firmware auto-cleans the SD card before our cleanup
  4152. # runs, every candidate FTP-DELE returns 550, and the old
  4153. # code burned 3 retries × 2 s × 3 candidates per print
  4154. # logging a misleading "may linger" WARNING on a successful
  4155. # print.
  4156. any_deleted = False
  4157. any_real_failure = False
  4158. any_not_found = False
  4159. for remote_path in candidate_paths:
  4160. # Retry only the FAILED case — 550 NOT_FOUND will never
  4161. # recover by waiting, so a "file isn't here" answer
  4162. # advances immediately to the next candidate without
  4163. # consuming the retry budget.
  4164. for attempt in range(1, 4):
  4165. try:
  4166. delete_result = await delete_file_async(
  4167. printer.ip_address,
  4168. printer.access_code,
  4169. remote_path,
  4170. printer_model=printer.model,
  4171. )
  4172. except Exception as e:
  4173. delete_result = DeleteResult.FAILED
  4174. logger.warning(
  4175. "SD card cleanup attempt %d/3 raised for %s: %s",
  4176. attempt,
  4177. remote_path,
  4178. e,
  4179. )
  4180. if delete_result == DeleteResult.DELETED:
  4181. any_deleted = True
  4182. logger.info("Deleted %s from printer %s SD card", remote_path, printer.name)
  4183. break
  4184. if delete_result == DeleteResult.NOT_FOUND:
  4185. any_not_found = True
  4186. break # 550 will not recover; try next candidate
  4187. # FAILED: real error — retry with backoff, then give up
  4188. if attempt < 3:
  4189. await asyncio.sleep(2)
  4190. else:
  4191. any_real_failure = True
  4192. logger.warning(
  4193. "SD card cleanup failed after 3 attempts for %s "
  4194. "(network/auth/transient error — file may linger on SD card)",
  4195. remote_path,
  4196. )
  4197. if not any_deleted and not any_real_failure and any_not_found:
  4198. # Every candidate said "not here." Either the printer
  4199. # firmware swept the SD card itself (common on A1) or the
  4200. # dispatcher's upload path doesn't match our candidate
  4201. # rule. Either way: nothing to clean up, no warning.
  4202. logger.debug(
  4203. "SD card cleanup: nothing to delete on %s — every candidate returned 550 "
  4204. "(printer likely self-cleaned)",
  4205. printer.name,
  4206. )
  4207. except Exception as e:
  4208. logger.warning("SD card file cleanup failed for printer %s: %s", printer_id, e)
  4209. log_timing("SD card cleanup")
  4210. # Update queue item status early — must run before the archive_id early-return
  4211. # so queue items don't get stuck in "printing" when archive lookup fails.
  4212. # Uses run_with_retry to handle SQLite "database is locked" errors (#897).
  4213. queue_item_id = None
  4214. queue_status = None
  4215. queue_auto_off = False
  4216. try:
  4217. from backend.app.core.database import run_with_retry
  4218. from backend.app.models.print_queue import PrintQueueItem
  4219. async def _update_queue_status(db):
  4220. nonlocal queue_item_id, queue_status, queue_auto_off
  4221. result = await db.execute(
  4222. select(PrintQueueItem)
  4223. .where(PrintQueueItem.printer_id == printer_id)
  4224. .where(PrintQueueItem.status == "printing")
  4225. )
  4226. printing_items = list(result.scalars().all())
  4227. if len(printing_items) > 1:
  4228. logger.warning(
  4229. "BUG: Multiple queue items in 'printing' status for printer %s: %s",
  4230. printer_id,
  4231. [(i.id, i.archive_id, i.library_file_id) for i in printing_items],
  4232. )
  4233. item = printing_items[0] if printing_items else None
  4234. if item:
  4235. queue_status = data.get("status", "completed")
  4236. # MQTT sends "aborted" for cancelled prints; normalise to
  4237. # "cancelled" so it matches the queue schema Literal.
  4238. if queue_status == "aborted":
  4239. queue_status = "cancelled"
  4240. item.status = queue_status
  4241. item.completed_at = datetime.now(timezone.utc)
  4242. if queue_status == "failed" and not item.error_message:
  4243. item.error_message = _format_hms_error_summary(data.get("hms_errors") or [])
  4244. # Bump usage counters on the source library file so admins can
  4245. # sort by "last printed" and (eventually) auto-purge stale
  4246. # files — #1008.
  4247. await _bump_library_file_usage_if_completed(db, item, queue_status)
  4248. await db.commit()
  4249. queue_item_id = item.id
  4250. queue_auto_off = item.auto_off_after
  4251. logger.info("Updated queue item %s status to %s", item.id, queue_status)
  4252. await run_with_retry(_update_queue_status, label="queue status update")
  4253. # Post-commit side effects (notifications, MQTT relay, auto-off) use
  4254. # their own sessions and have their own error handling — no retry needed.
  4255. if queue_item_id is not None:
  4256. # MQTT relay - publish queue job completed
  4257. try:
  4258. printer_info = printer_manager.get_printer(printer_id)
  4259. await mqtt_relay.on_queue_job_completed(
  4260. job_id=queue_item_id,
  4261. filename=filename or subtask_name,
  4262. printer_id=printer_id,
  4263. printer_name=printer_info.name if printer_info else "Unknown",
  4264. status=queue_status,
  4265. )
  4266. except Exception:
  4267. pass # Don't fail if MQTT fails
  4268. # Check if queue is now empty and send notification
  4269. try:
  4270. from sqlalchemy import func as sa_func
  4271. async with async_session() as db:
  4272. count_result = await db.execute(
  4273. select(sa_func.count(PrintQueueItem.id)).where(PrintQueueItem.status == "pending")
  4274. )
  4275. pending_count = count_result.scalar() or 0
  4276. if pending_count == 0:
  4277. today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
  4278. completed_result = await db.execute(
  4279. select(sa_func.count(PrintQueueItem.id)).where(
  4280. PrintQueueItem.status.in_(["completed", "failed", "skipped"]),
  4281. PrintQueueItem.completed_at >= today_start,
  4282. )
  4283. )
  4284. completed_count = completed_result.scalar() or 1
  4285. await notification_service.on_queue_completed(
  4286. completed_count=completed_count,
  4287. db=db,
  4288. )
  4289. except Exception:
  4290. pass # Don't fail if notification fails
  4291. # Handle auto_off_after - power off printer if the queue item opted
  4292. # in. Delegates to the smart-plug manager so the off honours each
  4293. # plug's configured strategy (time delay or temperature threshold),
  4294. # is cancelled if the printer starts printing again, and never cuts
  4295. # power on a loaded print (#1890). Previously an inline block here
  4296. # hardcoded a 50°C / 600s cooldown wait and powered off on the
  4297. # timeout regardless of print state — cutting a touchscreen reprint.
  4298. if queue_auto_off:
  4299. try:
  4300. async with async_session() as db:
  4301. await smart_plug_manager.schedule_off_after_queue_job(printer_id, db)
  4302. except Exception as e:
  4303. logger.warning("Failed to schedule queue auto-off for printer %s: %s", printer_id, e)
  4304. except Exception as e:
  4305. logging.getLogger(__name__).warning(f"Queue item update failed: {e}")
  4306. log_timing("Queue item update")
  4307. # Register bed cooldown waiter (event-driven via on_bed_temp_update callback).
  4308. # Must run before archive_id early-return so it fires for all prints (including
  4309. # prints started from BambuStudio/touchscreen that have no archive).
  4310. if data.get("status") == "completed":
  4311. try:
  4312. from backend.app.api.routes.settings import get_setting
  4313. async with async_session() as db:
  4314. threshold_str = await get_setting(db, "bed_cooled_threshold")
  4315. threshold = float(threshold_str) if threshold_str else 35.0
  4316. # Check if any provider has on_bed_cooled enabled (skip registration if none)
  4317. async with async_session() as db:
  4318. providers = await notification_service._get_providers_for_event(db, "on_bed_cooled", printer_id)
  4319. if providers:
  4320. _bed_cool_waiters[printer_id] = {
  4321. "threshold": threshold,
  4322. "filename": filename or subtask_name or "",
  4323. "registered_at": time.time(),
  4324. }
  4325. logger.info(
  4326. "[BED-COOL] Registered waiter for printer %s (threshold: %.0f°C)",
  4327. printer_id,
  4328. threshold,
  4329. )
  4330. else:
  4331. logger.debug("[BED-COOL] No providers enabled for bed_cooled on printer %s", printer_id)
  4332. except Exception as e:
  4333. logger.warning("[BED-COOL] Failed to register waiter: %s", e)
  4334. # --- Track filament consumption (must run before archive_id early-return so usage
  4335. # is recorded even when auto-archive is disabled) ---
  4336. usage_results: list[dict] = []
  4337. # Prefer ams_mapping captured from MQTT request topic (works for all print sources)
  4338. stored_ams_mapping = data.get("ams_mapping")
  4339. # Fallback to _print_ams_mappings for queue/reprint (set before print starts)
  4340. if not stored_ams_mapping and archive_id:
  4341. stored_ams_mapping = _print_ams_mappings.pop(archive_id, None)
  4342. # Always drain the plate_id register on completion — the session already
  4343. # consumed it at print-start injection; leaving it would leak into the next
  4344. # print on the same archive_id (rare but possible with reprints) (#1697).
  4345. # Capture the popped value so the completion notification can scope the
  4346. # archive-level (summed-across-plates per #1593) filament + time totals
  4347. # down to the single plate that was actually printed (#1785).
  4348. notify_plate_id: int | None = None
  4349. if archive_id:
  4350. notify_plate_id = _print_plate_ids.pop(archive_id, None)
  4351. # Internal inventory: track AMS remain% deltas (skip if Spoolman handles usage)
  4352. try:
  4353. async with async_session() as db:
  4354. from backend.app.api.routes.settings import get_setting
  4355. _spoolman_on = await get_setting(db, "spoolman_enabled")
  4356. if not _spoolman_on or _spoolman_on.lower() != "true":
  4357. from backend.app.services.usage_tracker import on_print_complete as usage_on_print_complete
  4358. async with async_session() as db:
  4359. usage_results = await usage_on_print_complete(
  4360. printer_id,
  4361. data,
  4362. printer_manager,
  4363. db,
  4364. archive_id=archive_id,
  4365. ams_mapping=stored_ams_mapping,
  4366. )
  4367. if usage_results:
  4368. await ws_manager.broadcast(
  4369. {
  4370. "type": "spool_usage_logged",
  4371. "printer_id": printer_id,
  4372. "usage": usage_results,
  4373. }
  4374. )
  4375. log_timing("Usage tracker")
  4376. except Exception as e:
  4377. logger.warning("Usage tracker on_print_complete failed: %s", e)
  4378. # Spoolman: report filament usage (requires archive_id for tracking data lookup)
  4379. if archive_id:
  4380. if data.get("status") == "completed":
  4381. try:
  4382. await _report_spoolman_usage(printer_id, archive_id)
  4383. log_timing("Spoolman usage report")
  4384. except Exception as e:
  4385. logger.warning("Spoolman usage reporting failed: %s", e)
  4386. else:
  4387. # Report partial usage if tracking data exists (only stored when weight sync is disabled)
  4388. try:
  4389. async with async_session() as db:
  4390. await _cleanup_spoolman_tracking(
  4391. printer_id,
  4392. archive_id,
  4393. db,
  4394. last_layer_num=data.get("last_layer_num"),
  4395. last_progress=data.get("last_progress"),
  4396. )
  4397. except Exception as e:
  4398. logger.debug("[SPOOLMAN] Cleanup failed: %s", e)
  4399. log_timing("Filament usage tracking")
  4400. if not archive_id:
  4401. logger.warning("Could not find archive for print complete: filename=%s, subtask=%s", filename, subtask_name)
  4402. # Still send print-complete/failed/stopped notifications even without an archive.
  4403. # Try to enrich with queue/library-file data so user-specific emails work too.
  4404. async def _notify_no_archive():
  4405. try:
  4406. async with async_session() as db:
  4407. from backend.app.models.library import LibraryFile
  4408. from backend.app.models.print_queue import PrintQueueItem
  4409. from backend.app.models.printer import Printer
  4410. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  4411. printer_obj = result.scalar_one_or_none()
  4412. p_name = printer_obj.name if printer_obj else f"Printer {printer_id}"
  4413. # Try to find the most-recent queue item for this printer so we can
  4414. # recover created_by_id and estimated print time.
  4415. # NOTE: By the time this task runs the queue item status has already
  4416. # been updated to a terminal state (completed/failed/cancelled), so
  4417. # we look for recently-completed items (within the last 5 minutes).
  4418. no_archive_data: dict | None = None
  4419. try:
  4420. cutoff = datetime.now(timezone.utc) - timedelta(minutes=5)
  4421. q_result = await db.execute(
  4422. select(PrintQueueItem)
  4423. .where(PrintQueueItem.printer_id == printer_id)
  4424. .where(PrintQueueItem.status.in_(["completed", "failed", "cancelled"]))
  4425. .where(PrintQueueItem.completed_at >= cutoff)
  4426. .order_by(PrintQueueItem.completed_at.desc())
  4427. .limit(1)
  4428. )
  4429. queue_item = q_result.scalar_one_or_none()
  4430. if queue_item:
  4431. no_archive_data = {"created_by_id": queue_item.created_by_id}
  4432. # Pull estimated time from library file when available
  4433. if queue_item.library_file_id:
  4434. lib_result = await db.execute(
  4435. select(LibraryFile).where(LibraryFile.id == queue_item.library_file_id)
  4436. )
  4437. lib_file = lib_result.scalar_one_or_none()
  4438. if lib_file and lib_file.print_time_seconds:
  4439. no_archive_data["print_time_seconds"] = lib_file.print_time_seconds
  4440. except Exception as lookup_err:
  4441. logger.debug(
  4442. "[NOTIFY-BG] Could not look up queue item for no-archive notification: %s", lookup_err
  4443. )
  4444. # Enrich with usage tracker results (captured in enclosing scope)
  4445. if usage_results:
  4446. if no_archive_data is None:
  4447. no_archive_data = {}
  4448. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  4449. if total_from_usage > 0:
  4450. no_archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  4451. no_archive_data["usage_results"] = usage_results
  4452. # Try MQTT remaining_time for print duration when no queue/library data
  4453. if no_archive_data and not no_archive_data.get("print_time_seconds"):
  4454. mqtt_remaining = data.get("remaining_time")
  4455. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  4456. no_archive_data["print_time_seconds"] = int(mqtt_remaining)
  4457. ps = data.get("status", "completed")
  4458. logger.info(
  4459. "[NOTIFY-BG] Sending notification without archive: printer=%s, status=%s", printer_id, ps
  4460. )
  4461. await notification_service.on_print_complete(
  4462. printer_id, p_name, ps, data, db, archive_data=no_archive_data
  4463. )
  4464. # Send user-specific email if we have a created_by_id
  4465. if no_archive_data and no_archive_data.get("created_by_id"):
  4466. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  4467. await _dispatch_user_print_email(
  4468. ps,
  4469. no_archive_data["created_by_id"],
  4470. p_name,
  4471. raw_filename,
  4472. db,
  4473. )
  4474. logger.info("[NOTIFY-BG] Completed (no-archive path)")
  4475. except Exception as e:
  4476. logger.warning("[NOTIFY-BG] Failed to send notification without archive: %s", e, exc_info=True)
  4477. spawn_background_task(_notify_no_archive(), name="notify-no-archive")
  4478. return
  4479. log_timing("Archive lookup")
  4480. # Update archive status
  4481. logger.info("[ARCHIVE] Updating archive %s status...", archive_id)
  4482. try:
  4483. async with async_session() as db:
  4484. service = ArchiveService(db)
  4485. status = data.get("status", "completed")
  4486. hms_errors = data.get("hms_errors", []) if status == "failed" else None
  4487. if hms_errors:
  4488. logger.info("[ARCHIVE] HMS errors at failure: %s", hms_errors)
  4489. failure_reason = derive_failure_reason(status, hms_errors)
  4490. if data.get("_reconciled"):
  4491. # A reconciled completion closes out a stale archive at
  4492. # reconnect — it is not a user action, so don't mislabel it
  4493. # "User cancelled". The "Stale" prefix matches the existing
  4494. # stale-cleanup convention and records that the real end time
  4495. # is unknown, which is also why its logged duration is 0 (#2592).
  4496. failure_reason = "Stale - reconciled after reconnect, end time unknown"
  4497. if failure_reason:
  4498. logger.info("[ARCHIVE] failure_reason=%r (status=%s)", failure_reason, status)
  4499. elif status == "failed" and hms_errors:
  4500. logger.info("[ARCHIVE] HMS errors present but none matched a known failure-reason short code")
  4501. await service.update_archive_status(
  4502. archive_id,
  4503. status=status,
  4504. completed_at=(
  4505. datetime.now(timezone.utc) if status in ("completed", "failed", "aborted", "cancelled") else None
  4506. ),
  4507. failure_reason=failure_reason,
  4508. )
  4509. logger.info(
  4510. "[ARCHIVE] Archive %s status updated to %s, failure_reason=%s", archive_id, status, failure_reason
  4511. )
  4512. await ws_manager.send_archive_updated(
  4513. {
  4514. "id": archive_id,
  4515. "status": status,
  4516. }
  4517. )
  4518. logger.info("[ARCHIVE] WebSocket notification sent for archive %s", archive_id)
  4519. # MQTT relay - publish archive updated
  4520. try:
  4521. await mqtt_relay.on_archive_updated(
  4522. archive_id=archive_id,
  4523. print_name=filename or subtask_name,
  4524. status=status,
  4525. )
  4526. except Exception:
  4527. pass # Don't fail if MQTT fails
  4528. except Exception as e:
  4529. logger.error("[ARCHIVE] Failed to update archive %s status: %s", archive_id, e, exc_info=True)
  4530. # Continue with other operations even if archive update fails
  4531. log_timing("Archive status update")
  4532. # Write independent print log entry (separate table, never touches archives)
  4533. try:
  4534. async with async_session() as db:
  4535. from backend.app.models.archive import PrintArchive
  4536. from backend.app.services.print_log import write_log_entry
  4537. archive = await db.get(PrintArchive, archive_id)
  4538. if archive:
  4539. # Back-fill created_by_id on reprint (#730): reprint reuses the
  4540. # source archive row rather than creating a new one, so an
  4541. # archive that was auto-created from a printer-initiated
  4542. # print (created_by_id=NULL) would otherwise stay unattributed
  4543. # forever. When we have a print-session user AND the archive
  4544. # has no attribution yet, credit the current user. Never
  4545. # overwrite an existing attribution — the original uploader
  4546. # keeps ownership.
  4547. _print_user_id = _print_user_info.get("user_id") if _print_user_info else None
  4548. if archive.created_by_id is None and _print_user_id is not None:
  4549. archive.created_by_id = _print_user_id
  4550. p_info = printer_manager.get_printer(printer_id)
  4551. # Per-run actuals — written to PrintLogEntry so stats reflect
  4552. # what THIS print actually used, not the source archive's
  4553. # first-run values (#1378). Helper handles the partial-print
  4554. # math (failed / cancelled / stopped get scaled to progress
  4555. # or to tracked spool deltas).
  4556. _run_status = data.get("status", "completed")
  4557. # #2614: scope the per-run estimate to the printed plate. For a
  4558. # multi-plate 3MF dispatched one plate at a time, the archive's
  4559. # filament/cost are the whole-file totals; the PrintLogEntry must
  4560. # reflect only this plate. No effect on single-plate archives (the
  4561. # plate estimate equals the whole-file value) or on the tracker
  4562. # path (measured spool deltas win in _compute_run_filament_grams).
  4563. _est_full_path = (
  4564. app_settings.base_dir / archive.file_path if archive.file_path else None
  4565. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  4566. _est_grams, _est_cost = _plate_scoped_run_estimate(archive, _est_full_path)
  4567. _run_grams = _compute_run_filament_grams(
  4568. _run_status,
  4569. _est_grams,
  4570. data.get("progress"),
  4571. usage_results,
  4572. )
  4573. # Per-run cost — prefer usage_results sum. For partial prints
  4574. # we deliberately skip the topup-to-estimate logic in
  4575. # usage_tracker (which assumes the print completed); the raw
  4576. # tracked-spool sum is closer to what THIS run actually cost.
  4577. _run_cost: float | None = None
  4578. if usage_results:
  4579. _run_cost = sum(r.get("cost") or 0 for r in usage_results) or None
  4580. if _run_cost is None and _run_status == "completed":
  4581. _run_cost = _est_cost
  4582. await write_log_entry(
  4583. db,
  4584. archive_id=archive.id,
  4585. status=_run_status,
  4586. print_name=archive.print_name,
  4587. printer_name=p_info.name if p_info else None,
  4588. printer_id=printer_id,
  4589. started_at=archive.started_at,
  4590. completed_at=archive.completed_at,
  4591. filament_type=archive.filament_type,
  4592. filament_color=archive.filament_color,
  4593. filament_used_grams=_run_grams,
  4594. cost=_run_cost,
  4595. failure_reason=archive.failure_reason,
  4596. thumbnail_path=archive.thumbnail_path,
  4597. created_by_id=archive.created_by_id,
  4598. created_by_username=_print_user_info.get("username") if _print_user_info else None,
  4599. # Reconciled completions have an unknown real end time —
  4600. # log 0 duration instead of the whole disconnect gap (#2592).
  4601. reconciled=bool(data.get("_reconciled")),
  4602. )
  4603. await db.commit()
  4604. logger.info("[PRINT_LOG] Log entry written for archive %s", archive_id)
  4605. except Exception as e:
  4606. logger.warning("[PRINT_LOG] Failed to write log entry for archive %s: %s", archive_id, e)
  4607. log_timing("Print log entry")
  4608. # Run slow operations as background tasks to avoid blocking the event loop
  4609. # These operations can take 5-10+ seconds and would freeze the UI if awaited
  4610. async def _background_energy_calculation():
  4611. """Calculate and save energy usage in background.
  4612. Reads the starting kWh from the archive row (#941: persisted so a mid-print
  4613. backend restart no longer loses per-print energy data).
  4614. """
  4615. try:
  4616. logger.info("[ENERGY-BG] Starting energy calculation for archive %s", archive_id)
  4617. async with async_session() as db:
  4618. from backend.app.models.archive import PrintArchive
  4619. archive = await db.get(PrintArchive, archive_id)
  4620. if archive is None:
  4621. logger.warning("[ENERGY-BG] Archive %s no longer exists", archive_id)
  4622. return
  4623. starting_kwh = archive.energy_start_kwh
  4624. if starting_kwh is None:
  4625. logger.info("[ENERGY-BG] No start kWh recorded for archive %s", archive_id)
  4626. return
  4627. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  4628. plug = plug_result.scalar_one_or_none()
  4629. if plug is None:
  4630. logger.info("[ENERGY-BG] No smart plug for printer %s", printer_id)
  4631. return
  4632. energy = await _get_plug_energy(plug, db)
  4633. logger.info("[ENERGY-BG] Energy response: %s", energy)
  4634. if not energy or energy.get("total") is None:
  4635. logger.warning("[ENERGY-BG] No 'total' in energy response")
  4636. return
  4637. energy_used = round(energy["total"] - starting_kwh, 4)
  4638. logger.info("[ENERGY-BG] Per-print energy: %s kWh", energy_used)
  4639. if energy_used < 0:
  4640. logger.warning(
  4641. "[ENERGY-BG] Negative energy delta for archive %s (start=%s, end=%s) — counter reset?",
  4642. archive_id,
  4643. starting_kwh,
  4644. energy["total"],
  4645. )
  4646. return
  4647. from backend.app.api.routes.settings import get_setting
  4648. energy_cost_per_kwh = await get_setting(db, "energy_cost_per_kwh")
  4649. cost_per_kwh = float(energy_cost_per_kwh) if energy_cost_per_kwh else 0.15
  4650. energy_cost_value = round(energy_used * cost_per_kwh, 3)
  4651. # First-run-only overwrite of archive.energy_kwh / energy_cost so a
  4652. # reprint doesn't visually clobber the source archive's energy data
  4653. # (#1378). Reprint energy lives in the matching PrintLogEntry below.
  4654. from sqlalchemy import func
  4655. from backend.app.models.print_log import PrintLogEntry
  4656. existing_runs = await db.scalar(
  4657. select(func.count(PrintLogEntry.id)).where(PrintLogEntry.archive_id == archive_id)
  4658. )
  4659. if (existing_runs or 0) <= 1:
  4660. # 0 = legacy archive that pre-dates per-run logging; 1 = the row
  4661. # we just wrote for THIS print. Either way it's the first run.
  4662. archive.energy_kwh = energy_used
  4663. archive.energy_cost = energy_cost_value
  4664. # Backfill the latest PrintLogEntry for this archive with energy
  4665. # (write_log_entry above ran before this background task completed,
  4666. # so energy fields are still NULL on that row).
  4667. latest_run = await db.execute(
  4668. select(PrintLogEntry)
  4669. .where(PrintLogEntry.archive_id == archive_id)
  4670. .order_by(PrintLogEntry.id.desc())
  4671. .limit(1)
  4672. )
  4673. run_row = latest_run.scalar_one_or_none()
  4674. if run_row is not None:
  4675. run_row.energy_kwh = energy_used
  4676. run_row.energy_cost = energy_cost_value
  4677. await db.commit()
  4678. logger.info("[ENERGY-BG] Saved: %s kWh, cost=%s", energy_used, energy_cost_value)
  4679. except Exception as e:
  4680. logger.warning("[ENERGY-BG] Failed: %s", e)
  4681. async def _background_finish_photo() -> str | None:
  4682. """Capture finish photo in background. Returns photo filename if captured."""
  4683. try:
  4684. logger.info("[PHOTO-BG] Starting finish photo capture for archive %s", archive_id)
  4685. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  4686. # Read phase: settings + printer + archive in a short session, released
  4687. # BEFORE the capture pipeline below. The capture (timelapse last-frame,
  4688. # stage-22 wait, external-camera grab, or a fresh RTSP shot) can take
  4689. # tens of seconds; holding this session across it pinned one pooled
  4690. # connection idle-in-transaction per finishing print (issue #2572).
  4691. async with async_session() as db:
  4692. from backend.app.api.routes.settings import get_setting
  4693. from backend.app.models.archive import PrintArchive
  4694. from backend.app.models.printer import Printer
  4695. capture_enabled = await get_setting(db, "capture_finish_photo")
  4696. if capture_enabled is not None and capture_enabled.lower() != "true":
  4697. return None
  4698. if not archive_id:
  4699. return None
  4700. printer = (await db.execute(select(Printer).where(Printer.id == printer_id))).scalar_one_or_none()
  4701. archive = (
  4702. await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  4703. ).scalar_one_or_none()
  4704. if not printer or not archive:
  4705. return None
  4706. import uuid
  4707. from datetime import datetime
  4708. from pathlib import Path
  4709. if archive.file_path:
  4710. archive_dir = app_settings.base_dir / Path(archive.file_path).parent
  4711. else:
  4712. logger.warning("[PHOTO-BG] Archive %s has no file_path, using fallback dir", archive_id)
  4713. archive_dir = app_settings.archive_dir / str(archive.id)
  4714. photo_filename = None
  4715. # Prefer the timelapse last-frame source when a timelapse was
  4716. # recording — it captures the moment after the toolhead parks
  4717. # but before the bed drops, which the live-camera grab below
  4718. # would miss (#1397). Skipped for external cameras (those have
  4719. # their own framing and don't see a Bambu timelapse). Only
  4720. # runs when the USER explicitly enabled timelapse for this
  4721. # print — #1721 removed Bambuddy's force-on at dispatch
  4722. # because it caused per-layer nozzle parking on Smooth-mode
  4723. # slicer profiles.
  4724. prefer_timelapse_source = bool(data.get("timelapse_was_active")) and not (
  4725. printer.external_camera_enabled and printer.external_camera_url
  4726. )
  4727. timelapse_still_pending = False
  4728. if prefer_timelapse_source:
  4729. photo_filename, timelapse_still_pending = await _capture_finish_photo_from_timelapse(
  4730. archive_id=archive_id,
  4731. archive_dir=archive_dir,
  4732. rotation=getattr(printer, "camera_rotation", 0),
  4733. )
  4734. # #1721: replacement framing path — on_finish_photo_moment
  4735. # pre-captured a frame at the stage-22 / FINISH edge (toolhead
  4736. # parked, bed not yet dropped) and cached the JPEG bytes in
  4737. # _stage22_finish_frames. Consume them now so the saved photo
  4738. # has the better framing instead of the post-bed-drop angle
  4739. # the live-camera fallback below would give.
  4740. if not photo_filename:
  4741. # #1790: on the FINISH-state fallback path the producer
  4742. # task is dispatched back-to-back with this consumer, so
  4743. # a bare pop would race past with an empty result and
  4744. # the RTSP fallback below would collide with the
  4745. # producer's still-in-flight grab (single-client RTSP
  4746. # on Bambu printers). Wait for the producer to finish
  4747. # or give up before touching the cache.
  4748. in_flight = _stage22_finish_in_flight.pop(printer_id, None)
  4749. if in_flight is not None:
  4750. try:
  4751. await asyncio.wait_for(in_flight.wait(), timeout=20.0)
  4752. except asyncio.TimeoutError:
  4753. logger.warning(
  4754. "[PHOTO-BG] timed out waiting for stage-22 producer for printer %s — proceeding to fallback",
  4755. printer_id,
  4756. )
  4757. cached_frame = _stage22_finish_frames.pop(printer_id, None)
  4758. if cached_frame:
  4759. # Already rotated by the producer (#2708) — rotating again
  4760. # here would undo the fix on the banked-frame path, whose
  4761. # bytes reach the cache having been rotated once already.
  4762. photos_dir = archive_dir / "photos"
  4763. photos_dir.mkdir(parents=True, exist_ok=True)
  4764. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  4765. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  4766. photo_path = photos_dir / photo_filename
  4767. await asyncio.to_thread(photo_path.write_bytes, cached_frame)
  4768. logger.info(
  4769. "[PHOTO-BG] Saved stage-22 pre-captured frame: %s (%d bytes)",
  4770. photo_filename,
  4771. len(cached_frame),
  4772. )
  4773. # Fallback chain: external camera → buffered live frame →
  4774. # fresh RTSP capture. Only runs if the timelapse path above
  4775. # didn't already produce a photo.
  4776. if not photo_filename:
  4777. if printer.external_camera_enabled and printer.external_camera_url:
  4778. logger.info("[PHOTO-BG] Using external camera")
  4779. from backend.app.api.routes.camera import live_frame_for_capture
  4780. from backend.app.services.external_camera import capture_frame
  4781. # #2707: the second half of the finish-photo failure — the
  4782. # pre-capture and this fallback both collided with the live
  4783. # view. None here continues down the fallback chain.
  4784. defer, buffered = live_frame_for_capture(printer_id)
  4785. if defer:
  4786. frame_data = buffered
  4787. else:
  4788. frame_data = await capture_frame(
  4789. printer.external_camera_url,
  4790. printer.external_camera_type or "mjpeg",
  4791. snapshot_url=printer.external_camera_snapshot_url,
  4792. )
  4793. if frame_data:
  4794. frame_data = _apply_camera_rotation(frame_data, printer, logger)
  4795. photos_dir = archive_dir / "photos"
  4796. photos_dir.mkdir(parents=True, exist_ok=True)
  4797. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  4798. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  4799. photo_path = photos_dir / photo_filename
  4800. await asyncio.to_thread(photo_path.write_bytes, frame_data)
  4801. logger.info("[PHOTO-BG] Saved external camera frame: %s", photo_filename)
  4802. else:
  4803. # Check if camera stream is active - use buffered frame to avoid freeze
  4804. # Check both RTSP streams (_active_streams) and chamber image streams (_active_chamber_streams)
  4805. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  4806. active_chamber_for_printer = [k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")]
  4807. buffered_frame = get_buffered_frame(printer_id)
  4808. if (active_for_printer or active_chamber_for_printer) and buffered_frame:
  4809. # Use frame from active stream
  4810. logger.info("[PHOTO-BG] Using buffered frame from active stream")
  4811. buffered_frame = _apply_camera_rotation(buffered_frame, printer, logger)
  4812. photos_dir = archive_dir / "photos"
  4813. photos_dir.mkdir(parents=True, exist_ok=True)
  4814. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  4815. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  4816. photo_path = photos_dir / photo_filename
  4817. await asyncio.to_thread(photo_path.write_bytes, buffered_frame)
  4818. logger.info("[PHOTO-BG] Saved buffered frame: %s", photo_filename)
  4819. else:
  4820. # No active stream - capture new frame
  4821. from backend.app.services.camera import capture_finish_photo
  4822. photo_filename = await capture_finish_photo(
  4823. printer_id=printer_id,
  4824. ip_address=printer.ip_address,
  4825. access_code=printer.access_code,
  4826. model=printer.model,
  4827. archive_dir=archive_dir,
  4828. rotation=getattr(printer, "camera_rotation", 0),
  4829. )
  4830. # Write phase: attach the photo in a fresh short-lived session.
  4831. if photo_filename:
  4832. async with async_session() as db:
  4833. from backend.app.models.archive import PrintArchive
  4834. arch = await db.get(PrintArchive, archive_id)
  4835. if arch is not None:
  4836. photos = arch.photos or []
  4837. photos.append(photo_filename)
  4838. arch.photos = photos
  4839. await db.commit()
  4840. logger.info("[PHOTO-BG] Saved: %s", photo_filename)
  4841. # The short wait above is bounded so a slow printer can't hold up
  4842. # the print-complete notification, which is what the caller is
  4843. # blocking on. When it ran out with the video still on its way,
  4844. # keep waiting off to the side and add the better frame to the
  4845. # archive once it arrives (#2704 follow-up) — otherwise P1-series
  4846. # users, whose videos routinely take minutes to transfer, never get
  4847. # the pre-bed-drop framing this path exists to provide.
  4848. #
  4849. # Spawned here rather than at the point the wait gave up: both this
  4850. # function and the upgrade do a read-modify-write on `photos`, and
  4851. # the live-camera fallback above can take tens of seconds. Starting
  4852. # the upgrade before that write means the two can interleave and one
  4853. # silently drops the other's entry, leaving a JPEG on disk that the
  4854. # gallery never lists.
  4855. if timelapse_still_pending:
  4856. spawn_background_task(
  4857. _upgrade_finish_photo_from_timelapse(
  4858. archive_id, archive_dir, rotation=getattr(printer, "camera_rotation", 0)
  4859. ),
  4860. name=f"finish-photo-upgrade-{archive_id}",
  4861. )
  4862. return photo_filename
  4863. except Exception as e:
  4864. logger.warning("[PHOTO-BG] Failed: %s", e)
  4865. return None
  4866. spawn_background_task(_background_energy_calculation(), name="background-energy-calc")
  4867. # Photo capture task - result will be used by notifications
  4868. photo_task = spawn_background_task(_background_finish_photo(), name="background-finish-photo")
  4869. log_timing("Background tasks scheduled (energy, photo)")
  4870. # Also run smart plug, notifications, and maintenance as background tasks
  4871. print_status = data.get("status", "completed")
  4872. async def _background_smart_plug():
  4873. """Handle smart plug automation in background."""
  4874. try:
  4875. logger.info("[AUTO-OFF-BG] Starting smart plug automation for printer %s", printer_id)
  4876. async with async_session() as db:
  4877. await smart_plug_manager.on_print_complete(printer_id, print_status, db)
  4878. logger.info("[AUTO-OFF-BG] Completed")
  4879. except Exception as e:
  4880. logger.warning("[AUTO-OFF-BG] Failed: %s", e)
  4881. async def _background_notifications(finish_photo_filename: str | None = None):
  4882. """Send print complete notifications in background."""
  4883. try:
  4884. logger.info(
  4885. "[NOTIFY-BG] Starting notifications for printer %s, photo=%s", printer_id, finish_photo_filename
  4886. )
  4887. async with async_session() as db:
  4888. from backend.app.models.archive import PrintArchive
  4889. from backend.app.models.printer import Printer
  4890. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  4891. printer = result.scalar_one_or_none()
  4892. printer_name = printer.name if printer else f"Printer {printer_id}"
  4893. archive_data = None
  4894. if archive_id:
  4895. archive_result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  4896. archive = archive_result.scalar_one_or_none()
  4897. if archive:
  4898. # Actual elapsed time from started_at/completed_at when both are
  4899. # populated (every terminal status sets completed_at after #1198).
  4900. # Falls back to None so the notification path can decide whether to
  4901. # render the slicer estimate as a last resort.
  4902. actual_time_seconds = None
  4903. if archive.started_at and archive.completed_at:
  4904. elapsed = (archive.completed_at - archive.started_at).total_seconds()
  4905. if elapsed > 0:
  4906. actual_time_seconds = int(elapsed)
  4907. archive_data = {
  4908. "print_time_seconds": archive.print_time_seconds,
  4909. "actual_time_seconds": actual_time_seconds,
  4910. "actual_filament_grams": archive.filament_used_grams,
  4911. "failure_reason": archive.failure_reason,
  4912. "created_by_id": archive.created_by_id,
  4913. }
  4914. # Scale filament usage for partial prints
  4915. if print_status != "completed" and archive.filament_used_grams:
  4916. progress = data.get("progress") or 0
  4917. scale = _partial_progress_scale(progress)
  4918. archive_data["actual_filament_grams"] = round(archive.filament_used_grams * scale, 1)
  4919. archive_data["progress"] = progress
  4920. # Pass per-slot data from archive.extra_data
  4921. if archive.extra_data and archive.extra_data.get("filament_slots"):
  4922. slots = archive.extra_data["filament_slots"]
  4923. if print_status != "completed":
  4924. scale = _partial_progress_scale(data.get("progress"))
  4925. slots = [{**s, "used_g": round(s["used_g"] * scale, 1)} for s in slots]
  4926. archive_data["filament_slots"] = slots
  4927. # Scope project-summed totals down to the plate that was
  4928. # actually printed — see _scope_notification_archive_data_to_plate
  4929. # for the why (#1785).
  4930. archive_data = _scope_notification_archive_data_to_plate(
  4931. archive_data,
  4932. archive.file_path,
  4933. notify_plate_id,
  4934. print_status,
  4935. data.get("progress"),
  4936. app_settings.base_dir,
  4937. )
  4938. # Enrich filament_grams from usage_results when archive has no 3MF data
  4939. if not archive_data.get("actual_filament_grams") and usage_results:
  4940. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  4941. if total_from_usage > 0:
  4942. archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  4943. # Pass usage tracker results for AMS slot info in notifications
  4944. if usage_results:
  4945. archive_data["usage_results"] = usage_results
  4946. # Add finish photo URL and image bytes if available
  4947. if finish_photo_filename:
  4948. from backend.app.api.routes.settings import get_setting
  4949. external_url = await get_setting(db, "external_url")
  4950. if external_url:
  4951. external_url = external_url.rstrip("/")
  4952. archive_data["finish_photo_url"] = (
  4953. f"{external_url}/api/v1/archives/{archive_id}/photos/{finish_photo_filename}"
  4954. )
  4955. else:
  4956. # Fallback to relative URL (won't work for external services)
  4957. archive_data["finish_photo_url"] = (
  4958. f"/api/v1/archives/{archive_id}/photos/{finish_photo_filename}"
  4959. )
  4960. # Read finish photo bytes for image attachment (e.g. Pushover)
  4961. try:
  4962. from pathlib import Path
  4963. photo_path = (
  4964. app_settings.base_dir
  4965. / Path(archive.file_path).parent
  4966. / "photos"
  4967. / finish_photo_filename
  4968. )
  4969. if photo_path.exists():
  4970. photo_bytes = await asyncio.to_thread(photo_path.read_bytes)
  4971. if len(photo_bytes) <= 2_500_000:
  4972. archive_data["image_data"] = photo_bytes
  4973. logger.info("[NOTIFY-BG] Loaded finish photo bytes: %s bytes", len(photo_bytes))
  4974. else:
  4975. logger.warning(
  4976. f"[NOTIFY-BG] Finish photo too large for attachment: "
  4977. f"{len(photo_bytes)} bytes"
  4978. )
  4979. except Exception as e:
  4980. logger.warning("[NOTIFY-BG] Failed to read finish photo bytes: %s", e)
  4981. await notification_service.on_print_complete(
  4982. printer_id, printer_name, print_status, data, db, archive_data=archive_data
  4983. )
  4984. # Send user-specific email notification
  4985. if archive_data:
  4986. created_by_id = archive_data.get("created_by_id")
  4987. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  4988. await _dispatch_user_print_email(
  4989. print_status,
  4990. created_by_id,
  4991. printer_name,
  4992. raw_filename,
  4993. db,
  4994. )
  4995. logger.info("[NOTIFY-BG] Completed")
  4996. except Exception as e:
  4997. logger.error("[NOTIFY-BG] Failed: %s", e, exc_info=True)
  4998. async def _background_maintenance_check():
  4999. """Check for maintenance due in background."""
  5000. if print_status != "completed":
  5001. return
  5002. try:
  5003. logger.info("[MAINT-BG] Starting maintenance check for printer %s", printer_id)
  5004. async with async_session() as db:
  5005. from backend.app.models.printer import Printer
  5006. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  5007. printer = result.scalar_one_or_none()
  5008. printer_name = printer.name if printer else f"Printer {printer_id}"
  5009. await ensure_default_types(db)
  5010. overview = await _get_printer_maintenance_internal(printer_id, db, commit=True)
  5011. items_needing_attention = [
  5012. {"name": item.maintenance_type_name, "is_due": item.is_due, "is_warning": item.is_warning}
  5013. for item in overview.maintenance_items
  5014. if item.enabled and (item.is_due or item.is_warning)
  5015. ]
  5016. if items_needing_attention:
  5017. await notification_service.on_maintenance_due(printer_id, printer_name, items_needing_attention, db)
  5018. logger.info("[MAINT-BG] Sent notification: %s items need attention", len(items_needing_attention))
  5019. # MQTT relay - publish maintenance alerts
  5020. for item in items_needing_attention:
  5021. try:
  5022. await mqtt_relay.on_maintenance_alert(
  5023. printer_id=printer_id,
  5024. printer_name=printer_name,
  5025. maintenance_type=item["name"],
  5026. current_value=0, # Not easily available here
  5027. threshold=0, # Not easily available here
  5028. )
  5029. except Exception:
  5030. pass # Don't fail if MQTT fails
  5031. else:
  5032. logger.info("[MAINT-BG] Completed (no items need attention)")
  5033. except Exception as e:
  5034. logger.warning("[MAINT-BG] Failed: %s", e)
  5035. spawn_background_task(_background_smart_plug(), name="background-smart-plug")
  5036. spawn_background_task(_background_maintenance_check(), name="background-maintenance-check")
  5037. # Notification task waits for photo capture to complete first (with timeout).
  5038. # When a timelapse was recording, photo sourcing polls the per-print
  5039. # timelapse for up to 60s (#1397) — extend the budget so the notification
  5040. # carries the correct bed-up photo instead of falling through to the
  5041. # live-cam grab. Adds ~30s of notification latency at worst on slow links.
  5042. photo_wait_timeout = 75 if data.get("timelapse_was_active") else 45
  5043. async def _photo_then_notify():
  5044. """Wait for photo capture, then send notification with photo URL."""
  5045. finish_photo = None
  5046. try:
  5047. finish_photo = await asyncio.wait_for(photo_task, timeout=photo_wait_timeout)
  5048. logger.info("[PHOTO-NOTIFY] Photo task returned: %s", finish_photo)
  5049. except TimeoutError:
  5050. logger.warning(
  5051. "[PHOTO-NOTIFY] Photo capture timed out after %ss, sending notification without photo",
  5052. photo_wait_timeout,
  5053. )
  5054. except Exception as e:
  5055. logger.warning("[PHOTO-NOTIFY] Photo task failed: %s", e)
  5056. try:
  5057. await _background_notifications(finish_photo)
  5058. except Exception as e:
  5059. logger.error("[PHOTO-NOTIFY] Notification sending failed: %s", e, exc_info=True)
  5060. spawn_background_task(_photo_then_notify(), name="photo-then-notify")
  5061. # Stitch external camera layer timelapse if session was active
  5062. print_status = data.get("status", "completed")
  5063. async def _background_layer_timelapse():
  5064. """Stitch layer timelapse and attach to archive."""
  5065. from backend.app.services.layer_timelapse import cancel_session, on_print_complete as tl_complete
  5066. try:
  5067. if print_status == "completed":
  5068. logger.info("[LAYER-TL] Stitching layer timelapse for printer %s", printer_id)
  5069. timelapse_path = await tl_complete(printer_id)
  5070. if timelapse_path and archive_id:
  5071. logger.info("[LAYER-TL] Attaching timelapse %s to archive %s", timelapse_path, archive_id)
  5072. async with async_session() as db:
  5073. service = ArchiveService(db)
  5074. timelapse_data = await asyncio.to_thread(timelapse_path.read_bytes)
  5075. await service.attach_timelapse(archive_id, timelapse_data, "layer_timelapse.mp4")
  5076. # Clean up the temp file
  5077. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  5078. logger.info("[LAYER-TL] Layer timelapse attached successfully")
  5079. elif timelapse_path:
  5080. # Timelapse created but no archive - just clean up
  5081. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  5082. else:
  5083. # Print failed or cancelled - cancel timelapse session
  5084. cancel_session(printer_id)
  5085. logger.info(
  5086. "[LAYER-TL] Cancelled layer timelapse for printer %s (status: %s)", printer_id, print_status
  5087. )
  5088. except Exception as e:
  5089. logger.warning("[LAYER-TL] Failed: %s", e)
  5090. # Try to cancel session on error
  5091. try:
  5092. cancel_session(printer_id)
  5093. except Exception:
  5094. pass # Best-effort timelapse session cancellation on error
  5095. spawn_background_task(_background_layer_timelapse(), name="background-layer-timelapse")
  5096. log_timing("All background tasks scheduled")
  5097. # Auto-scan for timelapse if recording was active during the print
  5098. if archive_id and data.get("timelapse_was_active") and data.get("status") == "completed":
  5099. logger.info("[TIMELAPSE] Timelapse was active during print, scheduling auto-scan for archive %s", archive_id)
  5100. # Schedule timelapse scan as background task with retries
  5101. # The printer needs time to encode the video after print completion
  5102. baseline = _timelapse_baselines.pop(printer_id, None)
  5103. spawn_background_task(
  5104. _scan_for_timelapse_with_retries(archive_id, baseline),
  5105. name=f"scan-timelapse-{archive_id}",
  5106. )
  5107. log_timing("Timelapse scan scheduled")
  5108. logger.info("[CALLBACK] on_print_complete finished for printer %s, archive %s", printer_id, archive_id)
  5109. # AMS sensor history recording
  5110. _ams_history_task: asyncio.Task | None = None
  5111. AMS_HISTORY_INTERVAL = 300 # Record every 5 minutes
  5112. AMS_HISTORY_RETENTION_DAYS = 30 # Keep data for 30 days
  5113. _ams_cleanup_counter = 0 # Track recordings to trigger periodic cleanup
  5114. # Track alarm cooldowns (printer_id:ams_id:type -> last_alarm_time)
  5115. _ams_alarm_cooldown: dict[str, datetime] = {}
  5116. AMS_ALARM_COOLDOWN_MINUTES = 60 # Don't send same alarm more than once per hour
  5117. def _ams_has_filament(ams_data: dict) -> bool:
  5118. """True if this AMS unit has at least one tray slot holding filament.
  5119. Bambu firmware reports loaded slots via `tray_exist_bits`, a per-AMS hex
  5120. bitmap (one bit per tray slot — bit set = spool present). Empty AMS units
  5121. still report sensor readings, but those readings are ambient and not
  5122. actionable: no filament to dry, no humidity to push down. #1619 — gate
  5123. humidity/temperature alarms on this check so empty units don't generate
  5124. hourly noise. Sensor history still records regardless so the UI charts
  5125. stay continuous.
  5126. Fallback path inspects the `tray` array's `tray_type` fields for setups
  5127. where `tray_exist_bits` is missing (some early-connection pushall shapes).
  5128. """
  5129. bits = ams_data.get("tray_exist_bits")
  5130. if isinstance(bits, str) and bits.strip():
  5131. try:
  5132. return int(bits, 16) > 0
  5133. except ValueError:
  5134. pass
  5135. trays = ams_data.get("tray")
  5136. if isinstance(trays, list):
  5137. return any(
  5138. isinstance(t, dict) and isinstance(t.get("tray_type"), str) and t["tray_type"].strip() for t in trays
  5139. )
  5140. return False
  5141. async def record_ams_history():
  5142. """Background task to record AMS humidity and temperature data."""
  5143. logger = logging.getLogger(__name__)
  5144. # Wait a short time for MQTT connections to establish on startup
  5145. await asyncio.sleep(10)
  5146. while True:
  5147. try:
  5148. from backend.app.models.ams_history import AMSSensorHistory
  5149. from backend.app.models.printer import Printer
  5150. from backend.app.models.settings import Settings
  5151. async with async_session() as db:
  5152. # Get all active printers
  5153. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  5154. printers = result.scalars().all()
  5155. # Get alarm thresholds from settings
  5156. humidity_threshold = 60.0 # Default: fair threshold
  5157. temp_threshold = 35.0 # Default: fair threshold
  5158. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  5159. setting = result.scalar_one_or_none()
  5160. if setting:
  5161. try:
  5162. humidity_threshold = float(setting.value)
  5163. except (ValueError, TypeError):
  5164. pass # Keep default threshold if stored value is invalid
  5165. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_fair"))
  5166. setting = result.scalar_one_or_none()
  5167. if setting:
  5168. try:
  5169. temp_threshold = float(setting.value)
  5170. except (ValueError, TypeError):
  5171. pass # Keep default threshold if stored value is invalid
  5172. # Per-filament humidity threshold overrides (#1605) — resolved
  5173. # per-AMS below from the loaded tray types. Reuses the same
  5174. # resolver as the auto-drying scheduler so behavior stays in
  5175. # lockstep across both consumers.
  5176. from backend.app.services.print_scheduler import PrintScheduler
  5177. per_type_humidity_thresholds: dict[str, int] = {}
  5178. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_thresholds"))
  5179. setting = result.scalar_one_or_none()
  5180. if setting and setting.value:
  5181. try:
  5182. raw = json.loads(setting.value)
  5183. if isinstance(raw, dict):
  5184. for k, v in raw.items():
  5185. try:
  5186. per_type_humidity_thresholds[str(k).upper() if k != "default" else "default"] = int(
  5187. v
  5188. )
  5189. except (TypeError, ValueError):
  5190. continue
  5191. except (ValueError, TypeError):
  5192. pass # Invalid JSON → no overrides, fall through to global threshold
  5193. recorded_count = 0
  5194. for printer in printers:
  5195. # Get current state from printer manager
  5196. state = printer_manager.get_status(printer.id)
  5197. if not state or not state.connected or not state.raw_data:
  5198. continue # Skip disconnected printers - don't use stale data
  5199. raw_data = state.raw_data
  5200. if "ams" not in raw_data or not isinstance(raw_data["ams"], list):
  5201. continue
  5202. # Record data for each AMS unit
  5203. for ams_data in raw_data["ams"]:
  5204. ams_id = int(ams_data.get("id", 0))
  5205. # Get humidity (prefer humidity_raw)
  5206. humidity_raw = ams_data.get("humidity_raw")
  5207. humidity_idx = ams_data.get("humidity")
  5208. humidity = None
  5209. if humidity_raw is not None:
  5210. try:
  5211. humidity = float(humidity_raw)
  5212. except (ValueError, TypeError):
  5213. pass # Skip unparseable humidity; will try fallback
  5214. if humidity is None and humidity_idx is not None:
  5215. try:
  5216. humidity = float(humidity_idx)
  5217. except (ValueError, TypeError):
  5218. pass # Skip unparseable humidity index value
  5219. # Get temperature
  5220. temperature = None
  5221. temp_str = ams_data.get("temp")
  5222. if temp_str is not None:
  5223. try:
  5224. temperature = float(temp_str)
  5225. except (ValueError, TypeError):
  5226. pass # Skip unparseable temperature value
  5227. # Skip if no data
  5228. if humidity is None and temperature is None:
  5229. continue
  5230. # Record the data point
  5231. history = AMSSensorHistory(
  5232. printer_id=printer.id,
  5233. ams_id=ams_id,
  5234. humidity=humidity,
  5235. humidity_raw=float(humidity_raw) if humidity_raw else None,
  5236. temperature=temperature,
  5237. )
  5238. db.add(history)
  5239. recorded_count += 1
  5240. # Generate AMS label and determine if it's AMS-HT (A, B, C, D or HT-A for AMS-Lite/Hub)
  5241. is_ams_ht = ams_id >= 128
  5242. if is_ams_ht:
  5243. ams_label = f"HT-{chr(65 + (ams_id - 128))}"
  5244. else:
  5245. ams_label = f"AMS-{chr(65 + ams_id)}"
  5246. # Skip alarm dispatch for empty AMS units — humidity /
  5247. # temperature readings are ambient with no filament to
  5248. # protect, and the hourly notification just becomes
  5249. # noise. Sensor history was already recorded above so
  5250. # the UI charts stay continuous (#1619). Per-AMS check
  5251. # so a multi-AMS setup with one loaded + one empty
  5252. # still alarms on the loaded unit.
  5253. if not _ams_has_filament(ams_data):
  5254. continue
  5255. # Resolve per-filament humidity threshold for this AMS
  5256. # unit (#1605). Falls back to the global ams_humidity_fair
  5257. # when no per-type overrides are configured.
  5258. trays = ams_data.get("tray", []) or []
  5259. effective_humidity_threshold = float(
  5260. PrintScheduler.resolve_humidity_threshold(
  5261. trays, per_type_humidity_thresholds, int(humidity_threshold)
  5262. )
  5263. )
  5264. # Check humidity alarm (only if above threshold)
  5265. if humidity is not None and humidity > effective_humidity_threshold:
  5266. cooldown_key = f"{printer.id}:{ams_id}:humidity"
  5267. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  5268. now = datetime.now(timezone.utc)
  5269. if (
  5270. last_alarm is None
  5271. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  5272. ):
  5273. _ams_alarm_cooldown[cooldown_key] = now
  5274. logger.info(
  5275. f"Sending humidity alarm for {printer.name} {ams_label}: {humidity}% > {effective_humidity_threshold}%"
  5276. )
  5277. try:
  5278. # Call different notification method based on AMS type
  5279. if is_ams_ht:
  5280. await notification_service.on_ams_ht_humidity_high(
  5281. printer.id,
  5282. printer.name,
  5283. ams_label,
  5284. humidity,
  5285. effective_humidity_threshold,
  5286. db,
  5287. )
  5288. else:
  5289. await notification_service.on_ams_humidity_high(
  5290. printer.id,
  5291. printer.name,
  5292. ams_label,
  5293. humidity,
  5294. effective_humidity_threshold,
  5295. db,
  5296. )
  5297. except Exception as e:
  5298. logger.warning("Failed to send humidity alarm: %s", e)
  5299. # Check temperature alarm (only if above threshold)
  5300. if temperature is not None and temperature > temp_threshold:
  5301. cooldown_key = f"{printer.id}:{ams_id}:temperature"
  5302. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  5303. now = datetime.now(timezone.utc)
  5304. if (
  5305. last_alarm is None
  5306. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  5307. ):
  5308. _ams_alarm_cooldown[cooldown_key] = now
  5309. logger.info(
  5310. f"Sending temperature alarm for {printer.name} {ams_label}: {temperature}°C > {temp_threshold}°C"
  5311. )
  5312. try:
  5313. # Call different notification method based on AMS type
  5314. if is_ams_ht:
  5315. await notification_service.on_ams_ht_temperature_high(
  5316. printer.id, printer.name, ams_label, temperature, temp_threshold, db
  5317. )
  5318. else:
  5319. await notification_service.on_ams_temperature_high(
  5320. printer.id, printer.name, ams_label, temperature, temp_threshold, db
  5321. )
  5322. except Exception as e:
  5323. logger.warning("Failed to send temperature alarm: %s", e)
  5324. await db.commit()
  5325. if recorded_count > 0:
  5326. logger.info("Recorded %s AMS sensor history entries", recorded_count)
  5327. # Periodic cleanup of old data (every ~288 recordings = ~24 hours at 5min interval)
  5328. global _ams_cleanup_counter
  5329. _ams_cleanup_counter += 1
  5330. if _ams_cleanup_counter >= 288:
  5331. _ams_cleanup_counter = 0
  5332. # Get retention days from settings
  5333. from backend.app.models.settings import Settings
  5334. result = await db.execute(select(Settings).where(Settings.key == "ams_history_retention_days"))
  5335. setting = result.scalar_one_or_none()
  5336. retention_days = int(setting.value) if setting else AMS_HISTORY_RETENTION_DAYS
  5337. cutoff = datetime.utcnow() - timedelta(days=retention_days)
  5338. result = await db.execute(delete(AMSSensorHistory).where(AMSSensorHistory.recorded_at < cutoff))
  5339. await db.commit()
  5340. if result.rowcount > 0:
  5341. logger.info(
  5342. f"Cleaned up {result.rowcount} old AMS sensor history entries (older than {retention_days} days)"
  5343. )
  5344. # Wait until next recording interval
  5345. await asyncio.sleep(AMS_HISTORY_INTERVAL)
  5346. except asyncio.CancelledError:
  5347. break
  5348. except Exception as e:
  5349. logger.warning("AMS history recording failed: %s", e)
  5350. await asyncio.sleep(60) # Wait a bit before retrying
  5351. def start_ams_history_recording():
  5352. """Start the AMS history recording background task."""
  5353. global _ams_history_task
  5354. if _ams_history_task is None:
  5355. _ams_history_task = asyncio.create_task(record_ams_history())
  5356. logging.getLogger(__name__).info("AMS history recording started")
  5357. def stop_ams_history_recording():
  5358. """Stop the AMS history recording background task."""
  5359. global _ams_history_task
  5360. if _ams_history_task:
  5361. _ams_history_task.cancel()
  5362. _ams_history_task = None
  5363. logging.getLogger(__name__).info("AMS history recording stopped")
  5364. # Printer sensor history recording (nozzle / bed / chamber)
  5365. _printer_sensor_history_task: asyncio.Task | None = None
  5366. PRINTER_SENSOR_HISTORY_INTERVAL = 60 # Record every minute — heaters move faster than AMS humidity
  5367. PRINTER_SENSOR_HISTORY_RETENTION_DAYS = 30
  5368. _printer_sensor_cleanup_counter = 0
  5369. # Sensor kinds tracked in state.temperatures — these are the normalised keys the
  5370. # MQTT parser writes, so we don't need to handle per-model field aliases here
  5371. # (nozzle_temper / left_nozzle_temper / right_nozzle_temper / chamber_temper
  5372. # are all collapsed by services/bambu_mqtt.py before they reach this loop).
  5373. _SENSOR_KINDS = ("nozzle", "nozzle_2", "bed", "chamber")
  5374. _SENSOR_TARGET_KEYS = {
  5375. "nozzle": "nozzle_target",
  5376. "nozzle_2": "nozzle_2_target",
  5377. "bed": "bed_target",
  5378. "chamber": "chamber_target",
  5379. }
  5380. async def record_printer_sensor_history():
  5381. """Background task to record nozzle / bed / chamber readings.
  5382. Pulls from `state.temperatures` (already normalised across all printer
  5383. models by the MQTT parser) rather than re-parsing raw_data, so we get
  5384. free coverage of dual-nozzle H2D, sensor-only X1C chamber, etc.
  5385. """
  5386. logger = logging.getLogger(__name__)
  5387. await asyncio.sleep(10)
  5388. while True:
  5389. try:
  5390. from backend.app.models.printer import Printer
  5391. from backend.app.models.printer_sensor_history import PrinterSensorHistory
  5392. from backend.app.models.settings import Settings
  5393. async with async_session() as db:
  5394. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  5395. printers = result.scalars().all()
  5396. recorded_count = 0
  5397. for printer in printers:
  5398. state = printer_manager.get_status(printer.id)
  5399. if not state or not state.connected:
  5400. continue
  5401. temps = getattr(state, "temperatures", None) or {}
  5402. if not isinstance(temps, dict):
  5403. continue
  5404. for kind in _SENSOR_KINDS:
  5405. if kind not in temps:
  5406. continue
  5407. try:
  5408. value = float(temps[kind])
  5409. except (ValueError, TypeError):
  5410. continue
  5411. target_raw = temps.get(_SENSOR_TARGET_KEYS[kind])
  5412. target_val: float | None = None
  5413. if target_raw is not None:
  5414. try:
  5415. target_val = float(target_raw)
  5416. except (ValueError, TypeError):
  5417. target_val = None
  5418. db.add(
  5419. PrinterSensorHistory(
  5420. printer_id=printer.id,
  5421. sensor_kind=kind,
  5422. value=value,
  5423. target=target_val,
  5424. )
  5425. )
  5426. recorded_count += 1
  5427. await db.commit()
  5428. if recorded_count > 0:
  5429. logger.debug("Recorded %s printer sensor history entries", recorded_count)
  5430. # Periodic cleanup — once every ~24h at this interval.
  5431. global _printer_sensor_cleanup_counter
  5432. _printer_sensor_cleanup_counter += 1
  5433. cleanup_every = max(1, (24 * 60 * 60) // PRINTER_SENSOR_HISTORY_INTERVAL)
  5434. if _printer_sensor_cleanup_counter >= cleanup_every:
  5435. _printer_sensor_cleanup_counter = 0
  5436. result = await db.execute(
  5437. select(Settings).where(Settings.key == "printer_sensor_history_retention_days")
  5438. )
  5439. setting = result.scalar_one_or_none()
  5440. retention_days = int(setting.value) if setting else PRINTER_SENSOR_HISTORY_RETENTION_DAYS
  5441. cutoff = datetime.utcnow() - timedelta(days=retention_days)
  5442. cleanup = await db.execute(
  5443. delete(PrinterSensorHistory).where(PrinterSensorHistory.recorded_at < cutoff)
  5444. )
  5445. await db.commit()
  5446. if cleanup.rowcount > 0:
  5447. logger.info(
  5448. "Cleaned up %s old printer sensor history entries (older than %s days)",
  5449. cleanup.rowcount,
  5450. retention_days,
  5451. )
  5452. await asyncio.sleep(PRINTER_SENSOR_HISTORY_INTERVAL)
  5453. except asyncio.CancelledError:
  5454. break
  5455. except Exception as e:
  5456. logger.warning("Printer sensor history recording failed: %s", e)
  5457. await asyncio.sleep(60)
  5458. def start_printer_sensor_history_recording():
  5459. global _printer_sensor_history_task
  5460. if _printer_sensor_history_task is None:
  5461. _printer_sensor_history_task = asyncio.create_task(record_printer_sensor_history())
  5462. logging.getLogger(__name__).info("Printer sensor history recording started")
  5463. def stop_printer_sensor_history_recording():
  5464. global _printer_sensor_history_task
  5465. if _printer_sensor_history_task:
  5466. _printer_sensor_history_task.cancel()
  5467. _printer_sensor_history_task = None
  5468. logging.getLogger(__name__).info("Printer sensor history recording stopped")
  5469. # Printer runtime tracking
  5470. _runtime_tracking_task: asyncio.Task | None = None
  5471. RUNTIME_TRACKING_INTERVAL = 30 # Update every 30 seconds
  5472. async def track_printer_runtime():
  5473. """Background task to track printer active runtime (RUNNING state only).
  5474. PAUSE is intentionally excluded — the runtime counter feeds hours-based
  5475. maintenance intervals (rod lubrication, belt checks, nozzle cleaning)
  5476. which track mechanical wear. Pause time has no motion and no wear, so
  5477. counting it inflates maintenance warnings (#1521).
  5478. """
  5479. logger = logging.getLogger(__name__)
  5480. # Wait for MQTT connections to establish on startup
  5481. await asyncio.sleep(15)
  5482. while True:
  5483. try:
  5484. from backend.app.models.printer import Printer
  5485. # Fetch printer IDs in a short-lived read-only session
  5486. async with async_session() as db:
  5487. result = await db.execute(
  5488. select(Printer.id, Printer.name, Printer.runtime_seconds, Printer.last_runtime_update).where(
  5489. Printer.is_active.is_(True)
  5490. )
  5491. )
  5492. printer_rows = result.all()
  5493. now = datetime.now(timezone.utc)
  5494. updated_count = 0
  5495. # Update each printer in its own short session to minimise write-lock
  5496. # hold time and avoid blocking critical commits like queue status
  5497. # updates (#897).
  5498. for pid, pname, runtime_secs, last_update in printer_rows:
  5499. state = printer_manager.get_status(pid)
  5500. if not state:
  5501. logger.debug("[%s] Runtime tracking: no state available", pname)
  5502. continue
  5503. if not state.connected:
  5504. logger.debug("[%s] Runtime tracking: not connected", pname)
  5505. continue
  5506. needs_commit = False
  5507. new_runtime = runtime_secs
  5508. new_last_update = last_update
  5509. if state.state == "RUNNING":
  5510. if last_update:
  5511. lu = last_update if last_update.tzinfo else last_update.replace(tzinfo=timezone.utc)
  5512. elapsed = (now - lu).total_seconds()
  5513. if elapsed > 0:
  5514. new_runtime = runtime_secs + int(elapsed)
  5515. updated_count += 1
  5516. needs_commit = True
  5517. logger.debug(
  5518. f"[{pname}] Runtime tracking: added {int(elapsed)}s, "
  5519. f"total={new_runtime}s ({new_runtime / 3600:.2f}h)"
  5520. )
  5521. else:
  5522. needs_commit = True
  5523. logger.debug("[%s] Runtime tracking: first active detection", pname)
  5524. new_last_update = now
  5525. else:
  5526. if last_update is not None:
  5527. logger.debug(f"[{pname}] Runtime tracking: state={state.state}, clearing last_runtime_update")
  5528. new_last_update = None
  5529. needs_commit = True
  5530. if needs_commit:
  5531. try:
  5532. async with async_session() as db:
  5533. result = await db.execute(select(Printer).where(Printer.id == pid))
  5534. printer = result.scalar_one_or_none()
  5535. if printer:
  5536. printer.runtime_seconds = new_runtime
  5537. printer.last_runtime_update = new_last_update
  5538. await db.commit()
  5539. except Exception as e:
  5540. logger.warning("[%s] Runtime tracking commit failed: %s", pname, e)
  5541. if updated_count > 0:
  5542. logger.debug("Updated runtime for %s printer(s)", updated_count)
  5543. except asyncio.CancelledError:
  5544. logger.info("Runtime tracking cancelled")
  5545. break
  5546. except Exception as e:
  5547. logger.warning("Runtime tracking failed: %s", e)
  5548. await asyncio.sleep(RUNTIME_TRACKING_INTERVAL)
  5549. def start_runtime_tracking():
  5550. """Start the printer runtime tracking background task."""
  5551. global _runtime_tracking_task
  5552. if _runtime_tracking_task is None:
  5553. _runtime_tracking_task = asyncio.create_task(track_printer_runtime())
  5554. logging.getLogger(__name__).info("Printer runtime tracking started")
  5555. def stop_runtime_tracking():
  5556. """Stop the printer runtime tracking background task."""
  5557. global _runtime_tracking_task
  5558. if _runtime_tracking_task:
  5559. _runtime_tracking_task.cancel()
  5560. _runtime_tracking_task = None
  5561. logging.getLogger(__name__).info("Printer runtime tracking stopped")
  5562. # SpoolBuddy device watchdog
  5563. _spoolbuddy_watchdog_task: asyncio.Task | None = None
  5564. SPOOLBUDDY_WATCHDOG_INTERVAL = 15
  5565. async def _spoolbuddy_watchdog_loop():
  5566. """Periodic check for SpoolBuddy devices that have gone offline."""
  5567. from backend.app.api.routes.spoolbuddy import spoolbuddy_watchdog
  5568. while True:
  5569. try:
  5570. await spoolbuddy_watchdog()
  5571. except asyncio.CancelledError:
  5572. break
  5573. except Exception as e:
  5574. logging.getLogger(__name__).warning("SpoolBuddy watchdog failed: %s", e)
  5575. await asyncio.sleep(SPOOLBUDDY_WATCHDOG_INTERVAL)
  5576. def start_spoolbuddy_watchdog():
  5577. global _spoolbuddy_watchdog_task
  5578. if _spoolbuddy_watchdog_task is None:
  5579. _spoolbuddy_watchdog_task = asyncio.create_task(_spoolbuddy_watchdog_loop())
  5580. logging.getLogger(__name__).info("SpoolBuddy watchdog started")
  5581. def stop_spoolbuddy_watchdog():
  5582. global _spoolbuddy_watchdog_task
  5583. if _spoolbuddy_watchdog_task:
  5584. _spoolbuddy_watchdog_task.cancel()
  5585. _spoolbuddy_watchdog_task = None
  5586. logging.getLogger(__name__).info("SpoolBuddy watchdog stopped")
  5587. # Camera stream orphan cleanup
  5588. _camera_cleanup_task: asyncio.Task | None = None
  5589. CAMERA_CLEANUP_INTERVAL = 60
  5590. async def _camera_cleanup_loop():
  5591. """Periodically clean up orphaned ffmpeg processes."""
  5592. from backend.app.api.routes.camera import cleanup_orphaned_streams
  5593. while True:
  5594. try:
  5595. await cleanup_orphaned_streams()
  5596. except asyncio.CancelledError:
  5597. break
  5598. except Exception as e:
  5599. logging.getLogger(__name__).warning("Camera stream cleanup failed: %s", e)
  5600. await asyncio.sleep(CAMERA_CLEANUP_INTERVAL)
  5601. def start_camera_cleanup():
  5602. global _camera_cleanup_task
  5603. if _camera_cleanup_task is None:
  5604. _camera_cleanup_task = asyncio.create_task(_camera_cleanup_loop())
  5605. logging.getLogger(__name__).info("Camera stream cleanup started")
  5606. def stop_camera_cleanup():
  5607. global _camera_cleanup_task
  5608. if _camera_cleanup_task:
  5609. _camera_cleanup_task.cancel()
  5610. _camera_cleanup_task = None
  5611. logging.getLogger(__name__).info("Camera stream cleanup stopped")
  5612. # ---------------------------------------------------------------------------
  5613. # Expected-print TTL eviction
  5614. # ---------------------------------------------------------------------------
  5615. def _evict_stale_expected_prints() -> None:
  5616. """Remove entries from _expected_prints / _expected_print_creators that are
  5617. older than _EXPECTED_PRINT_TTL_SECONDS.
  5618. This prevents unbounded growth when a print is registered (via
  5619. register_expected_print) but on_print_start never fires — e.g. because the
  5620. printer disconnects, the app restarts, or the print is started directly from
  5621. the printer panel without going through the queue.
  5622. """
  5623. # Use monotonic time so the TTL is unaffected by system clock adjustments
  5624. # (e.g. NTP sync, DST changes).
  5625. cutoff = time.monotonic() - _EXPECTED_PRINT_TTL_SECONDS
  5626. stale_keys = [k for k, t in _expected_print_registered_at.items() if t < cutoff]
  5627. if not stale_keys:
  5628. return
  5629. evicted_archive_ids: set[int] = set()
  5630. for key in stale_keys:
  5631. archive_id = _expected_prints.pop(key, None)
  5632. if archive_id is not None:
  5633. evicted_archive_ids.add(archive_id)
  5634. _expected_print_creators.pop(key, None)
  5635. _expected_print_registered_at.pop(key, None)
  5636. # Also clean up _print_ams_mappings and _print_plate_ids for archive_ids
  5637. # that have no remaining live keys in _expected_prints (all variants
  5638. # were just evicted).
  5639. live_archive_ids = set(_expected_prints.values())
  5640. for archive_id in evicted_archive_ids:
  5641. if archive_id not in live_archive_ids:
  5642. _print_ams_mappings.pop(archive_id, None)
  5643. _print_plate_ids.pop(archive_id, None)
  5644. logging.getLogger(__name__).info(
  5645. "Evicted %d stale expected-print entries (TTL=%ds)", len(stale_keys), _EXPECTED_PRINT_TTL_SECONDS
  5646. )
  5647. async def _expected_prints_cleanup_loop() -> None:
  5648. """Background task: periodically evict stale expected-print entries."""
  5649. while True:
  5650. try:
  5651. _evict_stale_expected_prints()
  5652. except asyncio.CancelledError:
  5653. raise
  5654. except Exception as e:
  5655. logging.getLogger(__name__).warning("Expected prints cleanup failed: %s", e)
  5656. await asyncio.sleep(_EXPECTED_PRINT_CLEANUP_INTERVAL)
  5657. def start_expected_prints_cleanup() -> None:
  5658. global _expected_prints_cleanup_task
  5659. if _expected_prints_cleanup_task is None:
  5660. _expected_prints_cleanup_task = asyncio.create_task(_expected_prints_cleanup_loop())
  5661. logging.getLogger(__name__).info("Expected prints cleanup started")
  5662. def stop_expected_prints_cleanup() -> None:
  5663. global _expected_prints_cleanup_task
  5664. if _expected_prints_cleanup_task:
  5665. _expected_prints_cleanup_task.cancel()
  5666. _expected_prints_cleanup_task = None
  5667. logging.getLogger(__name__).info("Expected prints cleanup stopped")
  5668. # ---------------------------------------------------------------------------
  5669. # L-2: Periodic auth-token cleanup (stale TOTP + expired revoked JTIs)
  5670. # ---------------------------------------------------------------------------
  5671. _auth_cleanup_task: asyncio.Task | None = None
  5672. _AUTH_CLEANUP_INTERVAL = 3600 # seconds (hourly)
  5673. async def _run_auth_cleanup() -> None:
  5674. """Single cleanup pass: remove stale TOTP records, expired revoked JTIs, and old rate-limit events."""
  5675. from backend.app.core.database import async_session
  5676. from backend.app.models.auth_ephemeral import AuthEphemeralToken, AuthRateLimitEvent
  5677. from backend.app.models.user_totp import UserTOTP
  5678. now = datetime.now(timezone.utc)
  5679. # Remove unconfirmed (is_enabled=False) TOTP records older than 1 hour.
  5680. try:
  5681. async with async_session() as db:
  5682. stale_cutoff = now - timedelta(hours=1)
  5683. result = await db.execute(
  5684. select(UserTOTP).where(
  5685. UserTOTP.is_enabled.is_(False),
  5686. UserTOTP.created_at < stale_cutoff,
  5687. )
  5688. )
  5689. stale_records = result.scalars().all()
  5690. if stale_records:
  5691. for rec in stale_records:
  5692. await db.delete(rec)
  5693. await db.commit()
  5694. logging.info("Auth cleanup: removed %d stale unconfirmed TOTP record(s)", len(stale_records))
  5695. except Exception as e:
  5696. logging.warning("Auth cleanup: failed to purge stale TOTP records: %s", e)
  5697. # Remove expired revoked-JTI entries (they are no longer needed once the
  5698. # original token's exp has passed — the token would be rejected by JWT
  5699. # signature verification regardless).
  5700. try:
  5701. async with async_session() as db:
  5702. await db.execute(
  5703. delete(AuthEphemeralToken).where(
  5704. AuthEphemeralToken.token_type == "revoked_jti",
  5705. AuthEphemeralToken.expires_at < now,
  5706. )
  5707. )
  5708. await db.commit()
  5709. except Exception as e:
  5710. logging.warning("Auth cleanup: failed to purge expired revoked JTIs: %s", e)
  5711. # L-R6-B: Purge AuthRateLimitEvent rows older than the lockout window (15 min).
  5712. # Events outside this window can never affect rate-limit decisions — they only
  5713. # consume DB space. Use the same window constant as the rate limiter so the
  5714. # two are always in sync.
  5715. try:
  5716. from backend.app.api.routes.mfa import LOCKOUT_WINDOW
  5717. async with async_session() as db:
  5718. await db.execute(
  5719. delete(AuthRateLimitEvent).where(
  5720. AuthRateLimitEvent.occurred_at < now - LOCKOUT_WINDOW,
  5721. )
  5722. )
  5723. await db.commit()
  5724. except Exception as e:
  5725. logging.warning("Auth cleanup: failed to purge stale rate-limit events: %s", e)
  5726. async def _auth_cleanup_loop() -> None:
  5727. """Periodic background task: run auth cleanup every hour."""
  5728. while True:
  5729. try:
  5730. await _run_auth_cleanup()
  5731. except asyncio.CancelledError:
  5732. break
  5733. except Exception as e:
  5734. logging.warning("Auth cleanup loop error: %s", e)
  5735. await asyncio.sleep(_AUTH_CLEANUP_INTERVAL)
  5736. def start_auth_cleanup() -> None:
  5737. global _auth_cleanup_task
  5738. if _auth_cleanup_task is None:
  5739. _auth_cleanup_task = asyncio.create_task(_auth_cleanup_loop())
  5740. logging.getLogger(__name__).info("Auth periodic cleanup started")
  5741. def stop_auth_cleanup() -> None:
  5742. global _auth_cleanup_task
  5743. if _auth_cleanup_task:
  5744. _auth_cleanup_task.cancel()
  5745. _auth_cleanup_task = None
  5746. logging.getLogger(__name__).info("Auth periodic cleanup stopped")
  5747. @asynccontextmanager
  5748. async def lifespan(app: FastAPI):
  5749. # Startup
  5750. # Install Windows-only asyncio Proactor cleanup-RST filter (#1113) before
  5751. # anything else can spawn tasks that might trip it.
  5752. from backend.app.core.asyncio_handlers import install_proactor_reset_filter
  5753. install_proactor_reset_filter()
  5754. await init_db()
  5755. # Register an app-scoped httpx client for Bambu Cloud services so
  5756. # per-request BambuCloudService instances reuse the same connection pool
  5757. # (important for routes like /cloud/filament-info that chain many
  5758. # get_setting_detail calls). The shared client stores no region/token
  5759. # state, so the per-request ownership pattern that fixed the region-bleed
  5760. # bug is preserved.
  5761. import httpx as _httpx
  5762. from backend.app.services.bambu_cloud import set_shared_http_client
  5763. from backend.app.services.makerworld import (
  5764. set_shared_http_client as set_shared_makerworld_http_client,
  5765. )
  5766. from backend.app.services.orca_cloud import (
  5767. set_shared_http_client as set_shared_orca_http_client,
  5768. )
  5769. _shared_cloud_http_client = _httpx.AsyncClient(timeout=30.0)
  5770. set_shared_http_client(_shared_cloud_http_client)
  5771. # Reuse the same connection pool for MakerWorld — different host, same
  5772. # keep-alive pool saves a TLS handshake per request.
  5773. set_shared_makerworld_http_client(_shared_cloud_http_client)
  5774. # Same for Orca Cloud — without this the per-request OrcaCloudService()
  5775. # each spun up (and never closed) its own client, leaking sockets.
  5776. set_shared_orca_http_client(_shared_cloud_http_client)
  5777. # Fix queue items stuck with invalid "aborted" status (should be "cancelled").
  5778. # This can happen when a print was cancelled mid-print on versions before this fix.
  5779. try:
  5780. async with async_session() as db:
  5781. from backend.app.models.print_queue import PrintQueueItem
  5782. result = await db.execute(select(PrintQueueItem).where(PrintQueueItem.status == "aborted"))
  5783. aborted_items = result.scalars().all()
  5784. if aborted_items:
  5785. for item in aborted_items:
  5786. item.status = "cancelled"
  5787. await db.commit()
  5788. logging.info("Fixed %d queue item(s) with invalid 'aborted' status → 'cancelled'", len(aborted_items))
  5789. except Exception as e:
  5790. logging.warning("Failed to fix aborted queue items: %s", e)
  5791. # Restore debug logging state from previous session
  5792. await init_debug_logging()
  5793. # Set up printer manager callbacks
  5794. loop = asyncio.get_event_loop()
  5795. printer_manager.set_event_loop(loop)
  5796. printer_manager.set_status_change_callback(on_printer_status_change)
  5797. printer_manager.set_print_start_callback(on_print_start)
  5798. printer_manager.set_print_complete_callback(on_print_complete)
  5799. printer_manager.set_print_running_observed_callback(on_print_running_observed)
  5800. printer_manager.set_finish_photo_moment_callback(on_finish_photo_moment)
  5801. printer_manager.set_ams_change_callback(on_ams_change)
  5802. # Rehydrate persisted awaiting-plate-clear gate (#961) so prompts survive restarts
  5803. await printer_manager.load_awaiting_plate_clear_from_db()
  5804. # Layer change callback for external camera timelapse
  5805. async def on_layer_change(printer_id: int, layer_num: int):
  5806. """Capture timelapse frame on layer change + first layer notification."""
  5807. from backend.app.services.layer_timelapse import on_layer_change as tl_layer_change
  5808. await tl_layer_change(printer_id, layer_num)
  5809. # #1867: bank a recent in-print frame so the FINISH-state finish-photo
  5810. # path (firmware that never emits stg_cur=22, e.g. A1 Mini) has a
  5811. # pre-swap image to fall back on instead of a live grab of the swapped
  5812. # plate. Layer-driven, so it freezes at the final object layer.
  5813. await _maybe_bank_inprint_frame(printer_id, layer_num)
  5814. # First layer complete notification (layer_num >= 2 means layer 1 is done).
  5815. # Gate on actual printing state — Bambu firmware ticks layer_num during
  5816. # the pre-print calibration sequence (homing / mesh-level / bed scan /
  5817. # nozzle clean), so a bare layer_num check can fire minutes before the
  5818. # first real extrusion. We require gcode_state == RUNNING and
  5819. # mc_print_sub_stage in (0 = "Printing", None) so calibration sub-stages
  5820. # (1, 9, 14, ...) are excluded. The window widens to [2, 10] because if
  5821. # the layer counter advanced past 2 during PREPARE, the next on_layer_change
  5822. # edge fires later; _first_layer_notified stays clear until we actually send
  5823. # so a deferred re-evaluation can win. See issue #1837.
  5824. if 2 <= layer_num <= 10 and not _first_layer_notified.get(printer_id, False):
  5825. client = printer_manager.get_client(printer_id)
  5826. state = client.state if client else None
  5827. if not state or state.state != "RUNNING":
  5828. return
  5829. if state.mc_print_sub_stage not in (None, 0):
  5830. return
  5831. _first_layer_notified[printer_id] = True
  5832. try:
  5833. async with async_session() as db:
  5834. from backend.app.models.printer import Printer
  5835. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  5836. printer = result.scalar_one_or_none()
  5837. if not printer:
  5838. return
  5839. printer_name = printer.name
  5840. filename = (state.subtask_name or state.gcode_file or "Unknown") if state else "Unknown"
  5841. total_layers = state.total_layers if state else 0
  5842. image_data = await _capture_snapshot_for_notification(
  5843. printer_id, printer, logging.getLogger(__name__)
  5844. )
  5845. await notification_service.on_first_layer_complete(
  5846. printer_id, printer_name, filename, total_layers, db, image_data=image_data
  5847. )
  5848. except Exception as e:
  5849. logging.getLogger(__name__).warning("First layer notification failed: %s", e)
  5850. printer_manager.set_layer_change_callback(on_layer_change)
  5851. # Event-driven bed cooldown: fires whenever bed_temper arrives via MQTT
  5852. async def on_bed_temp_update(printer_id: int, bed_temp: float):
  5853. waiter = _bed_cool_waiters.get(printer_id)
  5854. if not waiter:
  5855. return
  5856. threshold = waiter["threshold"]
  5857. if bed_temp > threshold:
  5858. return
  5859. # Bed is at or below threshold — fire notification and remove waiter
  5860. waiter_info = _bed_cool_waiters.pop(printer_id, None)
  5861. if not waiter_info:
  5862. return # Another callback already handled it
  5863. bed_cool_logger = logging.getLogger(__name__)
  5864. bed_cool_logger.info(
  5865. "[BED-COOL] Bed cooled to %.1f°C on printer %s (threshold: %.0f°C)",
  5866. bed_temp,
  5867. printer_id,
  5868. threshold,
  5869. )
  5870. try:
  5871. printer_info = printer_manager.get_printer(printer_id)
  5872. p_name = printer_info.name if printer_info else "Unknown"
  5873. async with async_session() as db:
  5874. await notification_service.on_bed_cooled(
  5875. printer_id=printer_id,
  5876. printer_name=p_name,
  5877. bed_temp=bed_temp,
  5878. threshold=threshold,
  5879. filename=waiter_info["filename"],
  5880. db=db,
  5881. )
  5882. except Exception as e:
  5883. bed_cool_logger.warning("[BED-COOL] Failed to send notification: %s", e)
  5884. printer_manager.set_bed_temp_update_callback(on_bed_temp_update)
  5885. async def on_drying_complete(printer_id: int, ams_id: int):
  5886. """Smart-plug auto-off-after-drying trigger (#1349).
  5887. Fires once per AMS unit when ``dry_time`` falls from >0 to 0. The
  5888. manager walks all plugs linked to this printer and turns off only
  5889. the ones with ``auto_off_after_drying`` enabled, after their
  5890. per-plug delay. Multiple AMS units finishing close together (e.g. a
  5891. dual-AMS dry that ends within the same MQTT push) call this once
  5892. per unit — the manager's ``_cancel_pending_off`` collapses
  5893. repeated scheduling on the same plug to one timer, so duplicate
  5894. fires are safe.
  5895. """
  5896. try:
  5897. async with async_session() as db:
  5898. await smart_plug_manager.on_drying_complete(printer_id, db)
  5899. except Exception as e:
  5900. logging.getLogger(__name__).warning(
  5901. "Failed to schedule auto-off-after-drying for printer %d (AMS %d): %s",
  5902. printer_id,
  5903. ams_id,
  5904. e,
  5905. )
  5906. printer_manager.set_drying_complete_callback(on_drying_complete)
  5907. async def on_assignment_verified(printer_id: int, ams_id: int, tray_id: int, verified: bool, detail: dict):
  5908. """Surface the read-back result of a spool assignment to the UI (#2582).
  5909. The MQTT client confirms (or fails to confirm) that the tray telemetry
  5910. echoed back the filament id we pushed. We relay that as a websocket
  5911. event so the frontend can toast "loaded" / "assignment didn't take"
  5912. instead of the historic silent fire-and-forget, which made the
  5913. AMS→Studio hand-off feel random to users.
  5914. """
  5915. try:
  5916. from backend.app.services.spool_assignment_notifications import (
  5917. _slot_label_from_global_tray,
  5918. )
  5919. if ams_id == 255:
  5920. global_id = 254 + tray_id
  5921. elif ams_id >= 128:
  5922. global_id = ams_id
  5923. else:
  5924. global_id = ams_id * 4 + tray_id
  5925. slot_label = _slot_label_from_global_tray(global_id)
  5926. printer_info = printer_manager.get_printer(printer_id)
  5927. printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  5928. await ws_manager.broadcast(
  5929. {
  5930. "type": "spool_assignment_verified",
  5931. "printer_id": printer_id,
  5932. "printer_name": printer_name,
  5933. "ams_id": ams_id,
  5934. "tray_id": tray_id,
  5935. "slot": slot_label,
  5936. "verified": verified,
  5937. # Present on success: False means the filament setting landed
  5938. # but the K-profile (cali_idx) did not — the reporter's exact
  5939. # "loaded but no flow profile" symptom.
  5940. "kprofile_applied": detail.get("kprofile_applied", True),
  5941. # Present on failure: whether any tray telemetry was seen in
  5942. # the window (distinguishes "printer silent" from "printer
  5943. # stored something else").
  5944. "saw_tray": detail.get("saw_tray", False),
  5945. }
  5946. )
  5947. except Exception as e:
  5948. logging.getLogger(__name__).warning(
  5949. "Failed to broadcast assignment verification for printer %d AMS%d-T%d: %s",
  5950. printer_id,
  5951. ams_id,
  5952. tray_id,
  5953. e,
  5954. )
  5955. printer_manager.set_assignment_verified_callback(on_assignment_verified)
  5956. # Initialize MQTT relay from settings
  5957. async with async_session() as db:
  5958. from backend.app.api.routes.settings import get_setting
  5959. mqtt_settings = {
  5960. "mqtt_enabled": (await get_setting(db, "mqtt_enabled") or "false") == "true",
  5961. "mqtt_broker": await get_setting(db, "mqtt_broker") or "",
  5962. "mqtt_port": int(await get_setting(db, "mqtt_port") or "1883"),
  5963. "mqtt_username": await get_setting(db, "mqtt_username") or "",
  5964. "mqtt_password": await get_setting(db, "mqtt_password") or "",
  5965. "mqtt_topic_prefix": await get_setting(db, "mqtt_topic_prefix") or "bambuddy",
  5966. "mqtt_use_tls": (await get_setting(db, "mqtt_use_tls") or "false") == "true",
  5967. }
  5968. await mqtt_relay.configure(mqtt_settings)
  5969. # Restore MQTT smart plug subscriptions
  5970. if mqtt_settings.get("mqtt_enabled"):
  5971. from backend.app.models.smart_plug import SmartPlug
  5972. from backend.app.services.mqtt_smart_plug import subscribe_plug_to_mqtt
  5973. result = await db.execute(select(SmartPlug).where(SmartPlug.plug_type == "mqtt"))
  5974. mqtt_plugs = result.scalars().all()
  5975. restored = 0
  5976. for plug in mqtt_plugs:
  5977. if subscribe_plug_to_mqtt(mqtt_relay.smart_plug_service, plug):
  5978. restored += 1
  5979. if restored:
  5980. logging.info("Restored %s MQTT smart plug subscriptions", restored)
  5981. # Connect to all active printers
  5982. async with async_session() as db:
  5983. await init_printer_connections(db)
  5984. # Auto-connect to Spoolman if enabled
  5985. async with async_session() as db:
  5986. from backend.app.api.routes.settings import get_setting
  5987. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  5988. spoolman_url = await get_setting(db, "spoolman_url")
  5989. if spoolman_enabled and spoolman_enabled.lower() == "true" and spoolman_url:
  5990. try:
  5991. client = await init_spoolman_client(spoolman_url)
  5992. if await client.health_check():
  5993. logging.info("Auto-connected to Spoolman at %s", spoolman_url)
  5994. # Ensure the 'tag' extra field exists for RFID/UUID storage
  5995. field_ok = await client.ensure_tag_extra_field()
  5996. if not field_ok:
  5997. logging.error("Spoolman tag extra field registration failed — NFC tag links may not persist")
  5998. # Register the BambuStudio slicer-preset fields used by the
  5999. # spool-edit / assign flow. Spoolman rejects PATCHes with
  6000. # unknown extra keys, so these must exist before any update
  6001. # that touches them.
  6002. for field_name in ("bambu_slicer_filament", "bambu_slicer_filament_name"):
  6003. if not await client.ensure_extra_field(field_name):
  6004. logging.warning(
  6005. "Spoolman extra field %r registration failed — "
  6006. "spool slicer-preset edits will return 502",
  6007. field_name,
  6008. )
  6009. else:
  6010. logging.warning("Spoolman at %s is not reachable", spoolman_url)
  6011. except Exception as e:
  6012. logging.warning("Failed to auto-connect to Spoolman: %s", e)
  6013. # Start the print scheduler
  6014. spawn_background_task(print_scheduler.run(), name="print-scheduler")
  6015. # Start the smart plug scheduler for time-based on/off
  6016. smart_plug_manager.start_scheduler()
  6017. # Resume any pending auto-offs that were interrupted by restart
  6018. await smart_plug_manager.resume_pending_auto_offs()
  6019. # Start the notification digest scheduler
  6020. notification_service.start_digest_scheduler()
  6021. # Start the GitHub backup scheduler
  6022. await github_backup_service.start_scheduler()
  6023. # Start the local backup scheduler
  6024. await local_backup_service.start_scheduler()
  6025. await obico_detection_service.start()
  6026. # Start the library trash sweeper (#1008)
  6027. await library_trash_service.start_scheduler()
  6028. # Start the archive auto-purge sweeper (#1008 follow-up)
  6029. await archive_purge_service.start_scheduler()
  6030. # Start AMS history recording
  6031. start_ams_history_recording()
  6032. # Start printer sensor (nozzle / bed / chamber) history recording
  6033. start_printer_sensor_history_recording()
  6034. # Start printer runtime tracking
  6035. start_runtime_tracking()
  6036. # Start SpoolBuddy device watchdog
  6037. start_spoolbuddy_watchdog()
  6038. # Start camera stream orphan cleanup
  6039. start_camera_cleanup()
  6040. # One-shot sweep for timelapse session directories orphaned by a crash
  6041. # or restart that happened mid-print (in-memory session tracking can't
  6042. # survive that, and nothing else reaps the leftover frames/output file)
  6043. try:
  6044. from backend.app.services.layer_timelapse import cleanup_orphaned_timelapse_sessions
  6045. removed = cleanup_orphaned_timelapse_sessions()
  6046. if removed:
  6047. logging.getLogger(__name__).info("Removed %d orphaned timelapse session artifact(s)", removed)
  6048. except Exception as e:
  6049. logging.getLogger(__name__).warning("Orphaned timelapse session cleanup failed: %s", e)
  6050. # Start expected-print TTL eviction (prevents memory leak when prints are
  6051. # registered but on_print_start never fires)
  6052. start_expected_prints_cleanup()
  6053. # L-2: Start periodic auth cleanup (stale TOTP + expired revoked JTIs)
  6054. start_auth_cleanup()
  6055. # Event-loop stall watchdog: dumps all thread stacks to stderr if the loop
  6056. # freezes (#1486 — silent "container hangs after adding a printer" reports).
  6057. from backend.app.services.loop_watchdog import start_loop_watchdog
  6058. start_loop_watchdog()
  6059. # Initialize virtual printer manager and sync from DB
  6060. from backend.app.services.virtual_printer import virtual_printer_manager
  6061. virtual_printer_manager.set_session_factory(async_session)
  6062. virtual_printer_manager.set_printer_manager(printer_manager)
  6063. try:
  6064. await virtual_printer_manager.sync_from_db()
  6065. logging.info("Virtual printer manager synced from database")
  6066. except Exception as e:
  6067. logging.warning("Failed to sync virtual printers: %s", e)
  6068. yield
  6069. # Shutdown
  6070. print_scheduler.stop()
  6071. smart_plug_manager.stop_scheduler()
  6072. notification_service.stop_digest_scheduler()
  6073. github_backup_service.stop_scheduler()
  6074. local_backup_service.stop_scheduler()
  6075. library_trash_service.stop_scheduler()
  6076. archive_purge_service.stop_scheduler()
  6077. obico_detection_service.stop()
  6078. stop_ams_history_recording()
  6079. stop_printer_sensor_history_recording()
  6080. stop_runtime_tracking()
  6081. stop_spoolbuddy_watchdog()
  6082. stop_camera_cleanup()
  6083. from backend.app.services.loop_watchdog import stop_loop_watchdog
  6084. stop_loop_watchdog()
  6085. # Tear down all camera fan-out broadcasters (#1089) so subscribers exit
  6086. # cleanly rather than waiting on a queue that nothing will ever fill.
  6087. try:
  6088. from backend.app.services.camera_fanout import shutdown_all_broadcasters
  6089. await shutdown_all_broadcasters()
  6090. except Exception as e:
  6091. logging.warning("Failed to shut down camera broadcasters: %s", e)
  6092. stop_expected_prints_cleanup()
  6093. stop_auth_cleanup()
  6094. printer_manager.disconnect_all()
  6095. await close_spoolman_client()
  6096. # Stop all virtual printer services
  6097. await virtual_printer_manager.stop_all()
  6098. await mqtt_smart_plug_service.disconnect(timeout=2)
  6099. await mqtt_relay.disconnect(timeout=2)
  6100. # Drop the shared Bambu Cloud HTTP client we registered at startup.
  6101. set_shared_http_client(None)
  6102. set_shared_makerworld_http_client(None)
  6103. set_shared_orca_http_client(None)
  6104. await _shared_cloud_http_client.aclose()
  6105. # Checkpoint WAL (SQLite only) and close all database connections
  6106. from backend.app.core.db_dialect import is_sqlite
  6107. if is_sqlite():
  6108. try:
  6109. async with engine.begin() as conn:
  6110. await conn.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
  6111. logging.info("WAL checkpoint completed")
  6112. except Exception as e:
  6113. logging.warning("WAL checkpoint failed: %s", e)
  6114. await engine.dispose()
  6115. app = FastAPI(
  6116. title=app_settings.app_name,
  6117. description="Archive and manage Bambu Lab 3MF files",
  6118. version=APP_VERSION,
  6119. lifespan=lifespan,
  6120. )
  6121. # =============================================================================
  6122. # Authentication Middleware - Secures ALL API routes by default
  6123. # =============================================================================
  6124. # Public routes that don't require authentication even when auth is enabled
  6125. PUBLIC_API_ROUTES = {
  6126. # Auth routes needed before/during login
  6127. "/api/v1/auth/status",
  6128. "/api/v1/auth/login",
  6129. "/api/v1/auth/setup", # Needed for initial setup and recovery
  6130. # Advanced auth status needed for login page
  6131. "/api/v1/auth/advanced-auth/status",
  6132. "/api/v1/auth/forgot-password", # Password reset for advanced auth
  6133. "/api/v1/auth/forgot-password/confirm", # Complete password reset with token (H-6)
  6134. # 2FA routes that are called BEFORE a JWT is issued (pre-auth flow)
  6135. "/api/v1/auth/2fa/verify", # Exchange pre_auth_token + 2FA code for JWT
  6136. "/api/v1/auth/2fa/email/send", # Send OTP email (pre_auth_token based)
  6137. # OIDC routes that must be reachable without a JWT
  6138. "/api/v1/auth/oidc/providers", # Public list of enabled providers
  6139. "/api/v1/auth/oidc/callback", # Redirect target from OIDC provider
  6140. "/api/v1/auth/oidc/exchange", # Exchange short-lived OIDC token for JWT
  6141. # Version check for updates (no sensitive data)
  6142. "/api/v1/updates/version",
  6143. # Metrics endpoint handles its own prometheus_token authentication
  6144. "/api/v1/metrics",
  6145. # Appliance bootstrap (#1589 follow-up): the SPA's i18n setup polls
  6146. # this BEFORE a JWT is available to pick up the firstboot wizard's
  6147. # hostname / timezone / locale and the chrony NTP-gate state. The
  6148. # response contains user-set defaults and a public sync flag — no
  6149. # secrets. Without this entry the global auth middleware returns 401
  6150. # before the route handler runs, regardless of the route's own
  6151. # "no auth required" intent.
  6152. "/api/v1/system/appliance",
  6153. # Cam Wall kiosk feed (#2531): a TV or Pi in kiosk mode has no login, so it
  6154. # authenticates with a long-lived ``camwall``-scoped token in the query
  6155. # string — exactly like the camera streams two lists below, and for the same
  6156. # reason (no header to put a JWT in). "Public" here only means the middleware
  6157. # steps aside; the route still runs RequireCamWallTokenIfAuthEnabled, which
  6158. # rejects an absent, expired, revoked, or wrong-scoped token. In particular a
  6159. # plain ``camera_stream`` token does NOT open this door.
  6160. "/api/v1/camwall/printers",
  6161. }
  6162. # Route prefixes that are public (for routes with dynamic segments)
  6163. PUBLIC_API_PREFIXES = [
  6164. # WebSocket connections handle their own auth
  6165. "/api/v1/ws",
  6166. # OIDC authorize redirects — include provider_id in path
  6167. "/api/v1/auth/oidc/authorize/",
  6168. ]
  6169. # Route patterns that are public (read-only display data)
  6170. # These are checked with "in path" - needed because browsers load images/videos
  6171. # via <img src> and <video src> which don't include Authorization headers
  6172. PUBLIC_API_PATTERNS = [
  6173. # Thumbnails
  6174. "/thumbnail", # /archives/{id}/thumbnail, /library/files/{id}/thumbnail
  6175. "/plate-thumbnail/", # /archives/{id}/plate-thumbnail/{plate_id}
  6176. # Images and media
  6177. "/photos/", # /archives/{id}/photos/{filename}
  6178. "/project-image/", # /archives/{id}/project-image/{path}
  6179. "/qrcode", # /archives/{id}/qrcode
  6180. "/timelapse", # /archives/{id}/timelapse (video)
  6181. "/cover", # /printers/{id}/cover
  6182. "/icon", # /external-links/{id}/icon
  6183. # Camera (streams loaded via <img> tag)
  6184. "/camera/stream", # /printers/{id}/camera/stream
  6185. "/camera/snapshot", # /printers/{id}/camera/snapshot
  6186. # Streaming-overlay status feed (#2613): OBS loads /overlay/{id} with no login
  6187. # and this backs it, authenticated by an ``overlay``-scoped token in the query
  6188. # string (same reasoning as the camera streams above — no header to carry a
  6189. # JWT). "Public" only means the middleware steps aside; the route still runs
  6190. # RequireOverlayTokenIfAuthEnabled, which rejects an absent, expired, revoked,
  6191. # or wrong-scoped token — a camwall or camera_stream token does NOT open it.
  6192. "/overlay-status", # /printers/{id}/overlay-status
  6193. # Slicer token-authenticated downloads — protocol handlers (bambustudioopen://,
  6194. # orcaslicer://) cannot send auth headers. These endpoints validate a short-lived
  6195. # download token in the URL path instead.
  6196. "/dl/", # /archives/{id}/dl/{token}/{filename}, /library/files/{id}/dl/{token}/{filename}
  6197. # Obico ML API fetches JPEG frames by one-shot nonce (issue #172 follow-up).
  6198. # The nonce itself is the credential: 32-byte random, single-use, ~30s TTL.
  6199. "/obico/cached-frame/", # /obico/cached-frame/{nonce}
  6200. ]
  6201. _security_headers_logger = logging.getLogger("backend.app.main.security_headers")
  6202. def _parse_trusted_frame_origins() -> tuple[str, ...]:
  6203. """Parse TRUSTED_FRAME_ORIGINS env var into a validated allowlist (#1191).
  6204. Format: comma-separated list of ``scheme://host[:port]`` origins.
  6205. Used by ``security_headers_middleware`` to relax ``frame-ancestors`` for
  6206. trusted same-LAN deployments (e.g. Home Assistant Webpage panel embedding
  6207. Bambuddy from a different port). Defaults to empty — strict ``'none'``.
  6208. Invalid entries are dropped with a warning rather than failing startup, so
  6209. a typo in one origin doesn't take the whole deployment down.
  6210. """
  6211. raw = os.environ.get("TRUSTED_FRAME_ORIGINS", "").strip()
  6212. if not raw:
  6213. return ()
  6214. valid: list[str] = []
  6215. for item in raw.split(","):
  6216. candidate = item.strip()
  6217. if not candidate:
  6218. continue
  6219. try:
  6220. parsed = urlparse(candidate)
  6221. except ValueError as e:
  6222. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — %s", candidate, e)
  6223. continue
  6224. if parsed.scheme not in ("http", "https"):
  6225. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — must be http(s)", candidate)
  6226. continue
  6227. if not parsed.netloc:
  6228. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — missing host", candidate)
  6229. continue
  6230. if parsed.path and parsed.path != "/":
  6231. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — paths not allowed", candidate)
  6232. continue
  6233. if parsed.query or parsed.fragment:
  6234. _security_headers_logger.warning(
  6235. "TRUSTED_FRAME_ORIGINS: dropping %r — query/fragment not allowed", candidate
  6236. )
  6237. continue
  6238. if "*" in parsed.netloc:
  6239. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — wildcards not allowed", candidate)
  6240. continue
  6241. valid.append(f"{parsed.scheme}://{parsed.netloc}")
  6242. if valid:
  6243. _security_headers_logger.info("TRUSTED_FRAME_ORIGINS: %s", ", ".join(valid))
  6244. return tuple(valid)
  6245. _TRUSTED_FRAME_ORIGINS: tuple[str, ...] = _parse_trusted_frame_origins()
  6246. def _frame_ancestors(default_value: str) -> str:
  6247. """Compose the ``frame-ancestors`` CSP directive (#1191).
  6248. ``default_value`` is the strict directive used when the operator has not
  6249. configured ``TRUSTED_FRAME_ORIGINS`` — typically ``'none'`` (catch-all and
  6250. docs) or ``'self'`` (gcode-viewer, served same-origin). When trusted origins
  6251. are configured, ``'self'`` is always included so same-origin embedding never
  6252. breaks even if an operator forgets to add their own origin to the list.
  6253. """
  6254. if _TRUSTED_FRAME_ORIGINS:
  6255. return "frame-ancestors 'self' " + " ".join(_TRUSTED_FRAME_ORIGINS) + ";"
  6256. return f"frame-ancestors {default_value};"
  6257. @app.middleware("http")
  6258. async def security_headers_middleware(request, call_next):
  6259. """Add standard HTTP security headers to every response."""
  6260. # Per-request nonce stamped into `script-src` (#1460). On its own this
  6261. # changes nothing for Bambuddy's own pages — index.html has no inline
  6262. # scripts since the SW registration moved to /sw-register.js. The reason
  6263. # it's here is Cloudflare: a CF-fronted deployment has the bot-detection
  6264. # script injected into the HTML on the edge, with a fresh hash on every
  6265. # load (so hashes can't be allowlisted). When CF sees a nonce in our CSP,
  6266. # it clones the same nonce onto its injected <script>, and the inline
  6267. # script passes the policy without us needing 'unsafe-inline'. See
  6268. # https://developers.cloudflare.com/cloudflare-challenges/challenge-types/javascript-detections/#if-you-have-a-content-security-policy-csp
  6269. csp_nonce = secrets.token_urlsafe(16)
  6270. response = await call_next(request)
  6271. response.headers["X-Content-Type-Options"] = "nosniff"
  6272. # X-Frame-Options is the legacy cross-origin embedding control. Modern
  6273. # browsers honour CSP frame-ancestors instead, and the legacy
  6274. # `ALLOW-FROM <url>` syntax is deprecated and inconsistent across vendors.
  6275. # When operators have explicitly allowlisted trusted frame origins (#1191
  6276. # — typically Home Assistant on a different port), drop X-Frame-Options
  6277. # and let the CSP-side frame-ancestors directive govern embedding.
  6278. if not _TRUSTED_FRAME_ORIGINS:
  6279. response.headers["X-Frame-Options"] = "SAMEORIGIN"
  6280. response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
  6281. # Content-Security-Policy for the React SPA.
  6282. # Notes:
  6283. # - 'unsafe-inline' for style-src: React and UI libs inject inline styles at runtime.
  6284. # - connect-src ws:/wss:: MQTT/printer WebSocket connections.
  6285. # - img-src data: / blob:: base64 thumbnails and Blob-URL timelapse previews.
  6286. # - media-src blob:: timelapse video player uses Blob URLs.
  6287. # - font-src data:: some icon fonts are embedded as data URIs.
  6288. if request.url.path.startswith("/gcode-viewer"):
  6289. # The gcode viewer is embedded in an iframe served by this same origin,
  6290. # so frame-ancestors must allow 'self'. prettygcode.js also uses eval()
  6291. # internally, so script-src needs 'unsafe-eval'.
  6292. response.headers["Content-Security-Policy"] = (
  6293. "default-src 'self'; "
  6294. "script-src 'self' 'unsafe-eval'; "
  6295. "style-src 'self' 'unsafe-inline'; "
  6296. "img-src 'self' data: blob:; "
  6297. "media-src 'self' blob:; "
  6298. "connect-src 'self' ws: wss:; "
  6299. "font-src 'self' data:; "
  6300. "object-src 'none'; "
  6301. "base-uri 'self'; "
  6302. "frame-src 'self' http: https:; " + _frame_ancestors("'self'")
  6303. )
  6304. elif request.url.path in ("/docs", "/redoc", "/docs/oauth2-redirect"):
  6305. # FastAPI's built-in Swagger UI / ReDoc pages load assets from
  6306. # cdn.jsdelivr.net and bootstrap with an inline <script>, so the
  6307. # default CSP would render a blank page.
  6308. response.headers["Content-Security-Policy"] = (
  6309. "default-src 'self'; "
  6310. "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
  6311. "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com; "
  6312. "img-src 'self' data: blob: https://fastapi.tiangolo.com https://cdn.redoc.ly; "
  6313. "connect-src 'self'; "
  6314. "font-src 'self' data: https://fonts.gstatic.com; "
  6315. "worker-src 'self' blob:; "
  6316. "object-src 'none'; "
  6317. "base-uri 'self'; " + _frame_ancestors("'none'")
  6318. )
  6319. else:
  6320. response.headers["Content-Security-Policy"] = (
  6321. "default-src 'self'; "
  6322. f"script-src 'self' 'nonce-{csp_nonce}'; "
  6323. "style-src 'self' 'unsafe-inline'; "
  6324. "img-src 'self' data: blob:; "
  6325. "media-src 'self' blob:; "
  6326. "connect-src 'self' ws: wss:; "
  6327. "font-src 'self' data:; "
  6328. "object-src 'none'; "
  6329. "base-uri 'self'; "
  6330. "frame-src 'self' http: https:; " + _frame_ancestors("'none'")
  6331. )
  6332. if request.url.scheme == "https":
  6333. response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
  6334. return response
  6335. @app.middleware("http")
  6336. async def auth_middleware(request, call_next):
  6337. """Enforce authentication on all API routes when auth is enabled.
  6338. This middleware provides defense-in-depth by checking auth at the API gateway level,
  6339. regardless of whether individual routes have auth dependencies.
  6340. """
  6341. from starlette.responses import JSONResponse
  6342. path = request.url.path
  6343. # Only apply to API routes
  6344. if not path.startswith("/api/"):
  6345. return await call_next(request)
  6346. # Allow public routes
  6347. if path in PUBLIC_API_ROUTES:
  6348. return await call_next(request)
  6349. # Allow public prefixes
  6350. for prefix in PUBLIC_API_PREFIXES:
  6351. if path.startswith(prefix):
  6352. return await call_next(request)
  6353. # Allow public patterns (read-only display data like thumbnails)
  6354. for pattern in PUBLIC_API_PATTERNS:
  6355. if pattern in path:
  6356. return await call_next(request)
  6357. # Check if auth is enabled. Fail CLOSED on any exception during the
  6358. # probe — GHSA-6mf4-q26m-47pv: the previous fail-open path here let
  6359. # an attacker who could force a DB exception (e.g. file-descriptor
  6360. # exhaustion via login flood) bypass auth on every protected endpoint.
  6361. try:
  6362. async with async_session() as db:
  6363. from backend.app.core.auth import is_auth_enabled
  6364. auth_enabled = await is_auth_enabled(db)
  6365. if not auth_enabled:
  6366. # Auth disabled, allow all requests
  6367. return await call_next(request)
  6368. except Exception:
  6369. logging.getLogger(__name__).exception("auth_middleware: failing closed on auth-probe error from %s", path)
  6370. return JSONResponse(
  6371. status_code=503,
  6372. content={"detail": "Authentication service temporarily unavailable"},
  6373. )
  6374. # Auth is enabled - require valid token
  6375. auth_header = request.headers.get("Authorization")
  6376. x_api_key = request.headers.get("X-API-Key")
  6377. # Check for API key auth first
  6378. if x_api_key or (auth_header and auth_header.startswith("Bearer bb_")):
  6379. # API key authentication - let the request through to be validated by route handler
  6380. # API keys are validated per-route since they have different permission levels
  6381. return await call_next(request)
  6382. # Check for JWT auth
  6383. if not auth_header or not auth_header.startswith("Bearer "):
  6384. return JSONResponse(
  6385. status_code=401,
  6386. content={"detail": "Authentication required"},
  6387. headers={"WWW-Authenticate": "Bearer"},
  6388. )
  6389. # Validate JWT token
  6390. import jwt
  6391. try:
  6392. from backend.app.core.auth import (
  6393. ALGORITHM,
  6394. SECRET_KEY,
  6395. _is_token_fresh,
  6396. get_user_by_username,
  6397. is_jti_revoked,
  6398. )
  6399. token = auth_header.replace("Bearer ", "")
  6400. payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
  6401. username = payload.get("sub")
  6402. if not username:
  6403. raise ValueError("No username in token")
  6404. jti = payload.get("jti")
  6405. if not jti:
  6406. raise ValueError("No jti in token")
  6407. iat = payload.get("iat")
  6408. # Verify user exists, is active, and token is still fresh (L-R8-A).
  6409. # Reject revoked tokens first (defense-in-depth gateway check), reusing
  6410. # this session so the gateway adds a single pooled checkout, not two (#2572).
  6411. async with async_session() as db:
  6412. if await is_jti_revoked(jti, db):
  6413. return JSONResponse(
  6414. status_code=401,
  6415. content={"detail": "Token has been revoked"},
  6416. headers={"WWW-Authenticate": "Bearer"},
  6417. )
  6418. user = await get_user_by_username(db, username)
  6419. if not user or not user.is_active:
  6420. return JSONResponse(
  6421. status_code=401,
  6422. content={"detail": "User not found or inactive"},
  6423. headers={"WWW-Authenticate": "Bearer"},
  6424. )
  6425. if not _is_token_fresh(iat, user):
  6426. return JSONResponse(
  6427. status_code=401,
  6428. content={"detail": "Token no longer valid"},
  6429. headers={"WWW-Authenticate": "Bearer"},
  6430. )
  6431. except jwt.ExpiredSignatureError:
  6432. return JSONResponse(
  6433. status_code=401,
  6434. content={"detail": "Token has expired"},
  6435. headers={"WWW-Authenticate": "Bearer"},
  6436. )
  6437. except (jwt.InvalidTokenError, ValueError, Exception):
  6438. return JSONResponse(
  6439. status_code=401,
  6440. content={"detail": "Invalid token"},
  6441. headers={"WWW-Authenticate": "Bearer"},
  6442. )
  6443. return await call_next(request)
  6444. @app.middleware("http")
  6445. async def trace_id_middleware(request, call_next):
  6446. """Stamp every HTTP request with a trace ID and echo it back.
  6447. Decorated AFTER auth_middleware on purpose: Starlette stacks
  6448. @app.middleware decorators LIFO, so the last-decorated runs first
  6449. inbound. Putting the trace stamp last makes it the OUTERMOST layer,
  6450. which means auth-middleware log lines (and every line emitted on the
  6451. way down to and back from the route handler) all carry the same
  6452. trace ID. If we put it before auth, auth's logs would be stamped
  6453. with the *previous* request's ID — useless for correlation.
  6454. Honours an inbound ``X-Trace-Id`` header so callers running their
  6455. own tracing can correlate their span IDs with our log lines, but
  6456. only if the value passes the whitelist gate in
  6457. ``backend.app.core.trace.normalise_inbound_trace_id`` — anything
  6458. rejected (too long, contains control chars, etc.) silently triggers
  6459. a freshly minted server-side ID rather than failing the request.
  6460. The minted (or echoed) ID is set on a ContextVar so that every log
  6461. record emitted during the request — application logs *and* uvicorn's
  6462. access log — carries it via TraceIDFilter, and is also written to
  6463. the ``X-Trace-Id`` response header so clients can pin a server-side
  6464. log search to the exact request they made.
  6465. """
  6466. from backend.app.core.trace import (
  6467. generate_trace_id,
  6468. normalise_inbound_trace_id,
  6469. trace_id_var,
  6470. )
  6471. inbound = normalise_inbound_trace_id(request.headers.get("X-Trace-Id"))
  6472. trace_id = inbound if inbound is not None else generate_trace_id()
  6473. token = trace_id_var.set(trace_id)
  6474. try:
  6475. response = await call_next(request)
  6476. finally:
  6477. # Reset the ContextVar so a record emitted in a totally
  6478. # unrelated background task that just happens to inherit this
  6479. # context doesn't keep referencing this request's ID forever.
  6480. # In practice ContextVar.reset is best-effort under asyncio
  6481. # task-spawn semantics, but the cost is one attribute write so
  6482. # we may as well do it.
  6483. trace_id_var.reset(token)
  6484. response.headers["X-Trace-Id"] = trace_id
  6485. return response
  6486. # API routes
  6487. app.include_router(auth.router, prefix=app_settings.api_prefix)
  6488. app.include_router(mfa.router, prefix=app_settings.api_prefix)
  6489. app.include_router(bug_report.router, prefix=app_settings.api_prefix)
  6490. app.include_router(users.router, prefix=app_settings.api_prefix)
  6491. app.include_router(groups.router, prefix=app_settings.api_prefix)
  6492. app.include_router(printers.router, prefix=app_settings.api_prefix)
  6493. app.include_router(archives.router, prefix=app_settings.api_prefix)
  6494. app.include_router(filaments.router, prefix=app_settings.api_prefix)
  6495. app.include_router(inventory.router, prefix=app_settings.api_prefix)
  6496. app.include_router(labels.router, prefix=app_settings.api_prefix)
  6497. app.include_router(settings_routes.router, prefix=app_settings.api_prefix)
  6498. app.include_router(cloud.router, prefix=app_settings.api_prefix)
  6499. app.include_router(orca_cloud.router, prefix=app_settings.api_prefix)
  6500. app.include_router(local_presets.router, prefix=app_settings.api_prefix)
  6501. app.include_router(smart_plugs.router, prefix=app_settings.api_prefix)
  6502. app.include_router(print_log.router, prefix=app_settings.api_prefix)
  6503. app.include_router(print_queue.router, prefix=app_settings.api_prefix)
  6504. app.include_router(kprofiles.router, prefix=app_settings.api_prefix)
  6505. app.include_router(notifications.router, prefix=app_settings.api_prefix)
  6506. app.include_router(notification_templates.router, prefix=app_settings.api_prefix)
  6507. app.include_router(user_notifications.router, prefix=app_settings.api_prefix)
  6508. app.include_router(spoolman.router, prefix=app_settings.api_prefix)
  6509. app.include_router(spoolman_inventory.router, prefix=app_settings.api_prefix)
  6510. app.include_router(updates.router, prefix=app_settings.api_prefix)
  6511. app.include_router(sponsor_prompt.router, prefix=app_settings.api_prefix)
  6512. app.include_router(maintenance.router, prefix=app_settings.api_prefix)
  6513. app.include_router(camera.router, prefix=app_settings.api_prefix)
  6514. app.include_router(camwall.router, prefix=app_settings.api_prefix)
  6515. app.include_router(external_links.router, prefix=app_settings.api_prefix)
  6516. app.include_router(projects.router, prefix=app_settings.api_prefix)
  6517. app.include_router(library.router, prefix=app_settings.api_prefix)
  6518. app.include_router(library_tags.router, prefix=app_settings.api_prefix)
  6519. app.include_router(library_trash.router, prefix=app_settings.api_prefix)
  6520. app.include_router(slice_jobs.router, prefix=app_settings.api_prefix)
  6521. app.include_router(slicer_pipelines.router, prefix=app_settings.api_prefix)
  6522. app.include_router(pipeline_runs.pipeline_run_create_router, prefix=app_settings.api_prefix)
  6523. app.include_router(pipeline_runs.pipeline_run_router, prefix=app_settings.api_prefix)
  6524. app.include_router(slicer_presets.router, prefix=app_settings.api_prefix)
  6525. app.include_router(archive_purge.router, prefix=app_settings.api_prefix)
  6526. app.include_router(makerworld.router, prefix=app_settings.api_prefix)
  6527. app.include_router(api_keys.router, prefix=app_settings.api_prefix)
  6528. app.include_router(webhook.router, prefix=app_settings.api_prefix)
  6529. app.include_router(ams_history.router, prefix=app_settings.api_prefix)
  6530. app.include_router(printer_sensor_history.router, prefix=app_settings.api_prefix)
  6531. app.include_router(system.router, prefix=app_settings.api_prefix)
  6532. app.include_router(support.router, prefix=app_settings.api_prefix)
  6533. app.include_router(websocket.router, prefix=app_settings.api_prefix)
  6534. app.include_router(discovery.router, prefix=app_settings.api_prefix)
  6535. app.include_router(pending_uploads.router, prefix=app_settings.api_prefix)
  6536. app.include_router(firmware.router, prefix=app_settings.api_prefix)
  6537. app.include_router(github_backup.router, prefix=app_settings.api_prefix)
  6538. app.include_router(local_backup.router, prefix=app_settings.api_prefix)
  6539. app.include_router(obico.router, prefix=app_settings.api_prefix)
  6540. app.include_router(metrics.router, prefix=app_settings.api_prefix)
  6541. app.include_router(virtual_printers.router, prefix=app_settings.api_prefix)
  6542. app.include_router(spoolbuddy.router, prefix=app_settings.api_prefix)
  6543. # Serve static files (React build)
  6544. if app_settings.static_dir.exists() and any(app_settings.static_dir.iterdir()):
  6545. app.mount(
  6546. "/assets",
  6547. StaticFiles(directory=app_settings.static_dir / "assets"),
  6548. name="assets",
  6549. )
  6550. if (app_settings.static_dir / "img").exists():
  6551. app.mount(
  6552. "/img",
  6553. StaticFiles(directory=app_settings.static_dir / "img"),
  6554. name="img",
  6555. )
  6556. if (app_settings.static_dir / "icons").exists():
  6557. app.mount(
  6558. "/icons",
  6559. StaticFiles(directory=app_settings.static_dir / "icons"),
  6560. name="icons",
  6561. )
  6562. # Self-hosted Inter woff2 files (#1460). Without this mount /fonts/*.woff2
  6563. # falls through to the SPA catch-all and returns index.html, which the
  6564. # browser's font sanitizer rejects ("downloadable font: rejected by
  6565. # sanitizer").
  6566. if (app_settings.static_dir / "fonts").exists():
  6567. app.mount(
  6568. "/fonts",
  6569. StaticFiles(directory=app_settings.static_dir / "fonts"),
  6570. name="fonts",
  6571. )
  6572. @app.get("/")
  6573. async def serve_frontend():
  6574. """Serve the React frontend."""
  6575. index_file = app_settings.static_dir / "index.html"
  6576. if index_file.exists():
  6577. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  6578. return {
  6579. "message": "Bambuddy API",
  6580. "docs": "/docs",
  6581. "frontend": "Build and place React app in /static directory",
  6582. }
  6583. # index.html must always be revalidated — Vite emits content-hashed JS/CSS
  6584. # bundles (e.g. `index-JRaF_JhW.js`), so the JS itself is safe to cache
  6585. # forever, but the HTML wrapping it is the only file that knows which hash
  6586. # is current. Without explicit cache-control headers Chromium decides
  6587. # heuristically (typically 10% of the time since Last-Modified) and on
  6588. # long-running kiosks happily serves stale HTML across browser restarts.
  6589. # That stale HTML references an old bundle hash, the old bundle is also
  6590. # in the disk cache, and the user ends up running pre-update JS forever
  6591. # without ever knowing why. ``no-cache`` (revalidate every time, but a
  6592. # 304 is cheap) is the correct setting for an SPA's entry HTML.
  6593. _HTML_CACHE_HEADERS = {"Cache-Control": "no-cache, must-revalidate"}
  6594. @app.get("/health")
  6595. async def health_check():
  6596. """Health check endpoint."""
  6597. return {"status": "healthy"}
  6598. # GET + HEAD on the three PWA bootstrap routes (#1460). Scanners and a plain
  6599. # `curl -I` use HEAD; FastAPI's @app.get only registers GET, so HEAD answers
  6600. # with 405 Method Not Allowed and shows up as a "broken manifest" red herring
  6601. # in deployment debugging.
  6602. @app.api_route("/manifest.json", methods=["GET", "HEAD"])
  6603. async def serve_manifest():
  6604. """Serve PWA manifest."""
  6605. manifest_file = app_settings.static_dir / "manifest.json"
  6606. if manifest_file.exists():
  6607. return FileResponse(manifest_file, media_type="application/manifest+json")
  6608. return {"error": "Manifest not found"}
  6609. @app.api_route("/sw.js", methods=["GET", "HEAD"])
  6610. async def serve_service_worker():
  6611. """Serve service worker."""
  6612. sw_file = app_settings.static_dir / "sw.js"
  6613. if sw_file.exists():
  6614. return FileResponse(
  6615. sw_file,
  6616. media_type="application/javascript",
  6617. headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
  6618. )
  6619. return {"error": "Service worker not found"}
  6620. @app.api_route("/sw-register.js", methods=["GET", "HEAD"])
  6621. async def serve_sw_register():
  6622. """Serve the service-worker registration bootstrap script.
  6623. Served as a real JS file so the strict `script-src 'self'` CSP covers it
  6624. without needing 'unsafe-inline' or per-build hashes on the inline tag.
  6625. """
  6626. reg_file = app_settings.static_dir / "sw-register.js"
  6627. if reg_file.exists():
  6628. return FileResponse(reg_file, media_type="application/javascript")
  6629. return {"error": "sw-register.js not found"}
  6630. # ── GCode viewer static files ────────────────────────────────────────────────
  6631. # Served via explicit routes so ordering is guaranteed (app.mount() loses
  6632. # to the /{full_path:path} catch-all in some Starlette versions).
  6633. _gcode_viewer_dir = (app_settings.static_dir.parent / "gcode_viewer").resolve()
  6634. # Surface packaging gaps at startup instead of as silent runtime 404s. If the
  6635. # directory is missing the explicit @app.get("/gcode-viewer/...") routes below
  6636. # return bare HTTPException(404) which renders as {"detail":"Not Found"} in
  6637. # the 3D Preview iframe (#1218) — easy to miss in normal operation, easy to
  6638. # spot if the operator scans the startup log or a support bundle.
  6639. if not (_gcode_viewer_dir / "index.html").is_file():
  6640. logging.getLogger(__name__).error(
  6641. "Embedded GCode viewer assets missing at %s — /gcode-viewer/ will return 404 "
  6642. "and 3D Preview will fail. This indicates a packaging bug; the gcode_viewer/ "
  6643. "directory must be present alongside static/.",
  6644. _gcode_viewer_dir,
  6645. )
  6646. def _gcode_viewer_response(rel: str) -> FileResponse:
  6647. from fastapi import HTTPException as _HTTPException
  6648. safe = (_gcode_viewer_dir / rel).resolve()
  6649. if not safe.is_relative_to(_gcode_viewer_dir):
  6650. raise _HTTPException(status_code=403)
  6651. if safe.is_file():
  6652. mt, _ = _mimetypes.guess_type(str(safe))
  6653. return FileResponse(str(safe), media_type=mt or "application/octet-stream")
  6654. raise _HTTPException(status_code=404)
  6655. @app.get("/gcode-viewer/")
  6656. async def serve_gcode_viewer_index() -> FileResponse:
  6657. """Raw PrettyGCode viewer for the iframe. The bare ``/gcode-viewer``
  6658. (no trailing slash) intentionally falls through to the SPA catch-all so a
  6659. full-page reload re-enters the React layout instead of serving the iframe
  6660. contents standalone."""
  6661. return _gcode_viewer_response("index.html")
  6662. @app.get("/gcode-viewer/{file_path:path}")
  6663. async def serve_gcode_viewer_file(file_path: str) -> FileResponse:
  6664. return _gcode_viewer_response(file_path)
  6665. # Catch-all route for React Router (must be last)
  6666. @app.get("/{full_path:path}")
  6667. async def serve_spa(full_path: str):
  6668. """Serve React app for client-side routing."""
  6669. # Don't intercept API routes - raise proper 404 so FastAPI can handle redirects
  6670. if full_path.startswith("api/"):
  6671. from fastapi import HTTPException
  6672. raise HTTPException(status_code=404, detail="Not found")
  6673. index_file = app_settings.static_dir / "index.html"
  6674. if index_file.exists():
  6675. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  6676. return {"error": "Frontend not built"}