main.py 648 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458945994609461946294639464946594669467946894699470947194729473947494759476947794789479948094819482948394849485948694879488948994909491949294939494949594969497949894999500950195029503950495059506950795089509951095119512951395149515951695179518951995209521952295239524952595269527952895299530953195329533953495359536953795389539954095419542954395449545954695479548954995509551955295539554955595569557955895599560956195629563956495659566956795689569957095719572957395749575957695779578957995809581958295839584958595869587958895899590959195929593959495959596959795989599960096019602960396049605960696079608960996109611961296139614961596169617961896199620962196229623962496259626962796289629963096319632963396349635963696379638963996409641964296439644964596469647964896499650965196529653965496559656965796589659966096619662966396649665966696679668966996709671967296739674967596769677967896799680968196829683968496859686968796889689969096919692969396949695969696979698969997009701970297039704970597069707970897099710971197129713971497159716971797189719972097219722972397249725972697279728972997309731973297339734973597369737973897399740974197429743974497459746974797489749975097519752975397549755975697579758975997609761976297639764976597669767976897699770977197729773977497759776977797789779978097819782978397849785978697879788978997909791979297939794979597969797979897999800980198029803980498059806980798089809981098119812981398149815981698179818981998209821982298239824982598269827982898299830983198329833983498359836983798389839984098419842984398449845984698479848984998509851985298539854985598569857985898599860986198629863986498659866986798689869987098719872987398749875987698779878987998809881988298839884988598869887988898899890989198929893989498959896989798989899990099019902990399049905990699079908990999109911991299139914991599169917991899199920992199229923992499259926992799289929993099319932993399349935993699379938993999409941994299439944994599469947994899499950995199529953995499559956995799589959996099619962996399649965996699679968996999709971997299739974997599769977997899799980998199829983998499859986998799889989999099919992999399949995999699979998999910000100011000210003100041000510006100071000810009100101001110012100131001410015100161001710018100191002010021100221002310024100251002610027100281002910030100311003210033100341003510036100371003810039100401004110042100431004410045100461004710048100491005010051100521005310054100551005610057100581005910060100611006210063100641006510066100671006810069100701007110072100731007410075100761007710078100791008010081100821008310084100851008610087100881008910090100911009210093100941009510096100971009810099101001010110102101031010410105101061010710108101091011010111101121011310114101151011610117101181011910120101211012210123101241012510126101271012810129101301013110132101331013410135101361013710138101391014010141101421014310144101451014610147101481014910150101511015210153101541015510156101571015810159101601016110162101631016410165101661016710168101691017010171101721017310174101751017610177101781017910180101811018210183101841018510186101871018810189101901019110192101931019410195101961019710198101991020010201102021020310204102051020610207102081020910210102111021210213102141021510216102171021810219102201022110222102231022410225102261022710228102291023010231102321023310234102351023610237102381023910240102411024210243102441024510246102471024810249102501025110252102531025410255102561025710258102591026010261102621026310264102651026610267102681026910270102711027210273102741027510276102771027810279102801028110282102831028410285102861028710288102891029010291102921029310294102951029610297102981029910300103011030210303103041030510306103071030810309103101031110312103131031410315103161031710318103191032010321103221032310324103251032610327103281032910330103311033210333103341033510336103371033810339103401034110342103431034410345103461034710348103491035010351103521035310354103551035610357103581035910360103611036210363103641036510366103671036810369103701037110372103731037410375103761037710378103791038010381103821038310384103851038610387103881038910390103911039210393103941039510396103971039810399104001040110402104031040410405104061040710408104091041010411104121041310414104151041610417104181041910420104211042210423104241042510426104271042810429104301043110432104331043410435104361043710438104391044010441104421044310444104451044610447104481044910450104511045210453104541045510456104571045810459104601046110462104631046410465104661046710468104691047010471104721047310474104751047610477104781047910480104811048210483104841048510486104871048810489104901049110492104931049410495104961049710498104991050010501105021050310504105051050610507105081050910510105111051210513105141051510516105171051810519105201052110522105231052410525105261052710528105291053010531105321053310534105351053610537105381053910540105411054210543105441054510546105471054810549105501055110552105531055410555105561055710558105591056010561105621056310564105651056610567105681056910570105711057210573105741057510576105771057810579105801058110582105831058410585105861058710588105891059010591105921059310594105951059610597105981059910600106011060210603106041060510606106071060810609106101061110612106131061410615106161061710618106191062010621106221062310624106251062610627106281062910630106311063210633106341063510636106371063810639106401064110642106431064410645106461064710648106491065010651106521065310654106551065610657106581065910660106611066210663106641066510666106671066810669106701067110672106731067410675106761067710678106791068010681106821068310684106851068610687106881068910690106911069210693106941069510696106971069810699107001070110702107031070410705107061070710708107091071010711107121071310714107151071610717107181071910720107211072210723107241072510726107271072810729107301073110732107331073410735107361073710738107391074010741107421074310744107451074610747107481074910750107511075210753107541075510756107571075810759107601076110762107631076410765107661076710768107691077010771107721077310774107751077610777107781077910780107811078210783107841078510786107871078810789107901079110792107931079410795107961079710798107991080010801108021080310804108051080610807108081080910810108111081210813108141081510816108171081810819108201082110822108231082410825108261082710828108291083010831108321083310834108351083610837108381083910840108411084210843108441084510846108471084810849108501085110852108531085410855108561085710858108591086010861108621086310864108651086610867108681086910870108711087210873108741087510876108771087810879108801088110882108831088410885108861088710888108891089010891108921089310894108951089610897108981089910900109011090210903109041090510906109071090810909109101091110912109131091410915109161091710918109191092010921109221092310924109251092610927109281092910930109311093210933109341093510936109371093810939109401094110942109431094410945109461094710948109491095010951109521095310954109551095610957109581095910960109611096210963109641096510966109671096810969109701097110972109731097410975109761097710978109791098010981109821098310984109851098610987109881098910990109911099210993109941099510996109971099810999110001100111002110031100411005110061100711008110091101011011110121101311014110151101611017110181101911020110211102211023110241102511026110271102811029110301103111032110331103411035110361103711038110391104011041110421104311044110451104611047110481104911050110511105211053110541105511056110571105811059110601106111062110631106411065110661106711068110691107011071110721107311074110751107611077110781107911080110811108211083110841108511086110871108811089110901109111092110931109411095110961109711098110991110011101111021110311104111051110611107111081110911110111111111211113111141111511116111171111811119111201112111122111231112411125111261112711128111291113011131111321113311134111351113611137111381113911140111411114211143111441114511146111471114811149111501115111152111531115411155111561115711158111591116011161111621116311164111651116611167111681116911170111711117211173111741117511176111771117811179111801118111182111831118411185111861118711188111891119011191111921119311194111951119611197111981119911200112011120211203112041120511206112071120811209112101121111212112131121411215112161121711218112191122011221112221122311224112251122611227112281122911230112311123211233112341123511236112371123811239112401124111242112431124411245112461124711248112491125011251112521125311254112551125611257112581125911260112611126211263112641126511266112671126811269112701127111272112731127411275112761127711278112791128011281112821128311284112851128611287112881128911290112911129211293112941129511296112971129811299113001130111302113031130411305113061130711308113091131011311113121131311314113151131611317113181131911320113211132211323113241132511326113271132811329113301133111332113331133411335113361133711338113391134011341113421134311344113451134611347113481134911350113511135211353113541135511356113571135811359113601136111362113631136411365113661136711368113691137011371113721137311374113751137611377113781137911380113811138211383113841138511386113871138811389113901139111392113931139411395113961139711398113991140011401114021140311404114051140611407114081140911410114111141211413114141141511416114171141811419114201142111422114231142411425114261142711428114291143011431114321143311434114351143611437114381143911440114411144211443114441144511446114471144811449114501145111452114531145411455114561145711458114591146011461114621146311464114651146611467114681146911470114711147211473114741147511476114771147811479114801148111482114831148411485114861148711488114891149011491114921149311494114951149611497114981149911500115011150211503115041150511506115071150811509115101151111512115131151411515115161151711518115191152011521115221152311524115251152611527115281152911530115311153211533115341153511536115371153811539115401154111542115431154411545115461154711548115491155011551115521155311554115551155611557115581155911560115611156211563115641156511566115671156811569115701157111572115731157411575115761157711578115791158011581115821158311584115851158611587115881158911590115911159211593115941159511596115971159811599116001160111602116031160411605116061160711608116091161011611116121161311614116151161611617116181161911620116211162211623116241162511626116271162811629116301163111632116331163411635116361163711638116391164011641116421164311644116451164611647116481164911650116511165211653116541165511656116571165811659116601166111662116631166411665116661166711668116691167011671116721167311674116751167611677116781167911680116811168211683116841168511686116871168811689116901169111692116931169411695116961169711698116991170011701117021170311704117051170611707117081170911710117111171211713117141171511716117171171811719117201172111722117231172411725117261172711728117291173011731117321173311734117351173611737117381173911740117411174211743117441174511746117471174811749117501175111752117531175411755117561175711758117591176011761117621176311764117651176611767117681176911770117711177211773117741177511776117771177811779117801178111782117831178411785117861178711788117891179011791117921179311794117951179611797117981179911800118011180211803118041180511806118071180811809118101181111812118131181411815118161181711818118191182011821118221182311824118251182611827118281182911830118311183211833118341183511836118371183811839118401184111842118431184411845118461184711848118491185011851118521185311854118551185611857118581185911860118611186211863118641186511866118671186811869118701187111872118731187411875118761187711878118791188011881118821188311884118851188611887118881188911890118911189211893118941189511896118971189811899119001190111902119031190411905119061190711908119091191011911119121191311914119151191611917119181191911920119211192211923119241192511926119271192811929119301193111932119331193411935119361193711938119391194011941119421194311944119451194611947119481194911950119511195211953119541195511956119571195811959119601196111962119631196411965119661196711968119691197011971119721197311974119751197611977119781197911980119811198211983119841198511986119871198811989119901199111992119931199411995119961199711998119991200012001120021200312004120051200612007120081200912010120111201212013120141201512016120171201812019120201202112022120231202412025120261202712028120291203012031120321203312034120351203612037120381203912040120411204212043120441204512046120471204812049120501205112052120531205412055120561205712058120591206012061120621206312064120651206612067120681206912070120711207212073120741207512076120771207812079120801208112082120831208412085120861208712088120891209012091120921209312094120951209612097120981209912100121011210212103121041210512106121071210812109121101211112112121131211412115121161211712118121191212012121121221212312124121251212612127121281212912130121311213212133121341213512136121371213812139121401214112142121431214412145121461214712148121491215012151121521215312154121551215612157121581215912160121611216212163121641216512166121671216812169121701217112172121731217412175121761217712178121791218012181121821218312184121851218612187121881218912190121911219212193121941219512196121971219812199122001220112202122031220412205122061220712208122091221012211122121221312214122151221612217122181221912220122211222212223122241222512226122271222812229122301223112232122331223412235122361223712238122391224012241122421224312244122451224612247122481224912250122511225212253122541225512256122571225812259122601226112262122631226412265122661226712268122691227012271122721227312274122751227612277122781227912280122811228212283122841228512286122871228812289122901229112292122931229412295122961229712298122991230012301123021230312304123051230612307123081230912310123111231212313123141231512316123171231812319123201232112322123231232412325123261232712328123291233012331123321233312334123351233612337123381233912340123411234212343123441234512346123471234812349123501235112352123531235412355123561235712358123591236012361123621236312364123651236612367123681236912370123711237212373123741237512376123771237812379123801238112382123831238412385123861238712388123891239012391123921239312394123951239612397123981239912400124011240212403124041240512406124071240812409124101241112412124131241412415124161241712418124191242012421124221242312424124251242612427124281242912430124311243212433124341243512436124371243812439124401244112442124431244412445124461244712448124491245012451124521245312454124551245612457124581245912460124611246212463124641246512466124671246812469124701247112472124731247412475124761247712478124791248012481124821248312484124851248612487124881248912490124911249212493124941249512496124971249812499125001250112502125031250412505125061250712508125091251012511125121251312514125151251612517125181251912520125211252212523125241252512526125271252812529125301253112532125331253412535125361253712538125391254012541125421254312544125451254612547125481254912550125511255212553125541255512556125571255812559125601256112562125631256412565125661256712568125691257012571125721257312574125751257612577125781257912580125811258212583125841258512586125871258812589125901259112592125931259412595125961259712598125991260012601126021260312604126051260612607126081260912610126111261212613126141261512616126171261812619126201262112622126231262412625126261262712628126291263012631126321263312634126351263612637126381263912640126411264212643126441264512646126471264812649126501265112652126531265412655126561265712658126591266012661126621266312664126651266612667126681266912670126711267212673126741267512676126771267812679126801268112682126831268412685126861268712688126891269012691126921269312694126951269612697126981269912700127011270212703127041270512706127071270812709127101271112712127131271412715127161271712718127191272012721127221272312724127251272612727127281272912730127311273212733127341273512736127371273812739127401274112742127431274412745127461274712748127491275012751127521275312754127551275612757127581275912760127611276212763127641276512766127671276812769127701277112772127731277412775127761277712778127791278012781127821278312784127851278612787127881278912790127911279212793127941279512796127971279812799128001280112802128031280412805128061280712808128091281012811128121281312814128151281612817128181281912820128211282212823128241282512826128271282812829128301283112832128331283412835128361283712838128391284012841128421284312844128451284612847128481284912850128511285212853128541285512856128571285812859128601286112862128631286412865128661286712868128691287012871128721287312874128751287612877128781287912880128811288212883128841288512886128871288812889128901289112892128931289412895128961289712898128991290012901129021290312904129051290612907129081290912910129111291212913129141291512916129171291812919129201292112922129231292412925129261292712928129291293012931129321293312934129351293612937129381293912940129411294212943129441294512946129471294812949129501295112952129531295412955129561295712958129591296012961129621296312964129651296612967129681296912970129711297212973129741297512976129771297812979129801298112982129831298412985129861298712988129891299012991129921299312994129951299612997129981299913000130011300213003130041300513006130071300813009130101301113012130131301413015130161301713018130191302013021130221302313024130251302613027130281302913030130311303213033130341303513036130371303813039130401304113042130431304413045130461304713048130491305013051130521305313054130551305613057130581305913060130611306213063130641306513066130671306813069130701307113072130731307413075130761307713078130791308013081130821308313084130851308613087130881308913090130911309213093130941309513096130971309813099131001310113102131031310413105131061310713108131091311013111131121311313114131151311613117131181311913120131211312213123131241312513126131271312813129131301313113132131331313413135131361313713138131391314013141131421314313144131451314613147131481314913150131511315213153131541315513156131571315813159131601316113162131631316413165131661316713168131691317013171131721317313174131751317613177131781317913180131811318213183131841318513186131871318813189131901319113192131931319413195131961319713198131991320013201132021320313204132051320613207132081320913210132111321213213132141321513216132171321813219132201322113222132231322413225132261322713228132291323013231132321323313234132351323613237132381323913240132411324213243132441324513246132471324813249132501325113252132531325413255132561325713258132591326013261132621326313264132651326613267132681326913270132711327213273132741327513276132771327813279132801328113282132831328413285132861328713288132891329013291132921329313294132951329613297132981329913300133011330213303133041330513306133071330813309133101331113312133131331413315133161331713318133191332013321133221332313324133251332613327133281332913330133311333213333133341333513336133371333813339133401334113342133431334413345133461334713348133491335013351133521335313354133551335613357133581335913360133611336213363
  1. import asyncio
  2. import json
  3. import logging
  4. import math
  5. import os
  6. import posixpath
  7. import re
  8. import secrets
  9. import shutil
  10. import time
  11. from contextlib import asynccontextmanager
  12. from datetime import datetime, timedelta, timezone
  13. from logging.handlers import RotatingFileHandler
  14. from pathlib import Path, PurePosixPath
  15. from typing import NamedTuple, cast
  16. from urllib.parse import urlparse
  17. from fastapi import FastAPI
  18. from fastapi.encoders import jsonable_encoder
  19. from fastapi.exceptions import RequestValidationError
  20. from fastapi.responses import FileResponse, JSONResponse
  21. from fastapi.staticfiles import StaticFiles
  22. from sqlalchemy import delete, func, inspect as sa_inspect, or_, select, text, update
  23. from sqlalchemy.engine import CursorResult
  24. from sqlalchemy.exc import IntegrityError
  25. from sqlalchemy.ext.asyncio import AsyncSession
  26. from sqlalchemy.orm.exc import StaleDataError
  27. from backend.app.api.routes import (
  28. ams_history,
  29. announcements,
  30. api_keys,
  31. archive_purge,
  32. archives,
  33. auth,
  34. bug_report,
  35. bulk,
  36. camera,
  37. camwall,
  38. cloud,
  39. connected_apps,
  40. discovery,
  41. external_links,
  42. filaments,
  43. finance,
  44. firmware,
  45. github_backup,
  46. groups,
  47. ha_sensors,
  48. inventory,
  49. kprofiles,
  50. label_templates,
  51. labels,
  52. library,
  53. library_tags,
  54. library_trash,
  55. library_variants,
  56. local_backup,
  57. local_presets,
  58. location_ha_sensors,
  59. maintenance,
  60. makerworld,
  61. manyfold,
  62. metrics,
  63. mfa,
  64. notification_templates,
  65. notifications,
  66. obico,
  67. octoeverywhere,
  68. orca_cloud,
  69. overlay_branding,
  70. pending_uploads,
  71. pipeline_runs,
  72. print_log,
  73. print_queue,
  74. printer_locations,
  75. printer_sensor_history,
  76. printers,
  77. projects,
  78. scheduled_dryings,
  79. settings as settings_routes,
  80. slice_jobs,
  81. slicer_pipelines,
  82. slicer_presets,
  83. smart_plugs,
  84. sponsor_prompt,
  85. spoolbuddy,
  86. spoolman,
  87. spoolman_inventory,
  88. support,
  89. system,
  90. updates,
  91. user_notifications,
  92. users,
  93. virtual_printers,
  94. webhook,
  95. websocket,
  96. )
  97. from backend.app.api.routes.maintenance import (
  98. _get_printer_maintenance_internal,
  99. ensure_default_types,
  100. maintenance_alert_values,
  101. )
  102. from backend.app.api.routes.support import init_debug_logging
  103. from backend.app.core.config import APP_VERSION, settings as app_settings
  104. from backend.app.core.database import async_session, engine, init_db
  105. from backend.app.core.static_assets import AssetStaticFiles
  106. from backend.app.core.tasks import spawn_background_task
  107. from backend.app.core.websocket import ws_manager
  108. from backend.app.services import kprofile_drift, pause_eta, print_dispatch_context, slot_unlink_grace
  109. from backend.app.services.ams_assignment_lock import get_ams_assignment_lock
  110. from backend.app.services.archive import ArchiveService, peek_plate_index_in_3mf, swap_plate_suffix
  111. from backend.app.services.archive_cost_estimate import (
  112. schedule_archive_cost_estimate as _schedule_archive_cost_estimate,
  113. settle_archive_cost_estimate as _settle_archive_cost_estimate,
  114. )
  115. from backend.app.services.archive_purge import archive_purge_service
  116. from backend.app.services.bambu_ftp import (
  117. RECOVERY_DOWNLOAD_MAX_TIMEOUT,
  118. FileNotOnPrinterError,
  119. cache_3mf_download,
  120. clear_3mf_cache,
  121. download_file_async,
  122. download_file_try_paths_async,
  123. ftps_handshake_blocked,
  124. get_cached_3mf,
  125. get_ftp_retry_settings,
  126. normalize_3mf_name,
  127. with_ftp_retry,
  128. )
  129. from backend.app.services.bambu_mqtt import PrinterState
  130. from backend.app.services.camera_light import camera_light
  131. from backend.app.services.energy_plug import energy_plug_candidates, select_energy_reading
  132. from backend.app.services.github_backup import github_backup_service
  133. from backend.app.services.ha_sensor_manager import ha_sensor_manager
  134. from backend.app.services.hms_errors import hms_fault_counts as _hms_fault_counts
  135. from backend.app.services.library_trash import library_trash_service
  136. from backend.app.services.local_backup import local_backup_service
  137. from backend.app.services.location_ha_sensor_manager import location_ha_sensor_manager
  138. from backend.app.services.mqtt_relay import mqtt_relay
  139. from backend.app.services.mqtt_smart_plug import mqtt_smart_plug_service
  140. from backend.app.services.notification_service import notification_service
  141. from backend.app.services.notify_live_activities import notify_live_activities
  142. from backend.app.services.notify_widgets import notify_widgets
  143. from backend.app.services.obico_detection import obico_detection_service
  144. from backend.app.services.octoeverywhere_detection import octoeverywhere_detection_service
  145. from backend.app.services.plug_energy import get_plug_energy as _get_plug_energy
  146. from backend.app.services.print_cost_estimate import plate_scoped_run_estimate as _plate_scoped_run_estimate
  147. from backend.app.services.print_scheduler import scheduler as print_scheduler
  148. from backend.app.services.print_storage import (
  149. REASON_FTP_TRANSFER_FAILED,
  150. REASON_FTPS_COOLOFF,
  151. external_storage_present,
  152. ftp_probe_paths,
  153. print_file_reachable_over_ftp,
  154. )
  155. from backend.app.services.printer_manager import (
  156. init_printer_connections,
  157. parse_plate_id,
  158. printer_manager,
  159. printer_state_to_dict,
  160. printer_tray_layout,
  161. resolve_plate_id,
  162. )
  163. from backend.app.services.slot_kprofile import find_slot_kprofile_for_extruder
  164. from backend.app.services.slot_nozzle import (
  165. nozzle_diameter_for_extruder,
  166. nozzle_flow_for_extruder,
  167. )
  168. from backend.app.services.smart_plug_manager import smart_plug_manager
  169. from backend.app.services.spool_assignment_notifications import (
  170. notify_missing_spool_assignments_on_print_start,
  171. )
  172. from backend.app.services.spool_filament_preset import printer_safe_filament_id
  173. from backend.app.services.spoolman import close_spoolman_client, get_spoolman_client, init_spoolman_client
  174. from backend.app.services.spoolman_tracking import (
  175. cleanup_tracking as _cleanup_spoolman_tracking,
  176. release_archived_slot_rows as _release_archived_spoolman_slots,
  177. report_usage as _report_spoolman_usage,
  178. store_print_data as _store_spoolman_print_data,
  179. )
  180. from backend.app.services.telegram_reactions import telegram_reaction_poller
  181. from backend.app.services.wled import wled_manager
  182. from backend.app.utils.ams_drying import is_drying_active, temperature_alarm_suppressed
  183. from backend.app.utils.ams_humidity import ams_humidity_percent
  184. from backend.app.utils.filament_types import printer_filament_type
  185. from backend.app.utils.fts_routing import extruder_for_inlet
  186. from backend.app.utils.local_time import to_naive_utc, utcnow_naive
  187. from backend.app.utils.print_jobs import is_internal_printer_job
  188. from backend.app.utils.tray_ids import AmsType, TrayLayout, external_tray_id, unit_ams_type, unit_label
  189. # =============================================================================
  190. # Dependency Check - runs before other imports to give helpful error messages
  191. # =============================================================================
  192. def _start_error_server(missing_packages: list):
  193. """Start a minimal HTTP server to display dependency errors in browser."""
  194. import os
  195. import signal
  196. from http.server import BaseHTTPRequestHandler, HTTPServer
  197. packages_html = "".join(f"<li><code>{p}</code></li>" for p in missing_packages)
  198. html = f"""<!DOCTYPE html>
  199. <html>
  200. <head>
  201. <title>Bambuddy - Setup Required</title>
  202. <style>
  203. body {{
  204. font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
  205. background: #0f172a; color: #e2e8f0;
  206. display: flex; justify-content: center; align-items: center;
  207. min-height: 100vh; margin: 0; padding: 20px; box-sizing: border-box;
  208. }}
  209. .container {{
  210. background: #1e293b; border-radius: 12px; padding: 40px;
  211. max-width: 600px; text-align: center; box-shadow: 0 4px 20px rgba(0,0,0,0.3);
  212. }}
  213. h1 {{ color: #f87171; margin-bottom: 10px; }}
  214. h2 {{ color: #94a3b8; font-weight: normal; margin-top: 0; }}
  215. .packages {{
  216. background: #0f172a; border-radius: 8px; padding: 20px;
  217. margin: 20px 0; text-align: left;
  218. }}
  219. .packages ul {{ margin: 0; padding-left: 20px; }}
  220. .packages li {{ color: #fbbf24; margin: 8px 0; }}
  221. .command {{
  222. background: #0f172a; border-radius: 8px; padding: 15px 20px;
  223. margin: 15px 0; font-family: monospace; color: #4ade80;
  224. text-align: left; overflow-x: auto;
  225. }}
  226. .note {{ color: #94a3b8; font-size: 14px; margin-top: 20px; }}
  227. </style>
  228. </head>
  229. <body>
  230. <div class="container">
  231. <h1>Setup Required</h1>
  232. <h2>Missing Python packages</h2>
  233. <div class="packages"><ul>{packages_html}</ul></div>
  234. <p>To fix, run this command on your server:</p>
  235. <div class="command">pip install -r requirements.lock</div>
  236. <p>Or if using a virtual environment:</p>
  237. <div class="command">./venv/bin/pip install -r requirements.lock</div>
  238. <p class="note">After installing, restart Bambuddy:<br>
  239. <code>sudo systemctl restart bambuddy</code></p>
  240. </div>
  241. </body>
  242. </html>"""
  243. class ErrorHandler(BaseHTTPRequestHandler):
  244. def do_GET(self):
  245. self.send_response(503)
  246. self.send_header("Content-type", "text/html")
  247. self.end_headers()
  248. self.wfile.write(html.encode())
  249. def log_message(self, format, *args):
  250. print(f"[Error Server] {args[0]}")
  251. port = int(os.environ.get("PORT", 8000))
  252. print(f"\nStarting error server on http://0.0.0.0:{port}")
  253. print("Visit this URL in your browser to see the error details.\n")
  254. server = HTTPServer(("0.0.0.0", port), ErrorHandler) # nosec B104
  255. def shutdown(signum, frame):
  256. print("\nShutting down error server...")
  257. raise SystemExit(0)
  258. signal.signal(signal.SIGTERM, shutdown)
  259. signal.signal(signal.SIGINT, shutdown)
  260. server.serve_forever()
  261. def check_dependencies():
  262. """Check that all required packages are installed."""
  263. missing = []
  264. # Map of import name -> package name (for pip install)
  265. required = {
  266. "jwt": "PyJWT",
  267. "fastapi": "fastapi",
  268. "uvicorn": "uvicorn",
  269. "sqlalchemy": "sqlalchemy",
  270. "aiosqlite": "aiosqlite",
  271. "pydantic": "pydantic",
  272. "paho.mqtt": "paho-mqtt",
  273. }
  274. for module, package in required.items():
  275. try:
  276. __import__(module)
  277. except ImportError:
  278. missing.append(package)
  279. if missing:
  280. print("\n" + "=" * 60)
  281. print("ERROR: Missing required Python packages!")
  282. print("=" * 60)
  283. print(f"\nMissing packages: {', '.join(missing)}")
  284. print("\nTo fix, run:")
  285. print(" pip install -r requirements.lock")
  286. print("\nOr if using a virtual environment:")
  287. print(" ./venv/bin/pip install -r requirements.lock")
  288. print("=" * 60 + "\n")
  289. _start_error_server(missing)
  290. check_dependencies()
  291. # =============================================================================
  292. # Import settings first for logging configuration
  293. # Configure logging based on settings
  294. # DEBUG=true -> DEBUG level, else use LOG_LEVEL setting
  295. log_level_str = "DEBUG" if app_settings.debug else app_settings.log_level.upper()
  296. log_level = getattr(logging, log_level_str, logging.INFO)
  297. # Trace ID column ([-] when no request scope is active — startup, MQTT
  298. # callbacks, scheduled tasks not chained from a request — so the column
  299. # stays visually aligned and missing values are obvious in grep). See
  300. # backend/app/core/trace.py for the ContextVar that feeds this slot.
  301. log_format = "%(asctime)s %(levelname)s [%(name)s] [%(trace_id)s] %(message)s"
  302. # Create root logger
  303. root_logger = logging.getLogger()
  304. root_logger.setLevel(log_level)
  305. # Trace-ID injection: this filter populates record.trace_id from the
  306. # per-request ContextVar so the format string above can reference it.
  307. # Attached to each HANDLER (not the root logger) because Python's
  308. # logging semantics only invoke a logger's filters on records that
  309. # *originated* at that logger — records propagated up from child
  310. # loggers (every named logger in the app) never trigger root's filter.
  311. # Putting it on the handlers means every record any handler emits gets
  312. # trace_id injected just before the formatter runs, regardless of which
  313. # logger created the record. Without this, the formatter raises
  314. # KeyError on every child-logger record and the record is silently
  315. # dropped — which is exactly the "logs/bambuddy.log only shows logs
  316. # partially" bug we hit. See backend/app/core/trace.py for the
  317. # ContextVar the filter reads.
  318. from backend.app.core.logging_filters import RefusedHeaderRedactFilter
  319. from backend.app.core.response_redaction import RefusedHeaderResponseMask
  320. from backend.app.core.trace import TraceIDFilter
  321. _trace_id_filter = TraceIDFilter()
  322. # Console handler - always enabled
  323. console_handler = logging.StreamHandler()
  324. console_handler.setLevel(log_level)
  325. console_handler.setFormatter(logging.Formatter(log_format))
  326. console_handler.addFilter(_trace_id_filter)
  327. # A refused header's value (often a token) masked before the line is written.
  328. _refused_header_filter = RefusedHeaderRedactFilter()
  329. console_handler.addFilter(_refused_header_filter)
  330. root_logger.addHandler(console_handler)
  331. def _attach_file_handler_to_uvicorn(handler: logging.Handler) -> None:
  332. """Send uvicorn's own loggers to bambuddy.log as well.
  333. Neither reaches the root handler: uvicorn.access has propagate=False, and
  334. uvicorn.error propagates only to "uvicorn", which has propagate=False.
  335. uvicorn.access is the record of which endpoint changed server state;
  336. uvicorn.error is where an unhandled route error is logged with its
  337. traceback ("Exception in ASGI application").
  338. """
  339. for name in ("uvicorn.access", "uvicorn.error"):
  340. logging.getLogger(name).addHandler(handler)
  341. # File handler - only in production or if explicitly enabled
  342. if app_settings.log_to_file:
  343. log_file = app_settings.log_dir / "bambuddy.log"
  344. file_handler = RotatingFileHandler(
  345. log_file,
  346. maxBytes=app_settings.log_max_bytes,
  347. backupCount=app_settings.log_backup_count,
  348. encoding="utf-8",
  349. )
  350. file_handler.setLevel(log_level)
  351. file_handler.setFormatter(logging.Formatter(log_format))
  352. file_handler.addFilter(_trace_id_filter)
  353. file_handler.addFilter(_refused_header_filter)
  354. root_logger.addHandler(file_handler)
  355. logging.info("Logging to file: %s", log_file)
  356. # Pipe uvicorn's HTTP access log to bambuddy.log too. Uvicorn ships its
  357. # access logger with propagate=False by default, so without this attach
  358. # there is no on-disk record of which endpoint triggered a server-state
  359. # change — the rogue stop_print mystery on 2026-04-26 was untraceable
  360. # for exactly this reason. Filtered to write methods only
  361. # (POST/PUT/PATCH/DELETE) so the high-volume status-poll GETs from the
  362. # frontend don't churn the rotation window faster than it's useful.
  363. from backend.app.core.logging_filters import (
  364. CancelledPoolNoiseFilter,
  365. WriteRequestsOnlyFilter,
  366. )
  367. _attach_file_handler_to_uvicorn(file_handler)
  368. uvicorn_access_logger = logging.getLogger("uvicorn.access")
  369. uvicorn_access_logger.addFilter(WriteRequestsOnlyFilter())
  370. # Uvicorn's access logger has propagate=False (its own default), so the
  371. # root-attached TraceIDFilter never sees these records. Attach a
  372. # second instance directly so HTTP access lines carry the same trace
  373. # ID column as the application logs they correlate with.
  374. uvicorn_access_logger.addFilter(TraceIDFilter())
  375. # Drop SQLAlchemy connection-pool log noise that's caused by Starlette's
  376. # BaseHTTPMiddleware cancelling the inner task scope on client
  377. # disconnect (#1112). The cancel-safe `get_db` already prevents the
  378. # underlying transaction leak; this filter only suppresses the residual
  379. # log records that pre-existing pools still emit during their cleanup.
  380. logging.getLogger("sqlalchemy.pool").addFilter(CancelledPoolNoiseFilter())
  381. # Query-string tokens out of uvicorn's request and WebSocket lines, on every
  382. # handler (console included), whether or not file logging is on.
  383. from backend.app.core.logging_filters import ( # noqa: E402
  384. QueryTokenRedactFilter,
  385. ShutdownCancelledRequestFilter,
  386. )
  387. # First, so the traceback of a stream cancelled at shutdown is dropped before
  388. # the filters below render it.
  389. logging.getLogger("uvicorn.error").addFilter(ShutdownCancelledRequestFilter())
  390. for _uvicorn_logger in ("uvicorn.access", "uvicorn.error"):
  391. logging.getLogger(_uvicorn_logger).addFilter(QueryTokenRedactFilter())
  392. logging.getLogger(_uvicorn_logger).addFilter(_refused_header_filter)
  393. # These stay at WARNING even in debug mode: at INFO/DEBUG httpx/httpcore log
  394. # full request URLs, and some carry credentials (bot tokens, webhook URLs, API
  395. # keys in the query). Same as the runtime log-level toggle in support.py.
  396. logging.getLogger("httpcore").setLevel(logging.WARNING)
  397. logging.getLogger("httpx").setLevel(logging.WARNING)
  398. # aiosqlite logs every statement with its parameters at DEBUG.
  399. logging.getLogger("aiosqlite").setLevel(logging.WARNING)
  400. # Reduce noise from third-party libraries in production
  401. if not app_settings.debug:
  402. logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
  403. logging.getLogger("paho.mqtt").setLevel(logging.WARNING)
  404. logging.info("Bambuddy starting - debug=%s, log_level=%s", app_settings.debug, log_level_str)
  405. # Track active prints: {(printer_id, filename): archive_id}
  406. _active_prints: dict[tuple[int, str], int] = {}
  407. # #1721: stage-22 pre-captured finish photo bytes per printer. on_finish_photo_moment
  408. # fires when stg_cur enters 22 ("Filament unloading") at end-of-print — toolhead
  409. # parked, bed not yet dropped — and grabs a single camera frame into this cache.
  410. # `_background_finish_photo` (inside on_print_complete) consumes the cached bytes
  411. # instead of running its own grab-now chain when present, so the finish photo
  412. # captures the better-framed pre-bed-drop moment without us having to force
  413. # timelapse on at dispatch (the #1397 mechanism that caused #1721's per-layer
  414. # nozzle parking on slicer profiles with Timelapse Type = Smooth).
  415. #
  416. # #2708: the bytes in here are ALWAYS already rotated by the printer's
  417. # camera_rotation. `on_finish_photo_moment` owns that, because one of its
  418. # sources (the #1867 in-print bank) is rotated before it ever reaches the
  419. # bank and the others are not — so the consumer can't tell them apart and
  420. # must not rotate again.
  421. _stage22_finish_frames: dict[int, bytes] = {}
  422. # #1790: per-printer producer-done event. Set by `on_finish_photo_moment` in its
  423. # `finally` block (whether it captured a frame or not). The consumer in
  424. # `_background_finish_photo` waits on it before reading `_stage22_finish_frames`
  425. # so the FINISH-state fallback path — where moment and completion are dispatched
  426. # back-to-back — doesn't race past the producer with an empty pop, and the
  427. # consumer's RTSP fallback can't collide with the producer's still-in-flight RTSP
  428. # grab (Bambu printers allow only one RTSP client at a time).
  429. _stage22_finish_in_flight: dict[int, asyncio.Event] = {}
  430. # #1867: rolling "last in-print camera frame" per printer. Refreshed on
  431. # layer-change and on print-progress advances (#2547) while the model is still
  432. # printing, then consumed by the FINISH-state finish-photo path when the
  433. # dispatcher recorded that it injected End G-code into this print. Bambu
  434. # reports gcode_state=FINISH AFTER the user End G-code (e.g. SwapMod
  435. # plate-swap) has run, so a live grab there would capture the swapped/empty
  436. # plate.
  437. #
  438. # The load-bearing property: both drivers are print telemetry that stops before
  439. # the End G-code executes — no further layer_num increases, and mc_percent
  440. # freezes — so the last banked frame is always the finished print before the
  441. # swap. Anything added as a third driver must hold that same property.
  442. _inprint_frame_bank: dict[int, bytes] = {}
  443. # Monotonic timestamp of the last banked frame per printer — throttles banking
  444. # so tall prints don't add a camera grab on every layer.
  445. _inprint_frame_bank_ts: dict[int, float] = {}
  446. # Minimum seconds between banked frames, except the final object layer which
  447. # always refreshes for the best framing.
  448. _INPRINT_BANK_MIN_INTERVAL = 25.0
  449. # Per-printer "connected" edge tracker. Used by `on_printer_status_change`
  450. # to fire `reconcile_stale_active_prints` exactly once per (re)connection
  451. # (#1542 follow-up — power-cycle ghost prints). The value is True after
  452. # the first connected status update for that connection; transitions back
  453. # to False whenever we observe `state.connected = False` so the next
  454. # reconnect re-arms reconciliation. Keyed by printer_id.
  455. _printer_reconciled_since_connect: dict[int, bool] = {}
  456. # Same edge, same keying, for priming the printer's calibration table exactly
  457. # once per (re)connection. Nothing else asks for it on connect: state.kprofiles
  458. # is otherwise filled only when someone opens the Profiles page or Configure
  459. # Slot, when a GitHub backup runs, or when the printer happens to answer
  460. # somebody else's query on the report topic. Until then the AMS slot card has
  461. # no K value to show on the printers whose trays carry none of their own
  462. # (#2854 — H2-series report cali_idx and nothing more).
  463. _printer_kprofiles_primed_since_connect: dict[int, bool] = {}
  464. # Track expected prints from reprint/scheduled (skip auto-archiving for these)
  465. # {(printer_id, filename): archive_id}
  466. _expected_prints: dict[tuple[int, str], int] = {}
  467. # Archives of the running prints Bambuddy dispatched itself: adopted from an
  468. # expected print, or found with Bambuddy's dispatch marker. _active_prints holds
  469. # every print on_print_start archived, those started from Bambu Studio, Handy or
  470. # the printer's screen included, so it is no proof of ownership on its own.
  471. _owned_print_archive_ids: set[int] = set()
  472. # What a dispatch registered for its print, keyed (printer_id, archive_id):
  473. # plates of one archive print on two printers at once, each with its own.
  474. # Track AMS mapping for prints: {(printer_id, archive_id): [global_tray_id_per_slot]}
  475. # Used by usage tracker to map 3MF slots to physical AMS trays
  476. _print_ams_mappings: dict[tuple[int, int], list[int]] = {}
  477. # Track cost center selection for the current print run: {(printer_id, archive_id): cost_center_id}
  478. _print_cost_center_ids: dict[tuple[int, int], int] = {}
  479. # Track plate_id for prints from multi-plate 3MFs: {(printer_id, archive_id): plate_id}
  480. # Used by usage tracker to scope 3MF parsing to the dispatched plate (#1697).
  481. # Populated by direct-Print and queue dispatch paths; queue prints also have a
  482. # redundant queue-item lookup in on_print_start so this dict isn't load-bearing
  483. # for the queue path. Cleared on print completion or TTL eviction.
  484. _print_plate_ids: dict[tuple[int, int], int] = {}
  485. # Track progress milestones for notifications: {printer_id: last_milestone_notified}
  486. # Milestones are 25, 50, 75. Value of 0 means no milestone notified yet for current print.
  487. _last_progress_milestone: dict[int, int] = {}
  488. # Track whether first layer complete notification has been sent for current print
  489. _first_layer_notified: dict[int, bool] = {}
  490. # Track whether we already sent a kill-switch stop for the current unauthorized print
  491. _unauthorized_print_kill_sent: set[int] = set()
  492. # The MQTT status callback is a hot path. Cache the two-setting kill-switch
  493. # lookup briefly so an unknown active print does not query the database on
  494. # every status frame. A short TTL keeps settings changes responsive.
  495. _KILL_SWITCH_SETTING_CACHE_TTL_SECONDS = 5.0
  496. _kill_switch_setting_cache: tuple[bool, float] | None = None
  497. # Provider notification started when the kill switch stops a print. The later
  498. # MQTT print-complete callback awaits this task and only sends its regular
  499. # provider notification when the immediate attempt failed.
  500. _kill_switch_notification_tasks: dict[int, asyncio.Task[bool]] = {}
  501. # HMS faults already notified, with when each was last seen:
  502. # {printer_id: {fault key: timestamp}}. Tracked per fault, not per printer: a
  503. # fault that flickers beside a held notice was forgotten on every gap and
  504. # notified on every return (#3226). The window is long because the measured
  505. # gaps reach 64 s.
  506. _notified_hms_errors: dict[int, dict[str, float]] = {}
  507. _HMS_CLEAR_GRACE_SECONDS = 600.0
  508. # The print state each printer was last seen in, and the faults recorded before
  509. # it last changed, which are forgotten as soon as they are gone (see
  510. # _take_new_hms_faults).
  511. _hms_print_state: dict[int, str] = {}
  512. _hms_forget_when_gone: dict[int, set[str]] = {}
  513. # Track timelapse file baselines at print start: {printer_id: set of video filenames}
  514. # Used for snapshot-diff detection at print completion
  515. _timelapse_baselines: dict[int, set[str]] = {}
  516. # Printers whose baseline above was taken off a card that could not be read,
  517. # so it is empty without the card being empty (#3412).
  518. _timelapse_baselines_unread: set[int] = set()
  519. # Track printers waiting for bed to cool after print completion.
  520. # Event-driven: fires when bed_temper arrives via MQTT below threshold.
  521. # {printer_id: {"threshold": float, "filename": str, "registered_at": float}}
  522. _bed_cool_waiters: dict[int, dict] = {}
  523. # Track printers where the user explicitly stopped the print from the queue UI.
  524. # When on_print_complete fires with status "failed" for these printers we treat it
  525. # as "cancelled" (stopped by user) so the correct notification email is sent.
  526. _user_stopped_printers: set[int] = set()
  527. # Printers whose running print the kill switch stopped. Their completion is a
  528. # cancel with the reason "killSwitch", not the "failed" the printer reports.
  529. _kill_switch_stopped: set[int] = set()
  530. # Offline-notification edge state (#1752): fire `on_printer_offline` exactly
  531. # once when a printer transitions connected → disconnected. `_printer_last_connected`
  532. # holds the previous observation so we only fire on the True → False edge (a
  533. # False → False repeat doesn't notify; an initial False at startup doesn't
  534. # notify either, since there's no prior True). `_printer_offline_notify_tasks`
  535. # holds the per-printer pending asyncio task that fires the notification
  536. # after a debounce window — cancelled if the printer reconnects before the
  537. # window elapses, so transient MQTT blips don't flood the user.
  538. _printer_last_connected: dict[int, bool] = {}
  539. # Printers whose relay offline event went out, so the reconnect publishes the
  540. # matching online event and a blip shorter than the debounce publishes neither.
  541. _printer_offline_published: set[int] = set()
  542. _printer_offline_notify_tasks: dict[int, asyncio.Task] = {}
  543. # Debounce: a printer must stay offline this long before we notify. Sized
  544. # against the staleness path (`bambu_mqtt.py::STALE_RECONNECT_COOLDOWN = 30s`)
  545. # so a single stale-trigger cooldown isn't enough to fire — only a real
  546. # offline that survives one reconnect attempt notifies.
  547. _PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS = 60.0
  548. # HMS short-code → failure_reason key. Used by _dispatch_archive_update
  549. # when status="failed" to label the print's failure_reason in archives.
  550. #
  551. # Earlier code matched on `module` alone (e.g. "any module 0x0C HMS → Layer shift"),
  552. # which is wrong on two counts:
  553. # 1. Real layer-shift codes live in module 0x03 (see Bambu wiki), not 0x0C.
  554. # 2. Module 0x0C is "Motion Controller" — broad category that also covers cameras
  555. # and visual markers, AND the H2D firmware emits a 0x0C HMS (0C00_001B, not in
  556. # the public wiki) as part of its user-cancel sequence. Matching on the module
  557. # alone caused user-cancellations to be archived as "Layer shift" failures.
  558. # We now match by full short code only — anything not in this map leaves
  559. # failure_reason=None rather than guessing.
  560. # Values are the canonical camelCase failure-reason keys, NOT display labels
  561. # (issue #2974). The vocabulary is enforced on writes by
  562. # ``_FAILURE_REASON_KEYS`` in ``api/routes/print_log.py`` and rendered through
  563. # ``t('editArchive.failureReasons.<key>')`` on both the archive editor and the
  564. # Statistics breakdown. Storing a label here instead put a second spelling of
  565. # the same cause into one column: the Failure Analysis widget groups on the raw
  566. # value, so a print the backend classified and an identical one a user
  567. # classified counted as two different reasons, and the label form could never
  568. # be translated because there was no key for ``t()`` to resolve.
  569. _HMS_FAILURE_REASONS: dict[str, str] = {
  570. # Layer shift / step loss
  571. "0300_4057": "layerShift",
  572. "0300_4068": "layerShift",
  573. "0300_800C": "layerShift",
  574. # Filament runout (printer-side & per-AMS-slot)
  575. "0300_8004": "filamentRunout",
  576. "0700_8011": "filamentRunout",
  577. "0701_8011": "filamentRunout",
  578. "0702_8011": "filamentRunout",
  579. "0703_8011": "filamentRunout",
  580. "0704_8011": "filamentRunout",
  581. "0705_8011": "filamentRunout",
  582. "0706_8011": "filamentRunout",
  583. "0707_8011": "filamentRunout",
  584. "07FF_8011": "filamentRunout",
  585. # Clogged nozzle / extruder
  586. "0300_4006": "cloggedNozzle",
  587. "0300_8016": "cloggedNozzle",
  588. "0300_801C": "cloggedNozzle",
  589. "0700_8003": "cloggedNozzle",
  590. "0700_8007": "cloggedNozzle",
  591. "0700_8013": "cloggedNozzle",
  592. "0701_8003": "cloggedNozzle",
  593. "0701_8007": "cloggedNozzle",
  594. "0701_8013": "cloggedNozzle",
  595. "0702_8003": "cloggedNozzle",
  596. # AI print monitoring — spaghetti / the model coming off the plate.
  597. # `spaghettiDetached` is a key the archive editor already offers for this
  598. # failure mode, not a new one, so a derived reason opens the dropdown on
  599. # that option rather than blank — and survives the next save, which clears
  600. # any value the editor does not recognise.
  601. #
  602. # A module-0x0C row is safe here despite the warning above: that warning is
  603. # about matching on the module alone, and 0C00_8042 is a full short code
  604. # with a documented meaning ("The AI print monitor has detected a spaghetti
  605. # defect", hms_errors.py). The H2D cancel echo is 0C00_001B, so the two
  606. # cannot collide.
  607. #
  608. # 0300_8003's own text ends "before continuing your print", but on an X2D
  609. # it arrives with the print already paused, offering only
  610. # RESUME_PRINTING_DEFECTS / STOP_PRINTING — a halt waiting on the user.
  611. #
  612. # Two neighbours are left out on purpose, so this does not get re-derived:
  613. # * 0C00_C004 "Possible spaghetti failure was detected." — "possible"
  614. # reads as a warning about a print that is still running, not a halt.
  615. # * 0300_800A is AI monitoring too, but it reports a filament pile-up in
  616. # the waste chute. That is not the print failing.
  617. #
  618. # That line is drawn from the text, not from `severity`, because severity
  619. # cannot draw it: 0300_8003 reaches us through `print_error`, a bare
  620. # module/error word with no level in it, and bambu_mqtt.py gives every
  621. # print_error entry a flat severity=3. Matching on the short code alone is
  622. # the right shape for derive_failure_reason, not an omission.
  623. "0300_8003": "spaghettiDetached",
  624. "0C00_8042": "spaghettiDetached",
  625. }
  626. def _hms_short_code(attr: int, code: int | str) -> str:
  627. """Build the canonical "MMMM_CCCC" HMS short code from raw attr/code values."""
  628. if isinstance(code, str):
  629. code_int = int(code.replace("0x", ""), 16) if code else 0
  630. else:
  631. code_int = int(code or 0)
  632. attr_int = int(attr or 0)
  633. return f"{(attr_int >> 16) & 0xFFFF:04X}_{code_int & 0xFFFF:04X}"
  634. def derive_failure_reason(status: str, hms_errors: list[dict] | None) -> str | None:
  635. """Derive the failure_reason key for an archived print.
  636. Returns "userCancelled" for cancelled/aborted prints; for failed prints,
  637. returns the first matching key from _HMS_FAILURE_REASONS, or None when
  638. no HMS code matches (don't guess — null is honest). The keys are the
  639. archive editor's vocabulary (_FAILURE_REASON_KEYS in print_log.py) and are
  640. translated at render time.
  641. """
  642. if status in ("aborted", "cancelled"):
  643. return "userCancelled"
  644. if status != "failed":
  645. return None
  646. for err in hms_errors or []:
  647. short_code = _hms_short_code(err.get("attr", 0), err.get("code", 0))
  648. if short_code in _HMS_FAILURE_REASONS:
  649. return _HMS_FAILURE_REASONS[short_code]
  650. return None
  651. # Track created_by_id for expected prints so the user email can be sent even when
  652. # the archive itself doesn't have created_by_id set (e.g. library-file-based prints).
  653. # {(printer_id, filename): created_by_id}
  654. _expected_print_creators: dict[tuple[int, str], int] = {}
  655. # Per-printer dedup for unknown_tag WS broadcasts. Keyed by
  656. # (ams_id, tray_id) -> (tag_uid, tray_uuid); we only re-broadcast when the
  657. # tag tuple changes for the slot. Cleared when the slot is reported empty
  658. # so remove + reinsert reliably re-prompts the UI.
  659. _unknown_tag_last_broadcast: dict[int, dict[tuple[int, int], tuple[str, str]]] = {}
  660. async def _broadcast_unknown_tag(
  661. *,
  662. printer_id: int,
  663. ams_id: int,
  664. tray_id: int,
  665. tag_uid: str,
  666. tray_uuid: str,
  667. tray_type: str | None = None,
  668. tray_color: str | None = None,
  669. tray_sub_brands: str | None = None,
  670. tray_count: int | None = None,
  671. ) -> None:
  672. """Broadcast unknown_tag, deduped so repeated MQTT pushes for the same slot+tag don't spam the UI."""
  673. _logger = logging.getLogger(__name__)
  674. slot_key = (ams_id, tray_id)
  675. tag_key = (tag_uid or "", tray_uuid or "")
  676. per_printer = _unknown_tag_last_broadcast.setdefault(printer_id, {})
  677. if per_printer.get(slot_key) == tag_key:
  678. _logger.debug(
  679. "unknown_tag deduped for printer=%d AMS=%d slot=%d tag=%s",
  680. printer_id,
  681. ams_id,
  682. tray_id,
  683. tag_key[0][:8] or tag_key[1][:8] or "(none)",
  684. )
  685. return
  686. _logger.info(
  687. "unknown_tag broadcast: printer=%d AMS=%d slot=%d type=%r color=%r tag=%s",
  688. printer_id,
  689. ams_id,
  690. tray_id,
  691. tray_type,
  692. tray_color,
  693. tag_key[0][:8] or tag_key[1][:8] or "(none)",
  694. )
  695. # Broadcast first; only commit the dedup if the WS write succeeds.
  696. # If broadcast raises, the next MQTT push retries instead of being
  697. # permanently silenced by a poisoned dedup entry.
  698. await ws_manager.broadcast(
  699. {
  700. "type": "unknown_tag",
  701. "printer_id": printer_id,
  702. "ams_id": ams_id,
  703. "tray_id": tray_id,
  704. "tag_uid": tag_uid,
  705. "tray_uuid": tray_uuid,
  706. "tray_type": tray_type,
  707. "tray_color": tray_color,
  708. "tray_sub_brands": tray_sub_brands,
  709. "tray_count": tray_count,
  710. }
  711. )
  712. per_printer[slot_key] = tag_key
  713. def _clear_unknown_tag_dedup(printer_id: int, ams_id: int, tray_id: int) -> None:
  714. """Drop the cached last-broadcast tag for a slot (called when slot reports empty or gets matched)."""
  715. per_printer = _unknown_tag_last_broadcast.get(printer_id)
  716. if per_printer is None:
  717. return
  718. per_printer.pop((ams_id, tray_id), None)
  719. # TTL for expected-print entries: evict registrations older than this to prevent
  720. # unbounded growth when a print is registered but never starts (e.g. printer
  721. # disconnect, app restart, print started from the printer panel).
  722. _EXPECTED_PRINT_TTL_SECONDS: int = 2 * 60 * 60 # 2 hours
  723. # Registration timestamps used for TTL eviction: {(printer_id, filename): monotonic_time}
  724. _expected_print_registered_at: dict[tuple[int, str], float] = {}
  725. # Cleanup loop interval
  726. _EXPECTED_PRINT_CLEANUP_INTERVAL: int = 15 * 60 # 15 minutes
  727. _expected_prints_cleanup_task: asyncio.Task | None = None
  728. _ACTIVE_PRINT_STATES: set[str] = {"RUNNING", "PRINTING", "PAUSE"}
  729. def _build_status_print_keys(printer_id: int, state: PrinterState) -> list[tuple[int, str]]:
  730. """Build filename keys for matching a printer status update to Bambuddy-owned jobs."""
  731. possible_keys: list[tuple[int, str]] = []
  732. filename = (state.gcode_file or state.current_print or "").strip()
  733. subtask_name = (state.subtask_name or "").strip()
  734. if subtask_name:
  735. possible_keys.append((printer_id, subtask_name))
  736. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  737. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  738. if filename:
  739. base_name = filename.rsplit("/", 1)[-1]
  740. if base_name.endswith(".gcode.3mf"):
  741. root_name = base_name[: -len(".gcode.3mf")]
  742. possible_keys.append((printer_id, root_name))
  743. possible_keys.append((printer_id, base_name))
  744. possible_keys.append((printer_id, f"{root_name}.gcode"))
  745. possible_keys.append((printer_id, f"{root_name}.3mf"))
  746. elif base_name.endswith(".3mf"):
  747. root_name = base_name[: -len(".3mf")]
  748. possible_keys.append((printer_id, root_name))
  749. possible_keys.append((printer_id, base_name))
  750. elif base_name.endswith(".gcode"):
  751. root_name = base_name[: -len(".gcode")]
  752. possible_keys.append((printer_id, root_name))
  753. possible_keys.append((printer_id, f"{root_name}.3mf"))
  754. possible_keys.append((printer_id, base_name))
  755. else:
  756. possible_keys.append((printer_id, base_name))
  757. possible_keys.append((printer_id, f"{base_name}.3mf"))
  758. return possible_keys
  759. async def _print_user_id_if_still_there(db, print_user: dict | None) -> int | None:
  760. """The id of who started the print, unless that user has been deleted since.
  761. Deleting a user forgets them in printer_manager, but a print that ended a
  762. moment before still holds them.
  763. """
  764. if not print_user or print_user.get("user_id") is None:
  765. return None
  766. from backend.app.models.user import User
  767. return await db.scalar(select(User.id).where(User.id == print_user["user_id"]))
  768. def _is_bambuddy_authorized_print_in_memory(printer_id: int, state: PrinterState) -> bool:
  769. """Check the cheap, process-local print ownership signals."""
  770. if printer_manager.get_current_print_user(printer_id):
  771. return True
  772. return any(
  773. key in _expected_prints or _active_prints.get(key) in _owned_print_archive_ids
  774. for key in _build_status_print_keys(printer_id, state)
  775. )
  776. def _reported_print_names(printer_id: int, state: PrinterState) -> set[str]:
  777. """The names the printer reports for its print, as it echoes them (spaces become underscores)."""
  778. return {_normalise_subtask_name(name) for _pid, name in _build_status_print_keys(printer_id, state)} - {""}
  779. async def _printing_archive_without_job_id(
  780. db, printer_id: int, reported: set[str], *, dispatched_only: bool, live_only: bool = False
  781. ):
  782. """The newest printing archive on ``printer_id`` of a print that reported no job id, under a reported name.
  783. An archive with a real id of its own is a print that reported one: a "0"
  784. print is not it, even under the same name (one whose completion was missed
  785. would otherwise cover it). ``dispatched_only`` keeps to archives carrying
  786. Bambuddy's dispatch marker; ``live_only`` leaves out deleted ones.
  787. """
  788. from backend.app.models.archive import PrintArchive
  789. query = select(PrintArchive).where(
  790. PrintArchive.printer_id == printer_id,
  791. PrintArchive.status == "printing",
  792. or_(PrintArchive.subtask_id.is_(None), PrintArchive.subtask_id.in_(("", "0"))),
  793. )
  794. if dispatched_only:
  795. query = query.where(or_(PrintArchive.billing_run_id.is_not(None), PrintArchive.created_by_id.is_not(None)))
  796. if live_only:
  797. query = query.where(PrintArchive.deleted_at.is_(None))
  798. archives = await db.execute(query.order_by(PrintArchive.created_at.desc()))
  799. return next((a for a in archives.scalars() if _archive_name_forms(a) & reported), None)
  800. async def running_print_archive(db, printer_id: int, state: PrinterState, *, printing_only: bool, by_name: bool):
  801. """The archive of the print running on ``printer_id``, or None (#3327).
  802. By the print's subtask_id first, which the firmware mints per print; with
  803. ``printing_only`` only an archive that is still printing (a leftover row of
  804. a completion never seen must not stand in for another job). A print
  805. started outside the cloud (Bambu Studio over LAN, Handy, the printer's
  806. screen) reports subtask_id "0", which links it to nothing: the archive
  807. Bambuddy bound to the running print comes next, the one completion charges
  808. and photographs. A restart forgets that binding and rebinds only Bambuddy's
  809. own prints, so with ``by_name`` a "0" print last falls back to its printing
  810. archive under the name the printer reports. That can be an earlier run of
  811. the same name whose completion was missed while Bambuddy was down: good
  812. enough for a plate label or a cover, not for which objects skip-objects
  813. offers, so those callers leave it off. None until the archive exists, and
  814. never a deleted one.
  815. """
  816. from backend.app.models.archive import PrintArchive
  817. from backend.app.services.print_log import open_run_on_printer
  818. subtask_id = str(getattr(state, "subtask_id", "") or "").strip()
  819. if subtask_id not in ("", "0"):
  820. # The run open on the printer for this job: the archive row is shared
  821. # by every plate of it, its printer and job id the latest start's.
  822. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  823. if run is not None and run.archive_id is not None:
  824. archive = await db.scalar(
  825. select(PrintArchive).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  826. )
  827. if archive is not None:
  828. return archive
  829. query = select(PrintArchive).where(
  830. PrintArchive.printer_id == printer_id,
  831. PrintArchive.subtask_id == subtask_id,
  832. PrintArchive.deleted_at.is_(None),
  833. )
  834. if printing_only:
  835. query = query.where(PrintArchive.status == "printing")
  836. archive = await db.scalar(query.order_by(PrintArchive.created_at.desc()).limit(1))
  837. if archive is not None:
  838. return archive
  839. bound = next(
  840. (_active_prints[key] for key in _build_status_print_keys(printer_id, state) if key in _active_prints),
  841. None,
  842. )
  843. if bound is not None:
  844. return await db.scalar(select(PrintArchive).where(PrintArchive.id == bound, PrintArchive.deleted_at.is_(None)))
  845. if not by_name or subtask_id != "0":
  846. return None
  847. reported = _reported_print_names(printer_id, state)
  848. if not reported:
  849. return None
  850. return await _printing_archive_without_job_id(db, printer_id, reported, dispatched_only=False, live_only=True)
  851. def _bind_active_print(
  852. printer_id: int,
  853. archive_id: int,
  854. archive_filename: str | None,
  855. filename: str | None = None,
  856. subtask_name: str | None = None,
  857. ) -> None:
  858. """Track a running print under every name its completion may report.
  859. One set for every path that binds a print (new archive, expected print,
  860. restart resume, fallback): the archive's file name, the name the printer
  861. reported, and the subtask with and without ``.3mf``. Paths that registered
  862. fewer names lost the bound archive at completion, and with it the plate
  863. restore's height (#3240), whenever MQTT reported a name they had left out.
  864. """
  865. for name in {archive_filename, filename, f"{subtask_name}.3mf" if subtask_name else None, subtask_name}:
  866. if name:
  867. _active_prints[(printer_id, name)] = archive_id
  868. async def _archive_of_open_run(printer_id: int, subtask_id: str | None = None) -> int | None:
  869. """The archive of the run open on the printer, if it is still there.
  870. With ``subtask_id``, only a run of that job."""
  871. try:
  872. from backend.app.models.archive import PrintArchive
  873. from backend.app.services.print_log import open_run_on_printer
  874. async with async_session() as db:
  875. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  876. if run is None or run.archive_id is None:
  877. return None
  878. return await db.scalar(
  879. select(PrintArchive.id).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  880. )
  881. except Exception as e:
  882. logging.getLogger(__name__).warning("[RUN] Could not read the open run of printer %s: %s", printer_id, e)
  883. return None
  884. async def _announce_discarded_runs(discarded, printer_id: int) -> None:
  885. """Publish the runs ``discard_open_runs`` dropped (archive redesign):
  886. each announced by run/started, now gone without an end."""
  887. if not discarded:
  888. return
  889. try:
  890. from backend.app.models.printer import Printer
  891. async with async_session() as db:
  892. printer = (
  893. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  894. ).first()
  895. if printer is not None:
  896. for run in discarded:
  897. await mqtt_relay.on_run_discarded(run, printer.serial_number, printer.name)
  898. except Exception as e:
  899. logging.getLogger(__name__).debug("[RUN] Could not announce discarded runs on printer %s: %s", printer_id, e)
  900. async def _publish_run_completed(run_id: int | None, printer_id: int) -> None:
  901. """Publish the run a print's end just wrote on the MQTT relay (archive
  902. redesign); nothing without one. Never in the way of the completion."""
  903. if run_id is None:
  904. return
  905. try:
  906. from backend.app.models.archive import PrintArchive
  907. from backend.app.models.print_log import PrintLogEntry
  908. from backend.app.models.printer import Printer
  909. async with async_session() as db:
  910. run = await db.get(PrintLogEntry, run_id)
  911. printer = (
  912. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  913. ).first()
  914. archive = await db.get(PrintArchive, run.archive_id) if run is not None and run.archive_id else None
  915. if run is not None and printer is not None:
  916. await mqtt_relay.on_run_completed(run, archive, printer.serial_number, printer.name)
  917. except Exception as e:
  918. logging.getLogger(__name__).debug("[RUN] Could not publish the end of run %s: %s", run_id, e)
  919. async def _open_run_for(archive_id: int, printer_id: int, subtask_id: str | None = None) -> None:
  920. """Write the run of the print that just started on the archive (archive
  921. redesign). In a session of its own and never in the way of the print
  922. start: until its readers move over, the archive still carries the print.
  923. The archive's start must be committed before. ``subtask_id`` is the
  924. printer's job, which finds the run again at a resume."""
  925. try:
  926. from backend.app.models.archive import PrintArchive
  927. from backend.app.models.printer import Printer
  928. from backend.app.services.print_log import open_run, open_run_on_printer, run_snapshot
  929. async with async_session() as db:
  930. archive = await db.get(PrintArchive, archive_id)
  931. if archive is None:
  932. return
  933. printer = (
  934. await db.execute(select(Printer.name, Printer.serial_number).where(Printer.id == printer_id))
  935. ).first()
  936. printer_name = printer.name if printer is not None else None
  937. already = await open_run_on_printer(db, printer_id)
  938. before = run_snapshot(already) if already is not None else None
  939. run = await open_run(
  940. db,
  941. archive,
  942. printer_id=printer_id,
  943. printer_name=printer_name,
  944. plate_id=_print_plate_ids.get((printer_id, archive_id)),
  945. subtask_id=subtask_id,
  946. )
  947. await db.commit()
  948. # Once per print: a resume or a repeated start finds the run it opened.
  949. # A new one replaced the run still open there (open_run): that one's
  950. # print ended unseen.
  951. # Compared by start too: SQLite can give the new run the id of the one
  952. # it replaced.
  953. replaced = (
  954. run is not None and before is not None and (before.id != run.id or before.started_at != run.started_at)
  955. )
  956. if run is not None and printer is not None and (before is None or replaced):
  957. if replaced:
  958. await _announce_discarded_runs([before], printer_id)
  959. try:
  960. await mqtt_relay.on_run_started(run, archive, printer.serial_number, printer_name)
  961. except Exception:
  962. pass # Don't fail the print start if MQTT fails
  963. except Exception as e:
  964. logging.getLogger(__name__).warning("[RUN] Could not open the run of archive %s: %s", archive_id, e)
  965. async def _drop_replaced_plate_videos(
  966. db, archive_id: int, plate_id: int | None, stale_relpath: str | None, logger: logging.Logger
  967. ) -> None:
  968. """Delete the videos a reprint of a plate replaces: ``stale_relpath``, the
  969. one the archive showed (when print start judged it this plate's), and
  970. those of the archive's earlier runs of the same plate. A plate keeps one
  971. video, as when the archive held the only one; another plate's stays. The
  972. runs that held them lose them. Commits."""
  973. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, PrintLogEntry
  974. from backend.app.utils.safe_path import PathTraversalError, safe_join_under
  975. runs = select(PrintLogEntry).where(
  976. PrintLogEntry.archive_id == archive_id, PrintLogEntry.timelapse_path.is_not(None)
  977. )
  978. with_video = list(await db.scalars(runs.execution_options(**{INCLUDE_OPEN_RUNS: True})))
  979. paths = {stale_relpath} if stale_relpath else set()
  980. if plate_id is not None:
  981. paths |= {r.timelapse_path for r in with_video if r.plate_id == plate_id}
  982. # A video another plate's run holds is that plate's.
  983. paths -= {r.timelapse_path for r in with_video if r.plate_id is not None and r.plate_id != plate_id}
  984. if not paths:
  985. return
  986. for run in with_video:
  987. if run.timelapse_path in paths:
  988. run.timelapse_path = None
  989. await db.commit()
  990. for relpath in sorted(p for p in paths if p):
  991. try:
  992. stale_path = safe_join_under(app_settings.base_dir, relpath, http=False)
  993. if stale_path.is_file():
  994. stale_path.unlink()
  995. logger.info("Deleted stale timelapse %s on reprint of archive %s", relpath, archive_id)
  996. except (OSError, PathTraversalError) as e:
  997. logger.warning("Failed to delete stale timelapse %s on reprint: %s", relpath, e)
  998. def _mark_owned_if_dispatched(archive) -> None:
  999. """Remember a running print's archive as Bambuddy's when it carries a
  1000. dispatch marker Bambuddy writes itself: ``billing_run_id`` (minted per
  1001. dispatch in the scheduler) or ``created_by_id`` (from the queue item)."""
  1002. if archive.billing_run_id is not None or archive.created_by_id is not None:
  1003. _owned_print_archive_ids.add(archive.id)
  1004. async def _is_printer_kill_switch_enabled_cached() -> bool:
  1005. """Return the kill-switch setting without querying on every MQTT frame."""
  1006. global _kill_switch_setting_cache
  1007. now = time.monotonic()
  1008. if _kill_switch_setting_cache is not None:
  1009. enabled, expires_at = _kill_switch_setting_cache
  1010. if now < expires_at:
  1011. return enabled
  1012. async with async_session() as db:
  1013. from backend.app.services.finance_budget import is_printer_kill_switch_enabled
  1014. enabled = await is_printer_kill_switch_enabled(db)
  1015. _kill_switch_setting_cache = (enabled, now + _KILL_SWITCH_SETTING_CACHE_TTL_SECONDS)
  1016. return enabled
  1017. async def _is_bambuddy_authorized_print(printer_id: int, state: PrinterState, db) -> bool | None:
  1018. """Resolve whether the current print was started by Bambuddy.
  1019. ``None`` means identity is not yet safe to decide. The kill switch must
  1020. defer in that case: stopping a print is irreversible, and the first status
  1021. frames after a restart may arrive before all subtask fields are populated.
  1022. """
  1023. if _is_bambuddy_authorized_print_in_memory(printer_id, state):
  1024. return True
  1025. # The printer's built-in calibration runs (Bambuddy's Calibrate, the
  1026. # printer's screen) play gcode from its read-only system partition, where no
  1027. # user file can be. Only that path counts: a calibration *name* proves
  1028. # nothing, since Bambu Studio's pressure-advance print reports a user-style
  1029. # path, and any print can be given that name.
  1030. if str(getattr(state, "gcode_file", "") or "").startswith("/usr/"):
  1031. return True
  1032. # A1, A1 Mini, P1 and A2L report the same calibration as a bare file name,
  1033. # which nothing distinguishes from a user's file given that name, so it is
  1034. # not exempt: with the kill switch on, calibrate on those printers with the
  1035. # switch off.
  1036. possible_keys = _build_status_print_keys(printer_id, state)
  1037. # In-memory ownership is lost on every Bambuddy restart, so fall back to what
  1038. # is on disk. subtask_id is minted per print and pins the answer to the job
  1039. # actually running, rather than to an unrelated one that reuses a filename.
  1040. raw_subtask_id = getattr(state, "subtask_id", None)
  1041. subtask_id = str(raw_subtask_id).strip() if raw_subtask_id is not None else ""
  1042. if subtask_id == "":
  1043. # Not reported yet: the first frames after a restart can lack it.
  1044. return None
  1045. from backend.app.models.archive import PrintArchive
  1046. if subtask_id == "0":
  1047. # A print started outside the cloud from Bambu Studio, Handy or the
  1048. # printer's screen reports "0", so there is no id to match: one of
  1049. # Bambuddy's printing archives on this printer under the name the
  1050. # printer reports, compared as the printer echoes names (spaces become
  1051. # underscores). Nothing reported to compare yet: wait.
  1052. reported = _reported_print_names(printer_id, state)
  1053. if not reported:
  1054. return None
  1055. archive = await _printing_archive_without_job_id(db, printer_id, reported, dispatched_only=True)
  1056. else:
  1057. from backend.app.services.print_log import open_run_on_printer
  1058. # This job's run open on the printer: the archive is shared by every
  1059. # plate of it, and its status, printer and job id are the latest
  1060. # start's, which may be another printer's.
  1061. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  1062. archive = await db.get(PrintArchive, run.archive_id) if run is not None and run.archive_id else None
  1063. if archive is None:
  1064. result = await db.execute(
  1065. select(PrintArchive)
  1066. .where(
  1067. PrintArchive.printer_id == printer_id,
  1068. PrintArchive.status == "printing",
  1069. PrintArchive.subtask_id == subtask_id,
  1070. )
  1071. .order_by(PrintArchive.created_at.desc())
  1072. .limit(1)
  1073. )
  1074. archive = result.scalar_one_or_none()
  1075. # An archive row on its own proves nothing: `on_print_start` archives every
  1076. # print it observes, including ones started from Bambu Studio or Handy, and
  1077. # stamps them with the same status and subtask_id. Authorizing on its mere
  1078. # existence would disable the kill switch the moment the 3MF finishes
  1079. # downloading. Only a dispatch marker Bambuddy writes itself counts —
  1080. # `billing_run_id` (minted per dispatch in the scheduler) or `created_by_id`
  1081. # (carried over from the queue item that started it).
  1082. if archive is not None and (archive.billing_run_id is not None or archive.created_by_id is not None):
  1083. # Rehydrate the fast in-memory path for subsequent status frames. Include
  1084. # both the archive filename and every normalized key reported by MQTT.
  1085. _owned_print_archive_ids.add(archive.id)
  1086. _active_prints[(printer_id, archive.filename)] = archive.id
  1087. for key in possible_keys:
  1088. _active_prints[key] = archive.id
  1089. return True
  1090. # No dispatch marker. Before calling this someone else's print, check whether
  1091. # Bambuddy has a job of its own running on this printer: a library-file
  1092. # dispatch has no archive at send time, and an archive created seconds later
  1093. # by `on_print_start` carries neither marker. The queue row, which the
  1094. # scheduler commits to status="printing" before the MQTT send, is the one
  1095. # durable record every Bambuddy print has. It cannot be tied to this
  1096. # subtask_id, so it is grounds to defer, never to authorize — stopping a
  1097. # print is irreversible, and refusing to act costs nothing but a log line.
  1098. from backend.app.models.print_queue import PrintQueueItem
  1099. dispatched_here = await db.scalar(
  1100. select(PrintQueueItem.id)
  1101. .where(
  1102. PrintQueueItem.printer_id == printer_id,
  1103. PrintQueueItem.status == "printing",
  1104. )
  1105. .limit(1)
  1106. )
  1107. if dispatched_here is not None:
  1108. return None
  1109. return False
  1110. async def _send_kill_switch_provider_notification(
  1111. printer_id: int,
  1112. printer_name: str,
  1113. data: dict,
  1114. ) -> bool:
  1115. """Send the immediate print-stopped provider notification.
  1116. Returning a success flag lets the normal MQTT completion path retry when
  1117. this early notification could not be delivered.
  1118. """
  1119. logger = logging.getLogger(__name__)
  1120. try:
  1121. async with async_session() as db:
  1122. from backend.app.services.print_log import open_run_on_printer
  1123. # The print it stopped, for {archive_id} / {run_id}: the completion
  1124. # that would name them skips its notification after this one.
  1125. run = await open_run_on_printer(db, printer_id)
  1126. await notification_service.on_print_complete(
  1127. printer_id,
  1128. printer_name,
  1129. "stopped",
  1130. data,
  1131. db,
  1132. archive_data={"archive_id": run.archive_id, "run_id": run.id} if run is not None else None,
  1133. )
  1134. return True
  1135. except Exception as e:
  1136. logger.warning(
  1137. "[KILL SWITCH] Immediate provider notification failed for printer %s: %s",
  1138. printer_id,
  1139. e,
  1140. )
  1141. return False
  1142. async def _kill_switch_notification_already_sent(task: asyncio.Task[bool] | None) -> bool:
  1143. """Wait for an immediate kill-switch notification, if one was scheduled."""
  1144. if task is None:
  1145. return False
  1146. try:
  1147. return await task
  1148. except Exception as e:
  1149. logging.getLogger(__name__).warning("[KILL SWITCH] Notification task failed: %s", e)
  1150. return False
  1151. async def _run_needs_energy_start(archive, printer_id: int, db) -> bool:
  1152. """At a resume: whether the print's run still lacks its energy start
  1153. reading. The archive's may be an earlier print's. Commits a reading
  1154. found on the archive for this run onto it."""
  1155. try:
  1156. from backend.app.services.print_log import run_needs_energy_start
  1157. needs = await run_needs_energy_start(db, archive, printer_id=printer_id)
  1158. await db.commit()
  1159. return needs
  1160. except Exception as e:
  1161. logging.getLogger(__name__).warning("[RUN] Energy start of archive %s's run not checked: %s", archive.id, e)
  1162. return archive.energy_start_kwh is None
  1163. async def _record_energy_start(archive, printer_id: int, db, *, context: str = "") -> bool:
  1164. """Capture the smart plug lifetime counter on the archive at print start.
  1165. Persists `energy_start_kwh` on the archive row (#941) so per-print energy
  1166. tracking survives a backend restart mid-print. The print-end handler reads
  1167. this value back from the DB and computes the delta against the current
  1168. plug counter.
  1169. """
  1170. _logger = logging.getLogger(__name__)
  1171. try:
  1172. candidates = await energy_plug_candidates(db, printer_id)
  1173. if not candidates:
  1174. _logger.info("[ENERGY] No smart plug for printer %s (archive %s)", printer_id, archive.id)
  1175. return False
  1176. selected = await select_energy_reading(candidates, _get_plug_energy, db)
  1177. if selected is None:
  1178. # Naming the plugs matters here: with several linked to one printer
  1179. # this is the difference between "the meter is offline" and "you
  1180. # linked only accessories" (#2859).
  1181. _logger.warning(
  1182. "[ENERGY] No plug on printer %s reports a lifetime energy counter for archive %s (tried: %s)",
  1183. printer_id,
  1184. archive.id,
  1185. ", ".join(plug.name for plug in candidates),
  1186. )
  1187. return False
  1188. plug, energy = selected
  1189. archive.energy_start_kwh = float(energy["total"])
  1190. # With the plug's snapshots, what lets the cost follow the price over
  1191. # the print rather than take the price at its end (#1251). A price that
  1192. # can't be had must not cost the print its energy reading.
  1193. archive.energy_start_at = utcnow_naive()
  1194. archive.energy_start_plug_id = plug.id
  1195. try:
  1196. from backend.app.services.energy_price import current_price
  1197. archive.energy_start_price = await current_price(db)
  1198. except Exception as e:
  1199. _logger.warning("[ENERGY] Could not read the electricity price for archive %s: %s", archive.id, e)
  1200. archive.energy_start_price = None
  1201. try:
  1202. from backend.app.services.print_log import mirror_energy_start
  1203. await mirror_energy_start(db, archive, printer_id=printer_id)
  1204. except Exception as e:
  1205. _logger.warning("[RUN] Energy start not copied onto the run of archive %s: %s", archive.id, e)
  1206. await db.commit()
  1207. _logger.info(
  1208. "[ENERGY] Recorded starting energy%s for archive %s from plug '%s': %s kWh",
  1209. f" ({context})" if context else "",
  1210. archive.id,
  1211. plug.name,
  1212. energy["total"],
  1213. )
  1214. return True
  1215. except Exception as e:
  1216. _logger.warning("[ENERGY] Failed to record starting energy for archive %s: %s", archive.id, e)
  1217. return False
  1218. def forget_print_creator(user_id: int) -> None:
  1219. """The print-start emails of a deleted user's jobs go to nobody, not to
  1220. whoever is given the id next (SQLite reuses it)."""
  1221. for key in [key for key, creator in _expected_print_creators.items() if creator == user_id]:
  1222. del _expected_print_creators[key]
  1223. async def forget_printer(printer_id: int) -> None:
  1224. """Drop what is kept for a deleted printer's id, here and in the services.
  1225. SQLite used to give the id to the next printer created, which inherited it
  1226. all: the running print's owner (its prints passed the kill switch and were
  1227. billed to that user), the plate-clear gate, pending and suppressed
  1228. notifications, the camera feed, the plate references. Ids are no longer
  1229. reused (AUTOINCREMENT), but nothing kept for a printer that is gone should
  1230. stay. Called after the delete is committed, and again by work that was in
  1231. flight and wrote state back.
  1232. Every printer-keyed map of this module is listed here or, with the reason,
  1233. in tests/unit/test_forget_printer.py.
  1234. """
  1235. from backend.app.api.routes import camera
  1236. from backend.app.services import (
  1237. bambu_ftp,
  1238. kprofile_drift,
  1239. layer_timelapse,
  1240. plate_detection,
  1241. print_dispatch_context,
  1242. usage_tracker,
  1243. )
  1244. from backend.app.services.notify_live_activities import notify_live_activities
  1245. from backend.app.services.obico_detection import obico_detection_service
  1246. from backend.app.services.printer_file_cache import printer_file_cache
  1247. # Keyed (printer_id, filename). The archive-keyed maps of those prints go too.
  1248. archive_ids = set()
  1249. for tracked in (_active_prints, _expected_prints):
  1250. for key in [key for key in tracked if key[0] == printer_id]:
  1251. archive_ids.add(tracked.pop(key))
  1252. for archive_id in archive_ids:
  1253. _owned_print_archive_ids.discard(archive_id)
  1254. for registered in (_print_ams_mappings, _print_cost_center_ids, _print_plate_ids):
  1255. for key in [key for key in registered if key[0] == printer_id]:
  1256. del registered[key]
  1257. for tracked in (_expected_print_creators, _expected_print_registered_at):
  1258. for key in [key for key in tracked if key[0] == printer_id]:
  1259. del tracked[key]
  1260. for key in [key for key in _ams_index_only_logged if key[0] == printer_id]:
  1261. _ams_index_only_logged.discard(key)
  1262. for key in [key for key in _ams_alarm_cooldown if key.split(":", 1)[0] == str(printer_id)]:
  1263. del _ams_alarm_cooldown[key]
  1264. # Pending work that would act on the id: an offline notice, a 3MF retry.
  1265. # A kill-switch notice is about a stop that happened and is left to finish.
  1266. for tasks in (_printer_offline_notify_tasks, _fallback_3mf_retry_tasks):
  1267. task = tasks.pop(printer_id, None)
  1268. if task is not None:
  1269. task.cancel()
  1270. for keyed in (
  1271. _stage22_finish_frames,
  1272. _stage22_finish_in_flight,
  1273. _inprint_frame_bank,
  1274. _inprint_frame_bank_ts,
  1275. _printer_reconciled_since_connect,
  1276. _printer_kprofiles_primed_since_connect,
  1277. _last_progress_milestone,
  1278. _first_layer_notified,
  1279. _kill_switch_notification_tasks,
  1280. _notified_hms_errors,
  1281. _hms_print_state,
  1282. _hms_forget_when_gone,
  1283. _timelapse_baselines,
  1284. _bed_cool_waiters,
  1285. _printer_last_connected,
  1286. _unknown_tag_last_broadcast,
  1287. _last_status_broadcast,
  1288. _connection_watchdog_last_attempt,
  1289. ):
  1290. keyed.pop(printer_id, None)
  1291. for marked in (
  1292. _unauthorized_print_kill_sent,
  1293. _user_stopped_printers,
  1294. _kill_switch_stopped,
  1295. _printer_offline_published,
  1296. _nozzle_count_updated,
  1297. _runtime_seen_connected,
  1298. _timelapse_baselines_unread,
  1299. ):
  1300. marked.discard(printer_id)
  1301. printer_manager.forget_printer(printer_id)
  1302. printer_file_cache.forget_printer(printer_id)
  1303. print_dispatch_context.clear(printer_id)
  1304. kprofile_drift.forget_printer(printer_id, deleted=True)
  1305. layer_timelapse.cancel_session(printer_id)
  1306. usage_tracker.forget_printer(printer_id)
  1307. obico_detection_service.forget_printer(printer_id)
  1308. octoeverywhere_detection_service.forget_printer(printer_id)
  1309. bambu_ftp.clear_3mf_cache(printer_id)
  1310. await camera.forget_printer(printer_id)
  1311. printers.clear_cover_cache(printer_id)
  1312. print_scheduler.forget_printer(printer_id)
  1313. notify_live_activities.forget_printer(printer_id)
  1314. plate_detection.remove_printer_references(printer_id)
  1315. def register_expected_print(
  1316. printer_id: int,
  1317. filename: str,
  1318. archive_id: int,
  1319. ams_mapping: list[int] | None = None,
  1320. created_by_id: int | None = None,
  1321. cost_center_id: int | None = None,
  1322. plate_id: int | None = None,
  1323. ):
  1324. """Register an expected print from reprint/scheduled so we don't create duplicate archives."""
  1325. # Store with multiple filename variations to catch different naming patterns
  1326. _expected_prints[(printer_id, filename)] = archive_id
  1327. # Also store without .3mf extension if present
  1328. if filename.endswith(".3mf"):
  1329. base = filename[:-4]
  1330. _expected_prints[(printer_id, base)] = archive_id
  1331. _expected_prints[(printer_id, f"{base}.gcode")] = archive_id
  1332. # Store AMS mapping for usage tracking at print completion, and plate_id for
  1333. # a single-plate dispatch from a multi-plate 3MF — without it, the direct-Print
  1334. # path attributes the whole file's filament total to the spool instead of just
  1335. # the printed plate (#1697). They describe this dispatch: one that has none
  1336. # drops what an earlier dispatch of the archive left (sent, never started),
  1337. # unless a plate of the archive is printing now and still needs them.
  1338. dispatch = (printer_id, archive_id)
  1339. archive_printing = _printing_here(printer_id, archive_id)
  1340. if ams_mapping is not None:
  1341. _print_ams_mappings[dispatch] = ams_mapping
  1342. elif not archive_printing:
  1343. _print_ams_mappings.pop(dispatch, None)
  1344. if cost_center_id is not None:
  1345. _print_cost_center_ids[dispatch] = cost_center_id
  1346. elif not archive_printing:
  1347. _print_cost_center_ids.pop(dispatch, None)
  1348. if plate_id is not None:
  1349. _print_plate_ids[dispatch] = plate_id
  1350. elif not archive_printing:
  1351. _print_plate_ids.pop(dispatch, None)
  1352. # Store created_by_id so the user start email can be sent even when the archive
  1353. # itself has no created_by_id (e.g. library-file-based queue prints)
  1354. if created_by_id is not None:
  1355. _expected_print_creators[(printer_id, filename)] = created_by_id
  1356. if filename.endswith(".3mf"):
  1357. base = filename[:-4]
  1358. _expected_print_creators[(printer_id, base)] = created_by_id
  1359. _expected_print_creators[(printer_id, f"{base}.gcode")] = created_by_id
  1360. # Record registration time for TTL-based eviction
  1361. _registered_at = time.monotonic()
  1362. _expected_print_registered_at[(printer_id, filename)] = _registered_at
  1363. if filename.endswith(".3mf"):
  1364. base = filename[:-4]
  1365. _expected_print_registered_at[(printer_id, base)] = _registered_at
  1366. _expected_print_registered_at[(printer_id, f"{base}.gcode")] = _registered_at
  1367. logging.getLogger(__name__).info(
  1368. f"Registered expected print: printer={printer_id}, file={filename}, archive={archive_id}, ams_mapping={ams_mapping}, plate_id={plate_id}"
  1369. )
  1370. def unregister_expected_print(printer_id: int, filename: str, archive_id: int) -> None:
  1371. """Undo :func:`register_expected_print` when the print never went out.
  1372. Registration has to happen *before* the MQTT print command, because the
  1373. printer can report the print before the line after the send executes. So
  1374. every path that registers and then fails to send — a cancel winning the
  1375. #1853 CAS race, a ``start_print()`` that returns False, or any exception in
  1376. between — leaves an expectation for a print that will never arrive.
  1377. The TTL sweep evicts those after two hours, which is far longer than it
  1378. takes a user to react to a failed dispatch by pressing print again: that
  1379. reprint would be folded into the *old* archive and take the stale
  1380. ``ams_mapping`` / ``plate_id`` with it. Hence the explicit inverse.
  1381. Mirrors the sweep's rules, including the one that is easy to get wrong:
  1382. ``_print_ams_mappings`` / ``_print_plate_ids`` / ``_print_cost_center_ids``
  1383. are keyed by printer and archive, not by file, so they may only be dropped
  1384. once no live key of the printer still points at that archive.
  1385. """
  1386. keys = [(printer_id, filename)]
  1387. if filename.endswith(".3mf"):
  1388. base = filename[:-4]
  1389. keys.append((printer_id, base))
  1390. keys.append((printer_id, f"{base}.gcode"))
  1391. removed = False
  1392. for key in keys:
  1393. if _expected_prints.pop(key, None) is not None:
  1394. removed = True
  1395. _expected_print_creators.pop(key, None)
  1396. _expected_print_registered_at.pop(key, None)
  1397. # Still needed by another expected print of the archive on the printer,
  1398. # or a plate of it printing there, as in the sweep.
  1399. if not _expected_or_printing_here(printer_id, archive_id):
  1400. _drop_dispatch(printer_id, archive_id)
  1401. if removed:
  1402. logging.getLogger(__name__).info(
  1403. "Unregistered expected print: printer=%s, file=%s, archive=%s (print was never sent)",
  1404. printer_id,
  1405. filename,
  1406. archive_id,
  1407. )
  1408. def _compute_run_filament_grams(
  1409. status: str,
  1410. archive_filament_used_grams: float | None,
  1411. progress: float | int | None,
  1412. usage_results: list[dict] | None,
  1413. ) -> float | None:
  1414. """Per-run filament for PrintLogEntry, partial- and tracker-aware (#1378, #1390).
  1415. Priority for every status:
  1416. 1. Sum of tracked spool deltas in ``usage_results`` (AMS-measured
  1417. weight delta — same source that drives "Total Consumed" on the
  1418. Inventory page, so Stats and Inventory totals stay aligned).
  1419. 2. For ``completed``: the slicer estimate (no tracker available, fall
  1420. back to the canonical "this print used X" value).
  1421. 3. For partial statuses: ``estimate * progress%``.
  1422. 4. ``None`` if nothing is known.
  1423. """
  1424. tracked_grams = sum(r.get("weight_used") or 0 for r in (usage_results or []))
  1425. if tracked_grams > 0:
  1426. return round(tracked_grams, 1)
  1427. if status == "completed":
  1428. return archive_filament_used_grams
  1429. if archive_filament_used_grams:
  1430. scale = max(0.0, min(((progress or 0) / 100.0), 1.0))
  1431. if scale > 0:
  1432. return round(archive_filament_used_grams * scale, 1)
  1433. return None
  1434. def _printing_here(printer_id: int, archive_id: int) -> bool:
  1435. """Whether a plate of the archive is printing on the printer."""
  1436. return any(key[0] == printer_id and bound == archive_id for key, bound in _active_prints.items())
  1437. def _expected_or_printing_here(printer_id: int, archive_id: int) -> bool:
  1438. """Whether a print of the archive is expected or printing on the printer."""
  1439. return _printing_here(printer_id, archive_id) or any(
  1440. key[0] == printer_id and expected == archive_id for key, expected in _expected_prints.items()
  1441. )
  1442. def _drop_dispatch(printer_id: int, archive_id: int) -> None:
  1443. """Forget what the dispatch of the archive to the printer registered."""
  1444. dispatch = (printer_id, archive_id)
  1445. _print_ams_mappings.pop(dispatch, None)
  1446. _print_cost_center_ids.pop(dispatch, None)
  1447. _print_plate_ids.pop(dispatch, None)
  1448. def _get_start_ams_mapping(data: dict, printer_id: int, archive_id: int | None) -> list[int] | None:
  1449. """Resolve AMS mapping for print start without consuming stored queue/reprint state."""
  1450. stored_ams_mapping = data.get("ams_mapping")
  1451. if not stored_ams_mapping and archive_id:
  1452. stored_ams_mapping = _print_ams_mappings.get((printer_id, archive_id))
  1453. return stored_ams_mapping
  1454. def _get_start_plate_id(printer_id: int, archive_id: int | None) -> int | None:
  1455. """Resolve plate_id for print start without consuming stored direct-Print state.
  1456. Direct-Print of a single plate from a multi-plate 3MF registers plate_id in
  1457. ``_print_plate_ids`` at dispatch time; this lets the spoolman / usage tracker
  1458. read it back at print-start without popping (the entry is popped on print
  1459. completion or TTL eviction, mirroring ``_print_ams_mappings``).
  1460. """
  1461. if archive_id is None:
  1462. return None
  1463. return _print_plate_ids.get((printer_id, archive_id))
  1464. def _run_print_time_seconds(archive, printer_id: int) -> int | None:
  1465. """The slicer's time for the plate this print prints, for its start
  1466. notification (#3375): ``archive.print_time_seconds`` adds every plate of
  1467. the file up (#1593). The plate is the dispatch's, else the archive's. The
  1468. whole file's time when its plates are not known; None for a plate not
  1469. known of several, and the notification takes the printer's remaining
  1470. time."""
  1471. from backend.app.services.print_cost_estimate import run_estimate
  1472. plate_id = _print_plate_ids.get((printer_id, archive.id), archive.plate_id)
  1473. estimate = run_estimate(archive, plate_id)
  1474. return estimate.seconds if estimate is not None else None
  1475. _plate_estimates_task: asyncio.Task | None = None
  1476. async def _fill_plate_estimates() -> None:
  1477. from backend.app.services.print_cost_estimate import fill_plate_estimates
  1478. try:
  1479. filled = await fill_plate_estimates(async_session)
  1480. except asyncio.CancelledError:
  1481. raise
  1482. except Exception as exc:
  1483. logging.getLogger(__name__).warning("[ARCHIVE] Reading plate estimates failed: %s", exc)
  1484. return
  1485. if filled:
  1486. logging.getLogger(__name__).info("[ARCHIVE] Read the plate estimates of %d archive(s)", filled)
  1487. def _partial_progress_scale(progress: int | float | None) -> float:
  1488. """Clamp ``progress / 100`` into [0.0, 1.0] for partial-print scaling.
  1489. Used by every site that multiplies a "would-have-used" slicer estimate
  1490. down to "actually-used" for failed / cancelled / stopped prints. Centralised
  1491. so the three sites in ``_background_notifications`` (and the per-plate
  1492. override helper) can't drift apart on the coercion shape.
  1493. """
  1494. return max(0.0, min((progress or 0) / 100.0, 1.0))
  1495. def _scope_notification_archive_data_to_plate(
  1496. archive_data: dict,
  1497. archive_file_path: str | None,
  1498. plate_id: int | None,
  1499. print_status: str,
  1500. progress: int | float | None,
  1501. base_dir: Path,
  1502. ) -> dict:
  1503. """Override summed-across-plates totals in ``archive_data`` with the values
  1504. for ``plate_id`` so the completion notification reports what was actually
  1505. printed, not the whole project (#1785).
  1506. The 3MF parser at services/archive.py:200-264 sums ``prediction`` and
  1507. ``weight`` across every plate of a multi-plate file (#1593) — correct for
  1508. the archive card's "whole project" headline, wrong for the completion
  1509. notification of a single-plate print. The queue UI already re-reads the
  1510. 3MF per-plate at print_queue.py:272-285; this helper mirrors that for the
  1511. notification payload (filament grams, time estimate, per-slot breakdown).
  1512. No-ops when ``plate_id`` is None, the file is missing, or the 3MF carries
  1513. no per-plate values — in every fail case the original ``archive_data`` is
  1514. returned unchanged so the notification still sends.
  1515. """
  1516. if plate_id is None or not archive_file_path:
  1517. return archive_data
  1518. from backend.app.utils.threemf_tools import (
  1519. extract_filament_usage_from_3mf,
  1520. extract_print_time_from_3mf,
  1521. )
  1522. archive_path = base_dir / archive_file_path
  1523. if not archive_path.exists():
  1524. return archive_data
  1525. plate_slots = extract_filament_usage_from_3mf(archive_path, plate_id)
  1526. plate_grams = sum(f.get("used_g", 0) for f in plate_slots)
  1527. plate_time = extract_print_time_from_3mf(archive_path, plate_id)
  1528. scale = 1.0 if print_status == "completed" else _partial_progress_scale(progress)
  1529. if plate_time:
  1530. archive_data["print_time_seconds"] = plate_time
  1531. # Gate both the grams headline AND the per-slot breakdown on the same
  1532. # `plate_grams > 0` signal: if the 3MF carries per-plate filament rows but
  1533. # they all sum to zero (slicer bug / re-slice without estimate), drop back
  1534. # to the project-level grams the archive columns already provide rather
  1535. # than ship a project-level headline next to an all-zero per-plate
  1536. # breakdown.
  1537. if plate_grams > 0:
  1538. archive_data["actual_filament_grams"] = round(plate_grams * scale, 1)
  1539. archive_data["filament_slots"] = [
  1540. {
  1541. "slot_id": s.get("slot_id"),
  1542. "used_g": round((s.get("used_g") or 0) * scale, 1),
  1543. "type": s.get("type", ""),
  1544. "color": s.get("color", ""),
  1545. }
  1546. for s in plate_slots
  1547. ]
  1548. return archive_data
  1549. def _extract_filament_data_from_mqtt(data: dict, ams_mapping: list[int] | None = None) -> dict[str, str]:
  1550. """Best-effort filament metadata from the MQTT print-start snapshot.
  1551. Used when the 3MF can't be downloaded (P1S/A1/P2S firmwares lock the
  1552. file during print, see #1533) so the fallback PrintArchive still has
  1553. enough filament info to support the inventory views and AMS-expansion
  1554. planning the operator opens it for. Returns a dict with optional
  1555. ``filament_type`` and ``filament_color`` keys in the same
  1556. comma-separated format the 3MF extractor produces, so the rest of the
  1557. codebase treats the fallback archive identically to a normal one.
  1558. ``ams_mapping`` is the slicer's slot-per-print-filament list captured
  1559. from the MQTT print payload (global tray IDs, possibly -1 for VT-tray
  1560. entries). When supplied, only the slots actually consumed by this
  1561. print contribute. Without it the function falls back to every loaded
  1562. AMS slot — less accurate but still useful.
  1563. Accepts both the raw inner payload (``{"ams": {"ams": [...]}, ...}``)
  1564. that the unit tests pass directly, AND the on_print_start callback
  1565. shape (``{"raw_data": {"ams": {"ams": [...]}, ...}, ...}``) the
  1566. bambu_mqtt service hands to main.py at runtime. The original
  1567. ``_extract_filament_data_from_mqtt(data)`` shipped in #1533 only
  1568. handled the inner shape and silently returned ``{}`` for every real
  1569. print start, leaving fallback archives' filament fields NULL — the
  1570. exact regression the fix was meant to close. Reported with a log
  1571. proving the AMS state was right there at
  1572. ``data["raw_data"]["ams"]["ams"][0]["tray"][0]`` (#1533 follow-up).
  1573. """
  1574. result: dict[str, str] = {}
  1575. # Look at the on_print_start wrapper first, then the inner shape.
  1576. raw_data = (data or {}).get("raw_data")
  1577. ams_root = (raw_data or {}).get("ams") if isinstance(raw_data, dict) else None
  1578. if not isinstance(ams_root, dict):
  1579. ams_root = (data or {}).get("ams") or {}
  1580. ams_units = ams_root.get("ams") if isinstance(ams_root, dict) else None
  1581. if not isinstance(ams_units, list) or not ams_units:
  1582. return result
  1583. from backend.app.services.print_scheduler import _global_tray_id
  1584. # Map global tray id → (type, color), addressed as the print's ams_mapping
  1585. # is (utils.tray_ids).
  1586. layout = TrayLayout.from_units(ams_units)
  1587. loaded: dict[int, tuple[str, str]] = {}
  1588. for unit in ams_units:
  1589. if not isinstance(unit, dict):
  1590. continue
  1591. try:
  1592. unit_id = int(unit.get("id", 0))
  1593. except (TypeError, ValueError):
  1594. continue
  1595. for tray in unit.get("tray") or []:
  1596. if not isinstance(tray, dict):
  1597. continue
  1598. try:
  1599. tray_id = int(tray.get("id", 0))
  1600. except (TypeError, ValueError):
  1601. continue
  1602. ttype = (tray.get("tray_type") or "").strip()
  1603. # RRGGBBAA from the printer, stored as the 3MF path stores it:
  1604. # "#RRGGBB" (the Archives page's swatches and colour filter).
  1605. hex_color = (tray.get("tray_color") or "").strip().upper()[:6]
  1606. tcolor = f"#{hex_color}" if re.fullmatch(r"[0-9A-F]{6}", hex_color) else ""
  1607. if not ttype:
  1608. continue # Empty / unloaded slot.
  1609. loaded[_global_tray_id(unit_id, tray_id, layout)] = (ttype, tcolor)
  1610. if not loaded:
  1611. return result
  1612. if ams_mapping:
  1613. used_ids = [int(x) for x in ams_mapping if isinstance(x, (int, float)) and int(x) >= 0]
  1614. filaments = [loaded[g] for g in used_ids if g in loaded]
  1615. if not filaments:
  1616. return result # Mapping points entirely at slots we have no data for.
  1617. else:
  1618. filaments = [loaded[g] for g in sorted(loaded.keys())]
  1619. types_joined = ",".join(f[0] for f in filaments)
  1620. colors_joined = ",".join(f[1] for f in filaments if f[1])
  1621. # Column limits per backend/app/models/archive.py: filament_type=50,
  1622. # filament_color=200.
  1623. if types_joined:
  1624. result["filament_type"] = types_joined[:50]
  1625. if colors_joined:
  1626. result["filament_color"] = colors_joined[:200]
  1627. return result
  1628. def _maybe_start_layer_timelapse(printer, printer_id: int, archive_id: int) -> bool:
  1629. """Start a layer-timelapse session for *archive_id* when the printer has
  1630. an external camera configured. Returns True if a session was started.
  1631. Three call sites in on_print_start (expected-archive promotion, fallback
  1632. archive creation, fresh-archive creation) used to inline this same
  1633. if-block; the inline copies kept drifting (#1353 fixed only one of them
  1634. on the first pass). Centralising the conditional + call here makes the
  1635. contract testable in isolation and keeps the three sites locked in step.
  1636. """
  1637. if not (printer.external_camera_enabled and printer.external_camera_url):
  1638. return False
  1639. from backend.app.services.layer_timelapse import start_session
  1640. start_session(
  1641. printer_id,
  1642. archive_id,
  1643. printer.external_camera_url,
  1644. printer.external_camera_type or "mjpeg",
  1645. snapshot_url=printer.external_camera_snapshot_url,
  1646. rotation=getattr(printer, "camera_rotation", 0),
  1647. )
  1648. logging.getLogger(__name__).info("Started layer timelapse for printer %s, archive %s", printer_id, archive_id)
  1649. return True
  1650. def _format_hms_error_summary(hms_errors: list[dict]) -> str | None:
  1651. """Build a human-readable failure reason from MQTT hms_errors for PrintQueueItem.error_message.
  1652. Each entry has keys: code ('0x4038'), attr (32-bit int), module, severity, and
  1653. — since #2926 — the description the parser already resolved, which is preferred
  1654. when present so the queue's failure reason reads the same as the status
  1655. response. The short code still produces the bracketed label, and still
  1656. resolves the sentence for a caller whose entries predate the field. An entry
  1657. with a 16-char ``full_code`` is labelled with it instead, since the short code
  1658. of an ``hms[]`` fault is not a code anyone can look up. Falls back to the bare
  1659. label when no description is on file. Returns None for an empty list so
  1660. callers can leave error_message unset.
  1661. """
  1662. if not hms_errors:
  1663. return None
  1664. from backend.app.services.hms_errors import get_error_description
  1665. parts: list[str] = []
  1666. for err in hms_errors:
  1667. try:
  1668. # `_hms_short_code` rather than a local derivation: this one used to
  1669. # format the error without masking it to 16 bits, so an `hms[]` entry
  1670. # whose code carries an alert-level group produced a five-digit label
  1671. # like "0500_3000A" — not a code the user can look up, and never a
  1672. # catalogue key, so the sentence was lost with it.
  1673. short_code = _hms_short_code(err.get("attr", 0), err.get("code", 0))
  1674. except (TypeError, ValueError):
  1675. continue
  1676. description = err.get("description") or get_error_description(short_code)
  1677. # An `hms[]` fault's short code ("0500_000E") drops the part and level
  1678. # groups and is not a code anyone can look up; its full code in the
  1679. # printer screen's four groups is (#2728).
  1680. full_code = str(err.get("full_code") or "")
  1681. label = "-".join(full_code[i : i + 4] for i in range(0, 16, 4)) if len(full_code) == 16 else short_code
  1682. parts.append(f"[{label}] {description}" if description else f"[{label}]")
  1683. return "; ".join(parts) if parts else None
  1684. async def _bump_library_file_usage_if_completed(db, item, queue_status: str) -> None:
  1685. """Increment LibraryFile.print_count and stamp last_printed_at when a queued
  1686. print completes successfully. Gated to status=='completed': failed, cancelled
  1687. and aborted prints do not count as usage. Caller is responsible for committing
  1688. the session. No-op when the queue item has no linked library file (e.g. reprints
  1689. from an archive). See #1008."""
  1690. if queue_status != "completed" or item.library_file_id is None:
  1691. return
  1692. from backend.app.models.library import LibraryFile
  1693. lib_file = await db.scalar(select(LibraryFile).where(LibraryFile.id == item.library_file_id))
  1694. if lib_file is None:
  1695. return
  1696. lib_file.print_count = (lib_file.print_count or 0) + 1
  1697. lib_file.last_printed_at = datetime.now(timezone.utc)
  1698. def mark_printer_stopped_by_user(printer_id: int) -> None:
  1699. """Mark that the active print on this printer was stopped by the user from the queue UI.
  1700. When on_print_complete fires with status 'failed' for a printer in this set we
  1701. reclassify it as 'cancelled' so the correct 'print stopped' notification is sent
  1702. rather than a 'print failed' notification.
  1703. """
  1704. _user_stopped_printers.add(printer_id)
  1705. logging.getLogger(__name__).info("Marked printer %s as user-stopped from queue", printer_id)
  1706. _last_status_broadcast: dict[int, str] = {}
  1707. # Track printers where we've updated nozzle_count
  1708. _nozzle_count_updated: set[int] = set()
  1709. async def _maybe_notify_printer_offline(printer_id: int) -> None:
  1710. """Wait the debounce window then fire `on_printer_offline` if the printer
  1711. is still offline.
  1712. Scheduled by `on_printer_status_change` on the connected → disconnected
  1713. edge (#1752). Cancelled by the same handler if the printer reconnects
  1714. before the window elapses, so a single MQTT blip + recovery doesn't
  1715. notify. Both the staleness-detector path (`bambu_mqtt.py::check_staleness`)
  1716. and the smart-plug power-off path (`printer_manager.mark_printer_offline`)
  1717. route through the same status-change callback, so this covers both.
  1718. """
  1719. logger = logging.getLogger(__name__)
  1720. try:
  1721. await asyncio.sleep(_PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS)
  1722. still_offline = not printer_manager.is_connected(printer_id)
  1723. logger.info(
  1724. "[#1752] Printer %s offline debounce elapsed: still_offline=%s",
  1725. printer_id,
  1726. still_offline,
  1727. )
  1728. if not still_offline:
  1729. return
  1730. async with async_session() as db:
  1731. from backend.app.models.printer import Printer
  1732. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1733. printer = result.scalar_one_or_none()
  1734. if not printer:
  1735. logger.warning(
  1736. "[#1752] Printer %s missing from DB at offline-notify time; skipping",
  1737. printer_id,
  1738. )
  1739. return
  1740. # Recorded only when the relay can publish it, so the reconnect's
  1741. # online event always has an offline before it.
  1742. if mqtt_relay.enabled and mqtt_relay.connected:
  1743. _printer_offline_published.add(printer_id)
  1744. try:
  1745. await mqtt_relay.on_printer_offline(printer_id, printer.name, printer.serial_number)
  1746. except Exception as e:
  1747. logger.warning("MQTT relay offline event failed for printer %s: %s", printer_id, e)
  1748. logger.info(
  1749. "[#1752] Dispatching on_printer_offline for printer %s (%s)",
  1750. printer_id,
  1751. printer.name,
  1752. )
  1753. await notification_service.on_printer_offline(printer_id, printer.name, db)
  1754. except asyncio.CancelledError:
  1755. raise
  1756. except Exception as e:
  1757. logger.warning("Printer offline notification failed for printer %s: %s", printer_id, e)
  1758. finally:
  1759. _printer_offline_notify_tasks.pop(printer_id, None)
  1760. def _hms_notify_key(error) -> str:
  1761. """What identifies a fault for notification de-duplication.
  1762. The full code, which is unique per fault. ``attr`` alone used to be the key:
  1763. unique for a ``print_error``, but for an ``hms[]`` fault it is only the
  1764. module and part, so two faults on one part (#1840's H2C held 0500-0600-0002-0005
  1765. and -0006 together) shared a key and only the first was ever notified. An
  1766. entry without a full code falls back to attr and code together.
  1767. """
  1768. full_code = getattr(error, "full_code", "") or ""
  1769. if full_code:
  1770. return full_code.upper()
  1771. return f"{error.attr:08X}:{error.code}"
  1772. def _hms_errors_to_notify(errors: list, new_error_codes: set[str]) -> list:
  1773. """The new faults that count (see ``_hms_fault_counts``).
  1774. These go to the MQTT relay; the caller also needs a description before it
  1775. sends a notification. Level 0, Bambu's "invalid" level, never counts. This
  1776. used to read ``severity >= 2`` when severity held the part byte; on the real
  1777. level that would drop the task-stopping errors (#2728).
  1778. """
  1779. return [e for e in errors if _hms_notify_key(e) in new_error_codes and _hms_fault_counts(e)]
  1780. # What each HMS module byte is about, read off Bambu's own texts for its codes
  1781. # (hms_catalog.json). The table used to call 0x05 "Nozzle/Extruder", 0x0C
  1782. # "Motion Controller" and 0x12 "Chamber", which titled USB and MQTT errors as
  1783. # nozzle faults and an A1 mini's AMS Lite runouts as chamber errors (#3486).
  1784. _HMS_MODULE_NAMES = {
  1785. 0x03: "Print/Task", # bed, nozzle temperatures, motors, the print itself
  1786. 0x05: "System", # USB/SD, network, firmware, files, AMS communication
  1787. 0x07: "AMS/Filament",
  1788. 0x08: "Toolhead",
  1789. 0x0C: "Camera", # cameras, lidar, first-layer and AI inspection
  1790. 0x12: "AMS Lite",
  1791. 0x18: "AMS-HT",
  1792. 0x1A: "Tool Changer",
  1793. 0x29: "Air Filter",
  1794. 0x2A: "AMS 3H",
  1795. }
  1796. def _hms_error_title(module: int) -> str:
  1797. """The notification title for a fault of *module*."""
  1798. name = _HMS_MODULE_NAMES.get(module)
  1799. return f"{name} Error" if name else "Printer Error"
  1800. def _take_new_hms_faults(printer_id: int, errors: list, print_state: str | None = None) -> list:
  1801. """The faults on this printer not notified yet, and record them as notified.
  1802. Tracking is updated before anything is sent, so concurrent status callbacks
  1803. cannot notify the same fault twice. A notified fault is forgotten once it
  1804. has been gone for ``_HMS_CLEAR_GRACE_SECONDS``, so one that flickers off
  1805. and on is notified once (#3226), while one that clears and comes back much
  1806. later is notified again.
  1807. When ``print_state`` changes (a print resumed, started or finished), every
  1808. fault recorded so far is forgotten the first time it is gone, without the
  1809. wait: a runout fixed before resuming must be notified if it happens again a
  1810. few minutes later, while the printer sits paused. Not only the faults gone
  1811. at the change itself, because the update that flips the state can still
  1812. carry the old fault (a ``print_error`` entry stays until a payload brings
  1813. ``hms``).
  1814. """
  1815. now = time.time()
  1816. current = {_hms_notify_key(e) for e in errors}
  1817. seen = _notified_hms_errors.setdefault(printer_id, {})
  1818. forget_when_gone = _hms_forget_when_gone.setdefault(printer_id, set())
  1819. if print_state is not None:
  1820. print_state = print_state.upper()
  1821. if _hms_print_state.get(printer_id, print_state) != print_state:
  1822. forget_when_gone.update(seen)
  1823. _hms_print_state[printer_id] = print_state
  1824. for key, last_seen in list(seen.items()):
  1825. if key not in current and (key in forget_when_gone or now - last_seen >= _HMS_CLEAR_GRACE_SECONDS):
  1826. del seen[key]
  1827. forget_when_gone.discard(key)
  1828. new = current - seen.keys()
  1829. for key in current:
  1830. seen[key] = now
  1831. if not seen:
  1832. _notified_hms_errors.pop(printer_id, None)
  1833. if not forget_when_gone:
  1834. _hms_forget_when_gone.pop(printer_id, None)
  1835. return _hms_errors_to_notify(errors, new)
  1836. async def _take_new_hms_faults_and_save(printer_id: int, errors: list, print_state: str | None = None) -> list:
  1837. """``_take_new_hms_faults``, and store the notified set when it changed (#3486).
  1838. The set lives in memory, so without this every restart sent each fault the
  1839. printer still lists once more. A failed write only costs that protection
  1840. for the next restart, so it is logged and the notification goes ahead.
  1841. """
  1842. before = set(_notified_hms_errors.get(printer_id, ()))
  1843. new = _take_new_hms_faults(printer_id, errors, print_state)
  1844. if set(_notified_hms_errors.get(printer_id, ())) != before:
  1845. from backend.app.services.hms_notified import save_notified_hms_faults
  1846. try:
  1847. await save_notified_hms_faults(printer_id, lambda: _notified_hms_errors.get(printer_id, {}).keys())
  1848. except Exception as e:
  1849. logging.getLogger(__name__).warning(
  1850. "Could not store the notified HMS faults of printer %s: %s", printer_id, e
  1851. )
  1852. return new
  1853. async def _load_notified_hms_faults() -> None:
  1854. """Restore the faults notified before the restart (#3486).
  1855. Each counts as seen now: one still listed is not sent again, and one that
  1856. is gone is forgotten after the usual grace period.
  1857. """
  1858. from backend.app.services.hms_notified import load_notified_hms_faults
  1859. try:
  1860. loaded = await load_notified_hms_faults()
  1861. except Exception as e:
  1862. logging.getLogger(__name__).warning("Could not load the notified HMS faults: %s", e)
  1863. return
  1864. now = time.time()
  1865. for printer_id, keys in loaded.items():
  1866. seen = _notified_hms_errors.setdefault(printer_id, {})
  1867. for key in keys:
  1868. seen.setdefault(key, now)
  1869. # `stg_cur` values that mean the printer is not in a preparation stage: 0 is
  1870. # "Printing", -1 and 255 are "no stage" (#3211).
  1871. _NO_PREPARATION_STAGES = frozenset({0, -1, 255})
  1872. def _progress_milestone_to_notify(printer_id: int, state: PrinterState) -> int | None:
  1873. """The milestone (25, 50 or 75) this status update reaches for the first time, if any.
  1874. Records it as notified. Called only for a printing state with progress above 0.
  1875. Progress before the first layer is not progress through the print (#3211).
  1876. An A1 mini reports ``mc_percent`` 85 in the first frame of a print, while
  1877. still preheating the bed at layer 0, then 3, 7, 40 and 44 through its
  1878. calibration, and starts layer 1 at 45. Read as progress, that 85 sent the
  1879. 75% notification together with Print Started, and since 75 was then on
  1880. record, 25 and 50 could never fire. So nothing counts while the printer
  1881. reports a layer count but has not started layer 1.
  1882. Without a layer count -- a print whose first frame carried no
  1883. ``total_layer_num``, until the pushall that asks for it is answered
  1884. (#2702) -- the preparation stage says the same thing: before layer 1 the
  1885. A1 mini reported ``stg_cur`` 2, 4, 14 and 1 (bed preheating, calibration,
  1886. homing), and 0 from layer 1 on. -1 and 255 are "no stage". A printer that
  1887. reports neither keeps the old behaviour rather than never notifying.
  1888. """
  1889. if (state.layer_num or 0) < 1:
  1890. if (state.total_layers or 0) > 0:
  1891. return None
  1892. if state.stg_cur not in _NO_PREPARATION_STAGES:
  1893. return None
  1894. progress = state.progress or 0
  1895. current_milestone = 0
  1896. if progress >= 75:
  1897. current_milestone = 75
  1898. elif progress >= 50:
  1899. current_milestone = 50
  1900. elif progress >= 25:
  1901. current_milestone = 25
  1902. if current_milestone > _last_progress_milestone.get(printer_id, 0):
  1903. _last_progress_milestone[printer_id] = current_milestone
  1904. return current_milestone
  1905. return None
  1906. async def on_printer_status_change(printer_id: int, state: PrinterState):
  1907. """Handle printer status changes - broadcast via WebSocket."""
  1908. try:
  1909. notify_live_activities.observe(printer_id, state)
  1910. except Exception:
  1911. logging.getLogger(__name__).exception("Notify Live Activity status hook failed for printer %s", printer_id)
  1912. try:
  1913. pause_eta.observe(printer_id, state)
  1914. except Exception:
  1915. logging.getLogger(__name__).exception("Pause heads-up check failed for printer %s", printer_id)
  1916. wled_manager.handle_status(
  1917. printer_id,
  1918. state,
  1919. awaiting_plate_clear=printer_manager.is_awaiting_plate_clear(printer_id),
  1920. )
  1921. # Connected-edge reconciliation (#1542 follow-up). When the printer
  1922. # transitions disconnected → connected — which covers both Bambuddy
  1923. # startup (no prior connection) and a mid-session MQTT reconnect — fire
  1924. # `reconcile_stale_active_prints` exactly once for this connection so
  1925. # any archive still in `status="printing"` that can't actually be
  1926. # running anymore (printer IDLE / different subtask / empty subtask)
  1927. # gets a synthesised PRINT COMPLETE. Without this, a print that
  1928. # finished during a disconnect window + a smart-plug power cycle
  1929. # leaves the .3mf on the SD card and the firmware ghost-replays it on
  1930. # next boot. Reconciliation runs concurrently — it must not block the
  1931. # WebSocket dedup / broadcast logic below, and the connected edge is
  1932. # marked True BEFORE the await so concurrent status updates inside
  1933. # the same connection don't re-trigger reconciliation.
  1934. #
  1935. # Wait for a real push_status before reconciling (#1679): MQTT
  1936. # `_on_connect` broadcasts `state` IMMEDIATELY after the broker accepts
  1937. # the connection, BEFORE `_request_push_all` round-trips. At that
  1938. # instant the `PrinterState` is still on construction defaults — most
  1939. # importantly `state.state == "unknown"` and `state.subtask_name == ""`.
  1940. # If reconcile spawns here, every in-flight archive falls through to
  1941. # the empty-subtask_name trigger and gets synthesised `aborted`, which
  1942. # creates a duplicate archive on the real PRINT COMPLETE and
  1943. # double-counts filament. Gating on `state.state ∉ ("", "unknown")`
  1944. # keeps the #1542 mechanism intact: once the first real push_status
  1945. # updates `state.state` (RUNNING / IDLE / FINISH / …), this handler
  1946. # fires again with the flag still False — reconcile then runs against
  1947. # actual evidence.
  1948. state_known = bool(state.state) and state.state.upper() not in ("", "UNKNOWN")
  1949. if state.connected and state_known and not _printer_reconciled_since_connect.get(printer_id, False):
  1950. _printer_reconciled_since_connect[printer_id] = True
  1951. spawn_background_task(
  1952. reconcile_stale_active_prints(printer_id),
  1953. name=f"reconcile-stale-prints-{printer_id}",
  1954. )
  1955. elif not state.connected and _printer_reconciled_since_connect.get(printer_id, False):
  1956. # Re-arm so the next reconnect triggers reconciliation again.
  1957. _printer_reconciled_since_connect[printer_id] = False
  1958. # Same edge, for the calibration table the AMS card reads its K values from.
  1959. #
  1960. # Also gated on knowing a nozzle diameter, which is what decides *which*
  1961. # tables to ask for. A `state_known` gate alone is not enough: the first
  1962. # real push_status is what makes the state known, and the nozzle fields do
  1963. # not always arrive in it. Latching there would spend this connection's one
  1964. # attempt on a printer that could not yet say what was fitted.
  1965. nozzle_known = any(n.nozzle_diameter for n in (state.nozzles or []))
  1966. if (
  1967. state.connected
  1968. and state_known
  1969. and nozzle_known
  1970. and not _printer_kprofiles_primed_since_connect.get(printer_id, False)
  1971. ):
  1972. _printer_kprofiles_primed_since_connect[printer_id] = True
  1973. spawn_background_task(
  1974. prime_kprofile_table(printer_id),
  1975. name=f"prime-kprofiles-{printer_id}",
  1976. )
  1977. elif not state.connected and _printer_kprofiles_primed_since_connect.get(printer_id, False):
  1978. _printer_kprofiles_primed_since_connect[printer_id] = False
  1979. # A slot can lose its K-profile selection (cali_idx back to -1) with nothing
  1980. # else in the AMS report changing -- a power cycle does it to every slot --
  1981. # and on_ams_change never hears of it (#3219). Checked here, on every push
  1982. # while the printer is idle; needs_check is cheap and throttled per slot.
  1983. # Guarded: nothing here may stop the status broadcast below.
  1984. try:
  1985. if not state.connected:
  1986. kprofile_drift.forget_printer(printer_id)
  1987. elif kprofile_drift.needs_check(printer_id, state):
  1988. spawn_background_task(
  1989. kprofile_drift.reapply_lost_kprofiles(printer_id),
  1990. name=f"reapply-kprofiles-{printer_id}",
  1991. )
  1992. except Exception:
  1993. logging.getLogger(__name__).exception("[Printer %s] K-profile check failed", printer_id)
  1994. # Offline-notification edge (#1752): schedule `on_printer_offline` on
  1995. # connected → disconnected. The "back online" channel is already covered
  1996. # by the print-failure notification (firmware reports gcode_state=FAILED
  1997. # on reconnect of an interrupted print), so there is no online
  1998. # notification. The MQTT relay gets both: offline after the debounce, and
  1999. # online on the reconnect that follows a published offline.
  2000. prev_connected = _printer_last_connected.get(printer_id)
  2001. _printer_last_connected[printer_id] = state.connected
  2002. if prev_connected is True and not state.connected:
  2003. existing = _printer_offline_notify_tasks.get(printer_id)
  2004. if existing is None or existing.done():
  2005. logging.getLogger(__name__).info(
  2006. "[#1752] Printer %s connected→disconnected edge; scheduling offline notification in %.0fs",
  2007. printer_id,
  2008. _PRINTER_OFFLINE_NOTIFY_DEBOUNCE_SECONDS,
  2009. )
  2010. _printer_offline_notify_tasks[printer_id] = asyncio.create_task(
  2011. _maybe_notify_printer_offline(printer_id),
  2012. name=f"printer-offline-notify-{printer_id}",
  2013. )
  2014. elif state.connected:
  2015. # Kept until the relay can actually send online: with its broker down
  2016. # now, the next status update after the reconnect sends it.
  2017. if printer_id in _printer_offline_published and mqtt_relay.enabled and mqtt_relay.connected:
  2018. _printer_offline_published.discard(printer_id)
  2019. printer_info = printer_manager.get_printer(printer_id)
  2020. if printer_info:
  2021. try:
  2022. await mqtt_relay.on_printer_online(printer_id, printer_info.name, printer_info.serial_number)
  2023. except Exception as e:
  2024. logging.getLogger(__name__).warning(
  2025. "MQTT relay online event failed for printer %s: %s", printer_id, e
  2026. )
  2027. pending = _printer_offline_notify_tasks.pop(printer_id, None)
  2028. if pending is not None and not pending.done():
  2029. logging.getLogger(__name__).info(
  2030. "[#1752] Printer %s reconnected before debounce; cancelling pending offline notification",
  2031. printer_id,
  2032. )
  2033. pending.cancel()
  2034. # Only broadcast if something meaningful changed (reduce WebSocket spam)
  2035. # Include rounded temperatures to detect meaningful temp changes (within 1 degree)
  2036. temps = state.temperatures or {}
  2037. nozzle_temp = round(temps.get("nozzle", 0))
  2038. bed_temp = round(temps.get("bed", 0))
  2039. nozzle_2_temp = round(temps.get("nozzle_2", 0)) if "nozzle_2" in temps else ""
  2040. chamber_temp = round(temps.get("chamber", 0)) if "chamber" in temps else ""
  2041. # Auto-detect dual-nozzle printers from MQTT temperature data
  2042. if "nozzle_2" in temps and printer_id not in _nozzle_count_updated:
  2043. _nozzle_count_updated.add(printer_id)
  2044. # Update nozzle_count in database
  2045. async with async_session() as db:
  2046. from backend.app.models.printer import Printer
  2047. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2048. printer = result.scalar_one_or_none()
  2049. if printer and printer.nozzle_count != 2:
  2050. printer.nozzle_count = 2
  2051. await db.commit()
  2052. logging.getLogger(__name__).info(
  2053. f"Auto-detected dual-nozzle printer {printer_id}, updated nozzle_count=2"
  2054. )
  2055. # Include target temps for heating phase detection
  2056. bed_target = round(temps.get("bed_target", 0))
  2057. nozzle_target = round(temps.get("nozzle_target", 0))
  2058. # Include tray_now and vt_tray hash so external spool changes trigger broadcasts
  2059. vt_tray_key = hash(str(state.raw_data.get("vt_tray", []))) if state.raw_data else 0
  2060. # Include AMS dry_time and tray state values so drying/slot changes trigger broadcasts.
  2061. #
  2062. # dry_countdown_stalled rides along because it is the one drying signal the
  2063. # countdown itself cannot carry: the MQTT layer raises it precisely BECAUSE
  2064. # dry_time stopped moving, so on the frame that flips it every other member
  2065. # of this key is identical and the push would be deduplicated away. Mid-print
  2066. # a temperature would eventually break the tie, but a parked command on an
  2067. # idle machine changes nothing else at all — AMS temp and humidity are not in
  2068. # the key — so the badge could sit unreachable indefinitely. The flag flips at
  2069. # most once per drying cycle, so it costs no mid-print broadcast traffic.
  2070. ams_dry_key = (
  2071. tuple((a.get("dry_time", 0), bool(a.get("dry_countdown_stalled"))) for a in (state.raw_data.get("ams") or []))
  2072. if state.raw_data
  2073. else ()
  2074. )
  2075. # Include tray states so load/unload transitions (state 11→10) trigger broadcasts (#784)
  2076. #
  2077. # The filament identity fields are here because Configure Slot writes
  2078. # exactly those and nothing else. Re-configuring a slot from PLA to another
  2079. # brand or colour of PLA leaves id/tray_type/state identical, so the key
  2080. # matched, this function returned before broadcasting, and the card kept
  2081. # showing the old filament until the 30s fallback poll or a page reload —
  2082. # even though the configure route asks the printer for a fresh pushall and
  2083. # that push does carry the new values. Reset always worked, because it
  2084. # clears tray_type.
  2085. #
  2086. # These fields only change when someone configures a slot or swaps a spool,
  2087. # so unlike temperature or progress they add no broadcast traffic mid-print.
  2088. ams_tray_key = (
  2089. tuple(
  2090. (
  2091. t.get("id"),
  2092. t.get("tray_type", ""),
  2093. t.get("state"),
  2094. t.get("tray_color", ""),
  2095. t.get("tray_info_idx", ""),
  2096. t.get("tray_sub_brands", ""),
  2097. t.get("cali_idx"),
  2098. )
  2099. for a in (state.raw_data.get("ams") or [])
  2100. for t in a.get("tray", [])
  2101. )
  2102. if state.raw_data
  2103. else ()
  2104. )
  2105. # Filament Track Switch: which inlet each AMS is bound to, and whether the
  2106. # accessory is fitted at all. Neither is in ams_tray_key (it is per-tray) nor
  2107. # in the AMS change-hash (tray fields only, and widening that would fire
  2108. # spurious Spoolman syncs), so without them a "Join IN-B" on the printer
  2109. # screen changed no key at all and the card's inlet badges sat stale until a
  2110. # reload. Like the filament-backup flag, these only move when someone
  2111. # reconfigures the machine, so they add no mid-print broadcast traffic.
  2112. fts_key = (
  2113. state.fila_switch.installed if state.fila_switch else False,
  2114. tuple(sorted(state.ams_switch_inlet.items())),
  2115. # Which hotend holds which slot. Unlike the two above this does move
  2116. # mid-print, on every filament change — but only between discrete slots,
  2117. # so it adds a push per toolchange, not a stream. The AMS slot menu needs
  2118. # it live: it decides which hotend the Load dialog may offer and whether
  2119. # Unload has anything to act on.
  2120. tuple(
  2121. sorted(
  2122. ((ext, slot.ams_id, slot.slot_id, slot.has_filament) for ext, slot in state.extruder_slots.items()),
  2123. # Sort on the extruder id alone: the other members are nullable
  2124. # and comparing None with an int raises.
  2125. key=lambda entry: entry[0],
  2126. )
  2127. ),
  2128. )
  2129. status_key = (
  2130. f"{state.connected}:{state.state}:{state.progress}:{state.layer_num}:"
  2131. f"{nozzle_temp}:{bed_temp}:{nozzle_2_temp}:{chamber_temp}:"
  2132. f"{state.stg_cur}:{bed_target}:{nozzle_target}:"
  2133. f"{state.cooling_fan_speed}:{state.big_fan1_speed}:{state.big_fan2_speed}:"
  2134. f"{state.chamber_light}:{state.active_extruder}:{state.tray_now}:{vt_tray_key}:"
  2135. f"{ams_dry_key}:{ams_tray_key}:{state.door_open}:{state.ams_filament_backup}:{fts_key}"
  2136. )
  2137. # Fields that change only when the machine does (a nozzle swap, a firmware
  2138. # update, developer mode) or once per print (its AMS mapping): no traffic,
  2139. # but on an idle printer nothing else moves the key when they change.
  2140. machine_key = (
  2141. tuple((n.nozzle_type, n.nozzle_diameter) for n in (state.nozzles or [])),
  2142. state.firmware_version,
  2143. state.developer_mode,
  2144. tuple(state.print_ams_mapping) if isinstance(state.print_ams_mapping, list) else None,
  2145. # Cooling/heating, switched on the printer's screen too: on an idle
  2146. # printer nothing else moves the key (PSF-4).
  2147. state.airduct_mode,
  2148. )
  2149. status_key = f"{status_key}:{machine_key}"
  2150. is_active_print = state.state in _ACTIVE_PRINT_STATES
  2151. if not is_active_print:
  2152. _unauthorized_print_kill_sent.discard(printer_id)
  2153. elif printer_id in _unauthorized_print_kill_sent:
  2154. # stop_print() was already sent for this print; avoid all further
  2155. # ownership and settings work until the printer leaves an active state.
  2156. pass
  2157. elif _is_bambuddy_authorized_print_in_memory(printer_id, state):
  2158. # Normal Bambuddy-started prints stay entirely on the in-memory path.
  2159. _unauthorized_print_kill_sent.discard(printer_id)
  2160. else:
  2161. kill_switch_enabled = False
  2162. authorization: bool | None = None
  2163. status_logger = logging.getLogger(__name__)
  2164. try:
  2165. kill_switch_enabled = await _is_printer_kill_switch_enabled_cached()
  2166. if kill_switch_enabled:
  2167. async with async_session() as db:
  2168. authorization = await _is_bambuddy_authorized_print(printer_id, state, db)
  2169. except Exception as e:
  2170. # Fail safe: a database/reconciliation error must never turn into an
  2171. # irreversible stop of a print whose ownership is still unknown.
  2172. authorization = None
  2173. status_logger.warning(
  2174. "[KILL SWITCH] Failed to reconcile print authorization for printer %s: %s", printer_id, e
  2175. )
  2176. if not kill_switch_enabled or authorization is True:
  2177. _unauthorized_print_kill_sent.discard(printer_id)
  2178. elif authorization is None:
  2179. _unauthorized_print_kill_sent.discard(printer_id)
  2180. status_logger.debug(
  2181. "[KILL SWITCH] Deferring authorization for printer %s until archive state is reconciled",
  2182. printer_id,
  2183. )
  2184. else:
  2185. try:
  2186. stopped = printer_manager.stop_print(printer_id)
  2187. if stopped:
  2188. _unauthorized_print_kill_sent.add(printer_id)
  2189. _kill_switch_stopped.add(printer_id)
  2190. printer_info = printer_manager.get_printer(printer_id)
  2191. printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  2192. filename = state.subtask_name or state.gcode_file or state.current_print or "Unknown"
  2193. notification_data = {
  2194. "status": "stopped",
  2195. "filename": state.gcode_file or state.current_print or "",
  2196. "subtask_name": state.subtask_name or "",
  2197. "progress": state.progress,
  2198. "reason": "unauthorized_print",
  2199. }
  2200. status_logger.warning(
  2201. "[KILL SWITCH] Stopped unauthorized print on printer %s (state=%s)",
  2202. printer_id,
  2203. state.state,
  2204. )
  2205. try:
  2206. await ws_manager.broadcast(
  2207. {
  2208. "type": "kill_switch_triggered",
  2209. "printer_id": printer_id,
  2210. "printer_name": printer_name,
  2211. "filename": filename,
  2212. "reason": "unauthorized_print",
  2213. }
  2214. )
  2215. except Exception as e:
  2216. status_logger.warning(
  2217. "[KILL SWITCH] WebSocket notification failed for printer %s: %s", printer_id, e
  2218. )
  2219. previous_task = _kill_switch_notification_tasks.pop(printer_id, None)
  2220. if previous_task is not None and not previous_task.done():
  2221. previous_task.cancel()
  2222. _kill_switch_notification_tasks[printer_id] = spawn_background_task(
  2223. _send_kill_switch_provider_notification(printer_id, printer_name, notification_data),
  2224. name=f"kill-switch-notification-{printer_id}",
  2225. )
  2226. else:
  2227. status_logger.warning(
  2228. "[KILL SWITCH] Could not stop unauthorized print on printer %s (state=%s)",
  2229. printer_id,
  2230. state.state,
  2231. )
  2232. except Exception as e:
  2233. status_logger.warning(
  2234. "[KILL SWITCH] Failed to stop unauthorized print on printer %s: %s", printer_id, e
  2235. )
  2236. # MQTT relay - publish status (before dedup check - always publish to MQTT)
  2237. try:
  2238. printer_info = printer_manager.get_printer(printer_id)
  2239. if printer_info:
  2240. await mqtt_relay.on_printer_status(
  2241. printer_id,
  2242. state,
  2243. printer_info.name,
  2244. printer_info.serial_number,
  2245. printer_manager.is_awaiting_plate_clear(printer_id),
  2246. )
  2247. except Exception:
  2248. pass # Don't fail status callback if MQTT fails
  2249. if _last_status_broadcast.get(printer_id) == status_key:
  2250. return # No change, skip WebSocket broadcast
  2251. _last_status_broadcast[printer_id] = status_key
  2252. # Check for progress milestone notifications (25%, 50%, 75%)
  2253. progress = state.progress or 0
  2254. is_printing = state.state in ("RUNNING", "PRINTING")
  2255. if is_printing and progress > 0:
  2256. current_milestone = _progress_milestone_to_notify(printer_id, state)
  2257. if current_milestone is not None:
  2258. try:
  2259. from backend.app.models.printer import Printer
  2260. # Read the printer in a short session and release the connection
  2261. # BEFORE the ~15s camera snapshot below — holding it across the grab
  2262. # pinned a pooled connection per milestone, per printer (issue #2572).
  2263. async with async_session() as db:
  2264. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2265. printer = result.scalar_one_or_none()
  2266. printer_name = printer.name if printer else f"Printer {printer_id}"
  2267. filename = state.subtask_name or state.gcode_file or "Unknown"
  2268. # remaining_time is in minutes, convert to seconds for notification
  2269. remaining_time_seconds = state.remaining_time * 60 if state.remaining_time else None
  2270. # Capture camera snapshot for notification image attachment (no DB held).
  2271. image_data = await _capture_snapshot_for_notification(printer_id, printer, logging.getLogger(__name__))
  2272. # Notification send needs a session (provider/template lookups).
  2273. async with async_session() as db:
  2274. await notification_service.on_print_progress(
  2275. printer_id,
  2276. printer_name,
  2277. filename,
  2278. current_milestone,
  2279. db,
  2280. remaining_time_seconds,
  2281. image_data=image_data,
  2282. )
  2283. except Exception as e:
  2284. logging.getLogger(__name__).warning(f"Progress milestone notification failed: {e}")
  2285. elif progress < 5:
  2286. # Reset milestone tracking when print restarts or new print begins
  2287. _last_progress_milestone[printer_id] = 0
  2288. _first_layer_notified[printer_id] = False
  2289. # HMS error codes that should not trigger notifications even though they
  2290. # have known descriptions (e.g. user-initiated actions, not real errors).
  2291. _HMS_NOTIFICATION_SUPPRESS = {
  2292. "0500_400E", # Printing was cancelled (user action, not an error)
  2293. }
  2294. # Check for new HMS errors and send notifications
  2295. current_hms_errors = getattr(state, "hms_errors", []) or []
  2296. if current_hms_errors:
  2297. new_errors = await _take_new_hms_faults_and_save(printer_id, current_hms_errors, state.state)
  2298. if new_errors:
  2299. try:
  2300. from backend.app.models.printer import Printer
  2301. # Read the printer in a short session and release the connection
  2302. # BEFORE the ~15s camera snapshot below (issue #2572).
  2303. async with async_session() as db:
  2304. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2305. printer = result.scalar_one_or_none()
  2306. printer_name = printer.name if printer else f"Printer {printer_id}"
  2307. # Capture camera snapshot once for all error notifications (no DB held).
  2308. error_image_data = await _capture_snapshot_for_notification(
  2309. printer_id, printer, logging.getLogger(__name__)
  2310. )
  2311. # Notification sends need a session (provider/template lookups).
  2312. async with async_session() as db:
  2313. sent_count = 0
  2314. for error in new_errors:
  2315. # Build short code like "0700_8010"
  2316. # Mask to 16 bits to handle printers that send larger values
  2317. error_code_int = int(error.code.replace("0x", ""), 16) if error.code else 0
  2318. error_code_masked = error_code_int & 0xFFFF
  2319. short_code = f"{(error.attr >> 16) & 0xFFFF:04X}_{error_code_masked:04X}"
  2320. # Only notify for errors Bambu publishes text for — printers
  2321. # send undocumented codes that aren't real errors, and Bambu
  2322. # lists some with empty text. Resolved at parse time (#2926);
  2323. # short_code is still needed for the suppression set below.
  2324. description = error.description
  2325. if not description or short_code in _HMS_NOTIFICATION_SUPPRESS:
  2326. continue
  2327. error_type = _hms_error_title(error.module)
  2328. error_detail = description
  2329. await notification_service.on_printer_error(
  2330. printer_id, printer_name, error_type, db, error_detail, image_data=error_image_data
  2331. )
  2332. sent_count += 1
  2333. if sent_count:
  2334. logging.getLogger(__name__).info(
  2335. f"[HMS] Sent notification for {sent_count} error(s) on printer {printer_id}"
  2336. )
  2337. # Also publish to MQTT relay (no DB): every new fault that
  2338. # counts, with or without text, the same set the UI counts.
  2339. printer_info = printer_manager.get_printer(printer_id)
  2340. if printer_info:
  2341. errors_data = [
  2342. {
  2343. "code": e.code,
  2344. "attr": e.attr,
  2345. "module": e.module,
  2346. "severity": e.severity,
  2347. }
  2348. for e in new_errors
  2349. ]
  2350. await mqtt_relay.on_printer_error(
  2351. printer_id, printer_info.name, printer_info.serial_number, errors_data
  2352. )
  2353. except Exception as e:
  2354. logging.getLogger(__name__).warning(f"HMS error notification failed: {e}")
  2355. else:
  2356. # No HMS errors: nothing to send, but faults gone long enough (or gone
  2357. # across a print state change) are forgotten. Some codes, e.g. chamber
  2358. # temperature regulation during PETG prints, toggle every few seconds.
  2359. await _take_new_hms_faults_and_save(printer_id, [], state.state)
  2360. await ws_manager.send_printer_status(
  2361. printer_id,
  2362. printer_state_to_dict(
  2363. state,
  2364. printer_id,
  2365. printer_manager.get_model(printer_id),
  2366. printer_manager.get_drying_targets(printer_id),
  2367. ),
  2368. )
  2369. def _is_bambu_uuid(tray_uuid: str) -> bool:
  2370. """Check if a tray UUID looks like a valid Bambu Lab RFID UUID (non-empty, non-zero)."""
  2371. return bool(tray_uuid) and tray_uuid not in ("", "0" * len(tray_uuid))
  2372. async def on_fts_inlet_change(printer_id: int, ams_id: int, inlet: str):
  2373. """Re-point a moved AMS's K-profiles at the nozzle it now feeds.
  2374. K-profiles are per-nozzle and the printer's calibration table is numbered
  2375. per-nozzle, but a tray holds exactly one ``cali_idx``. Moving an AMS to the
  2376. switch's other inlet therefore silently invalidates every configured slot in
  2377. it: the index stays put and now resolves against the other nozzle's table.
  2378. Measured on the maintainer's H2C — one spool calibrated 0.018 on the left
  2379. and 0.020 on the right kept the left profile after the move, and a manual
  2380. RFID re-read only re-asserted the same wrong one.
  2381. Configuring a slot is a deliberate preparation step, so this re-selects
  2382. rather than re-configures: only the calibration binding moves, and only for
  2383. slots whose spool already has a stored profile for the new nozzle. A slot
  2384. Bambuddy knows nothing about is left exactly as the operator left it.
  2385. """
  2386. logger = logging.getLogger(__name__)
  2387. target_extruder = extruder_for_inlet(inlet)
  2388. if target_extruder is None:
  2389. return
  2390. client = printer_manager.get_client(printer_id)
  2391. state = printer_manager.get_status(printer_id)
  2392. if not client or not state or not state.raw_data:
  2393. return
  2394. # The nozzle the AMS now feeds -- the diameter of the TARGET extruder, not
  2395. # of nozzle 0. On a machine with two sizes fitted, moving the inlet changes
  2396. # the nozzle width, which changes both the K profile to select and the
  2397. # preset the slot should carry.
  2398. nozzle_diameter = nozzle_diameter_for_extruder(state, target_extruder, printer_manager.get_model(printer_id))
  2399. ams_raw = state.raw_data.get("ams")
  2400. ams_list = ams_raw.get("ams", []) if isinstance(ams_raw, dict) else ams_raw if isinstance(ams_raw, list) else []
  2401. unit = next((u for u in ams_list if str(u.get("id")) == str(ams_id)), None)
  2402. if not unit:
  2403. return
  2404. try:
  2405. async with async_session() as db:
  2406. for tray in unit.get("tray", []):
  2407. tray_id = int(tray.get("id", -1))
  2408. if tray_id < 0 or not tray.get("tray_type"):
  2409. continue
  2410. current_idx = tray.get("cali_idx")
  2411. profile = await find_slot_kprofile_for_extruder(
  2412. db,
  2413. printer_id,
  2414. ams_id,
  2415. tray_id,
  2416. target_extruder,
  2417. nozzle_diameter,
  2418. printer_manager.get_model(printer_id),
  2419. nozzle_flow_for_extruder(state, target_extruder, printer_manager.get_model(printer_id)),
  2420. )
  2421. if profile is None or profile.cali_idx is None:
  2422. continue
  2423. if current_idx == profile.cali_idx:
  2424. continue # Already on the right one.
  2425. logger.info(
  2426. "[Printer %s] AMS %s slot %s moved to inlet %s (nozzle %s): "
  2427. "re-selecting K-profile %s (cali_idx %s -> %s, K=%s)",
  2428. printer_id,
  2429. ams_id,
  2430. tray_id,
  2431. inlet,
  2432. target_extruder,
  2433. profile.name,
  2434. current_idx,
  2435. profile.cali_idx,
  2436. profile.k_value,
  2437. )
  2438. client.extrusion_cali_sel(
  2439. ams_id=ams_id,
  2440. tray_id=tray_id,
  2441. cali_idx=profile.cali_idx,
  2442. filament_id=printer_safe_filament_id(profile.filament_id, tray.get("tray_info_idx", "")),
  2443. nozzle_diameter=nozzle_diameter,
  2444. )
  2445. except Exception as e:
  2446. logger.warning("[Printer %s] Could not re-apply K-profiles after inlet move: %s", printer_id, e)
  2447. def _slot_feeds_print(printer_id: int, ams_id: int, tray_id: int) -> bool:
  2448. """Is this slot the one feeding the printer right now?
  2449. ``tray_now`` names one slot for the whole printer, so on a dual-nozzle
  2450. machine each extruder's own fed slot is asked as well. Only meaningful
  2451. while a print runs; the caller checks that.
  2452. """
  2453. status = printer_manager.get_status(printer_id)
  2454. if status is None:
  2455. return False
  2456. global_id = printer_tray_layout(printer_id, printer_manager).tray_from_slot_key(ams_id, tray_id)
  2457. if getattr(status, "tray_now", None) == global_id:
  2458. return True
  2459. slots = getattr(status, "extruder_slots", None) or {}
  2460. return any(slot.has_filament and slot.holds(ams_id, tray_id) for slot in slots.values())
  2461. def _tray_reports_no_filament(color: str, tray_type: str) -> bool:
  2462. """Does this tray report carry no filament data at all?
  2463. No type, and a colour that is either empty or ``00000000``: firmware sends
  2464. both for a blank slot, and ``00000000`` is what Bambuddy's own slot reset
  2465. writes. A reconnect mid-print brought it in that form and got past every
  2466. guard that only knew the empty string (#3526). A clear spool also reports
  2467. ``00000000``, but always with a type, so it is not blank.
  2468. """
  2469. return not tray_type.strip() and color.strip().upper() in ("", "00000000")
  2470. async def _unlink_stale_assignments(printer_id: int, ams_data: list, printing_now: bool) -> None:
  2471. """Unlink built-in inventory assignments whose slot no longer holds their spool.
  2472. Runs from ``on_ams_change`` and, for removals held by ``slot_unlink_grace``,
  2473. from the delayed re-check -- which is why it takes the AMS data and print
  2474. state as arguments instead of reading them from the push (#3186).
  2475. """
  2476. logger = logging.getLogger(__name__)
  2477. from backend.app.utils.color_utils import colors_similar as _colors_similar
  2478. # Auto-unlink spool assignments with stale fingerprints. Under the
  2479. # per-printer assignment lock since #3186: the held-removal re-check runs
  2480. # this outside any MQTT push, so it can now overlap one.
  2481. try:
  2482. async with get_ams_assignment_lock(printer_id), async_session() as db:
  2483. from sqlalchemy.orm import selectinload
  2484. from backend.app.api.routes.inventory import _find_tray_in_ams_data
  2485. from backend.app.models.spool import Spool as _Spool
  2486. from backend.app.models.spool_assignment import SpoolAssignment as SA
  2487. from backend.app.services.ams_slot_presence import spool_present
  2488. from backend.app.services.inventory_mode import spoolman_owns_assignments
  2489. # Built-in assignments only. Since #2812 they survive a switch to
  2490. # Spoolman mode rather than being deleted by it, and this pass ends
  2491. # in ``db.delete`` — left ungated it would unlink them one slot at a
  2492. # time as the AMS contents changed under the other mode, undoing the
  2493. # preservation more slowly but just as completely.
  2494. assignments = []
  2495. if not await spoolman_owns_assignments(db):
  2496. result = await db.execute(
  2497. select(SA)
  2498. .where(SA.printer_id == printer_id)
  2499. .options(selectinload(SA.spool).selectinload(_Spool.k_profiles))
  2500. )
  2501. assignments = result.scalars().all()
  2502. # ``printing_now`` (top of this function) keeps a runout from
  2503. # unlinking the spool that fed the print — the next idle-time pass
  2504. # unlinks it if the user really did take it out.
  2505. stale = []
  2506. # Removals this pass is holding rather than unlinking (#3186).
  2507. held: set[tuple] = set()
  2508. for assignment in assignments:
  2509. # External spool assignments (ams_id=255) live in vt_tray, not AMS data
  2510. if assignment.ams_id == 255:
  2511. ps = printer_manager.get_status(printer_id)
  2512. vt_tray_raw = ps.raw_data.get("vt_tray", []) if ps else []
  2513. ext_id = external_tray_id(assignment.tray_id) # 0→254, 1→255, rack 2-8→247-253
  2514. current_tray = None
  2515. for vt in vt_tray_raw:
  2516. if isinstance(vt, dict) and int(vt.get("id", 254)) == ext_id:
  2517. current_tray = vt
  2518. break
  2519. if not current_tray:
  2520. # vt_tray data may not have arrived yet — keep assignment
  2521. continue
  2522. else:
  2523. current_tray = _find_tray_in_ams_data(ams_data, assignment.ams_id, assignment.tray_id)
  2524. if not current_tray:
  2525. if printing_now:
  2526. logger.info(
  2527. "Auto-unlink skipped: spool %d AMS%d-T%d — slot empty during a running print (runout?)",
  2528. assignment.spool_id,
  2529. assignment.ams_id,
  2530. assignment.tray_id,
  2531. )
  2532. continue
  2533. # A whole AMS unit can drop out of one push and come back in
  2534. # the next; only a slot that stays gone is a removal (#3186).
  2535. hold_key = ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id)
  2536. if not slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2537. held.add(hold_key)
  2538. logger.info(
  2539. "Auto-unlink held: spool %d AMS%d-T%d — tray not found in AMS data; "
  2540. "unlinking if it is still gone in %ds",
  2541. assignment.spool_id,
  2542. assignment.ams_id,
  2543. assignment.tray_id,
  2544. int(slot_unlink_grace.GRACE_SECONDS),
  2545. )
  2546. continue
  2547. logger.info(
  2548. "Auto-unlink: spool %d AMS%d-T%d — tray not found in AMS data (slot empty?)",
  2549. assignment.spool_id,
  2550. assignment.ams_id,
  2551. assignment.tray_id,
  2552. )
  2553. stale.append(assignment) # Slot empty
  2554. elif _is_bambu_uuid(current_tray.get("tray_uuid", "")):
  2555. # A Bambu Lab spool is in this slot — check if it's the same spool
  2556. # that's currently assigned. If yes, keep the assignment (avoids
  2557. # unnecessary unlink/re-assign/ams_filament_setting cycle that clears
  2558. # the printer's filament preset on every startup).
  2559. tray_uuid = current_tray.get("tray_uuid", "")
  2560. tag_uid = current_tray.get("tag_uid", "")
  2561. spool = assignment.spool
  2562. spool_matches = False
  2563. if spool:
  2564. if (spool.tray_uuid and spool.tray_uuid.upper() == tray_uuid.upper()) or (
  2565. spool.tag_uid
  2566. and tag_uid
  2567. and tag_uid != "0000000000000000"
  2568. and spool.tag_uid.upper() == tag_uid.upper()
  2569. ):
  2570. spool_matches = True
  2571. if spool_matches:
  2572. # Same BL spool still in slot — keep assignment, update fingerprint if needed
  2573. cur_color = current_tray.get("tray_color", "")
  2574. cur_type = current_tray.get("tray_type", "")
  2575. fp_color = assignment.fingerprint_color or ""
  2576. fp_type = assignment.fingerprint_type or ""
  2577. if cur_color.upper() != fp_color.upper() or cur_type.upper() != fp_type.upper():
  2578. assignment.fingerprint_color = cur_color
  2579. assignment.fingerprint_type = cur_type
  2580. logger.debug(
  2581. "Auto-unlink: spool %d AMS%d-T%d — same BL spool, updated fingerprint",
  2582. assignment.spool_id,
  2583. assignment.ams_id,
  2584. assignment.tray_id,
  2585. )
  2586. continue
  2587. # Different BL spool or unrecognized — unlink so auto-assign can match
  2588. logger.info(
  2589. "Auto-unlink: spool %d AMS%d-T%d — different Bambu Lab spool detected (uuid=%s)",
  2590. assignment.spool_id,
  2591. assignment.ams_id,
  2592. assignment.tray_id,
  2593. tray_uuid,
  2594. )
  2595. stale.append(assignment)
  2596. else:
  2597. cur_color = current_tray.get("tray_color", "")
  2598. cur_type = current_tray.get("tray_type", "")
  2599. cur_state = current_tray.get("state")
  2600. fp_color = assignment.fingerprint_color or ""
  2601. fp_type = assignment.fingerprint_type or ""
  2602. # SpoolBuddy pre-config replay: fingerprint_type empty means
  2603. # the slot was empty when the user pre-assigned via SpoolBuddy
  2604. # (the firmware drops ams_filament_setting on empty slots, so
  2605. # MQTT was deferred). The moment any filament gets inserted
  2606. # — Bambu RFID, 3rd-party, or even an existing-but-now-
  2607. # reconfigured spool — fire the deferred configuration.
  2608. # The "loaded" signal is state == 11 (Bambu's "filament fed to
  2609. # extruder" code) OR, on firmwares that don't use the state
  2610. # enum meaningfully, a non-empty tray_type when state is
  2611. # NOT one of the firmware's explicit empty signals (9, 10).
  2612. # state-only was wrong for firmwares that never set 11 — A1
  2613. # Mini BMCU 01.07.02.00 and P1S Standard AMS 00.00.06.75 both
  2614. # always report state=3 — so the replay never fired for them
  2615. # (#1322). The state ∉ {9,10} guard keeps the firmware's
  2616. # explicit "empty" signals authoritative over any stale
  2617. # tray_type that might survive the relay's auto-clearing.
  2618. #
  2619. # tray_exist_bits comes first because that guard cannot tell
  2620. # a firmware "empty" from Bambuddy's own: apply_tray_exist_bits
  2621. # writes state=9 when the bit is 0 and leaves it there when the
  2622. # bit returns. A non-RFID spool inserted into a pre-assigned
  2623. # slot brings no tray_type with it, so the stale 9 made this
  2624. # expression false forever and the deferred config never fired
  2625. # — the deadlock #1322 removed from the assign path, still in
  2626. # place here (#3084, #3100).
  2627. loaded = spool_present(current_tray) is True or (
  2628. cur_state == 11 or (cur_state not in (9, 10) and cur_type.strip())
  2629. )
  2630. # Not into the slot feeding a running print: the printer refuses
  2631. # the setting for it (``ams_filament_setting refused``), and a
  2632. # spool swapped in after a runout sat through one refused send
  2633. # per AMS change. Held, it fires on the first change once the
  2634. # print is over (#3434).
  2635. if (
  2636. not fp_type.strip()
  2637. and loaded
  2638. and assignment.spool
  2639. and printing_now
  2640. and _slot_feeds_print(printer_id, assignment.ams_id, assignment.tray_id)
  2641. ):
  2642. logger.debug(
  2643. "Pre-config held for spool %d AMS%d-T%d: the slot is feeding the print",
  2644. assignment.spool_id,
  2645. assignment.ams_id,
  2646. assignment.tray_id,
  2647. )
  2648. continue
  2649. if not fp_type.strip() and loaded and assignment.spool:
  2650. # The fingerprint ends this replay. Not recorded when nothing
  2651. # could be sent (no MQTT client while the printer reconnects),
  2652. # so the next AMS change tries again. An attempt that raised is
  2653. # recorded as before: it may have sent part of the commands and
  2654. # would raise again on every change, remain% steps included.
  2655. configured: bool | None = False
  2656. try:
  2657. from backend.app.api.routes.inventory import (
  2658. apply_spool_to_slot_via_mqtt,
  2659. load_preset_lookup_user,
  2660. )
  2661. from backend.app.core.auth import is_auth_enabled
  2662. # Whoever made the assignment: their cloud sign-in
  2663. # resolves a custom preset, which nobody's did here,
  2664. # so the slot got Generic PLA on insert (#3519). With
  2665. # login off the shared sign-in applies, as before.
  2666. replay_user = None
  2667. if assignment.assigned_by_user_id is not None and await is_auth_enabled(db):
  2668. replay_user = await load_preset_lookup_user(db, assignment.assigned_by_user_id)
  2669. configured = await apply_spool_to_slot_via_mqtt(
  2670. db=db,
  2671. current_user=replay_user,
  2672. spool=assignment.spool,
  2673. printer_id=printer_id,
  2674. ams_id=assignment.ams_id,
  2675. tray_id=assignment.tray_id,
  2676. current_tray_info_idx=current_tray.get("tray_info_idx", ""),
  2677. current_tray_type=cur_type,
  2678. current_tray_sub_brands=current_tray.get("tray_sub_brands", ""),
  2679. )
  2680. except Exception:
  2681. logger.exception(
  2682. "Pre-config apply failed for spool %d on printer %d AMS%d-T%d",
  2683. assignment.spool_id,
  2684. printer_id,
  2685. assignment.ams_id,
  2686. assignment.tray_id,
  2687. )
  2688. configured = None
  2689. if configured is False:
  2690. logger.warning(
  2691. "Pre-config not sent for spool %d on printer %d AMS%d-T%d (no MQTT client); "
  2692. "retrying on the next AMS change",
  2693. assignment.spool_id,
  2694. printer_id,
  2695. assignment.ams_id,
  2696. assignment.tray_id,
  2697. )
  2698. else:
  2699. if configured:
  2700. logger.info(
  2701. "SpoolBuddy pre-config applied on insert: spool %d → printer %d AMS%d-T%d",
  2702. assignment.spool_id,
  2703. printer_id,
  2704. assignment.ams_id,
  2705. assignment.tray_id,
  2706. )
  2707. assignment.fingerprint_color = cur_color
  2708. assignment.fingerprint_type = cur_type
  2709. continue
  2710. if not _colors_similar(cur_color, fp_color) or cur_type.upper() != fp_type.upper():
  2711. # Blank tray data mid-print is a runout, not a swap: the
  2712. # firmware clears colour and type when it unloads a spool
  2713. # it just emptied. Unlinking here would erase the record
  2714. # of which spool fed the print so far.
  2715. if printing_now and _tray_reports_no_filament(cur_color, cur_type):
  2716. logger.info(
  2717. "Auto-unlink skipped: spool %d AMS%d-T%d — tray data cleared during a running print "
  2718. "(runout?)",
  2719. assignment.spool_id,
  2720. assignment.ams_id,
  2721. assignment.tray_id,
  2722. )
  2723. continue
  2724. # Same reasoning off the print, on firmware's own say-so:
  2725. # a blank tray report from a slot whose tray_exist_bits
  2726. # bit is set describes a spool the AMS cannot identify —
  2727. # a non-RFID one, or one whose slot was reset — not a
  2728. # spool that was taken out. Deleting the assignment there
  2729. # threw away the identity the user had supplied, which is
  2730. # the only place it existed (#3100). A slot the bit calls
  2731. # empty, or one that carries no bit at all, still unlinks.
  2732. #
  2733. # Unless the slot was reported empty first: a removal
  2734. # already held means a spool came out and another went
  2735. # in, so the blank report keeps the hold running below
  2736. # rather than cancelling it (#3186).
  2737. if (
  2738. spool_present(current_tray) is True
  2739. and _tray_reports_no_filament(cur_color, cur_type)
  2740. and not slot_unlink_grace.is_held(
  2741. printer_id,
  2742. ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id),
  2743. )
  2744. ):
  2745. logger.info(
  2746. "Auto-unlink skipped: spool %d AMS%d-T%d — slot still occupied, "
  2747. "tray reports no filament data yet",
  2748. assignment.spool_id,
  2749. assignment.ams_id,
  2750. assignment.tray_id,
  2751. )
  2752. continue
  2753. # A blank report the presence bit does not vouch for is
  2754. # still only one push. An idle X1C cleared a whole AMS
  2755. # unit's bits, colour and type for a moment and lost
  2756. # four saved assignments that way (#3186); unlink only
  2757. # if the slot is still blank after the grace period.
  2758. if _tray_reports_no_filament(cur_color, cur_type):
  2759. hold_key = ("inventory", assignment.ams_id, assignment.tray_id, assignment.spool_id)
  2760. if not slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2761. held.add(hold_key)
  2762. logger.info(
  2763. "Auto-unlink held: spool %d AMS%d-T%d — tray reports no filament data; "
  2764. "unlinking if it is still blank in %ds",
  2765. assignment.spool_id,
  2766. assignment.ams_id,
  2767. assignment.tray_id,
  2768. int(slot_unlink_grace.GRACE_SECONDS),
  2769. )
  2770. continue
  2771. # Fingerprint mismatch — but check if tray now matches the
  2772. # assigned spool (e.g. auto-configure changed the tray).
  2773. # Both sides are reduced to the type the slot can carry
  2774. # before comparing: the assign path writes that rather
  2775. # than the spool's raw material (#2902), so a spool whose
  2776. # material is a product line — "PLA+", "HTPLA" — reports
  2777. # back as "PLA" and would otherwise fail this check and
  2778. # be auto-unlinked from the slot it was just assigned to.
  2779. # Reducing the printer's side too keeps slots configured
  2780. # by an older Bambuddy, still reporting "PLA+", matching.
  2781. spool = assignment.spool
  2782. if spool:
  2783. spool_color = (spool.rgba or "FFFFFFFF").upper()
  2784. # Two ways the assign path can have arrived at the
  2785. # slot's type, so both count as "we wrote this".
  2786. # The material column is one; the spool's preset is
  2787. # the other, and it outranks the material when the
  2788. # spool has one -- a spool whose material says PLA
  2789. # and whose preset is "Bambu PLA Aero" puts
  2790. # PLA-AERO in the slot (#2902). Read from the stored
  2791. # preset name rather than resolving the preset,
  2792. # because this runs on every AMS push and a cloud
  2793. # lookup here would be both slow and unavailable on
  2794. # the unauthenticated replay path.
  2795. spool_types = {printer_filament_type(spool.material).upper()}
  2796. if spool.slicer_filament_name:
  2797. spool_types.add(printer_filament_type(spool.slicer_filament_name).upper())
  2798. # An imported local preset stores its type outright,
  2799. # which is what the assign path used -- and the name
  2800. # above may be unset. One keyed read, and only on a
  2801. # mismatch, which is rare.
  2802. #
  2803. # slicer_filament is free text up to fifty characters,
  2804. # so the digits have to be checked against the range
  2805. # of the integer primary key they are about to be
  2806. # compared with. Postgres raises on an out-of-range
  2807. # integer rather than simply not matching, and that
  2808. # would poison this session and abandon the rest of
  2809. # the cleanup pass.
  2810. lp_ref = (spool.slicer_filament or "").strip()
  2811. if lp_ref.isdigit() and int(lp_ref) <= 2147483647:
  2812. from backend.app.models.local_preset import LocalPreset as _LP
  2813. lp_type = await db.scalar(select(_LP.filament_type).where(_LP.id == int(lp_ref)))
  2814. if lp_type:
  2815. spool_types.add(printer_filament_type(lp_type).upper())
  2816. if (
  2817. _colors_similar(cur_color, spool_color)
  2818. and printer_filament_type(cur_type).upper() in spool_types
  2819. ):
  2820. logger.info(
  2821. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch but tray matches spool, updating fp",
  2822. assignment.spool_id,
  2823. assignment.ams_id,
  2824. assignment.tray_id,
  2825. )
  2826. assignment.fingerprint_color = cur_color
  2827. assignment.fingerprint_type = cur_type
  2828. continue
  2829. logger.info(
  2830. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch (cur=%s/%s fp=%s/%s spool=%s/%s)",
  2831. assignment.spool_id,
  2832. assignment.ams_id,
  2833. assignment.tray_id,
  2834. cur_color,
  2835. cur_type,
  2836. fp_color,
  2837. fp_type,
  2838. spool.rgba if spool else "?",
  2839. spool.material if spool else "?",
  2840. )
  2841. stale.append(assignment) # Spool changed
  2842. slot_unlink_grace.settle(printer_id, "inventory", held)
  2843. # Snapshot slots before delete — ORM attribute access after the
  2844. # commit would refresh against a deleted row.
  2845. unlinked_slots = [(a.ams_id, a.tray_id) for a in stale]
  2846. for a in stale:
  2847. await db.delete(a)
  2848. if stale:
  2849. logger.info("Auto-unlinked %d stale spool assignments for printer %d", len(stale), printer_id)
  2850. # Commit any changes (stale deletions and/or fingerprint updates)
  2851. await db.commit()
  2852. # Tell open browsers the assignment is gone (#2575). Only the manual
  2853. # REST assign/unassign endpoints broadcast this event; without it the
  2854. # frontend's spool-assignments cache keeps rendering the unlinked
  2855. # spool on the slot until an unrelated refetch — which reads exactly
  2856. # like "the fix didn't work" (reporter verified: a browser refresh
  2857. # after the swap showed the correct state all along).
  2858. for ams_id, tray_id in unlinked_slots:
  2859. await ws_manager.broadcast(
  2860. {
  2861. "type": "spool_assignment_changed",
  2862. "printer_id": printer_id,
  2863. "ams_id": ams_id,
  2864. "tray_id": tray_id,
  2865. }
  2866. )
  2867. except Exception as e:
  2868. logger.warning("Spool assignment cleanup failed: %s", e, exc_info=True)
  2869. async def _expire_spoolman_empty_slots(printer_id: int, ams_data: list, printing_now: bool) -> None:
  2870. """Delete Spoolman slot rows whose held removal has run its grace period.
  2871. The Spoolman half of the #3186 re-check. The full sync in ``on_ams_change``
  2872. talks to Spoolman for every tray; this only needs the local rows, so it
  2873. repeats that pass's empty-slot decision -- a tray with no type or no colour
  2874. (``parse_ams_tray`` returns None for exactly those), not during a print, and
  2875. not in a slot the presence bit calls occupied -- and nothing else.
  2876. """
  2877. logger = logging.getLogger(__name__)
  2878. try:
  2879. async with async_session() as db:
  2880. from backend.app.api.routes.settings import get_setting
  2881. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  2882. from backend.app.services.ams_slot_presence import spool_present
  2883. enabled = await get_setting(db, "spoolman_enabled")
  2884. if not enabled or enabled.lower() != "true":
  2885. return
  2886. sync_mode = await get_setting(db, "spoolman_sync_mode")
  2887. if sync_mode and sync_mode != "auto":
  2888. return
  2889. trays: dict[tuple[int, int], dict] = {}
  2890. for ams_unit in ams_data or []:
  2891. if not isinstance(ams_unit, dict):
  2892. continue
  2893. for tray in ams_unit.get("tray", []):
  2894. if isinstance(tray, dict):
  2895. trays[(int(ams_unit.get("id", 0)), int(tray.get("id", 0)))] = tray
  2896. rows = (
  2897. (
  2898. await db.execute(
  2899. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  2900. )
  2901. )
  2902. .scalars()
  2903. .all()
  2904. )
  2905. held: set[tuple] = set()
  2906. expired: list[tuple[int, int]] = []
  2907. for row in rows:
  2908. tray = trays.get((row.ams_id, row.tray_id))
  2909. if tray is None or printing_now:
  2910. continue
  2911. if (tray.get("tray_type") or "").strip() and (tray.get("tray_color") or "").strip():
  2912. continue
  2913. hold_key = ("spoolman", row.ams_id, row.tray_id, row.spoolman_spool_id)
  2914. if spool_present(tray) is True and not slot_unlink_grace.is_held(printer_id, hold_key):
  2915. continue
  2916. if slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  2917. expired.append((row.ams_id, row.tray_id))
  2918. else:
  2919. held.add(hold_key)
  2920. slot_unlink_grace.settle(printer_id, "spoolman", held)
  2921. if not expired:
  2922. return
  2923. # A statement rather than ORM deletes, like the sync pass: the two
  2924. # can overlap, and a row the other already removed must not fail
  2925. # this commit.
  2926. for ams_id, tray_id in expired:
  2927. await db.execute(
  2928. delete(SpoolmanSlotAssignment).where(
  2929. SpoolmanSlotAssignment.printer_id == printer_id,
  2930. SpoolmanSlotAssignment.ams_id == ams_id,
  2931. SpoolmanSlotAssignment.tray_id == tray_id,
  2932. )
  2933. )
  2934. await db.commit()
  2935. logger.info("Unlinked %d Spoolman slot(s) that stayed empty for printer %d", len(expired), printer_id)
  2936. for ams_id, tray_id in expired:
  2937. await ws_manager.broadcast(
  2938. {
  2939. "type": "spool_assignment_changed",
  2940. "printer_id": printer_id,
  2941. "ams_id": ams_id,
  2942. "tray_id": tray_id,
  2943. }
  2944. )
  2945. except Exception as e:
  2946. logger.warning("Spoolman slot re-check failed for printer %s: %s", printer_id, e, exc_info=True)
  2947. async def _recheck_held_unlinks(printer_id: int) -> None:
  2948. """Re-run just the slot cleanup against the printer's current AMS state.
  2949. Scheduled by ``slot_unlink_grace`` when it holds a removal. The unlink
  2950. passes otherwise run only when the AMS hash changes, and a slot that went
  2951. empty and stayed empty may never change it again.
  2952. """
  2953. status = printer_manager.get_status(printer_id)
  2954. # A disconnected printer's state is its last report, not a new one: acting
  2955. # on it would "confirm" a removal nobody has seen for the whole grace
  2956. # period. Leave the holds; the first push after reconnecting decides.
  2957. if status is None or not status.connected:
  2958. return
  2959. ams_raw = status.raw_data.get("ams")
  2960. ams_data = ams_raw.get("ams", []) if isinstance(ams_raw, dict) else ams_raw if isinstance(ams_raw, list) else []
  2961. printing_now = (getattr(status, "state", "") or "").upper() in ("RUNNING", "PAUSE")
  2962. await _unlink_stale_assignments(printer_id, ams_data, printing_now)
  2963. await _expire_spoolman_empty_slots(printer_id, ams_data, printing_now)
  2964. slot_unlink_grace.set_recheck(_recheck_held_unlinks)
  2965. async def on_ams_change(printer_id: int, ams_data: list):
  2966. """Handle AMS data changes - sync to Spoolman if enabled and auto mode."""
  2967. logger = logging.getLogger(__name__)
  2968. # Snapshot BEFORE any await: if a print is active, or a finished one is
  2969. # still being charged, skip weight sync later. on_print_complete may pop
  2970. # _active_sessions during our awaits (#880).
  2971. from backend.app.services.usage_tracker import printer_is_charging
  2972. _print_active = printer_is_charging(printer_id)
  2973. # A slot that reports empty while a print is running is a filament runout,
  2974. # not a spool swap: the spool is still physically in the AMS, just
  2975. # consumed. Dropping either inventory backend's slot link there loses the
  2976. # only record of which spool fed the print, so the completion path can't
  2977. # charge the runout segment to anything. Both cleanup passes below consult
  2978. # this; computed once, up front, so neither depends on the other having run.
  2979. _unlink_state = printer_manager.get_status(printer_id)
  2980. printing_now = (getattr(_unlink_state, "state", "") or "").upper() in ("RUNNING", "PAUSE")
  2981. # The session alone is not enough: a restart that finds a stale persisted
  2982. # session registers none for the print that is running.
  2983. _print_active = _print_active or printing_now
  2984. # MQTT relay - publish AMS change
  2985. try:
  2986. printer_info = printer_manager.get_printer(printer_id)
  2987. if printer_info:
  2988. await mqtt_relay.on_ams_change(printer_id, printer_info.name, printer_info.serial_number, ams_data)
  2989. except Exception:
  2990. pass # Don't fail AMS callback if MQTT fails
  2991. # Broadcast AMS change via WebSocket (bypasses status_key deduplication)
  2992. # This ensures frontend gets immediate updates when AMS slots are configured
  2993. try:
  2994. state = printer_manager.get_status(printer_id)
  2995. if state:
  2996. logger.info("[Printer %s] Broadcasting AMS change via WebSocket", printer_id)
  2997. await ws_manager.send_printer_status(
  2998. printer_id,
  2999. printer_state_to_dict(
  3000. state,
  3001. printer_id,
  3002. printer_manager.get_model(printer_id),
  3003. printer_manager.get_drying_targets(printer_id),
  3004. ),
  3005. )
  3006. except Exception as e:
  3007. logger.warning("Failed to broadcast AMS change for printer %s: %s", printer_id, e)
  3008. await _unlink_stale_assignments(printer_id, ams_data, printing_now)
  3009. # Spools whose tag this pass assigned to a slot; their other slots are
  3010. # released below, once this printer's lock is let go (#3427).
  3011. rfid_assigned: list[tuple[int, tuple[int, int, int]]] = []
  3012. # Auto-manage inventory spools from AMS tray data (skip if Spoolman manages AMS).
  3013. # Serialised per-printer via services/ams_assignment_lock: MQTT bursts can deliver
  3014. # two AMS pushes ~30 ms apart, and without the lock both callbacks read
  3015. # "no existing assignment" for the same (printer, ams, tray) and race to
  3016. # INSERT, hitting the spool_assignment_printer_id_ams_id_tray_id_key
  3017. # unique constraint on Postgres. SQLite's WAL serialises writes so the
  3018. # bug stayed latent there. See services/ams_assignment_lock for details.
  3019. try:
  3020. async with get_ams_assignment_lock(printer_id), async_session() as db:
  3021. from sqlalchemy.orm import selectinload
  3022. from backend.app.api.routes.settings import get_setting
  3023. from backend.app.models.spool import Spool
  3024. from backend.app.models.spool_assignment import SpoolAssignment as SA
  3025. from backend.app.services.spool_tag_matcher import (
  3026. auto_assign_spool,
  3027. create_spool_from_tray,
  3028. find_matching_untagged_spool,
  3029. get_spool_by_tag,
  3030. is_bambu_tag,
  3031. is_valid_tag,
  3032. link_tag_to_inventory_spool,
  3033. tag_belongs_to_archived_spool,
  3034. )
  3035. _spoolman_on = await get_setting(db, "spoolman_enabled")
  3036. _auto_add_raw = await get_setting(db, "auto_add_unknown_rfid")
  3037. _auto_add_unknown = _auto_add_raw is None or _auto_add_raw.lower() == "true"
  3038. if not _spoolman_on or _spoolman_on.lower() != "true":
  3039. for ams_unit in ams_data:
  3040. if not isinstance(ams_unit, dict):
  3041. continue
  3042. ams_id = int(ams_unit.get("id", 0))
  3043. for tray in ams_unit.get("tray", []):
  3044. if not isinstance(tray, dict):
  3045. continue
  3046. tray_id = int(tray.get("id", 0))
  3047. tag_uid = tray.get("tag_uid", "")
  3048. tray_uuid = tray.get("tray_uuid", "")
  3049. tray_info_idx = tray.get("tray_info_idx", "")
  3050. if not tray.get("tray_type"):
  3051. # Slot reported empty — drop any cached unknown-tag
  3052. # broadcast so reinserting the same spool re-prompts.
  3053. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id)
  3054. continue # Empty slot
  3055. # Check if assignment already exists for this slot
  3056. existing = await db.execute(
  3057. select(SA)
  3058. .options(selectinload(SA.spool).selectinload(Spool.k_profiles))
  3059. .where(SA.printer_id == printer_id, SA.ams_id == ams_id, SA.tray_id == tray_id)
  3060. )
  3061. existing_assignment = existing.scalar_one_or_none()
  3062. if existing_assignment:
  3063. # Sync spool weight_used from AMS remain — only INCREASE, never decrease.
  3064. # The AMS remain% is low-resolution (integer %, i.e. 10g steps for 1kg spool)
  3065. # and must not overwrite precise values from the usage tracker (3MF/G-code).
  3066. # Skip during active prints: the usage tracker handles deduction
  3067. # precisely via 3MF data on print completion. Without this guard the
  3068. # AMS remain% SET and the usage tracker ADD both fire from the same
  3069. # MQTT message, doubling the deduction (#880).
  3070. if _print_active:
  3071. continue
  3072. remain_raw = tray.get("remain")
  3073. if (
  3074. remain_raw is not None
  3075. and existing_assignment.spool
  3076. and not existing_assignment.spool.weight_locked
  3077. ):
  3078. try:
  3079. remain_val = int(remain_raw)
  3080. except (TypeError, ValueError):
  3081. remain_val = -1
  3082. if 1 <= remain_val <= 100:
  3083. lw = existing_assignment.spool.label_weight or 1000
  3084. new_used = round(lw * (100 - remain_val) / 100.0, 1)
  3085. current_used = existing_assignment.spool.weight_used or 0
  3086. if new_used > current_used + 1:
  3087. # Rechecked in the statement: the spool was loaded
  3088. # before this, and a weight set by hand (or weighed)
  3089. # and locked since then must stay (WL-6).
  3090. result = cast(
  3091. CursorResult,
  3092. await db.execute(
  3093. update(Spool)
  3094. .where(
  3095. Spool.id == existing_assignment.spool_id,
  3096. Spool.weight_locked.is_not(True),
  3097. func.coalesce(Spool.weight_used, 0) < new_used - 1,
  3098. )
  3099. .values(weight_used=new_used)
  3100. ),
  3101. )
  3102. await db.commit()
  3103. if result.rowcount:
  3104. logger.info(
  3105. "Weight sync: spool %d weight_used %s -> %s (remain=%d)",
  3106. existing_assignment.spool_id,
  3107. current_used,
  3108. new_used,
  3109. remain_val,
  3110. )
  3111. await ws_manager.broadcast({"type": "inventory_changed"})
  3112. # A K-profile the slot lost (the default K after a reset or
  3113. # power cycle) is put back by kprofile_drift on the next idle
  3114. # push; a different real profile was picked on purpose and
  3115. # stays (#3219, item 44).
  3116. continue
  3117. if is_bambu_tag(tag_uid, tray_uuid, tray_info_idx):
  3118. # BL spool with RFID tag: auto-match → inventory match → auto-create
  3119. spool = await get_spool_by_tag(db, tag_uid, tray_uuid)
  3120. # Only a spool already known by this tag is proven to be
  3121. # here; a material/colour match or a new spool is not
  3122. # grounds to take it out of other slots (#3427).
  3123. known_by_tag = spool is not None
  3124. if not spool and await tag_belongs_to_archived_spool(db, tag_uid, tray_uuid):
  3125. # An archived spool still in the AMS: the slot
  3126. # stays free rather than adopting its tag.
  3127. continue
  3128. if not spool:
  3129. # Try matching an untagged inventory spool (same material/color)
  3130. spool = await find_matching_untagged_spool(db, tray)
  3131. if spool:
  3132. await link_tag_to_inventory_spool(db, spool, tray)
  3133. elif _auto_add_unknown:
  3134. spool = await create_spool_from_tray(db, tray)
  3135. else:
  3136. # Auto-add disabled: surface the slot so the
  3137. # user can add it manually via the UI.
  3138. await _broadcast_unknown_tag(
  3139. printer_id=printer_id,
  3140. ams_id=ams_id,
  3141. tray_id=tray_id,
  3142. tag_uid=tag_uid,
  3143. tray_uuid=tray_uuid,
  3144. tray_type=tray.get("tray_type"),
  3145. tray_color=tray.get("tray_color"),
  3146. tray_sub_brands=tray.get("tray_sub_brands"),
  3147. tray_count=len(ams_unit.get("tray", [])),
  3148. )
  3149. continue
  3150. # Slot matched (existing tag, untagged inventory
  3151. # match, or freshly auto-created spool) — drop any
  3152. # stale dedup so a future tag swap re-prompts.
  3153. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id)
  3154. await auto_assign_spool(
  3155. printer_id,
  3156. ams_id,
  3157. tray_id,
  3158. spool,
  3159. printer_manager,
  3160. db,
  3161. tray_info_idx=tray_info_idx,
  3162. )
  3163. await db.commit()
  3164. if known_by_tag:
  3165. rfid_assigned.append((spool.id, (printer_id, ams_id, tray_id)))
  3166. await ws_manager.broadcast(
  3167. {
  3168. "type": "spool_auto_assigned",
  3169. "printer_id": printer_id,
  3170. "ams_id": ams_id,
  3171. "tray_id": tray_id,
  3172. "spool_id": spool.id,
  3173. }
  3174. )
  3175. logger.info(
  3176. "RFID auto-assigned spool %d to printer %d AMS%d-T%d",
  3177. spool.id,
  3178. printer_id,
  3179. ams_id,
  3180. tray_id,
  3181. )
  3182. elif is_valid_tag(tag_uid, tray_uuid) and not await tag_belongs_to_archived_spool(
  3183. db, tag_uid, tray_uuid
  3184. ):
  3185. # Non-BL spool with some tag — let user choose (not
  3186. # an archived one's, whose slot stays free)
  3187. await _broadcast_unknown_tag(
  3188. printer_id=printer_id,
  3189. ams_id=ams_id,
  3190. tray_id=tray_id,
  3191. tag_uid=tag_uid,
  3192. tray_uuid=tray_uuid,
  3193. tray_type=tray.get("tray_type"),
  3194. tray_color=tray.get("tray_color"),
  3195. tray_sub_brands=tray.get("tray_sub_brands"),
  3196. tray_count=len(ams_unit.get("tray", [])),
  3197. )
  3198. # No-tag slots (generic non-RFID filament) are left alone:
  3199. # nothing to identify, prompting "+ Add" would just create
  3200. # ghost spools with empty tags on every confirm.
  3201. except Exception as e:
  3202. logger.warning("RFID spool auto-assign failed: %s", e, exc_info=True)
  3203. if rfid_assigned:
  3204. from backend.app.services.spool_relocation import INTERNAL, release_other_slots
  3205. for spool_id, slot in rfid_assigned:
  3206. try:
  3207. await release_other_slots(INTERNAL, spool_id, slot)
  3208. except Exception as e:
  3209. logger.warning("Releasing spool %s from its other slots failed: %s", spool_id, e, exc_info=True)
  3210. try:
  3211. async with async_session() as db:
  3212. from backend.app.api.routes.settings import get_setting
  3213. from backend.app.models.printer import Printer
  3214. # Check if Spoolman is enabled
  3215. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  3216. if not spoolman_enabled or spoolman_enabled.lower() != "true":
  3217. return
  3218. # Check sync mode
  3219. sync_mode = await get_setting(db, "spoolman_sync_mode")
  3220. if sync_mode and sync_mode != "auto":
  3221. return # Only sync on auto mode
  3222. _auto_add_raw_sm = await get_setting(db, "auto_add_unknown_rfid")
  3223. auto_add_unknown_rfid = _auto_add_raw_sm is None or _auto_add_raw_sm.lower() == "true"
  3224. # `spoolman_disable_weight_sync` is deprecated (#1119) — weight is now
  3225. # always owned by per-print tracking, never by AMS auto-sync. The
  3226. # setting is still read by the settings UI for backwards compat but
  3227. # has no effect on the sync path here.
  3228. # Get Spoolman URL
  3229. spoolman_url = await get_setting(db, "spoolman_url")
  3230. if not spoolman_url:
  3231. return
  3232. # Get or create Spoolman client
  3233. client = await get_spoolman_client()
  3234. if not client:
  3235. try:
  3236. client = await init_spoolman_client(spoolman_url)
  3237. except ValueError as exc:
  3238. logger.warning("Spoolman URL %r rejected by SSRF guard: %s", spoolman_url, exc)
  3239. return
  3240. # Check if Spoolman is reachable
  3241. if not await client.health_check():
  3242. logger.warning("Spoolman not reachable at %s", spoolman_url)
  3243. return
  3244. # Get printer name for location
  3245. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3246. printer = result.scalar_one_or_none()
  3247. printer_name = printer.name if printer else f"Printer {printer_id}"
  3248. # OPTIMIZATION: Fetch all spools once before processing trays
  3249. # This eliminates redundant API calls (one per tray) when syncing multiple trays
  3250. logger.debug("[Printer %s] Fetching spools cache for AMS sync...", printer_id)
  3251. try:
  3252. cached_spools = await client.get_spools()
  3253. logger.debug("[Printer %s] Cached %d spools for batch sync", printer_id, len(cached_spools))
  3254. except Exception as e:
  3255. logger.error(
  3256. "[Printer %s] Failed to fetch spools cache after retries, aborting AMS sync: %s",
  3257. printer_id,
  3258. e,
  3259. )
  3260. return
  3261. # Load inventory weights as fallback (when AMS MQTT data lacks remain values)
  3262. from sqlalchemy.orm import selectinload
  3263. from backend.app.models.spool_assignment import SpoolAssignment
  3264. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  3265. from backend.app.services.ams_slot_presence import spool_present
  3266. from backend.app.services.inventory_mode import spoolman_owns_assignments
  3267. # Built-in remaining weight, used by sync_ams_tray only when the
  3268. # firmware reports an unusable remain%/tray_weight for a slot.
  3269. #
  3270. # Left empty since #2812. This block runs in Spoolman mode only,
  3271. # and until then the built-in table was emptied on the switch, so
  3272. # there was never anything here to read and the fallback was inert.
  3273. # Preserving those rows makes it live again, and it is keyed by slot
  3274. # rather than by spool: after a mode switch the tray may well hold
  3275. # different filament, and ``create_spool`` writes ``remaining_weight``
  3276. # unconditionally, so a stale figure would be seeded into a brand new
  3277. # Spoolman spool. Deliberately kept inert rather than deleted, so the
  3278. # intent survives for whoever revisits the cross-mode fallback.
  3279. inventory_weights: dict[tuple[int, int], float] = {}
  3280. if not await spoolman_owns_assignments(db):
  3281. try:
  3282. assign_result = await db.execute(
  3283. select(SpoolAssignment)
  3284. .options(selectinload(SpoolAssignment.spool))
  3285. .where(SpoolAssignment.printer_id == printer_id)
  3286. )
  3287. for assignment in assign_result.scalars().all():
  3288. spool = assignment.spool
  3289. if spool and spool.label_weight > 0:
  3290. remaining = max(0.0, spool.label_weight - (spool.weight_used or 0))
  3291. inventory_weights[(assignment.ams_id, assignment.tray_id)] = remaining
  3292. except Exception as e:
  3293. logger.warning("Could not load inventory weights for printer %s: %s", printer_id, e)
  3294. # Load existing Spoolman slot assignments for the no-RFID fallback path
  3295. spoolman_slot_map: dict[tuple[int, int], int] = {}
  3296. slot_map_read = False
  3297. try:
  3298. slot_result = await db.execute(
  3299. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  3300. )
  3301. for slot in slot_result.scalars().all():
  3302. spoolman_slot_map[(slot.ams_id, slot.tray_id)] = slot.spoolman_spool_id
  3303. slot_map_read = True
  3304. except Exception as e:
  3305. logger.warning("Could not load Spoolman slot assignments for printer %s: %s", printer_id, e)
  3306. # Sync each AMS tray and collect slot changes for DB persistence
  3307. synced = 0
  3308. slot_changes: list[tuple[int, int, int]] = [] # (ams_id, tray_id, spoolman_spool_id) to upsert
  3309. empty_slots: list[tuple[int, int]] = [] # (ams_id, tray_id) whose tray is now empty
  3310. spoolman_held: set[tuple] = set() # removals held for the grace period (#3186)
  3311. for ams_unit in ams_data:
  3312. if not isinstance(ams_unit, dict):
  3313. continue
  3314. ams_id = int(ams_unit.get("id", 0))
  3315. trays = ams_unit.get("tray", [])
  3316. for tray_data in trays:
  3317. if not isinstance(tray_data, dict):
  3318. continue
  3319. tray_id_raw = int(tray_data.get("id", 0))
  3320. tray = client.parse_ams_tray(ams_id, tray_data)
  3321. if not tray:
  3322. # Empty tray slot — record for local assignment cleanup
  3323. # and drop any cached unknown-tag broadcast so a
  3324. # reinserted spool re-prompts.
  3325. #
  3326. # Not during a running print: a slot that empties there
  3327. # is a filament runout, and the spool is still in the
  3328. # AMS. `spoolman_slot_assignments` is how a tag-less
  3329. # spool assigned through the Bambuddy UI is resolved at
  3330. # completion (#1459), so deleting the row mid-print
  3331. # loses the runout segment's usage — the same failure
  3332. # the internal inventory's auto-unlink had.
  3333. #
  3334. # Nor when firmware's presence bit says the slot is
  3335. # occupied. parse_ams_tray calls a tray with no type or
  3336. # no colour empty, and a spool the AMS cannot read has
  3337. # neither until something configures it — so a tag-less
  3338. # spool assigned through the UI had its row deleted by
  3339. # the first idle push after it was inserted. Same
  3340. # deletion as the internal inventory's in #3100, same
  3341. # answer, so the two modes stay in step.
  3342. #
  3343. # And only once the slot has stayed empty for the grace
  3344. # period -- the internal inventory's #3186 answer, for
  3345. # the same one-push blank.
  3346. linked_spool = spoolman_slot_map.get((ams_id, tray_id_raw))
  3347. hold_key = ("spoolman", ams_id, tray_id_raw, linked_spool)
  3348. if not printing_now and (
  3349. spool_present(tray_data) is not True or slot_unlink_grace.is_held(printer_id, hold_key)
  3350. ):
  3351. if linked_spool is None or slot_unlink_grace.removal_confirmed(printer_id, hold_key):
  3352. empty_slots.append((ams_id, tray_id_raw))
  3353. else:
  3354. spoolman_held.add(hold_key)
  3355. logger.info(
  3356. "Spoolman slot unlink held: AMS%d-T%d (spool %d) reports empty; "
  3357. "unlinking if it is still empty in %ds",
  3358. ams_id,
  3359. tray_id_raw,
  3360. linked_spool,
  3361. int(slot_unlink_grace.GRACE_SECONDS),
  3362. )
  3363. _clear_unknown_tag_dedup(printer_id, ams_id, tray_id_raw)
  3364. continue
  3365. spool_tag = (
  3366. tray.tray_uuid
  3367. if tray.tray_uuid and tray.tray_uuid != "00000000000000000000000000000000"
  3368. else tray.tag_uid
  3369. )
  3370. # Provide the hint only when no RFID is available
  3371. hint = spoolman_slot_map.get((ams_id, tray.tray_id)) if not spool_tag else None
  3372. try:
  3373. inv_remaining = inventory_weights.get((ams_id, tray.tray_id))
  3374. result = await client.sync_ams_tray(
  3375. tray,
  3376. printer_name,
  3377. db,
  3378. # Per-print tracking is the only weight writer (#1119).
  3379. # AMS auto-sync still maintains spool metadata / slot
  3380. # assignments but no longer touches remaining_weight.
  3381. disable_weight_sync=True,
  3382. cached_spools=cached_spools,
  3383. inventory_remaining=inv_remaining,
  3384. spoolman_spool_id_hint=hint,
  3385. auto_add_unknown_rfid=auto_add_unknown_rfid,
  3386. )
  3387. if result is None and spool_tag and not auto_add_unknown_rfid:
  3388. # Spoolman skipped auto-create per user setting — surface
  3389. # the slot so the UI can offer "+ Add to inventory".
  3390. await _broadcast_unknown_tag(
  3391. printer_id=printer_id,
  3392. ams_id=ams_id,
  3393. tray_id=tray.tray_id,
  3394. tag_uid=tray.tag_uid or "",
  3395. tray_uuid=tray.tray_uuid or "",
  3396. tray_type=tray.tray_type,
  3397. tray_color=tray.tray_color,
  3398. tray_sub_brands=tray.tray_sub_brands,
  3399. tray_count=len(trays),
  3400. )
  3401. elif result:
  3402. _clear_unknown_tag_dedup(printer_id, ams_id, tray.tray_id)
  3403. if result:
  3404. synced += 1
  3405. if result.get("id"):
  3406. slot_changes.append((ams_id, tray.tray_id, result["id"]))
  3407. # If a new spool was created, add it to the cache
  3408. # so subsequent trays can find it if they reference the same tag
  3409. spool_exists = any(s.get("id") == result["id"] for s in cached_spools)
  3410. if not spool_exists:
  3411. cached_spools.append(result)
  3412. logger.debug(
  3413. "[Printer %s] Added newly created spool %s to cache",
  3414. printer_id,
  3415. result["id"],
  3416. )
  3417. # Reconcile slot_preset_mappings (the same row internal
  3418. # mode keeps in sync via inventory + spool_tag_matcher).
  3419. # Without this the slot card surfaces the previous spool's
  3420. # preset name — same bug shape, different inventory mode.
  3421. from backend.app.services.slot_preset_writer import (
  3422. upsert_slot_preset_for_spoolman_spool,
  3423. )
  3424. await upsert_slot_preset_for_spoolman_spool(
  3425. db=db,
  3426. spoolman_spool=result,
  3427. tray_info_idx=tray.tray_info_idx or "",
  3428. tray_sub_brands=tray.tray_sub_brands or "",
  3429. tray_type=tray.tray_type or "",
  3430. printer_id=printer_id,
  3431. ams_id=ams_id,
  3432. tray_id=tray.tray_id,
  3433. )
  3434. except Exception as e:
  3435. logger.error("Error syncing AMS %s tray %s: %s", ams_id, tray.tray_id, e)
  3436. slot_unlink_grace.settle(printer_id, "spoolman", spoolman_held)
  3437. if synced > 0:
  3438. logger.info("Auto-synced %s AMS trays to Spoolman for printer %s", synced, printer_id)
  3439. # Persist slot assignment changes to the local table: only slots
  3440. # whose spool this pass changed, and only while they still hold what
  3441. # it read before awaiting Spoolman, so a spool assigned by hand in
  3442. # between stays (SSA-3). The changed slots are announced: the slot
  3443. # card reads slot_preset_mappings (rewritten above) ahead of the live
  3444. # tray, so with no event it kept the previous spool's preset name.
  3445. if slot_changes or empty_slots:
  3446. from backend.app.services.spoolman_slot_sync import write_slot_results
  3447. try:
  3448. await write_slot_results(
  3449. db,
  3450. {(printer_id, a_id, t_id): spool_id for (a_id, t_id), spool_id in spoolman_slot_map.items()}
  3451. if slot_map_read
  3452. else None,
  3453. [(printer_id, a_id, t_id, spool_id) for a_id, t_id, spool_id in slot_changes],
  3454. [(printer_id, a_id, t_id) for a_id, t_id in empty_slots],
  3455. )
  3456. except Exception as e:
  3457. await db.rollback()
  3458. logger.error("Error persisting Spoolman slot assignments for printer %s: %s", printer_id, e)
  3459. except Exception as e:
  3460. logging.getLogger(__name__).error("Spoolman AMS sync failed for printer %s: %s", printer_id, e)
  3461. # Largest finish photo attached to a notification; a bigger one is still linked.
  3462. _FINISH_PHOTO_ATTACH_MAX_BYTES = 2_500_000
  3463. async def _finish_photo_for_notification(
  3464. db, archive, archive_id: int, filename: str
  3465. ) -> tuple[str | None, bytes | None]:
  3466. """The ``{finish_photo_url}`` link and the bytes to attach, for a print's finish photo.
  3467. With authentication off the link is the archive's own photo route, as it
  3468. always was: it opens without a login and doesn't expire. With
  3469. authentication on that route needs a media token, which nothing tapping a
  3470. link in Telegram, CallMeBot or a Home Assistant notification has, so the
  3471. link only ever answered 401. The photo is then saved as a notification
  3472. photo as well (utils/notification_photos.py) and the link points there: an
  3473. unguessable name that opens this one photo and nothing else, for 3 days.
  3474. The link is relative when no External URL is set. Bytes over
  3475. ``_FINISH_PHOTO_ATTACH_MAX_BYTES`` are linked but not attached. Returns
  3476. ``(None, None)`` when the photo can't be found.
  3477. """
  3478. from backend.app.api.routes.settings import get_setting
  3479. from backend.app.core.auth import is_auth_enabled
  3480. from backend.app.utils.archive_paths import find_archive_photo
  3481. from backend.app.utils.notification_photos import save_notification_photo
  3482. log = logging.getLogger(__name__)
  3483. base = ((await get_setting(db, "external_url")) or "").strip().rstrip("/")
  3484. url: str | None = f"{base}/api/v1/archives/{archive_id}/photos/{filename}"
  3485. photo_bytes: bytes | None = None
  3486. try:
  3487. photo_path = find_archive_photo(archive, filename)
  3488. if photo_path is not None:
  3489. photo_bytes = await asyncio.to_thread(photo_path.read_bytes)
  3490. except Exception as e:
  3491. log.warning("[NOTIFY-BG] Failed to read finish photo bytes: %s", e)
  3492. try:
  3493. auth_on = await is_auth_enabled(db)
  3494. except Exception:
  3495. auth_on = True # Unknown: the archive link may need a login, so don't rely on it.
  3496. if auth_on:
  3497. url = None
  3498. if photo_bytes:
  3499. try:
  3500. name = await asyncio.to_thread(save_notification_photo, photo_bytes, "print_complete")
  3501. url = f"{base}/api/v1/notifications/photos/{name}"
  3502. except Exception as e:
  3503. log.warning("[NOTIFY-BG] Failed to save finish photo for its link: %s", e)
  3504. if photo_bytes is not None and len(photo_bytes) > _FINISH_PHOTO_ATTACH_MAX_BYTES:
  3505. log.warning("[NOTIFY-BG] Finish photo too large for attachment: %s bytes", len(photo_bytes))
  3506. return url, None
  3507. if photo_bytes:
  3508. log.info("[NOTIFY-BG] Loaded finish photo bytes: %s bytes", len(photo_bytes))
  3509. return url, photo_bytes
  3510. async def _capture_snapshot_for_notification(printer_id: int, printer, logger, *, light: bool = True) -> bytes | None:
  3511. """Capture a camera snapshot for notification image attachment.
  3512. Returns JPEG bytes (max 2.5MB) or None if capture fails or is unavailable.
  3513. Uses: external camera > buffered frame > fresh capture.
  3514. Turns the chamber light on for the picture when the printer asks for it
  3515. (#1655). ``light=False`` is for callers that capture all through a print,
  3516. where the light would flash with every frame.
  3517. """
  3518. if not printer:
  3519. return None
  3520. async with camera_light(printer if light else None):
  3521. return await _capture_snapshot_frame(printer_id, printer, logger)
  3522. async def _capture_snapshot_frame(printer_id: int, printer, logger) -> bytes | None:
  3523. """The capture behind ``_capture_snapshot_for_notification``."""
  3524. try:
  3525. from backend.app.api.routes.settings import get_setting
  3526. async with async_session() as db:
  3527. capture_enabled = await get_setting(db, "capture_finish_photo")
  3528. if capture_enabled is not None and capture_enabled.lower() != "true":
  3529. return None
  3530. # Try external camera first
  3531. if printer.external_camera_enabled and printer.external_camera_url:
  3532. logger.info("[SNAPSHOT] Capturing from external camera for printer %s", printer_id)
  3533. from backend.app.api.routes.camera import live_frame_for_capture
  3534. from backend.app.services.external_camera import capture_frame
  3535. # An external camera allows one reader, so capturing while a viewer
  3536. # is attached fails (#2707). A None here falls through to the paths
  3537. # below exactly as a failed capture did.
  3538. defer, buffered = live_frame_for_capture(printer_id)
  3539. if defer:
  3540. frame_data = buffered
  3541. else:
  3542. frame_data = await capture_frame(
  3543. printer.external_camera_url,
  3544. printer.external_camera_type or "mjpeg",
  3545. snapshot_url=printer.external_camera_snapshot_url,
  3546. )
  3547. if frame_data and len(frame_data) <= 2_500_000:
  3548. logger.info("[SNAPSHOT] External camera frame: %s bytes", len(frame_data))
  3549. return _apply_camera_rotation(frame_data, printer, logger)
  3550. # Try buffered frame from active stream
  3551. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  3552. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  3553. active_chamber = [k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")]
  3554. buffered_frame = get_buffered_frame(printer_id)
  3555. if (active_for_printer or active_chamber) and buffered_frame:
  3556. logger.info("[SNAPSHOT] Using buffered frame for printer %s: %s bytes", printer_id, len(buffered_frame))
  3557. if len(buffered_frame) <= 2_500_000:
  3558. return _apply_camera_rotation(buffered_frame, printer, logger)
  3559. # Fresh capture from printer camera
  3560. logger.info("[SNAPSHOT] Capturing fresh frame for printer %s", printer_id)
  3561. from backend.app.services.camera import capture_camera_frame_bytes
  3562. frame_data = await capture_camera_frame_bytes(
  3563. printer.ip_address, printer.access_code, printer.model, timeout=15
  3564. )
  3565. if frame_data and len(frame_data) <= 2_500_000:
  3566. logger.info("[SNAPSHOT] Fresh camera frame: %s bytes", len(frame_data))
  3567. return _apply_camera_rotation(frame_data, printer, logger)
  3568. except Exception as e:
  3569. logger.warning("[SNAPSHOT] Failed to capture snapshot for printer %s: %s", printer_id, e)
  3570. return None
  3571. async def _maybe_bank_inprint_frame(printer_id: int, layer_num: int) -> None:
  3572. """#1867: bank a recent in-print camera frame for the finish photo.
  3573. Called on every layer change and (#2547) on every print-progress advance.
  3574. Grabs one frame (throttled) into ``_inprint_frame_bank`` so the finish-photo
  3575. path has a pre-End-G-code image for prints that end with a plate swap.
  3576. Both drivers are print telemetry that stops the instant printing ends: no
  3577. further layers, and progress freezes before the End G-code (e.g. SwapMod
  3578. plate swap) executes. So the last banked frame is always the finished print,
  3579. never the swapped plate — that property is what the #1867 path relies on and
  3580. it must survive any change to the throttle below.
  3581. Layer changes alone were not enough: they stop when the *final* layer
  3582. begins, which on a three-minute last layer left the bank stale by the whole
  3583. length of that layer (#2547). Progress keeps ticking through it.
  3584. Best-effort: any failure just leaves the previous banked frame.
  3585. """
  3586. logger = logging.getLogger(__name__)
  3587. client = printer_manager.get_client(printer_id)
  3588. state = client.state if client else None
  3589. if not state or state.state != "RUNNING":
  3590. return
  3591. # Only during actual extrusion — firmware ticks layer_num during the
  3592. # pre-print calibration sequence, whose sub-stages are non-zero.
  3593. if state.mc_print_sub_stage not in (None, 0):
  3594. return
  3595. # #2547: throttled uniformly, with no last-layer exemption. The old code
  3596. # bypassed the throttle on the final layer to guarantee a fresh frame there;
  3597. # now that progress advances also drive banking, that exemption would fire a
  3598. # camera grab on every percent tick of the last layer. Bambu printers accept
  3599. # one RTSP client at a time, so each grab contends with the live view.
  3600. now = time.monotonic()
  3601. last = _inprint_frame_bank_ts.get(printer_id, 0.0)
  3602. if (now - last) < _INPRINT_BANK_MIN_INTERVAL:
  3603. return
  3604. total = state.total_layers or 0
  3605. try:
  3606. async with async_session() as db:
  3607. from backend.app.models.printer import Printer
  3608. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3609. printer = result.scalar_one_or_none()
  3610. if not printer:
  3611. return
  3612. # Reuses the notification snapshot path, which honours the
  3613. # `capture_finish_photo` setting (returns None when disabled) so we
  3614. # don't bank frames the user never asked for. Without the chamber light
  3615. # (#1655): this runs every 25 seconds for the whole print.
  3616. frame = await _capture_snapshot_for_notification(printer_id, printer, logger, light=False)
  3617. if frame:
  3618. _inprint_frame_bank[printer_id] = frame
  3619. _inprint_frame_bank_ts[printer_id] = now
  3620. logger.debug(
  3621. "[FINISH-PHOTO-BANK] banked in-print frame for printer %s at layer %s/%s (%d bytes)",
  3622. printer_id,
  3623. layer_num,
  3624. total,
  3625. len(frame),
  3626. )
  3627. except Exception as e:
  3628. logger.debug("[FINISH-PHOTO-BANK] bank failed for printer %s: %s", printer_id, e)
  3629. def _apply_camera_rotation(image_data: bytes, printer, logger) -> bytes:
  3630. """Apply camera rotation to snapshot image if configured."""
  3631. from backend.app.services.camera import apply_camera_rotation
  3632. return apply_camera_rotation(image_data, getattr(printer, "camera_rotation", 0), logger)
  3633. async def _send_print_start_notification(
  3634. printer_id: int,
  3635. data: dict,
  3636. archive_data: dict | None = None,
  3637. logger=None,
  3638. ):
  3639. """Helper to send print start notification with optional archive data."""
  3640. if logger is None:
  3641. logger = logging.getLogger(__name__)
  3642. try:
  3643. async with async_session() as db:
  3644. from backend.app.models.printer import Printer
  3645. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3646. printer = result.scalar_one_or_none()
  3647. printer_name = printer.name if printer else f"Printer {printer_id}"
  3648. # Capture camera snapshot for notification image attachment
  3649. image_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  3650. if image_data:
  3651. if archive_data is None:
  3652. archive_data = {}
  3653. archive_data["image_data"] = image_data
  3654. await notification_service.on_print_start(printer_id, printer_name, data, db, archive_data=archive_data)
  3655. # Send user-specific email notification for print start
  3656. if archive_data and archive_data.get("created_by_id"):
  3657. await notification_service.send_user_print_email(
  3658. event_type="user_print_start",
  3659. created_by_id=archive_data["created_by_id"],
  3660. printer_name=printer_name,
  3661. filename=data.get("subtask_name") or data.get("filename", "Unknown"),
  3662. db=db,
  3663. )
  3664. except Exception as e:
  3665. logger.warning("Notification on_print_start failed: %s", e)
  3666. async def _dispatch_user_print_email(
  3667. status: str,
  3668. created_by_id: int | None,
  3669. printer_name: str,
  3670. filename: str,
  3671. db,
  3672. image_data: bytes | None = None,
  3673. finish_photo_url: str | None = None,
  3674. ) -> None:
  3675. """Send a user-specific print-completion email based on print status.
  3676. Maps the normalised print status to the correct event type and delegates
  3677. to :meth:`NotificationService.send_user_print_email`. A single helper
  3678. avoids duplicating the ``if status == "completed" / elif "failed" / elif
  3679. "stopped"`` dispatch block at every call site.
  3680. Does nothing if *created_by_id* is ``None``.
  3681. """
  3682. if created_by_id is None:
  3683. return
  3684. if status == "completed":
  3685. event_type = "user_print_complete"
  3686. elif status == "failed":
  3687. event_type = "user_print_failed"
  3688. elif status in ("stopped", "aborted", "cancelled"):
  3689. event_type = "user_print_stopped"
  3690. else:
  3691. return
  3692. await notification_service.send_user_print_email(
  3693. event_type=event_type,
  3694. created_by_id=created_by_id,
  3695. printer_name=printer_name,
  3696. filename=filename,
  3697. db=db,
  3698. image_data=image_data,
  3699. finish_photo_url=finish_photo_url,
  3700. )
  3701. def _load_objects_from_archive(archive, printer_id: int, logger) -> None:
  3702. """Extract printable objects from an archive's 3MF file and store in printer state."""
  3703. try:
  3704. from backend.app.services.archive import extract_printable_objects_from_archive
  3705. client = printer_manager.get_client(printer_id)
  3706. if not client:
  3707. return
  3708. # Extract with positions for UI overlay, scoped to the plate that
  3709. # is printing — resolve_plate_id is the same resolver /cover uses,
  3710. # so the object list can't disagree with the thumbnail it is drawn
  3711. # over (#2522).
  3712. printable_objects, bbox_all = extract_printable_objects_from_archive(
  3713. app_settings.base_dir / archive.file_path,
  3714. plate_number=resolve_plate_id(client.state),
  3715. )
  3716. if printable_objects:
  3717. client.state.printable_objects = printable_objects
  3718. client.state.printable_objects_bbox_all = bbox_all
  3719. client.state.skipped_objects = []
  3720. logger.info("Loaded %s printable objects for printer %s", len(printable_objects), printer_id)
  3721. except Exception as e:
  3722. logger.debug("Failed to extract printable objects from archive: %s", e)
  3723. def _load_pauses_from_3mf(threemf_path, printer_id: int, *, warned_layers=(), passed_through_layer: int = 0) -> None:
  3724. """Read the running print's slicer pauses from its 3MF, in the background (#2984).
  3725. In a thread: a plate's G-code runs to tens of MB, and the print-start
  3726. handler must not wait for it. Kept only while the printer is still on
  3727. the print they were read for: a new print replaces the list.
  3728. ``warned_layers``: pauses this print was already warned about, kept on its
  3729. archive, so a restart does not send the heads-up again.
  3730. ``passed_through_layer``: for a print resumed after a restart, the layer it
  3731. is on. Its pauses up to there are behind it, though the printer's layer
  3732. reads 0 until it reports the next one.
  3733. """
  3734. client = printer_manager.get_client(printer_id)
  3735. if not client:
  3736. return
  3737. state = client.state
  3738. read_for = state.pause_points
  3739. try:
  3740. plate_id = resolve_plate_id(state)
  3741. except Exception:
  3742. # Never at the cost of the print-start handler this runs inside.
  3743. logging.getLogger(__name__).warning("[Printer %s] Could not tell the plate to read pauses for", printer_id)
  3744. return
  3745. async def _load() -> None:
  3746. from backend.app.utils.threemf_tools import extract_pause_points_from_3mf
  3747. points = await asyncio.to_thread(extract_pause_points_from_3mf, Path(threemf_path), plate_id)
  3748. if state.pause_points is not read_for:
  3749. return
  3750. state.pause_points = [p for p in points if p.layer > passed_through_layer]
  3751. state.pause_warned_layers = set(state.pause_warned_layers) | set(warned_layers)
  3752. if points:
  3753. logging.getLogger(__name__).info(
  3754. "Loaded %s slicer pause(s) for printer %s at layer(s) %s",
  3755. len(points),
  3756. printer_id,
  3757. [p.layer for p in points],
  3758. )
  3759. spawn_background_task(_load(), name=f"load-pauses-{printer_id}")
  3760. def _load_pauses_from_resumed_archive(archive, printer_id: int) -> None:
  3761. """Pauses for a print start taken as the resumption of a running print.
  3762. A restart while the print was paused makes its resume look like a new
  3763. print (#2984): the pauses warned about and passed are kept from before.
  3764. """
  3765. client = printer_manager.get_client(printer_id)
  3766. state = client.state if client else None
  3767. _load_pauses_from_3mf(
  3768. app_settings.base_dir / archive.file_path,
  3769. printer_id,
  3770. warned_layers=pause_eta.warned_layers_of(archive, state) if state is not None else (),
  3771. passed_through_layer=getattr(client, "layer_before_new_print", 0) if client else 0,
  3772. )
  3773. async def _restore_printable_objects(printer_id: int, state, db, logger) -> None:
  3774. """Put the skip-objects list back after a restart mid-print.
  3775. ``PrinterState.printable_objects`` is in-memory only, and the only thing
  3776. that fills it is ``_load_objects_from_archive`` on the print-start paths —
  3777. which the #1304 guard suppresses on the first RUNNING push after startup.
  3778. Everything else this hook restores (the archive, the usage-tracking session,
  3779. the timelapse baseline) was already handled; the object list was not, so a
  3780. restart mid-print took skip-objects away for the rest of that print.
  3781. Nothing recovered it either: the printer card gates its Skip button on the
  3782. object count, and the one endpoint that can rebuild the list is reachable
  3783. only from the modal that button opens.
  3784. The archive is the running print's as ``running_print_archive`` finds it:
  3785. by subtask_id, or for a print that reports none by the archive bound to it
  3786. (#3327), never by name: a leftover ``status="printing"`` row from a
  3787. completion we never saw must not hand this print someone else's objects.
  3788. After a restart that binding exists only for Bambuddy's own prints, so a
  3789. print that reports no subtask_id loads nothing rather than a guess; the
  3790. reload path on ``GET /print/objects`` covers that case on demand.
  3791. """
  3792. client = printer_manager.get_client(printer_id)
  3793. if client is None or (client.state.printable_objects and client.state.pause_points):
  3794. return
  3795. archive = await running_print_archive(db, printer_id, state, printing_only=True, by_name=False)
  3796. if archive is not None and archive.file_path:
  3797. if not client.state.printable_objects:
  3798. _load_objects_from_archive(archive, printer_id, logger)
  3799. # The slicer's pauses are in-memory only as well (#2984).
  3800. if not client.state.pause_points:
  3801. _load_pauses_from_3mf(
  3802. app_settings.base_dir / archive.file_path,
  3803. printer_id,
  3804. warned_layers=pause_eta.warned_layers_of(archive, state),
  3805. )
  3806. # Retry ladder for a fallback archive created while the printer's FTPS cool-off
  3807. # was running (#2957). The cool-off is 300s, so the first attempt is placed just
  3808. # past it; the second covers a handshake that failed again on the way back and
  3809. # armed a fresh one. Module-level so tests can shrink them.
  3810. _FALLBACK_3MF_RETRY_DELAYS_SECONDS: tuple[float, ...] = (310.0, 620.0)
  3811. # Retry ladder for the other temporary give-up: the file service answered and
  3812. # the transfer still did not finish, which at print start is usually the printer
  3813. # serving MQTT, the camera and a job upload at the same time (#3063). Nothing has
  3814. # to expire here, so the first attempt comes early -- #3063's reporter had the
  3815. # same 19MB file complete 48 seconds after the download budget ran out. The later
  3816. # two cover a printer that stays busy well into the print.
  3817. _FALLBACK_3MF_TRANSFER_RETRY_DELAYS_SECONDS: tuple[float, ...] = (60.0, 240.0, 600.0)
  3818. # printer_id -> the in-flight retry task, so print completion can cancel it.
  3819. _fallback_3mf_retry_tasks: dict[int, asyncio.Task] = {}
  3820. # printer_id -> lock serialising recovery attempts for that printer. Three callers
  3821. # can reach one archive at once: the cover endpoint (whose single-flight coalesces
  3822. # by view, so two views race), the cool-off retry task, and print completion.
  3823. # Without this they each read file_path == "" and each run a full copy, so the row
  3824. # ends up pointing at one timestamped directory while the others sit orphaned.
  3825. #
  3826. # Keyed by printer rather than archive because a printer runs one print at a time,
  3827. # which makes the two equally strong here — and it bounds the dict by printer
  3828. # count instead of needing a cleanup pass. Popping a per-archive entry cannot be
  3829. # done safely: `Lock.locked()` reads False between release and the queued waiter
  3830. # resuming, so "no waiters" is not a question this API can answer.
  3831. _fallback_recovery_locks: dict[int, asyncio.Lock] = {}
  3832. # How many of a printer's newest archives a touchscreen reprint is compared
  3833. # against (#3307). A repeat is usually of a recent job; the bound keeps a
  3834. # print start from reading a long history's extra_data.
  3835. _REPEAT_LOOKBACK = 200
  3836. async def _archived_3mf_for_repeat(
  3837. db, printer_id: int, subtask_name: str | None, gcode_file: str | None, total_layers: int | None
  3838. ):
  3839. """The newest archive of the job this printer is running again, with its 3MF.
  3840. A "Print again" on the printer's screen never names the file on the card:
  3841. it reports its own history copy under a generic name
  3842. (``file:///userdata/project_file.gcode.3mf``), the 3MF's internal title as
  3843. ``subtask_name`` and the plate as ``gcode_file`` -- and the title is often
  3844. not the card file's name, so every lookup built from it misses (#3307).
  3845. An earlier run that was archived recorded that same title and plate in its
  3846. ``_print_data``, and holds the 3MF.
  3847. Same printer, same title, same plate, a 3MF still on disk. A layer count
  3848. known on both sides must agree: the same title and plate re-sliced is
  3849. another job. None when nothing matches.
  3850. """
  3851. from backend.app.models.archive import PrintArchive
  3852. if not subtask_name or not gcode_file:
  3853. return None
  3854. rows = (
  3855. (
  3856. await db.execute(
  3857. select(PrintArchive)
  3858. .where(
  3859. PrintArchive.printer_id == printer_id,
  3860. PrintArchive.deleted_at.is_(None),
  3861. PrintArchive.file_path.isnot(None),
  3862. PrintArchive.file_path != "",
  3863. )
  3864. .order_by(PrintArchive.created_at.desc(), PrintArchive.id.desc())
  3865. .limit(_REPEAT_LOOKBACK)
  3866. )
  3867. )
  3868. .scalars()
  3869. .all()
  3870. )
  3871. for archive in rows:
  3872. extra = archive.extra_data if isinstance(archive.extra_data, dict) else {}
  3873. run = extra.get("_print_data")
  3874. if not isinstance(run, dict):
  3875. continue
  3876. if run.get("subtask_name") != subtask_name or run.get("filename") != gcode_file:
  3877. continue
  3878. if total_layers and archive.total_layers and archive.total_layers != total_layers:
  3879. continue
  3880. if not (app_settings.base_dir / archive.file_path).is_file():
  3881. continue
  3882. return archive
  3883. return None
  3884. async def _recover_fallback_archive(archive_id: int, source_3mf: Path, printer_id: int, *, price: bool = True) -> bool:
  3885. """Fill in a no-3MF archive from a 3MF that turned up later.
  3886. Returns True when the row was upgraded. Safe to call speculatively: it
  3887. verifies the archive still exists, is still a fallback, and that the file
  3888. is a readable 3MF before touching anything.
  3889. Serialised per printer — see ``_fallback_recovery_locks``.
  3890. """
  3891. lock = _fallback_recovery_locks.setdefault(printer_id, asyncio.Lock())
  3892. async with lock:
  3893. return await _recover_fallback_archive_locked(archive_id, source_3mf, printer_id, price=price)
  3894. async def _recover_fallback_archive_locked(
  3895. archive_id: int, source_3mf: Path, printer_id: int, *, price: bool = True
  3896. ) -> bool:
  3897. """The body of :func:`_recover_fallback_archive`, under its per-printer lock.
  3898. ``price`` is False when the print's end is being handled: the archive
  3899. still reads "printing" then, and its end prices it.
  3900. """
  3901. import zipfile
  3902. from backend.app.models.archive import PrintArchive
  3903. from backend.app.services.archive import ArchiveService
  3904. logger = logging.getLogger(__name__)
  3905. if not source_3mf.exists() or source_3mf.stat().st_size == 0:
  3906. return False
  3907. if not await asyncio.to_thread(zipfile.is_zipfile, source_3mf):
  3908. # A truncated or half-written download is worse than no download: it
  3909. # would replace an honest empty archive with wrong metadata.
  3910. logger.warning("[RECOVER] %s is not a readable 3MF; leaving archive %s as-is", source_3mf, archive_id)
  3911. return False
  3912. async with async_session() as db:
  3913. archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  3914. if archive is None or archive.deleted_at is not None:
  3915. return False
  3916. if archive.file_path:
  3917. # Already recovered, or never was a fallback. Either way there is a
  3918. # real 3MF attached and overwriting it is not this function's job.
  3919. return False
  3920. print_data = (archive.extra_data or {}).get("_print_data") or {}
  3921. service = ArchiveService(db)
  3922. recovered = await service.archive_print(
  3923. printer_id=printer_id,
  3924. source_file=source_3mf,
  3925. print_data={**print_data, "status": archive.status or "printing"},
  3926. subtask_id=archive.subtask_id,
  3927. update_archive_id=archive.id,
  3928. )
  3929. if recovered is None:
  3930. return False
  3931. logger.info(
  3932. "[RECOVER] Archive %s filled in from %s (%s bytes) — it started as a no-3MF fallback",
  3933. archive_id,
  3934. source_3mf,
  3935. recovered.file_size,
  3936. )
  3937. # Spoolman's tracking stored no 3MF usage at print start and would
  3938. # charge the AMS remain% change; the built-in tracker reads the archive
  3939. # at completion and needs nothing. Also at the print's end (``price``
  3940. # False): that recovery runs inside the close-out, before the charge.
  3941. try:
  3942. from backend.app.services.spoolman_tracking import attach_recovered_3mf
  3943. from backend.app.services.usage_tracker import is_completing
  3944. await attach_recovered_3mf(
  3945. printer_id,
  3946. recovered.id,
  3947. recovered.file_path,
  3948. db,
  3949. printer_manager,
  3950. plate_id=_get_start_plate_id(printer_id, recovered.id) or recovered.plate_id,
  3951. may_start=not price or (recovered.status == "printing" and not is_completing(printer_id)),
  3952. )
  3953. except Exception:
  3954. logger.warning(
  3955. "[RECOVER] Archive %s: could not add the 3MF to Spoolman's usage tracking", archive_id, exc_info=True
  3956. )
  3957. if price and recovered.status == "printing" and recovered.file_path:
  3958. # Print start had no 3MF to read the slicer's pauses from (#2984).
  3959. _load_pauses_from_3mf(app_settings.base_dir / recovered.file_path, printer_id)
  3960. if price and recovered.status == "printing":
  3961. # Print start priced it from the default rate, having no 3MF to
  3962. # estimate from; now it has one. An ended print is left alone: its
  3963. # end prices it. A print Bambuddy didn't dispatch has no start
  3964. # plate; the one the archive just read from the 3MF stands in.
  3965. _schedule_archive_cost_estimate(
  3966. printer_id,
  3967. recovered.id,
  3968. printer_manager,
  3969. ams_mapping=_get_start_ams_mapping(print_data, printer_id, recovered.id),
  3970. plate_id=_get_start_plate_id(printer_id, recovered.id) or recovered.plate_id,
  3971. session_factory=async_session,
  3972. )
  3973. # `archive_updated`, not `archive_created` — the row was already on the
  3974. # Archives page as an empty card and is now filled in, not new.
  3975. await ws_manager.send_archive_updated(
  3976. {
  3977. "id": recovered.id,
  3978. "printer_id": recovered.printer_id,
  3979. "filename": recovered.filename,
  3980. "print_name": recovered.print_name,
  3981. "status": recovered.status,
  3982. }
  3983. )
  3984. return True
  3985. async def try_recover_fallback_archive(printer_id: int, name: str, path: Path, *, price: bool = True) -> bool:
  3986. """Offer a freshly-downloaded 3MF to this printer's running fallback archive.
  3987. Called from the paths that pull a 3MF for a print that is already under way
  3988. — chiefly the cover endpoint, which downloads the very file the archive flow
  3989. could not get and, before #2957, used it for a thumbnail and nothing else.
  3990. The bytes are already local, so this costs a parse and a row update.
  3991. No-op when the running print has a real archive, which is the common case.
  3992. """
  3993. from backend.app.models.archive import PrintArchive
  3994. logger = logging.getLogger(__name__)
  3995. # `_active_prints` is keyed on the raw names seen at print start — the
  3996. # dispatch filename, the subtask name, and the subtask name plus ".3mf".
  3997. # Callers here arrive with whichever variant their own path produced, so
  3998. # match on the same normalization the download cache uses rather than on an
  3999. # exact string; that is what makes "Desktop_Goose.gcode.3mf" from the cover
  4000. # endpoint find an archive registered under "Desktop_Goose".
  4001. wanted = normalize_3mf_name(name)
  4002. archive_id = None
  4003. for (key_printer_id, key_name), value in list(_active_prints.items()):
  4004. if key_printer_id == printer_id and normalize_3mf_name(key_name) == wanted:
  4005. archive_id = value
  4006. break
  4007. if archive_id is None:
  4008. return False
  4009. async with async_session() as db:
  4010. archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  4011. # Cheap pre-check so the common case (a normal archive) does no work.
  4012. if archive is None or archive.file_path or archive.deleted_at is not None:
  4013. return False
  4014. try:
  4015. return await _recover_fallback_archive(archive_id, path, printer_id, price=price)
  4016. except Exception as e:
  4017. # Recovery is opportunistic. A failure here must never take down the
  4018. # caller, which is usually just trying to render a thumbnail.
  4019. logger.warning("[RECOVER] Could not fill in archive %s from %s: %s", archive_id, path, e)
  4020. return False
  4021. def _schedule_fallback_3mf_retry(
  4022. printer_id: int,
  4023. archive_id: int,
  4024. filenames: list[str],
  4025. delays: tuple[float, ...] | None = None,
  4026. reason: str = REASON_FTPS_COOLOFF,
  4027. ) -> None:
  4028. """Re-attempt the 3MF download after a temporary give-up.
  4029. ``reason`` says which give-up this is, and picks the default ladder: an
  4030. FTPS cool-off has to be waited out, while a transfer that timed out under
  4031. contention is worth asking about again straight away (#3063). It is only
  4032. read for the ladder and the log line -- the retry itself is identical, since
  4033. in both cases the file is on the printer and the last attempt at it failed
  4034. for a reason that does not last.
  4035. """
  4036. logger = logging.getLogger(__name__)
  4037. if delays is None:
  4038. delays = (
  4039. _FALLBACK_3MF_TRANSFER_RETRY_DELAYS_SECONDS
  4040. if reason == REASON_FTP_TRANSFER_FAILED
  4041. else _FALLBACK_3MF_RETRY_DELAYS_SECONDS
  4042. )
  4043. async def _retry() -> None:
  4044. from backend.app.models.archive import PrintArchive
  4045. from backend.app.models.printer import Printer
  4046. for delay in delays:
  4047. await asyncio.sleep(delay)
  4048. async with async_session() as db:
  4049. archive = (
  4050. await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  4051. ).scalar_one_or_none()
  4052. if archive is None or archive.deleted_at is not None or archive.file_path:
  4053. return
  4054. printer = (await db.execute(select(Printer).where(Printer.id == printer_id))).scalar_one_or_none()
  4055. if printer is None:
  4056. return
  4057. # Read the fields while the session is open rather than touching
  4058. # a detached instance minutes later, mid-download.
  4059. printer_ip = printer.ip_address
  4060. printer_code = printer.access_code
  4061. printer_model = printer.model
  4062. # Someone else may have fetched it in the meantime — the cover
  4063. # endpoint routinely does, and its copy is the same bytes.
  4064. for name in filenames:
  4065. cached = get_cached_3mf(printer_id, name)
  4066. if cached and await _recover_fallback_archive(archive_id, cached, printer_id):
  4067. return
  4068. if ftps_handshake_blocked(printer_ip):
  4069. logger.info(
  4070. "[RECOVER] Printer %s is still in its FTPS cool-off; archive %s retry deferred",
  4071. printer_id,
  4072. archive_id,
  4073. )
  4074. continue
  4075. _, _, _, ftp_timeout = await get_ftp_retry_settings()
  4076. for candidate in filenames:
  4077. # Bare name only. These come from the print-start flow, which
  4078. # already strips the path, but the local temp write must not
  4079. # depend on that holding for every future caller — a name that
  4080. # is absolute or contains ".." would otherwise escape the data
  4081. # volume via the `/` operator.
  4082. name = Path(candidate).name
  4083. if not name or name in (".", ".."):
  4084. continue
  4085. if not name.endswith(".3mf"):
  4086. name = f"{name}.3mf"
  4087. temp_path = app_settings.archive_dir / "temp" / name
  4088. temp_path.parent.mkdir(parents=True, exist_ok=True)
  4089. try:
  4090. hit = await download_file_try_paths_async(
  4091. printer_ip,
  4092. printer_code,
  4093. ftp_probe_paths(name),
  4094. temp_path,
  4095. socket_timeout=ftp_timeout,
  4096. printer_model=printer_model,
  4097. max_total_timeout=RECOVERY_DOWNLOAD_MAX_TIMEOUT,
  4098. )
  4099. except Exception as e:
  4100. logger.debug("[RECOVER] Retry download of %s failed: %s", name, e)
  4101. continue
  4102. if not hit:
  4103. continue
  4104. cache_3mf_download(printer_id, name, temp_path)
  4105. if await _recover_fallback_archive(archive_id, temp_path, printer_id):
  4106. return
  4107. logger.info("[RECOVER] Archive %s still has no 3MF after a retry", archive_id)
  4108. async def _guarded() -> None:
  4109. try:
  4110. await _retry()
  4111. except asyncio.CancelledError:
  4112. raise
  4113. except Exception as e:
  4114. logger.warning("[RECOVER] Retry task for archive %s failed: %s", archive_id, e)
  4115. finally:
  4116. if _fallback_3mf_retry_tasks.get(printer_id) is asyncio.current_task():
  4117. _fallback_3mf_retry_tasks.pop(printer_id, None)
  4118. existing = _fallback_3mf_retry_tasks.pop(printer_id, None)
  4119. if existing and not existing.done():
  4120. existing.cancel()
  4121. task = asyncio.create_task(_guarded())
  4122. _fallback_3mf_retry_tasks[printer_id] = task
  4123. logger.info(
  4124. "[RECOVER] Archive %s has no 3MF (%s) and the file should still be on printer %s; will retry in %s",
  4125. archive_id,
  4126. reason,
  4127. printer_id,
  4128. ", ".join(f"{d:g}s" for d in delays),
  4129. )
  4130. async def _ask_outcome_for_external_print(db, printer_id: int, observed_name: str | None = None) -> bool:
  4131. """Whether an archive created here should ask for the print's outcome (#1898).
  4132. Only a print Bambuddy did not dispatch reaches the archive-*creating*
  4133. branches below: a queued job already has its archive and takes the
  4134. expected-print branch, where the queue item's own ``confirm_outcome``
  4135. decides. The queue is still consulted, because a restart mid-print empties
  4136. ``_expected_prints`` — without the check the setting could override a queue
  4137. item that deliberately has the flag off. Any failure answers "don't ask":
  4138. an unwanted prompt is worse than a missing one, and this must never be the
  4139. reason a print goes unarchived.
  4140. A ``printing`` row is not proof on its own, though: Bambuddy deliberately
  4141. leaves one behind when a completion cannot be matched to it
  4142. (``_completion_belongs_to_queue_item``), and the scheduler's stranded sweep
  4143. only takes it back once the printer has sat connected and terminal for
  4144. minutes. Until then every screen-started print on that printer would
  4145. silently lose its prompt, so the row is held against ``observed_name`` by
  4146. the same comparison a completion uses: a positive disagreement means the row
  4147. is about some other run and this print is external after all.
  4148. """
  4149. logger = logging.getLogger(__name__)
  4150. try:
  4151. # A savepoint rather than a rollback of the caller's transaction on
  4152. # failure: a failed statement leaves the transaction unusable, but a
  4153. # full rollback also expires every object the caller has loaded (the
  4154. # printer, the archive it just created), and on an async session the
  4155. # next attribute read then raises instead of reloading. Everything here
  4156. # is a read, so undoing the savepoint loses nothing.
  4157. async with db.begin_nested():
  4158. return await _decide_outcome_for_external_print(db, printer_id, observed_name, logger)
  4159. except Exception as e:
  4160. logger.warning("[CALLBACK] Could not decide the outcome prompt for printer %s: %s", printer_id, e)
  4161. return False
  4162. async def _decide_outcome_for_external_print(db, printer_id: int, observed_name: str | None, logger) -> bool:
  4163. """The reads behind ``_ask_outcome_for_external_print``; see there."""
  4164. from backend.app.api.routes.settings import get_setting, setting_is_true
  4165. if not setting_is_true(await get_setting(db, "confirm_outcome_external_prints")):
  4166. return False
  4167. from backend.app.models.print_queue import PrintQueueItem
  4168. dispatched_here = await db.scalar(
  4169. select(PrintQueueItem)
  4170. .where(
  4171. PrintQueueItem.printer_id == printer_id,
  4172. PrintQueueItem.status == "printing",
  4173. )
  4174. .limit(1)
  4175. )
  4176. if dispatched_here is None:
  4177. return True
  4178. expected = await _queue_item_dispatched_name(db, dispatched_here)
  4179. observed = (observed_name or "").strip()
  4180. if not expected or not observed or _dispatched_name_matches_echo(expected, observed):
  4181. logger.info(
  4182. "[CALLBACK] Not asking for the outcome on printer %s: queue item %s is still printing, so "
  4183. "Bambuddy dispatched this run and the item's own ask-for-outcome flag decides.",
  4184. printer_id,
  4185. dispatched_here.id,
  4186. )
  4187. return False
  4188. logger.info(
  4189. "[CALLBACK] Queue item %s is still marked printing on printer %s but was dispatched as %r, not "
  4190. "%r; treating this as an externally started print.",
  4191. dispatched_here.id,
  4192. printer_id,
  4193. expected,
  4194. observed,
  4195. )
  4196. return True
  4197. async def _dispatch_outcome_confirmation_safely(
  4198. db,
  4199. printer_id: int,
  4200. printer_name: str,
  4201. data: dict,
  4202. archive_id: int,
  4203. archive_data: dict | None = None,
  4204. run_id: int | None = None,
  4205. ) -> None:
  4206. """``dispatch_outcome_confirmation`` for a caller that has more to do on ``db``.
  4207. The completion task sends the per-user print email on the same session
  4208. right after the prompt. A failed statement in the prompt leaves that
  4209. session needing a rollback, and without one the email step fails with
  4210. PendingRollbackError; the prompt is the optional part, so it must not be
  4211. the reason the email never goes out.
  4212. """
  4213. try:
  4214. await dispatch_outcome_confirmation(db, printer_id, printer_name, data, archive_id, archive_data, run_id=run_id)
  4215. except Exception as e:
  4216. logging.getLogger(__name__).error("[NOTIFY-BG] Outcome-confirmation dispatch failed: %s", e, exc_info=True)
  4217. try:
  4218. await db.rollback()
  4219. except Exception:
  4220. pass
  4221. async def _outcome_answerers(db, owner_id: int | None) -> set[int] | None:
  4222. """Who may answer a print's outcome prompt in the app: its owner and anyone
  4223. who may change everyone's archives. None (everyone on the printer) with auth
  4224. off, or for a print nobody owns."""
  4225. from sqlalchemy.orm import selectinload
  4226. from backend.app.core.auth import is_auth_enabled
  4227. from backend.app.core.permissions import Permission
  4228. from backend.app.models.user import User
  4229. if owner_id is None or not await is_auth_enabled(db):
  4230. return None
  4231. users = (await db.execute(select(User).options(selectinload(User.groups)).where(User.is_active))).scalars().all()
  4232. return {owner_id} | {u.id for u in users if u.has_permission(Permission.ARCHIVES_UPDATE_ALL.value)}
  4233. async def dispatch_outcome_confirmation(
  4234. db,
  4235. printer_id: int,
  4236. printer_name: str,
  4237. data: dict,
  4238. archive_id: int,
  4239. archive_data: dict | None = None,
  4240. *,
  4241. run_id: int | None = None,
  4242. ) -> bool:
  4243. """Emit the post-print outcome prompt for a completed print (#1898).
  4244. Gated on the run that ended (``run_id``): ``confirm_requested`` is the
  4245. opt-in (from the queue item, or from ``confirm_outcome_external_prints``
  4246. for a print Bambuddy did not start, copied onto the run at its start) and
  4247. ``user_verdict`` being unset is what makes the question still open. Mints
  4248. the run's capability token the one-tap verdict links carry: plates of one
  4249. archive each have their own question. Without a run, the archive's, as
  4250. before runs.
  4251. Lives here rather than inline in ``on_print_complete``'s notification task
  4252. because that task swallows every exception: extracted, the gate and the two
  4253. emissions can be driven by a test, which is the only thing standing between
  4254. a regression here and a farm that quietly stops asking.
  4255. Returns whether a prompt was sent.
  4256. """
  4257. logger = logging.getLogger(__name__)
  4258. from backend.app.models.archive import PrintArchive
  4259. from backend.app.models.print_log import PrintLogEntry
  4260. confirm_archive = (await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))).scalar_one_or_none()
  4261. if confirm_archive is None:
  4262. return False
  4263. run = await db.get(PrintLogEntry, run_id) if run_id is not None else None
  4264. if run is not None and run.archive_id != archive_id:
  4265. run = None
  4266. # The run carries the opt-in from its start; one logged before it did
  4267. # (None) asks as its archive does.
  4268. asks = confirm_archive if run is None or run.confirm_requested is None else run
  4269. holder = run if run is not None else confirm_archive
  4270. if not (asks.confirm_requested and holder.user_verdict is None):
  4271. return False
  4272. import secrets as _secrets
  4273. # A spent token stays on the row so the one-tap route can recognise it, so
  4274. # "has a token" no longer means "answerable". A prompt going out now needs a
  4275. # live one: mint a new token whenever the stored one is already spent, or
  4276. # every button in the new message would land on the already-answered page.
  4277. if not holder.confirm_token or holder.confirm_token_used_at is not None:
  4278. holder.confirm_token = _secrets.token_urlsafe(32)
  4279. holder.confirm_token_used_at = None
  4280. await db.commit()
  4281. from backend.app.api.routes.settings import get_external_base_url, get_setting
  4282. base = await get_external_base_url(db)
  4283. if not await get_setting(db, "external_url"):
  4284. # Both the Telegram inline keyboard and the ntfy action buttons need an
  4285. # absolute URL, so an unconfigured install used to get a message body
  4286. # with two unusable relative paths and no buttons at all. The shared
  4287. # fallback at least produces tappable links; say which setting makes
  4288. # them resolve from a phone.
  4289. logger.warning(
  4290. "[#1898] No external_url configured — the outcome prompt's Good/Reject links point at %s. "
  4291. "Set Settings → External URL so they resolve away from this host.",
  4292. base,
  4293. )
  4294. token = holder.confirm_token
  4295. good_url = f"{base}/api/v1/archives/confirm/{token}/good"
  4296. reject_url = f"{base}/api/v1/archives/confirm/{token}/reject"
  4297. confirm_url = f"{base}/archives?confirm={archive_id}" + (f"&run={run.id}" if run is not None else "")
  4298. await ws_manager.send_print_confirm_request(
  4299. printer_id,
  4300. {
  4301. "archive_id": archive_id,
  4302. "run_id": run.id if run is not None else None,
  4303. "print_name": confirm_archive.print_name or confirm_archive.filename,
  4304. },
  4305. user_ids=await _outcome_answerers(db, confirm_archive.created_by_id),
  4306. )
  4307. await notification_service.on_print_confirm_request(
  4308. printer_id,
  4309. printer_name,
  4310. data,
  4311. db,
  4312. archive_data=archive_data,
  4313. good_url=good_url,
  4314. reject_url=reject_url,
  4315. confirm_url=confirm_url,
  4316. archive_id=archive_id,
  4317. run_id=run.id if run is not None else None,
  4318. )
  4319. return True
  4320. async def _printer_gone(db, printer_id: int) -> bool:
  4321. """Whether the printer was deleted. A column read: the session's loaded
  4322. Printer would otherwise answer for the database."""
  4323. from backend.app.models.printer import Printer
  4324. return await db.scalar(select(Printer.id).where(Printer.id == printer_id)) is None
  4325. async def _drop_archive_of_deleted_printer(db, printer_id: int, archive_id: int) -> bool:
  4326. """Remove the archive just written if its printer was deleted meanwhile.
  4327. The delete route sweeps the printer's archives before it commits, so a row
  4328. committed after that sweep (the check before writing passed while the
  4329. delete was still uncommitted) is left for a printer that will never report
  4330. the print's end. A row the sweep reached is the route's: deleted with the
  4331. printer's archives, or detached (printer_id NULL) when the user kept them,
  4332. and then left as it is. Either way nothing is registered for it. Run after
  4333. the archive's commit, before anything is registered for it.
  4334. """
  4335. from backend.app.models.archive import PrintArchive
  4336. if not await _printer_gone(db, printer_id):
  4337. return False
  4338. owner = await db.scalar(select(PrintArchive.printer_id).where(PrintArchive.id == archive_id))
  4339. if owner == printer_id:
  4340. logging.getLogger(__name__).info(
  4341. "Printer %s was deleted while archive %s was written; removing it", printer_id, archive_id
  4342. )
  4343. await ArchiveService(db).delete_archive(archive_id)
  4344. await forget_printer(printer_id)
  4345. return True
  4346. async def _refused_for_deleted_printer(db, printer_id: int, what: str) -> bool:
  4347. """After an IntegrityError on a print start's archive write: whether it was
  4348. the printer being deleted, forgotten then (the caller returns).
  4349. PostgreSQL: the delete route locks the printer from its archive sweep to its
  4350. commit, so an archive written or adopted for it meanwhile (one that sets
  4351. printer_id) waits and the printer's foreign key then refuses it. That is
  4352. the printer being deleted, not an error; anything else is re-raised by the
  4353. caller.
  4354. """
  4355. await db.rollback()
  4356. if not await _printer_gone(db, printer_id):
  4357. return False
  4358. logging.getLogger(__name__).info("Printer %s was deleted while %s; refused", printer_id, what)
  4359. await forget_printer(printer_id)
  4360. return True
  4361. async def _forget_if_printer_gone(printer_id: int) -> None:
  4362. """At the end of a print start: the slow steps after the last check (smart
  4363. plug, notifications, the timelapse baseline's FTP listing) can outlast the
  4364. printer's delete and its forget_printer, and then wrote state back.
  4365. (An ActivePrintSpoolman row written then is the init_db orphan sweep's on
  4366. SQLite; PostgreSQL refuses it.)
  4367. Its own session: those steps can leave the handler's needing a rollback (a
  4368. write to the archive the delete removed). Never raises: it runs inside the
  4369. fallback branch's try, whose except reports the archive as failed.
  4370. """
  4371. try:
  4372. async with async_session() as db:
  4373. gone = await _printer_gone(db, printer_id)
  4374. if gone:
  4375. await forget_printer(printer_id)
  4376. except Exception:
  4377. logging.getLogger(__name__).warning(
  4378. "Could not check whether printer %s was deleted during its print start", printer_id, exc_info=True
  4379. )
  4380. async def _announce_plate_check_failed(printer_id: int, printer_name: str, reason: str) -> None:
  4381. """Tell the UI a plate check could not run. It does not block the print,
  4382. but an unchecked plate is not an empty one."""
  4383. try:
  4384. await ws_manager.broadcast(
  4385. {"type": "plate_check_failed", "printer_id": printer_id, "printer_name": printer_name, "reason": reason}
  4386. )
  4387. except Exception as e:
  4388. logging.getLogger(__name__).warning("[PLATE CHECK] Could not announce the failed check: %s", e)
  4389. def _expected_print_keys(printer_id: int, filename: str | None, subtask_name: str | None) -> list[tuple[int, str]]:
  4390. """The ``_expected_prints`` keys a print start may be registered under."""
  4391. keys: list[tuple[int, str]] = []
  4392. if subtask_name:
  4393. keys.append((printer_id, subtask_name))
  4394. keys.append((printer_id, f"{subtask_name}.3mf"))
  4395. keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  4396. if filename:
  4397. fname = filename.split("/")[-1] if "/" in filename else filename
  4398. keys.append((printer_id, fname))
  4399. # Strip extensions to match
  4400. base = fname.replace(".gcode", "").replace(".3mf", "")
  4401. keys.append((printer_id, base))
  4402. keys.append((printer_id, f"{base}.3mf"))
  4403. return keys
  4404. async def _dispatched_by_bambuddy(db, printer_id: int, filename: str | None, subtask_name: str | None) -> bool:
  4405. """Whether this print start is one Bambuddy sent to the printer."""
  4406. if any(key in _expected_prints for key in _expected_print_keys(printer_id, filename, subtask_name)):
  4407. return True
  4408. from backend.app.models.archive import PrintArchive
  4409. from backend.app.models.print_queue import PrintQueueItem
  4410. # The queue row the scheduler sets to printing before it sends -- but only
  4411. # when it was sent as this print: a row left printing by a run whose end
  4412. # was missed names another file.
  4413. dispatched_names = (
  4414. await db.execute(
  4415. select(PrintArchive.filename)
  4416. .join(PrintQueueItem, PrintQueueItem.archive_id == PrintArchive.id)
  4417. .where(PrintQueueItem.printer_id == printer_id, PrintQueueItem.status == "printing")
  4418. )
  4419. ).scalars()
  4420. observed = [name for name in (subtask_name, (filename or "").rsplit("/", 1)[-1]) if name]
  4421. return any(
  4422. _subtask_names_match(_subtask_name_from_filename(dispatched), _subtask_name_from_filename(seen))
  4423. for dispatched in dispatched_names
  4424. if dispatched
  4425. for seen in observed
  4426. )
  4427. async def on_print_start(printer_id: int, data: dict):
  4428. """Handle print start - archive the 3MF file immediately."""
  4429. logger = logging.getLogger(__name__)
  4430. # The print's start, before the 3MF download that the archive waits for
  4431. # (#3327): a minute of it would otherwise be missing from the print time.
  4432. detected_at = datetime.now(timezone.utc)
  4433. logger.info("[CALLBACK] on_print_start called for printer %s, data keys: %s", printer_id, list(data.keys()))
  4434. octoeverywhere_detection_service.reset_printer(printer_id)
  4435. try:
  4436. notify_live_activities.print_started(printer_id, printer_manager.get_status(printer_id), data)
  4437. except Exception:
  4438. logger.exception("Notify Live Activity start hook failed for printer %s", printer_id)
  4439. # Clear any stale user-stopped flag from previous print cycles
  4440. _user_stopped_printers.discard(printer_id)
  4441. _kill_switch_stopped.discard(printer_id)
  4442. # A new print starts its milestones from zero (#3211). The status path only
  4443. # resets on progress below 5 while not printing, which a printer that goes
  4444. # from FINISH at 100% straight into a new print at a preparation-phase 85%
  4445. # never shows. This callback does not fire after a Bambuddy restart (#1304)
  4446. # or on resume from pause, so it cannot repeat a milestone mid-print.
  4447. _last_progress_milestone[printer_id] = 0
  4448. _kill_switch_notification_tasks.pop(printer_id, None)
  4449. # #1721: drop any leftover pre-captured finish frame from a prior print
  4450. # so a never-consumed cache entry can't bleed into the new print's photo.
  4451. _stage22_finish_frames.pop(printer_id, None)
  4452. # #1867: same for the in-print frame bank — a queued print must not reuse
  4453. # the previous job's banked frame.
  4454. _inprint_frame_bank.pop(printer_id, None)
  4455. _inprint_frame_bank_ts.pop(printer_id, None)
  4456. # #2547: bind (or clear) the "this print ends with injected End G-code" flag.
  4457. # Unconditional, so a print Bambuddy didn't dispatch drops the previous
  4458. # print's flag instead of inheriting it.
  4459. print_dispatch_context.adopt(printer_id)
  4460. # Cancel any active bed cooldown waiter for this printer
  4461. if _bed_cool_waiters.pop(printer_id, None):
  4462. logger.info("[BED-COOL] Cancelled bed cooldown waiter for printer %s (new print started)", printer_id)
  4463. # Clear cached cover images so the new print's thumbnail is fetched fresh
  4464. from backend.app.api.routes.printers import clear_cover_cache
  4465. clear_cover_cache(printer_id)
  4466. await ws_manager.send_print_start(printer_id, data)
  4467. # Notify when the print-start AMS mapping references tray slots without spool assignments.
  4468. await notify_missing_spool_assignments_on_print_start(printer_id, data, logger)
  4469. # MQTT relay - publish print start
  4470. try:
  4471. printer_info = printer_manager.get_printer(printer_id)
  4472. if printer_info:
  4473. await mqtt_relay.on_print_start(
  4474. printer_id,
  4475. printer_info.name,
  4476. printer_info.serial_number,
  4477. data.get("filename", ""),
  4478. data.get("subtask_name", ""),
  4479. )
  4480. except Exception:
  4481. pass # Don't fail print start callback if MQTT fails
  4482. # Capture AMS tray remain%, the assignment snapshot, the dispatched plate
  4483. # and mapping, and the seeded tray-change log.
  4484. #
  4485. # Unconditional, for both inventory backends. This only *captures* — the
  4486. # writing is still split, with the internal tracker skipped at completion
  4487. # when Spoolman owns usage. Spoolman's own durable row (#1820) already
  4488. # carries its plate-scoped 3MF figures and stored mapping, but not the
  4489. # tray-change log, and that log is the only record of which spool fed
  4490. # which layers when AMS Filament Backup swaps trays mid-print. Capturing
  4491. # it on one side only would leave Spoolman users with the mid-print
  4492. # restart bug this fixes for everyone else.
  4493. try:
  4494. async with async_session() as db:
  4495. from backend.app.api.routes.settings import get_setting
  4496. from backend.app.services.usage_tracker import on_print_start as usage_on_print_start
  4497. _spoolman_on = await get_setting(db, "spoolman_enabled")
  4498. await usage_on_print_start(
  4499. printer_id,
  4500. data,
  4501. printer_manager,
  4502. db=db,
  4503. spoolman_owns_usage=bool(_spoolman_on) and _spoolman_on.lower() == "true",
  4504. )
  4505. except Exception as e:
  4506. logger.warning("Usage tracker on_print_start failed: %s", e)
  4507. # Track if notification was sent (to avoid sending twice)
  4508. notification_sent = False
  4509. # Smart plug automation: turn on plug when print starts
  4510. try:
  4511. async with async_session() as db:
  4512. await smart_plug_manager.on_print_start(printer_id, db)
  4513. except Exception as e:
  4514. logger.warning("Smart plug on_print_start failed: %s", e)
  4515. async with async_session() as db:
  4516. from backend.app.models.printer import Printer
  4517. from backend.app.services.bambu_ftp import list_files_async
  4518. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  4519. printer = result.scalar_one_or_none()
  4520. # Plate detection check - pause if objects detected on build plate
  4521. logger.info(
  4522. f"[PLATE CHECK] printer_id={printer_id}, plate_detection_enabled={printer.plate_detection_enabled if printer else 'NO PRINTER'}"
  4523. )
  4524. if printer and printer.plate_detection_enabled:
  4525. logger.info("[PLATE CHECK] ENTERING plate detection code for printer %s", printer_id)
  4526. # Release the pooled DB connection before the plate-detection camera
  4527. # work (a 2.5s light-settle sleep + FTP/camera capture). Only the
  4528. # printer SELECT has run so far — nothing to persist — so this commit
  4529. # is a data-noop that ends the read transaction and returns the
  4530. # connection to the pool during the I/O (issue #2572). expire_on_commit
  4531. # =False keeps printer.* readable; on_plate_not_empty (rare) and the
  4532. # archive lookups below re-acquire a fresh connection on next execute.
  4533. await db.commit()
  4534. try:
  4535. from backend.app.services.plate_detection import check_plate_empty
  4536. # Build ROI tuple from printer settings if available
  4537. roi = None
  4538. if all(
  4539. [
  4540. printer.plate_detection_roi_x is not None,
  4541. printer.plate_detection_roi_y is not None,
  4542. printer.plate_detection_roi_w is not None,
  4543. printer.plate_detection_roi_h is not None,
  4544. ]
  4545. ):
  4546. roi = (
  4547. printer.plate_detection_roi_x,
  4548. printer.plate_detection_roi_y,
  4549. printer.plate_detection_roi_w,
  4550. printer.plate_detection_roi_h,
  4551. )
  4552. # The light is held, not switched, so a live view opened
  4553. # meanwhile keeps it when the check lets go (SP-C11). Every
  4554. # printer, whatever the camera light mode: detection needs it.
  4555. plate_photo_data = None
  4556. objects_detected = False
  4557. check_failure: str | None = None
  4558. async with camera_light(printer, any_mode=True, settle=2.5):
  4559. logger.info("[PLATE CHECK] Running plate detection for printer %s", printer_id)
  4560. plate_result = await check_plate_empty(
  4561. printer_id=printer_id,
  4562. ip_address=printer.ip_address,
  4563. access_code=printer.access_code,
  4564. model=printer.model,
  4565. include_debug_image=False,
  4566. external_camera_url=printer.external_camera_url,
  4567. external_camera_type=printer.external_camera_type,
  4568. use_external=printer.external_camera_enabled,
  4569. roi=roi,
  4570. external_camera_snapshot_url=printer.external_camera_snapshot_url,
  4571. )
  4572. # No reference to compare against is a check that did not
  4573. # run either: say so rather than call the plate empty.
  4574. check_failure = getattr(plate_result, "failure_reason", None) or (
  4575. "not_calibrated" if plate_result.needs_calibration else None
  4576. )
  4577. objects_detected = not check_failure and not plate_result.is_empty
  4578. if objects_detected:
  4579. # Objects detected - pause the print!
  4580. logger.warning(
  4581. f"[PLATE CHECK] Objects detected on plate for printer {printer_id}! "
  4582. f"Confidence: {plate_result.confidence:.0%}, Diff: {plate_result.difference_percent:.1f}%"
  4583. )
  4584. pause_client = printer_manager.get_client(printer_id)
  4585. if pause_client:
  4586. pause_client.pause_print()
  4587. logger.info("[PLATE CHECK] Print paused for printer %s", printer_id)
  4588. # Snapshot while the light's still on — restoring it first
  4589. # would leave the notification with a dark photo.
  4590. try:
  4591. plate_photo_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  4592. except Exception as snap_err:
  4593. logger.warning(
  4594. "[PLATE CHECK] Failed to capture snapshot for printer %s: %s", printer_id, snap_err
  4595. )
  4596. if objects_detected:
  4597. # Send notification about plate not empty
  4598. await ws_manager.broadcast(
  4599. {
  4600. "type": "plate_not_empty",
  4601. "printer_id": printer_id,
  4602. "printer_name": printer.name,
  4603. "message": f"Objects detected on build plate! Print paused. (Diff: {plate_result.difference_percent:.1f}%)",
  4604. }
  4605. )
  4606. # Also send push notification
  4607. try:
  4608. await notification_service.on_plate_not_empty(
  4609. printer_id=printer_id,
  4610. printer_name=printer.name,
  4611. db=db,
  4612. difference_percent=plate_result.difference_percent,
  4613. image_data=plate_photo_data,
  4614. )
  4615. except Exception as notif_err:
  4616. logger.warning("[PLATE CHECK] Failed to send notification: %s", notif_err)
  4617. elif check_failure:
  4618. logger.warning(
  4619. "[PLATE CHECK] Plate check could not run for printer %s (%s): %s; proceeding with print",
  4620. printer_id,
  4621. check_failure,
  4622. plate_result.message,
  4623. )
  4624. await _announce_plate_check_failed(printer_id, printer.name, check_failure)
  4625. else:
  4626. logger.info("[PLATE CHECK] Plate is empty for printer %s, proceeding with print", printer_id)
  4627. except Exception as plate_err:
  4628. # Don't block print on plate detection errors
  4629. logger.warning(
  4630. "[PLATE CHECK] Plate check could not run for printer %s: %s; proceeding with print",
  4631. printer_id,
  4632. plate_err,
  4633. )
  4634. await _announce_plate_check_failed(printer_id, printer.name, "error")
  4635. if not printer:
  4636. logger.info("[CALLBACK] Skipping archive - printer not found in database")
  4637. if not notification_sent:
  4638. await _send_print_start_notification(printer_id, data, logger=logger)
  4639. return
  4640. # Skip the printer's own jobs — a calibration run is not a user's print.
  4641. # See is_internal_printer_job for what counts and why both fields are
  4642. # tested; the pressure-advance line reports as a subtask name with no
  4643. # /usr/ path, which the old prefix-only test here missed entirely.
  4644. #
  4645. # No notification either. The event describes the printer calibrating
  4646. # itself, so "Print started" is as wrong as the archive was, and the
  4647. # matching completion is suppressed in on_print_complete for the same
  4648. # reason. Checked before the auto-archive switches, whose "off" path
  4649. # still notifies.
  4650. # A /usr/ path is the printer's own file. A name alone is not proof: a
  4651. # print Bambuddy dispatched is never one, whatever it is called.
  4652. # Dispatched = an expected print, or, after a restart that lost those,
  4653. # a queue job printing on the printer.
  4654. if is_internal_printer_job(data.get("filename", ""), data.get("subtask_name", "")) and (
  4655. (data.get("filename") or "").startswith("/usr/")
  4656. or not await _dispatched_by_bambuddy(db, printer_id, data.get("filename", ""), data.get("subtask_name", ""))
  4657. ):
  4658. logger.info(
  4659. "[CALLBACK] Skipping archive — internal printer job detected: filename=%s, subtask=%s",
  4660. data.get("filename", ""),
  4661. data.get("subtask_name", ""),
  4662. )
  4663. return
  4664. # The global switch is the master: off, no printer archives prints
  4665. # started outside Bambuddy; on, each printer's own switch decides.
  4666. from backend.app.api.routes.settings import get_setting
  4667. _global_auto_archive = await get_setting(db, "auto_archive")
  4668. _global_archives = _global_auto_archive is None or _global_auto_archive.lower() == "true"
  4669. archives_prints = printer.auto_archive and _global_archives
  4670. if not archives_prints:
  4671. # auto-archive disabled — check if there's an expected print (dispatched
  4672. # by BamBuddy via queue/reprint) that already has an archive to promote.
  4673. # If so, fall through to the expected-print handling below so the archive
  4674. # is tracked in _active_prints and usage tracking works at completion.
  4675. _fn = data.get("filename", "")
  4676. _sn = data.get("subtask_name", "")
  4677. _check_keys: list[tuple[int, str]] = []
  4678. if _sn:
  4679. _check_keys += [
  4680. (printer_id, _sn),
  4681. (printer_id, f"{_sn}.3mf"),
  4682. (printer_id, f"{_sn}.gcode.3mf"),
  4683. ]
  4684. if _fn:
  4685. _base_fn = _fn.split("/")[-1] if "/" in _fn else _fn
  4686. _check_keys.append((printer_id, _base_fn))
  4687. _no_archive_base = _base_fn.replace(".gcode", "").replace(".3mf", "")
  4688. _check_keys += [
  4689. (printer_id, _no_archive_base),
  4690. (printer_id, f"{_no_archive_base}.3mf"),
  4691. ]
  4692. _has_expected = any(k in _expected_prints for k in _check_keys)
  4693. if not _has_expected:
  4694. # No expected print — truly external print (started from slicer/touchscreen).
  4695. # Say which switch it was: before 1.3.0 the global one was saved
  4696. # and never read, so an install upgraded with it off stopped
  4697. # archiving with nothing to point at it (#3477).
  4698. if not _global_archives:
  4699. logger.info(
  4700. "[CALLBACK] Skipping archive: the global 'Auto-archive prints' switch "
  4701. "(Settings > General > Archive Settings) is off, and Bambuddy did not start this print"
  4702. )
  4703. else:
  4704. logger.info(
  4705. "[CALLBACK] Skipping archive: auto-archive is off for printer %s, "
  4706. "and Bambuddy did not start this print",
  4707. printer_id,
  4708. )
  4709. if not notification_sent:
  4710. _no_archive_creator: int | None = None
  4711. for _key in _check_keys:
  4712. _expected_prints.pop(_key, None)
  4713. _expected_print_registered_at.pop(_key, None)
  4714. popped_creator = _expected_print_creators.pop(_key, None)
  4715. if _no_archive_creator is None:
  4716. _no_archive_creator = popped_creator
  4717. _creator_data = {"created_by_id": _no_archive_creator} if _no_archive_creator else None
  4718. await _send_print_start_notification(printer_id, data, _creator_data, logger)
  4719. return
  4720. else:
  4721. logger.info("[CALLBACK] auto_archive disabled but expected print found — promoting archive")
  4722. # Get the filename and subtask_name
  4723. filename = data.get("filename", "")
  4724. subtask_name = data.get("subtask_name", "")
  4725. # MQTT subtask_id uniquely identifies a print job on the printer. When
  4726. # present, it lets us match an archive across a backend restart (#972):
  4727. # same id → same print → resume the existing row instead of cancelling
  4728. # it and recreating from scratch (which loses started_at). Treat "0"
  4729. # and "" as absent — Bambu reports "0" for non-cloud / local prints.
  4730. raw_mqtt = data.get("raw_data") or {}
  4731. subtask_id = raw_mqtt.get("subtask_id")
  4732. if subtask_id is not None:
  4733. subtask_id = str(subtask_id).strip()
  4734. if subtask_id in ("", "0"):
  4735. subtask_id = None
  4736. logger.info("[CALLBACK] Print start detected - filename: %s, subtask: %s", filename, subtask_name)
  4737. if not filename and not subtask_name:
  4738. # Send notification without archive data (no filename)
  4739. logger.info("[CALLBACK] Skipping archive - no filename or subtask_name")
  4740. if not notification_sent:
  4741. await _send_print_start_notification(printer_id, data, logger=logger)
  4742. return
  4743. # Check if this is an expected print from reprint/scheduled
  4744. expected_keys = _expected_print_keys(printer_id, filename, subtask_name)
  4745. expected_archive_id = None
  4746. # Who queued it, taken with the entries it was registered beside: left
  4747. # behind once those are gone (the archive deleted mid-dispatch, or the
  4748. # start notification already sent), the expiry never saw it, and an
  4749. # unrelated later print of that name was attributed to that user (EP-2).
  4750. fallback_creator = None
  4751. for key in expected_keys:
  4752. expected_archive_id = _expected_prints.pop(key, None)
  4753. _expected_print_registered_at.pop(key, None)
  4754. if expected_archive_id:
  4755. # Clean up other possible keys for this print
  4756. for other_key in expected_keys:
  4757. _expected_prints.pop(other_key, None)
  4758. _expected_print_registered_at.pop(other_key, None)
  4759. creator = _expected_print_creators.pop(other_key, None)
  4760. if fallback_creator is None:
  4761. fallback_creator = creator
  4762. break
  4763. if expected_archive_id:
  4764. # This is a reprint/scheduled print - use existing archive, don't create new one
  4765. logger.info("Using expected archive %s for print (skipping duplicate)", expected_archive_id)
  4766. # Bambuddy dispatched it: the kill switch's fast path keeps it as
  4767. # Bambuddy's once the expected entry is consumed here.
  4768. _owned_print_archive_ids.add(expected_archive_id)
  4769. from backend.app.models.archive import PrintArchive
  4770. result = await db.execute(select(PrintArchive).where(PrintArchive.id == expected_archive_id))
  4771. archive = result.scalar_one_or_none()
  4772. # Deleted after this handler loaded it (the plate check waits
  4773. # seconds) but before the route forgot the expected print: the
  4774. # archive the route kept or deleted is left as the route left it.
  4775. if archive and await _printer_gone(db, printer_id):
  4776. logger.info(
  4777. "Printer %s was deleted before its print started; not adopting archive %s", printer_id, archive.id
  4778. )
  4779. await forget_printer(printer_id)
  4780. return
  4781. if archive:
  4782. # As it was before this adoption: put back if the printer turns
  4783. # out to have been deleted while it was adopted (below).
  4784. before_adoption = {
  4785. attr.columns[0].name: getattr(archive, attr.key)
  4786. for attr in sa_inspect(PrintArchive).column_attrs
  4787. if attr.key != "id"
  4788. }
  4789. # Update archive status to printing
  4790. archive.status = "printing"
  4791. archive.started_at = datetime.now(timezone.utc)
  4792. # The previous run's outcome is not this one's: a reprint that
  4793. # completes must not keep showing why the last one failed.
  4794. archive.failure_reason = None
  4795. archive.completed_at = None
  4796. # The previous run's answer is still on this row and the
  4797. # completion prompt is gated on ``user_verdict is None`` (#1898),
  4798. # so without a reset the second run inherits the first run's
  4799. # verdict: no prompt at all, and a green "good" badge on a run
  4800. # nobody ever judged.
  4801. if archive.confirm_requested:
  4802. archive.user_verdict = None
  4803. archive.user_verdict_source = None
  4804. archive.user_verdict_at = None
  4805. archive.confirm_token = None
  4806. archive.confirm_token_used_at = None
  4807. # Reprint of an archive reuses the source row. Without resetting
  4808. # ``timelapse_path`` _scan_for_timelapse_with_retries early-returns
  4809. # ("already has timelapse") and _capture_finish_photo_from_timelapse
  4810. # extracts the *original* print's last frame, which then ships in
  4811. # the completion notification (#1707). Clear the path so the
  4812. # scanner runs fresh; also unlink the old video file so reprints
  4813. # don't accumulate orphans in the archive directory. Photos list
  4814. # is left alone — accumulating one finish photo per run is fine.
  4815. # The print-start baseline (#2704) is stale for the same reason:
  4816. # it describes the printer before the previous run. The capture
  4817. # below overwrites it, but clear it here too so an early failure
  4818. # can't leave the scan diffing against the wrong snapshot.
  4819. #
  4820. # Unlinked only when it is the same plate printing again. Every
  4821. # plate of a Send All shares this archive, and the video on it
  4822. # can be the only copy of another plate's run -- the printer's
  4823. # was deleted once it attached (#3275). That one is left in the
  4824. # archive directory, which goes with the archive. A video whose
  4825. # plate is not on record (attached by hand, or before the
  4826. # column existed) is kept too where the 3MF has other plates,
  4827. # or cannot be read: nothing tells it apart from another plate's,
  4828. # and keeping costs one file where deleting can cost the only
  4829. # copy. On an archive with exactly one plate it is that plate's.
  4830. archive.timelapse_baseline = None
  4831. stale_timelapse_relpath = archive.timelapse_path
  4832. stale_timelapse_plate_id = archive.timelapse_plate_id
  4833. starting_plate_id = _print_plate_ids.get((printer_id, expected_archive_id))
  4834. if stale_timelapse_relpath:
  4835. archive.timelapse_path = None
  4836. archive.timelapse_plate_id = None
  4837. stale_is_this_plates = stale_timelapse_plate_id == starting_plate_id
  4838. if stale_timelapse_relpath and stale_timelapse_plate_id is None and archive.file_path:
  4839. from backend.app.services.archive import plate_indexes_in_3mf
  4840. plates = await asyncio.to_thread(
  4841. plate_indexes_in_3mf,
  4842. app_settings.base_dir / archive.file_path, # SEC-PATH-OK: stored relative archive path
  4843. )
  4844. stale_is_this_plates = len(plates) == 1
  4845. if stale_timelapse_relpath and not stale_is_this_plates:
  4846. logger.info(
  4847. "Kept timelapse %s of plate %s on archive %s: plate %s is starting",
  4848. stale_timelapse_relpath,
  4849. stale_timelapse_plate_id,
  4850. expected_archive_id,
  4851. starting_plate_id,
  4852. )
  4853. # The video is deleted once the adoption stands (below).
  4854. delete_stale_video = bool(stale_timelapse_relpath) and stale_is_this_plates
  4855. # Persist a restart-stable id so a later restart resumes this
  4856. # archive by subtask_id instead of name-matching + duplicating
  4857. # it (#1485). The printer often hasn't echoed subtask_id back
  4858. # this soon after dispatch, so fall back to the id Bambuddy
  4859. # minted when it sent the print command. Scoped to this
  4860. # expected-print branch on purpose: an expected match means
  4861. # Bambuddy dispatched this exact print in this process, so the
  4862. # client's last-dispatch id genuinely belongs to it — using it
  4863. # for an externally-started print could mis-tag the archive.
  4864. effective_subtask_id = subtask_id
  4865. if not effective_subtask_id:
  4866. _client = printer_manager.get_client(printer_id)
  4867. _dispatched = getattr(_client, "last_dispatch_subtask_id", None) if _client else None
  4868. if _dispatched:
  4869. effective_subtask_id = str(_dispatched).strip() or None
  4870. # Update on first-set OR on reprint (the queue dispatcher mints
  4871. # a fresh subtask_id per dispatch in bambu_mqtt:3647). Skipping
  4872. # the rewrite for reprints leaves the archive holding the FIRST
  4873. # run's id; if MQTT then reconnects mid-print, the reconciler
  4874. # (#1542) compares the stale stored id against the printer's
  4875. # live id, sees a mismatch, and synthesises a bogus PRINT
  4876. # COMPLETE — exactly the false-positive "Print Stopped" reported
  4877. # in #1807. Inequality check preserves the noop-on-stable-push
  4878. # behaviour the earlier `not archive.subtask_id` guard provided.
  4879. if effective_subtask_id and archive.subtask_id != effective_subtask_id:
  4880. archive.subtask_id = effective_subtask_id
  4881. # #1403 follow-up: VP-queue archives are created with
  4882. # printer_id=None at queue-add time (we don't know which
  4883. # printer will run the job yet). When the print actually
  4884. # starts on a specific printer the expected-archive lookup
  4885. # used to skip this assignment, leaving printer_id=None
  4886. # forever — which then disables the "Scan for timelapse"
  4887. # button in ArchivesPage (gated on !archive.printer_id).
  4888. if archive.printer_id != printer_id:
  4889. archive.printer_id = printer_id
  4890. adopting_id = archive.id # a rollback expires the object
  4891. try:
  4892. await db.commit()
  4893. except (IntegrityError, StaleDataError):
  4894. # The archive stays unchanged, or is gone: "delete archives"
  4895. # removed the row under the adoption, which then matched no
  4896. # row (StaleDataError) rather than being refused.
  4897. if await _refused_for_deleted_printer(db, printer_id, f"archive {adopting_id} was adopted"):
  4898. return
  4899. raise
  4900. # The printer can be deleted between the check above and this
  4901. # commit (the route's transaction lasts seconds), and the commit
  4902. # then adopted the archive anyway: status, verdict and timelapse
  4903. # changed, and the deleted printer's id written on a row the
  4904. # route had detached or never touched. Undo the adoption.
  4905. if await _printer_gone(db, printer_id):
  4906. logger.info(
  4907. "Printer %s was deleted while archive %s was adopted; undoing the adoption",
  4908. printer_id,
  4909. archive.id,
  4910. )
  4911. # Everything as before this adoption. The printer it had
  4912. # then stays, unless it was the deleted one: the route
  4913. # detached that row (kept) or deleted it (no row matches).
  4914. previous_printer = before_adoption["printer_id"]
  4915. await db.execute(
  4916. update(PrintArchive)
  4917. .where(PrintArchive.id == archive.id)
  4918. .values(
  4919. {
  4920. **before_adoption,
  4921. "printer_id": None if previous_printer == printer_id else previous_printer,
  4922. }
  4923. )
  4924. )
  4925. await db.commit()
  4926. await forget_printer(printer_id)
  4927. return
  4928. await _drop_replaced_plate_videos(
  4929. db,
  4930. archive.id,
  4931. starting_plate_id,
  4932. stale_timelapse_relpath if delete_stale_video else None,
  4933. logger,
  4934. )
  4935. # Track as active print
  4936. _bind_active_print(printer_id, archive.id, archive.filename, filename, subtask_name)
  4937. await _open_run_for(archive.id, printer_id, subtask_id)
  4938. # Start timelapse session if external camera is enabled (#1353).
  4939. # Queue / VP-dispatched prints land here in the expected-archive
  4940. # branch and used to skip start_session entirely — frames were
  4941. # never captured and the post-print stitch silently returned None.
  4942. _maybe_start_layer_timelapse(printer, printer_id, archive.id)
  4943. # Inject ams_mapping into usage tracker session — the session was created
  4944. # before expected-print promotion, so it may have ams_mapping=None when
  4945. # the MQTT request topic subscription failed (common on P1S/A1).
  4946. _stored_map = _print_ams_mappings.get((printer_id, expected_archive_id))
  4947. _stored_plate_id = _print_plate_ids.get((printer_id, expected_archive_id))
  4948. if _stored_map or _stored_plate_id is not None:
  4949. try:
  4950. from backend.app.services.usage_tracker import _active_sessions
  4951. _ut_session = _active_sessions.get(printer_id)
  4952. if _ut_session and _stored_map and not _ut_session.ams_mapping:
  4953. _ut_session.ams_mapping = _stored_map
  4954. logger.info("[CALLBACK] Injected ams_mapping into usage tracker session: %s", _stored_map)
  4955. # plate_id injection covers direct-Print of plate N of a multi-plate
  4956. # 3MF — queue prints already capture it via the on_print_start queue
  4957. # lookup, but direct-Print never goes through the queue (#1697).
  4958. if _ut_session and _stored_plate_id is not None and _ut_session.plate_id is None:
  4959. _ut_session.plate_id = _stored_plate_id
  4960. logger.info("[CALLBACK] Injected plate_id into usage tracker session: %s", _stored_plate_id)
  4961. except Exception:
  4962. pass
  4963. # Set up energy tracking (#941: persist start on archive row)
  4964. await _record_energy_start(archive, printer_id, db, context="expected-print")
  4965. await ws_manager.send_archive_updated(
  4966. {
  4967. "id": archive.id,
  4968. "status": "printing",
  4969. }
  4970. )
  4971. # Send notification with archive data (reprint/scheduled)
  4972. if not notification_sent:
  4973. # Use archive's created_by_id; fall back to the creator registered via
  4974. # register_expected_print (handles library-file-based queue items where
  4975. # the freshly-created archive has no created_by_id yet), taken above.
  4976. archive_data = {
  4977. "print_time_seconds": _run_print_time_seconds(archive, printer_id),
  4978. "created_by_id": archive.created_by_id or fallback_creator,
  4979. }
  4980. await _send_print_start_notification(printer_id, data, archive_data, logger)
  4981. # Extract printable objects from the archived 3MF file
  4982. _load_objects_from_archive(archive, printer_id, logger)
  4983. if archive.file_path:
  4984. _load_pauses_from_3mf(app_settings.base_dir / archive.file_path, printer_id)
  4985. # Store Spoolman tracking data for per-filament usage reporting
  4986. try:
  4987. await _store_spoolman_print_data(
  4988. printer_id,
  4989. archive.id,
  4990. archive.file_path,
  4991. db,
  4992. printer_manager,
  4993. ams_mapping=_get_start_ams_mapping(data, printer_id, archive.id),
  4994. plate_id=_get_start_plate_id(printer_id, archive.id),
  4995. )
  4996. except Exception as e:
  4997. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  4998. # Price the print from its spools now rather than only at completion (#3261)
  4999. _schedule_archive_cost_estimate(
  5000. printer_id,
  5001. archive.id,
  5002. printer_manager,
  5003. ams_mapping=_get_start_ams_mapping(data, printer_id, archive.id),
  5004. plate_id=_get_start_plate_id(printer_id, archive.id),
  5005. session_factory=async_session,
  5006. )
  5007. # Capture timelapse file baseline for snapshot-diff on completion
  5008. # (mirrors the new-archive branch). Queue / VP-dispatched prints
  5009. # hit this branch — without the baseline the completion-time scan
  5010. # falls into its "take baseline now" fallback, which snapshots
  5011. # AFTER the new MP4 already exists and never matches a diff
  5012. # (#1403 follow-up — see pwostran's 2026-05-18 support bundle).
  5013. await _capture_timelapse_baseline_at_start(printer, printer_id, logger, archive_id=archive.id)
  5014. await _forget_if_printer_gone(printer_id)
  5015. return # Skip creating a new archive
  5016. # The archive was deleted after the job was queued.
  5017. if not archives_prints:
  5018. # Only the expected print brought a printer that does not
  5019. # archive this far; with it gone, nothing is archived.
  5020. return
  5021. # Otherwise archive the print anew rather than drop it, so it still
  5022. # gets its notification, usage tracking and history.
  5023. logger.warning(
  5024. "Expected archive %s for printer %s no longer exists; archiving the print anew",
  5025. expected_archive_id,
  5026. printer_id,
  5027. )
  5028. # Check if there's already a "printing" archive for this printer/file
  5029. # This prevents duplicates when backend restarts during an active print
  5030. from backend.app.models.archive import PrintArchive
  5031. existing_archive: PrintArchive | None = None
  5032. resumed_by_run = False
  5033. # Best match: this job's run, open on the printer since the print
  5034. # started. The archive row is shared by every plate of it, and its
  5035. # printer, job id and status are the latest start's, which may be
  5036. # another printer's.
  5037. if subtask_id:
  5038. from backend.app.services.print_log import open_run_on_printer
  5039. run = await open_run_on_printer(db, printer_id, subtask_id=subtask_id)
  5040. if run is not None and run.archive_id is not None:
  5041. existing_archive = await db.scalar(
  5042. select(PrintArchive).where(PrintArchive.id == run.archive_id, PrintArchive.deleted_at.is_(None))
  5043. )
  5044. resumed_by_run = existing_archive is not None
  5045. # Preferred match: subtask_id equality. MQTT reports the same subtask_id
  5046. # across a backend restart for the same print, so this is the most
  5047. # reliable way to reattach. Only a row still printing: a cancelled one
  5048. # has been through completion (usage, history, notifications) already,
  5049. # and reopening it would count the print twice.
  5050. if subtask_id and existing_archive is None:
  5051. by_id = await db.execute(
  5052. select(PrintArchive)
  5053. .where(PrintArchive.printer_id == printer_id)
  5054. .where(PrintArchive.subtask_id == subtask_id)
  5055. .where(PrintArchive.status == "printing")
  5056. .order_by(PrintArchive.created_at.desc())
  5057. .limit(1)
  5058. )
  5059. existing_archive = by_id.scalar_one_or_none()
  5060. # Fallback match: name-based lookup. Kept as-is for prints whose
  5061. # subtask_id is missing ("0" / local / non-cloud prints).
  5062. if existing_archive is None:
  5063. check_name = subtask_name or filename.split("/")[-1].replace(".gcode", "").replace(".3mf", "")
  5064. archive_filenames = [f"{check_name}.3mf", f"{check_name}.gcode.3mf"]
  5065. # A print started from the printer's screen names its file in full,
  5066. # so the forms above double its extension. The file itself is the
  5067. # one exact match, and nothing looser is added (#3009).
  5068. if _subtask_is_the_file(subtask_name, filename):
  5069. archive_filenames.append(subtask_name)
  5070. existing = await db.execute(
  5071. select(PrintArchive)
  5072. .where(PrintArchive.printer_id == printer_id)
  5073. .where(PrintArchive.status == "printing")
  5074. .where(
  5075. or_(
  5076. PrintArchive.print_name == check_name,
  5077. PrintArchive.filename.in_(archive_filenames),
  5078. )
  5079. )
  5080. .order_by(PrintArchive.created_at.desc())
  5081. .limit(1)
  5082. )
  5083. existing_archive = existing.scalar_one_or_none()
  5084. if existing_archive:
  5085. # subtask_id match → always resume, regardless of age. Same print,
  5086. # just a backend restart.
  5087. subtask_match = resumed_by_run or bool(subtask_id and existing_archive.subtask_id == subtask_id)
  5088. if subtask_match:
  5089. logger.info("Resuming archive %s on subtask_id match (%s)", existing_archive.id, subtask_id)
  5090. _bind_active_print(printer_id, existing_archive.id, existing_archive.filename, filename, subtask_name)
  5091. await _open_run_for(existing_archive.id, printer_id, subtask_id)
  5092. _mark_owned_if_dispatched(existing_archive)
  5093. if await _run_needs_energy_start(existing_archive, printer_id, db):
  5094. await _record_energy_start(existing_archive, printer_id, db, context="subtask-resume")
  5095. if not notification_sent:
  5096. archive_data = {
  5097. "print_time_seconds": _run_print_time_seconds(existing_archive, printer_id),
  5098. "created_by_id": existing_archive.created_by_id,
  5099. }
  5100. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5101. _load_objects_from_archive(existing_archive, printer_id, logger)
  5102. if existing_archive.file_path:
  5103. _load_pauses_from_resumed_archive(existing_archive, printer_id)
  5104. return
  5105. # Name-match only (no subtask_id to anchor on): decide resume vs.
  5106. # stale from the printer's *current* progress, not wall-clock age.
  5107. # A genuinely long print used to trip a blind 4h cutoff and have its
  5108. # live archive cancelled + duplicated on every backend restart
  5109. # (#1485). If the printer reports real progress, this name-matched
  5110. # 'printing' archive IS that ongoing print — resume it whatever its
  5111. # age. Only treat it as a stale leftover when the printer clearly
  5112. # shows a different, freshly-started print: near-0% progress on an
  5113. # archive far too old to still be at 0%. Unknown progress (printer
  5114. # not connected) never cancels — resuming is the safe default.
  5115. archive_age = datetime.now(timezone.utc) - existing_archive.created_at.replace(tzinfo=timezone.utc)
  5116. live_status = printer_manager.get_status(printer_id)
  5117. live_progress = getattr(live_status, "progress", None) if live_status else None
  5118. looks_stale = (
  5119. live_progress is not None and live_progress < 1.0 and archive_age.total_seconds() > 2 * 60 * 60
  5120. )
  5121. # Two subtask ids are two prints, however young the archive: one
  5122. # whose completion was missed must not swallow the next print of
  5123. # the same file. Except Bambuddy's own reprint, which reuses the
  5124. # row and mints a fresh id (#1807): the id it last sent says so,
  5125. # and after a restart, which forgets it, this printer's queue job
  5126. # running the archive.
  5127. if (
  5128. not looks_stale
  5129. and subtask_id
  5130. and existing_archive.subtask_id
  5131. and existing_archive.subtask_id != subtask_id
  5132. ):
  5133. _client = printer_manager.get_client(printer_id)
  5134. _dispatched = str(getattr(_client, "last_dispatch_subtask_id", None) or "").strip()
  5135. if _dispatched:
  5136. looks_stale = _dispatched != subtask_id
  5137. else:
  5138. from backend.app.models.print_queue import PrintQueueItem as _PQI
  5139. queue_run = await db.scalar(
  5140. select(_PQI.id)
  5141. .where(_PQI.archive_id == existing_archive.id)
  5142. .where(_PQI.printer_id == printer_id)
  5143. .where(_PQI.status == "printing")
  5144. .limit(1)
  5145. )
  5146. looks_stale = queue_run is None
  5147. if looks_stale:
  5148. logger.warning(
  5149. "Found stale 'printing' archive %s (age: %s, printer progress %s%%, subtask %s, printer reports %s)"
  5150. " — marking cancelled and creating new archive",
  5151. existing_archive.id,
  5152. archive_age,
  5153. live_progress,
  5154. existing_archive.subtask_id,
  5155. subtask_id,
  5156. )
  5157. existing_archive.status = "cancelled"
  5158. # Canonical key, not a sentence (issue #2974). "No status update
  5159. # received" is what both stale paths actually observed; which of
  5160. # the two it was is already carried by ``status`` -- cancelled
  5161. # here, the reconciled outcome at the reconnect site -- so one
  5162. # key loses no information and gives the Statistics breakdown a
  5163. # single bucket instead of two untranslatable prose strings.
  5164. existing_archive.failure_reason = "noStatusUpdate"
  5165. # And its run, as before runs were opened at the start: a new
  5166. # print starts on the printer, so whatever is open there ended
  5167. # unseen. Left open, the next reconnect would close it again.
  5168. from backend.app.services.print_log import discard_open_runs
  5169. discarded = await discard_open_runs(db, printer_id)
  5170. await db.commit()
  5171. await _announce_discarded_runs(discarded, printer_id)
  5172. # Its print keys go with it; the new archive registers its own.
  5173. for _key in [k for k, v in _active_prints.items() if v == existing_archive.id and k[0] == printer_id]:
  5174. del _active_prints[_key]
  5175. # Fall through to create new archive (don't return)
  5176. else:
  5177. logger.info(
  5178. f"Skipping duplicate - already have printing archive {existing_archive.id} for {check_name}"
  5179. )
  5180. # Track this as the active print
  5181. _bind_active_print(printer_id, existing_archive.id, existing_archive.filename, filename, subtask_name)
  5182. _mark_owned_if_dispatched(existing_archive)
  5183. # Attach subtask_id retroactively so future restarts can resume.
  5184. # Compare for inequality (not "is empty") to also pick up reprint
  5185. # dispatches that mint a fresh id — see #1807 for the bogus
  5186. # "Print Stopped" the strict-empty guard caused on reconnect.
  5187. if subtask_id and existing_archive.subtask_id != subtask_id:
  5188. existing_archive.subtask_id = subtask_id
  5189. await db.commit()
  5190. await _open_run_for(existing_archive.id, printer_id, subtask_id)
  5191. # Also set up energy tracking if not already tracked (#941: persisted column)
  5192. if await _run_needs_energy_start(existing_archive, printer_id, db):
  5193. await _record_energy_start(existing_archive, printer_id, db, context="existing-printing")
  5194. # Send notification with archive data (existing archive)
  5195. if not notification_sent:
  5196. archive_data = {
  5197. "print_time_seconds": _run_print_time_seconds(existing_archive, printer_id),
  5198. "created_by_id": existing_archive.created_by_id,
  5199. }
  5200. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5201. # Extract printable objects from the archived 3MF file
  5202. _load_objects_from_archive(existing_archive, printer_id, logger)
  5203. if existing_archive.file_path:
  5204. _load_pauses_from_resumed_archive(existing_archive, printer_id)
  5205. return
  5206. # Build list of possible 3MF filenames to try
  5207. possible_names = []
  5208. # Bambu printers typically store files as "Name.gcode.3mf"
  5209. # The subtask_name is usually the best source for the filename
  5210. if subtask_name:
  5211. if _subtask_is_the_file(subtask_name, filename):
  5212. # A print started from the printer's own screen reports the
  5213. # file's full name, extension included. That is the file, so
  5214. # no extension is appended to it (#3009).
  5215. possible_names.append(subtask_name)
  5216. else:
  5217. # Try common Bambu naming patterns
  5218. possible_names.append(f"{subtask_name}.gcode.3mf")
  5219. possible_names.append(f"{subtask_name}.3mf")
  5220. # Try original filename with .3mf extension
  5221. if filename:
  5222. # Extract just the filename part, not the full path
  5223. fname = filename.split("/")[-1] if "/" in filename else filename
  5224. if fname.endswith(".3mf"):
  5225. possible_names.append(fname)
  5226. elif fname.endswith(".gcode"):
  5227. base = fname.rsplit(".", 1)[0]
  5228. possible_names.append(f"{base}.gcode.3mf")
  5229. possible_names.append(f"{base}.3mf")
  5230. else:
  5231. possible_names.append(f"{fname}.gcode.3mf")
  5232. possible_names.append(f"{fname}.3mf")
  5233. # Also try with spaces converted to underscores (Bambu Studio may normalize filenames)
  5234. space_variants = []
  5235. for name in possible_names:
  5236. if " " in name:
  5237. space_variants.append(name.replace(" ", "_"))
  5238. possible_names.extend(space_variants)
  5239. # Remove duplicates while preserving order
  5240. seen = set()
  5241. possible_names = [x for x in possible_names if not (x in seen or seen.add(x))]
  5242. logger.info("Trying filenames: %s", possible_names)
  5243. # Release the pooled DB connection before the 3MF FTP download. Reaching
  5244. # here means none of the expected-/existing-archive write branches ran
  5245. # (they all return earlier) — only SELECTs have executed on this path, so
  5246. # this commit persists nothing; it ends the read transaction so the
  5247. # connection returns to the pool during the download. That download tries
  5248. # up to five remote paths per candidate filename with retry/backoff and
  5249. # can run for minutes under FTP contention; holding the session across it
  5250. # pinned one pooled connection idle-in-transaction (issue #2572). No DB
  5251. # work runs during the download — the new-archive writes below re-acquire
  5252. # a fresh connection, and expire_on_commit=False keeps printer.* readable.
  5253. await db.commit()
  5254. # Try to find and download the 3MF file
  5255. temp_path = None
  5256. downloaded_filename = None
  5257. # Cache check: cover endpoint may have already pulled this 3MF during
  5258. # the print (frontend opens the card and shows the thumbnail) — reuse
  5259. # that file instead of re-downloading 36MB over the same FTP link that
  5260. # just served it (#972). The cache keys on a normalized filename so
  5261. # variants like "X", "X.3mf", "X.gcode.3mf" all collapse to one entry.
  5262. for try_filename in possible_names:
  5263. if not try_filename.endswith(".3mf"):
  5264. continue
  5265. cached = get_cached_3mf(printer_id, try_filename)
  5266. if cached:
  5267. logger.info("Reusing cached 3MF from %s (avoided duplicate FTP)", cached)
  5268. temp_path = cached
  5269. downloaded_filename = try_filename
  5270. break
  5271. # Does this printer keep the sliced file somewhere FTPS can reach? On
  5272. # H2-series and P2S the answer is routinely no — the file stays on
  5273. # internal eMMC and port 990 only ever serves external storage — and
  5274. # then the whole sweep below (six filenames x five directories x four
  5275. # retries, then the directory walk) is ~110 connections that cannot
  5276. # succeed. Skip it and say why (#2780).
  5277. storage = print_file_reachable_over_ftp(printer_manager.get_status(printer_id))
  5278. # Set when a lookup is abandoned because the printer's FTPS cool-off is
  5279. # running rather than because the file is somewhere unreachable. The
  5280. # distinction is the whole of #2957: one is permanent, the other clears
  5281. # in minutes with the file still sitting on the printer.
  5282. blocked_by_ftps_cooloff = False
  5283. # Set when a probe reached the printer and still came back without the
  5284. # file -- a timeout mid-transfer, a refused connection, anything that is
  5285. # not a clean "not here". A 550 raises FileNotOnPrinterError and is
  5286. # caught by name below, so a file that genuinely is not on the card
  5287. # leaves this False and schedules nothing. Anything else means the
  5288. # transfer, not the file, is what failed, and that does not last (#3063).
  5289. ftp_transfer_failed = False
  5290. # The narrower case of the above: the directory walk *listed* a file
  5291. # for this print and could not fetch it, so the file is on the card
  5292. # under a name that matches. The only failure that rules out an
  5293. # earlier run's 3MF standing in for it (#3307).
  5294. walk_transfer_failed = False
  5295. # Paths the sweep reached and could not transfer -- a timeout, not a
  5296. # 550. The directory walk lists the same folders and would download
  5297. # the same file again: on a slow A1 that was a second 300 s on the
  5298. # printer's one FTPS slot, at print start, for a file the recovery
  5299. # retry fetches with a deadline twice as long (#3308).
  5300. sweep_failed_paths: set[str] = set()
  5301. # The print's name, for a fallback archive whose `subtask_name` the
  5302. # plate guard below had to disown. Display only, and deliberately kept
  5303. # apart from `subtask_name`: that variable is what every file lookup
  5304. # here is built from, and once a name has been shown to fetch another
  5305. # plate's 3MF it must not key `_active_prints` either, or the cover
  5306. # endpoint hands the same contradicted file to
  5307. # `_recover_fallback_archive` and fills the row in with it (#3126).
  5308. display_name_after_plate_reject: str | None = None
  5309. # Get FTP retry settings
  5310. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  5311. # ...but "the printer put it on eMMC" is where it went, not whether we
  5312. # can read it. An H2D with a card in mirrors the job to /cache and
  5313. # serves it happily, and skipping on the URL alone cost that reporter
  5314. # every archive for two days (#2856). So ask the printer instead of
  5315. # guessing: the dispatch named the exact file, which is one connection
  5316. # walking five paths rather than the sweep's ~110. Only when the probe
  5317. # comes back empty does the verdict's reason stand.
  5318. if not storage.reachable and not downloaded_filename and storage.probe_filename:
  5319. if ftps_handshake_blocked(printer.ip_address):
  5320. # Deliberately NOT recorded as a cool-off give-up. This branch
  5321. # only runs on an unreachable verdict, and that verdict is the
  5322. # honest, permanent reason the archive is empty — the probe was
  5323. # a long shot on top of it. Blaming the cool-off here would
  5324. # schedule a retry for a file sitting on internal eMMC, which is
  5325. # the sweep #2780 removed (#2957).
  5326. logger.debug(
  5327. "Not probing for %s on printer %s: its file service is not answering over TLS",
  5328. storage.probe_filename,
  5329. printer_id,
  5330. )
  5331. else:
  5332. probe_path = app_settings.archive_dir / "temp" / storage.probe_filename
  5333. probe_path.parent.mkdir(parents=True, exist_ok=True)
  5334. try:
  5335. probe_hit = await download_file_try_paths_async(
  5336. printer.ip_address,
  5337. printer.access_code,
  5338. ftp_probe_paths(storage.probe_filename),
  5339. probe_path,
  5340. socket_timeout=ftp_timeout,
  5341. printer_model=printer.model,
  5342. )
  5343. except Exception as e:
  5344. logger.debug("3MF probe for %s failed: %s", storage.probe_filename, e)
  5345. probe_hit = False
  5346. if probe_hit:
  5347. downloaded_filename = storage.probe_filename
  5348. temp_path = probe_path
  5349. cache_3mf_download(printer_id, downloaded_filename, probe_path)
  5350. # Naming the path, not just the file: a printer that keeps
  5351. # uploads around for weeks can serve a same-named copy of an
  5352. # earlier slice, and without the directory in the log that
  5353. # mismatch is invisible rather than merely rare (#1820).
  5354. logger.info(
  5355. "Found %s at %s over FTPS for printer %s even though the printer reported %s",
  5356. downloaded_filename,
  5357. probe_hit,
  5358. printer_id,
  5359. storage.reason,
  5360. )
  5361. if not storage.reachable and not downloaded_filename:
  5362. # Same opening words whether or not a probe ran, because that is
  5363. # the phrase support asks people to grep for — only the tail says
  5364. # which of the two happened.
  5365. logger.info(
  5366. "Skipping the 3MF lookup for printer %s: %s — %s",
  5367. printer_id,
  5368. storage.reason,
  5369. "no copy of it on external storage either"
  5370. if storage.probe_filename
  5371. else "the print file is not on storage Bambuddy can read over FTPS, so no path would find it",
  5372. )
  5373. for try_filename in possible_names if not downloaded_filename and storage.reachable else []:
  5374. if not try_filename.endswith(".3mf"):
  5375. continue
  5376. # Root (/) is where BambuStudio/OrcaSlicer uploads land on A1/P1-series
  5377. # printers, so try it first — deferring it to last cost #972's reporter
  5378. # ~48 minutes of retries on /cache//model//data//data/Metadata before
  5379. # landing on the path that actually had the file.
  5380. remote_paths = [
  5381. f"/{try_filename}",
  5382. f"/cache/{try_filename}",
  5383. f"/model/{try_filename}",
  5384. f"/data/{try_filename}",
  5385. f"/data/Metadata/{try_filename}",
  5386. ]
  5387. temp_path = app_settings.archive_dir / "temp" / try_filename
  5388. temp_path.parent.mkdir(parents=True, exist_ok=True)
  5389. for remote_path in remote_paths:
  5390. if ftps_handshake_blocked(printer.ip_address):
  5391. # The printer's FTPS service is not completing a TLS
  5392. # handshake, so it has no path we could reach — walking the
  5393. # remaining candidates only re-runs the same failure
  5394. # (#2780). Fall through to the no-3MF archive now.
  5395. #
  5396. # Remember *why*, though. This is the one give-up that is
  5397. # temporary: the cool-off clears in minutes and the file was
  5398. # on the printer the whole time. The fallback archive is
  5399. # stamped with it so a retry can be scheduled, and so the
  5400. # Archives banner stops blaming storage (#2957).
  5401. blocked_by_ftps_cooloff = True
  5402. logger.warning(
  5403. "Giving up on the 3MF for printer %s: its file service is not answering over TLS",
  5404. printer_id,
  5405. )
  5406. break
  5407. logger.debug("Trying FTP download: %s", remote_path)
  5408. try:
  5409. if ftp_retry_enabled:
  5410. downloaded = await with_ftp_retry(
  5411. download_file_async,
  5412. printer.ip_address,
  5413. printer.access_code,
  5414. remote_path,
  5415. temp_path,
  5416. timeout=ftp_timeout,
  5417. socket_timeout=ftp_timeout,
  5418. printer_model=printer.model,
  5419. max_retries=ftp_retry_count,
  5420. retry_delay=ftp_retry_delay,
  5421. operation_name=f"Download 3MF from {remote_path}",
  5422. cooloff_ip=printer.ip_address,
  5423. non_retry_exceptions=(FileNotOnPrinterError,),
  5424. )
  5425. else:
  5426. downloaded = await download_file_async(
  5427. printer.ip_address,
  5428. printer.access_code,
  5429. remote_path,
  5430. temp_path,
  5431. timeout=ftp_timeout,
  5432. socket_timeout=ftp_timeout,
  5433. printer_model=printer.model,
  5434. )
  5435. if downloaded:
  5436. downloaded_filename = try_filename
  5437. logger.info("Downloaded: %s", remote_path)
  5438. # Populate shared cache so the cover endpoint (if it
  5439. # runs next) doesn't refetch the same 36MB over FTP.
  5440. cache_3mf_download(printer_id, try_filename, temp_path)
  5441. break
  5442. # with_ftp_retry returns None once it has spent its budget,
  5443. # and download_file_async returns False on a timeout, so an
  5444. # exhausted transfer arrives here rather than as an
  5445. # exception (#3063).
  5446. ftp_transfer_failed = True
  5447. sweep_failed_paths.add(remote_path)
  5448. except FileNotOnPrinterError:
  5449. # 550 — file isn't at this path. Advance to next candidate
  5450. # without burning the retry budget.
  5451. logger.debug("3MF not at %s (550), trying next path", remote_path)
  5452. except Exception as e:
  5453. ftp_transfer_failed = True
  5454. sweep_failed_paths.add(remote_path)
  5455. logger.debug("FTP download failed for %s: %s", remote_path, e)
  5456. if downloaded_filename or ftps_handshake_blocked(printer.ip_address):
  5457. break
  5458. # If still not found, try listing directories to find matching file
  5459. # Different printer models use different directory structures. Skipped
  5460. # when the printer's FTPS handshake is failing — the directory walk is
  5461. # five more connections that cannot get further than the download did.
  5462. if (
  5463. not downloaded_filename
  5464. and storage.reachable
  5465. and (filename or subtask_name)
  5466. and not ftps_handshake_blocked(printer.ip_address)
  5467. ):
  5468. search_term = (subtask_name or filename).lower().replace(".gcode", "").replace(".3mf", "")
  5469. logger.info("Direct FTP download failed, searching directories for '%s'", search_term)
  5470. search_dirs = ["/cache", "/model", "/data", "/data/Metadata", "/"]
  5471. # Every folder is listed before one file is picked, so a file named
  5472. # for the print in a later folder wins over one that merely
  5473. # contains the name in an earlier folder.
  5474. listed_paths: list[str] = []
  5475. for search_dir in search_dirs:
  5476. try:
  5477. dir_files = await list_files_async(
  5478. printer.ip_address, printer.access_code, search_dir, printer_model=printer.model
  5479. )
  5480. except Exception as e:
  5481. logger.debug("Failed to list %s: %s", search_dir, e)
  5482. continue
  5483. threemf_files = [f.get("name") for f in dir_files if f.get("name", "").endswith(".3mf")]
  5484. if threemf_files:
  5485. logger.info(
  5486. f"Found {len(threemf_files)} 3MF files in {search_dir}: {threemf_files[:5]}{'...' if len(threemf_files) > 5 else ''}"
  5487. )
  5488. listed_paths += [
  5489. posixpath.join(search_dir, f.get("name", "")) for f in dir_files if not f.get("is_directory")
  5490. ]
  5491. for remote_full_path in _3mf_files_for_print(listed_paths, search_term):
  5492. if remote_full_path in sweep_failed_paths:
  5493. # Listed for this print, and its transfer already failed:
  5494. # the same as a failed walk download, without spending a
  5495. # second deadline on it. Skipping it cannot hand the walk a
  5496. # look-alike: while a file of the print's own name is
  5497. # listed, files that merely contain the name are not.
  5498. logger.info(
  5499. "Not downloading %s again: its transfer already failed; the recovery retry will fetch it",
  5500. remote_full_path,
  5501. )
  5502. walk_transfer_failed = True
  5503. continue
  5504. fname = posixpath.basename(remote_full_path)
  5505. logger.info("Found matching file: %s", remote_full_path)
  5506. temp_path = app_settings.archive_dir / "temp" / fname
  5507. temp_path.parent.mkdir(parents=True, exist_ok=True)
  5508. try:
  5509. if ftp_retry_enabled:
  5510. downloaded = await with_ftp_retry(
  5511. download_file_async,
  5512. printer.ip_address,
  5513. printer.access_code,
  5514. remote_full_path,
  5515. temp_path,
  5516. timeout=ftp_timeout,
  5517. socket_timeout=ftp_timeout,
  5518. printer_model=printer.model,
  5519. max_retries=ftp_retry_count,
  5520. retry_delay=ftp_retry_delay,
  5521. operation_name=f"Download 3MF from {remote_full_path}",
  5522. cooloff_ip=printer.ip_address,
  5523. )
  5524. else:
  5525. downloaded = await download_file_async(
  5526. printer.ip_address,
  5527. printer.access_code,
  5528. remote_full_path,
  5529. temp_path,
  5530. timeout=ftp_timeout,
  5531. socket_timeout=ftp_timeout,
  5532. printer_model=printer.model,
  5533. )
  5534. except Exception as e:
  5535. logger.debug("Failed to download %s: %s", remote_full_path, e)
  5536. continue
  5537. if downloaded:
  5538. downloaded_filename = fname
  5539. logger.info("Found and downloaded: %s", remote_full_path)
  5540. cache_3mf_download(printer_id, fname, temp_path)
  5541. break
  5542. # The listing named the file, so it is on the card;
  5543. # only the transfer failed (#3063).
  5544. ftp_transfer_failed = True
  5545. walk_transfer_failed = True
  5546. # A "Print again" on the printer's screen names no file Bambuddy can
  5547. # find: only its history copy under a generic name, and the 3MF's
  5548. # internal title (#3307). When an earlier run of the same job was
  5549. # archived, its 3MF is the one being printed again. Not when the
  5550. # directory walk listed a file for this print and only its transfer
  5551. # failed: that file is on the card, and the retry fetches it. A
  5552. # cool-off or a failed sweep path is no such evidence -- the retry
  5553. # tries the same names, which this print's file never had.
  5554. reused_3mf_from: int | None = None
  5555. if not downloaded_filename and not walk_transfer_failed:
  5556. reported_layers = raw_mqtt.get("total_layer_num")
  5557. earlier = await _archived_3mf_for_repeat(
  5558. db,
  5559. printer_id,
  5560. subtask_name,
  5561. filename,
  5562. reported_layers if isinstance(reported_layers, int) and reported_layers > 0 else None,
  5563. )
  5564. if earlier is not None:
  5565. reuse_name = Path(earlier.filename or "").name or f"archive_{earlier.id}.3mf"
  5566. # Per printer: two printers reprinting the same job at once
  5567. # must not write one temp file (archive_print names the
  5568. # archive after original_filename, not this path).
  5569. reuse_path = app_settings.archive_dir / "temp" / f"reprint_{printer_id}_{reuse_name}"
  5570. try:
  5571. reuse_path.parent.mkdir(parents=True, exist_ok=True)
  5572. await asyncio.to_thread(shutil.copyfile, app_settings.base_dir / earlier.file_path, reuse_path)
  5573. except OSError as e:
  5574. # A copy that fails leaves the print the fallback archive
  5575. # it would have had, not no archive at all.
  5576. logger.warning("Could not copy the 3MF of archive %s for printer %s: %s", earlier.id, printer_id, e)
  5577. else:
  5578. temp_path = reuse_path
  5579. downloaded_filename = reuse_name
  5580. reused_3mf_from = earlier.id
  5581. cache_3mf_download(printer_id, reuse_name, temp_path)
  5582. logger.info(
  5583. "No 3MF on printer %s for %s (%s); using the 3MF of archive %s, an earlier run of the same job",
  5584. printer_id,
  5585. subtask_name,
  5586. filename,
  5587. earlier.id,
  5588. )
  5589. # Validate the downloaded 3MF actually matches the plate that's running
  5590. # (#1204): subtask_name lags across consecutive plates of the same model,
  5591. # so the first FTP candidate (built from subtask_name) can land on the
  5592. # previous plate's still-resident upload. Cross-check the slice_info
  5593. # plate index against the plate parsed from gcode_file (always fresh —
  5594. # it's the field whose change triggered this callback).
  5595. if downloaded_filename and temp_path:
  5596. expected_plate = parse_plate_id(filename)
  5597. actual_plate = peek_plate_index_in_3mf(temp_path) if expected_plate is not None else None
  5598. if expected_plate is not None and actual_plate is not None and actual_plate != expected_plate:
  5599. logger.warning(
  5600. "[CALLBACK] 3MF plate mismatch: downloaded %s reports plate %s but printer is "
  5601. "running plate %s — subtask_name=%r appears stale, retrying with corrected name",
  5602. downloaded_filename,
  5603. actual_plate,
  5604. expected_plate,
  5605. subtask_name,
  5606. )
  5607. corrected_subtask = swap_plate_suffix(subtask_name, expected_plate)
  5608. retry_succeeded = False
  5609. if corrected_subtask and corrected_subtask != subtask_name:
  5610. for try_filename in (f"{corrected_subtask}.gcode.3mf", f"{corrected_subtask}.3mf"):
  5611. retry_temp_path = app_settings.archive_dir / "temp" / try_filename
  5612. retry_temp_path.parent.mkdir(parents=True, exist_ok=True)
  5613. for remote_path in (
  5614. f"/{try_filename}",
  5615. f"/cache/{try_filename}",
  5616. f"/model/{try_filename}",
  5617. f"/data/{try_filename}",
  5618. f"/data/Metadata/{try_filename}",
  5619. ):
  5620. try:
  5621. if ftp_retry_enabled:
  5622. downloaded = await with_ftp_retry(
  5623. download_file_async,
  5624. printer.ip_address,
  5625. printer.access_code,
  5626. remote_path,
  5627. retry_temp_path,
  5628. timeout=ftp_timeout,
  5629. socket_timeout=ftp_timeout,
  5630. printer_model=printer.model,
  5631. max_retries=ftp_retry_count,
  5632. retry_delay=ftp_retry_delay,
  5633. operation_name=f"Re-download 3MF from {remote_path}",
  5634. cooloff_ip=printer.ip_address,
  5635. non_retry_exceptions=(FileNotOnPrinterError,),
  5636. )
  5637. else:
  5638. downloaded = await download_file_async(
  5639. printer.ip_address,
  5640. printer.access_code,
  5641. remote_path,
  5642. retry_temp_path,
  5643. timeout=ftp_timeout,
  5644. socket_timeout=ftp_timeout,
  5645. printer_model=printer.model,
  5646. )
  5647. if downloaded and peek_plate_index_in_3mf(retry_temp_path) == expected_plate:
  5648. logger.info(
  5649. "[CALLBACK] Re-download succeeded with corrected name %s "
  5650. "(plate %s) — replacing wrong file",
  5651. try_filename,
  5652. expected_plate,
  5653. )
  5654. try:
  5655. temp_path.unlink(missing_ok=True)
  5656. except OSError:
  5657. pass
  5658. temp_path = retry_temp_path
  5659. downloaded_filename = try_filename
  5660. subtask_name = corrected_subtask
  5661. cache_3mf_download(printer_id, try_filename, temp_path)
  5662. retry_succeeded = True
  5663. break
  5664. elif downloaded:
  5665. # Wrong plate again — discard and keep trying
  5666. try:
  5667. retry_temp_path.unlink(missing_ok=True)
  5668. except OSError:
  5669. pass
  5670. except FileNotOnPrinterError:
  5671. continue
  5672. except Exception as e:
  5673. logger.debug("Re-download failed for %s: %s", remote_path, e)
  5674. if retry_succeeded:
  5675. break
  5676. # If the retry didn't find a matching file, drop the wrong 3MF
  5677. # so the no-3MF fallback below creates an archive whose name
  5678. # at least reflects the right plate.
  5679. if not retry_succeeded:
  5680. logger.warning(
  5681. "[CALLBACK] Could not re-download correct plate %s — falling back to no-3MF archive",
  5682. expected_plate,
  5683. )
  5684. try:
  5685. temp_path.unlink(missing_ok=True)
  5686. except OSError:
  5687. pass
  5688. temp_path = None
  5689. downloaded_filename = None
  5690. # Whatever the sweep's transport did earlier, it is not why
  5691. # this archive ends up empty: a 3MF downloaded fine, it was
  5692. # just the wrong plate. Retrying would re-fetch that same
  5693. # contradicted file under the same stale names and hand it
  5694. # to _recover_fallback_archive, which checks that a
  5695. # candidate is a readable 3MF but not which plate it is --
  5696. # so the row would be filled in with another plate's
  5697. # filament and cost, the exact swap #2957 removed (#3063).
  5698. ftp_transfer_failed = False
  5699. # Disown the name for *lookups*: it has just been shown to
  5700. # fetch another plate's 3MF, and it keys `_active_prints`
  5701. # below, where the cover endpoint's own download of that
  5702. # same name would find this archive and fill it in with the
  5703. # file we are discarding here.
  5704. #
  5705. # Keep it for the *title*, which is a separate question.
  5706. # ``swap_plate_suffix`` returns None both for a name that
  5707. # carries no "- Plate N" / "_plate_N" suffix and for no
  5708. # name at all, and those are not the same situation: a name
  5709. # without a suffix holds no stale plate number to be wrong
  5710. # about. Blanking both uses at once dropped the project
  5711. # name too, and the row fell through to the gcode_file path
  5712. # titled "plate_1" though the real name was in hand.
  5713. # #1204's own premise is consecutive plates *of the same
  5714. # model*, so the project part is right either way (#3126).
  5715. display_name_after_plate_reject = corrected_subtask or subtask_name or None
  5716. subtask_name = corrected_subtask or ""
  5717. # Deleted while the 3MF was being fetched (that can take minutes): no
  5718. # archive, timelapse or tracking for a printer that will never report
  5719. # the print's end. forget_printer again: the download registered its
  5720. # 3MF cache entry after the delete cleared the id. (A delete committing
  5721. # while the archive is written is caught after its commit.)
  5722. if await _printer_gone(db, printer_id):
  5723. logger.info("Printer %s was deleted during the print-start download; not archiving", printer_id)
  5724. await forget_printer(printer_id)
  5725. return
  5726. if not downloaded_filename or not temp_path:
  5727. logger.warning("Could not find 3MF file for print: %s", filename or subtask_name)
  5728. # Create a fallback archive without 3MF data so the print is still tracked
  5729. # This commonly happens with P1S/A1 printers where FTP has file size limitations
  5730. try:
  5731. from backend.app.models.archive import PrintArchive
  5732. # Why the card is empty. The two temporary causes outrank the
  5733. # storage verdict because they say the sweep never got a fair
  5734. # answer: a cool-off skipped it at the transport, and a failed
  5735. # transfer reached the printer but never finished. Either way
  5736. # the file is still on the card, so reporting where the printer
  5737. # files its jobs would describe a setting that is not the
  5738. # problem (#2957, #3063).
  5739. if blocked_by_ftps_cooloff:
  5740. no_3mf_reason = REASON_FTPS_COOLOFF
  5741. elif storage.reachable and ftp_transfer_failed:
  5742. no_3mf_reason = REASON_FTP_TRANSFER_FAILED
  5743. else:
  5744. no_3mf_reason = storage.reason
  5745. # Derive print name from subtask_name or filename. The
  5746. # plate guard's disowned name comes second: it is a real name
  5747. # for a real print, and only the gcode_file path is left
  5748. # otherwise -- which titles the row "plate_1" (#3126).
  5749. print_name = subtask_name or display_name_after_plate_reject or filename
  5750. if print_name:
  5751. # Clean up the name (remove extensions, path parts)
  5752. print_name = print_name.split("/")[-1]
  5753. print_name = print_name.replace(".gcode.3mf", "").replace(".gcode", "").replace(".3mf", "")
  5754. else:
  5755. print_name = "Unknown Print"
  5756. # Recover estimated print time from MQTT (best-effort for notifications)
  5757. fallback_print_time = None
  5758. mqtt_remaining = data.get("remaining_time")
  5759. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  5760. fallback_print_time = int(mqtt_remaining)
  5761. if fallback_print_time is None:
  5762. mc_remaining = (data.get("raw_data") or {}).get("mc_remaining_time")
  5763. if mc_remaining and isinstance(mc_remaining, (int, float)) and mc_remaining > 0:
  5764. fallback_print_time = int(mc_remaining * 60)
  5765. # Best-effort filament metadata from MQTT — see
  5766. # _extract_filament_data_from_mqtt. Without this the fallback
  5767. # archive's filament fields stayed NULL even though the AMS
  5768. # state at print start was sitting right there in `data`.
  5769. # The slicer's ams_mapping (when present) narrows the result
  5770. # to slots actually used by the print (#1533).
  5771. mqtt_filament_meta = _extract_filament_data_from_mqtt(
  5772. data, _get_start_ams_mapping(data, printer_id, None)
  5773. )
  5774. # Create minimal archive entry
  5775. fallback_archive = PrintArchive(
  5776. printer_id=printer_id,
  5777. filename=filename or f"{print_name}.3mf",
  5778. file_path="", # Empty - no 3MF file available
  5779. file_size=0,
  5780. print_name=print_name,
  5781. print_time_seconds=fallback_print_time,
  5782. status="printing",
  5783. started_at=detected_at,
  5784. subtask_id=subtask_id,
  5785. confirm_requested=await _ask_outcome_for_external_print(db, printer_id, subtask_name),
  5786. filament_type=mqtt_filament_meta.get("filament_type"),
  5787. filament_color=mqtt_filament_meta.get("filament_color"),
  5788. extra_data={
  5789. "no_3mf_available": True,
  5790. # Why the card is empty, when we know -- see above. The
  5791. # banner reads this to stop telling H2/P2 owners to
  5792. # switch on a setting that is already on and would not
  5793. # have helped (#2780).
  5794. "no_3mf_reason": no_3mf_reason,
  5795. "original_subtask": subtask_name or display_name_after_plate_reject or "",
  5796. "_print_data": data,
  5797. },
  5798. )
  5799. db.add(fallback_archive)
  5800. try:
  5801. await db.commit()
  5802. except IntegrityError:
  5803. # Not "Failed to create fallback archive" below, and no
  5804. # start notification for a printer that is gone.
  5805. if await _refused_for_deleted_printer(db, printer_id, "its fallback archive was written"):
  5806. return
  5807. raise
  5808. await db.refresh(fallback_archive)
  5809. logger.info("Created fallback archive %s for %s (no 3MF available)", fallback_archive.id, print_name)
  5810. if await _drop_archive_of_deleted_printer(db, printer_id, fallback_archive.id):
  5811. return
  5812. _maybe_start_layer_timelapse(printer, printer_id, fallback_archive.id)
  5813. # Track as active print
  5814. _bind_active_print(printer_id, fallback_archive.id, fallback_archive.filename, filename, subtask_name)
  5815. await _open_run_for(fallback_archive.id, printer_id, subtask_id)
  5816. # Record starting energy if smart plug available (#941: persisted column)
  5817. await _record_energy_start(fallback_archive, printer_id, db, context="fallback")
  5818. # Send WebSocket notification
  5819. await ws_manager.send_archive_created(
  5820. {
  5821. "id": fallback_archive.id,
  5822. "printer_id": fallback_archive.printer_id,
  5823. "filename": fallback_archive.filename,
  5824. "print_name": fallback_archive.print_name,
  5825. "status": fallback_archive.status,
  5826. }
  5827. )
  5828. # MQTT relay - publish archive created
  5829. try:
  5830. await mqtt_relay.on_archive_created(
  5831. archive_id=fallback_archive.id,
  5832. print_name=fallback_archive.print_name,
  5833. printer_name=printer.name,
  5834. status=fallback_archive.status,
  5835. )
  5836. except Exception:
  5837. pass # Don't fail if MQTT fails
  5838. # Store Spoolman tracking data (may not work for fallback since no 3MF)
  5839. try:
  5840. await _store_spoolman_print_data(
  5841. printer_id,
  5842. fallback_archive.id,
  5843. fallback_archive.file_path,
  5844. db,
  5845. printer_manager,
  5846. ams_mapping=_get_start_ams_mapping(data, printer_id, fallback_archive.id),
  5847. plate_id=_get_start_plate_id(printer_id, fallback_archive.id),
  5848. )
  5849. except Exception as e:
  5850. logger.debug("[SPOOLMAN] Could not store tracking for fallback archive: %s", e)
  5851. # Both temporary give-ups are worth coming back for, and for
  5852. # the same reason: the file is on the printer and the last look
  5853. # failed at the transport rather than finding nothing. One waits
  5854. # out the handshake block (#2957), the other waits for the
  5855. # printer to stop being busy (#3063). Deliberately not scheduled
  5856. # for a storage verdict: a file on internal eMMC will not appear
  5857. # at any FTPS path however long we wait, and retrying it is
  5858. # exactly the sweep #2780 removed.
  5859. if no_3mf_reason in (REASON_FTPS_COOLOFF, REASON_FTP_TRANSFER_FAILED) and possible_names:
  5860. # `possible_names`, not the raw MQTT strings: it is the exact
  5861. # list this flow just tried, already stripped of any path
  5862. # (`filename` arrives as "/data/Metadata/plate_1.gcode" on
  5863. # some firmware) and deduped.
  5864. _schedule_fallback_3mf_retry(
  5865. printer_id=printer_id,
  5866. archive_id=fallback_archive.id,
  5867. filenames=list(possible_names),
  5868. reason=no_3mf_reason,
  5869. )
  5870. # Send notification without archive data (file not found)
  5871. if not notification_sent:
  5872. await _send_print_start_notification(printer_id, data, logger=logger)
  5873. # The same baseline the other two on_print_start branches take
  5874. # (#2704), and last for the same reason they are: it lists the
  5875. # printer's timelapse directory, so a slow card must not delay
  5876. # the _active_prints registration, the energy reading, the
  5877. # archive-created event or the start notification above it.
  5878. #
  5879. # This branch never took one, so every no-3MF archive reached
  5880. # completion with no baseline in memory and none on the row, and
  5881. # the completion scan fell into its "snapshot now" fallback --
  5882. # which runs after the printer has written the video, so the new
  5883. # file landed inside the baseline and no diff ever matched
  5884. # (#2957 follow-up).
  5885. #
  5886. # Skipped when the FTPS cool-off is what produced this fallback:
  5887. # the listing needs the same connection that just failed, so it
  5888. # could only record that the card was unreadable. The scan
  5889. # handles that case by refusing to choose between candidates.
  5890. if not blocked_by_ftps_cooloff:
  5891. await _capture_timelapse_baseline_at_start(
  5892. printer, printer_id, logger, archive_id=fallback_archive.id
  5893. )
  5894. await _forget_if_printer_gone(printer_id)
  5895. return
  5896. except Exception as e:
  5897. logger.error("Failed to create fallback archive: %s", e)
  5898. # Send notification without archive data (file not found)
  5899. if not notification_sent:
  5900. await _send_print_start_notification(printer_id, data, logger=logger)
  5901. return
  5902. try:
  5903. # Archive the file with status "printing"
  5904. service = ArchiveService(db)
  5905. try:
  5906. archive = await service.archive_print(
  5907. printer_id=printer_id,
  5908. source_file=temp_path,
  5909. print_data={
  5910. **data,
  5911. "status": "printing",
  5912. # Where the 3MF came from when the printer had none to
  5913. # give (#3307), for whoever reads this archive later.
  5914. **({"_3mf_from_archive": reused_3mf_from} if reused_3mf_from is not None else {}),
  5915. },
  5916. subtask_id=subtask_id,
  5917. started_at=detected_at,
  5918. # The reused copy's temp name carries the printer id; a
  5919. # download's is the file's own name, as before.
  5920. original_filename=downloaded_filename if reused_3mf_from is not None else None,
  5921. )
  5922. except IntegrityError:
  5923. # archive_print removed the folder it had copied the 3MF into.
  5924. if await _refused_for_deleted_printer(db, printer_id, "its archive was written"):
  5925. return
  5926. raise
  5927. if archive:
  5928. # Ask-for-outcome for a print Bambuddy did not dispatch (#1898).
  5929. # Set on the row rather than passed to archive_print, which also
  5930. # serves the queue dispatcher — there the queue item decides.
  5931. # Guarded because this branch has no ``except``: an unhandled
  5932. # write error would take the _active_prints registration, the
  5933. # start notification, the energy reading and the timelapse
  5934. # baseline below it with it, and a missing prompt is by far the
  5935. # cheaper failure.
  5936. archive_id = archive.id
  5937. try:
  5938. if await _ask_outcome_for_external_print(db, printer_id, subtask_name):
  5939. archive.confirm_requested = True
  5940. await db.commit()
  5941. except Exception as e:
  5942. logger.warning("Could not flag archive %s for the outcome prompt: %s", archive_id, e)
  5943. # The rollback expires every loaded object, and on an async
  5944. # session reading one afterwards raises instead of
  5945. # reloading, so the two this branch goes on to use are
  5946. # fetched again. The archive itself was committed by
  5947. # archive_print; only the flag is lost.
  5948. try:
  5949. await db.rollback()
  5950. archive = await db.get(PrintArchive, archive_id)
  5951. printer = await db.get(Printer, printer_id)
  5952. except Exception as reload_error:
  5953. logger.warning(
  5954. "Could not reload archive %s after the failed flag write: %s", archive_id, reload_error
  5955. )
  5956. if await _drop_archive_of_deleted_printer(db, printer_id, archive_id):
  5957. return
  5958. # Track this active print (use both original filename and downloaded filename)
  5959. _bind_active_print(printer_id, archive_id, downloaded_filename, filename, subtask_name)
  5960. await _open_run_for(archive_id, printer_id, subtask_id)
  5961. logger.info("Created archive %s for %s", archive.id, downloaded_filename)
  5962. _maybe_start_layer_timelapse(printer, printer_id, archive.id)
  5963. # Record starting energy from smart plug if available (#941: persisted column)
  5964. await _record_energy_start(archive, printer_id, db, context="auto-archive")
  5965. await ws_manager.send_archive_created(
  5966. {
  5967. "id": archive.id,
  5968. "printer_id": archive.printer_id,
  5969. "filename": archive.filename,
  5970. "print_name": archive.print_name,
  5971. "status": archive.status,
  5972. }
  5973. )
  5974. # MQTT relay - publish archive created
  5975. try:
  5976. await mqtt_relay.on_archive_created(
  5977. archive_id=archive.id,
  5978. print_name=archive.print_name,
  5979. printer_name=printer.name,
  5980. status=archive.status,
  5981. )
  5982. except Exception:
  5983. pass # Don't fail if MQTT fails
  5984. # Send notification with archive data (new archive created)
  5985. if not notification_sent:
  5986. archive_data = {
  5987. "print_time_seconds": _run_print_time_seconds(archive, printer_id),
  5988. "created_by_id": archive.created_by_id,
  5989. }
  5990. await _send_print_start_notification(printer_id, data, archive_data, logger)
  5991. # Extract printable objects for skip object functionality
  5992. try:
  5993. from backend.app.services.archive import extract_printable_objects_from_3mf
  5994. client = printer_manager.get_client(printer_id)
  5995. if client:
  5996. with open(temp_path, "rb") as f:
  5997. threemf_data = f.read()
  5998. # Extract with positions for UI overlay, scoped to the
  5999. # plate that is printing — an all-plates 3MF carries
  6000. # every plate's objects (#2522).
  6001. printable_objects, bbox_all = extract_printable_objects_from_3mf(
  6002. threemf_data,
  6003. plate_number=resolve_plate_id(client.state),
  6004. include_positions=True,
  6005. )
  6006. if printable_objects:
  6007. # Store objects in printer state
  6008. client.state.printable_objects = printable_objects
  6009. client.state.printable_objects_bbox_all = bbox_all
  6010. client.state.skipped_objects = [] # Reset skipped objects for new print
  6011. logger.info(
  6012. "Loaded %s printable objects for printer %s", len(printable_objects), printer_id
  6013. )
  6014. except Exception as e:
  6015. logger.debug("Failed to extract printable objects: %s", e)
  6016. # From the archived copy: the downloaded one may be removed
  6017. # below while the pauses are still being read.
  6018. if archive is not None and archive.file_path:
  6019. _load_pauses_from_3mf(app_settings.base_dir / archive.file_path, printer_id)
  6020. # Store Spoolman tracking data for per-filament usage reporting
  6021. try:
  6022. await _store_spoolman_print_data(
  6023. printer_id,
  6024. archive.id,
  6025. archive.file_path,
  6026. db,
  6027. printer_manager,
  6028. ams_mapping=_get_start_ams_mapping(data, printer_id, archive_id),
  6029. plate_id=_get_start_plate_id(printer_id, archive_id),
  6030. )
  6031. except Exception as e:
  6032. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  6033. # Price the print from its spools now rather than only at completion (#3261)
  6034. _schedule_archive_cost_estimate(
  6035. printer_id,
  6036. archive.id,
  6037. printer_manager,
  6038. ams_mapping=_get_start_ams_mapping(data, printer_id, archive_id),
  6039. plate_id=_get_start_plate_id(printer_id, archive_id),
  6040. session_factory=async_session,
  6041. )
  6042. # Capture timelapse file baseline for snapshot-diff on completion
  6043. await _capture_timelapse_baseline_at_start(printer, printer_id, logger, archive_id=archive.id)
  6044. await _forget_if_printer_gone(printer_id)
  6045. finally:
  6046. # Keep temp_path around until print completes so the cover endpoint
  6047. # can reuse it (#972). Cache eviction in on_print_complete deletes
  6048. # the file. If the cache entry was evicted early (file vanished),
  6049. # clean up any stragglers here to avoid leaking disk on retries.
  6050. cached_now = get_cached_3mf(printer_id, downloaded_filename) if downloaded_filename else None
  6051. if temp_path and temp_path.exists() and cached_now != temp_path:
  6052. temp_path.unlink()
  6053. _TIMELAPSE_VIDEO_EXTENSIONS = (".mp4", ".avi")
  6054. _TIMELAPSE_DIRECTORIES = ("/timelapse", "/timelapse/video", "/record", "/recording")
  6055. # A baseline that could not be read at print start is read again on this
  6056. # schedule while the print runs (#3412). The printer writes the video only once
  6057. # the print has ended, so a reading taken mid-print is as good as one taken at
  6058. # the start. The interval is shorter than the FTPS cool-offs (300 s), which are
  6059. # what usually makes the card unreadable, so the first reading after one ends
  6060. # is not long behind; while a cool-off runs a round costs nothing, as no
  6061. # connection is opened. The cap bounds a print that never reports an end.
  6062. #
  6063. # A round that could not read the card doubles the wait before the next, up to
  6064. # the ceiling, and after that many failed rounds in a row the re-read gives up
  6065. # and leaves the baseline unread for the completion scan to handle (#3474). Each
  6066. # connection to a hung file service can leave a session behind on the printer.
  6067. _TIMELAPSE_BASELINE_REREAD_INTERVAL_SECONDS = 120.0
  6068. _TIMELAPSE_BASELINE_REREAD_MAX_INTERVAL_SECONDS = 1800.0
  6069. _TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS = 5
  6070. _TIMELAPSE_BASELINE_REREAD_MAX_ATTEMPTS = 720
  6071. # Poll schedule for the post-print timelapse scan (#2704). Module-level so
  6072. # tests can shrink them without waiting out real delays.
  6073. #
  6074. # This replaced a fixed [5, 10, 20, 30] retry ladder, i.e. roughly 65 s of
  6075. # looking. Across 247 support bundles the attempt that found the video was #1
  6076. # 272 times, then 17 / 13 / 13 — a flat tail against the cutoff rather than a
  6077. # decaying one, which is the signature of a budget that expires while files are
  6078. # still arriving. 457 scans were scheduled and only 262 ever attached. Big
  6079. # prints make big videos and the printer writes them after the print ends, so
  6080. # the poll now runs for minutes and costs one FTP LIST per round.
  6081. _TIMELAPSE_SCAN_FIRST_DELAY_SECONDS: float = 5.0
  6082. _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS: float = 30.0
  6083. _TIMELAPSE_SCAN_TIMEOUT_SECONDS: float = 900.0
  6084. def _timelapse_scan_max_attempts() -> int:
  6085. """Round cap for the poll, derived from the wall-clock budget.
  6086. The deadline alone is not a sufficient bound: it assumes each round really
  6087. waits, which stops being true the moment ``asyncio.sleep`` is patched out,
  6088. and an FTP list that fails immediately would otherwise spin against the
  6089. printer at full speed for the whole window. Whichever bound is reached
  6090. first ends the poll.
  6091. """
  6092. if _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS <= 0:
  6093. # A zero interval makes the wall-clock budget meaningless; fall back to
  6094. # the round count the production interval would have given.
  6095. return 32
  6096. return max(1, int(_TIMELAPSE_SCAN_TIMEOUT_SECONDS // _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS) + 1)
  6097. async def _read_timelapse_baseline(printer) -> tuple[set[str], bool]:
  6098. """The video names on *printer*'s card, and whether the card was read.
  6099. ``_list_timelapse_videos`` answers ``[]`` when its connect fails rather
  6100. than raising, so a card behind the FTPS handshake cool-off, or one whose
  6101. listing timed out, is indistinguishable from one holding no videos.
  6102. Everywhere that only wants to know "is there a video yet" the difference
  6103. does not matter -- both mean "not yet, retry".
  6104. It matters where an empty listing is recorded as a *baseline*. Recording
  6105. "the card held nothing" for a card that was never read means every video on
  6106. it counts as new, and the completion scan then attaches a stale video to
  6107. this print and deletes it from the printer (#2957 follow-up, #3412). So an
  6108. card counts as read only when every directory walked answered, with a
  6109. listing or with "no such directory". One that failed may be the one
  6110. holding the videos.
  6111. One walk, one connection per directory. It used to list the card with
  6112. ``_list_timelapse_videos`` first and walk the same directories again when
  6113. that came back empty, which against a hung file service was eight dead
  6114. connections a round instead of one (#3474). A directory that fails does
  6115. not end the walk unless the failure put the printer in a cool-off: a later
  6116. one holding videos still answers the question.
  6117. """
  6118. from backend.app.services.bambu_ftp import ftps_background_blocked, list_files_result_async
  6119. ip_address = getattr(printer, "ip_address", None)
  6120. printer_id = getattr(printer, "id", None)
  6121. if not ip_address:
  6122. return set(), True
  6123. # No card in the slot: nothing to read, and nothing a video could land on.
  6124. if printer_id is not None and not external_storage_present(printer_manager.get_status(printer_id)):
  6125. return set(), True
  6126. read = True
  6127. for timelapse_path in _TIMELAPSE_DIRECTORIES:
  6128. if ftps_background_blocked(ip_address):
  6129. # Below any path: every remaining one would fail the same way.
  6130. return set(), False
  6131. listing = await list_files_result_async(
  6132. ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  6133. )
  6134. if not (listing.available or listing.missing):
  6135. read = False
  6136. continue
  6137. videos = [
  6138. f
  6139. for f in listing.files
  6140. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  6141. ]
  6142. if videos:
  6143. return {f.get("name", "") for f in videos}, True
  6144. return set(), read
  6145. async def _list_timelapse_videos(printer) -> tuple[list[dict], str | None]:
  6146. """List video files from printer's timelapse directory.
  6147. Finds MP4 (X1/A1 series) and AVI (P1 series) timelapse files.
  6148. Returns (video_files, found_path) where video_files is a list of file dicts
  6149. and found_path is the directory where they were found, or ([], None).
  6150. An empty return does not distinguish "no videos" from "could not read the
  6151. card" — see :func:`_read_timelapse_baseline`, which the two baseline
  6152. callers use instead.
  6153. """
  6154. from backend.app.services.bambu_ftp import ftps_background_blocked, list_files_async
  6155. logger = logging.getLogger(__name__)
  6156. # No card in the slot means no /timelapse to walk — four connections that
  6157. # can only fail, on a path whose failures are swallowed and so would go on
  6158. # costing time silently forever (#2780).
  6159. #
  6160. # ``getattr`` rather than ``printer.id``: every dereference below happens
  6161. # inside the loop's own try/except, so a caller that passed something
  6162. # unexpected used to get an empty listing rather than an exception. Keep
  6163. # that, instead of making this gate the first thing that can raise here.
  6164. printer_id = getattr(printer, "id", None)
  6165. if printer_id is not None and not external_storage_present(printer_manager.get_status(printer_id)):
  6166. logger.debug("[TIMELAPSE] Skipping the scan for printer %s: it reports no external storage", printer_id)
  6167. return [], None
  6168. for timelapse_path in _TIMELAPSE_DIRECTORIES:
  6169. if ftps_background_blocked(getattr(printer, "ip_address", None) or ""):
  6170. # The printer's file service is refusing or hung: every remaining
  6171. # path is another connection it may keep a session for (#3474).
  6172. logger.debug("[TIMELAPSE] Skipping the scan for printer %s: its file service is cooling off", printer_id)
  6173. return [], None
  6174. try:
  6175. found_files = await list_files_async(
  6176. printer.ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  6177. )
  6178. if found_files:
  6179. video_files = [
  6180. f
  6181. for f in found_files
  6182. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  6183. ]
  6184. if video_files:
  6185. return video_files, timelapse_path
  6186. except Exception as e:
  6187. logger.debug("[TIMELAPSE] Path %s failed: %s", timelapse_path, e)
  6188. continue
  6189. return [], None
  6190. async def _capture_timelapse_baseline_at_start(
  6191. printer, printer_id: int, logger: logging.Logger, archive_id: int | None = None
  6192. ) -> None:
  6193. """Snapshot the printer's timelapse directory at print start so the
  6194. completion-time scan can pick the new file by set-difference.
  6195. Must be called from every on_print_start path that proceeds to a real
  6196. print — both the new-archive branch and the expected-archive branch (which
  6197. queue / VP-dispatched prints take). Without a baseline,
  6198. _scan_for_timelapse_with_retries falls into its "take baseline now"
  6199. fallback that runs AFTER the new MP4 has already landed on the SD card,
  6200. so the new file ends up in the "baseline" set and no diff ever matches.
  6201. Bambu printers in LAN-only mode don't sync NTP, so mtime ordering is
  6202. unreliable — the snapshot-diff approach sidesteps that entirely.
  6203. When ``archive_id`` is known the baseline is also written to the archive
  6204. row, so it survives a restart and the manual "Scan for Timelapse" button
  6205. can run the same diff instead of falling back to clock-based matching
  6206. (#2704). Only baselines taken at print start are persisted — one taken at
  6207. completion already contains the new video and would poison a later scan.
  6208. """
  6209. names: set[str] | None = None
  6210. unread = False
  6211. _timelapse_baselines_unread.discard(printer_id)
  6212. try:
  6213. names, read = await _read_timelapse_baseline(printer)
  6214. unread = not read
  6215. _timelapse_baselines[printer_id] = names
  6216. if unread:
  6217. # Recorded anyway, deliberately, and marked. An empty baseline taken
  6218. # off a card we could not read is not evidence, but it is still the
  6219. # right *default*: Bambuddy deletes each video from the printer once
  6220. # it is attached, so the usual card holds exactly one video at
  6221. # completion and an empty baseline resolves it. Persisting NULL
  6222. # instead would send completion to take its own snapshot, by which
  6223. # point this print's video is on the card and would be swallowed by
  6224. # it. The mark keeps the scan from choosing between several videos
  6225. # (#3412), and the card is read again while the print runs.
  6226. _timelapse_baselines_unread.add(printer_id)
  6227. logger.warning(
  6228. "[TIMELAPSE] Baseline for printer %s taken while its card could not be read — "
  6229. "treating it as empty for now and reading it again during the print",
  6230. printer_id,
  6231. )
  6232. logger.info(
  6233. "[TIMELAPSE] Baseline at print start: %s video files for printer %s",
  6234. len(names),
  6235. printer_id,
  6236. )
  6237. except Exception as e:
  6238. logger.warning("[TIMELAPSE] Failed to capture baseline at print start: %s", e)
  6239. run_started_at: datetime | None = None
  6240. run_id: int | None = None
  6241. if archive_id is not None:
  6242. try:
  6243. async with async_session() as db:
  6244. from backend.app.models.archive import PrintArchive
  6245. from backend.app.services.print_log import open_run_on_printer
  6246. archive = await db.get(PrintArchive, archive_id)
  6247. if archive is not None:
  6248. # Written even when the listing failed, and then as NULL. A
  6249. # reprint reuses the archive row, so leaving the previous
  6250. # run's baseline in place would have the scan diff this print
  6251. # against the state of the printer before the *last* one —
  6252. # and a stale baseline reads as authoritative, where NULL
  6253. # correctly falls back to a fresh snapshot.
  6254. baseline = sorted(names) if names is not None else None
  6255. archive.timelapse_baseline = baseline
  6256. archive.timelapse_baseline_unread = unread
  6257. run_started_at = archive.started_at
  6258. # And on the run this print opened, whose scan reads it:
  6259. # the archive's is its latest start's, which another plate
  6260. # of it on another printer can take over.
  6261. run = await open_run_on_printer(db, printer_id)
  6262. if run is not None and run.archive_id == archive_id:
  6263. run.timelapse_baseline = baseline
  6264. run.timelapse_baseline_unread = unread
  6265. run_id = run.id
  6266. await db.commit()
  6267. except Exception as e:
  6268. # In-memory baseline still covers the normal completion path.
  6269. logger.warning("[TIMELAPSE] Failed to persist baseline for archive %s: %s", archive_id, e)
  6270. if unread and names is not None:
  6271. spawn_background_task(
  6272. _reread_unread_timelapse_baseline(printer_id, names, archive_id, run_started_at, run_id),
  6273. name=f"reread-timelapse-baseline-{printer_id}",
  6274. )
  6275. # Before the print has ended there is no video yet: a long PREPARE (heating,
  6276. # levelling) counts as much as the print itself.
  6277. _BASELINE_REREAD_STATES = _ACTIVE_PRINT_STATES | {"PREPARE", "SLICING"}
  6278. def _schedule_timelapse_scan(
  6279. archive_id: int, printer_id: int, *, plate_id: int | None, run_id: int | None = None
  6280. ) -> None:
  6281. """Start the completion scan with the print-start baseline: the names, and
  6282. whether they were taken off a card that could not be read. Taking them
  6283. ends any re-read still running for it. ``run_id`` is the print that
  6284. ended, whose video it looks for."""
  6285. unread = printer_id in _timelapse_baselines_unread
  6286. _timelapse_baselines_unread.discard(printer_id)
  6287. baseline = _timelapse_baselines.pop(printer_id, None)
  6288. from backend.app.services import archive_work
  6289. # Held while it looks: it finds its print by the archive's id, which a
  6290. # merge would move away (archive redesign P2).
  6291. spawn_background_task(
  6292. archive_work.held_while(
  6293. archive_id,
  6294. _scan_for_timelapse_with_retries(
  6295. archive_id, baseline, plate_id=plate_id, baseline_unread=unread, run_id=run_id
  6296. ),
  6297. ),
  6298. name=f"scan-timelapse-{archive_id}",
  6299. )
  6300. def _unread_baseline_round(printer_id: int, names: set[str]) -> bool | None:
  6301. """Whether to read an unread baseline again now: True to read, None to
  6302. skip this round, False once it is over -- the print it was taken for is
  6303. no longer the one on the printer, has stopped printing, or the baseline
  6304. has been read.
  6305. ``names`` is the very set object recorded at print start: completion pops
  6306. it and the next print records a new one, so identity marks the run. A
  6307. printer that is momentarily disconnected says nothing about its print, so
  6308. that only skips the round.
  6309. So does a print that is not recording a timelapse: completion only looks
  6310. for a video when one was recorded, so reading the card for it is a
  6311. connection for nothing (#3474). Skipped rather than ended, as the flag can
  6312. arrive after the print has started.
  6313. """
  6314. if _timelapse_baselines.get(printer_id) is not names or printer_id not in _timelapse_baselines_unread:
  6315. return False
  6316. state = printer_manager.get_status(printer_id)
  6317. if state is None or not state.connected:
  6318. return None
  6319. if state.state not in _BASELINE_REREAD_STATES:
  6320. return False
  6321. return True if state.timelapse else None
  6322. async def _reread_unread_timelapse_baseline(
  6323. printer_id: int,
  6324. names: set[str],
  6325. archive_id: int | None,
  6326. run_started_at: datetime | None,
  6327. run_id: int | None = None,
  6328. ) -> None:
  6329. """Read again a print-start baseline the card could not give (#3412).
  6330. The printer writes the video only once the print has ended, so the card
  6331. read at any point during the print holds exactly what it held at the
  6332. start. Each round stops once the print is no longer the one the baseline
  6333. was taken for or has stopped printing, and checks that again after the
  6334. listing returns, before anything is recorded.
  6335. A round that cannot read the card doubles the wait before the next one,
  6336. and enough of them in a row end the re-read (#3474): see
  6337. ``_TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS``.
  6338. """
  6339. from backend.app.models.archive import PrintArchive
  6340. from backend.app.models.printer import Printer
  6341. from backend.app.services.print_log import run_of
  6342. logger = logging.getLogger(__name__)
  6343. interval = _TIMELAPSE_BASELINE_REREAD_INTERVAL_SECONDS
  6344. failed_rounds = 0
  6345. for _ in range(_TIMELAPSE_BASELINE_REREAD_MAX_ATTEMPTS):
  6346. await asyncio.sleep(interval)
  6347. now = _unread_baseline_round(printer_id, names)
  6348. if now is False:
  6349. return
  6350. if now is None:
  6351. continue
  6352. try:
  6353. async with async_session() as db:
  6354. printer = await db.get(Printer, printer_id)
  6355. if printer is None:
  6356. return
  6357. fresh, read = await _read_timelapse_baseline(printer)
  6358. except Exception as e:
  6359. logger.debug("[TIMELAPSE] Re-reading the baseline for printer %s failed: %s", printer_id, e)
  6360. fresh, read = set(), False
  6361. if not read:
  6362. failed_rounds += 1
  6363. if failed_rounds >= _TIMELAPSE_BASELINE_REREAD_MAX_FAILED_ROUNDS:
  6364. logger.warning(
  6365. "[TIMELAPSE] Card of printer %s still unreadable after %s tries; no longer reading "
  6366. "the baseline for this print, it stays unread",
  6367. printer_id,
  6368. failed_rounds,
  6369. )
  6370. return
  6371. interval = min(interval * 2, _TIMELAPSE_BASELINE_REREAD_MAX_INTERVAL_SECONDS)
  6372. continue
  6373. now = _unread_baseline_round(printer_id, names)
  6374. if now is False:
  6375. return
  6376. if now is None:
  6377. # The printer dropped off while the listing was out, so nothing
  6378. # says it did not end meanwhile: keep the reading for a round that
  6379. # can vouch for it.
  6380. continue
  6381. names.update(fresh)
  6382. _timelapse_baselines_unread.discard(printer_id)
  6383. logger.info(
  6384. "[TIMELAPSE] Baseline for printer %s read during the print: %s video files",
  6385. printer_id,
  6386. len(fresh),
  6387. )
  6388. if archive_id is None:
  6389. return
  6390. try:
  6391. async with async_session() as db:
  6392. archive = await db.get(PrintArchive, archive_id)
  6393. # Only the run it was taken for: by now the row can belong to a
  6394. # reprint, whose baseline is its own.
  6395. if archive is not None and archive.started_at == run_started_at and archive.timelapse_baseline_unread:
  6396. archive.timelapse_baseline = sorted(names)
  6397. archive.timelapse_baseline_unread = False
  6398. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6399. if run is not None and run.timelapse_baseline_unread:
  6400. run.timelapse_baseline = sorted(names)
  6401. run.timelapse_baseline_unread = False
  6402. await db.commit()
  6403. except Exception as e:
  6404. # The in-memory baseline is read now, which covers completion
  6405. # unless Bambuddy restarts first.
  6406. logger.warning("[TIMELAPSE] Failed to persist the re-read baseline for archive %s: %s", archive_id, e)
  6407. return
  6408. async def _timelapse_still_wanted(db, archive, run_id: int | None, run_started_at: datetime | None) -> bool:
  6409. """Whether the print a timelapse is for still has none: its run, by id,
  6410. exists without a video and no later print took over (``superseded``).
  6411. Without a run, the archive is still on that print's start (a newer run's
  6412. video is its own) and has none."""
  6413. from backend.app.services.print_log import run_of, superseded
  6414. if run_id is not None:
  6415. run = await run_of(db, archive.id, run_id)
  6416. return run is not None and run.timelapse_path is None and not await superseded(db, run)
  6417. return archive.timelapse_path is None and archive.started_at == run_started_at
  6418. async def _scan_for_timelapse_with_retries(
  6419. archive_id: int,
  6420. baseline_names: set[str] | None = None,
  6421. *,
  6422. plate_id: int | None = None,
  6423. baseline_unread: bool = False,
  6424. run_id: int | None = None,
  6425. ):
  6426. """Poll the printer for this print's timelapse and attach it.
  6427. Snapshot diff, not timestamp matching: a printer in LAN-only mode cannot
  6428. reach Bambu's NTP server, so the clock behind both the filename and the FTP
  6429. mtime is arbitrarily wrong — one reporter's P1S was six and a half days out
  6430. (#2704). Comparing the current listing against the set of filenames that
  6431. existed when the print started needs no clock at all, because the printer
  6432. writes the video only once the print has ended.
  6433. Baseline precedence: the caller's in-memory set, then the one persisted on
  6434. the archive at print start, then a snapshot taken now. The last of those is
  6435. a poor substitute — by completion the new video may already be on the card,
  6436. in which case it lands in the "baseline" and no diff can ever match — but it
  6437. is all that is available for a print that began before Bambuddy started.
  6438. On success the video is deleted from the printer, which keeps ``/timelapse``
  6439. down to the unclaimed files and makes the next diff unambiguous.
  6440. ``baseline_unread`` marks the caller's baseline as taken off a card that
  6441. could not be read; a stored one carries the same mark on the archive.
  6442. ``run_id`` is the print that ended (archive redesign): the video is that
  6443. run's, the stored baseline is the run's own, and the poll ends once the
  6444. run has a video or is gone. Without it (no run was written), the run is
  6445. whichever the archive is on when the scan starts.
  6446. """
  6447. from backend.app.services.print_log import run_of
  6448. logger = logging.getLogger(__name__)
  6449. # Cleared when the baseline was taken off a card we could not read, so the
  6450. # attach step refuses to choose between several candidates (#2957, #3412).
  6451. baseline_trusted = True
  6452. # --- Phase 1: establish the baseline -------------------------------------
  6453. try:
  6454. async with async_session() as db:
  6455. from backend.app.models.printer import Printer
  6456. service = ArchiveService(db)
  6457. archive = await service.get_archive(archive_id)
  6458. if not archive:
  6459. logger.warning("[TIMELAPSE] Archive %s not found, aborting", archive_id)
  6460. return
  6461. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6462. if run_id is not None and run is None:
  6463. logger.info("[TIMELAPSE] Run %s of archive %s is gone, aborting", run_id, archive_id)
  6464. return
  6465. if run.timelapse_path if run is not None else archive.timelapse_path:
  6466. logger.info("[TIMELAPSE] Archive %s already has timelapse attached", archive_id)
  6467. return
  6468. # The printer the print ran on: plates of one archive can print on
  6469. # two printers, and the archive's is the one that started last.
  6470. scan_printer_id = run.printer_id if run is not None and run.printer_id else archive.printer_id
  6471. if not scan_printer_id:
  6472. logger.warning("[TIMELAPSE] Archive %s has no printer, aborting", archive_id)
  6473. return
  6474. # The run this scan is for. The archive row passes to a newer run
  6475. # when it prints again (the next plate, a reprint), and that run's
  6476. # video is its own.
  6477. run_started_at = run.started_at if run is not None else archive.started_at
  6478. # The baseline stored for it: the run's own, else the archive's
  6479. # while that is still this run's start's.
  6480. stored_baseline, stored_unread = None, False
  6481. if run is not None and run.timelapse_baseline is not None:
  6482. stored_baseline, stored_unread = run.timelapse_baseline, bool(run.timelapse_baseline_unread)
  6483. elif archive.started_at == run_started_at:
  6484. stored_baseline, stored_unread = archive.timelapse_baseline, archive.timelapse_baseline_unread
  6485. if baseline_names is not None and baseline_unread and stored_baseline is not None and not stored_unread:
  6486. # The card was read at print start, and the in-memory copy is
  6487. # one Bambuddy took after a restart off a card it could not
  6488. # read: the stored one is this run's and is evidence.
  6489. baseline_names = set(stored_baseline)
  6490. logger.info(
  6491. "[TIMELAPSE] Using stored baseline over an unread one: %s existing video files for archive %s",
  6492. len(baseline_names),
  6493. archive_id,
  6494. )
  6495. elif baseline_names is not None:
  6496. baseline_trusted = not baseline_unread
  6497. logger.info(
  6498. "[TIMELAPSE] Using print-start baseline: %s existing video files for archive %s",
  6499. len(baseline_names),
  6500. archive_id,
  6501. )
  6502. elif stored_baseline is not None:
  6503. # Persisted at print start — survives a restart mid-print.
  6504. baseline_names = set(stored_baseline)
  6505. baseline_trusted = not stored_unread
  6506. logger.info(
  6507. "[TIMELAPSE] Using stored baseline: %s existing video files for archive %s",
  6508. len(baseline_names),
  6509. archive_id,
  6510. )
  6511. else:
  6512. result = await db.execute(select(Printer).where(Printer.id == scan_printer_id))
  6513. printer = result.scalar_one_or_none()
  6514. if not printer:
  6515. logger.warning("[TIMELAPSE] Printer not found for archive %s, aborting", archive_id)
  6516. return
  6517. baseline_names, baseline_trusted = await _read_timelapse_baseline(printer)
  6518. if not baseline_trusted:
  6519. # The card is unreadable at the one moment a baseline has to
  6520. # be taken, so the empty listing means "we never looked",
  6521. # not "these are all new". Carry on with it anyway — the
  6522. # usual card holds exactly one video, which resolves
  6523. # correctly — but stop the poll from *choosing* between
  6524. # several, which is how a stale video got attached to this
  6525. # print and then deleted off the printer (#2957 follow-up).
  6526. logger.warning(
  6527. "[TIMELAPSE] Baseline for archive %s taken while printer %s's card could not be "
  6528. "read. A single new video still resolves; several will not be guessed between — "
  6529. "use Scan for Timelapse to pick one by hand",
  6530. archive_id,
  6531. scan_printer_id,
  6532. )
  6533. logger.info(
  6534. "[TIMELAPSE] Baseline snapshot (fallback): %s existing video files for archive %s",
  6535. len(baseline_names),
  6536. archive_id,
  6537. )
  6538. except Exception as e:
  6539. logger.warning("[TIMELAPSE] Failed to take baseline snapshot for archive %s: %s", archive_id, e)
  6540. return
  6541. # --- Phase 2: poll for a file that was not there when the print began -----
  6542. deadline = time.monotonic() + _TIMELAPSE_SCAN_TIMEOUT_SECONDS
  6543. max_attempts = _timelapse_scan_max_attempts()
  6544. seen_names: set[str] = set()
  6545. delay = _TIMELAPSE_SCAN_FIRST_DELAY_SECONDS
  6546. attempt = 0
  6547. while True:
  6548. await asyncio.sleep(delay)
  6549. delay = _TIMELAPSE_SCAN_POLL_INTERVAL_SECONDS
  6550. attempt += 1
  6551. try:
  6552. from backend.app.models.printer import Printer
  6553. # Read phase: fetch archive + printer in a short session and release
  6554. # the pooled connection BEFORE the FTP list/download below. Holding it
  6555. # across the FTP round-trips left one connection idle-in-transaction per
  6556. # in-flight scan (issue #2572).
  6557. async with async_session() as db:
  6558. from backend.app.services.print_log import claimed_timelapse_stems
  6559. service = ArchiveService(db)
  6560. archive = await service.get_archive(archive_id)
  6561. if not archive:
  6562. logger.warning("[TIMELAPSE] Archive %s not found, stopping poll", archive_id)
  6563. return
  6564. if not await _timelapse_still_wanted(db, archive, run_id, run_started_at):
  6565. logger.info(
  6566. "[TIMELAPSE] Archive %s has its video or is printing again, stopping the run's poll",
  6567. archive_id,
  6568. )
  6569. return
  6570. result = await db.execute(select(Printer).where(Printer.id == scan_printer_id))
  6571. printer = result.scalar_one_or_none()
  6572. if not printer:
  6573. logger.warning("[TIMELAPSE] Printer not found for archive %s, stopping poll", archive_id)
  6574. return
  6575. claimed = await claimed_timelapse_stems(db, scan_printer_id, archive_id=archive_id, run_id=run_id)
  6576. # I/O phase (no DB connection held): FTP list + download.
  6577. video_files, found_path = await _list_timelapse_videos(printer)
  6578. # The poll can run for dozens of rounds, so only narrate a round
  6579. # that saw something change. Repeating the whole listing every 30 s
  6580. # would bury the one interesting line in the support bundle.
  6581. names_now = {f.get("name", "") for f in video_files}
  6582. changed = attempt == 1 or names_now != seen_names
  6583. seen_names = names_now
  6584. speak = logger.info if changed else logger.debug
  6585. if video_files:
  6586. speak("[TIMELAPSE] Attempt %s: Found %s video files in %s", attempt, len(video_files), found_path)
  6587. if changed:
  6588. for f in video_files[:5]:
  6589. logger.info("[TIMELAPSE] - %s", f.get("name"))
  6590. attached = await _attach_first_unclaimed_timelapse(
  6591. archive_id,
  6592. printer,
  6593. video_files,
  6594. baseline_names,
  6595. claimed,
  6596. attempt,
  6597. logger,
  6598. quiet=not changed,
  6599. require_unambiguous=not baseline_trusted,
  6600. plate_id=plate_id,
  6601. run_started_at=run_started_at,
  6602. run_id=run_id,
  6603. )
  6604. if attached:
  6605. return
  6606. else:
  6607. speak("[TIMELAPSE] Attempt %s: No video files found, will retry", attempt)
  6608. except Exception as e:
  6609. logger.warning("[TIMELAPSE] Attempt %s failed with error: %s", attempt, e)
  6610. if attempt >= max_attempts or time.monotonic() >= deadline:
  6611. break
  6612. # No name-match fallback: it compared the print name against the filename,
  6613. # and Bambu firmware only ever writes "video_<timestamp>". Across 247 support
  6614. # bundles it fired 159 times and matched zero times, so all it added was a
  6615. # misleading log line before giving up.
  6616. logger.warning(
  6617. "[TIMELAPSE] No new video appeared for archive %s within %ss, giving up",
  6618. archive_id,
  6619. int(_TIMELAPSE_SCAN_TIMEOUT_SECONDS),
  6620. )
  6621. async def _attach_layer_timelapse(
  6622. archive_id: int,
  6623. timelapse_path: Path,
  6624. plate_id: int | None,
  6625. run_started_at: datetime | None,
  6626. run_id: int | None = None,
  6627. ) -> None:
  6628. """Attach the external camera's stitched timelapse to the archive.
  6629. The camera's video is the one the user set up in Bambuddy, so it wins
  6630. over the printer's own when both are on: it replaces the printer's video
  6631. (usually there first; stitching takes minutes) and that file is deleted,
  6632. leaving nothing in the archive folder unreferenced. ``run_started_at`` is
  6633. the run the frames were taken for: when the archive is printing again by
  6634. the time the stitch is done, the video is not the new run's and is
  6635. dropped. The stitched file is a temporary either way.
  6636. ``run_id`` is the print that ended (archive redesign): the video is that
  6637. run's and replaces that run's printer video; the poll by start above is
  6638. for a print without a run.
  6639. """
  6640. from backend.app.services.print_log import run_of, superseded
  6641. from backend.app.utils.safe_path import PathTraversalError, safe_join_under
  6642. logger = logging.getLogger(__name__)
  6643. try:
  6644. async with async_session() as db:
  6645. service = ArchiveService(db)
  6646. archive = await service.get_archive(archive_id)
  6647. if archive is None:
  6648. return
  6649. run = await run_of(db, archive_id, run_id) if run_id is not None else None
  6650. if run_id is not None and run is None:
  6651. logger.info("[LAYER-TL] Run %s of archive %s is gone; dropping its layer timelapse", run_id, archive_id)
  6652. return
  6653. if run is not None and await superseded(db, run):
  6654. logger.info(
  6655. "[LAYER-TL] A later print took over run %s of archive %s; dropping its layer timelapse",
  6656. run_id,
  6657. archive_id,
  6658. )
  6659. return
  6660. if run is None and archive.started_at != run_started_at:
  6661. logger.info(
  6662. "[LAYER-TL] Archive %s is printing again; dropping the previous run's layer timelapse", archive_id
  6663. )
  6664. return
  6665. holder = run if run is not None else archive
  6666. replaced = holder.timelapse_path
  6667. logger.info("[LAYER-TL] Attaching timelapse %s to archive %s", timelapse_path, archive_id)
  6668. timelapse_data = await asyncio.to_thread(timelapse_path.read_bytes)
  6669. if not await service.attach_timelapse(
  6670. archive_id, timelapse_data, "layer_timelapse.mp4", plate_id=plate_id, run_id=run_id
  6671. ):
  6672. return
  6673. attached = holder.timelapse_path
  6674. logger.info("[LAYER-TL] Layer timelapse attached successfully")
  6675. if replaced and replaced != attached:
  6676. try:
  6677. old = safe_join_under(app_settings.base_dir, replaced, http=False)
  6678. except PathTraversalError:
  6679. return
  6680. await asyncio.to_thread(old.unlink, missing_ok=True)
  6681. logger.info("[LAYER-TL] Replaced the printer's timelapse %s on archive %s", replaced, archive_id)
  6682. finally:
  6683. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  6684. # ``run_started_at`` not given: attach whichever run the archive is on.
  6685. _ANY_RUN = object()
  6686. async def _attach_first_unclaimed_timelapse(
  6687. archive_id: int,
  6688. printer,
  6689. video_files: list[dict],
  6690. baseline_names: set[str],
  6691. claimed: set[str],
  6692. attempt: int,
  6693. logger: logging.Logger,
  6694. *,
  6695. quiet: bool = False,
  6696. require_unambiguous: bool = False,
  6697. plate_id: int | None = None,
  6698. run_started_at: datetime | None | object = _ANY_RUN,
  6699. run_id: int | None = None,
  6700. ) -> bool:
  6701. """Download and attach the one video that belongs to this print.
  6702. A candidate is any file absent from the print-start baseline. More than one
  6703. can qualify when a previous print's video landed late, after this print's
  6704. baseline was taken — those are filtered out by name, because they are
  6705. already attached to another archive. Sorting the candidates instead would
  6706. mean sorting on mtime or on the filename timestamp, both of which come from
  6707. the printer's unsynced clock.
  6708. Returns True once a video is attached. The printer's copy is deleted only
  6709. after the attach succeeds on bytes whose length matched the listing.
  6710. ``quiet`` downgrades the "nothing yet" lines to DEBUG when the caller has
  6711. already seen this exact listing — the poll runs for many rounds and only the
  6712. rounds where something changed are worth an INFO line.
  6713. """
  6714. from backend.app.services.bambu_ftp import (
  6715. DownloadDeadlineExceeded,
  6716. delete_archived_timelapse,
  6717. download_file_bytes_async,
  6718. remote_file_settled,
  6719. )
  6720. speak = logger.debug if quiet else logger.info
  6721. new_files = [f for f in video_files if f.get("name", "") not in baseline_names]
  6722. if not new_files:
  6723. speak("[TIMELAPSE] Attempt %s: No new files since baseline, will retry", attempt)
  6724. return False
  6725. candidates = [f for f in new_files if Path(f.get("name", "")).stem not in claimed]
  6726. if not candidates:
  6727. speak(
  6728. "[TIMELAPSE] Attempt %s: %s new file(s), all already attached to other archives, will retry",
  6729. attempt,
  6730. len(new_files),
  6731. )
  6732. return False
  6733. if len(candidates) > 1:
  6734. if require_unambiguous:
  6735. # The baseline is not evidence -- it was taken off a card that could
  6736. # not be read -- so "new since the baseline" does not narrow these
  6737. # down at all. Taking the first would attach an arbitrary video to
  6738. # this print and then delete it from the printer.
  6739. logger.warning(
  6740. "[TIMELAPSE] Attempt %s: %s unclaimed videos (%s) and no baseline to tell them apart — "
  6741. "leaving all of them on the printer for manual selection",
  6742. attempt,
  6743. len(candidates),
  6744. ", ".join(str(f.get("name")) for f in candidates),
  6745. )
  6746. return False
  6747. logger.warning(
  6748. "[TIMELAPSE] Attempt %s: %s unclaimed new files (%s) — taking the first; "
  6749. "the rest stay on the printer for manual selection",
  6750. attempt,
  6751. len(candidates),
  6752. ", ".join(str(f.get("name")) for f in candidates),
  6753. )
  6754. target = candidates[0]
  6755. file_name = str(target.get("name") or "") # every listed entry has one
  6756. remote_path = target.get("path") or f"/timelapse/{file_name}"
  6757. logger.info(
  6758. "[TIMELAPSE] Attempt %s: New file detected: %s (downloading for archive %s)",
  6759. attempt,
  6760. file_name,
  6761. archive_id,
  6762. )
  6763. # The listing always carries a size (`list_files` skips entries it can't
  6764. # parse), but read it explicitly: the delete below is destructive and must
  6765. # depend on a size we actually had, not on one we hoped was there.
  6766. expected_size = target.get("size")
  6767. try:
  6768. timelapse_data = await download_file_bytes_async(
  6769. printer.ip_address,
  6770. printer.access_code,
  6771. remote_path,
  6772. printer_model=printer.model,
  6773. expected_size=expected_size,
  6774. )
  6775. except DownloadDeadlineExceeded:
  6776. # Too slow for its size this round; the printer keeps its copy.
  6777. timelapse_data = None
  6778. if not timelapse_data:
  6779. # Short or failed transfer. The printer keeps its copy, so the next
  6780. # round can try again — which is exactly why the delete below is
  6781. # gated on a verified download.
  6782. logger.warning("[TIMELAPSE] Attempt %s: Failed to download new file, will retry", attempt)
  6783. return False
  6784. # The length check above proves we got what the listing said, not that the
  6785. # printer had finished writing. A video still being written can be listed
  6786. # short, served short, and pass — so confirm it has stopped growing before
  6787. # committing to it and deleting the original (#2704).
  6788. if not await remote_file_settled(
  6789. printer.ip_address,
  6790. printer.access_code,
  6791. remote_path,
  6792. len(timelapse_data),
  6793. printer_model=printer.model,
  6794. ):
  6795. return False
  6796. # Write phase: attach in a fresh short-lived session. The download took a
  6797. # while: by now the archive may have its video, or be printing again for a
  6798. # newer run, whose video this is not. Then nothing is attached or deleted,
  6799. # and the poll ends (True).
  6800. async with async_session() as db:
  6801. service = ArchiveService(db)
  6802. if run_id is not None or run_started_at is not _ANY_RUN:
  6803. current = await service.get_archive(archive_id)
  6804. started = None if run_started_at is _ANY_RUN else cast("datetime | None", run_started_at)
  6805. if current is None or not await _timelapse_still_wanted(db, current, run_id, started):
  6806. logger.info(
  6807. "[TIMELAPSE] Archive %s moved on during the download (video attached or printing again); "
  6808. "%s stays on the printer",
  6809. archive_id,
  6810. file_name,
  6811. )
  6812. return True
  6813. success = await service.attach_timelapse(
  6814. archive_id, timelapse_data, file_name, plate_id=plate_id, run_id=run_id
  6815. )
  6816. if not success:
  6817. logger.warning("[TIMELAPSE] Failed to attach timelapse to archive %s", archive_id)
  6818. return False
  6819. logger.info("[TIMELAPSE] Successfully attached timelapse to archive %s", archive_id)
  6820. await delete_archived_timelapse(
  6821. printer.ip_address,
  6822. printer.access_code,
  6823. remote_path,
  6824. verified=expected_size is not None,
  6825. printer_model=printer.model,
  6826. printer_name=printer.name,
  6827. )
  6828. return True
  6829. # Defaults for the finish-photo-from-timelapse polling loop (#1397). These are
  6830. # module-level so tests can monkeypatch them down to ~0 without timing out.
  6831. _FINISH_PHOTO_TIMELAPSE_POLL_INTERVAL_SECONDS: float = 3.0
  6832. _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS: float = 60.0
  6833. # How long the *background* upgrade keeps waiting after the notification has
  6834. # already gone out (#2704 follow-up). The short bound above exists so a slow
  6835. # printer can't hold up the print-complete notification; this one exists so the
  6836. # archive still ends up with the better frame afterwards.
  6837. #
  6838. # Measured across 261 attaches in the support bundles, the video lands a median
  6839. # 13s after the print ends — but the P1 series writes MJPEG AVI rather than
  6840. # H.264 MP4 and serves it slowly, so its p90 is 167s and the worst observed case
  6841. # was 546s. Every other model was inside 26s. The long budget is therefore
  6842. # almost entirely for P1-series users; on everything else the short wait already
  6843. # wins and this task never runs.
  6844. _FINISH_PHOTO_UPGRADE_TIMEOUT_SECONDS: float = 900.0
  6845. async def _capture_finish_photo_from_timelapse(
  6846. archive_id: int,
  6847. archive_dir: Path,
  6848. timeout: float | None = None,
  6849. rotation: int = 0,
  6850. run_id: int | None = None,
  6851. ) -> tuple[str | None, bool]:
  6852. """Wait for the per-print timelapse to land on the archive and extract its
  6853. last frame as the finish photo (#1397).
  6854. Bambu firmware stops timelapse recording after the toolhead parks but
  6855. before the bed-drop end-gcode runs, so the last frame frames the finished
  6856. print correctly. A live camera grab at gcode_state=FINISH captures the
  6857. bed already lowered.
  6858. ``_scan_for_timelapse_with_retries`` runs in parallel and writes
  6859. ``archive.timelapse_path`` when the file lands. This function polls for
  6860. that field.
  6861. Returns ``(filename, still_pending)``. ``still_pending`` is True only when
  6862. the wait ran out with no video on the archive yet — i.e. the video may
  6863. still be coming and a later attempt could succeed. It is False when the
  6864. video landed (whether or not extraction worked), because in that case
  6865. waiting longer changes nothing. The caller uses that to decide between
  6866. falling back permanently and scheduling a background upgrade.
  6867. ``rotation`` is the printer's camera_rotation, applied to the extracted
  6868. still (#2708) so this source agrees with every other finish-photo source.
  6869. The archived video itself is the printer's own file and is left alone —
  6870. rotating it would mean re-encoding it.
  6871. ``run_id`` is the print that ended: its own video is the one waited for,
  6872. not the archive's, which can be another plate's.
  6873. """
  6874. import uuid
  6875. from backend.app.models.archive import PrintArchive
  6876. from backend.app.services.camera import apply_camera_rotation_to_file, extract_video_last_frame
  6877. from backend.app.services.print_log import run_of
  6878. logger = logging.getLogger(__name__)
  6879. budget = _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS if timeout is None else timeout
  6880. deadline = asyncio.get_event_loop().time() + budget
  6881. poll_interval = _FINISH_PHOTO_TIMELAPSE_POLL_INTERVAL_SECONDS
  6882. while True:
  6883. async with async_session() as db:
  6884. if run_id is not None:
  6885. run = await run_of(db, archive_id, run_id)
  6886. timelapse_relpath = run.timelapse_path if run else None
  6887. else:
  6888. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  6889. archive = result.scalar_one_or_none()
  6890. timelapse_relpath = archive.timelapse_path if archive else None
  6891. if timelapse_relpath:
  6892. video_path = app_settings.base_dir / timelapse_relpath
  6893. if video_path.exists() and video_path.stat().st_size > 0:
  6894. photos_dir = archive_dir / "photos"
  6895. photos_dir.mkdir(parents=True, exist_ok=True)
  6896. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  6897. filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  6898. output_path = photos_dir / filename
  6899. if await extract_video_last_frame(video_path, output_path):
  6900. await apply_camera_rotation_to_file(output_path, rotation, logger)
  6901. logger.info(
  6902. "[PHOTO-BG] Extracted finish photo from timelapse %s for archive %s",
  6903. video_path.name,
  6904. archive_id,
  6905. )
  6906. return filename, False
  6907. logger.warning(
  6908. "[PHOTO-BG] Timelapse %s landed but last-frame extraction failed for archive %s; falling back",
  6909. video_path.name,
  6910. archive_id,
  6911. )
  6912. return None, False
  6913. if asyncio.get_event_loop().time() >= deadline:
  6914. logger.info(
  6915. "[PHOTO-BG] Timelapse for archive %s didn't land within %.0fs; falling back to live camera",
  6916. archive_id,
  6917. budget,
  6918. )
  6919. return None, True
  6920. await asyncio.sleep(poll_interval)
  6921. async def _upgrade_finish_photo_from_timelapse(
  6922. archive_id: int,
  6923. archive_dir: Path,
  6924. rotation: int = 0,
  6925. run_id: int | None = None,
  6926. replace: str | None = None,
  6927. ) -> None:
  6928. """Put the timelapse's last frame on an archive after the fact (#2704).
  6929. The print-complete notification waits only ~60s for the video, because
  6930. holding a notification for minutes is worse than sending it with a live
  6931. camera grab. On a P1-series printer the video often lands well after that,
  6932. so the archive used to be stuck with the live grab — which is taken at
  6933. ``gcode_state=FINISH``, after the end G-code has dropped the bed, and is
  6934. the worse photo of the two.
  6935. This keeps waiting in the background and, when the video arrives, extracts
  6936. the frame and puts it *first* in the archive's photo list, so opening the
  6937. gallery shows it. ``replace`` is the live grab: the frame is written over
  6938. that file, under its name, rather than added next to it. Two finish photos
  6939. of the same moment is one too many, and the name has to survive: with
  6940. authentication off, the notification that already went out links to
  6941. ``/archives/<id>/photos/<name>``, which now shows the better photo. (With
  6942. it on, the link is to a copy in notification_photos, and attachments are
  6943. bytes, so neither cares.) Without ``replace``, or once that photo has been
  6944. deleted, the frame is added under its own name. ``run_id`` is the print it
  6945. is for, whose list leads with it too.
  6946. """
  6947. from backend.app.services.print_log import add_run_photo, run_of
  6948. logger = logging.getLogger(__name__)
  6949. filename, _ = await _capture_finish_photo_from_timelapse(
  6950. archive_id, archive_dir, timeout=_FINISH_PHOTO_UPGRADE_TIMEOUT_SECONDS, rotation=rotation, run_id=run_id
  6951. )
  6952. if not filename:
  6953. logger.info("[PHOTO-UPGRADE] No timelapse frame for archive %s; keeping the live grab", archive_id)
  6954. return
  6955. try:
  6956. async with async_session() as db:
  6957. from backend.app.models.archive import PrintArchive
  6958. archive = await db.get(PrintArchive, archive_id)
  6959. if archive is None:
  6960. return
  6961. photos = list(archive.photos or [])
  6962. if filename in photos:
  6963. return
  6964. photos_dir = archive_dir / "photos"
  6965. if replace and replace in photos and (photos_dir / replace).is_file():
  6966. # Atomic, so the gallery never reads half a JPEG.
  6967. await asyncio.to_thread(os.replace, photos_dir / filename, photos_dir / replace)
  6968. filename = replace
  6969. photos.remove(replace)
  6970. # Front of the list: PhotoGalleryModal opens at index 0.
  6971. archive.photos = [filename, *photos]
  6972. add_run_photo(await run_of(db, archive_id, run_id) if run_id is not None else None, filename, first=True)
  6973. await db.commit()
  6974. except Exception as e:
  6975. logger.warning("[PHOTO-UPGRADE] Failed to attach upgraded photo to archive %s: %s", archive_id, e)
  6976. return
  6977. logger.info(
  6978. "[PHOTO-UPGRADE] Archive %s now leads with the timelapse frame %s%s",
  6979. archive_id,
  6980. filename,
  6981. " (replacing the live grab)" if filename == replace else "",
  6982. )
  6983. await ws_manager.send_archive_updated({"id": archive_id, "photo_added": filename})
  6984. async def _restore_usage_tracking_session(printer_id: int, state, db, logger) -> None:
  6985. """Put the filament-attribution context back after a restart mid-print.
  6986. ``usage_tracker._active_sessions`` and ``PrinterState.tray_change_log``
  6987. both die with the process. The print keeps running, so at completion the
  6988. tracker would fall back to whatever the printer reports *now* — and AMS
  6989. filament backup makes "now" the substitute tray, charging the whole print
  6990. to the spool that only finished it.
  6991. The persisted row is only trusted when its print name still matches what
  6992. the printer says it is running: a row left behind by a completion we never
  6993. saw must not attach itself to the next print.
  6994. """
  6995. try:
  6996. from backend.app.api.routes.settings import get_setting
  6997. from backend.app.services.usage_tracker import (
  6998. clear_persisted_session,
  6999. get_persisted_ams_mapping,
  7000. get_persisted_print_name,
  7001. get_persisted_spool_change_log,
  7002. restore_session,
  7003. )
  7004. persisted_name = await get_persisted_print_name(db, printer_id)
  7005. current_name = (state.subtask_name or "").strip()
  7006. if persisted_name and current_name and persisted_name.strip() != current_name:
  7007. logger.info(
  7008. "[RESTART] Discarding stale print session for printer %s (%r != running %r)",
  7009. printer_id,
  7010. persisted_name,
  7011. current_name,
  7012. )
  7013. await clear_persisted_session(db, printer_id)
  7014. # Fall through to seeding: the print on the printer is real, it just
  7015. # isn't the one the row described.
  7016. persisted_log = None
  7017. else:
  7018. # Spoolman users get the tray-change log back but no in-memory
  7019. # session — see ``on_print_start`` on why that dict is load-bearing
  7020. # for the remain%-sync guard.
  7021. _spoolman_on = await get_setting(db, "spoolman_enabled")
  7022. persisted_log = await restore_session(
  7023. db,
  7024. printer_id,
  7025. register_active=not (bool(_spoolman_on) and _spoolman_on.lower() == "true"),
  7026. )
  7027. # The slot pills (#1762) read the mapping captured from the print
  7028. # command, which dies with the process as well. One this process
  7029. # has captured since is newer.
  7030. if state.print_ams_mapping is None:
  7031. state.print_ams_mapping = await get_persisted_ams_mapping(db, printer_id)
  7032. # Spools swapped on a tray before the restart (#3325); anything this
  7033. # process has recorded since goes after them.
  7034. persisted_swaps = await get_persisted_spool_change_log(db, printer_id)
  7035. if persisted_swaps:
  7036. seen = getattr(state, "spool_change_log", None) or []
  7037. state.spool_change_log = persisted_swaps + [item for item in seen if item not in persisted_swaps]
  7038. if persisted_log:
  7039. restored = [tuple(entry) for entry in persisted_log if isinstance(entry, (list, tuple)) and len(entry) == 2]
  7040. # Anything this process already observed goes after the persisted
  7041. # history — the log is ordered by layer, and a fresh process can
  7042. # only have seen changes from later in the print.
  7043. for entry in state.tray_change_log or []:
  7044. if tuple(entry) not in restored:
  7045. restored.append(tuple(entry))
  7046. state.tray_change_log = restored
  7047. tray_now = state.tray_now
  7048. if 0 <= tray_now <= 254:
  7049. if not state.tray_change_log:
  7050. # No persisted history — a print that started before this build,
  7051. # or before the row existed. Seed with the tray feeding right
  7052. # now so the remainder of the print is at least attributable to
  7053. # the right spool.
  7054. state.tray_change_log = [(tray_now, state.layer_num)]
  7055. logger.info(
  7056. "[RESTART] Seeded tray change log for printer %s: tray=%d at layer=%d",
  7057. printer_id,
  7058. tray_now,
  7059. state.layer_num,
  7060. )
  7061. # The tray handler updates ``last_loaded_tray`` on every push
  7062. # regardless of whether it logged a change, so re-align it to avoid
  7063. # a duplicate entry on the next push. Only ever with a real tray:
  7064. # ``last_loaded_tray`` is the "survives the end-of-print retract to
  7065. # 255" fallback, and writing 255 into it would defeat that.
  7066. state.last_loaded_tray = tray_now
  7067. except Exception:
  7068. # Never let attribution recovery cost the caller its timelapse
  7069. # baseline — that capture has to happen before the printer uploads
  7070. # the in-flight MP4 and there is no second chance at it.
  7071. logger.exception("[RESTART] Failed to restore usage-tracking session for printer %s", printer_id)
  7072. async def on_print_running_observed(printer_id: int, data: dict):
  7073. """Restart-recovery for a print that started before Bambuddy came up.
  7074. bambu_mqtt.py suppresses ``on_print_start`` on the first RUNNING push
  7075. after Bambuddy startup (#1304 guard, prevents duplicate archive
  7076. creation). This hook restores the persisted archive into ``_active_prints``
  7077. and captures the timelapse baseline that normally hangs off print start.
  7078. Fires once per session, in lieu of on_print_start when restart-recovery
  7079. kicks in. The printer doesn't upload the timelapse until after PRINT
  7080. COMPLETE, so a baseline captured any time during the print is still
  7081. pre-upload.
  7082. """
  7083. logger = logging.getLogger(__name__)
  7084. async with async_session() as db:
  7085. from backend.app.models.printer import Printer
  7086. state = printer_manager.get_status(printer_id)
  7087. if state is not None:
  7088. authorization = await _is_bambuddy_authorized_print(printer_id, state, db)
  7089. if authorization is True:
  7090. logger.info("[RESTART] Restored active Bambuddy print for printer %s", printer_id)
  7091. await _restore_usage_tracking_session(printer_id, state, db, logger)
  7092. await _restore_printable_objects(printer_id, state, db, logger)
  7093. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  7094. printer = result.scalar_one_or_none()
  7095. if not printer:
  7096. logger.warning(
  7097. "[TIMELAPSE] on_print_running_observed: printer %s not found in DB, skipping baseline",
  7098. printer_id,
  7099. )
  7100. return
  7101. # Avoid double-capture: ownership reconciliation above must still run when
  7102. # a baseline already exists, but the camera work itself is one-shot.
  7103. if printer_id in _timelapse_baselines:
  7104. logger.debug(
  7105. "[TIMELAPSE] on_print_running_observed: baseline already present for printer %s, skipping",
  7106. printer_id,
  7107. )
  7108. return
  7109. await _capture_timelapse_baseline_at_start(printer, printer_id, logger)
  7110. class _RunOfArchive:
  7111. """An archive as one of its runs printed it: the run's job id, start and
  7112. plate, the rest the archive's. What the reconcile judges and closes."""
  7113. def __init__(self, archive, run):
  7114. self._archive = archive
  7115. self.subtask_id = run.subtask_id
  7116. self.started_at = run.started_at
  7117. self.plate_id = run.plate_id if run.plate_id is not None else archive.plate_id
  7118. def __getattr__(self, name):
  7119. return getattr(self._archive, name)
  7120. def _is_active_archive_stale(archive, state) -> tuple[bool, str]:
  7121. """Return ``(is_stale, reason)`` for an archive in ``status="printing"``
  7122. against the printer's current MQTT state.
  7123. Reconciliation triggers (#1542 follow-up — recovers from missed PRINT
  7124. COMPLETE events, typically a print finishing during an MQTT disconnect
  7125. window followed by a smart-plug power cycle):
  7126. 1. Printer state is terminal (IDLE / FINISH / FAILED). The print is
  7127. provably not running anymore — only branch that should fire under
  7128. normal disconnect-then-reconnect timing.
  7129. 2. Printer has a different ``subtask_id`` than the archive. Bambu
  7130. firmware mints a fresh ``subtask_id`` for each print, including the
  7131. ghost replay it runs after a power cycle from a leftover SD file —
  7132. so a mismatch unambiguously means the in-DB archive is no longer
  7133. the print on the printer.
  7134. 3. Printer is running but ``subtask_name`` is empty. The printer
  7135. doesn't know what it's running; the archive's reference to it is
  7136. already broken.
  7137. Conservative on purpose: PAUSE / PREPARE / SLICING and any RUNNING state
  7138. with matching subtask_id+subtask_name is left alone. The cost of a false
  7139. positive is a print closed while it still runs: the reconcile takes its
  7140. ``_active_prints`` entry and the archive is no longer ``printing``, so
  7141. the real PRINT COMPLETE later finds no archive to close (#1679). The
  7142. cost of a false negative is the ghost-print loop in #1542. What the
  7143. archive is closed as is ``_reconciled_outcome``'s call.
  7144. Pre-push guard (#1679): when ``state.state`` is empty or ``"unknown"``,
  7145. MQTT has connected but the first ``push_status`` response hasn't been
  7146. applied yet — ``PrinterState`` is sitting on its construction defaults.
  7147. The reconcile caller in ``on_printer_status_change`` is already gated
  7148. on a real ``state.state``, so in normal operation this branch is
  7149. unreachable; it's kept as belt-and-braces for future callers and for
  7150. the narrow window where a partial state update could arrive
  7151. (``state.state`` set but ``subtask_name`` not yet populated). Returning
  7152. ``not stale`` on degenerate input is strictly conservative: a real
  7153. stale archive will still be caught by the next push_status arriving
  7154. with terminal state.
  7155. """
  7156. current_state = (state.state or "").upper()
  7157. if current_state in ("", "UNKNOWN"):
  7158. # No real push_status yet — PrinterState defaults are not evidence.
  7159. return False, ""
  7160. if current_state in ("IDLE", "FINISH", "FAILED"):
  7161. return True, f"printer state {current_state}"
  7162. # Below here the printer is in a running / pre-running state (RUNNING /
  7163. # PAUSE / PREPARE / SLICING / etc.) — decide based on subtask identity.
  7164. current_subtask_id = (state.subtask_id or "").strip()
  7165. if archive.subtask_id and current_subtask_id and archive.subtask_id != current_subtask_id:
  7166. return True, f"subtask_id changed ({archive.subtask_id!r} → {current_subtask_id!r})"
  7167. current_subtask_name = (state.subtask_name or "").strip()
  7168. if not current_subtask_name:
  7169. return True, "printer subtask_name empty"
  7170. return False, ""
  7171. async def prime_kprofile_table(printer_id: int) -> int:
  7172. """Read the printer's calibration table once per connection.
  7173. The AMS slot card shows a K value per slot (#2854). On the printers whose
  7174. trays carry no ``k`` field of their own -- the whole H2 series, whose trays
  7175. report ``cali_idx`` and nothing else -- that number can only come from
  7176. ``state.kprofiles``, and nothing used to fill it on connect. It arrived by
  7177. luck: someone opening the Profiles page or Configure Slot, a nightly GitHub
  7178. backup, or the printer answering a query BambuStudio made on the report
  7179. topic we share. A Bambuddy that nobody visited showed a card with no K
  7180. values at all.
  7181. Only the diameters actually fitted are asked for, which is one request on a
  7182. single-nozzle printer and two on a dual. Probing the four sizes blind is
  7183. what the backup does, and it is both wasteful and the thing that used to
  7184. blank the table.
  7185. Returns the number of nozzles whose table was read.
  7186. """
  7187. client = printer_manager.get_client(printer_id)
  7188. state = printer_manager.get_status(printer_id)
  7189. if client is None or state is None or not state.connected:
  7190. return 0
  7191. # Deduplicated, order preserved: a dual-nozzle printer with two 0.4s should
  7192. # ask once, and both entries are empty until the first push_status lands.
  7193. diameters = list(dict.fromkeys(n.nozzle_diameter for n in (state.nozzles or []) if n.nozzle_diameter))
  7194. if not diameters:
  7195. logging.getLogger(__name__).debug(
  7196. "[Printer %s] No nozzle diameter reported yet; leaving the K-profile table to the next reader",
  7197. printer_id,
  7198. )
  7199. return 0
  7200. primed = 0
  7201. for diameter in diameters:
  7202. try:
  7203. profiles = await client.get_kprofiles(nozzle_diameter=diameter, max_retries=2)
  7204. except Exception as exc: # noqa: BLE001
  7205. # A printer that won't answer costs the card its K values, nothing
  7206. # more — never the connection this runs on the back of.
  7207. logging.getLogger(__name__).warning(
  7208. "[Printer %s] Could not read the K-profile table for nozzle %s: %s", printer_id, diameter, exc
  7209. )
  7210. continue
  7211. primed += 1
  7212. logging.getLogger(__name__).info(
  7213. "[Printer %s] Primed K-profile table for nozzle %s: %d profiles", printer_id, diameter, len(profiles)
  7214. )
  7215. return primed
  7216. def _sd_files_in_use(state) -> set[str]:
  7217. """Names of the SD-card files the printer may be printing right now (#3009).
  7218. The post-print cleanup works out what to delete from the finished archive,
  7219. not from the printer. When reconciliation closes an old archive because the
  7220. printer has moved on to a new job, and that job is the same file reprinted
  7221. from the printer's screen, the cleanup would delete the file mid-print.
  7222. Empty unless the printer is busy: on an ordinary completion the state is
  7223. FINISH or FAILED and nothing is held back. ``gcode_file`` arrives as a bare
  7224. name, a path or a URL depending on firmware, and some report only the
  7225. plate's G-code, so the job name is matched as well, in the same spellings
  7226. the cleanup tries. Names are lower-cased; the card's FAT filesystem
  7227. ignores case.
  7228. """
  7229. def _text(value) -> str:
  7230. return value.strip() if isinstance(value, str) else ""
  7231. if state is None:
  7232. return set()
  7233. current_state = _text(getattr(state, "state", None)).upper()
  7234. if current_state in ("", "UNKNOWN", "IDLE", "FINISH", "FAILED"):
  7235. return set()
  7236. in_use: set[str] = set()
  7237. gcode_file = _text(getattr(state, "gcode_file", None))
  7238. if gcode_file:
  7239. # Drop a scheme by hand: urlparse would cut a name at "#" or "?",
  7240. # both legal in a file name, and raises on some inputs.
  7241. name = PurePosixPath(gcode_file.split("://", 1)[-1]).name
  7242. if name:
  7243. in_use.add(name.lower())
  7244. subtask_name = _text(getattr(state, "subtask_name", None))
  7245. if subtask_name:
  7246. for name in (subtask_name, subtask_name.replace(" ", "_")):
  7247. for ext in (".3mf", ".gcode"):
  7248. in_use.add(f"{name}{ext}".lower())
  7249. return in_use
  7250. def _reconciled_outcome(archive, state) -> str:
  7251. """What a stale archive is closed as at reconnect (#3310).
  7252. A print that ends while Bambuddy restarts is never seen to end: the new
  7253. MQTT client's first status is already FINISH, which is not a transition.
  7254. The printer keeps reporting the finished job's ``subtask_id`` until the
  7255. next print, so a matching id proves the FINISH or FAILED is this
  7256. archive's. Anything else -- IDLE, another job, an id of "0" (a print
  7257. started outside the cloud) or none -- says only that the print is over,
  7258. not how, and stays ``aborted``.
  7259. """
  7260. current_state = (state.state or "").upper()
  7261. if current_state not in ("FINISH", "FAILED"):
  7262. return "aborted"
  7263. archive_id = (archive.subtask_id or "").strip()
  7264. current_id = str(state.subtask_id or "").strip()
  7265. if archive_id in ("", "0") or archive_id != current_id:
  7266. return "aborted"
  7267. return "completed" if current_state == "FINISH" else "failed"
  7268. def _reconciled_end_time(archive, now: datetime, plate_id: int | None = None) -> datetime | None:
  7269. """When a print reconciled as completed most likely ended, or None (#3310).
  7270. The printer does not say when it finished, and the reconnect can be hours
  7271. later. Its start plus the slicer's estimate for the plate it printed,
  7272. never later than now, is the best guess; None when either is missing,
  7273. and the run then keeps no duration, as an aborted one does.
  7274. ``plate_id`` is the plate this run printed, from its queue job: a reprint
  7275. of another plate reuses the archive without changing ``archive.plate_id``.
  7276. """
  7277. started = archive.started_at
  7278. if not started:
  7279. return None
  7280. if started.tzinfo is None:
  7281. started = started.replace(tzinfo=timezone.utc)
  7282. plate = plate_id if plate_id is not None else archive.plate_id
  7283. estimate = None
  7284. if archive.file_path:
  7285. from backend.app.utils.threemf_tools import extract_print_time_from_3mf
  7286. full_path = (
  7287. app_settings.base_dir / archive.file_path
  7288. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  7289. try:
  7290. estimate = extract_print_time_from_3mf(full_path, plate)
  7291. except Exception as exc: # noqa: BLE001
  7292. logging.getLogger(__name__).debug(
  7293. "Plate estimate for reconciled archive %s unreadable: %s", archive.id, exc
  7294. )
  7295. if plate is None and estimate != archive.print_time_seconds:
  7296. # Without a plate the first plate's estimate is read. The archive
  7297. # carries the sum over all plates (#1593), so a difference means
  7298. # several plates and no telling which one printed.
  7299. estimate = None
  7300. elif plate is None:
  7301. # Nothing to read the plate from; the archive's own estimate, as for
  7302. # a single-plate file.
  7303. estimate = archive.print_time_seconds
  7304. if not estimate or estimate <= 0:
  7305. return None
  7306. return min(started + timedelta(seconds=int(estimate)), now)
  7307. async def reconcile_stale_active_prints(printer_id: int) -> int:
  7308. """Synthesise ``on_print_complete`` for archives whose print can't be
  7309. running on the printer anymore.
  7310. Called once per MQTT (re)connection (from on_printer_status_change when
  7311. the connected edge flips False → True) and at Bambuddy startup (from
  7312. the FastAPI lifespan). Without this, a print that completes during a
  7313. disconnect window — followed by a smart-plug-driven power cycle — leaves
  7314. the ``.3mf`` on the SD card, the firmware auto-replays it on next boot,
  7315. and Bambuddy fires a fresh PRINT START for the ghost rather than the
  7316. SD cleanup that PRINT COMPLETE was supposed to run. Repeats every
  7317. power cycle until the operator notices (#1542 follow-up). Reconciliation
  7318. closes the loop by faking the missed PRINT COMPLETE — the existing
  7319. cleanup chain handles SD-file deletion, status updates, usage tracking,
  7320. and notifications.
  7321. The status is ``_reconciled_outcome``'s: ``completed`` or ``failed``
  7322. when the printer still reports the archive's own job as finished or
  7323. failed, ``aborted`` otherwise. Nothing corrects it later: the archive is
  7324. no longer ``printing``, so a later PRINT COMPLETE finds no archive, and
  7325. after a restart the printer's FINISH is not one (#3310).
  7326. Returns the number of archives reconciled.
  7327. """
  7328. state = printer_manager.get_status(printer_id)
  7329. if not state:
  7330. return 0
  7331. # Don't reconcile while disconnected — we'd be making a decision against
  7332. # stale cached state. The connected → reconcile edge handles this.
  7333. if not state.connected:
  7334. return 0
  7335. from backend.app.models.archive import PrintArchive
  7336. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, RUN_OPEN, PrintLogEntry
  7337. reconciled = 0
  7338. async with async_session() as db:
  7339. # The prints on this printer Bambuddy saw start: their runs open here.
  7340. # The archive row is shared by every plate of a file, its printer, job
  7341. # and start the latest start's, so each is judged as its run.
  7342. open_runs = (
  7343. await db.scalars(
  7344. select(PrintLogEntry)
  7345. .where(PrintLogEntry.printer_id == printer_id, PrintLogEntry.status == RUN_OPEN)
  7346. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  7347. )
  7348. ).all()
  7349. by_run = {
  7350. archive.id: archive
  7351. for archive in (
  7352. await db.scalars(
  7353. select(PrintArchive).where(
  7354. PrintArchive.id.in_([run.archive_id for run in open_runs if run.archive_id is not None])
  7355. )
  7356. )
  7357. ).all()
  7358. }
  7359. active: list = [_RunOfArchive(by_run[run.archive_id], run) for run in open_runs if run.archive_id in by_run]
  7360. # And as before runs were opened at the start: a printing archive of
  7361. # this printer with no run open anywhere (printing since before).
  7362. archives_with_open_runs = select(PrintLogEntry.archive_id).where(
  7363. PrintLogEntry.status == RUN_OPEN, PrintLogEntry.archive_id.is_not(None)
  7364. )
  7365. result = await db.execute(
  7366. select(PrintArchive)
  7367. .where(
  7368. PrintArchive.printer_id == printer_id,
  7369. PrintArchive.status == "printing",
  7370. PrintArchive.id.not_in(archives_with_open_runs),
  7371. )
  7372. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  7373. )
  7374. active.extend(result.scalars().all())
  7375. if not active:
  7376. return 0
  7377. # The plate each run printed, from its queue job (#3310).
  7378. from backend.app.models.print_queue import PrintQueueItem
  7379. async with async_session() as db:
  7380. plate_rows = await db.execute(
  7381. select(PrintQueueItem.archive_id, PrintQueueItem.plate_id).where(
  7382. PrintQueueItem.printer_id == printer_id,
  7383. PrintQueueItem.status == "printing",
  7384. PrintQueueItem.archive_id.in_([a.id for a in active]),
  7385. )
  7386. )
  7387. run_plates = {archive_id: plate for archive_id, plate in plate_rows.all() if plate is not None}
  7388. logger = logging.getLogger(__name__)
  7389. for archive in active:
  7390. is_stale, reason = _is_active_archive_stale(archive, state)
  7391. if not is_stale:
  7392. continue
  7393. logger.info(
  7394. "[RECONCILE] Printer %s: synthesising missed PRINT COMPLETE for archive %s (%s) — %s",
  7395. printer_id,
  7396. archive.id,
  7397. archive.filename,
  7398. reason,
  7399. )
  7400. # Synthesised payload: minimal fields the on_print_complete chain
  7401. # needs. `_reconciled` marker lets downstream code distinguish this
  7402. # from a real MQTT-driven completion if it ever needs to (e.g. for
  7403. # metrics / debug logging). raw_data is the live printer state so
  7404. # the usage tracker can compare end-of-print remain% against the
  7405. # captured start values.
  7406. outcome = _reconciled_outcome(archive, state)
  7407. payload = {
  7408. "status": outcome,
  7409. "filename": archive.filename,
  7410. "subtask_name": archive.print_name or "",
  7411. "subtask_id": archive.subtask_id or "",
  7412. "raw_data": state.raw_data or {},
  7413. "_reconciled": True,
  7414. }
  7415. if isinstance(archive, _RunOfArchive):
  7416. # The archive it closes, which its names and printer may not find:
  7417. # another printer's plate of it can have started since.
  7418. payload["_reconciled_archive_id"] = archive.id
  7419. if outcome != "aborted":
  7420. # The printer's progress and layer are this job's, so a partial
  7421. # run is charged what it printed rather than nothing.
  7422. from backend.app.utils.printer_models import usage_progress
  7423. run_progress = usage_progress(
  7424. printer_manager.get_model(printer_id),
  7425. state.progress,
  7426. getattr(state, "layer_num", None),
  7427. getattr(state, "total_layers", None),
  7428. )
  7429. if run_progress:
  7430. payload["last_progress"] = run_progress
  7431. if state.layer_num:
  7432. payload["last_layer_num"] = state.layer_num
  7433. if outcome == "failed":
  7434. payload["hms_errors"] = [e.completion_payload() for e in state.hms_errors or []]
  7435. if outcome == "completed":
  7436. ended = _reconciled_end_time(archive, datetime.now(timezone.utc), run_plates.get(archive.id))
  7437. if ended:
  7438. payload["_reconciled_completed_at"] = ended
  7439. try:
  7440. await on_print_complete(printer_id, payload)
  7441. reconciled += 1
  7442. except Exception as e:
  7443. # Catch-all: a reconciliation failure must not block the
  7444. # printer's normal status flow. The archive stays in
  7445. # ``status="printing"`` and the next reconnect retries.
  7446. logger.warning(
  7447. "[RECONCILE] on_print_complete synthesis failed for archive %s: %s",
  7448. archive.id,
  7449. e,
  7450. )
  7451. return reconciled
  7452. # #2547: clearance left between the nozzle and the top of the print when the
  7453. # plate is commanded back into camera framing. The nozzle is parked away from
  7454. # the part by then, so this is belt-and-braces against a max_z_height that
  7455. # under-reports (e.g. a slicer that excludes a final Z hop).
  7456. _PLATE_RESTORE_CLEARANCE_MM = 10.0
  7457. # Feedrate for both moves. F600 is exactly what Bambu's own end G-code uses on
  7458. # this axis, so it is a proven-safe speed for the full travel.
  7459. _PLATE_RESTORE_FEEDRATE = 600
  7460. # Time allowed for the plate to reach the restored position before the camera
  7461. # grab. Sized for the ~100 mm the stock end G-code drops on X1/P1 at F600
  7462. # (10 mm/s); most H2 and P2 end G-code drops less, or not at all.
  7463. _PLATE_RESTORE_SETTLE_SECONDS = 12.0
  7464. # How long `_background_finish_photo` waits for this producer. Must cover the
  7465. # settle window plus a worst-case RTSP grab (15s), and stay below the
  7466. # notification path's own photo wait so a slow producer degrades to a
  7467. # photo-less notification rather than a missed one. The chamber light's
  7468. # snapshot delay (#1655) spends from the spare 8 seconds, which is why
  7469. # camera_light.MAX_DELAY_SECONDS stays below it.
  7470. _FINISH_PHOTO_PRODUCER_WAIT_SECONDS = _PLATE_RESTORE_SETTLE_SECONDS + 23.0
  7471. def _completion_print_keys(printer_id: int, filename: str, subtask_name: str) -> list[tuple[int, str]]:
  7472. """The ``_active_prints`` keys a finished print may be registered under.
  7473. Matching how they were registered in on_print_start. Shared by
  7474. ``on_print_complete`` and the finish-photo plate restore (#3240), which has
  7475. to find the same binding.
  7476. """
  7477. possible_keys = []
  7478. # Try subtask_name variations first (most reliable for matching)
  7479. if subtask_name:
  7480. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  7481. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  7482. possible_keys.append((printer_id, subtask_name))
  7483. # Try filename variations
  7484. if filename:
  7485. # Extract just the filename if it's a path
  7486. fname = filename.split("/")[-1] if "/" in filename else filename
  7487. if fname.endswith(".3mf"):
  7488. possible_keys.append((printer_id, fname))
  7489. elif fname.endswith(".gcode"):
  7490. base_name = fname.rsplit(".", 1)[0]
  7491. possible_keys.append((printer_id, f"{base_name}.gcode.3mf"))
  7492. possible_keys.append((printer_id, f"{base_name}.3mf"))
  7493. possible_keys.append((printer_id, fname))
  7494. else:
  7495. possible_keys.append((printer_id, f"{fname}.gcode.3mf"))
  7496. possible_keys.append((printer_id, f"{fname}.3mf"))
  7497. possible_keys.append((printer_id, fname))
  7498. # Also try full path versions
  7499. if filename.endswith(".3mf"):
  7500. possible_keys.append((printer_id, filename))
  7501. elif filename.endswith(".gcode"):
  7502. base_name = filename.rsplit(".", 1)[0]
  7503. possible_keys.append((printer_id, f"{base_name}.3mf"))
  7504. possible_keys.append((printer_id, filename))
  7505. else:
  7506. possible_keys.append((printer_id, f"{filename}.3mf"))
  7507. possible_keys.append((printer_id, filename))
  7508. return possible_keys
  7509. def _archive_name_forms(archive) -> set[str]:
  7510. """The names an archive could be echoed back as, in normalised form (#3240).
  7511. Its print name, and its filename with and without the 3MF extension.
  7512. """
  7513. forms = set()
  7514. if archive.print_name:
  7515. forms.add(_normalise_subtask_name(archive.print_name))
  7516. if archive.filename:
  7517. forms.add(_normalise_subtask_name(archive.filename))
  7518. for suffix in (".gcode.3mf", ".3mf"):
  7519. if archive.filename.lower().endswith(suffix):
  7520. forms.add(_normalise_subtask_name(archive.filename[: -len(suffix)]))
  7521. forms.discard("")
  7522. return forms
  7523. def _bound_print(printer_id: int, data: dict) -> tuple[int | None, int | None]:
  7524. """The archive and plate Bambuddy bound to the print ending on ``printer_id`` (#3240).
  7525. Read from ``_active_prints`` with the keys ``on_print_complete`` tries, and
  7526. from the plate recorded when the job was dispatched. Neither is touched:
  7527. ``on_print_complete`` pops both, which is why the finish-photo producer
  7528. calls this before its first await, while they are still there.
  7529. ``(None, None)`` for a print Bambuddy did not bind -- one it failed to
  7530. archive, or that it only knows by a name -- and on any error: the producer
  7531. calls this before the ``try`` that always releases the photo consumer, so
  7532. it must not raise.
  7533. """
  7534. try:
  7535. keys = _completion_print_keys(printer_id, str(data.get("filename") or ""), str(data.get("subtask_name") or ""))
  7536. archive_id = next((_active_prints[key] for key in keys if key in _active_prints), None)
  7537. if archive_id is None:
  7538. return None, None
  7539. return archive_id, _print_plate_ids.get((printer_id, archive_id))
  7540. except Exception:
  7541. logging.getLogger(__name__).debug("Could not read the print binding for printer %s", printer_id, exc_info=True)
  7542. return None, None
  7543. class PlateHeights(NamedTuple):
  7544. """Where the finished print is, and where its end G-code left the plate (#2547)."""
  7545. print_top: float
  7546. """The plate's ``max_z_height``: the Z the toolhead sat at for the last layer."""
  7547. end_z: float
  7548. """The last Z the end G-code moved to: where the plate is now, and where it is parked back to."""
  7549. async def _plate_heights_for_current_print(
  7550. printer_id: int,
  7551. data: dict,
  7552. logger,
  7553. archive_id: int | None,
  7554. plate_id: int | None = None,
  7555. ) -> PlateHeights | None:
  7556. """Heights of the print that just finished on ``printer_id``, or None (#2547).
  7557. These numbers become the targets of real Z moves, so every step here refuses
  7558. rather than guesses. Both have to be known: without the end Z there is no
  7559. position to park the plate back to that the printer's own G-code chose, and
  7560. the firmware won't stop a made-up one at the travel limit (#2579). A height belonging to some *other* print is the one
  7561. failure that could drive the nozzle into the model: 20 mm carried onto a
  7562. 200 mm print would command the plate up through the part.
  7563. Three independent things therefore have to agree before a height is
  7564. returned:
  7565. 1. **Binding.** ``archive_id`` is the archive Bambuddy bound to this print
  7566. when it started (see ``_bound_print``), never one found by name. A name
  7567. is not an identity: every plate of a multi-plate file shares one, and a
  7568. reprint reuses its archive row, so the newest archive with the right
  7569. name can be another plate of the same file. Support bundles show the
  7570. old name lookup doing exactly that (archive 288 for a print bound to
  7571. 169, 313 layers against 125), stopped only by the layer check below.
  7572. No binding, no move.
  7573. 2. **Identity.** The bound archive's name still has to be the finished
  7574. print's ``subtask_name``, compared in ``_normalise_subtask_name`` form
  7575. because the printer echoes spaces as underscores (#3240). Exact: not a
  7576. substring, and not the completion check's tolerance for a truncated
  7577. echo.
  7578. 3. **Corroboration.** The archive's layer count (parsed from the 3MF) has to
  7579. match the layer count the printer itself reported over MQTT for the print
  7580. that just ended. These come from genuinely different sources, so a
  7581. mismatch means the row is not this print, whatever its name says.
  7582. ``plate_id`` is the plate the job was dispatched with; a reprint of one
  7583. plate of a multi-plate file reuses the archive row without updating its
  7584. ``plate_id``. ``completed`` is accepted alongside ``printing`` only because
  7585. ``on_print_complete`` may already have flipped the status by the time we
  7586. run.
  7587. """
  7588. subtask_name = (data.get("subtask_name") or "").strip()
  7589. if not subtask_name:
  7590. # Nothing to identify the print by — refuse rather than fall back to
  7591. # "whatever ran last on this printer".
  7592. logger.info("[PLATE-RESTORE] printer %s: print has no name to match on — skipping", printer_id)
  7593. return None
  7594. if archive_id is None:
  7595. logger.info(
  7596. "[PLATE-RESTORE] printer %s: %r is not bound to an archive — skipping",
  7597. printer_id,
  7598. subtask_name,
  7599. )
  7600. return None
  7601. try:
  7602. from backend.app.models.archive import PrintArchive
  7603. from backend.app.utils.threemf_tools import extract_end_z_from_3mf, extract_max_z_height_from_3mf
  7604. async with async_session() as db:
  7605. result = await db.execute(
  7606. select(PrintArchive).where(
  7607. PrintArchive.id == archive_id,
  7608. PrintArchive.printer_id == printer_id,
  7609. PrintArchive.status.in_(("printing", "completed")),
  7610. PrintArchive.deleted_at.is_(None),
  7611. )
  7612. )
  7613. archive = result.scalar_one_or_none()
  7614. if archive is None or not archive.file_path:
  7615. logger.info("[PLATE-RESTORE] printer %s: archive %s has no usable file — skipping", printer_id, archive_id)
  7616. return None
  7617. if _normalise_subtask_name(subtask_name) not in _archive_name_forms(archive):
  7618. logger.warning(
  7619. "[PLATE-RESTORE] printer %s: archive %s is not named %r — refusing to move the plate",
  7620. printer_id,
  7621. archive_id,
  7622. subtask_name,
  7623. )
  7624. return None
  7625. client = printer_manager.get_client(printer_id)
  7626. reported_layers = getattr(getattr(client, "state", None), "total_layers", None)
  7627. if reported_layers and archive.total_layers and reported_layers != archive.total_layers:
  7628. logger.warning(
  7629. "[PLATE-RESTORE] printer %s: archive %s says %s layers but the printer reported %s "
  7630. "— refusing to move the plate on a height that may not be this print's",
  7631. printer_id,
  7632. archive.id,
  7633. archive.total_layers,
  7634. reported_layers,
  7635. )
  7636. return None
  7637. # Stored relative to the data directory, like every other archive path
  7638. # (#3240: this read a `data_dir` setting that does not exist).
  7639. path = app_settings.base_dir / archive.file_path
  7640. plate = plate_id or archive.plate_id or 1
  7641. print_top = await asyncio.to_thread(extract_max_z_height_from_3mf, path, plate)
  7642. if print_top is None:
  7643. return None
  7644. end_z = await asyncio.to_thread(extract_end_z_from_3mf, path, plate)
  7645. if end_z is None:
  7646. logger.info(
  7647. "[PLATE-RESTORE] printer %s: archive %s has no end G-code position to park at — skipping",
  7648. printer_id,
  7649. archive_id,
  7650. )
  7651. return None
  7652. return PlateHeights(print_top, end_z)
  7653. except Exception as e:
  7654. logger.debug("[PLATE-RESTORE] printer %s: no usable print height: %s", printer_id, e)
  7655. return None
  7656. async def _restore_plate_for_finish_photo(printer_id: int, max_z_height: float, logger) -> bool:
  7657. """Raise the plate back into camera framing before the finish photo (#2547).
  7658. On X1/P1/A1 Bambu's end G-code drops the plate ~100 mm as the last thing it
  7659. does, so by the time ``gcode_state`` reaches FINISH the finished print sits
  7660. far below the camera's natural framing — the complaint behind #1145, #1397
  7661. and #1565. This commands an absolute ``G1 Z`` back to just above the last
  7662. printed layer. Whether that is needed at all is ``_raise_plate_for_finish_photo``'s
  7663. call: plenty of end G-code (H2C, tall prints anywhere) leaves the plate in
  7664. view already.
  7665. Absolute, not relative, is the whole safety argument. ``max_z_height +
  7666. clearance`` is a height the toolhead was physically at seconds earlier, so
  7667. it is inside the travel limits by construction and leaves the nozzle above
  7668. the part. It is also unambiguous across model families: Z is the
  7669. nozzle-to-bed gap whether the bed moves (X1/P1/H2) or the toolhead does
  7670. (A1), so unlike the relative bed-jog path (#1334) there is no sign to get
  7671. wrong. ``M211`` is never touched — see the bed-jog docstring for why
  7672. (#2579).
  7673. Returns True if the move was sent and waited out, False if it was skipped.
  7674. """
  7675. client = printer_manager.get_client(printer_id)
  7676. if client is None:
  7677. return False
  7678. # Re-read state immediately before commanding motion. If the queue has
  7679. # already started the next print, the printer is no longer ours to move.
  7680. state = getattr(client, "state", None)
  7681. if state is None or state.state != "FINISH":
  7682. logger.info(
  7683. "[PLATE-RESTORE] printer %s is in state %s, not FINISH — skipping",
  7684. printer_id,
  7685. getattr(state, "state", "unknown"),
  7686. )
  7687. return False
  7688. target_z = max_z_height + _PLATE_RESTORE_CLEARANCE_MM
  7689. if not client.send_gcode(f"G90\nG1 Z{target_z:.2f} F{_PLATE_RESTORE_FEEDRATE}"):
  7690. logger.warning("[PLATE-RESTORE] printer %s: send failed — capturing where it is", printer_id)
  7691. return False
  7692. logger.info(
  7693. "[PLATE-RESTORE] printer %s: plate to Z%.2f (print top %.2f + %.1f clearance), settling %.0fs",
  7694. printer_id,
  7695. target_z,
  7696. max_z_height,
  7697. _PLATE_RESTORE_CLEARANCE_MM,
  7698. _PLATE_RESTORE_SETTLE_SECONDS,
  7699. )
  7700. await asyncio.sleep(_PLATE_RESTORE_SETTLE_SECONDS)
  7701. return True
  7702. def _park_plate_after_finish_photo(printer_id: int, end_z: float, logger) -> None:
  7703. """Put the plate back where the end G-code left it, after the finish photo (#2547).
  7704. ``end_z`` is the last Z the print's own end G-code moved to, never a height
  7705. worked out here. That is the safety argument: Bambu's templates keep their
  7706. end moves inside each model's travel, and the firmware does not clamp
  7707. G-code sent over MQTT (#2579). Parking at print top + 100 mm instead drove
  7708. the plate of a 238 mm H2C print to the bottom of its travel, a move Bambu's
  7709. own template had declined to make.
  7710. Fire-and-forget: if it doesn't land, the plate is merely high, and the next
  7711. print homes anyway.
  7712. """
  7713. client = printer_manager.get_client(printer_id)
  7714. state = getattr(client, "state", None) if client else None
  7715. if client is None or state is None or state.state != "FINISH":
  7716. return
  7717. client.send_gcode(f"G90\nG1 Z{end_z:.2f} F{_PLATE_RESTORE_FEEDRATE}")
  7718. logger.info("[PLATE-RESTORE] printer %s: plate back to the end G-code's Z%.2f", printer_id, end_z)
  7719. async def _raise_plate_for_finish_photo(
  7720. printer_id: int, data: dict, logger, archive_id: int | None, plate_id: int | None
  7721. ) -> float | None:
  7722. """Raise the plate into view for the finish photo if it needs it (#2547).
  7723. Returns the Z to park the plate back to once the photo is taken, or None
  7724. when the plate wasn't moved. It needs moving only when the end G-code
  7725. left it lower than the restore height; when it is already at or above it
  7726. (the H2C end G-code puts it back 2 mm under the nozzle), raising would be
  7727. a move away from the camera.
  7728. """
  7729. heights = await _plate_heights_for_current_print(printer_id, data, logger, archive_id, plate_id)
  7730. if heights is None:
  7731. logger.info("[PLATE-RESTORE] printer %s: print height unknown — capturing without restore", printer_id)
  7732. return None
  7733. if heights.end_z <= heights.print_top + _PLATE_RESTORE_CLEARANCE_MM:
  7734. logger.info(
  7735. "[PLATE-RESTORE] printer %s: end G-code left the plate at Z%.2f (print top %.2f) — already in view",
  7736. printer_id,
  7737. heights.end_z,
  7738. heights.print_top,
  7739. )
  7740. return None
  7741. if await _plate_restore_is_blocked_by_queue(printer_id):
  7742. logger.info("[PLATE-RESTORE] printer %s has queued work — skipping plate restore", printer_id)
  7743. return None
  7744. if await _restore_plate_for_finish_photo(printer_id, heights.print_top, logger):
  7745. return heights.end_z
  7746. return None
  7747. async def _plate_restore_is_blocked_by_queue(printer_id: int) -> bool:
  7748. """True if a queue item is about to take this printer (#2547).
  7749. The scheduler dispatches the next job the moment a print completes, and a
  7750. plate move interleaved with a print start is not a race worth having. The
  7751. state re-check in ``_restore_plate_for_finish_photo`` closes the tail of
  7752. this window; this closes the head of it.
  7753. """
  7754. try:
  7755. from backend.app.models.print_queue import PrintQueueItem
  7756. async with async_session() as db:
  7757. result = await db.execute(
  7758. select(PrintQueueItem.id)
  7759. .where(
  7760. PrintQueueItem.printer_id == printer_id,
  7761. PrintQueueItem.status.in_(("pending", "printing")),
  7762. )
  7763. .limit(1)
  7764. )
  7765. return result.scalar_one_or_none() is not None
  7766. except Exception as e:
  7767. # Fail closed: if we can't tell, don't move the plate.
  7768. logging.getLogger(__name__).debug(
  7769. "[PLATE-RESTORE] queue check failed for printer %s: %s — skipping restore", printer_id, e
  7770. )
  7771. return True
  7772. async def on_finish_photo_moment(printer_id: int, data: dict):
  7773. """Pre-capture a finish photo when the printer enters stage 22 / FINISH (#1721).
  7774. Fires either at the stage-22 ("Filament unloading") edge — toolhead
  7775. parked, bed not yet dropped, optimal framing — or as a FINISH-state
  7776. fallback for prints that skip stage 22 (cancel, external-spool-only,
  7777. HMS halt, firmware variants). Grabs one frame via the same
  7778. external-camera / RTSP path the post-completion fallback uses, stores
  7779. the JPEG bytes in ``_stage22_finish_frames[printer_id]``, and lets
  7780. ``_background_finish_photo`` consume the cached bytes when it runs.
  7781. Replaces the #1397 "force timelapse on at dispatch" mechanism, which
  7782. caused per-layer nozzle parking on slicer profiles with Timelapse Type
  7783. set to Smooth (#1721). No force-on now means the user's explicit
  7784. timelapse=off in the slicer send dialog is respected.
  7785. """
  7786. logger = logging.getLogger(__name__)
  7787. trigger = data.get("trigger", "unknown")
  7788. timelapse_was_active = bool(data.get("timelapse_was_active"))
  7789. logger.info(
  7790. "[FINISH-PHOTO-MOMENT] printer=%s trigger=%s timelapse_active=%s",
  7791. printer_id,
  7792. trigger,
  7793. timelapse_was_active,
  7794. )
  7795. # If a timelapse is actively recording, skip the pre-capture — the
  7796. # post-completion path will extract the last frame from the recorded
  7797. # video, which still provides the best framing (toolhead parked,
  7798. # before bed drop) without the per-layer parking side effects.
  7799. if timelapse_was_active:
  7800. logger.info(
  7801. "[FINISH-PHOTO-MOMENT] timelapse active for printer %s — skipping pre-capture (last-frame extraction will run post-completion)",
  7802. printer_id,
  7803. )
  7804. return
  7805. # #1790: register the producer-done event BEFORE the first await so the
  7806. # consumer in `_background_finish_photo` — which is dispatched back-to-back
  7807. # with us on the FINISH-state fallback path — sees it as soon as it polls.
  7808. # The `finally` below guarantees `set()` runs on every exit, including
  7809. # early returns and exceptions, so the consumer's bounded wait can't hang.
  7810. producer_done = asyncio.Event()
  7811. _stage22_finish_in_flight[printer_id] = producer_done
  7812. # #3240: also before the first await. On the FINISH-state path
  7813. # `on_print_complete` is dispatched right behind us and pops both the
  7814. # print's archive binding and its plate; read here, they are still there.
  7815. bound_archive_id, bound_plate_id = _bound_print(printer_id, data)
  7816. # #2547: the Z to park the plate back to, set once the plate has actually
  7817. # been raised, and read by the `finally` below. Declared out here so a
  7818. # failure anywhere after the move — a camera timeout, a DB error — still
  7819. # lowers the plate again.
  7820. park_z: float | None = None
  7821. try:
  7822. async with async_session() as db:
  7823. from backend.app.api.routes.settings import get_setting
  7824. from backend.app.models.printer import Printer
  7825. capture_setting = await get_setting(db, "capture_finish_photo")
  7826. if capture_setting is not None and capture_setting.lower() != "true":
  7827. logger.info("[FINISH-PHOTO-MOMENT] capture_finish_photo disabled — skipping pre-capture")
  7828. return
  7829. restore_setting = await get_setting(db, "finish_photo_restore_plate")
  7830. restore_plate_enabled = restore_setting is None or restore_setting.lower() == "true"
  7831. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  7832. printer = result.scalar_one_or_none()
  7833. if printer is None:
  7834. logger.warning(
  7835. "[FINISH-PHOTO-MOMENT] printer %s not found in DB",
  7836. printer_id,
  7837. )
  7838. return
  7839. frame_bytes: bytes | None = None
  7840. # #2708: the banked frame arrives already rotated — it comes from
  7841. # `_capture_snapshot_for_notification`, which rotates before returning.
  7842. # Every other source below is a raw grab. Tracking which lets us store
  7843. # exactly one rotation in `_stage22_finish_frames` either way.
  7844. frame_already_rotated = False
  7845. # On the FINISH-state path the End G-code has already run, and two very
  7846. # different situations arrive here needing opposite answers.
  7847. #
  7848. # #1867: if Bambuddy injected End G-code into this print, a SwapMod
  7849. # snippet may have ejected the plate — the scene in front of the camera
  7850. # is no longer the finished print, and no amount of moving the plate
  7851. # brings it back. Use the banked in-print frame instead.
  7852. #
  7853. # #2547: otherwise the print is still sitting there — on X1/P1/A1 ~100 mm
  7854. # lower than the camera frames well — and the toolhead is parked out of
  7855. # the way. That is the *best* moment available on firmware that never emits
  7856. # stage 22 (H2C, A1 Mini) — so capture live, after putting the plate
  7857. # back. Preferring the bank here unconditionally, as this code used to,
  7858. # is what shipped a mid-print photo with the toolhead over the part.
  7859. if trigger == "finish_state" and print_dispatch_context.end_gcode_injected(printer_id):
  7860. banked = _inprint_frame_bank.get(printer_id)
  7861. if banked:
  7862. frame_bytes = banked
  7863. frame_already_rotated = True
  7864. logger.info(
  7865. "[FINISH-PHOTO-MOMENT] End G-code was injected — using banked in-print "
  7866. "frame (%d bytes) instead of a post-swap live grab",
  7867. len(banked),
  7868. )
  7869. else:
  7870. logger.warning(
  7871. "[FINISH-PHOTO-MOMENT] End G-code was injected for printer %s but the "
  7872. "in-print bank is empty — falling back to a live grab, which may show a "
  7873. "swapped or empty plate",
  7874. printer_id,
  7875. )
  7876. # `park_z` is set only once the plate is actually up, because the
  7877. # `finally` reads it to decide whether it owes a move back down.
  7878. #
  7879. # Never on a print whose End G-code Bambuddy injected, even when the bank
  7880. # came up empty above: that machine may have just ejected its plate, and
  7881. # driving Z into whatever a swap mechanism is doing is not a risk worth
  7882. # taking for a photo of a bed we already know may be bare.
  7883. if (
  7884. frame_bytes is None
  7885. and trigger == "finish_state"
  7886. and restore_plate_enabled
  7887. and not print_dispatch_context.end_gcode_injected(printer_id)
  7888. ):
  7889. park_z = await _raise_plate_for_finish_photo(printer_id, data, logger, bound_archive_id, bound_plate_id)
  7890. # Chamber light for the live capture below (#1655). The banked frame
  7891. # above was taken without it, like every in-print bank.
  7892. async with camera_light(printer if frame_bytes is None else None):
  7893. if frame_bytes is None and printer.external_camera_enabled and printer.external_camera_url:
  7894. from backend.app.api.routes.camera import live_frame_for_capture
  7895. from backend.app.services.external_camera import capture_frame
  7896. # #2707: this used to collide with the live view and fail, which is
  7897. # how finish-photo notifications went out with no image attached.
  7898. # Leaving frame_bytes None keeps the rest of the fallback chain.
  7899. defer, buffered = live_frame_for_capture(printer_id)
  7900. if defer:
  7901. frame_bytes = buffered
  7902. else:
  7903. frame_bytes = await capture_frame(
  7904. printer.external_camera_url,
  7905. printer.external_camera_type or "mjpeg",
  7906. snapshot_url=printer.external_camera_snapshot_url,
  7907. )
  7908. if frame_bytes:
  7909. logger.info(
  7910. "[FINISH-PHOTO-MOMENT] captured external-camera frame (%d bytes)",
  7911. len(frame_bytes),
  7912. )
  7913. elif frame_bytes is None:
  7914. from backend.app.api.routes.camera import get_buffered_frame
  7915. buffered = get_buffered_frame(printer_id)
  7916. if buffered:
  7917. frame_bytes = buffered
  7918. logger.info(
  7919. "[FINISH-PHOTO-MOMENT] used buffered RTSP frame (%d bytes)",
  7920. len(frame_bytes),
  7921. )
  7922. else:
  7923. from backend.app.services.camera import capture_camera_frame_bytes
  7924. frame_bytes = await capture_camera_frame_bytes(
  7925. ip_address=printer.ip_address,
  7926. access_code=printer.access_code,
  7927. model=printer.model,
  7928. timeout=15,
  7929. )
  7930. if frame_bytes:
  7931. logger.info(
  7932. "[FINISH-PHOTO-MOMENT] captured RTSP frame (%d bytes)",
  7933. len(frame_bytes),
  7934. )
  7935. if frame_bytes:
  7936. if not frame_already_rotated:
  7937. frame_bytes = _apply_camera_rotation(frame_bytes, printer, logger)
  7938. _stage22_finish_frames[printer_id] = frame_bytes
  7939. else:
  7940. logger.warning(
  7941. "[FINISH-PHOTO-MOMENT] no frame captured for printer %s — post-completion fallback will retry",
  7942. printer_id,
  7943. )
  7944. except Exception as e:
  7945. logger.warning(
  7946. "[FINISH-PHOTO-MOMENT] pre-capture failed for printer %s: %s",
  7947. printer_id,
  7948. e,
  7949. )
  7950. finally:
  7951. # #2547: we raised the plate, so we own lowering it — including when the
  7952. # capture above failed or threw partway through.
  7953. if park_z is not None:
  7954. try:
  7955. _park_plate_after_finish_photo(printer_id, park_z, logger)
  7956. except Exception as e:
  7957. logger.warning("[PLATE-RESTORE] printer %s: could not lower plate: %s", printer_id, e)
  7958. # #1790: always unblock the consumer's bounded wait — whether we stored
  7959. # a frame, gave up, or hit an exception. Local ref means cleanup of the
  7960. # dict entry by the consumer doesn't affect signalling.
  7961. producer_done.set()
  7962. def _subtask_name_from_filename(filename: str) -> str:
  7963. """Recover the subtask name a print command would have carried for *filename*.
  7964. The dispatcher derives the printer-facing subtask name from the archive's
  7965. file name, so stripping the extensions back off gives the value MQTT echoes
  7966. on completion. Only the two extensions Bambuddy actually stores are removed,
  7967. and in the order they nest (``.gcode.3mf``), so a model whose own name
  7968. contains a dot -- ``My.Model.3mf`` -- keeps it.
  7969. """
  7970. name = PurePosixPath(filename).name
  7971. for suffix in (".3mf", ".gcode"):
  7972. if name.lower().endswith(suffix):
  7973. name = name[: -len(suffix)]
  7974. return name
  7975. def _subtask_is_the_file(subtask_name: str, filename: str) -> bool:
  7976. """Whether the subtask name is the printed 3MF's own file name.
  7977. A print started from the printer's screen reports the same full name, with
  7978. its extension, as both subtask and file (#3009). A dispatched print reports
  7979. a bare subtask name; one that merely ends in ".3mf" can be a model named
  7980. that, whose file is ``Foo.3mf.gcode.3mf``, so it does not count.
  7981. """
  7982. if not subtask_name or not subtask_name.lower().endswith(".3mf"):
  7983. return False
  7984. return PurePosixPath((filename or "").split("://", 1)[-1]).name == subtask_name
  7985. # How the printer marks a subtask name it had to cut short. Observed on real
  7986. # hardware at ~100 characters, but the cut-off is not a fixed character count
  7987. # (a name with multibyte characters came back at 98), so match the marker
  7988. # rather than a length.
  7989. _SUBTASK_TRUNCATION_MARKER = "..."
  7990. def _normalise_subtask_name(name: str) -> str:
  7991. """Canonical form for comparing a dispatched name against MQTT's echo.
  7992. The printer does not echo the name back verbatim: it substitutes
  7993. underscores for spaces. ``H2D_Carbon_Filter_(V2)_Body & Solid Lid`` is
  7994. dispatched and ``H2D_Carbon_Filter_(V2)_Body_&_Solid_Lid`` comes back.
  7995. The 3MF lookup in this module has always known that -- it builds
  7996. space-to-underscore variants of every candidate filename, and its
  7997. directory search normalises both sides before comparing. This exists so
  7998. the completion check reads the same rule from the same place instead of
  7999. growing its own, which is exactly how it came to disagree (#2829).
  8000. A space at either end is substituted too: ``Part .gcode.3mf`` is
  8001. dispatched as ``Part `` and echoed as ``Part_``. Underscores at the ends
  8002. are dropped after the substitution so both sides agree (#3241).
  8003. """
  8004. return name.strip().replace(" ", "_").strip("_").casefold()
  8005. def _3mf_files_for_print(paths: list[str], search_term: str) -> list[str]:
  8006. """The 3MF files of the printer's folder listings that are this print's.
  8007. A file of the print's own name wins, in whichever folder. Only without one
  8008. is a file that merely contains the name taken, and then only when it is
  8009. the one such file: "Cube_large.3mf" is not "Cube"'s file, and of several
  8010. candidates none can be told apart. Paths keep the listings' order.
  8011. """
  8012. threemf = [path for path in paths if path.lower().endswith(".3mf")]
  8013. exact = [path for path in threemf if _subtask_names_match(_subtask_name_from_filename(path), search_term)]
  8014. if exact:
  8015. return exact
  8016. term = _normalise_subtask_name(search_term)
  8017. partial = [path for path in threemf if term in _normalise_subtask_name(PurePosixPath(path).name)]
  8018. if len(partial) > 1:
  8019. logging.getLogger(__name__).info(
  8020. "Several 3MF files contain %r and none is named for it: %s", search_term, partial
  8021. )
  8022. return []
  8023. return partial
  8024. def _subtask_names_match(expected: str, observed: str) -> bool:
  8025. """Whether two subtask names describe the same print.
  8026. Beyond the space/underscore substitution, the printer truncates long names
  8027. and marks the cut with ``...``. A truncated echo has to count as a match or
  8028. every print with a long name strands its queue item the same way.
  8029. """
  8030. expected_n = _normalise_subtask_name(expected)
  8031. observed_n = _normalise_subtask_name(observed)
  8032. if expected_n == observed_n:
  8033. return True
  8034. # Either side can be the truncated one: the printer truncates what it
  8035. # echoes, and an archive whose own filename was recorded from a previous
  8036. # truncated echo carries the marker too.
  8037. for full, cut in ((expected_n, observed_n), (observed_n, expected_n)):
  8038. if not cut.endswith(_SUBTASK_TRUNCATION_MARKER):
  8039. continue
  8040. # The full side has lost its edge underscores; a cut that lands right
  8041. # after one keeps it, so drop it here too (#3241). Nothing left before
  8042. # the marker is no evidence of anything, and must not match every name.
  8043. kept = cut[: -len(_SUBTASK_TRUNCATION_MARKER)].rstrip("_")
  8044. if kept and full.startswith(kept):
  8045. return True
  8046. return False
  8047. def _dispatched_name_matches_echo(expected: str, observed: str) -> bool:
  8048. """Whether the printer's subtask echo is the name a print was dispatched as.
  8049. A dispatched print is echoed under its bare name at start, but a printer
  8050. that picks a print back up -- Bambuddy reconnecting to a print already
  8051. running, or an A1 mini resuming after power-loss recovery -- can report the
  8052. file name with its extension instead (#3485). The echo is tried both as it
  8053. came and with the extensions stripped; the raw form still has to count, so
  8054. a model named ``Foo.3mf`` (file ``Foo.3mf.gcode.3mf``) keeps matching.
  8055. """
  8056. return _subtask_names_match(expected, observed) or _subtask_names_match(
  8057. expected, _subtask_name_from_filename(observed)
  8058. )
  8059. async def _queue_item_dispatched_name(db, item) -> str:
  8060. """The subtask name *item* was dispatched under, or "" when unknowable.
  8061. A row with no archive, or an archive with no file name, is unverifiable
  8062. rather than wrong; every caller answers that with its permissive branch.
  8063. """
  8064. if item.archive_id is None:
  8065. return ""
  8066. from backend.app.models.archive import PrintArchive
  8067. archive = await db.get(PrintArchive, item.archive_id)
  8068. if archive is None or not archive.filename:
  8069. return ""
  8070. return _subtask_name_from_filename(archive.filename)
  8071. async def _completion_belongs_to_queue_item(db, item, data: dict) -> bool:
  8072. """Whether this completion event is plausibly about *item*'s print.
  8073. The caller finds its queue row by printer and ``status='printing'`` alone,
  8074. which is all a completion event gives it -- there is no run identifier in
  8075. the MQTT payload to match on. That makes the lookup indiscriminate: any
  8076. completion delivered for this printer closes whichever row happens to be
  8077. printing, however unrelated. Comparing the subtask name against the archive
  8078. the row was dispatched with costs one primary-key load and rules that out.
  8079. Deliberately permissive: it answers False only on a positive disagreement
  8080. between two names we actually have. A row with no archive, an archive with
  8081. no file name, or an event with no subtask name is unverifiable rather than
  8082. wrong, and refusing those would strand the item in ``printing`` and wedge
  8083. the printer's queue -- a worse failure than the one being prevented.
  8084. """
  8085. observed = (data.get("subtask_name") or "").strip()
  8086. if not observed or item.archive_id is None:
  8087. return True
  8088. from backend.app.models.archive import PrintArchive
  8089. archive = await db.get(PrintArchive, item.archive_id)
  8090. if archive is None or not archive.filename:
  8091. return True
  8092. expected = _subtask_name_from_filename(archive.filename)
  8093. if not expected or _dispatched_name_matches_echo(expected, observed):
  8094. return True
  8095. logging.getLogger(__name__).warning(
  8096. "Ignoring print completion for queue item %s: it was dispatched as %r "
  8097. "(archive %s, %s) but the completion reports subtask %r. Leaving the item "
  8098. "printing rather than closing a run this event is not about.",
  8099. item.id,
  8100. expected,
  8101. archive.id,
  8102. archive.filename,
  8103. observed,
  8104. )
  8105. return False
  8106. async def _recover_fallback_from_cache_before_eviction(printer_id: int, data: dict) -> None:
  8107. """Spend the 3MF download cache on a still-empty fallback archive.
  8108. ``on_print_complete`` drops the cache as its first act, which deletes the
  8109. file. If the cover endpoint (or anything else) pulled the 3MF while the
  8110. print ran and the archive never got one, this is the last moment those bytes
  8111. exist (#2957).
  8112. """
  8113. logger = logging.getLogger(__name__)
  8114. names = [
  8115. n
  8116. for n in (data.get("filename"), data.get("subtask_name"), (data.get("raw_data") or {}).get("subtask_name"))
  8117. if n
  8118. ]
  8119. for name in names:
  8120. try:
  8121. cached = get_cached_3mf(printer_id, name)
  8122. # The print has ended: its completion prices the archive, so no
  8123. # print-start estimate is scheduled to race it.
  8124. if cached and await try_recover_fallback_archive(printer_id, name, cached, price=False):
  8125. return
  8126. except Exception as e:
  8127. logger.debug("[RECOVER] Pre-eviction recovery for %s failed: %s", name, e)
  8128. # Longest the print's end waits for its file to be deleted from the SD card
  8129. _SD_CLEANUP_BUDGET_SECONDS = 45.0
  8130. async def _cleanup_sd_card_after_print(
  8131. printer_id: int, subtask_name: str | None, archive_id: int | None, logger
  8132. ) -> None:
  8133. """Delete a finished print's file from the printer's SD card."""
  8134. # Cleanup: delete uploaded file from printer SD card to prevent phantom prints (Issue #374, #1542)
  8135. # The print scheduler uploads files to the SD card root (/). Some printers (e.g. P1S, A1)
  8136. # auto-start files found in root on power cycle, causing ghost prints.
  8137. # Must run before the archive_id early-return so it executes even when archiving is disabled.
  8138. try:
  8139. if subtask_name:
  8140. archive_filename: str | None = None
  8141. async with async_session() as db:
  8142. from backend.app.models.archive import PrintArchive
  8143. from backend.app.models.printer import Printer
  8144. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8145. printer = result.scalar_one_or_none()
  8146. if archive_id:
  8147. archive_row = await db.execute(select(PrintArchive.filename).where(PrintArchive.id == archive_id))
  8148. archive_filename = archive_row.scalar_one_or_none()
  8149. if printer:
  8150. from backend.app.services.bambu_ftp import DeleteResult, delete_file_async
  8151. from backend.app.utils.filename import derive_remote_filename
  8152. # Primary candidate: the exact path the dispatcher uploaded to
  8153. # (derived from archive.filename via the same rule as upload).
  8154. # Without it, a library row that ended up with a doubled
  8155. # .gcode.3mf (#1542) leaves the real file behind because the
  8156. # subtask_name + ext fallbacks below don't match what's on the
  8157. # SD card. Fallbacks remain for archive-less prints (subtask
  8158. # never resolved to an archive) and for older naming variants.
  8159. candidate_paths: list[str] = []
  8160. if archive_filename:
  8161. candidate_paths.append(f"/{derive_remote_filename(archive_filename)}")
  8162. for ext in (".3mf", ".gcode"):
  8163. fallback = f"/{subtask_name}{ext}"
  8164. if fallback not in candidate_paths:
  8165. candidate_paths.append(fallback)
  8166. # Three outcomes track across all candidates so the final log
  8167. # line reflects what actually happened. The A1 in #1721 always
  8168. # ends here with ``any_not_found=True`` and the others False
  8169. # — its firmware auto-cleans the SD card before our cleanup
  8170. # runs, every candidate FTP-DELE returns 550, and the old
  8171. # code burned 3 retries × 2 s × 3 candidates per print
  8172. # logging a misleading "may linger" WARNING on a successful
  8173. # print.
  8174. any_deleted = False
  8175. any_real_failure = False
  8176. any_not_found = False
  8177. files_in_use = _sd_files_in_use(printer_manager.get_status(printer_id))
  8178. for remote_path in candidate_paths:
  8179. if PurePosixPath(remote_path).name.lower() in files_in_use:
  8180. logger.info(
  8181. "SD card cleanup: keeping %s on printer %s, which is printing it now",
  8182. remote_path,
  8183. printer.name,
  8184. )
  8185. continue
  8186. # Retry only the FAILED case — 550 NOT_FOUND will never
  8187. # recover by waiting, so a "file isn't here" answer
  8188. # advances immediately to the next candidate without
  8189. # consuming the retry budget.
  8190. for attempt in range(1, 4):
  8191. try:
  8192. delete_result = await delete_file_async(
  8193. printer.ip_address,
  8194. printer.access_code,
  8195. remote_path,
  8196. printer_model=printer.model,
  8197. )
  8198. except Exception as e:
  8199. delete_result = DeleteResult.FAILED
  8200. logger.warning(
  8201. "SD card cleanup attempt %d/3 raised for %s: %s",
  8202. attempt,
  8203. remote_path,
  8204. e,
  8205. )
  8206. if delete_result == DeleteResult.DELETED:
  8207. any_deleted = True
  8208. logger.info("Deleted %s from printer %s SD card", remote_path, printer.name)
  8209. break
  8210. if delete_result == DeleteResult.NOT_FOUND:
  8211. any_not_found = True
  8212. break # 550 will not recover; try next candidate
  8213. # FAILED: real error — retry with backoff, then give up
  8214. if attempt < 3:
  8215. await asyncio.sleep(2)
  8216. else:
  8217. any_real_failure = True
  8218. logger.warning(
  8219. "SD card cleanup failed after 3 attempts for %s "
  8220. "(network/auth/transient error — file may linger on SD card)",
  8221. remote_path,
  8222. )
  8223. if not any_deleted and not any_real_failure and any_not_found:
  8224. # Every candidate said "not here." Either the printer
  8225. # firmware swept the SD card itself (common on A1) or the
  8226. # dispatcher's upload path doesn't match our candidate
  8227. # rule. Either way: nothing to clean up, no warning.
  8228. logger.debug(
  8229. "SD card cleanup: nothing to delete on %s — every candidate returned 550 "
  8230. "(printer likely self-cleaned)",
  8231. printer.name,
  8232. )
  8233. except Exception as e:
  8234. logger.warning("SD card file cleanup failed for printer %s: %s", printer_id, e)
  8235. async def on_print_complete(printer_id: int, data: dict):
  8236. """Handle print completion - update the archive status.
  8237. The printer counts as printing until the print's usage is booked (or this
  8238. ends), so the AMS remain% weight sync can't charge it a second time while
  8239. the usage tracker works (WU-3). Marked before the first await.
  8240. """
  8241. from backend.app.services.usage_tracker import done_completing, mark_completing
  8242. mark_completing(printer_id)
  8243. try:
  8244. await _close_out_print(printer_id, data)
  8245. finally:
  8246. done_completing(printer_id)
  8247. async def _close_out_print(printer_id: int, data: dict):
  8248. import time
  8249. logger = logging.getLogger(__name__)
  8250. start_time = time.time()
  8251. def log_timing(section: str):
  8252. elapsed = time.time() - start_time
  8253. logger.info("[TIMING] %s: %.3fs elapsed", section, elapsed)
  8254. logger.info("[CALLBACK] on_print_complete started for printer %s", printer_id)
  8255. octoeverywhere_detection_service.reset_printer(printer_id)
  8256. # A kill-switch stop sends its provider notification immediately. Keep the
  8257. # task so the later notification path can await it and avoid a duplicate;
  8258. # if that immediate attempt failed, the regular completion path retries.
  8259. kill_switch_notification_task = _kill_switch_notification_tasks.pop(printer_id, None)
  8260. # Last chance before the bytes go: if this print's archive is still an empty
  8261. # fallback and something downloaded the 3MF while it ran, fill the archive in
  8262. # now. The cover endpoint's copy lives in exactly this cache, and clearing it
  8263. # below deletes the file (#2957).
  8264. await _recover_fallback_from_cache_before_eviction(printer_id, data)
  8265. # A pending cool-off retry has nothing left to recover for — the cache is
  8266. # about to be dropped and the print is over.
  8267. retry_task = _fallback_3mf_retry_tasks.pop(printer_id, None)
  8268. if retry_task and not retry_task.done():
  8269. retry_task.cancel()
  8270. # Drop the 3MF download cache for this printer (#972). The print is over,
  8271. # nothing else legitimately needs the bytes; keeping them would only risk
  8272. # handing a stale file to the next print if it reuses the same name.
  8273. clear_3mf_cache(printer_id)
  8274. try:
  8275. ws_data = {
  8276. "status": data.get("status"),
  8277. "filename": data.get("filename"),
  8278. "subtask_name": data.get("subtask_name"),
  8279. "timelapse_was_active": data.get("timelapse_was_active"),
  8280. }
  8281. await ws_manager.send_print_complete(printer_id, ws_data)
  8282. log_timing("WebSocket send_print_complete")
  8283. except Exception as e:
  8284. logger.warning("[CALLBACK] WebSocket send_print_complete failed: %s", e)
  8285. # Capture user info before clearing (needed for print log entry)
  8286. _print_user_info = printer_manager.get_current_print_user(printer_id)
  8287. # Clear current print user tracking (Issue #206)
  8288. printer_manager.clear_current_print_user(printer_id)
  8289. # If the user explicitly stopped this print from the queue UI the printer will
  8290. # report "failed" or "aborted" via MQTT. Override that to "cancelled" so the
  8291. # correct "print stopped" notification/email is sent instead of a failure alert.
  8292. _raw_status = data.get("status", "completed")
  8293. if printer_id in _user_stopped_printers and _raw_status in ("failed", "aborted"):
  8294. logger.info(
  8295. "[CALLBACK] Overriding status '%s' -> 'cancelled' for printer %s (print was stopped from queue by user)",
  8296. _raw_status,
  8297. printer_id,
  8298. )
  8299. data = {**data, "status": "cancelled"}
  8300. elif printer_id in _kill_switch_stopped and _raw_status in ("failed", "aborted"):
  8301. logger.info(
  8302. "[CALLBACK] Overriding status '%s' -> 'cancelled' for printer %s (stopped by the printer kill switch)",
  8303. _raw_status,
  8304. printer_id,
  8305. )
  8306. data = {**data, "status": "cancelled", "_kill_switch": True}
  8307. _user_stopped_printers.discard(printer_id)
  8308. _kill_switch_stopped.discard(printer_id)
  8309. # Raise the plate-clear gate for queued dispatch (#961). Any terminal status
  8310. # may have left material on the bed: a user can cancel ten hours into a
  8311. # twelve-hour print, a printer can self-abort mid-job after a clog, and a
  8312. # touchscreen-stop reports `aborted` rather than `cancelled` because
  8313. # `_user_stopped_printers` is only populated when the user stops via the
  8314. # Bambuddy queue UI. Earlier code raised the flag only for completed/failed,
  8315. # which auto-dispatched the next queued print onto a fouled bed two seconds
  8316. # after a touchscreen-abort (#1171). Persisted to DB so the gate survives
  8317. # Auto Off power cycles and Bambuddy restarts.
  8318. try:
  8319. notify_live_activities.print_finished(printer_id, data)
  8320. except Exception:
  8321. logger.exception("Notify Live Activity completion hook failed for printer %s", printer_id)
  8322. _final_status = data.get("status", "completed")
  8323. if _final_status in ("completed", "failed", "aborted", "cancelled"):
  8324. printer_manager.set_awaiting_plate_clear(printer_id, True)
  8325. # MQTT relay - publish print complete
  8326. try:
  8327. printer_info = printer_manager.get_printer(printer_id)
  8328. if printer_info:
  8329. await mqtt_relay.on_print_complete(
  8330. printer_id,
  8331. printer_info.name,
  8332. printer_info.serial_number,
  8333. data.get("filename", ""),
  8334. data.get("subtask_name", ""),
  8335. data.get("status", "completed"),
  8336. )
  8337. except Exception:
  8338. pass # Don't fail print complete callback if MQTT fails
  8339. filename = data.get("filename", "")
  8340. subtask_name = data.get("subtask_name", "")
  8341. if not filename and not subtask_name:
  8342. # Which print ended can't be told: the printer reported no name since
  8343. # Bambuddy started (a print that failed while preparing, a calibration).
  8344. # Charge nothing, but close the print session, which otherwise kept the
  8345. # AMS weight sync off until the next print (AS-5). A print archived as
  8346. # running is closed and charged by the reconnect reconcile.
  8347. logger.warning("Print complete without filename or subtask_name")
  8348. try:
  8349. from backend.app.services.usage_tracker import discard_session
  8350. async with async_session() as db:
  8351. await discard_session(db, printer_id)
  8352. except Exception as e:
  8353. logger.warning("Failed to clear print session for printer %s: %s", printer_id, e)
  8354. return
  8355. logger.info("Print complete - filename: %s, subtask: %s, status: %s", filename, subtask_name, data.get("status"))
  8356. # Build list of possible keys to try (matching how they were registered in on_print_start)
  8357. possible_keys = _completion_print_keys(printer_id, filename, subtask_name)
  8358. # Find the archive for this print
  8359. logger.info("Looking for archive in _active_prints, keys to try: %s...", possible_keys[:5])
  8360. logger.info("Current _active_prints: %s", list(_active_prints.keys()))
  8361. archive_id = None
  8362. for key in possible_keys:
  8363. archive_id = _active_prints.pop(key, None)
  8364. if archive_id:
  8365. _owned_print_archive_ids.discard(archive_id)
  8366. logger.info("Found archive %s with key %s", archive_id, key)
  8367. # Also clean up the printer's other keys pointing to this archive;
  8368. # another printer's are a plate of it still printing there.
  8369. keys_to_remove = [k for k, v in _active_prints.items() if v == archive_id and k[0] == printer_id]
  8370. for k in keys_to_remove:
  8371. _active_prints.pop(k, None)
  8372. break
  8373. # The archive bound when the print started, before the name search below.
  8374. # Only this one may give the finish-photo plate restore a height (#3240):
  8375. # the search matches names by substring.
  8376. bound_archive_id = archive_id
  8377. if not archive_id and data.get("_reconciled") and data.get("_reconciled_archive_id"):
  8378. # The reconcile names the archive whose run on this printer it
  8379. # closes. Only while that run is open: the print's real end may have
  8380. # closed it meanwhile, or closed the print without managing to write
  8381. # it, and either way there is nothing left to end twice.
  8382. if await _archive_of_open_run(printer_id) != data["_reconciled_archive_id"]:
  8383. logger.info(
  8384. "[RECONCILE] Archive %s has no run open on printer %s any more; nothing to close",
  8385. data["_reconciled_archive_id"],
  8386. printer_id,
  8387. )
  8388. return
  8389. archive_id = data["_reconciled_archive_id"]
  8390. ending_job = str(data.get("subtask_id") or "").strip()
  8391. if not archive_id and ending_job not in ("", "0"):
  8392. # Not bound (a restart since the start): this job's run, open on the
  8393. # printer. Only by its job id: a run open there can be of the print
  8394. # running now, while the end is of an earlier one closed late (the
  8395. # restart's reconcile of a missed end).
  8396. archive_id = await _archive_of_open_run(printer_id, subtask_id=ending_job)
  8397. if archive_id:
  8398. logger.info("Found archive %s by the run of job %s open on printer %s", archive_id, ending_job, printer_id)
  8399. if not archive_id:
  8400. # Try to find by filename or subtask_name if not tracked (for prints started before app)
  8401. async with async_session() as db:
  8402. from backend.app.models.archive import PrintArchive
  8403. # Try matching by subtask_name (stored as print_name) first. The
  8404. # same name rule as the queue's completion check: a name inside
  8405. # another ("Clip" in "Cable Clip") is a different print, and the
  8406. # printer's underscores for spaces and truncation still match.
  8407. if subtask_name:
  8408. printing = await db.scalars(
  8409. select(PrintArchive)
  8410. .where(PrintArchive.printer_id == printer_id)
  8411. .where(PrintArchive.status == "printing")
  8412. .order_by(PrintArchive.created_at.desc())
  8413. )
  8414. # A print started from the printer's screen reports its file
  8415. # name, extensions included, as the subtask (#3009).
  8416. reported = {subtask_name, _subtask_name_from_filename(subtask_name)}
  8417. archive = next(
  8418. (
  8419. candidate
  8420. for candidate in printing
  8421. if any(
  8422. (candidate.print_name and _subtask_names_match(candidate.print_name, name))
  8423. or (
  8424. candidate.filename
  8425. and _subtask_names_match(_subtask_name_from_filename(candidate.filename), name)
  8426. )
  8427. for name in reported
  8428. )
  8429. ),
  8430. None,
  8431. )
  8432. if archive:
  8433. archive_id = archive.id
  8434. logger.info("Found archive %s by subtask_name match: %s", archive_id, subtask_name)
  8435. # Also try by filename
  8436. if not archive_id and filename:
  8437. result = await db.execute(
  8438. select(PrintArchive)
  8439. .where(PrintArchive.printer_id == printer_id)
  8440. .where(PrintArchive.filename == filename)
  8441. .where(PrintArchive.status == "printing")
  8442. .order_by(PrintArchive.created_at.desc())
  8443. .limit(1)
  8444. )
  8445. archive = result.scalar_one_or_none()
  8446. if archive:
  8447. archive_id = archive.id
  8448. # Bounded: a printer whose FTP hangs on DELE took up to 3 candidates x 3
  8449. # attempts x a 60 s timeout, about nine minutes, before the queue item was
  8450. # closed below. The stranded-job closer gives up after five and closed it
  8451. # first, so the job's auto-off and its billing never ran. Inline rather
  8452. # than in the background, so the next job of the same file isn't released
  8453. # (and uploaded under the name being deleted) while the cleanup still
  8454. # works on it. A delete already sent may still finish in its FTP thread
  8455. # after the cutoff, as one past delete_file_async's own timeout always did.
  8456. try:
  8457. await asyncio.wait_for(
  8458. _cleanup_sd_card_after_print(printer_id, subtask_name, archive_id, logger),
  8459. timeout=_SD_CLEANUP_BUDGET_SECONDS,
  8460. )
  8461. except TimeoutError:
  8462. logger.warning(
  8463. "SD card cleanup on printer %s took over %ss; leaving the file there and closing the print",
  8464. printer_id,
  8465. _SD_CLEANUP_BUDGET_SECONDS,
  8466. )
  8467. log_timing("SD card cleanup")
  8468. # Update queue item status early — must run before the archive_id early-return
  8469. # so queue items don't get stuck in "printing" when archive lookup fails.
  8470. # Uses run_with_retry to handle SQLite "database is locked" errors (#897).
  8471. queue_item_id = None
  8472. billing_run_id: str | None = None
  8473. billing_user_id: int | None = None
  8474. billing_cost_center_id: int | None = None
  8475. billing_plate_id: int | None = None
  8476. queue_status = None
  8477. queue_auto_off = False
  8478. try:
  8479. from backend.app.core.database import run_with_retry
  8480. from backend.app.models.print_queue import PrintQueueItem
  8481. async def _update_queue_status(db):
  8482. nonlocal billing_run_id, billing_user_id, billing_cost_center_id, billing_plate_id
  8483. nonlocal queue_item_id, queue_status, queue_auto_off
  8484. result = await db.execute(
  8485. select(PrintQueueItem)
  8486. .where(PrintQueueItem.printer_id == printer_id)
  8487. .where(PrintQueueItem.status == "printing")
  8488. )
  8489. printing_items = list(result.scalars().all())
  8490. if len(printing_items) > 1:
  8491. logger.warning(
  8492. "BUG: Multiple queue items in 'printing' status for printer %s: %s",
  8493. printer_id,
  8494. [(i.id, i.archive_id, i.library_file_id) for i in printing_items],
  8495. )
  8496. item = printing_items[0] if printing_items else None
  8497. if item is not None and not await _completion_belongs_to_queue_item(db, item, data):
  8498. return
  8499. if item:
  8500. queue_status = data.get("status", "completed")
  8501. # MQTT sends "aborted" for cancelled prints; normalise to
  8502. # "cancelled" so it matches the queue schema Literal.
  8503. if queue_status == "aborted":
  8504. queue_status = "cancelled"
  8505. item.status = queue_status
  8506. item.completed_at = datetime.now(timezone.utc)
  8507. if queue_status == "failed" and not item.error_message:
  8508. item.error_message = _format_hms_error_summary(data.get("hms_errors") or [])
  8509. # Bump usage counters on the source library file so admins can
  8510. # sort by "last printed" and (eventually) auto-purge stale
  8511. # files — #1008.
  8512. await _bump_library_file_usage_if_completed(db, item, queue_status)
  8513. await db.commit()
  8514. queue_item_id = item.id
  8515. billing_run_id = item.billing_run_id
  8516. billing_user_id = item.created_by_id
  8517. billing_cost_center_id = item.cost_center_id
  8518. billing_plate_id = item.plate_id
  8519. queue_auto_off = item.auto_off_after
  8520. logger.info("Updated queue item %s status to %s", item.id, queue_status)
  8521. await run_with_retry(_update_queue_status, label="queue status update")
  8522. # Post-commit side effects (notifications, MQTT relay, auto-off) use
  8523. # their own sessions and have their own error handling — no retry needed.
  8524. if queue_item_id is not None:
  8525. # Batch orders (#342): this run may have been the last one an order
  8526. # owed. Re-evaluate here rather than lazily on read, so a finished
  8527. # order reports itself complete without someone opening the page.
  8528. try:
  8529. from backend.app.services.print_batch import refresh_batch_status_for_item
  8530. async with async_session() as db:
  8531. await refresh_batch_status_for_item(db, queue_item_id)
  8532. await db.commit()
  8533. except Exception as e:
  8534. logger.warning("[BATCH] Failed to refresh batch status for queue item %s: %s", queue_item_id, e)
  8535. # MQTT relay - publish queue job completed
  8536. try:
  8537. printer_info = printer_manager.get_printer(printer_id)
  8538. await mqtt_relay.on_queue_job_completed(
  8539. job_id=queue_item_id,
  8540. filename=filename or subtask_name,
  8541. printer_id=printer_id,
  8542. printer_name=printer_info.name if printer_info else "Unknown",
  8543. status=queue_status,
  8544. )
  8545. except Exception:
  8546. pass # Don't fail if MQTT fails
  8547. # Check if queue is now empty and send notification
  8548. try:
  8549. from sqlalchemy import func as sa_func
  8550. async with async_session() as db:
  8551. count_result = await db.execute(
  8552. select(sa_func.count(PrintQueueItem.id)).where(PrintQueueItem.status == "pending")
  8553. )
  8554. pending_count = count_result.scalar() or 0
  8555. if pending_count == 0:
  8556. today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
  8557. completed_result = await db.execute(
  8558. select(sa_func.count(PrintQueueItem.id)).where(
  8559. PrintQueueItem.status.in_(["completed", "failed", "skipped"]),
  8560. PrintQueueItem.completed_at >= today_start,
  8561. )
  8562. )
  8563. completed_count = completed_result.scalar() or 1
  8564. await notification_service.on_queue_completed(
  8565. completed_count=completed_count,
  8566. db=db,
  8567. )
  8568. except Exception:
  8569. pass # Don't fail if notification fails
  8570. # Handle auto_off_after - power off printer if the queue item opted
  8571. # in. Delegates to the smart-plug manager so the off honours each
  8572. # plug's configured strategy (time delay or temperature threshold),
  8573. # is cancelled if the printer starts printing again, and never cuts
  8574. # power on a loaded print (#1890). Previously an inline block here
  8575. # hardcoded a 50°C / 600s cooldown wait and powered off on the
  8576. # timeout regardless of print state — cutting a touchscreen reprint.
  8577. if queue_auto_off:
  8578. try:
  8579. async with async_session() as db:
  8580. await smart_plug_manager.schedule_off_after_queue_job(printer_id, db)
  8581. except Exception as e:
  8582. logger.warning("Failed to schedule queue auto-off for printer %s: %s", printer_id, e)
  8583. except Exception as e:
  8584. logging.getLogger(__name__).warning(f"Queue item update failed: {e}")
  8585. log_timing("Queue item update")
  8586. # Register bed cooldown waiter (event-driven via on_bed_temp_update callback).
  8587. # Must run before archive_id early-return so it fires for all prints (including
  8588. # prints started from BambuStudio/touchscreen that have no archive).
  8589. if data.get("status") == "completed":
  8590. try:
  8591. from backend.app.api.routes.settings import get_setting
  8592. async with async_session() as db:
  8593. threshold_str = await get_setting(db, "bed_cooled_threshold")
  8594. threshold = float(threshold_str) if threshold_str else 35.0
  8595. # Check if any provider has on_bed_cooled enabled (skip registration if none)
  8596. async with async_session() as db:
  8597. providers = await notification_service._get_providers_for_event(db, "on_bed_cooled", printer_id)
  8598. if providers:
  8599. _bed_cool_waiters[printer_id] = {
  8600. "threshold": threshold,
  8601. "filename": filename or subtask_name or "",
  8602. "registered_at": time.time(),
  8603. }
  8604. logger.info(
  8605. "[BED-COOL] Registered waiter for printer %s (threshold: %.0f°C)",
  8606. printer_id,
  8607. threshold,
  8608. )
  8609. else:
  8610. logger.debug("[BED-COOL] No providers enabled for bed_cooled on printer %s", printer_id)
  8611. except Exception as e:
  8612. logger.warning("[BED-COOL] Failed to register waiter: %s", e)
  8613. # Capture the slicer estimate before usage tracking runs. The tracker may
  8614. # update archive.cost with this run's measured cost; billing partial runs
  8615. # against that already-partial value would discount the charge twice.
  8616. billing_planned_grams: float | None = None
  8617. billing_base_cost: float | None = None
  8618. if archive_id:
  8619. try:
  8620. async with async_session() as db:
  8621. from backend.app.models.archive import PrintArchive
  8622. billing_archive = await db.get(PrintArchive, archive_id)
  8623. if billing_archive:
  8624. billing_path = (
  8625. app_settings.base_dir / billing_archive.file_path if billing_archive.file_path else None
  8626. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  8627. billing_planned_grams, billing_base_cost = _plate_scoped_run_estimate(
  8628. billing_archive,
  8629. billing_path,
  8630. billing_plate_id
  8631. if billing_plate_id is not None
  8632. else _get_start_plate_id(printer_id, archive_id),
  8633. )
  8634. except Exception as e:
  8635. logger.warning("[FINANCE] Failed to capture planned usage for archive %s: %s", archive_id, e)
  8636. # --- Track filament consumption (must run before archive_id early-return so usage
  8637. # is recorded even when auto-archive is disabled) ---
  8638. usage_results: list[dict] = []
  8639. # Prefer ams_mapping captured from MQTT request topic (works for all print sources)
  8640. stored_ams_mapping = data.get("ams_mapping")
  8641. # Fallback to _print_ams_mappings for queue/reprint (set before print starts).
  8642. # Drained either way, like the plate below: left over, it would be read by
  8643. # the archive's next run, dispatched without a mapping.
  8644. registered_ams_mapping = _print_ams_mappings.pop((printer_id, archive_id), None) if archive_id else None
  8645. if not stored_ams_mapping:
  8646. stored_ams_mapping = registered_ams_mapping
  8647. # Always drain the plate_id register on completion — the session already
  8648. # consumed it at print-start injection; leaving it would leak into the next
  8649. # print on the same archive_id (rare but possible with reprints) (#1697).
  8650. # Capture the popped value so the completion notification can scope the
  8651. # archive-level (summed-across-plates per #1593) filament + time totals
  8652. # down to the single plate that was actually printed (#1785).
  8653. notify_plate_id: int | None = None
  8654. if archive_id:
  8655. notify_plate_id = _print_plate_ids.pop((printer_id, archive_id), None)
  8656. # The start estimate must not outlive the real cost written below (COST-4).
  8657. if archive_id:
  8658. await _settle_archive_cost_estimate(archive_id)
  8659. # Internal inventory: track AMS remain% deltas (skip if Spoolman handles usage)
  8660. try:
  8661. async with async_session() as db:
  8662. from backend.app.api.routes.settings import get_setting
  8663. _spoolman_on = await get_setting(db, "spoolman_enabled")
  8664. if not _spoolman_on or _spoolman_on.lower() != "true":
  8665. from backend.app.services.usage_tracker import on_print_complete as usage_on_print_complete
  8666. async with async_session() as db:
  8667. usage_results = await usage_on_print_complete(
  8668. printer_id,
  8669. data,
  8670. printer_manager,
  8671. db,
  8672. archive_id=archive_id,
  8673. ams_mapping=stored_ams_mapping,
  8674. )
  8675. if usage_results:
  8676. await ws_manager.broadcast(
  8677. {
  8678. "type": "spool_usage_logged",
  8679. "printer_id": printer_id,
  8680. "usage": usage_results,
  8681. }
  8682. )
  8683. log_timing("Usage tracker")
  8684. except Exception as e:
  8685. logger.warning("Usage tracker on_print_complete failed: %s", e)
  8686. # Drop the print-start context unconditionally — the Spoolman branch above
  8687. # skips the internal tracker entirely, so nothing else would clear what
  8688. # print start captured, and a row surviving its print would be restored
  8689. # onto the next one after a restart.
  8690. try:
  8691. from backend.app.services.usage_tracker import discard_session
  8692. async with async_session() as db:
  8693. await discard_session(db, printer_id)
  8694. except Exception as e:
  8695. logger.warning("Failed to clear persisted print session for printer %s: %s", printer_id, e)
  8696. # Spoolman: report filament usage (requires archive_id for tracking data lookup)
  8697. # A partial run's cost from the Spoolman spools it was charged to; the
  8698. # built-in inventory's comes in usage_results.
  8699. spoolman_run_cost: float | None = None
  8700. if archive_id:
  8701. if data.get("status") == "completed":
  8702. try:
  8703. await _report_spoolman_usage(printer_id, archive_id)
  8704. log_timing("Spoolman usage report")
  8705. except Exception as e:
  8706. logger.warning("Spoolman usage reporting failed: %s", e)
  8707. else:
  8708. # Report partial usage if tracking data exists (only stored when weight sync is disabled)
  8709. try:
  8710. async with async_session() as db:
  8711. spoolman_run_cost = await _cleanup_spoolman_tracking(
  8712. printer_id,
  8713. archive_id,
  8714. db,
  8715. last_layer_num=data.get("last_layer_num"),
  8716. last_progress=data.get("last_progress"),
  8717. status=data.get("status") or "failed",
  8718. )
  8719. except Exception as e:
  8720. logger.debug("[SPOOLMAN] Cleanup failed: %s", e)
  8721. await _release_archived_spoolman_slots(printer_id)
  8722. log_timing("Filament usage tracking")
  8723. if not archive_id:
  8724. # The printer's own calibration run has no archive by design, so this
  8725. # arrives here every time one finishes. Returning before the no-archive
  8726. # notification keeps a calibration from being announced as a finished
  8727. # print. Everything above this point has already run — the plate-clear
  8728. # gate, the queue reconciliation, the SD-card cleanup — so only the
  8729. # notification is skipped.
  8730. if is_internal_printer_job(filename, subtask_name):
  8731. logger.info(
  8732. "[CALLBACK] Internal printer job completed, no notification: filename=%s, subtask=%s",
  8733. filename,
  8734. subtask_name,
  8735. )
  8736. return
  8737. logger.warning("Could not find archive for print complete: filename=%s, subtask=%s", filename, subtask_name)
  8738. # Still send print-complete/failed/stopped notifications even without an archive.
  8739. # Try to enrich with queue/library-file data so user-specific emails work too.
  8740. async def _notify_no_archive():
  8741. try:
  8742. async with async_session() as db:
  8743. from backend.app.models.library import LibraryFile
  8744. from backend.app.models.print_queue import PrintQueueItem
  8745. from backend.app.models.printer import Printer
  8746. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8747. printer_obj = result.scalar_one_or_none()
  8748. p_name = printer_obj.name if printer_obj else f"Printer {printer_id}"
  8749. # Only the queue item this completion itself closed (the
  8750. # reconciliation above, which checks the run belongs to it).
  8751. # A print started outside Bambuddy closed none, and must not
  8752. # borrow an earlier job's owner, email or times.
  8753. no_archive_data: dict | None = None
  8754. try:
  8755. queue_item = await db.get(PrintQueueItem, queue_item_id) if queue_item_id else None
  8756. if queue_item:
  8757. no_archive_data = {"created_by_id": queue_item.created_by_id}
  8758. # How long it really ran, as the archive path reports it (#1198).
  8759. started = to_naive_utc(queue_item.started_at)
  8760. ended = to_naive_utc(queue_item.completed_at)
  8761. # Not for a completion reconciled at reconnect: it
  8762. # ended at an unknown time while the printer was
  8763. # offline, and "now" is the reconnect (DB-C9).
  8764. if (
  8765. not data.get("_reconciled")
  8766. and started
  8767. and ended
  8768. and (ended - started).total_seconds() > 0
  8769. ):
  8770. no_archive_data["actual_time_seconds"] = int((ended - started).total_seconds())
  8771. # Estimated time: the queue item caches it from its
  8772. # archive or library file; otherwise read the library
  8773. # file's metadata (LibraryFile has no column for it).
  8774. if queue_item.print_time_seconds:
  8775. no_archive_data["print_time_seconds"] = queue_item.print_time_seconds
  8776. elif queue_item.library_file_id:
  8777. lib_result = await db.execute(
  8778. select(LibraryFile).where(LibraryFile.id == queue_item.library_file_id)
  8779. )
  8780. lib_file = lib_result.scalar_one_or_none()
  8781. estimate = (
  8782. (lib_file.file_metadata or {}).get("print_time_seconds") if lib_file else None
  8783. )
  8784. if estimate:
  8785. no_archive_data["print_time_seconds"] = estimate
  8786. except Exception as lookup_err:
  8787. logger.debug(
  8788. "[NOTIFY-BG] Could not look up queue item for no-archive notification: %s", lookup_err
  8789. )
  8790. # Enrich with usage tracker results (captured in enclosing scope)
  8791. if usage_results:
  8792. if no_archive_data is None:
  8793. no_archive_data = {}
  8794. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  8795. if total_from_usage > 0:
  8796. no_archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  8797. no_archive_data["usage_results"] = usage_results
  8798. # Try MQTT remaining_time for print duration when no queue/library data
  8799. if no_archive_data and not no_archive_data.get("print_time_seconds"):
  8800. mqtt_remaining = data.get("remaining_time")
  8801. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  8802. no_archive_data["print_time_seconds"] = int(mqtt_remaining)
  8803. ps = data.get("status", "completed")
  8804. logger.info(
  8805. "[NOTIFY-BG] Sending notification without archive: printer=%s, status=%s", printer_id, ps
  8806. )
  8807. if not await _kill_switch_notification_already_sent(kill_switch_notification_task):
  8808. await notification_service.on_print_complete(
  8809. printer_id, p_name, ps, data, db, archive_data=no_archive_data
  8810. )
  8811. else:
  8812. logger.info("[NOTIFY-BG] Skipped duplicate kill-switch provider notification")
  8813. # Send user-specific email if we have a created_by_id
  8814. if no_archive_data and no_archive_data.get("created_by_id"):
  8815. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  8816. await _dispatch_user_print_email(
  8817. ps,
  8818. no_archive_data["created_by_id"],
  8819. p_name,
  8820. raw_filename,
  8821. db,
  8822. )
  8823. logger.info("[NOTIFY-BG] Completed (no-archive path)")
  8824. except Exception as e:
  8825. logger.warning("[NOTIFY-BG] Failed to send notification without archive: %s", e, exc_info=True)
  8826. spawn_background_task(_notify_no_archive(), name="notify-no-archive")
  8827. return
  8828. log_timing("Archive lookup")
  8829. # Update archive status
  8830. logger.info("[ARCHIVE] Updating archive %s status...", archive_id)
  8831. try:
  8832. async with async_session() as db:
  8833. service = ArchiveService(db)
  8834. status = data.get("status", "completed")
  8835. hms_errors = data.get("hms_errors", []) if status == "failed" else None
  8836. if hms_errors:
  8837. logger.info("[ARCHIVE] HMS errors at failure: %s", hms_errors)
  8838. failure_reason = derive_failure_reason(status, hms_errors)
  8839. if data.get("_reconciled") and (status == "aborted" or (status == "failed" and not failure_reason)):
  8840. # A reconciled completion closes out a stale archive at
  8841. # reconnect — it is not a user action, so don't mislabel it
  8842. # "userCancelled". It shares the stale-cleanup path's key
  8843. # (issue #2974) and records that the real end time is unknown,
  8844. # which is also why its logged duration is 0 (#2592). A failed
  8845. # one the printer's errors explain keeps their reason (#3310).
  8846. failure_reason = "noStatusUpdate"
  8847. elif data.get("_kill_switch"):
  8848. # Not the user's cancel: the kill switch stopped a print that did
  8849. # not come from Bambuddy.
  8850. failure_reason = "killSwitch"
  8851. if failure_reason:
  8852. logger.info("[ARCHIVE] failure_reason=%r (status=%s)", failure_reason, status)
  8853. elif status == "failed" and hms_errors:
  8854. logger.info("[ARCHIVE] HMS errors present but none matched a known failure-reason short code")
  8855. await service.update_archive_status(
  8856. archive_id,
  8857. status=status,
  8858. completed_at=(
  8859. # A reconciled completed print ended at an estimated time,
  8860. # not at the reconnect (#3310).
  8861. data.get("_reconciled_completed_at") or datetime.now(timezone.utc)
  8862. if status in ("completed", "failed", "aborted", "cancelled")
  8863. else None
  8864. ),
  8865. failure_reason=failure_reason,
  8866. )
  8867. logger.info(
  8868. "[ARCHIVE] Archive %s status updated to %s, failure_reason=%s", archive_id, status, failure_reason
  8869. )
  8870. await ws_manager.send_archive_updated(
  8871. {
  8872. "id": archive_id,
  8873. "status": status,
  8874. }
  8875. )
  8876. logger.info("[ARCHIVE] WebSocket notification sent for archive %s", archive_id)
  8877. # MQTT relay - publish archive updated
  8878. try:
  8879. await mqtt_relay.on_archive_updated(
  8880. archive_id=archive_id,
  8881. print_name=filename or subtask_name,
  8882. status=status,
  8883. )
  8884. except Exception:
  8885. pass # Don't fail if MQTT fails
  8886. except Exception as e:
  8887. logger.error("[ARCHIVE] Failed to update archive %s status: %s", archive_id, e, exc_info=True)
  8888. # Continue with other operations even if archive update fails
  8889. log_timing("Archive status update")
  8890. # Apply finance wallet charge or release reservations once. For all partial
  8891. # terminal states (failed, aborted at the printer display, or cancelled via
  8892. # Bambuddy) use this run's measured spool delta, falling back to the last
  8893. # valid printer progress. PrintArchive.filament_used_grams is the slicer
  8894. # estimate and therefore cannot represent an interrupted run.
  8895. try:
  8896. if data.get("status") in ("completed", "failed", "aborted", "cancelled"):
  8897. async with async_session() as db:
  8898. from backend.app.models.archive import PrintArchive
  8899. from backend.app.services.finance_billing import apply_print_charge_for_archive
  8900. from backend.app.services.print_log import run_being_ended
  8901. archive = await db.get(PrintArchive, archive_id)
  8902. # The run this end fills in below: charged as its own, not as
  8903. # the archive's latest dispatch (another plate of it, maybe).
  8904. charged_run = await run_being_ended(db, archive_id, printer_id, archive.started_at) if archive else None
  8905. if archive and billing_run_id is None and charged_run is None:
  8906. billing_run_id = getattr(archive, "billing_run_id", None)
  8907. if archive and archive.created_by_id is None and _print_user_info:
  8908. archive.created_by_id = await _print_user_id_if_still_there(db, _print_user_info)
  8909. await db.flush()
  8910. run_status = data.get("status", "completed")
  8911. last_progress = data.get("last_progress")
  8912. if last_progress is None:
  8913. last_progress = data.get("progress")
  8914. actual_run_grams = _compute_run_filament_grams(
  8915. run_status,
  8916. billing_planned_grams,
  8917. last_progress,
  8918. usage_results,
  8919. )
  8920. filament_usage = (actual_run_grams, billing_planned_grams) if run_status != "completed" else None
  8921. in_memory_cost_center_id = _print_cost_center_ids.pop((printer_id, archive_id), None)
  8922. charged = await apply_print_charge_for_archive(
  8923. db,
  8924. archive_id,
  8925. charged_user_id=billing_user_id,
  8926. cost_center_id=(
  8927. billing_cost_center_id if billing_cost_center_id is not None else in_memory_cost_center_id
  8928. ),
  8929. print_queue_id=queue_item_id,
  8930. print_run_id=billing_run_id,
  8931. base_cost_override=billing_base_cost,
  8932. filament_usage=filament_usage,
  8933. run_entry_id=charged_run.id if charged_run is not None else None,
  8934. run_status=run_status,
  8935. )
  8936. await db.commit()
  8937. if charged:
  8938. logger.info("[FINANCE] Applied print charge for archive %s", archive_id)
  8939. except Exception as e:
  8940. logger.warning("[FINANCE] Failed to apply print charge for archive %s: %s", archive_id, e)
  8941. printer_info = printer_manager.get_printer(printer_id)
  8942. billing_printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  8943. billing_filename = filename or subtask_name or "Unknown"
  8944. billing_error = str(e)
  8945. try:
  8946. await ws_manager.broadcast(
  8947. {
  8948. "type": "billing_charge_failed",
  8949. "printer_id": printer_id,
  8950. "printer_name": billing_printer_name,
  8951. "filename": billing_filename,
  8952. "archive_id": archive_id,
  8953. }
  8954. )
  8955. except Exception as notification_error:
  8956. logger.error(
  8957. "[FINANCE] Failed to broadcast billing error for archive %s: %s",
  8958. archive_id,
  8959. notification_error,
  8960. )
  8961. async def _notify_billing_charge_failed() -> None:
  8962. try:
  8963. async with async_session() as notification_db:
  8964. await notification_service.on_billing_charge_failed(
  8965. printer_id,
  8966. billing_printer_name,
  8967. billing_filename,
  8968. archive_id,
  8969. billing_error,
  8970. notification_db,
  8971. )
  8972. except Exception as provider_error:
  8973. logger.error(
  8974. "[FINANCE] Failed to send provider billing alert for archive %s: %s",
  8975. archive_id,
  8976. provider_error,
  8977. exc_info=True,
  8978. )
  8979. spawn_background_task(
  8980. _notify_billing_charge_failed(),
  8981. name=f"billing-charge-failed-{archive_id}",
  8982. )
  8983. log_timing("Finance charge update")
  8984. # Write independent print log entry (separate table, never touches archives)
  8985. # The run's id, for the energy work below: the archive's newest run is
  8986. # the one started last, which with plates of it on two printers need not
  8987. # be the one that just ended.
  8988. completed_run_id: int | None = None
  8989. async def _write_run(db):
  8990. # Retried on a locked SQLite database: a run left open here would be
  8991. # closed again by the next reconnect's reconcile.
  8992. nonlocal completed_run_id
  8993. from backend.app.models.archive import PrintArchive
  8994. from backend.app.services.print_log import discard_open_runs, record_archive_wear, write_log_entry
  8995. archive = await db.get(PrintArchive, archive_id)
  8996. if archive is None:
  8997. # Deleted while it printed: its run has nothing left to end on.
  8998. discarded = await discard_open_runs(db, printer_id)
  8999. await db.commit()
  9000. await _announce_discarded_runs(discarded, printer_id)
  9001. if archive:
  9002. # Back-fill created_by_id on reprint (#730): reprint reuses the
  9003. # source archive row rather than creating a new one, so an
  9004. # archive that was auto-created from a printer-initiated
  9005. # print (created_by_id=NULL) would otherwise stay unattributed
  9006. # forever. When we have a print-session user AND the archive
  9007. # has no attribution yet, credit the current user. Never
  9008. # overwrite an existing attribution — the original uploader
  9009. # keeps ownership.
  9010. _print_user_id = await _print_user_id_if_still_there(db, _print_user_info)
  9011. if archive.created_by_id is None and _print_user_id is not None:
  9012. archive.created_by_id = _print_user_id
  9013. p_info = printer_manager.get_printer(printer_id)
  9014. # Per-run actuals — written to PrintLogEntry so stats reflect
  9015. # what THIS print actually used, not the source archive's
  9016. # first-run values (#1378). Helper handles the partial-print
  9017. # math (failed / cancelled / stopped get scaled to progress
  9018. # or to tracked spool deltas).
  9019. _run_status = data.get("status", "completed")
  9020. # #2614: scope the per-run estimate to the printed plate. For a
  9021. # multi-plate 3MF dispatched one plate at a time, the archive's
  9022. # filament/cost are the whole-file totals; the PrintLogEntry must
  9023. # reflect only this plate. No effect on single-plate archives (the
  9024. # plate estimate equals the whole-file value) or on the tracker
  9025. # path (measured spool deltas win in _compute_run_filament_grams).
  9026. _est_full_path = (
  9027. app_settings.base_dir / archive.file_path if archive.file_path else None
  9028. ) # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  9029. # The plate that printed, as billing reads it: a reprint of
  9030. # another plate must not log the archive's first plate.
  9031. _run_plate_id = billing_plate_id if billing_plate_id is not None else notify_plate_id
  9032. _est_grams, _est_cost = _plate_scoped_run_estimate(archive, _est_full_path, _run_plate_id)
  9033. if _run_plate_id is None:
  9034. _run_plate_id = archive.plate_id
  9035. _run_grams = _compute_run_filament_grams(
  9036. _run_status,
  9037. _est_grams,
  9038. data.get("last_progress", data.get("progress")),
  9039. usage_results,
  9040. )
  9041. # Per-run cost — prefer usage_results sum. For partial prints
  9042. # we deliberately skip the topup-to-estimate logic in
  9043. # usage_tracker (which assumes the print completed); the raw
  9044. # tracked-spool sum is closer to what THIS run actually cost.
  9045. _run_cost: float | None = None
  9046. if usage_results:
  9047. _run_cost = sum(r.get("cost") or 0 for r in usage_results) or None
  9048. if _run_cost is None and _run_status != "completed":
  9049. _run_cost = spoolman_run_cost
  9050. if _run_cost is None and _run_status == "completed":
  9051. _run_cost = _est_cost
  9052. from backend.app.models.printer import Printer as _Printer
  9053. _wear_rate = await db.scalar(select(_Printer.wear_cost_per_hour).where(_Printer.id == printer_id))
  9054. run_entry = await write_log_entry(
  9055. db,
  9056. archive_id=archive.id,
  9057. # Captured by _update_queue_status above; None for
  9058. # printer-initiated prints with no queue row. Batch
  9059. # cost/energy roll-up joins on it (#342).
  9060. queue_item_id=queue_item_id,
  9061. status=_run_status,
  9062. print_name=archive.print_name,
  9063. # The plate this run printed, as the estimate above takes
  9064. # it; the archive's own is the latest run's.
  9065. plate_id=_run_plate_id,
  9066. printer_name=p_info.name if p_info else None,
  9067. printer_id=printer_id,
  9068. started_at=archive.started_at,
  9069. completed_at=archive.completed_at,
  9070. filament_type=archive.filament_type,
  9071. filament_color=archive.filament_color,
  9072. filament_used_grams=_run_grams,
  9073. cost=_run_cost,
  9074. wear_cost_per_hour=_wear_rate,
  9075. failure_reason=archive.failure_reason,
  9076. thumbnail_path=archive.thumbnail_path,
  9077. created_by_id=archive.created_by_id,
  9078. created_by_username=_print_user_info.get("username") if _print_user_info else None,
  9079. # Reconciled completions have an unknown real end time —
  9080. # log 0 duration instead of the whole disconnect gap (#2592),
  9081. # unless it was estimated (#3310).
  9082. reconciled=bool(data.get("_reconciled")) and not data.get("_reconciled_completed_at"),
  9083. # Its time is the estimate or nothing, never measured (#3310).
  9084. duration_unmeasured=bool(data.get("_reconciled")),
  9085. # The real end of a print stopped while its printer was offline.
  9086. replaces_offline_stop=True,
  9087. )
  9088. await record_archive_wear(db, archive, run_entry)
  9089. await db.commit()
  9090. completed_run_id = run_entry.id
  9091. logger.info("[PRINT_LOG] Log entry written for archive %s", archive_id)
  9092. try:
  9093. from backend.app.core.database import run_with_retry
  9094. await run_with_retry(_write_run, label="print log entry")
  9095. except Exception as e:
  9096. logger.warning("[PRINT_LOG] Failed to write log entry for archive %s: %s", archive_id, e)
  9097. await _publish_run_completed(completed_run_id, printer_id)
  9098. log_timing("Print log entry")
  9099. # Run slow operations as background tasks to avoid blocking the event loop
  9100. # These operations can take 5-10+ seconds and would freeze the UI if awaited
  9101. async def _background_energy_calculation():
  9102. """Calculate and save energy usage in background.
  9103. Reads the starting kWh from the archive row (#941: persisted so a mid-print
  9104. backend restart no longer loses per-print energy data).
  9105. """
  9106. try:
  9107. logger.info("[ENERGY-BG] Starting energy calculation for archive %s", archive_id)
  9108. async with async_session() as db:
  9109. from backend.app.models.archive import PrintArchive
  9110. from backend.app.models.print_log import PrintLogEntry
  9111. from backend.app.services.print_log import energy_start_of
  9112. archive = await db.get(PrintArchive, archive_id)
  9113. if archive is None:
  9114. logger.warning("[ENERGY-BG] Archive %s no longer exists", archive_id)
  9115. return
  9116. # The ended run's own start reading: the archive's is its
  9117. # latest start's, maybe another plate's on another printer.
  9118. run_row = await db.get(PrintLogEntry, completed_run_id) if completed_run_id is not None else None
  9119. start = energy_start_of(run_row, archive)
  9120. if start is None:
  9121. logger.info("[ENERGY-BG] No start kWh recorded for archive %s's run", archive_id)
  9122. return
  9123. starting_kwh = start.kwh
  9124. candidates = await energy_plug_candidates(db, printer_id)
  9125. if not candidates:
  9126. logger.info("[ENERGY-BG] No smart plug for printer %s", printer_id)
  9127. return
  9128. # Same ordering as the start reading, so the delta below is
  9129. # against the counter that produced `starting_kwh` (#2859).
  9130. selected = await select_energy_reading(candidates, _get_plug_energy, db)
  9131. if selected is None:
  9132. logger.warning(
  9133. "[ENERGY-BG] No plug on printer %s reports a lifetime energy counter (tried: %s)",
  9134. printer_id,
  9135. ", ".join(plug.name for plug in candidates),
  9136. )
  9137. return
  9138. plug, energy = selected
  9139. logger.info("[ENERGY-BG] Energy response from plug '%s': %s", plug.name, energy)
  9140. energy_used = round(energy["total"] - starting_kwh, 4)
  9141. logger.info("[ENERGY-BG] Per-print energy: %s kWh", energy_used)
  9142. if energy_used < 0:
  9143. logger.warning(
  9144. "[ENERGY-BG] Negative energy delta for archive %s (start=%s, end=%s) — counter reset?",
  9145. archive_id,
  9146. starting_kwh,
  9147. energy["total"],
  9148. )
  9149. return
  9150. from backend.app.services.energy_price import current_price, print_energy_cost, stored_price
  9151. # Each hour of the print at the price of that hour (#1251). If
  9152. # that fails, the last price known, as before, rather than no
  9153. # energy at all. Read first: the session may be unusable after.
  9154. last_price = await stored_price(db)
  9155. try:
  9156. energy_cost_value = round(
  9157. await print_energy_cost(
  9158. db,
  9159. plug_id=start.plug_id,
  9160. start_at=start.at,
  9161. start_kwh=starting_kwh,
  9162. start_price=start.price,
  9163. end_kwh=energy["total"],
  9164. end_plug_id=plug.id,
  9165. price_now=await current_price(db),
  9166. ),
  9167. 3,
  9168. )
  9169. except Exception as e:
  9170. logger.warning("[ENERGY-BG] Hour-by-hour cost failed for archive %s: %s", archive_id, e)
  9171. energy_cost_value = round(energy_used * last_price, 3)
  9172. # First-run-only overwrite of archive.energy_kwh / energy_cost so a
  9173. # reprint doesn't visually clobber the source archive's energy data
  9174. # (#1378). Reprint energy lives in the matching PrintLogEntry below.
  9175. from sqlalchemy import func
  9176. existing_runs = await db.scalar(
  9177. select(func.count(PrintLogEntry.id)).where(PrintLogEntry.archive_id == archive_id)
  9178. )
  9179. if (existing_runs or 0) <= 1:
  9180. # 0 = legacy archive that pre-dates per-run logging; 1 = the row
  9181. # we just wrote for THIS print. Either way it's the first run.
  9182. archive.energy_kwh = energy_used
  9183. archive.energy_cost = energy_cost_value
  9184. # The ended run's energy (write_log_entry above ran before this
  9185. # background task, so its energy fields are still NULL). With
  9186. # no run written, none: the archive's newest is another print's.
  9187. if run_row is not None:
  9188. run_row.energy_kwh = energy_used
  9189. run_row.energy_cost = energy_cost_value
  9190. await db.commit()
  9191. logger.info("[ENERGY-BG] Saved: %s kWh, cost=%s", energy_used, energy_cost_value)
  9192. except Exception as e:
  9193. logger.warning("[ENERGY-BG] Failed: %s", e)
  9194. async def _background_finish_photo() -> str | None:
  9195. """Capture finish photo in background. Returns photo filename if captured."""
  9196. # #2547: the Z to park the plate back to, set once this function has
  9197. # raised the plate itself (the timelapse path, where the moment producer
  9198. # returned without doing it). Declared out here so the `finally` can
  9199. # lower it again no matter where the capture below fails.
  9200. park_z: float | None = None
  9201. try:
  9202. logger.info("[PHOTO-BG] Starting finish photo capture for archive %s", archive_id)
  9203. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  9204. # Read phase: settings + printer + archive in a short session, released
  9205. # BEFORE the capture pipeline below. The capture (timelapse last-frame,
  9206. # stage-22 wait, external-camera grab, or a fresh RTSP shot) can take
  9207. # tens of seconds; holding this session across it pinned one pooled
  9208. # connection idle-in-transaction per finishing print (issue #2572).
  9209. async with async_session() as db:
  9210. from backend.app.api.routes.settings import get_setting
  9211. from backend.app.models.archive import PrintArchive
  9212. from backend.app.models.printer import Printer
  9213. capture_enabled = await get_setting(db, "capture_finish_photo")
  9214. if capture_enabled is not None and capture_enabled.lower() != "true":
  9215. return None
  9216. if not archive_id:
  9217. return None
  9218. printer = (await db.execute(select(Printer).where(Printer.id == printer_id))).scalar_one_or_none()
  9219. archive = (
  9220. await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  9221. ).scalar_one_or_none()
  9222. if not printer or not archive:
  9223. return None
  9224. import uuid
  9225. from datetime import datetime
  9226. from backend.app.utils.archive_paths import archive_dir as resolve_archive_dir
  9227. if not archive.file_path:
  9228. logger.warning("[PHOTO-BG] Archive %s has no file_path, using fallback dir", archive_id)
  9229. archive_dir = resolve_archive_dir(archive)
  9230. photo_filename = None
  9231. # Prefer the timelapse last-frame source when a timelapse was
  9232. # recording — it captures the moment after the toolhead parks
  9233. # but before the bed drops, which the live-camera grab below
  9234. # would miss (#1397). Skipped for external cameras (those have
  9235. # their own framing and don't see a Bambu timelapse). Only
  9236. # runs when the USER explicitly enabled timelapse for this
  9237. # print — #1721 removed Bambuddy's force-on at dispatch
  9238. # because it caused per-layer nozzle parking on Smooth-mode
  9239. # slicer profiles.
  9240. prefer_timelapse_source = bool(data.get("timelapse_was_active")) and not (
  9241. printer.external_camera_enabled and printer.external_camera_url
  9242. )
  9243. timelapse_still_pending = False
  9244. if prefer_timelapse_source:
  9245. photo_filename, timelapse_still_pending = await _capture_finish_photo_from_timelapse(
  9246. archive_id=archive_id,
  9247. archive_dir=archive_dir,
  9248. rotation=getattr(printer, "camera_rotation", 0),
  9249. run_id=completed_run_id,
  9250. )
  9251. # #1721: replacement framing path — on_finish_photo_moment
  9252. # pre-captured a frame at the stage-22 / FINISH edge (toolhead
  9253. # parked, bed not yet dropped) and cached the JPEG bytes in
  9254. # _stage22_finish_frames. Consume them now so the saved photo
  9255. # has the better framing instead of the post-bed-drop angle
  9256. # the live-camera fallback below would give.
  9257. if not photo_filename:
  9258. # #1790: on the FINISH-state fallback path the producer
  9259. # task is dispatched back-to-back with this consumer, so
  9260. # a bare pop would race past with an empty result and
  9261. # the RTSP fallback below would collide with the
  9262. # producer's still-in-flight grab (single-client RTSP
  9263. # on Bambu printers). Wait for the producer to finish
  9264. # or give up before touching the cache.
  9265. #
  9266. # #2547: 20s was enough when the producer only ever grabbed a
  9267. # frame. It now also raises the plate first, which costs the
  9268. # settle window before the grab even starts — so the budget has
  9269. # to cover settle + a worst-case 15s RTSP timeout, and still sit
  9270. # under the notification's own photo wait below.
  9271. in_flight = _stage22_finish_in_flight.pop(printer_id, None)
  9272. if in_flight is not None:
  9273. try:
  9274. await asyncio.wait_for(in_flight.wait(), timeout=_FINISH_PHOTO_PRODUCER_WAIT_SECONDS)
  9275. except TimeoutError:
  9276. logger.warning(
  9277. "[PHOTO-BG] timed out waiting for stage-22 producer for printer %s — proceeding to fallback",
  9278. printer_id,
  9279. )
  9280. cached_frame = _stage22_finish_frames.pop(printer_id, None)
  9281. if cached_frame:
  9282. # Already rotated by the producer (#2708) — rotating again
  9283. # here would undo the fix on the banked-frame path, whose
  9284. # bytes reach the cache having been rotated once already.
  9285. photos_dir = archive_dir / "photos"
  9286. photos_dir.mkdir(parents=True, exist_ok=True)
  9287. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9288. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9289. photo_path = photos_dir / photo_filename
  9290. await asyncio.to_thread(photo_path.write_bytes, cached_frame)
  9291. logger.info(
  9292. "[PHOTO-BG] Saved stage-22 pre-captured frame: %s (%d bytes)",
  9293. photo_filename,
  9294. len(cached_frame),
  9295. )
  9296. # #2547: the timelapse path reaches the live grab below whenever the
  9297. # video hasn't landed in time — the documented usual outcome on
  9298. # P1-series, where transfers are slowest. `on_finish_photo_moment`
  9299. # returned early for those prints without raising the plate, so
  9300. # without this the photo that actually ships in the notification is
  9301. # of an already-dropped plate: exactly the framing #1145/#1397/#1565
  9302. # asked us to fix. The archive still gets the better video frame
  9303. # later; this is about the image the user is sent.
  9304. #
  9305. # Gated on `timelapse_was_active` precisely because that is the
  9306. # condition under which the producer skipped. On every other path it
  9307. # has already raised and lowered the plate, and repeating that here
  9308. # would be a second pointless round trip.
  9309. if (
  9310. not photo_filename
  9311. and data.get("timelapse_was_active")
  9312. and not print_dispatch_context.end_gcode_injected(printer_id)
  9313. ):
  9314. try:
  9315. async with async_session() as db:
  9316. from backend.app.api.routes.settings import get_setting
  9317. restore_setting = await get_setting(db, "finish_photo_restore_plate")
  9318. if restore_setting is None or restore_setting.lower() == "true":
  9319. park_z = await _raise_plate_for_finish_photo(
  9320. printer_id, data, logger, bound_archive_id, notify_plate_id
  9321. )
  9322. except Exception as e:
  9323. logger.warning("[PLATE-RESTORE] printer %s: restore failed: %s", printer_id, e)
  9324. # Fallback chain: external camera → buffered live frame →
  9325. # fresh RTSP capture. Only runs if the timelapse path above
  9326. # didn't already produce a photo.
  9327. # Chamber light for the live capture below (#1655).
  9328. async with camera_light(printer if not photo_filename else None):
  9329. if not photo_filename:
  9330. if printer.external_camera_enabled and printer.external_camera_url:
  9331. logger.info("[PHOTO-BG] Using external camera")
  9332. from backend.app.api.routes.camera import live_frame_for_capture
  9333. from backend.app.services.external_camera import capture_frame
  9334. # #2707: the second half of the finish-photo failure — the
  9335. # pre-capture and this fallback both collided with the live
  9336. # view. None here continues down the fallback chain.
  9337. defer, buffered = live_frame_for_capture(printer_id)
  9338. if defer:
  9339. frame_data = buffered
  9340. else:
  9341. frame_data = await capture_frame(
  9342. printer.external_camera_url,
  9343. printer.external_camera_type or "mjpeg",
  9344. snapshot_url=printer.external_camera_snapshot_url,
  9345. )
  9346. if frame_data:
  9347. frame_data = _apply_camera_rotation(frame_data, printer, logger)
  9348. photos_dir = archive_dir / "photos"
  9349. photos_dir.mkdir(parents=True, exist_ok=True)
  9350. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9351. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9352. photo_path = photos_dir / photo_filename
  9353. await asyncio.to_thread(photo_path.write_bytes, frame_data)
  9354. logger.info("[PHOTO-BG] Saved external camera frame: %s", photo_filename)
  9355. else:
  9356. # Check if camera stream is active - use buffered frame to avoid freeze
  9357. # Check both RTSP streams (_active_streams) and chamber image streams (_active_chamber_streams)
  9358. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  9359. active_chamber_for_printer = [
  9360. k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")
  9361. ]
  9362. buffered_frame = get_buffered_frame(printer_id)
  9363. if (active_for_printer or active_chamber_for_printer) and buffered_frame:
  9364. # Use frame from active stream
  9365. logger.info("[PHOTO-BG] Using buffered frame from active stream")
  9366. buffered_frame = _apply_camera_rotation(buffered_frame, printer, logger)
  9367. photos_dir = archive_dir / "photos"
  9368. photos_dir.mkdir(parents=True, exist_ok=True)
  9369. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  9370. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  9371. photo_path = photos_dir / photo_filename
  9372. await asyncio.to_thread(photo_path.write_bytes, buffered_frame)
  9373. logger.info("[PHOTO-BG] Saved buffered frame: %s", photo_filename)
  9374. else:
  9375. # No active stream - capture new frame
  9376. from backend.app.services.camera import capture_finish_photo
  9377. photo_filename = await capture_finish_photo(
  9378. printer_id=printer_id,
  9379. ip_address=printer.ip_address,
  9380. access_code=printer.access_code,
  9381. model=printer.model,
  9382. archive_dir=archive_dir,
  9383. rotation=getattr(printer, "camera_rotation", 0),
  9384. )
  9385. # Write phase: attach the photo in a fresh short-lived session.
  9386. if photo_filename:
  9387. async with async_session() as db:
  9388. from backend.app.models.archive import PrintArchive
  9389. from backend.app.services.print_log import add_run_photo, run_of
  9390. arch = await db.get(PrintArchive, archive_id)
  9391. # The ended print's own list takes it too: the archive's
  9392. # holds every print's.
  9393. run = await run_of(db, archive_id, completed_run_id) if completed_run_id is not None else None
  9394. if arch is not None:
  9395. # A new list: appended in place, the ORM sees no
  9396. # change and a second print's photo is never saved.
  9397. photos = list(arch.photos or [])
  9398. photos.append(photo_filename)
  9399. arch.photos = photos
  9400. add_run_photo(run, photo_filename)
  9401. await db.commit()
  9402. logger.info("[PHOTO-BG] Saved: %s", photo_filename)
  9403. # The short wait above is bounded so a slow printer can't hold up
  9404. # the print-complete notification, which is what the caller is
  9405. # blocking on. When it ran out with the video still on its way,
  9406. # keep waiting off to the side and swap the better frame into the
  9407. # archive once it arrives (#2704 follow-up) — otherwise P1-series
  9408. # users, whose videos routinely take minutes to transfer, never get
  9409. # the pre-bed-drop framing this path exists to provide.
  9410. #
  9411. # Spawned here rather than at the point the wait gave up: both this
  9412. # function and the upgrade do a read-modify-write on `photos`, and
  9413. # the live-camera fallback above can take tens of seconds. Starting
  9414. # the upgrade before that write means the two can interleave and one
  9415. # silently drops the other's entry, leaving a JPEG on disk that the
  9416. # gallery never lists.
  9417. if timelapse_still_pending:
  9418. from backend.app.services import archive_work
  9419. spawn_background_task(
  9420. archive_work.held_while(
  9421. archive_id,
  9422. _upgrade_finish_photo_from_timelapse(
  9423. archive_id,
  9424. archive_dir,
  9425. rotation=getattr(printer, "camera_rotation", 0),
  9426. run_id=completed_run_id,
  9427. replace=photo_filename,
  9428. ),
  9429. ),
  9430. name=f"finish-photo-upgrade-{archive_id}",
  9431. )
  9432. return photo_filename
  9433. except Exception as e:
  9434. logger.warning("[PHOTO-BG] Failed: %s", e)
  9435. return None
  9436. finally:
  9437. # #2547: we raised the plate, so we owe the move back down — even if
  9438. # the capture in between threw. Otherwise the user finds the print
  9439. # pinned under the nozzle.
  9440. if park_z is not None:
  9441. try:
  9442. _park_plate_after_finish_photo(printer_id, park_z, logger)
  9443. except Exception as e:
  9444. logger.warning("[PLATE-RESTORE] printer %s: could not lower plate: %s", printer_id, e)
  9445. spawn_background_task(_background_energy_calculation(), name="background-energy-calc")
  9446. # Photo capture task - result will be used by notifications
  9447. photo_task = spawn_background_task(_background_finish_photo(), name="background-finish-photo")
  9448. log_timing("Background tasks scheduled (energy, photo)")
  9449. # Also run smart plug, notifications, and maintenance as background tasks
  9450. print_status = data.get("status", "completed")
  9451. async def _background_smart_plug():
  9452. """Handle smart plug automation in background."""
  9453. try:
  9454. logger.info("[AUTO-OFF-BG] Starting smart plug automation for printer %s", printer_id)
  9455. async with async_session() as db:
  9456. await smart_plug_manager.on_print_complete(printer_id, print_status, db)
  9457. logger.info("[AUTO-OFF-BG] Completed")
  9458. except Exception as e:
  9459. logger.warning("[AUTO-OFF-BG] Failed: %s", e)
  9460. async def _background_notifications(finish_photo_filename: str | None = None):
  9461. """Send print complete notifications in background."""
  9462. try:
  9463. logger.info(
  9464. "[NOTIFY-BG] Starting notifications for printer %s, photo=%s", printer_id, finish_photo_filename
  9465. )
  9466. async with async_session() as db:
  9467. from backend.app.models.archive import PrintArchive
  9468. from backend.app.models.printer import Printer
  9469. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  9470. printer = result.scalar_one_or_none()
  9471. printer_name = printer.name if printer else f"Printer {printer_id}"
  9472. archive_data = None
  9473. if archive_id:
  9474. from backend.app.models.print_log import PrintLogEntry
  9475. archive_result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  9476. archive = archive_result.scalar_one_or_none()
  9477. # The run that ended: the archive shows its latest, which
  9478. # with another plate of it printing elsewhere is not this.
  9479. run = await db.get(PrintLogEntry, completed_run_id) if completed_run_id is not None else None
  9480. run_started = run.started_at if run is not None else archive.started_at if archive else None
  9481. run_completed = run.completed_at if run is not None else archive.completed_at if archive else None
  9482. if archive:
  9483. # Actual elapsed time from started_at/completed_at when both are
  9484. # populated (every terminal status sets completed_at after #1198).
  9485. # Falls back to None so the notification path can decide whether to
  9486. # render the slicer estimate as a last resort.
  9487. actual_time_seconds = None
  9488. # Unknown for a completion reconciled at reconnect: the
  9489. # print ended while the printer was offline and
  9490. # completed_at is the reconnect, so the gap would read
  9491. # as the duration -- 14 h for a night offline (DB-C9).
  9492. # An estimated end (#3310) gives the estimate.
  9493. if (
  9494. run_started
  9495. and run_completed
  9496. and (not data.get("_reconciled") or data.get("_reconciled_completed_at"))
  9497. ):
  9498. elapsed = (run_completed - run_started).total_seconds()
  9499. if elapsed > 0:
  9500. actual_time_seconds = int(elapsed)
  9501. archive_data = {
  9502. "print_time_seconds": archive.print_time_seconds,
  9503. "actual_time_seconds": actual_time_seconds,
  9504. "actual_filament_grams": archive.filament_used_grams,
  9505. "failure_reason": run.failure_reason if run is not None else archive.failure_reason,
  9506. "created_by_id": archive.created_by_id,
  9507. }
  9508. # Scale filament usage for partial prints
  9509. if print_status != "completed" and archive.filament_used_grams:
  9510. progress = data.get("progress") or 0
  9511. scale = _partial_progress_scale(progress)
  9512. archive_data["actual_filament_grams"] = round(archive.filament_used_grams * scale, 1)
  9513. archive_data["progress"] = progress
  9514. # Pass per-slot data from archive.extra_data
  9515. if archive.extra_data and archive.extra_data.get("filament_slots"):
  9516. slots = archive.extra_data["filament_slots"]
  9517. if print_status != "completed":
  9518. scale = _partial_progress_scale(data.get("progress"))
  9519. slots = [{**s, "used_g": round(s["used_g"] * scale, 1)} for s in slots]
  9520. archive_data["filament_slots"] = slots
  9521. # Scope project-summed totals down to the plate that was
  9522. # actually printed — see _scope_notification_archive_data_to_plate
  9523. # for the why (#1785).
  9524. archive_data = _scope_notification_archive_data_to_plate(
  9525. archive_data,
  9526. archive.file_path,
  9527. notify_plate_id,
  9528. print_status,
  9529. data.get("progress"),
  9530. app_settings.base_dir,
  9531. )
  9532. # Enrich filament_grams from usage_results when archive has no 3MF data
  9533. if not archive_data.get("actual_filament_grams") and usage_results:
  9534. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  9535. if total_from_usage > 0:
  9536. archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  9537. # Pass usage tracker results for AMS slot info in notifications
  9538. if usage_results:
  9539. archive_data["usage_results"] = usage_results
  9540. # Which archive and print it was (archive redesign).
  9541. archive_data["archive_id"] = archive_id
  9542. archive_data["run_id"] = completed_run_id
  9543. # Add finish photo URL and image bytes if available
  9544. if finish_photo_filename:
  9545. photo_url, photo_bytes = await _finish_photo_for_notification(
  9546. db, archive, archive_id, finish_photo_filename
  9547. )
  9548. if photo_url:
  9549. archive_data["finish_photo_url"] = photo_url
  9550. if photo_bytes:
  9551. archive_data["image_data"] = photo_bytes
  9552. if not await _kill_switch_notification_already_sent(kill_switch_notification_task):
  9553. await notification_service.on_print_complete(
  9554. printer_id, printer_name, print_status, data, db, archive_data=archive_data
  9555. )
  9556. else:
  9557. logger.info("[NOTIFY-BG] Skipped duplicate kill-switch provider notification")
  9558. # Post-print outcome confirmation (#1898). Runs in this
  9559. # background task so the finish photo fetched above rides
  9560. # along with the prompt.
  9561. if print_status == "completed" and archive_id:
  9562. await _dispatch_outcome_confirmation_safely(
  9563. db, printer_id, printer_name, data, archive_id, archive_data, run_id=completed_run_id
  9564. )
  9565. # Send user-specific email notification
  9566. if archive_data:
  9567. created_by_id = archive_data.get("created_by_id")
  9568. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  9569. await _dispatch_user_print_email(
  9570. print_status,
  9571. created_by_id,
  9572. printer_name,
  9573. raw_filename,
  9574. db,
  9575. image_data=archive_data.get("image_data"),
  9576. finish_photo_url=archive_data.get("finish_photo_url"),
  9577. )
  9578. logger.info("[NOTIFY-BG] Completed")
  9579. except Exception as e:
  9580. logger.error("[NOTIFY-BG] Failed: %s", e, exc_info=True)
  9581. async def _background_maintenance_check():
  9582. """Check for maintenance due in background."""
  9583. if print_status != "completed":
  9584. return
  9585. try:
  9586. logger.info("[MAINT-BG] Starting maintenance check for printer %s", printer_id)
  9587. async with async_session() as db:
  9588. from backend.app.models.printer import Printer
  9589. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  9590. printer = result.scalar_one_or_none()
  9591. printer_name = printer.name if printer else f"Printer {printer_id}"
  9592. await ensure_default_types(db)
  9593. overview = await _get_printer_maintenance_internal(printer_id, db, commit=True)
  9594. attention = [
  9595. item for item in overview.maintenance_items if item.enabled and (item.is_due or item.is_warning)
  9596. ]
  9597. items_needing_attention = [
  9598. {"name": item.maintenance_type_name, "is_due": item.is_due, "is_warning": item.is_warning}
  9599. for item in attention
  9600. ]
  9601. if items_needing_attention:
  9602. await notification_service.on_maintenance_due(printer_id, printer_name, items_needing_attention, db)
  9603. logger.info("[MAINT-BG] Sent notification: %s items need attention", len(items_needing_attention))
  9604. # MQTT relay - publish maintenance alerts
  9605. for item in attention:
  9606. current_value, threshold, unit = maintenance_alert_values(item)
  9607. try:
  9608. await mqtt_relay.on_maintenance_alert(
  9609. printer_id=printer_id,
  9610. printer_name=printer_name,
  9611. maintenance_type=item.maintenance_type_name,
  9612. current_value=current_value,
  9613. threshold=threshold,
  9614. unit=unit,
  9615. )
  9616. except Exception:
  9617. pass # Don't fail if MQTT fails
  9618. else:
  9619. logger.info("[MAINT-BG] Completed (no items need attention)")
  9620. except Exception as e:
  9621. logger.warning("[MAINT-BG] Failed: %s", e)
  9622. spawn_background_task(_background_smart_plug(), name="background-smart-plug")
  9623. spawn_background_task(_background_maintenance_check(), name="background-maintenance-check")
  9624. # Notification task waits for photo capture to complete first (with timeout).
  9625. # When a timelapse was recording, photo sourcing polls the per-print
  9626. # timelapse for up to 60s (#1397) — extend the budget so the notification
  9627. # carries the correct bed-up photo instead of falling through to the
  9628. # live-cam grab. Adds ~30s of notification latency at worst on slow links.
  9629. #
  9630. # #2547: both budgets now have to cover a plate restore as well.
  9631. #
  9632. # Without timelapse, the wait is on the moment producer, which raises the
  9633. # plate before its grab — so this has to outlast that producer's own budget.
  9634. #
  9635. # With timelapse, the capture polls up to
  9636. # `_FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS` for the video and only then
  9637. # falls back to a live grab, which is the case that raises the plate. At the
  9638. # old flat 75s that fallback was guaranteed to be cut off mid-settle, so the
  9639. # restore would have moved the plate for a photo nobody waited for.
  9640. photo_wait_timeout = (
  9641. _FINISH_PHOTO_TIMELAPSE_POLL_TIMEOUT_SECONDS + _FINISH_PHOTO_PRODUCER_WAIT_SECONDS
  9642. if data.get("timelapse_was_active")
  9643. else _FINISH_PHOTO_PRODUCER_WAIT_SECONDS + 15
  9644. )
  9645. async def _photo_then_notify():
  9646. """Wait for photo capture, then send notification with photo URL."""
  9647. finish_photo = None
  9648. try:
  9649. finish_photo = await asyncio.wait_for(photo_task, timeout=photo_wait_timeout)
  9650. logger.info("[PHOTO-NOTIFY] Photo task returned: %s", finish_photo)
  9651. except TimeoutError:
  9652. logger.warning(
  9653. "[PHOTO-NOTIFY] Photo capture timed out after %ss, sending notification without photo",
  9654. photo_wait_timeout,
  9655. )
  9656. except Exception as e:
  9657. logger.warning("[PHOTO-NOTIFY] Photo task failed: %s", e)
  9658. try:
  9659. await _background_notifications(finish_photo)
  9660. except Exception as e:
  9661. logger.error("[PHOTO-NOTIFY] Notification sending failed: %s", e, exc_info=True)
  9662. spawn_background_task(_photo_then_notify(), name="photo-then-notify")
  9663. # Stitch external camera layer timelapse if session was active
  9664. print_status = data.get("status", "completed")
  9665. async def _background_layer_timelapse():
  9666. """Stitch layer timelapse and attach to archive."""
  9667. from backend.app.services.layer_timelapse import cancel_session, on_print_complete as tl_complete
  9668. try:
  9669. if print_status == "completed":
  9670. logger.info("[LAYER-TL] Stitching layer timelapse for printer %s", printer_id)
  9671. # The run the frames belong to, read before the stitch: it can
  9672. # take long enough for the archive to start printing again.
  9673. run_started_at = None
  9674. if archive_id:
  9675. async with async_session() as db:
  9676. run_archive = await ArchiveService(db).get_archive(archive_id)
  9677. run_started_at = run_archive.started_at if run_archive else None
  9678. timelapse_path = await tl_complete(printer_id)
  9679. if timelapse_path and archive_id:
  9680. await _attach_layer_timelapse(
  9681. archive_id, timelapse_path, notify_plate_id, run_started_at, run_id=completed_run_id
  9682. )
  9683. elif timelapse_path:
  9684. # Timelapse created but no archive - just clean up
  9685. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  9686. else:
  9687. # Print failed or cancelled - cancel timelapse session
  9688. cancel_session(printer_id)
  9689. logger.info(
  9690. "[LAYER-TL] Cancelled layer timelapse for printer %s (status: %s)", printer_id, print_status
  9691. )
  9692. except Exception as e:
  9693. logger.warning("[LAYER-TL] Failed: %s", e)
  9694. # A completed print's session left _active_sessions when its stitch
  9695. # began: the one there now belongs to the next print, which may have
  9696. # started during the stitch, and must not lose its frames.
  9697. if print_status != "completed":
  9698. try:
  9699. cancel_session(printer_id)
  9700. except Exception:
  9701. pass # Best-effort timelapse session cancellation on error
  9702. from backend.app.services import archive_work
  9703. # Held through the stitch, which can take longer than a merge waits after
  9704. # a print (archive redesign P2).
  9705. layer_timelapse = _background_layer_timelapse()
  9706. if archive_id:
  9707. layer_timelapse = archive_work.held_while(archive_id, layer_timelapse)
  9708. spawn_background_task(layer_timelapse, name="background-layer-timelapse")
  9709. log_timing("All background tasks scheduled")
  9710. # Auto-scan for timelapse if recording was active during the print. A
  9711. # failed or cancelled print gets one too: the printer writes the video of
  9712. # the run so far when it stops (measured on an H2C, #3481), and that video
  9713. # is how the user sees what went wrong. A model that writes none costs one
  9714. # poll that finds nothing new.
  9715. if archive_id and data.get("timelapse_was_active"):
  9716. logger.info("[TIMELAPSE] Timelapse was active during print, scheduling auto-scan for archive %s", archive_id)
  9717. # Schedule timelapse scan as background task with retries
  9718. # The printer needs time to encode the video after print completion
  9719. _schedule_timelapse_scan(archive_id, printer_id, plate_id=notify_plate_id, run_id=completed_run_id)
  9720. log_timing("Timelapse scan scheduled")
  9721. logger.info("[CALLBACK] on_print_complete finished for printer %s, archive %s", printer_id, archive_id)
  9722. # AMS sensor history recording
  9723. _ams_history_task: asyncio.Task | None = None
  9724. AMS_HISTORY_INTERVAL = 300 # Record every 5 minutes
  9725. AMS_HISTORY_RETENTION_DAYS = 30 # Keep data for 30 days
  9726. def _retention_days(stored: str | None, default: int) -> int:
  9727. """A history retention setting as days to keep: at least 1, the default
  9728. when unset or unreadable. A stored 0 or less (the API took one before it
  9729. had a minimum) would put the cutoff at now and delete all history."""
  9730. try:
  9731. return max(1, int(stored)) if stored is not None else default
  9732. except (TypeError, ValueError):
  9733. return default
  9734. _ams_cleanup_counter = 0 # Track recordings to trigger periodic cleanup
  9735. # Track alarm cooldowns (printer_id:ams_id:type -> last_alarm_time)
  9736. _ams_alarm_cooldown: dict[str, datetime] = {}
  9737. AMS_ALARM_COOLDOWN_MINUTES = 60 # Don't send same alarm more than once per hour
  9738. # (printer_id, ams_id) already reported as sending the drop index and no
  9739. # percentage. Logged once each so a supported printer that turns out to do this
  9740. # shows up in a support bundle rather than as a user wondering where the
  9741. # humidity reading went -- see the note at the read site below (#3140).
  9742. _ams_index_only_logged: set[tuple[int, int]] = set()
  9743. def _resolve_temp_alarm_threshold(fair_threshold: float, raw_alarm_value: str | None) -> float:
  9744. """Temperature at which the AMS alarm fires, falling back to the display band.
  9745. ``ams_temp_fair`` decides when the AMS card turns amber. It used to decide
  9746. when a notification was sent as well, which is why a room above it made the
  9747. alarm fire once an hour for as long as the weather lasted -- and the only way
  9748. to stop that was to raise the display band and lose the colour that says the
  9749. unit is warm (#2905).
  9750. Unset resolves to the fair threshold, so an install that never sets one is
  9751. unchanged. Settings storage stringifies ``None`` to the literal ``"None"``,
  9752. so that arrives here as a string and is handled by the same branch as any
  9753. other unparseable value -- there is no separate sentinel to keep in sync.
  9754. A non-positive value is refused rather than honoured: zero would alarm
  9755. permanently, and it is far more likely to be a cleared field than a
  9756. deliberate choice.
  9757. """
  9758. if raw_alarm_value is None:
  9759. return fair_threshold
  9760. try:
  9761. value = float(raw_alarm_value)
  9762. except (TypeError, ValueError):
  9763. return fair_threshold
  9764. if not math.isfinite(value) or value <= 0:
  9765. return fair_threshold
  9766. return value
  9767. # Per-AMS "drying was live at" latch that suppresses the high-temperature alarm
  9768. # through a cycle and the cool-down after it (#1802). Stored in the settings
  9769. # table rather than alongside _ams_alarm_cooldown above, because a restart
  9770. # partway through a cool-down would otherwise resume alarming about heat the
  9771. # user asked for — the same internal-timestamp-row pattern as
  9772. # support.py's debug_logging_enabled_at.
  9773. AMS_DRYING_LATCH_KEY = "ams_drying_alarm_latch"
  9774. # Upper bound on that suppression. The latch normally clears as soon as the unit
  9775. # reads at or below the threshold; see utils.ams_drying for why this cap only
  9776. # matters when it never does.
  9777. AMS_DRYING_GRACE_MINUTES = 120
  9778. async def _load_ams_drying_latch(db) -> dict[str, datetime]:
  9779. """Read the persisted per-AMS drying latch, dropping entries out of window.
  9780. Anything older than the grace cap would expire on its next visit anyway, so
  9781. discarding it here costs nothing and stops rows for deleted printers from
  9782. accumulating.
  9783. Stamps ahead of now get two defences, because a box whose clock jumps
  9784. backwards (a Pi with no RTC coming up before NTP) writes them: wildly future
  9785. ones are discarded outright, and the rest are clamped to now. Without the
  9786. clamp the cap would measure from a moment that has not happened yet and hold
  9787. the alarm quiet for the skew on top of the cap. One unnecessary notification
  9788. after a clock jump is a far better failure than an alarm silently disabled
  9789. for hours.
  9790. """
  9791. from backend.app.models.settings import Settings
  9792. result = await db.execute(select(Settings).where(Settings.key == AMS_DRYING_LATCH_KEY))
  9793. setting = result.scalar_one_or_none()
  9794. if not setting or not setting.value:
  9795. return {}
  9796. try:
  9797. raw = json.loads(setting.value)
  9798. except (ValueError, TypeError):
  9799. return {} # Corrupted row → no latch, alarms behave as they did before
  9800. if not isinstance(raw, dict):
  9801. return {}
  9802. now = datetime.now(timezone.utc)
  9803. window = timedelta(minutes=AMS_DRYING_GRACE_MINUTES)
  9804. latch: dict[str, datetime] = {}
  9805. for key, value in raw.items():
  9806. try:
  9807. stamp = datetime.fromisoformat(str(value))
  9808. except (ValueError, TypeError):
  9809. continue
  9810. if stamp.tzinfo is None:
  9811. stamp = stamp.replace(tzinfo=timezone.utc)
  9812. if not (now - window <= stamp <= now + window):
  9813. continue
  9814. # Nothing may sit in the future: suppression is measured as now minus
  9815. # the stamp, so a stamp ahead of now would extend it by the skew on top
  9816. # of the cap. Clamping the survivors keeps the cap an actual cap.
  9817. latch[str(key)] = min(stamp, now)
  9818. return latch
  9819. async def _save_ams_drying_latch(db, latch: dict[str, datetime]) -> None:
  9820. """Persist the latch, writing only when it actually changed.
  9821. Adds the session change but does not commit — the caller's own commit
  9822. carries it, so the latch lands in the same transaction as the sensor rows
  9823. that produced it.
  9824. """
  9825. from backend.app.models.settings import Settings
  9826. payload = json.dumps({key: stamp.isoformat() for key, stamp in sorted(latch.items())})
  9827. result = await db.execute(select(Settings).where(Settings.key == AMS_DRYING_LATCH_KEY))
  9828. setting = result.scalar_one_or_none()
  9829. if setting is None:
  9830. # Don't create the row on installs that never dry anything.
  9831. if payload != "{}":
  9832. db.add(Settings(key=AMS_DRYING_LATCH_KEY, value=payload))
  9833. elif setting.value != payload:
  9834. setting.value = payload
  9835. def _ams_has_filament(ams_data: dict) -> bool:
  9836. """True if this AMS unit has at least one tray slot holding filament.
  9837. Bambu firmware reports loaded slots via `tray_exist_bits`, a per-AMS hex
  9838. bitmap (one bit per tray slot — bit set = spool present). Empty AMS units
  9839. still report sensor readings, but those readings are ambient and not
  9840. actionable: no filament to dry, no humidity to push down. #1619 — gate
  9841. humidity/temperature alarms on this check so empty units don't generate
  9842. hourly noise. Sensor history still records regardless so the UI charts
  9843. stay continuous.
  9844. Fallback path inspects the `tray` array's `tray_type` fields for setups
  9845. where `tray_exist_bits` is missing (some early-connection pushall shapes).
  9846. """
  9847. bits = ams_data.get("tray_exist_bits")
  9848. if isinstance(bits, str) and bits.strip():
  9849. try:
  9850. return int(bits, 16) > 0
  9851. except ValueError:
  9852. pass
  9853. trays = ams_data.get("tray")
  9854. if isinstance(trays, list):
  9855. return any(
  9856. isinstance(t, dict) and isinstance(t.get("tray_type"), str) and t["tray_type"].strip() for t in trays
  9857. )
  9858. return False
  9859. async def record_ams_history():
  9860. """Background task to record AMS humidity and temperature data."""
  9861. logger = logging.getLogger(__name__)
  9862. # Wait a short time for MQTT connections to establish on startup
  9863. await asyncio.sleep(10)
  9864. while True:
  9865. try:
  9866. from backend.app.models.ams_history import AMSSensorHistory
  9867. from backend.app.models.printer import Printer
  9868. from backend.app.models.settings import Settings
  9869. async with async_session() as db:
  9870. # Get all active printers
  9871. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  9872. printers = result.scalars().all()
  9873. # Get alarm thresholds from settings
  9874. humidity_threshold = 60.0 # Default: fair threshold
  9875. temp_fair_threshold = 35.0 # Display band default (ams_temp_fair)
  9876. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  9877. setting = result.scalar_one_or_none()
  9878. if setting:
  9879. try:
  9880. humidity_threshold = float(setting.value)
  9881. except (ValueError, TypeError):
  9882. pass # Keep default threshold if stored value is invalid
  9883. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_fair"))
  9884. setting = result.scalar_one_or_none()
  9885. if setting:
  9886. try:
  9887. temp_fair_threshold = float(setting.value)
  9888. except (ValueError, TypeError):
  9889. pass # Keep default threshold if stored value is invalid
  9890. # The alarm gets its own threshold, seeded from the resolved fair
  9891. # value so an install that has never set one behaves exactly as
  9892. # it did before (#2905). ams_temp_fair decides when the card turns
  9893. # amber; 35 C is a reasonable place to change a colour and not a
  9894. # reasonable place to page someone. A room above it makes the
  9895. # alarm fire once an hour for as long as the weather lasts, and
  9896. # the only way to stop it was to raise the display band and lose
  9897. # the colour that says the unit is warm.
  9898. #
  9899. # An unset value is stored as the literal "None", which the except
  9900. # below swallows the same way it swallows garbage -- so the
  9901. # fallback costs nothing and needs no sentinel of its own.
  9902. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_alarm"))
  9903. setting = result.scalar_one_or_none()
  9904. temp_alarm_threshold = _resolve_temp_alarm_threshold(
  9905. temp_fair_threshold, setting.value if setting else None
  9906. )
  9907. # Per-filament humidity threshold overrides (#1605) — resolved
  9908. # per-AMS below from the loaded tray types. Reuses the same
  9909. # resolver as the auto-drying scheduler so behavior stays in
  9910. # lockstep across both consumers.
  9911. from backend.app.services.print_scheduler import PrintScheduler
  9912. per_type_humidity_thresholds: dict[str, int] = {}
  9913. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_thresholds"))
  9914. setting = result.scalar_one_or_none()
  9915. if setting and setting.value:
  9916. try:
  9917. raw = json.loads(setting.value)
  9918. if isinstance(raw, dict):
  9919. for k, v in raw.items():
  9920. try:
  9921. per_type_humidity_thresholds[str(k).upper() if k != "default" else "default"] = int(
  9922. v
  9923. )
  9924. except (TypeError, ValueError):
  9925. continue
  9926. except (ValueError, TypeError):
  9927. pass # Invalid JSON → no overrides, fall through to global threshold
  9928. # Per-AMS drying latch (#1802), loaded once per pass and written
  9929. # back below only if a unit changed it.
  9930. drying_latch = await _load_ams_drying_latch(db)
  9931. drying_latch_before = dict(drying_latch)
  9932. recorded_count = 0
  9933. for printer in printers:
  9934. # Get current state from printer manager
  9935. state = printer_manager.get_status(printer.id)
  9936. if not state or not state.connected or not state.raw_data:
  9937. continue # Skip disconnected printers - don't use stale data
  9938. raw_data = state.raw_data
  9939. if "ams" not in raw_data or not isinstance(raw_data["ams"], list):
  9940. continue
  9941. # Record data for each AMS unit
  9942. for ams_data in raw_data["ams"]:
  9943. ams_id = int(ams_data.get("id", 0))
  9944. # Percentage only. The 1-5 index is inverted, so
  9945. # charting it as a percentage drew the wettest units as
  9946. # the driest (#3140); a unit that reports no percentage
  9947. # leaves a gap in the chart instead. See
  9948. # utils/ams_humidity.
  9949. humidity = ams_humidity_percent(ams_data)
  9950. # No supported printer is known to send the index
  9951. # alone -- the report came from unsupported firmware,
  9952. # and no install has been seen using the old fallback.
  9953. # "Known" is doing work there, so say so once per unit:
  9954. # the alternative is a silent blank card.
  9955. if humidity is None and ams_data.get("humidity") is not None:
  9956. unit_key = (printer.id, ams_id)
  9957. if unit_key not in _ams_index_only_logged:
  9958. _ams_index_only_logged.add(unit_key)
  9959. logger.info(
  9960. "[%s] AMS %d reports the 1-5 humidity index but no usable humidity_raw "
  9961. "percentage. The index is inverted and is not shown as a percentage "
  9962. "(#3140), so this unit has no humidity reading, chart or alarm. "
  9963. "Please report this with the printer and AMS firmware versions.",
  9964. printer.name,
  9965. ams_id,
  9966. )
  9967. # Get temperature
  9968. temperature = None
  9969. temp_str = ams_data.get("temp")
  9970. if temp_str is not None:
  9971. try:
  9972. temperature = float(temp_str)
  9973. except (ValueError, TypeError):
  9974. pass # Skip unparseable temperature value
  9975. # Skip if no data
  9976. if humidity is None and temperature is None:
  9977. continue
  9978. # Record the data point
  9979. history = AMSSensorHistory(
  9980. printer_id=printer.id,
  9981. ams_id=ams_id,
  9982. humidity=humidity,
  9983. # Both columns hold the same reading now that the
  9984. # index can no longer reach ``humidity``. Writing it
  9985. # through the same value also stops a genuine 0%
  9986. # from being stored as NULL, which the old truthiness
  9987. # test did.
  9988. humidity_raw=humidity,
  9989. temperature=temperature,
  9990. )
  9991. db.add(history)
  9992. recorded_count += 1
  9993. # Generate AMS label and determine if it's AMS-HT (A, B, C, D or HT-A for AMS-Lite/Hub)
  9994. ams_type = unit_ams_type(ams_id, ams_data.get("info"))
  9995. is_ams_ht = ams_type == AmsType.N3S
  9996. ams_label = unit_label(ams_id, ams_type)
  9997. # Skip alarm dispatch for empty AMS units — humidity /
  9998. # temperature readings are ambient with no filament to
  9999. # protect, and the hourly notification just becomes
  10000. # noise. Sensor history was already recorded above so
  10001. # the UI charts stay continuous (#1619). Per-AMS check
  10002. # so a multi-AMS setup with one loaded + one empty
  10003. # still alarms on the loaded unit.
  10004. if not _ams_has_filament(ams_data):
  10005. continue
  10006. # Resolve per-filament humidity threshold for this AMS
  10007. # unit (#1605). Falls back to the global ams_humidity_fair
  10008. # when no per-type overrides are configured.
  10009. trays = ams_data.get("tray", []) or []
  10010. effective_humidity_threshold = float(
  10011. PrintScheduler.resolve_humidity_threshold(
  10012. trays, per_type_humidity_thresholds, int(humidity_threshold)
  10013. )
  10014. )
  10015. # Check humidity alarm (only if above threshold)
  10016. if humidity is not None and humidity > effective_humidity_threshold:
  10017. cooldown_key = f"{printer.id}:{ams_id}:humidity"
  10018. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  10019. now = datetime.now(timezone.utc)
  10020. if (
  10021. last_alarm is None
  10022. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  10023. ):
  10024. _ams_alarm_cooldown[cooldown_key] = now
  10025. logger.info(
  10026. f"Sending humidity alarm for {printer.name} {ams_label}: {humidity}% > {effective_humidity_threshold}%"
  10027. )
  10028. try:
  10029. # Call different notification method based on AMS type
  10030. if is_ams_ht:
  10031. await notification_service.on_ams_ht_humidity_high(
  10032. printer.id,
  10033. printer.name,
  10034. ams_label,
  10035. humidity,
  10036. effective_humidity_threshold,
  10037. db,
  10038. )
  10039. else:
  10040. await notification_service.on_ams_humidity_high(
  10041. printer.id,
  10042. printer.name,
  10043. ams_label,
  10044. humidity,
  10045. effective_humidity_threshold,
  10046. db,
  10047. )
  10048. except Exception as e:
  10049. logger.warning("Failed to send humidity alarm: %s", e)
  10050. # A drying cycle heats the unit far past ams_temp_fair on
  10051. # purpose — 45 C for PLA, 65 C for PETG, 85 C on an
  10052. # AMS-HT, against a 35 C default — so the alarm fired
  10053. # once an hour for the whole cycle and kept firing while
  10054. # the unit cooled back down (#1802). Latch on the
  10055. # firmware's own drying state and hold until the reading
  10056. # returns to normal. Humidity is deliberately left alone:
  10057. # it falls during drying, which is the whole point.
  10058. latch_key = f"{printer.id}:{ams_id}"
  10059. # The latch releases at `threshold`, so it takes the alarm
  10060. # number too. Handing it the display band would strand the
  10061. # latch on any unit that settles back above it -- a room
  10062. # where the AMS rests at 37.7 C never returns under a 35 C
  10063. # band, so the latch could only expire on the grace cap
  10064. # rather than releasing when the unit had actually cooled.
  10065. suppress_temp_alarm, new_latch = temperature_alarm_suppressed(
  10066. drying_active=is_drying_active(ams_data),
  10067. temperature=temperature,
  10068. threshold=temp_alarm_threshold,
  10069. latched_at=drying_latch.get(latch_key),
  10070. now=datetime.now(timezone.utc),
  10071. grace_minutes=AMS_DRYING_GRACE_MINUTES,
  10072. )
  10073. if new_latch is None:
  10074. drying_latch.pop(latch_key, None)
  10075. else:
  10076. drying_latch[latch_key] = new_latch
  10077. # Check temperature alarm (only if above threshold)
  10078. if temperature is not None and temperature > temp_alarm_threshold and not suppress_temp_alarm:
  10079. cooldown_key = f"{printer.id}:{ams_id}:temperature"
  10080. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  10081. now = datetime.now(timezone.utc)
  10082. if (
  10083. last_alarm is None
  10084. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  10085. ):
  10086. _ams_alarm_cooldown[cooldown_key] = now
  10087. logger.info(
  10088. f"Sending temperature alarm for {printer.name} {ams_label}: "
  10089. f"{temperature}°C > {temp_alarm_threshold}°C"
  10090. )
  10091. try:
  10092. # Call different notification method based on AMS type
  10093. if is_ams_ht:
  10094. # The reported threshold has to be the one
  10095. # that fired, or the message says "> 35 °C"
  10096. # while firing at 45.
  10097. await notification_service.on_ams_ht_temperature_high(
  10098. printer.id, printer.name, ams_label, temperature, temp_alarm_threshold, db
  10099. )
  10100. else:
  10101. await notification_service.on_ams_temperature_high(
  10102. printer.id, printer.name, ams_label, temperature, temp_alarm_threshold, db
  10103. )
  10104. except Exception as e:
  10105. logger.warning("Failed to send temperature alarm: %s", e)
  10106. if drying_latch != drying_latch_before:
  10107. await _save_ams_drying_latch(db, drying_latch)
  10108. await db.commit()
  10109. if recorded_count > 0:
  10110. logger.info("Recorded %s AMS sensor history entries", recorded_count)
  10111. # Periodic cleanup of old data (every ~288 recordings = ~24 hours at 5min interval)
  10112. global _ams_cleanup_counter
  10113. _ams_cleanup_counter += 1
  10114. if _ams_cleanup_counter >= 288:
  10115. _ams_cleanup_counter = 0
  10116. # Get retention days from settings
  10117. from backend.app.models.settings import Settings
  10118. result = await db.execute(select(Settings).where(Settings.key == "ams_history_retention_days"))
  10119. setting = result.scalar_one_or_none()
  10120. retention_days = _retention_days(setting.value if setting else None, AMS_HISTORY_RETENTION_DAYS)
  10121. cutoff = utcnow_naive() - timedelta(days=retention_days)
  10122. result = await db.execute(delete(AMSSensorHistory).where(AMSSensorHistory.recorded_at < cutoff))
  10123. await db.commit()
  10124. if result.rowcount > 0:
  10125. logger.info(
  10126. f"Cleaned up {result.rowcount} old AMS sensor history entries (older than {retention_days} days)"
  10127. )
  10128. # Wait until next recording interval
  10129. await asyncio.sleep(AMS_HISTORY_INTERVAL)
  10130. except asyncio.CancelledError:
  10131. break
  10132. except Exception as e:
  10133. logger.warning("AMS history recording failed: %s", e)
  10134. await asyncio.sleep(60) # Wait a bit before retrying
  10135. def start_ams_history_recording():
  10136. """Start the AMS history recording background task."""
  10137. global _ams_history_task
  10138. if _ams_history_task is None:
  10139. _ams_history_task = asyncio.create_task(record_ams_history())
  10140. logging.getLogger(__name__).info("AMS history recording started")
  10141. def stop_ams_history_recording():
  10142. """Stop the AMS history recording background task."""
  10143. global _ams_history_task
  10144. if _ams_history_task:
  10145. _ams_history_task.cancel()
  10146. _ams_history_task = None
  10147. logging.getLogger(__name__).info("AMS history recording stopped")
  10148. # Printer sensor history recording (nozzle / bed / chamber)
  10149. _printer_sensor_history_task: asyncio.Task | None = None
  10150. PRINTER_SENSOR_HISTORY_INTERVAL = 60 # Record every minute — heaters move faster than AMS humidity
  10151. PRINTER_SENSOR_HISTORY_RETENTION_DAYS = 30
  10152. _printer_sensor_cleanup_counter = 0
  10153. # Sensor kinds tracked in state.temperatures — these are the normalised keys the
  10154. # MQTT parser writes, so we don't need to handle per-model field aliases here
  10155. # (nozzle_temper / left_nozzle_temper / right_nozzle_temper / chamber_temper
  10156. # are all collapsed by services/bambu_mqtt.py before they reach this loop).
  10157. _SENSOR_KINDS = ("nozzle", "nozzle_2", "bed", "chamber")
  10158. _SENSOR_TARGET_KEYS = {
  10159. "nozzle": "nozzle_target",
  10160. "nozzle_2": "nozzle_2_target",
  10161. "bed": "bed_target",
  10162. "chamber": "chamber_target",
  10163. }
  10164. async def record_printer_sensor_history():
  10165. """Background task to record nozzle / bed / chamber readings.
  10166. Pulls from `state.temperatures` (already normalised across all printer
  10167. models by the MQTT parser) rather than re-parsing raw_data, so we get
  10168. free coverage of dual-nozzle H2D, sensor-only X1C chamber, etc.
  10169. """
  10170. logger = logging.getLogger(__name__)
  10171. await asyncio.sleep(10)
  10172. while True:
  10173. try:
  10174. from backend.app.models.printer import Printer
  10175. from backend.app.models.printer_sensor_history import PrinterSensorHistory
  10176. from backend.app.models.settings import Settings
  10177. async with async_session() as db:
  10178. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  10179. printers = result.scalars().all()
  10180. recorded_count = 0
  10181. for printer in printers:
  10182. state = printer_manager.get_status(printer.id)
  10183. if not state or not state.connected:
  10184. continue
  10185. temps = getattr(state, "temperatures", None) or {}
  10186. if not isinstance(temps, dict):
  10187. continue
  10188. for kind in _SENSOR_KINDS:
  10189. if kind not in temps:
  10190. continue
  10191. try:
  10192. value = float(temps[kind])
  10193. except (ValueError, TypeError):
  10194. continue
  10195. target_raw = temps.get(_SENSOR_TARGET_KEYS[kind])
  10196. target_val: float | None = None
  10197. if target_raw is not None:
  10198. try:
  10199. target_val = float(target_raw)
  10200. except (ValueError, TypeError):
  10201. target_val = None
  10202. db.add(
  10203. PrinterSensorHistory(
  10204. printer_id=printer.id,
  10205. sensor_kind=kind,
  10206. value=value,
  10207. target=target_val,
  10208. )
  10209. )
  10210. recorded_count += 1
  10211. await db.commit()
  10212. if recorded_count > 0:
  10213. logger.debug("Recorded %s printer sensor history entries", recorded_count)
  10214. # Periodic cleanup — once every ~24h at this interval.
  10215. global _printer_sensor_cleanup_counter
  10216. _printer_sensor_cleanup_counter += 1
  10217. cleanup_every = max(1, (24 * 60 * 60) // PRINTER_SENSOR_HISTORY_INTERVAL)
  10218. if _printer_sensor_cleanup_counter >= cleanup_every:
  10219. _printer_sensor_cleanup_counter = 0
  10220. result = await db.execute(
  10221. select(Settings).where(Settings.key == "printer_sensor_history_retention_days")
  10222. )
  10223. setting = result.scalar_one_or_none()
  10224. retention_days = _retention_days(
  10225. setting.value if setting else None, PRINTER_SENSOR_HISTORY_RETENTION_DAYS
  10226. )
  10227. cutoff = utcnow_naive() - timedelta(days=retention_days)
  10228. cleanup = await db.execute(
  10229. delete(PrinterSensorHistory).where(PrinterSensorHistory.recorded_at < cutoff)
  10230. )
  10231. await db.commit()
  10232. if cleanup.rowcount > 0:
  10233. logger.info(
  10234. "Cleaned up %s old printer sensor history entries (older than %s days)",
  10235. cleanup.rowcount,
  10236. retention_days,
  10237. )
  10238. await asyncio.sleep(PRINTER_SENSOR_HISTORY_INTERVAL)
  10239. except asyncio.CancelledError:
  10240. break
  10241. except Exception as e:
  10242. logger.warning("Printer sensor history recording failed: %s", e)
  10243. await asyncio.sleep(60)
  10244. def start_printer_sensor_history_recording():
  10245. global _printer_sensor_history_task
  10246. if _printer_sensor_history_task is None:
  10247. _printer_sensor_history_task = asyncio.create_task(record_printer_sensor_history())
  10248. logging.getLogger(__name__).info("Printer sensor history recording started")
  10249. def stop_printer_sensor_history_recording():
  10250. global _printer_sensor_history_task
  10251. if _printer_sensor_history_task:
  10252. _printer_sensor_history_task.cancel()
  10253. _printer_sensor_history_task = None
  10254. logging.getLogger(__name__).info("Printer sensor history recording stopped")
  10255. # Printer runtime tracking
  10256. _runtime_tracking_task: asyncio.Task | None = None
  10257. RUNTIME_TRACKING_INTERVAL = 30 # Update every 30 seconds
  10258. # Printers the runtime tracker has seen connected since startup. One of them
  10259. # going offline clears its timestamp, so the time it was away is not counted
  10260. # when it comes back printing (DB-C5). A printer not yet seen -- still
  10261. # connecting after a restart -- keeps it for a grace period: a print that ran
  10262. # on through a Bambuddy restart is counted from the stored timestamp, as before.
  10263. _runtime_seen_connected: set[int] = set()
  10264. # A printer never seen connected this long after the tracker started is
  10265. # offline, not still connecting: its stored timestamp predates a gap too.
  10266. RUNTIME_STARTUP_GRACE_SECONDS = 120
  10267. async def track_printer_runtime():
  10268. """Background task to track printer active runtime (RUNNING state only).
  10269. PAUSE is intentionally excluded — the runtime counter feeds hours-based
  10270. maintenance intervals (rod lubrication, belt checks, nozzle cleaning)
  10271. which track mechanical wear. Pause time has no motion and no wear, so
  10272. counting it inflates maintenance warnings (#1521).
  10273. """
  10274. logger = logging.getLogger(__name__)
  10275. # Wait for MQTT connections to establish on startup
  10276. await asyncio.sleep(15)
  10277. started = time.monotonic()
  10278. while True:
  10279. try:
  10280. from backend.app.models.printer import Printer
  10281. # Fetch printer IDs in a short-lived read-only session
  10282. async with async_session() as db:
  10283. result = await db.execute(
  10284. select(Printer.id, Printer.name, Printer.runtime_seconds, Printer.last_runtime_update).where(
  10285. Printer.is_active.is_(True)
  10286. )
  10287. )
  10288. printer_rows = result.all()
  10289. now = datetime.now(timezone.utc)
  10290. updated_count = 0
  10291. # Update each printer in its own short session to minimise write-lock
  10292. # hold time and avoid blocking critical commits like queue status
  10293. # updates (#897).
  10294. for pid, pname, runtime_secs, last_update in printer_rows:
  10295. state = printer_manager.get_status(pid)
  10296. if not state:
  10297. logger.debug("[%s] Runtime tracking: no state available", pname)
  10298. continue
  10299. if not state.connected:
  10300. logger.debug("[%s] Runtime tracking: not connected", pname)
  10301. offline = pid in _runtime_seen_connected or (
  10302. time.monotonic() - started > RUNTIME_STARTUP_GRACE_SECONDS
  10303. )
  10304. if offline and last_update is not None:
  10305. try:
  10306. async with async_session() as db:
  10307. printer = await db.get(Printer, pid)
  10308. if printer:
  10309. printer.last_runtime_update = None
  10310. await db.commit()
  10311. except Exception as e:
  10312. # Still marked: the next offline tick tries again.
  10313. logger.warning("[%s] Runtime tracking commit failed: %s", pname, e)
  10314. continue
  10315. _runtime_seen_connected.discard(pid)
  10316. continue
  10317. _runtime_seen_connected.add(pid)
  10318. needs_commit = False
  10319. new_runtime = runtime_secs
  10320. new_last_update = last_update
  10321. if state.state == "RUNNING":
  10322. if last_update:
  10323. lu = last_update if last_update.tzinfo else last_update.replace(tzinfo=timezone.utc)
  10324. elapsed = (now - lu).total_seconds()
  10325. if elapsed > 0:
  10326. new_runtime = runtime_secs + int(elapsed)
  10327. updated_count += 1
  10328. needs_commit = True
  10329. logger.debug(
  10330. f"[{pname}] Runtime tracking: added {int(elapsed)}s, "
  10331. f"total={new_runtime}s ({new_runtime / 3600:.2f}h)"
  10332. )
  10333. else:
  10334. needs_commit = True
  10335. logger.debug("[%s] Runtime tracking: first active detection", pname)
  10336. new_last_update = now
  10337. else:
  10338. if last_update is not None:
  10339. logger.debug(f"[{pname}] Runtime tracking: state={state.state}, clearing last_runtime_update")
  10340. new_last_update = None
  10341. needs_commit = True
  10342. if needs_commit:
  10343. try:
  10344. async with async_session() as db:
  10345. result = await db.execute(select(Printer).where(Printer.id == pid))
  10346. printer = result.scalar_one_or_none()
  10347. if printer:
  10348. printer.runtime_seconds = new_runtime
  10349. printer.last_runtime_update = new_last_update
  10350. await db.commit()
  10351. except Exception as e:
  10352. logger.warning("[%s] Runtime tracking commit failed: %s", pname, e)
  10353. if updated_count > 0:
  10354. logger.debug("Updated runtime for %s printer(s)", updated_count)
  10355. except asyncio.CancelledError:
  10356. logger.info("Runtime tracking cancelled")
  10357. break
  10358. except Exception as e:
  10359. logger.warning("Runtime tracking failed: %s", e)
  10360. await asyncio.sleep(RUNTIME_TRACKING_INTERVAL)
  10361. def start_runtime_tracking():
  10362. """Start the printer runtime tracking background task."""
  10363. global _runtime_tracking_task
  10364. if _runtime_tracking_task is None:
  10365. _runtime_tracking_task = asyncio.create_task(track_printer_runtime())
  10366. logging.getLogger(__name__).info("Printer runtime tracking started")
  10367. def stop_runtime_tracking():
  10368. """Stop the printer runtime tracking background task."""
  10369. global _runtime_tracking_task
  10370. if _runtime_tracking_task:
  10371. _runtime_tracking_task.cancel()
  10372. _runtime_tracking_task = None
  10373. logging.getLogger(__name__).info("Printer runtime tracking stopped")
  10374. # SpoolBuddy device watchdog
  10375. _spoolbuddy_watchdog_task: asyncio.Task | None = None
  10376. SPOOLBUDDY_WATCHDOG_INTERVAL = 15
  10377. async def _spoolbuddy_watchdog_loop():
  10378. """Periodic check for SpoolBuddy devices that have gone offline."""
  10379. from backend.app.api.routes.spoolbuddy import spoolbuddy_watchdog
  10380. while True:
  10381. try:
  10382. await spoolbuddy_watchdog()
  10383. except asyncio.CancelledError:
  10384. break
  10385. except Exception as e:
  10386. logging.getLogger(__name__).warning("SpoolBuddy watchdog failed: %s", e)
  10387. await asyncio.sleep(SPOOLBUDDY_WATCHDOG_INTERVAL)
  10388. def start_spoolbuddy_watchdog():
  10389. global _spoolbuddy_watchdog_task
  10390. if _spoolbuddy_watchdog_task is None:
  10391. _spoolbuddy_watchdog_task = asyncio.create_task(_spoolbuddy_watchdog_loop())
  10392. logging.getLogger(__name__).info("SpoolBuddy watchdog started")
  10393. def stop_spoolbuddy_watchdog():
  10394. global _spoolbuddy_watchdog_task
  10395. if _spoolbuddy_watchdog_task:
  10396. _spoolbuddy_watchdog_task.cancel()
  10397. _spoolbuddy_watchdog_task = None
  10398. logging.getLogger(__name__).info("SpoolBuddy watchdog stopped")
  10399. # Dead-MQTT-session recovery
  10400. #
  10401. # check_staleness() covers the "connected but silent" half-broken session. It
  10402. # does nothing once ``state.connected`` is False, and paho's own auto-reconnect
  10403. # is the only thing left watching at that point. When paho stops making
  10404. # progress there is no backstop at all: the #2732 bundle has a P1S drop on a
  10405. # keep-alive timeout at 02:19 and not reconnect until 11:24 — nine hours
  10406. # offline with the UI open the whole time, recovered only when something
  10407. # happened to nudge it.
  10408. #
  10409. # This loop is that backstop. It only touches printers that had a working
  10410. # session and lost it, and only when the MQTT port still answers — a printer
  10411. # that is simply switched off is left to paho, since rebuilding a client
  10412. # against an unreachable host achieves nothing and would fill the log every
  10413. # night.
  10414. _connection_watchdog_task: asyncio.Task | None = None
  10415. CONNECTION_WATCHDOG_INTERVAL = 60
  10416. # How long a printer must have been silent before we stop trusting paho.
  10417. # Comfortably above STALE_TIMEOUT (60 s) and the max reconnect backoff (30 s),
  10418. # so a session that is recovering on its own is never interrupted.
  10419. CONNECTION_WATCHDOG_OFFLINE_GRACE = 300
  10420. # Per-printer floor between rebuild attempts.
  10421. CONNECTION_WATCHDOG_RETRY_INTERVAL = 300
  10422. _connection_watchdog_last_attempt: dict[int, float] = {}
  10423. async def _recover_dead_printer_sessions() -> int:
  10424. """Rebuild MQTT clients that have been offline too long to still be trying.
  10425. Returns the number of printers a rebuild was attempted for (for tests and
  10426. for the caller's logging). Never raises: one unreachable printer must not
  10427. stop the sweep for the rest of the farm.
  10428. """
  10429. logger = logging.getLogger(__name__)
  10430. from backend.app.services.printer_diagnostic import PORT_MQTT, check_port
  10431. now = time.monotonic()
  10432. recovered = 0
  10433. for printer_id, client in list(printer_manager._clients.items()):
  10434. try:
  10435. if client.state.connected:
  10436. _connection_watchdog_last_attempt.pop(printer_id, None)
  10437. continue
  10438. # Time since the last inbound message is the age of the last known
  10439. # good session — no extra bookkeeping needed, and it is the same
  10440. # clock is_stale() reads. 0 means this client has never had one:
  10441. # that is the initial-connect path, where paho retrying is the
  10442. # correct and only behaviour, so leave it be.
  10443. last_msg = client._last_message_time
  10444. if not last_msg:
  10445. continue
  10446. offline_for = time.time() - last_msg
  10447. if offline_for < CONNECTION_WATCHDOG_OFFLINE_GRACE:
  10448. continue
  10449. last_attempt = _connection_watchdog_last_attempt.get(printer_id)
  10450. if last_attempt is not None and now - last_attempt < CONNECTION_WATCHDOG_RETRY_INTERVAL:
  10451. continue
  10452. if not await check_port(client.ip_address, PORT_MQTT):
  10453. # Switched off, unplugged, or off the network. Paho's retry is
  10454. # the right handler; say so at debug level and move on.
  10455. logger.debug(
  10456. "[#2732] Printer %s offline for %.0fs and its MQTT port is not answering "
  10457. "— leaving the reconnect to paho",
  10458. printer_id,
  10459. offline_for,
  10460. )
  10461. _connection_watchdog_last_attempt[printer_id] = now
  10462. continue
  10463. _connection_watchdog_last_attempt[printer_id] = now
  10464. recovered += 1
  10465. logger.warning(
  10466. "[#2732] Printer %s has been offline for %.0fs but answers on MQTT port %d — "
  10467. "rebuilding the client with a fresh session (last connect error: %s)",
  10468. printer_id,
  10469. offline_for,
  10470. PORT_MQTT,
  10471. client.last_connect_error or "none recorded",
  10472. )
  10473. # Async context, so this takes the hard-reset path: fresh client_id,
  10474. # paho's QoS 1 queue dropped. That matters — a project_file left
  10475. # unacked on the dead session would otherwise replay into the new
  10476. # one and trip 0500_4003 on the printer (#1136).
  10477. client.force_reconnect_stale_session(f"offline for {offline_for:.0f}s, port still answering")
  10478. except Exception as e:
  10479. logger.warning("[#2732] Connection watchdog failed for printer %s: %s", printer_id, e)
  10480. return recovered
  10481. async def _connection_watchdog_loop():
  10482. logger = logging.getLogger(__name__)
  10483. # Let the initial connects settle before judging anyone offline.
  10484. await asyncio.sleep(CONNECTION_WATCHDOG_OFFLINE_GRACE)
  10485. while True:
  10486. try:
  10487. await _recover_dead_printer_sessions()
  10488. except asyncio.CancelledError:
  10489. break
  10490. except Exception as e:
  10491. logger.warning("Connection watchdog sweep failed: %s", e)
  10492. await asyncio.sleep(CONNECTION_WATCHDOG_INTERVAL)
  10493. def start_connection_watchdog():
  10494. global _connection_watchdog_task
  10495. if _connection_watchdog_task is None:
  10496. _connection_watchdog_task = asyncio.create_task(_connection_watchdog_loop())
  10497. logging.getLogger(__name__).info("Printer connection watchdog started")
  10498. def stop_connection_watchdog():
  10499. global _connection_watchdog_task
  10500. if _connection_watchdog_task:
  10501. _connection_watchdog_task.cancel()
  10502. _connection_watchdog_task = None
  10503. _connection_watchdog_last_attempt.clear()
  10504. logging.getLogger(__name__).info("Printer connection watchdog stopped")
  10505. # Camera stream orphan cleanup
  10506. _camera_cleanup_task: asyncio.Task | None = None
  10507. CAMERA_CLEANUP_INTERVAL = 60
  10508. # Every this many camera cleanups (about an hour), the timelapse janitor too:
  10509. # kept frames of a failed stitch go after a day, not at the next restart.
  10510. TIMELAPSE_CLEANUP_EVERY = 60
  10511. async def _camera_cleanup_loop():
  10512. """Periodically clean up orphaned ffmpeg processes and timelapse leftovers."""
  10513. from backend.app.api.routes.camera import cleanup_orphaned_streams
  10514. from backend.app.services.layer_timelapse import cleanup_orphaned_timelapse_sessions
  10515. rounds = 0
  10516. while True:
  10517. try:
  10518. rounds += 1
  10519. await cleanup_orphaned_streams()
  10520. except asyncio.CancelledError:
  10521. break
  10522. except Exception as e:
  10523. logging.getLogger(__name__).warning("Camera stream cleanup failed: %s", e)
  10524. try:
  10525. if rounds % TIMELAPSE_CLEANUP_EVERY == 0:
  10526. await asyncio.to_thread(cleanup_orphaned_timelapse_sessions)
  10527. except asyncio.CancelledError:
  10528. break
  10529. except Exception as e:
  10530. logging.getLogger(__name__).warning("Timelapse frames cleanup failed: %s", e)
  10531. await asyncio.sleep(CAMERA_CLEANUP_INTERVAL)
  10532. def start_camera_cleanup():
  10533. global _camera_cleanup_task
  10534. if _camera_cleanup_task is None:
  10535. _camera_cleanup_task = asyncio.create_task(_camera_cleanup_loop())
  10536. logging.getLogger(__name__).info("Camera stream cleanup started")
  10537. def stop_camera_cleanup():
  10538. global _camera_cleanup_task
  10539. if _camera_cleanup_task:
  10540. _camera_cleanup_task.cancel()
  10541. _camera_cleanup_task = None
  10542. logging.getLogger(__name__).info("Camera stream cleanup stopped")
  10543. # ---------------------------------------------------------------------------
  10544. # Expected-print TTL eviction
  10545. # ---------------------------------------------------------------------------
  10546. def _evict_stale_expected_prints() -> None:
  10547. """Remove entries from _expected_prints / _expected_print_creators that are
  10548. older than _EXPECTED_PRINT_TTL_SECONDS.
  10549. This prevents unbounded growth when a print is registered (via
  10550. register_expected_print) but on_print_start never fires — e.g. because the
  10551. printer disconnects, the app restarts, or the print is started directly from
  10552. the printer panel without going through the queue.
  10553. """
  10554. # Use monotonic time so the TTL is unaffected by system clock adjustments
  10555. # (e.g. NTP sync, DST changes).
  10556. # A print that ended without its archive being found at completion leaves
  10557. # its id here; only running prints are owned.
  10558. _owned_print_archive_ids.intersection_update(_active_prints.values())
  10559. cutoff = time.monotonic() - _EXPECTED_PRINT_TTL_SECONDS
  10560. stale_keys = [k for k, t in _expected_print_registered_at.items() if t < cutoff]
  10561. if not stale_keys:
  10562. return
  10563. evicted: set[tuple[int, int]] = set()
  10564. for key in stale_keys:
  10565. archive_id = _expected_prints.pop(key, None)
  10566. if archive_id is not None:
  10567. evicted.add((key[0], archive_id))
  10568. _expected_print_creators.pop(key, None)
  10569. _expected_print_registered_at.pop(key, None)
  10570. # Also clean up _print_ams_mappings and _print_plate_ids for archive_ids
  10571. # that have no remaining live keys in _expected_prints (all variants
  10572. # were just evicted) and are not printing right now. Print start pops
  10573. # only the names the printer reported, so the "<base>.gcode" variant
  10574. # outlives it; without the printing check its eviction two hours after
  10575. # dispatch threw away a running print's plate, AMS mapping and cost
  10576. # centre, which completion still reads (#3275).
  10577. for printer_id, archive_id in evicted:
  10578. if not _expected_or_printing_here(printer_id, archive_id):
  10579. _drop_dispatch(printer_id, archive_id)
  10580. logging.getLogger(__name__).info(
  10581. "Evicted %d stale expected-print entries (TTL=%ds)", len(stale_keys), _EXPECTED_PRINT_TTL_SECONDS
  10582. )
  10583. async def _expected_prints_cleanup_loop() -> None:
  10584. """Background task: periodically evict stale expected-print entries."""
  10585. while True:
  10586. try:
  10587. _evict_stale_expected_prints()
  10588. except asyncio.CancelledError:
  10589. raise
  10590. except Exception as e:
  10591. logging.getLogger(__name__).warning("Expected prints cleanup failed: %s", e)
  10592. await asyncio.sleep(_EXPECTED_PRINT_CLEANUP_INTERVAL)
  10593. def start_expected_prints_cleanup() -> None:
  10594. global _expected_prints_cleanup_task
  10595. if _expected_prints_cleanup_task is None:
  10596. _expected_prints_cleanup_task = asyncio.create_task(_expected_prints_cleanup_loop())
  10597. logging.getLogger(__name__).info("Expected prints cleanup started")
  10598. def stop_expected_prints_cleanup() -> None:
  10599. global _expected_prints_cleanup_task
  10600. if _expected_prints_cleanup_task:
  10601. _expected_prints_cleanup_task.cancel()
  10602. _expected_prints_cleanup_task = None
  10603. logging.getLogger(__name__).info("Expected prints cleanup stopped")
  10604. # ---------------------------------------------------------------------------
  10605. # L-2: Periodic auth-token cleanup (stale TOTP + expired revoked JTIs)
  10606. # ---------------------------------------------------------------------------
  10607. _auth_cleanup_task: asyncio.Task | None = None
  10608. _AUTH_CLEANUP_INTERVAL = 3600 # seconds (hourly)
  10609. async def _run_auth_cleanup() -> None:
  10610. """Single cleanup pass: remove stale TOTP records, expired revoked JTIs, and old rate-limit events."""
  10611. from backend.app.core.database import async_session
  10612. from backend.app.models.auth_ephemeral import AuthEphemeralToken, AuthRateLimitEvent
  10613. from backend.app.models.user_totp import UserTOTP
  10614. now = datetime.now(timezone.utc)
  10615. # Remove unconfirmed (is_enabled=False) TOTP records older than 1 hour.
  10616. try:
  10617. async with async_session() as db:
  10618. stale_cutoff = now - timedelta(hours=1)
  10619. result = await db.execute(
  10620. select(UserTOTP).where(
  10621. UserTOTP.is_enabled.is_(False),
  10622. UserTOTP.created_at < stale_cutoff,
  10623. )
  10624. )
  10625. stale_records = result.scalars().all()
  10626. if stale_records:
  10627. for rec in stale_records:
  10628. await db.delete(rec)
  10629. await db.commit()
  10630. logging.info("Auth cleanup: removed %d stale unconfirmed TOTP record(s)", len(stale_records))
  10631. except Exception as e:
  10632. logging.warning("Auth cleanup: failed to purge stale TOTP records: %s", e)
  10633. # Remove expired revoked-JTI entries (they are no longer needed once the
  10634. # original token's exp has passed — the token would be rejected by JWT
  10635. # signature verification regardless).
  10636. try:
  10637. async with async_session() as db:
  10638. await db.execute(
  10639. delete(AuthEphemeralToken).where(
  10640. AuthEphemeralToken.token_type == "revoked_jti",
  10641. AuthEphemeralToken.expires_at < now,
  10642. )
  10643. )
  10644. await db.commit()
  10645. except Exception as e:
  10646. logging.warning("Auth cleanup: failed to purge expired revoked JTIs: %s", e)
  10647. # L-R6-B: Purge AuthRateLimitEvent rows older than the lockout window (15 min).
  10648. # Events outside this window can never affect rate-limit decisions — they only
  10649. # consume DB space. Use the same window constant as the rate limiter so the
  10650. # two are always in sync.
  10651. try:
  10652. from backend.app.api.routes.mfa import LOCKOUT_WINDOW
  10653. async with async_session() as db:
  10654. await db.execute(
  10655. delete(AuthRateLimitEvent).where(
  10656. AuthRateLimitEvent.occurred_at < now - LOCKOUT_WINDOW,
  10657. )
  10658. )
  10659. await db.commit()
  10660. except Exception as e:
  10661. logging.warning("Auth cleanup: failed to purge stale rate-limit events: %s", e)
  10662. async def _auth_cleanup_loop() -> None:
  10663. """Periodic background task: run auth cleanup every hour."""
  10664. while True:
  10665. try:
  10666. await _run_auth_cleanup()
  10667. except asyncio.CancelledError:
  10668. break
  10669. except Exception as e:
  10670. logging.warning("Auth cleanup loop error: %s", e)
  10671. await asyncio.sleep(_AUTH_CLEANUP_INTERVAL)
  10672. def start_auth_cleanup() -> None:
  10673. global _auth_cleanup_task
  10674. if _auth_cleanup_task is None:
  10675. _auth_cleanup_task = asyncio.create_task(_auth_cleanup_loop())
  10676. logging.getLogger(__name__).info("Auth periodic cleanup started")
  10677. def stop_auth_cleanup() -> None:
  10678. global _auth_cleanup_task
  10679. if _auth_cleanup_task:
  10680. _auth_cleanup_task.cancel()
  10681. _auth_cleanup_task = None
  10682. logging.getLogger(__name__).info("Auth periodic cleanup stopped")
  10683. async def initialize_wled_printers(db: AsyncSession) -> None:
  10684. """Seed WLED settings and apply offline presets before MQTT connections start."""
  10685. from backend.app.api.routes.settings import get_setting, setting_is_true
  10686. from backend.app.models.printer import Printer
  10687. wled_manager.require_plate_clear = setting_is_true(await get_setting(db, "require_plate_clear"))
  10688. result = await db.execute(select(Printer.id, Printer.wled_config, Printer.is_active))
  10689. for printer_id, config, is_active in result.all():
  10690. wled_manager.configure_printer(printer_id, config)
  10691. if not is_active and config:
  10692. wled_manager.handle_offline(printer_id)
  10693. @asynccontextmanager
  10694. async def lifespan(app: FastAPI):
  10695. # Startup
  10696. # Install Windows-only asyncio Proactor cleanup-RST filter (#1113) before
  10697. # anything else can spawn tasks that might trip it.
  10698. from backend.app.core.asyncio_handlers import install_proactor_reset_filter, warn_if_running_on_uvloop
  10699. install_proactor_reset_filter()
  10700. # Before anything opens files or sockets in bulk: a soft limit of 1024 is
  10701. # what turned descriptor exhaustion into a corrupted database (#2883).
  10702. from backend.app.core.fd_limit import raise_open_file_limit
  10703. raise_open_file_limit()
  10704. # Before init_db, so the warning is near the top of the log rather than
  10705. # below a migration run. See warn_if_running_on_uvloop for what is at stake.
  10706. warn_if_running_on_uvloop()
  10707. await init_db()
  10708. # Browser download tokens expire after five minutes. Remove abandoned
  10709. # prepared ZIPs at startup as well as before each new preparation so a
  10710. # quiet appliance cannot retain an unusable bundle indefinitely.
  10711. try:
  10712. from backend.app.services.printer_media import prune_stale_printer_file_bundles
  10713. await prune_stale_printer_file_bundles()
  10714. except Exception as exc:
  10715. logging.warning("Failed to prune stale printer download bundles: %s", exc)
  10716. # After migrations, so the is_env_managed column exists. Never raises --
  10717. # a bad BAMBUDDY_OIDC_* value is logged and skipped rather than blocking
  10718. # startup (see apply_env_oidc_provider).
  10719. from backend.app.core.oidc_env import apply_env_oidc_provider
  10720. async with async_session() as oidc_db:
  10721. await apply_env_oidc_provider(oidc_db)
  10722. # Close out batches that finished before `completed` was a reachable status
  10723. # (#342). Without this the Batches tab opens on every batch created since
  10724. # the feature shipped, all still marked active. Never blocks startup.
  10725. try:
  10726. from backend.app.services.print_batch import backfill_batch_statuses
  10727. async with async_session() as batch_db:
  10728. await backfill_batch_statuses(batch_db)
  10729. except Exception as exc:
  10730. logging.warning("[BATCH] Startup status backfill failed: %s", exc)
  10731. # Each plate's estimate of archives made before they were kept (#3375),
  10732. # read from their 3MFs in the background: the readers use the whole
  10733. # file's figures until then.
  10734. global _plate_estimates_task
  10735. _plate_estimates_task = asyncio.create_task(_fill_plate_estimates())
  10736. # Register an app-scoped httpx client for Bambu Cloud services so
  10737. # per-request BambuCloudService instances reuse the same connection pool
  10738. # (important for routes like /cloud/filament-info that chain many
  10739. # get_setting_detail calls). The shared client stores no region/token
  10740. # state, so the per-request ownership pattern that fixed the region-bleed
  10741. # bug is preserved.
  10742. import httpx as _httpx
  10743. from backend.app.services.bambu_cloud import set_shared_http_client
  10744. from backend.app.services.model_providers.makerworld.service import (
  10745. set_shared_http_client as set_shared_makerworld_http_client,
  10746. )
  10747. from backend.app.services.orca_cloud import (
  10748. set_shared_http_client as set_shared_orca_http_client,
  10749. )
  10750. _shared_cloud_http_client = _httpx.AsyncClient(timeout=30.0)
  10751. set_shared_http_client(_shared_cloud_http_client)
  10752. # Reuse the same connection pool for MakerWorld — different host, same
  10753. # keep-alive pool saves a TLS handshake per request.
  10754. set_shared_makerworld_http_client(_shared_cloud_http_client)
  10755. # Same for Orca Cloud — without this the per-request OrcaCloudService()
  10756. # each spun up (and never closed) its own client, leaking sockets.
  10757. set_shared_orca_http_client(_shared_cloud_http_client)
  10758. # Fix queue items stuck with invalid "aborted" status (should be "cancelled").
  10759. # This can happen when a print was cancelled mid-print on versions before this fix.
  10760. try:
  10761. async with async_session() as db:
  10762. from backend.app.models.print_queue import PrintQueueItem
  10763. result = await db.execute(select(PrintQueueItem).where(PrintQueueItem.status == "aborted"))
  10764. aborted_items = result.scalars().all()
  10765. if aborted_items:
  10766. for item in aborted_items:
  10767. item.status = "cancelled"
  10768. await db.commit()
  10769. logging.info("Fixed %d queue item(s) with invalid 'aborted' status → 'cancelled'", len(aborted_items))
  10770. except Exception as e:
  10771. logging.warning("Failed to fix aborted queue items: %s", e)
  10772. # Restore debug logging state from previous session
  10773. await init_debug_logging()
  10774. # Set up printer manager callbacks
  10775. loop = asyncio.get_event_loop()
  10776. printer_manager.set_event_loop(loop)
  10777. printer_manager.set_status_change_callback(on_printer_status_change)
  10778. printer_manager.set_print_start_callback(on_print_start)
  10779. printer_manager.set_print_complete_callback(on_print_complete)
  10780. printer_manager.set_print_running_observed_callback(on_print_running_observed)
  10781. printer_manager.set_finish_photo_moment_callback(on_finish_photo_moment)
  10782. printer_manager.set_ams_change_callback(on_ams_change)
  10783. printer_manager.set_fts_inlet_change_callback(on_fts_inlet_change)
  10784. # Rehydrate persisted awaiting-plate-clear gate (#961) so prompts survive restarts
  10785. await printer_manager.load_awaiting_plate_clear_from_db()
  10786. # Before any printer connects, or its first report re-sends every fault
  10787. # still listed (#3486).
  10788. await _load_notified_hms_faults()
  10789. # WLED is entirely optional. Seed its in-memory per-printer configuration
  10790. # before MQTT connections begin emitting status callbacks.
  10791. async with async_session() as db:
  10792. await initialize_wled_printers(db)
  10793. # Layer change callback for external camera timelapse
  10794. async def on_layer_change(printer_id: int, layer_num: int):
  10795. """Capture timelapse frame on layer change + first layer notification."""
  10796. from backend.app.services.layer_timelapse import on_layer_change as tl_layer_change
  10797. await tl_layer_change(printer_id, layer_num)
  10798. # #1867: bank a recent in-print frame so the finish-photo path has a
  10799. # pre-End-G-code image to use instead of a live grab of a swapped plate.
  10800. # #2547 added `on_print_progress` as a second driver — this one alone
  10801. # stops firing once the final layer begins.
  10802. await _maybe_bank_inprint_frame(printer_id, layer_num)
  10803. # First layer complete notification (layer_num >= 2 means layer 1 is done).
  10804. # Gate on actual printing state — Bambu firmware ticks layer_num during
  10805. # the pre-print calibration sequence (homing / mesh-level / bed scan /
  10806. # nozzle clean), so a bare layer_num check can fire minutes before the
  10807. # first real extrusion. We require gcode_state == RUNNING and
  10808. # mc_print_sub_stage in (0 = "Printing", None) so calibration sub-stages
  10809. # (1, 9, 14, ...) are excluded. The window widens to [2, 10] because if
  10810. # the layer counter advanced past 2 during PREPARE, the next on_layer_change
  10811. # edge fires later; _first_layer_notified stays clear until we actually send
  10812. # so a deferred re-evaluation can win. See issue #1837.
  10813. if 2 <= layer_num <= 10 and not _first_layer_notified.get(printer_id, False):
  10814. client = printer_manager.get_client(printer_id)
  10815. state = client.state if client else None
  10816. if not state or state.state != "RUNNING":
  10817. return
  10818. if state.mc_print_sub_stage not in (None, 0):
  10819. return
  10820. _first_layer_notified[printer_id] = True
  10821. try:
  10822. async with async_session() as db:
  10823. from backend.app.models.printer import Printer
  10824. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  10825. printer = result.scalar_one_or_none()
  10826. if not printer:
  10827. return
  10828. printer_name = printer.name
  10829. filename = (state.subtask_name or state.gcode_file or "Unknown") if state else "Unknown"
  10830. total_layers = state.total_layers if state else 0
  10831. image_data = await _capture_snapshot_for_notification(
  10832. printer_id, printer, logging.getLogger(__name__)
  10833. )
  10834. await notification_service.on_first_layer_complete(
  10835. printer_id, printer_name, filename, total_layers, db, image_data=image_data
  10836. )
  10837. except Exception as e:
  10838. logging.getLogger(__name__).warning("First layer notification failed: %s", e)
  10839. printer_manager.set_layer_change_callback(on_layer_change)
  10840. async def on_print_progress(printer_id: int, percent: int):
  10841. """#2547: keep the in-print frame bank fresh through the final layer.
  10842. `on_layer_change` stops the moment the last layer starts, which on the
  10843. H2C capture that closed #2547 left the bank stale for the three minutes
  10844. that layer took. Progress is the only field that keeps advancing there,
  10845. and it freezes before the End G-code runs — so banking on it stays
  10846. inside the print and never sees a swapped plate.
  10847. """
  10848. client = printer_manager.get_client(printer_id)
  10849. state = client.state if client else None
  10850. await _maybe_bank_inprint_frame(printer_id, state.layer_num if state else 0)
  10851. printer_manager.set_print_progress_callback(on_print_progress)
  10852. # Event-driven bed cooldown: fires whenever bed_temper arrives via MQTT
  10853. async def on_bed_temp_update(printer_id: int, bed_temp: float):
  10854. waiter = _bed_cool_waiters.get(printer_id)
  10855. if not waiter:
  10856. return
  10857. threshold = waiter["threshold"]
  10858. if bed_temp > threshold:
  10859. return
  10860. # Bed is at or below threshold — fire notification and remove waiter
  10861. waiter_info = _bed_cool_waiters.pop(printer_id, None)
  10862. if not waiter_info:
  10863. return # Another callback already handled it
  10864. bed_cool_logger = logging.getLogger(__name__)
  10865. bed_cool_logger.info(
  10866. "[BED-COOL] Bed cooled to %.1f°C on printer %s (threshold: %.0f°C)",
  10867. bed_temp,
  10868. printer_id,
  10869. threshold,
  10870. )
  10871. try:
  10872. printer_info = printer_manager.get_printer(printer_id)
  10873. p_name = printer_info.name if printer_info else "Unknown"
  10874. async with async_session() as db:
  10875. await notification_service.on_bed_cooled(
  10876. printer_id=printer_id,
  10877. printer_name=p_name,
  10878. bed_temp=bed_temp,
  10879. threshold=threshold,
  10880. filename=waiter_info["filename"],
  10881. db=db,
  10882. )
  10883. except Exception as e:
  10884. bed_cool_logger.warning("[BED-COOL] Failed to send notification: %s", e)
  10885. printer_manager.set_bed_temp_update_callback(on_bed_temp_update)
  10886. async def on_drying_complete(printer_id: int, ams_id: int):
  10887. """Smart-plug auto-off-after-drying trigger (#1349).
  10888. Fires once per AMS unit when ``dry_time`` falls from >0 to 0. The
  10889. manager walks all plugs linked to this printer and turns off only
  10890. the ones with ``auto_off_after_drying`` enabled, after their
  10891. per-plug delay. Multiple AMS units finishing close together (e.g. a
  10892. dual-AMS dry that ends within the same MQTT push) call this once
  10893. per unit — the manager's ``_cancel_pending_off`` collapses
  10894. repeated scheduling on the same plug to one timer, so duplicate
  10895. fires are safe.
  10896. """
  10897. try:
  10898. async with async_session() as db:
  10899. await smart_plug_manager.on_drying_complete(printer_id, db)
  10900. except Exception as e:
  10901. logging.getLogger(__name__).warning(
  10902. "Failed to schedule auto-off-after-drying for printer %d (AMS %d): %s",
  10903. printer_id,
  10904. ams_id,
  10905. e,
  10906. )
  10907. printer_manager.set_drying_complete_callback(on_drying_complete)
  10908. async def on_drying_cycle_end(printer_id: int, cycle):
  10909. """Mark the spools in an AMS that just finished drying as dried (#2863)."""
  10910. from backend.app.services.spool_drying import record_drying_cycle
  10911. try:
  10912. async with async_session() as db:
  10913. await record_drying_cycle(db, printer_id, cycle)
  10914. except Exception as e:
  10915. logging.getLogger(__name__).warning(
  10916. "Failed to record drying for printer %d (AMS %d): %s",
  10917. printer_id,
  10918. cycle.ams_id,
  10919. e,
  10920. )
  10921. printer_manager.set_drying_cycle_end_callback(on_drying_cycle_end)
  10922. async def on_assignment_verified(printer_id: int, ams_id: int, tray_id: int, verified: bool, detail: dict):
  10923. """Surface the read-back result of a spool assignment to the UI (#2582).
  10924. The MQTT client confirms (or fails to confirm) that the tray telemetry
  10925. echoed back the filament id we pushed. We relay that as a websocket
  10926. event so the frontend can toast "loaded" / "assignment didn't take"
  10927. instead of the historic silent fire-and-forget, which made the
  10928. AMS→Studio hand-off feel random to users.
  10929. """
  10930. try:
  10931. from backend.app.services.spool_assignment_notifications import (
  10932. _slot_label_from_global_tray,
  10933. )
  10934. layout = printer_tray_layout(printer_id, printer_manager)
  10935. global_id = layout.tray_from_slot_key(ams_id, tray_id)
  10936. slot_label = _slot_label_from_global_tray(global_id, layout)
  10937. printer_info = printer_manager.get_printer(printer_id)
  10938. printer_name = printer_info.name if printer_info else f"Printer {printer_id}"
  10939. await ws_manager.broadcast(
  10940. {
  10941. "type": "spool_assignment_verified",
  10942. "printer_id": printer_id,
  10943. "printer_name": printer_name,
  10944. "ams_id": ams_id,
  10945. "tray_id": tray_id,
  10946. "slot": slot_label,
  10947. "verified": verified,
  10948. # Present on success: False means the filament setting landed
  10949. # but the K-profile (cali_idx) did not — the reporter's exact
  10950. # "loaded but no flow profile" symptom.
  10951. "kprofile_applied": detail.get("kprofile_applied", True),
  10952. }
  10953. )
  10954. except Exception as e:
  10955. logging.getLogger(__name__).warning(
  10956. "Failed to broadcast assignment verification for printer %d AMS%d-T%d: %s",
  10957. printer_id,
  10958. ams_id,
  10959. tray_id,
  10960. e,
  10961. )
  10962. printer_manager.set_assignment_verified_callback(on_assignment_verified)
  10963. async def on_tray_change(printer_id: int, tray_global: int, layer_num: int):
  10964. """Persist a mid-print tray change for completion-time attribution.
  10965. AMS filament backup switches trays without telling the slicer, so the
  10966. tray-change log is the only record of which spool fed which layers.
  10967. Keeping it only in memory meant a restart mid-print charged everything
  10968. to the tray that finished the job.
  10969. """
  10970. try:
  10971. from backend.app.services.usage_tracker import record_tray_change
  10972. async with async_session() as db:
  10973. await record_tray_change(db, printer_id, tray_global, layer_num)
  10974. except Exception as e:
  10975. logging.getLogger(__name__).warning(
  10976. "Failed to persist tray change for printer %d (tray=%d, layer=%d): %s",
  10977. printer_id,
  10978. tray_global,
  10979. layer_num,
  10980. e,
  10981. )
  10982. printer_manager.set_tray_change_callback(on_tray_change)
  10983. # Initialize MQTT relay from settings
  10984. async with async_session() as db:
  10985. from backend.app.api.routes.settings import get_setting
  10986. mqtt_settings = {
  10987. "mqtt_enabled": (await get_setting(db, "mqtt_enabled") or "false") == "true",
  10988. "mqtt_broker": await get_setting(db, "mqtt_broker") or "",
  10989. "mqtt_port": int(await get_setting(db, "mqtt_port") or "1883"),
  10990. "mqtt_username": await get_setting(db, "mqtt_username") or "",
  10991. "mqtt_password": await get_setting(db, "mqtt_password") or "",
  10992. "mqtt_topic_prefix": await get_setting(db, "mqtt_topic_prefix") or "bambuddy",
  10993. "mqtt_use_tls": (await get_setting(db, "mqtt_use_tls") or "false") == "true",
  10994. }
  10995. await mqtt_relay.configure(mqtt_settings)
  10996. # Restore MQTT smart plug subscriptions
  10997. if mqtt_settings.get("mqtt_enabled"):
  10998. from backend.app.models.smart_plug import SmartPlug
  10999. from backend.app.services.mqtt_smart_plug import subscribe_plug_to_mqtt
  11000. result = await db.execute(select(SmartPlug).where(SmartPlug.plug_type == "mqtt"))
  11001. mqtt_plugs = result.scalars().all()
  11002. restored = 0
  11003. for plug in mqtt_plugs:
  11004. if subscribe_plug_to_mqtt(mqtt_relay.smart_plug_service, plug):
  11005. restored += 1
  11006. if restored:
  11007. logging.info("Restored %s MQTT smart plug subscriptions", restored)
  11008. # Connect to all active printers
  11009. async with async_session() as db:
  11010. await init_printer_connections(db)
  11011. # Auto-connect to Spoolman if enabled
  11012. async with async_session() as db:
  11013. from backend.app.api.routes.settings import get_setting
  11014. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  11015. spoolman_url = await get_setting(db, "spoolman_url")
  11016. if spoolman_enabled and spoolman_enabled.lower() == "true" and spoolman_url:
  11017. try:
  11018. client = await init_spoolman_client(spoolman_url)
  11019. if await client.health_check():
  11020. logging.info("Auto-connected to Spoolman at %s", spoolman_url)
  11021. # Ensure the 'tag' extra field exists for RFID/UUID storage
  11022. field_ok = await client.ensure_tag_extra_field()
  11023. if not field_ok:
  11024. logging.error("Spoolman tag extra field registration failed — NFC tag links may not persist")
  11025. # Register the BambuStudio slicer-preset fields used by the
  11026. # spool-edit / assign flow. Spoolman rejects PATCHes with
  11027. # unknown extra keys, so these must exist before any update
  11028. # that touches them.
  11029. for field_name in ("bambu_slicer_filament", "bambu_slicer_filament_name"):
  11030. if not await client.ensure_extra_field(field_name):
  11031. logging.warning(
  11032. "Spoolman extra field %r registration failed — "
  11033. "spool slicer-preset edits will return 502",
  11034. field_name,
  11035. )
  11036. else:
  11037. logging.warning("Spoolman at %s is not reachable", spoolman_url)
  11038. except Exception as e:
  11039. logging.warning("Failed to auto-connect to Spoolman: %s", e)
  11040. # Start the print scheduler
  11041. spawn_background_task(print_scheduler.run(), name="print-scheduler")
  11042. # Start the smart plug scheduler for time-based on/off
  11043. smart_plug_manager.start_scheduler()
  11044. # Start the Home Assistant sensor poller (#1148)
  11045. ha_sensor_manager.start()
  11046. location_ha_sensor_manager.start()
  11047. # Resume any pending auto-offs that were interrupted by restart
  11048. await smart_plug_manager.resume_pending_auto_offs()
  11049. # Start the notification digest scheduler
  11050. notification_service.start_digest_scheduler()
  11051. notify_live_activities.start()
  11052. notify_widgets.start()
  11053. # Start the Telegram reaction pollers (#3046), one per bot token used by a
  11054. # provider in reactions/both mode; the notification routes resync them.
  11055. # Never fatal: a bad provider row or a DB hiccup here costs reactions
  11056. # until the next provider save, not the whole startup.
  11057. try:
  11058. await telegram_reaction_poller.start()
  11059. except Exception as e:
  11060. logging.warning("Telegram reaction poller did not start: %s", e)
  11061. # Start the GitHub backup scheduler
  11062. await github_backup_service.start_scheduler()
  11063. # Start the local backup scheduler
  11064. await local_backup_service.start_scheduler()
  11065. await obico_detection_service.start()
  11066. await octoeverywhere_detection_service.start()
  11067. # Start the library trash sweeper (#1008)
  11068. await library_trash_service.start_scheduler()
  11069. # Start the archive auto-purge sweeper (#1008 follow-up)
  11070. await archive_purge_service.start_scheduler()
  11071. # Start AMS history recording
  11072. start_ams_history_recording()
  11073. # Start printer sensor (nozzle / bed / chamber) history recording
  11074. start_printer_sensor_history_recording()
  11075. # Start printer runtime tracking
  11076. start_runtime_tracking()
  11077. # Start SpoolBuddy device watchdog
  11078. start_spoolbuddy_watchdog()
  11079. # Start camera stream orphan cleanup
  11080. start_camera_cleanup()
  11081. # Start the backstop for MQTT sessions paho has stopped recovering (#2732)
  11082. start_connection_watchdog()
  11083. # One-shot sweep for timelapse session directories orphaned by a crash
  11084. # or restart that happened mid-print (in-memory session tracking can't
  11085. # survive that, and nothing else reaps the leftover frames/output file)
  11086. try:
  11087. from backend.app.services.layer_timelapse import cleanup_orphaned_timelapse_sessions
  11088. removed = cleanup_orphaned_timelapse_sessions()
  11089. if removed:
  11090. logging.getLogger(__name__).info("Removed %d orphaned timelapse session artifact(s)", removed)
  11091. except Exception as e:
  11092. logging.getLogger(__name__).warning("Orphaned timelapse session cleanup failed: %s", e)
  11093. # Start expected-print TTL eviction (prevents memory leak when prints are
  11094. # registered but on_print_start never fires)
  11095. start_expected_prints_cleanup()
  11096. # L-2: Start periodic auth cleanup (stale TOTP + expired revoked JTIs)
  11097. start_auth_cleanup()
  11098. # Maintainer announcements: a signed feed fetched from GitHub every few hours.
  11099. from backend.app.services import announcements as announcements_service
  11100. announcements_service.start()
  11101. from backend.app.services.printer_media import start_printer_download_cleanup
  11102. start_printer_download_cleanup()
  11103. # Event-loop stall watchdog: dumps all thread stacks to stderr if the loop
  11104. # freezes (#1486 — silent "container hangs after adding a printer" reports).
  11105. from backend.app.services.loop_watchdog import start_loop_watchdog
  11106. start_loop_watchdog()
  11107. # Initialize virtual printer manager and sync from DB
  11108. from backend.app.services.virtual_printer import virtual_printer_manager
  11109. virtual_printer_manager.set_session_factory(async_session)
  11110. virtual_printer_manager.set_printer_manager(printer_manager)
  11111. try:
  11112. await virtual_printer_manager.sync_from_db()
  11113. logging.info("Virtual printer manager synced from database")
  11114. except Exception as e:
  11115. logging.warning("Failed to sync virtual printers: %s", e)
  11116. yield
  11117. # Shutdown
  11118. print_scheduler.stop()
  11119. smart_plug_manager.stop_scheduler()
  11120. ha_sensor_manager.stop()
  11121. location_ha_sensor_manager.stop()
  11122. notification_service.stop_digest_scheduler()
  11123. await notify_live_activities.close()
  11124. await notify_widgets.close()
  11125. await telegram_reaction_poller.aclose()
  11126. github_backup_service.stop_scheduler()
  11127. local_backup_service.stop_scheduler()
  11128. library_trash_service.stop_scheduler()
  11129. archive_purge_service.stop_scheduler()
  11130. obico_detection_service.stop()
  11131. octoeverywhere_detection_service.stop()
  11132. stop_ams_history_recording()
  11133. stop_printer_sensor_history_recording()
  11134. stop_runtime_tracking()
  11135. stop_spoolbuddy_watchdog()
  11136. stop_camera_cleanup()
  11137. stop_connection_watchdog()
  11138. from backend.app.services.loop_watchdog import stop_loop_watchdog
  11139. stop_loop_watchdog()
  11140. # Tear down all camera fan-out broadcasters (#1089) so subscribers exit
  11141. # cleanly rather than waiting on a queue that nothing will ever fill.
  11142. try:
  11143. from backend.app.services.camera_fanout import shutdown_all_broadcasters
  11144. await shutdown_all_broadcasters()
  11145. except Exception as e:
  11146. logging.warning("Failed to shut down camera broadcasters: %s", e)
  11147. stop_expected_prints_cleanup()
  11148. stop_auth_cleanup()
  11149. if _plate_estimates_task is not None and not _plate_estimates_task.done():
  11150. _plate_estimates_task.cancel()
  11151. from backend.app.services import announcements as announcements_service
  11152. announcements_service.stop()
  11153. from backend.app.services.printer_media import stop_printer_download_cleanup
  11154. await stop_printer_download_cleanup()
  11155. # Stops a timelapse editor's ffmpeg rather than leaving it to the OS.
  11156. from backend.app.services import timelapse_jobs
  11157. await timelapse_jobs.cancel_all()
  11158. printer_manager.disconnect_all()
  11159. slot_unlink_grace.reset()
  11160. await close_spoolman_client()
  11161. # Stop all virtual printer services
  11162. await virtual_printer_manager.stop_all()
  11163. await mqtt_smart_plug_service.disconnect(timeout=2)
  11164. await mqtt_relay.disconnect(timeout=2)
  11165. await wled_manager.shutdown()
  11166. # Drop the shared Bambu Cloud HTTP client we registered at startup.
  11167. set_shared_http_client(None)
  11168. set_shared_makerworld_http_client(None)
  11169. set_shared_orca_http_client(None)
  11170. await _shared_cloud_http_client.aclose()
  11171. # Checkpoint WAL (SQLite only) and close all database connections
  11172. from backend.app.core.db_dialect import is_sqlite
  11173. if is_sqlite():
  11174. try:
  11175. async with engine.begin() as conn:
  11176. await conn.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
  11177. logging.info("WAL checkpoint completed")
  11178. except Exception as e:
  11179. logging.warning("WAL checkpoint failed: %s", e)
  11180. await engine.dispose()
  11181. app = FastAPI(
  11182. title=app_settings.app_name,
  11183. description="Archive and manage Bambu Lab 3MF files",
  11184. version=APP_VERSION,
  11185. lifespan=lifespan,
  11186. )
  11187. # Added first so it sits innermost, next to the routes: a refused header's value
  11188. # (often a stored token) masked in every JSON body. See core.response_redaction.
  11189. app.add_middleware(RefusedHeaderResponseMask)
  11190. def _json_safe(value):
  11191. """The validation errors with every non-finite number written as text."""
  11192. if isinstance(value, float) and not math.isfinite(value):
  11193. return str(value)
  11194. if isinstance(value, dict):
  11195. return {key: _json_safe(item) for key, item in value.items()}
  11196. if isinstance(value, list | tuple):
  11197. return [_json_safe(item) for item in value]
  11198. return value
  11199. @app.exception_handler(RequestValidationError)
  11200. async def _validation_error(_request, exc: RequestValidationError) -> JSONResponse:
  11201. """FastAPI's own 422, except that it survives a non-finite number.
  11202. A body may carry ``Infinity`` or ``NaN`` (Python's JSON reader accepts
  11203. them), and the default handler echoes the input back in its errors,
  11204. which JSON cannot hold: the request ended in an error instead of a 422.
  11205. """
  11206. return JSONResponse(status_code=422, content={"detail": _json_safe(jsonable_encoder(exc.errors()))})
  11207. # =============================================================================
  11208. # Authentication Middleware - Secures ALL API routes by default
  11209. # =============================================================================
  11210. # Public routes that don't require authentication even when auth is enabled
  11211. PUBLIC_API_ROUTES = {
  11212. # Auth routes needed before/during login
  11213. "/api/v1/auth/status",
  11214. "/api/v1/auth/login",
  11215. "/api/v1/auth/setup", # Needed for initial setup and recovery
  11216. # Advanced auth status needed for login page
  11217. "/api/v1/auth/advanced-auth/status",
  11218. "/api/v1/auth/forgot-password", # Password reset for advanced auth
  11219. "/api/v1/auth/forgot-password/confirm", # Complete password reset with token (H-6)
  11220. # 2FA routes that are called BEFORE a JWT is issued (pre-auth flow)
  11221. "/api/v1/auth/2fa/verify", # Exchange pre_auth_token + 2FA code for JWT
  11222. "/api/v1/auth/2fa/email/send", # Send OTP email (pre_auth_token based)
  11223. # OIDC routes that must be reachable without a JWT
  11224. "/api/v1/auth/oidc/providers", # Public list of enabled providers
  11225. "/api/v1/auth/oidc/callback", # Redirect target from OIDC provider
  11226. "/api/v1/auth/oidc/exchange", # Exchange short-lived OIDC token for JWT
  11227. # Connected apps: the app's server swaps a code for the user's identity,
  11228. # authenticated by its client secret rather than a login token.
  11229. "/api/v1/connect/token",
  11230. # Version check for updates (no sensitive data)
  11231. "/api/v1/updates/version",
  11232. # Metrics endpoint handles its own prometheus_token authentication
  11233. "/api/v1/metrics",
  11234. # Appliance bootstrap (#1589 follow-up): the SPA's i18n setup polls
  11235. # this BEFORE a JWT is available to pick up the firstboot wizard's
  11236. # hostname / timezone / locale and the chrony NTP-gate state. The
  11237. # response contains user-set defaults and a public sync flag — no
  11238. # secrets. Without this entry the global auth middleware returns 401
  11239. # before the route handler runs, regardless of the route's own
  11240. # "no auth required" intent.
  11241. "/api/v1/system/appliance",
  11242. # Cam Wall kiosk feed (#2531): a TV or Pi in kiosk mode has no login, so it
  11243. # authenticates with a long-lived ``camwall``-scoped token in the query
  11244. # string — exactly like the camera streams two lists below, and for the same
  11245. # reason (no header to put a JWT in). "Public" here only means the middleware
  11246. # steps aside; the route still runs RequireCamWallTokenIfAuthEnabled, which
  11247. # rejects an absent, expired, revoked, or wrong-scoped token. In particular a
  11248. # plain ``camera_stream`` token does NOT open this door.
  11249. "/api/v1/camwall/printers",
  11250. # Overlay branding: the route enforces overlay-scoped token authentication.
  11251. "/api/v1/overlay-branding/logo",
  11252. }
  11253. # Route prefixes that are public (for routes with dynamic segments)
  11254. PUBLIC_API_PREFIXES = [
  11255. # WebSocket connections handle their own auth
  11256. "/api/v1/ws",
  11257. # OIDC authorize redirects — include provider_id in path
  11258. "/api/v1/auth/oidc/authorize/",
  11259. # One-tap outcome-verdict links from push notifications (#1898). Tapped on
  11260. # a phone with no session, so no header can carry a JWT — the single-use
  11261. # capability token in the path IS the credential (same reasoning as the
  11262. # /dl/ slicer downloads below). The route grants nothing beyond writing
  11263. # good/reject on the one archive the token was minted for.
  11264. "/api/v1/archives/confirm/",
  11265. ]
  11266. # Route patterns that are public (read-only display data)
  11267. # These are checked with "in path" - needed because browsers load images/videos
  11268. # via <img src> and <video src> which don't include Authorization headers
  11269. PUBLIC_API_PATTERNS = [
  11270. # Thumbnails
  11271. "/thumbnail", # /archives/{id}/thumbnail, /library/files/{id}/thumbnail
  11272. "/plate-thumbnail/", # /archives/{id}/plate-thumbnail/{plate_id}
  11273. # Images and media
  11274. "/photos/", # /archives/{id}/photos/{filename}
  11275. "/project-image/", # /archives/{id}/project-image/{path}
  11276. "/qrcode", # /archives/{id}/qrcode
  11277. "/timelapse", # /archives/{id}/timelapse (video)
  11278. "/cover", # /printers/{id}/cover
  11279. "/icon", # /external-links/{id}/icon
  11280. # Camera (streams loaded via <img> tag)
  11281. "/camera/stream", # /printers/{id}/camera/stream
  11282. "/camera/snapshot", # /printers/{id}/camera/snapshot
  11283. # Streaming-overlay status feed (#2613): OBS loads /overlay/{id} with no login
  11284. # and this backs it, authenticated by an ``overlay``-scoped token in the query
  11285. # string (same reasoning as the camera streams above — no header to carry a
  11286. # JWT). "Public" only means the middleware steps aside; the route still runs
  11287. # RequireOverlayTokenIfAuthEnabled, which rejects an absent, expired, revoked,
  11288. # or wrong-scoped token — a camwall or camera_stream token does NOT open it.
  11289. "/overlay-status", # /printers/{id}/overlay-status
  11290. # Slicer token-authenticated downloads — protocol handlers (bambustudioopen://,
  11291. # orcaslicer://) cannot send auth headers. These endpoints validate a short-lived
  11292. # download token in the URL path instead.
  11293. "/dl/", # /archives/{id}/dl/{token}/{filename}, /library/files/{id}/dl/{token}/{filename}
  11294. # Same family, but the segment is "source-dl" — which does NOT contain "/dl/",
  11295. # and these patterns match by substring. Without its own entry the middleware
  11296. # 401s the slicer's header-less request before the route's token check runs,
  11297. # so "Open source 3MF in slicer" failed whenever auth was enabled (#3029).
  11298. "/source-dl/", # /archives/{id}/source-dl/{token}/{filename}
  11299. # Obico ML API fetches JPEG frames by one-shot nonce (issue #172 follow-up).
  11300. # The nonce itself is the credential: 32-byte random, single-use, ~30s TTL.
  11301. "/obico/cached-frame/", # /obico/cached-frame/{nonce}
  11302. ]
  11303. _security_headers_logger = logging.getLogger("backend.app.main.security_headers")
  11304. def _parse_trusted_frame_origins() -> tuple[str, ...]:
  11305. """Parse TRUSTED_FRAME_ORIGINS env var into a validated allowlist (#1191).
  11306. Format: comma-separated list of ``scheme://host[:port]`` origins.
  11307. Used by ``security_headers_middleware`` to relax ``frame-ancestors`` for
  11308. trusted same-LAN deployments (e.g. Home Assistant Webpage panel embedding
  11309. Bambuddy from a different port). Defaults to empty — strict ``'none'``.
  11310. Invalid entries are dropped with a warning rather than failing startup, so
  11311. a typo in one origin doesn't take the whole deployment down.
  11312. """
  11313. raw = os.environ.get("TRUSTED_FRAME_ORIGINS", "").strip()
  11314. if not raw:
  11315. return ()
  11316. valid: list[str] = []
  11317. for item in raw.split(","):
  11318. candidate = item.strip()
  11319. if not candidate:
  11320. continue
  11321. try:
  11322. parsed = urlparse(candidate)
  11323. except ValueError as e:
  11324. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — %s", candidate, e)
  11325. continue
  11326. if parsed.scheme not in ("http", "https"):
  11327. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — must be http(s)", candidate)
  11328. continue
  11329. if not parsed.netloc:
  11330. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — missing host", candidate)
  11331. continue
  11332. if parsed.path and parsed.path != "/":
  11333. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — paths not allowed", candidate)
  11334. continue
  11335. if parsed.query or parsed.fragment:
  11336. _security_headers_logger.warning(
  11337. "TRUSTED_FRAME_ORIGINS: dropping %r — query/fragment not allowed", candidate
  11338. )
  11339. continue
  11340. if "*" in parsed.netloc:
  11341. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — wildcards not allowed", candidate)
  11342. continue
  11343. valid.append(f"{parsed.scheme}://{parsed.netloc}")
  11344. if valid:
  11345. _security_headers_logger.info("TRUSTED_FRAME_ORIGINS: %s", ", ".join(valid))
  11346. return tuple(valid)
  11347. _TRUSTED_FRAME_ORIGINS: tuple[str, ...] = _parse_trusted_frame_origins()
  11348. def _frame_ancestors(default_value: str) -> str:
  11349. """Compose the ``frame-ancestors`` CSP directive (#1191).
  11350. ``default_value`` is the strict directive used when the operator has not
  11351. configured ``TRUSTED_FRAME_ORIGINS`` — typically ``'none'`` (catch-all and
  11352. docs) or ``'self'`` (the streaming overlay, embedded same-origin by the
  11353. Settings URL builder's preview). When trusted origins
  11354. are configured, ``'self'`` is always included so same-origin embedding never
  11355. breaks even if an operator forgets to add their own origin to the list.
  11356. """
  11357. if _TRUSTED_FRAME_ORIGINS:
  11358. return "frame-ancestors 'self' " + " ".join(_TRUSTED_FRAME_ORIGINS) + ";"
  11359. return f"frame-ancestors {default_value};"
  11360. # The two Vite-emitted worker assets that compile WebAssembly (#2976). Both
  11361. # patterns are anchored on the exact emitted name so the relaxed policies
  11362. # below can never apply to any other asset.
  11363. # src/workers/stepPreview.worker.ts -> /assets/stepPreview.worker-<hash>.js
  11364. # pdfjs-dist/legacy/build/pdf.worker.min.mjs?worker&url
  11365. # -> /assets/pdf.worker.min-<hash>.js
  11366. # Vite also emits a one-line chunk under the second name that only exports the
  11367. # worker's URL. The page imports it as a module, and a module script is run
  11368. # under the importing document's policy, never its own response's, so it
  11369. # matching as well changes nothing.
  11370. _STEP_WORKER_ASSET_RE = re.compile(r"^/assets/stepPreview\.worker-[\w-]+\.js$")
  11371. _PDF_WORKER_ASSET_RE = re.compile(r"^/assets/pdf\.worker\.min-[\w-]+\.js$")
  11372. @app.middleware("http")
  11373. async def security_headers_middleware(request, call_next):
  11374. """Add standard HTTP security headers to every response."""
  11375. # Per-request nonce stamped into `script-src` (#1460). On its own this
  11376. # changes nothing for Bambuddy's own pages — index.html has no inline
  11377. # scripts since the SW registration moved to /sw-register.js. The reason
  11378. # it's here is Cloudflare: a CF-fronted deployment has the bot-detection
  11379. # script injected into the HTML on the edge, with a fresh hash on every
  11380. # load (so hashes can't be allowlisted). When CF sees a nonce in our CSP,
  11381. # it clones the same nonce onto its injected <script>, and the inline
  11382. # script passes the policy without us needing 'unsafe-inline'. See
  11383. # https://developers.cloudflare.com/cloudflare-challenges/challenge-types/javascript-detections/#if-you-have-a-content-security-policy-csp
  11384. csp_nonce = secrets.token_urlsafe(16)
  11385. # Routes that render their own HTML need it too, or their inline script is
  11386. # blocked by the policy below. The outcome-confirmation page (#1898) is the
  11387. # one that does: it submits its own form so a verdict still costs one tap.
  11388. request.state.csp_nonce = csp_nonce
  11389. response = await call_next(request)
  11390. response.headers["X-Content-Type-Options"] = "nosniff"
  11391. # X-Frame-Options is the legacy cross-origin embedding control. Modern
  11392. # browsers honour CSP frame-ancestors instead, and the legacy
  11393. # `ALLOW-FROM <url>` syntax is deprecated and inconsistent across vendors.
  11394. # When operators have explicitly allowlisted trusted frame origins (#1191
  11395. # — typically Home Assistant on a different port), drop X-Frame-Options
  11396. # and let the CSP-side frame-ancestors directive govern embedding.
  11397. if not _TRUSTED_FRAME_ORIGINS:
  11398. response.headers["X-Frame-Options"] = "SAMEORIGIN"
  11399. response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
  11400. # Content-Security-Policy for the React SPA.
  11401. # Notes:
  11402. # - 'unsafe-inline' for style-src: React and UI libs inject inline styles at runtime.
  11403. # - connect-src ws:/wss:: MQTT/printer WebSocket connections.
  11404. # - img-src data: / blob:: base64 thumbnails and Blob-URL timelapse previews.
  11405. # - media-src blob:: timelapse video player uses Blob URLs.
  11406. # - font-src data:: some icon fonts are embedded as data URIs.
  11407. if request.url.path in ("/docs", "/redoc", "/docs/oauth2-redirect"):
  11408. # FastAPI's built-in Swagger UI / ReDoc pages load assets from
  11409. # cdn.jsdelivr.net and bootstrap with an inline <script>, so the
  11410. # default CSP would render a blank page.
  11411. response.headers["Content-Security-Policy"] = (
  11412. "default-src 'self'; "
  11413. "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
  11414. "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com; "
  11415. "img-src 'self' data: blob: https://fastapi.tiangolo.com https://cdn.redoc.ly; "
  11416. "connect-src 'self'; "
  11417. "font-src 'self' data: https://fonts.gstatic.com; "
  11418. "worker-src 'self' blob:; "
  11419. "object-src 'none'; "
  11420. "base-uri 'self'; " + _frame_ancestors("'none'")
  11421. )
  11422. elif _STEP_WORKER_ASSET_RE.match(request.url.path):
  11423. # The STEP preview worker (#2976) runs OpenCascade compiled to WASM;
  11424. # its emscripten/embind glue generates invoker functions with `new
  11425. # Function(...)`, which needs 'unsafe-eval'. Per CSP3 a dedicated
  11426. # worker is governed by the policy delivered with the WORKER SCRIPT's
  11427. # own response — not the document's — so relaxing it here confines
  11428. # eval to that DOM-less worker context. The document policy below
  11429. # stays nonce-strict, and this response header has no effect when the
  11430. # file is merely fetched (a fetch's CSP is enforced against the
  11431. # requesting document, not the resource's own headers).
  11432. response.headers["Content-Security-Policy"] = (
  11433. "default-src 'self'; "
  11434. "script-src 'self' 'wasm-unsafe-eval' 'unsafe-eval'; "
  11435. "connect-src 'self'; "
  11436. "object-src 'none'; "
  11437. "base-uri 'self'; " + _frame_ancestors("'none'")
  11438. )
  11439. elif _PDF_WORKER_ASSET_RE.match(request.url.path):
  11440. # pdf.js decodes JPEG2000/JBIG2 images and ICC colour with WebAssembly
  11441. # and fetches those modules from /assets/pdfjs/wasm/ (#2976). Same CSP3
  11442. # rule as the STEP worker above: the policy that governs a dedicated
  11443. # worker is the one delivered with its own script, so the wasm compile
  11444. # has to be permitted here rather than on the document. Unlike the STEP
  11445. # worker this one needs no JS eval, so it gets 'wasm-unsafe-eval' only.
  11446. response.headers["Content-Security-Policy"] = (
  11447. "default-src 'self'; "
  11448. "script-src 'self' 'wasm-unsafe-eval'; "
  11449. "connect-src 'self'; "
  11450. "object-src 'none'; "
  11451. "base-uri 'self'; " + _frame_ancestors("'none'")
  11452. )
  11453. else:
  11454. # The streaming overlay is embedded same-origin by the URL builder's
  11455. # preview in Settings (#1422), so this branch allows 'self'.
  11456. # Embedding from anywhere else is still refused: 'self'
  11457. # only permits a framer on this origin, which is Bambuddy's own UI, so
  11458. # a clickjacking page on another host is blocked exactly as before.
  11459. # (The overlay draws status over a camera feed and its only interactive
  11460. # element is the logo link, so there is nothing to bait a click into
  11461. # even from a same-origin framer.) Cross-origin embedding of the
  11462. # overlay — Home Assistant on another port — remains what
  11463. # TRUSTED_FRAME_ORIGINS is for, and _frame_ancestors already folds that
  11464. # allowlist in.
  11465. #
  11466. # The connected-app consent page (/connect/authorize) gets the same
  11467. # 'self': an app opened from Bambuddy's sidebar runs in an iframe, and
  11468. # its "Sign in with Bambuddy" navigates that iframe to this page. With
  11469. # 'none' the browser refuses to show it even inside Bambuddy. 'self'
  11470. # requires every ancestor to be this origin, so a foreign page -- a
  11471. # sidebar link's site included -- still cannot frame the consent
  11472. # screen to bait a click.
  11473. embeddable_same_origin = request.url.path.startswith("/overlay/") or request.url.path == "/connect/authorize"
  11474. # No 'wasm-unsafe-eval' here: nothing compiles WebAssembly on the main
  11475. # thread. Both wasm consumers — the STEP preview and pdf.js's image
  11476. # decoders (#2976) — run in dedicated workers, which CSP3 governs by
  11477. # the policy served with their own script, so each gets it in its own
  11478. # branch above and the document policy stays as strict as it was.
  11479. response.headers["Content-Security-Policy"] = (
  11480. "default-src 'self'; "
  11481. f"script-src 'self' 'nonce-{csp_nonce}'; "
  11482. "style-src 'self' 'unsafe-inline'; "
  11483. "img-src 'self' data: blob:; "
  11484. "media-src 'self' blob:; "
  11485. "connect-src 'self' ws: wss:; "
  11486. "font-src 'self' data:; "
  11487. "object-src 'none'; "
  11488. "base-uri 'self'; "
  11489. "frame-src 'self' http: https:; " + _frame_ancestors("'self'" if embeddable_same_origin else "'none'")
  11490. )
  11491. if request.url.scheme == "https":
  11492. response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
  11493. return response
  11494. @app.middleware("http")
  11495. async def auth_middleware(request, call_next):
  11496. """Enforce authentication on all API routes when auth is enabled.
  11497. This middleware provides defense-in-depth by checking auth at the API gateway level,
  11498. regardless of whether individual routes have auth dependencies.
  11499. """
  11500. from starlette.responses import JSONResponse
  11501. path = request.url.path
  11502. # Only apply to API routes
  11503. if not path.startswith("/api/"):
  11504. return await call_next(request)
  11505. # Allow public routes
  11506. if path in PUBLIC_API_ROUTES:
  11507. return await call_next(request)
  11508. # Allow public prefixes
  11509. for prefix in PUBLIC_API_PREFIXES:
  11510. if path.startswith(prefix):
  11511. return await call_next(request)
  11512. # Allow public patterns (read-only display data like thumbnails)
  11513. for pattern in PUBLIC_API_PATTERNS:
  11514. if pattern in path:
  11515. return await call_next(request)
  11516. # Check if auth is enabled. Fail CLOSED on any exception during the
  11517. # probe — GHSA-6mf4-q26m-47pv: the previous fail-open path here let
  11518. # an attacker who could force a DB exception (e.g. file-descriptor
  11519. # exhaustion via login flood) bypass auth on every protected endpoint.
  11520. # Only the probe is guarded: the request itself runs outside the try, so an
  11521. # error in a route surfaces as the 500 it is rather than as an auth outage.
  11522. try:
  11523. async with async_session() as db:
  11524. from backend.app.core.auth import is_auth_enabled
  11525. auth_enabled = await is_auth_enabled(db)
  11526. except Exception:
  11527. logging.getLogger(__name__).exception("auth_middleware: failing closed on auth-probe error from %s", path)
  11528. return JSONResponse(
  11529. status_code=503,
  11530. content={"detail": "Authentication service temporarily unavailable"},
  11531. )
  11532. if not auth_enabled:
  11533. # Auth disabled, allow all requests
  11534. return await call_next(request)
  11535. # Auth is enabled - require valid token
  11536. auth_header = request.headers.get("Authorization")
  11537. x_api_key = request.headers.get("X-API-Key")
  11538. # Check for API key auth first
  11539. if x_api_key or (auth_header and auth_header.startswith("Bearer bb_")):
  11540. # API key authentication - let the request through to be validated by route handler
  11541. # API keys are validated per-route since they have different permission levels
  11542. return await call_next(request)
  11543. # Check for JWT auth
  11544. if not auth_header or not auth_header.startswith("Bearer "):
  11545. return JSONResponse(
  11546. status_code=401,
  11547. content={"detail": "Authentication required"},
  11548. headers={"WWW-Authenticate": "Bearer"},
  11549. )
  11550. # Validate JWT token
  11551. import jwt
  11552. try:
  11553. from backend.app.core.auth import (
  11554. ALGORITHM,
  11555. SECRET_KEY,
  11556. _is_token_fresh,
  11557. get_user_by_username,
  11558. is_jti_revoked,
  11559. )
  11560. token = auth_header.replace("Bearer ", "")
  11561. payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
  11562. username = payload.get("sub")
  11563. if not username:
  11564. raise ValueError("No username in token")
  11565. jti = payload.get("jti")
  11566. if not jti:
  11567. raise ValueError("No jti in token")
  11568. iat = payload.get("iat")
  11569. # Verify user exists, is active, and token is still fresh (L-R8-A).
  11570. # Reject revoked tokens first (defense-in-depth gateway check), reusing
  11571. # this session so the gateway adds a single pooled checkout, not two (#2572).
  11572. async with async_session() as db:
  11573. if await is_jti_revoked(jti, db):
  11574. return JSONResponse(
  11575. status_code=401,
  11576. content={"detail": "Token has been revoked"},
  11577. headers={"WWW-Authenticate": "Bearer"},
  11578. )
  11579. user = await get_user_by_username(db, username)
  11580. if not user or not user.is_active:
  11581. return JSONResponse(
  11582. status_code=401,
  11583. content={"detail": "User not found or inactive"},
  11584. headers={"WWW-Authenticate": "Bearer"},
  11585. )
  11586. if not _is_token_fresh(iat, user):
  11587. return JSONResponse(
  11588. status_code=401,
  11589. content={"detail": "Token no longer valid"},
  11590. headers={"WWW-Authenticate": "Bearer"},
  11591. )
  11592. except jwt.ExpiredSignatureError:
  11593. return JSONResponse(
  11594. status_code=401,
  11595. content={"detail": "Token has expired"},
  11596. headers={"WWW-Authenticate": "Bearer"},
  11597. )
  11598. except (jwt.InvalidTokenError, ValueError):
  11599. return JSONResponse(
  11600. status_code=401,
  11601. content={"detail": "Invalid token"},
  11602. headers={"WWW-Authenticate": "Bearer"},
  11603. )
  11604. except Exception:
  11605. # Not a bad token but a failure checking it (database, pool): refuse,
  11606. # as for the probe above, rather than answer 401 -- the frontend reads
  11607. # a 401 from /auth/me as final and signs the user out.
  11608. logging.getLogger(__name__).exception("auth_middleware: failing closed on token-check error from %s", path)
  11609. return JSONResponse(
  11610. status_code=503,
  11611. content={"detail": "Authentication service temporarily unavailable"},
  11612. )
  11613. return await call_next(request)
  11614. @app.middleware("http")
  11615. async def trace_id_middleware(request, call_next):
  11616. """Stamp every HTTP request with a trace ID and echo it back.
  11617. Decorated AFTER auth_middleware on purpose: Starlette stacks
  11618. @app.middleware decorators LIFO, so the last-decorated runs first
  11619. inbound. Putting the trace stamp last makes it the OUTERMOST layer,
  11620. which means auth-middleware log lines (and every line emitted on the
  11621. way down to and back from the route handler) all carry the same
  11622. trace ID. If we put it before auth, auth's logs would be stamped
  11623. with the *previous* request's ID — useless for correlation.
  11624. Honours an inbound ``X-Trace-Id`` header so callers running their
  11625. own tracing can correlate their span IDs with our log lines, but
  11626. only if the value passes the whitelist gate in
  11627. ``backend.app.core.trace.normalise_inbound_trace_id`` — anything
  11628. rejected (too long, contains control chars, etc.) silently triggers
  11629. a freshly minted server-side ID rather than failing the request.
  11630. The minted (or echoed) ID is set on a ContextVar so that every log
  11631. record emitted during the request — application logs *and* uvicorn's
  11632. access log — carries it via TraceIDFilter, and is also written to
  11633. the ``X-Trace-Id`` response header so clients can pin a server-side
  11634. log search to the exact request they made.
  11635. """
  11636. from backend.app.core.trace import (
  11637. generate_trace_id,
  11638. normalise_inbound_trace_id,
  11639. trace_id_var,
  11640. )
  11641. inbound = normalise_inbound_trace_id(request.headers.get("X-Trace-Id"))
  11642. trace_id = inbound if inbound is not None else generate_trace_id()
  11643. token = trace_id_var.set(trace_id)
  11644. try:
  11645. response = await call_next(request)
  11646. finally:
  11647. # Reset the ContextVar so a record emitted in a totally
  11648. # unrelated background task that just happens to inherit this
  11649. # context doesn't keep referencing this request's ID forever.
  11650. # In practice ContextVar.reset is best-effort under asyncio
  11651. # task-spawn semantics, but the cost is one attribute write so
  11652. # we may as well do it.
  11653. trace_id_var.reset(token)
  11654. response.headers["X-Trace-Id"] = trace_id
  11655. return response
  11656. # API routes
  11657. app.include_router(auth.router, prefix=app_settings.api_prefix)
  11658. app.include_router(mfa.router, prefix=app_settings.api_prefix)
  11659. app.include_router(bug_report.router, prefix=app_settings.api_prefix)
  11660. app.include_router(users.router, prefix=app_settings.api_prefix)
  11661. app.include_router(groups.router, prefix=app_settings.api_prefix)
  11662. app.include_router(printers.router, prefix=app_settings.api_prefix)
  11663. app.include_router(printer_locations.router, prefix=app_settings.api_prefix)
  11664. app.include_router(archives.router, prefix=app_settings.api_prefix)
  11665. app.include_router(filaments.router, prefix=app_settings.api_prefix)
  11666. app.include_router(finance.router, prefix=app_settings.api_prefix)
  11667. app.include_router(inventory.router, prefix=app_settings.api_prefix)
  11668. app.include_router(labels.router, prefix=app_settings.api_prefix)
  11669. app.include_router(label_templates.router, prefix=app_settings.api_prefix)
  11670. app.include_router(settings_routes.router, prefix=app_settings.api_prefix)
  11671. app.include_router(overlay_branding.router, prefix=app_settings.api_prefix)
  11672. app.include_router(cloud.router, prefix=app_settings.api_prefix)
  11673. app.include_router(orca_cloud.router, prefix=app_settings.api_prefix)
  11674. app.include_router(local_presets.router, prefix=app_settings.api_prefix)
  11675. app.include_router(smart_plugs.router, prefix=app_settings.api_prefix)
  11676. app.include_router(ha_sensors.router, prefix=app_settings.api_prefix)
  11677. app.include_router(location_ha_sensors.router, prefix=app_settings.api_prefix)
  11678. app.include_router(print_log.router, prefix=app_settings.api_prefix)
  11679. app.include_router(print_queue.router, prefix=app_settings.api_prefix)
  11680. app.include_router(scheduled_dryings.router, prefix=app_settings.api_prefix)
  11681. app.include_router(kprofiles.router, prefix=app_settings.api_prefix)
  11682. app.include_router(notifications.router, prefix=app_settings.api_prefix)
  11683. app.include_router(notification_templates.router, prefix=app_settings.api_prefix)
  11684. app.include_router(user_notifications.router, prefix=app_settings.api_prefix)
  11685. app.include_router(spoolman.router, prefix=app_settings.api_prefix)
  11686. app.include_router(spoolman_inventory.router, prefix=app_settings.api_prefix)
  11687. app.include_router(updates.router, prefix=app_settings.api_prefix)
  11688. app.include_router(sponsor_prompt.router, prefix=app_settings.api_prefix)
  11689. app.include_router(announcements.router, prefix=app_settings.api_prefix)
  11690. app.include_router(maintenance.router, prefix=app_settings.api_prefix)
  11691. app.include_router(camera.router, prefix=app_settings.api_prefix)
  11692. app.include_router(camwall.router, prefix=app_settings.api_prefix)
  11693. app.include_router(external_links.router, prefix=app_settings.api_prefix)
  11694. app.include_router(projects.router, prefix=app_settings.api_prefix)
  11695. app.include_router(library.router, prefix=app_settings.api_prefix)
  11696. app.include_router(library_tags.router, prefix=app_settings.api_prefix)
  11697. app.include_router(library_trash.router, prefix=app_settings.api_prefix)
  11698. app.include_router(library_variants.router, prefix=app_settings.api_prefix)
  11699. app.include_router(slice_jobs.router, prefix=app_settings.api_prefix)
  11700. app.include_router(slicer_pipelines.router, prefix=app_settings.api_prefix)
  11701. app.include_router(pipeline_runs.pipeline_run_create_router, prefix=app_settings.api_prefix)
  11702. app.include_router(pipeline_runs.pipeline_run_router, prefix=app_settings.api_prefix)
  11703. app.include_router(slicer_presets.router, prefix=app_settings.api_prefix)
  11704. app.include_router(archive_purge.router, prefix=app_settings.api_prefix)
  11705. app.include_router(makerworld.router, prefix=app_settings.api_prefix)
  11706. app.include_router(manyfold.router, prefix=app_settings.api_prefix)
  11707. app.include_router(api_keys.router, prefix=app_settings.api_prefix)
  11708. app.include_router(connected_apps.router, prefix=app_settings.api_prefix)
  11709. app.include_router(webhook.router, prefix=app_settings.api_prefix)
  11710. app.include_router(ams_history.router, prefix=app_settings.api_prefix)
  11711. app.include_router(printer_sensor_history.router, prefix=app_settings.api_prefix)
  11712. app.include_router(system.router, prefix=app_settings.api_prefix)
  11713. app.include_router(support.router, prefix=app_settings.api_prefix)
  11714. app.include_router(websocket.router, prefix=app_settings.api_prefix)
  11715. app.include_router(discovery.router, prefix=app_settings.api_prefix)
  11716. app.include_router(pending_uploads.router, prefix=app_settings.api_prefix)
  11717. app.include_router(firmware.router, prefix=app_settings.api_prefix)
  11718. app.include_router(bulk.router, prefix=app_settings.api_prefix)
  11719. app.include_router(github_backup.router, prefix=app_settings.api_prefix)
  11720. app.include_router(local_backup.router, prefix=app_settings.api_prefix)
  11721. app.include_router(obico.router, prefix=app_settings.api_prefix)
  11722. app.include_router(octoeverywhere.router, prefix=app_settings.api_prefix)
  11723. app.include_router(metrics.router, prefix=app_settings.api_prefix)
  11724. app.include_router(virtual_printers.router, prefix=app_settings.api_prefix)
  11725. app.include_router(spoolbuddy.router, prefix=app_settings.api_prefix)
  11726. # Serve static files (React build)
  11727. if app_settings.static_dir.exists() and any(app_settings.static_dir.iterdir()):
  11728. # Compressed, and cached for good where Vite hashed the name (#3175).
  11729. app.mount(
  11730. "/assets",
  11731. AssetStaticFiles(directory=app_settings.static_dir / "assets"),
  11732. name="assets",
  11733. )
  11734. if (app_settings.static_dir / "img").exists():
  11735. app.mount(
  11736. "/img",
  11737. StaticFiles(directory=app_settings.static_dir / "img"),
  11738. name="img",
  11739. )
  11740. if (app_settings.static_dir / "icons").exists():
  11741. app.mount(
  11742. "/icons",
  11743. StaticFiles(directory=app_settings.static_dir / "icons"),
  11744. name="icons",
  11745. )
  11746. # Self-hosted Inter woff2 files (#1460). Without this mount /fonts/*.woff2
  11747. # falls through to the SPA catch-all and returns index.html, which the
  11748. # browser's font sanitizer rejects ("downloadable font: rejected by
  11749. # sanitizer").
  11750. if (app_settings.static_dir / "fonts").exists():
  11751. app.mount(
  11752. "/fonts",
  11753. StaticFiles(directory=app_settings.static_dir / "fonts"),
  11754. name="fonts",
  11755. )
  11756. @app.get("/")
  11757. async def serve_frontend():
  11758. """Serve the React frontend."""
  11759. index_file = app_settings.static_dir / "index.html"
  11760. if index_file.exists():
  11761. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  11762. return {
  11763. "message": "Bambuddy API",
  11764. "docs": "/docs",
  11765. "frontend": "Build and place React app in /static directory",
  11766. }
  11767. # index.html must always be revalidated — Vite emits content-hashed JS/CSS
  11768. # bundles (e.g. `index-JRaF_JhW.js`), so the JS itself is safe to cache
  11769. # forever, but the HTML wrapping it is the only file that knows which hash
  11770. # is current. Without explicit cache-control headers Chromium decides
  11771. # heuristically (typically 10% of the time since Last-Modified) and on
  11772. # long-running kiosks happily serves stale HTML across browser restarts.
  11773. # That stale HTML references an old bundle hash, the old bundle is also
  11774. # in the disk cache, and the user ends up running pre-update JS forever
  11775. # without ever knowing why. ``no-cache`` (revalidate every time, but a
  11776. # 304 is cheap) is the correct setting for an SPA's entry HTML.
  11777. _HTML_CACHE_HEADERS = {"Cache-Control": "no-cache, must-revalidate"}
  11778. @app.get("/health")
  11779. async def health_check():
  11780. """Health check endpoint."""
  11781. return {"status": "healthy"}
  11782. # GET + HEAD on the three PWA bootstrap routes (#1460). Scanners and a plain
  11783. # `curl -I` use HEAD; FastAPI's @app.get only registers GET, so HEAD answers
  11784. # with 405 Method Not Allowed and shows up as a "broken manifest" red herring
  11785. # in deployment debugging. One decorator per method, not api_route(methods=
  11786. # [...]): api_route gives both methods the same OpenAPI operation id.
  11787. @app.get("/manifest.json")
  11788. @app.head("/manifest.json")
  11789. async def serve_manifest():
  11790. """Serve PWA manifest."""
  11791. manifest_file = app_settings.static_dir / "manifest.json"
  11792. if manifest_file.exists():
  11793. return FileResponse(manifest_file, media_type="application/manifest+json")
  11794. return {"error": "Manifest not found"}
  11795. @app.get("/sw.js")
  11796. @app.head("/sw.js")
  11797. async def serve_service_worker():
  11798. """Serve service worker."""
  11799. sw_file = app_settings.static_dir / "sw.js"
  11800. if sw_file.exists():
  11801. return FileResponse(
  11802. sw_file,
  11803. media_type="application/javascript",
  11804. headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
  11805. )
  11806. return {"error": "Service worker not found"}
  11807. @app.get("/sw-register.js")
  11808. @app.head("/sw-register.js")
  11809. async def serve_sw_register():
  11810. """Serve the service-worker registration bootstrap script.
  11811. Served as a real JS file so the strict `script-src 'self'` CSP covers it
  11812. without needing 'unsafe-inline' or per-build hashes on the inline tag.
  11813. """
  11814. reg_file = app_settings.static_dir / "sw-register.js"
  11815. if reg_file.exists():
  11816. return FileResponse(reg_file, media_type="application/javascript")
  11817. return {"error": "sw-register.js not found"}
  11818. # ── GCode viewer static files ────────────────────────────────────────────────
  11819. # Catch-all route for React Router (must be last)
  11820. @app.get("/{full_path:path}")
  11821. async def serve_spa(full_path: str):
  11822. """Serve React app for client-side routing."""
  11823. # Don't intercept API routes - raise proper 404 so FastAPI can handle redirects
  11824. if full_path.startswith("api/"):
  11825. from fastapi import HTTPException
  11826. raise HTTPException(status_code=404, detail="Not found")
  11827. index_file = app_settings.static_dir / "index.html"
  11828. if index_file.exists():
  11829. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  11830. return {"error": "Frontend not built"}