print_scheduler.py 577 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748974997509751975297539754975597569757975897599760976197629763976497659766976797689769977097719772977397749775977697779778977997809781978297839784978597869787978897899790979197929793979497959796979797989799980098019802980398049805980698079808980998109811981298139814981598169817981898199820982198229823982498259826982798289829983098319832983398349835983698379838983998409841984298439844984598469847984898499850985198529853985498559856985798589859986098619862986398649865986698679868986998709871987298739874987598769877987898799880988198829883988498859886988798889889989098919892989398949895989698979898989999009901990299039904990599069907990899099910991199129913991499159916991799189919992099219922992399249925992699279928992999309931993299339934993599369937993899399940994199429943994499459946994799489949995099519952995399549955995699579958995999609961996299639964996599669967996899699970997199729973997499759976997799789979998099819982998399849985998699879988998999909991999299939994999599969997999899991000010001100021000310004100051000610007100081000910010100111001210013100141001510016100171001810019100201002110022100231002410025100261002710028100291003010031100321003310034100351003610037100381003910040100411004210043100441004510046100471004810049100501005110052100531005410055100561005710058100591006010061100621006310064100651006610067100681006910070100711007210073100741007510076100771007810079100801008110082100831008410085100861008710088100891009010091100921009310094100951009610097100981009910100101011010210103101041010510106101071010810109101101011110112101131011410115101161011710118101191012010121101221012310124101251012610127101281012910130101311013210133101341013510136101371013810139101401014110142101431014410145101461014710148101491015010151101521015310154101551015610157101581015910160101611016210163101641016510166101671016810169101701017110172101731017410175101761017710178101791018010181101821018310184101851018610187101881018910190101911019210193101941019510196101971019810199102001020110202102031020410205102061020710208102091021010211102121021310214102151021610217102181021910220102211022210223102241022510226102271022810229102301023110232102331023410235102361023710238102391024010241102421024310244102451024610247102481024910250102511025210253102541025510256102571025810259102601026110262102631026410265102661026710268102691027010271102721027310274102751027610277102781027910280102811028210283102841028510286102871028810289102901029110292102931029410295102961029710298102991030010301103021030310304103051030610307103081030910310103111031210313103141031510316103171031810319103201032110322103231032410325103261032710328103291033010331103321033310334103351033610337103381033910340103411034210343103441034510346103471034810349103501035110352103531035410355103561035710358103591036010361103621036310364103651036610367103681036910370103711037210373103741037510376103771037810379103801038110382103831038410385103861038710388103891039010391103921039310394103951039610397103981039910400104011040210403104041040510406104071040810409104101041110412104131041410415104161041710418104191042010421104221042310424104251042610427104281042910430104311043210433104341043510436104371043810439104401044110442104431044410445104461044710448104491045010451104521045310454104551045610457104581045910460104611046210463104641046510466104671046810469104701047110472104731047410475104761047710478104791048010481104821048310484104851048610487104881048910490104911049210493104941049510496104971049810499105001050110502105031050410505105061050710508105091051010511105121051310514105151051610517105181051910520105211052210523105241052510526105271052810529105301053110532105331053410535105361053710538105391054010541105421054310544105451054610547105481054910550105511055210553105541055510556105571055810559105601056110562105631056410565105661056710568105691057010571105721057310574105751057610577105781057910580105811058210583105841058510586105871058810589105901059110592105931059410595105961059710598105991060010601106021060310604106051060610607106081060910610106111061210613106141061510616106171061810619106201062110622106231062410625106261062710628106291063010631106321063310634106351063610637106381063910640106411064210643106441064510646106471064810649106501065110652106531065410655106561065710658106591066010661106621066310664106651066610667106681066910670106711067210673106741067510676106771067810679106801068110682106831068410685106861068710688106891069010691106921069310694106951069610697106981069910700107011070210703107041070510706107071070810709107101071110712107131071410715107161071710718107191072010721107221072310724107251072610727107281072910730107311073210733107341073510736107371073810739107401074110742107431074410745107461074710748107491075010751107521075310754107551075610757107581075910760107611076210763107641076510766107671076810769107701077110772107731077410775107761077710778107791078010781107821078310784107851078610787107881078910790107911079210793107941079510796107971079810799108001080110802108031080410805108061080710808108091081010811108121081310814108151081610817108181081910820108211082210823108241082510826108271082810829108301083110832108331083410835108361083710838108391084010841108421084310844108451084610847108481084910850108511085210853108541085510856108571085810859108601086110862108631086410865108661086710868108691087010871108721087310874108751087610877108781087910880108811088210883108841088510886108871088810889108901089110892108931089410895108961089710898108991090010901109021090310904109051090610907109081090910910109111091210913109141091510916109171091810919109201092110922109231092410925109261092710928109291093010931109321093310934109351093610937109381093910940109411094210943109441094510946109471094810949109501095110952109531095410955109561095710958109591096010961109621096310964109651096610967109681096910970109711097210973109741097510976109771097810979109801098110982109831098410985109861098710988109891099010991109921099310994109951099610997109981099911000110011100211003110041100511006110071100811009110101101111012110131101411015110161101711018110191102011021110221102311024110251102611027110281102911030110311103211033110341103511036110371103811039110401104111042110431104411045110461104711048110491105011051110521105311054110551105611057110581105911060110611106211063110641106511066110671106811069110701107111072110731107411075110761107711078110791108011081110821108311084110851108611087110881108911090110911109211093110941109511096110971109811099111001110111102111031110411105111061110711108111091111011111111121111311114111151111611117111181111911120111211112211123111241112511126111271112811129111301113111132111331113411135111361113711138111391114011141111421114311144111451114611147111481114911150111511115211153111541115511156111571115811159111601116111162111631116411165111661116711168111691117011171111721117311174111751117611177111781117911180111811118211183111841118511186111871118811189111901119111192111931119411195111961119711198111991120011201112021120311204112051120611207112081120911210112111121211213112141121511216112171121811219112201122111222112231122411225112261122711228112291123011231112321123311234112351123611237112381123911240112411124211243112441124511246112471124811249112501125111252112531125411255112561125711258112591126011261112621126311264112651126611267112681126911270112711127211273112741127511276112771127811279112801128111282112831128411285112861128711288112891129011291112921129311294112951129611297112981129911300113011130211303113041130511306113071130811309113101131111312113131131411315113161131711318113191132011321113221132311324113251132611327113281132911330113311133211333113341133511336113371133811339113401134111342113431134411345113461134711348113491135011351113521135311354113551135611357113581135911360113611136211363113641136511366113671136811369113701137111372113731137411375113761137711378113791138011381113821138311384113851138611387
  1. """Print scheduler service - processes the print queue."""
  2. import asyncio
  3. import json
  4. import logging
  5. import time
  6. import uuid
  7. import zipfile
  8. from collections import deque
  9. from collections.abc import Awaitable, Callable, Mapping
  10. from dataclasses import dataclass
  11. from datetime import datetime, timedelta, timezone
  12. from pathlib import Path
  13. from typing import cast
  14. from fastapi import HTTPException
  15. from sqlalchemy import delete, false, func, or_, select, true, update
  16. from sqlalchemy.engine import CursorResult
  17. from sqlalchemy.ext.asyncio import AsyncSession
  18. from sqlalchemy.orm import selectinload
  19. from backend.app.core.config import settings
  20. from backend.app.core.database import async_session, run_with_retry
  21. from backend.app.core.filament_scope import (
  22. ALL_FILAMENTS,
  23. FilamentScope,
  24. file_hold_message,
  25. judged_requirements,
  26. print_anyway_hold_message,
  27. resolve_user_id_filament_scope,
  28. spool_hold_message,
  29. unlisted_file_hold_message,
  30. )
  31. from backend.app.core.printer_scope import ALL_PRINTERS, PrinterScope, resolve_user_id_printer_scope
  32. from backend.app.core.tasks import spawn_background_task
  33. from backend.app.core.websocket import ws_manager
  34. from backend.app.models.archive import PrintArchive
  35. from backend.app.models.library import LibraryFile
  36. from backend.app.models.print_queue import PrintQueueItem, PrintQueueVariant
  37. from backend.app.models.printer import Printer
  38. from backend.app.models.scheduled_drying import ScheduledDrying
  39. from backend.app.models.settings import Settings
  40. from backend.app.models.smart_plug import SmartPlug
  41. from backend.app.models.spool_assignment import SpoolAssignment
  42. from backend.app.models.spoolman_slot_assignment import SpoolmanSlotAssignment
  43. from backend.app.services import (
  44. drying_preflight,
  45. kprofile_drift,
  46. plate_clear,
  47. print_dispatch_context,
  48. slicer_ams_pick,
  49. stock_forecast,
  50. )
  51. from backend.app.services.bambu_ftp import (
  52. BambuFTPClient,
  53. FtpFailureKind,
  54. FtpFailureReport,
  55. UploadCancelled,
  56. cache_3mf_download,
  57. delete_file_async,
  58. describe_upload_failure,
  59. get_ftp_retry_settings,
  60. upload_file_async,
  61. with_ftp_retry,
  62. )
  63. from backend.app.services.bambu_mqtt import _RACK_NOZZLE_IDS, HMS_MQTT_VERIFY_FAILED, resolve_rack_plan_mapping
  64. from backend.app.services.filament_deficit import compute_deficit_for_queue_item
  65. from backend.app.services.finance_budget import (
  66. create_budget_reservation,
  67. release_budget_reservation,
  68. validate_print_budget,
  69. )
  70. from backend.app.services.ha_sensor_manager import ha_sensor_manager
  71. from backend.app.services.mqtt_relay import mqtt_relay
  72. from backend.app.services.notification_service import notification_service
  73. from backend.app.services.print_cost_estimate import estimate_queue_source_cost
  74. from backend.app.services.printer_manager import (
  75. printer_manager,
  76. printer_tray_layout,
  77. supports_airduct,
  78. supports_chamber_heater,
  79. supports_chamber_temp,
  80. supports_drying,
  81. supports_drying_while_printing,
  82. )
  83. from backend.app.services.queue_item_printer import drop_printer_resolved
  84. from backend.app.services.slot_nozzle import reports_two_nozzles
  85. from backend.app.services.smart_plug_manager import is_monitor_only, publish_plug_state, smart_plug_manager
  86. from backend.app.services.spool_assignment_notifications import (
  87. _global_tray_from_assignment,
  88. _slot_label_from_global_tray,
  89. )
  90. from backend.app.utils.ams_drying import is_countdown_parked
  91. from backend.app.utils.ams_humidity import ams_humidity_percent
  92. from backend.app.utils.archive_paths import archive_photos_dir
  93. from backend.app.utils.color_utils import perceptual_color_distance
  94. from backend.app.utils.filament_types import canonical_filament_type
  95. from backend.app.utils.filename import InvalidFilenameError, derive_remote_filename, validate_print_filename
  96. from backend.app.utils.library_paths import move_library_photos
  97. from backend.app.utils.local_time import utcnow_naive
  98. from backend.app.utils.printer_models import (
  99. has_ams3h,
  100. is_dual_nozzle_model,
  101. is_gcode_compatible,
  102. is_nozzle_rack_model,
  103. normalize_printer_model,
  104. )
  105. from backend.app.utils.threemf_tools import (
  106. default_plate_number,
  107. extract_rack_plan_from_3mf,
  108. extract_slot_extruders_from_3mf,
  109. plate_bed_temperature,
  110. select_plate_gcode_name,
  111. )
  112. from backend.app.utils.tray_ids import (
  113. TrayLayout,
  114. is_ams_ht_id,
  115. is_external_tray,
  116. reconcile_use_ams,
  117. reports_no_ams,
  118. unit_label,
  119. )
  120. logger = logging.getLogger(__name__)
  121. def _plate_clear_holds(printer_id: int, require_plate_clear: bool) -> bool:
  122. """Whether the plate-clear gate holds this printer's queue right now.
  123. ``require_plate_clear`` is the app setting; the printer's own setting
  124. overrides it (#2860).
  125. """
  126. return plate_clear.applies(printer_id, require_plate_clear) and printer_manager.is_awaiting_plate_clear(printer_id)
  127. def _ams_slot_label(ams_id: int, tray_id: int, layout: TrayLayout | None = None) -> str:
  128. """Human slot name for an assignment tuple, in the spelling already in use.
  129. Composed from the notification side's own pair rather than spelled out again
  130. here. The hand-rolled version this replaces got three real slot kinds wrong:
  131. ams_id 254 fell into its ``>= 128`` branch and came out as ``HT-`` plus
  132. ``chr(191)``; both externals collapsed to one label, because an external
  133. assignment stores ams_id 255 with tray_id picking left or right
  134. (inventory.py:1771, ``ext_id = data.tray_id + 254``); and an A2L AMS-Lite,
  135. normalised to unit 6, read as ``AMS-G`` instead of ``Lite-``.
  136. Naming the same slot two ways is its own defect once an alert and the
  137. Inventory page are meant to be talking about the same tray, so this defers
  138. rather than adding a fifth spelling.
  139. ``layout`` is the printer's (``printer_tray_layout``): an H3C names its
  140. per-hotend external inputs "Ext-n".
  141. """
  142. return _slot_label_from_global_tray(_global_tray_from_assignment(ams_id, tray_id, layout), layout)
  143. # Minimum seconds between low-filament checks. Matches the scheduler's idle
  144. # interval: the fast path runs every 3 s while an upload is in flight, and a
  145. # low spool does not need answering at that resolution (#2913).
  146. _FILAMENT_LOW_MIN_INTERVAL = 30.0
  147. # Minimum seconds between stock forecast checks (#2955). The forecast is in whole
  148. # days, so a finer check would only repeat the same answer.
  149. _STOCK_FORECAST_MIN_INTERVAL = 3600.0
  150. # The settings row that holds which stock alerts have already been sent (#2955).
  151. _STOCK_ALERTS_SETTING_KEY = "stock_alerts_notified"
  152. # The Inventory page asks for this many usage records and the forecast panel
  153. # builds its rates from them; the alert reads the same window so the two agree.
  154. _STOCK_FORECAST_HISTORY_LIMIT = 5000
  155. def _spoolman_time(value: object) -> datetime | None:
  156. """A Spoolman ISO timestamp, or None when missing or unreadable."""
  157. if not value:
  158. return None
  159. try:
  160. return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
  161. except ValueError:
  162. return None
  163. def _remaining_percent(label_weight: int | float | None, weight_used: float | None) -> float | None:
  164. """Remaining filament as a percentage of the label weight.
  165. The same arithmetic the Inventory page's Low Stock count uses, and it works
  166. unchanged in both inventory modes: internal spools store ``weight_used``
  167. directly, and ``_map_spoolman_spool`` derives it from Spoolman's
  168. ``remaining_weight`` so the shape matches. Returns None when there is no
  169. label weight to be a percentage of -- a spool that cannot say how full it
  170. started cannot say how empty it is.
  171. """
  172. if not label_weight or label_weight <= 0:
  173. return None
  174. remaining = max(0.0, float(label_weight) - float(weight_used or 0.0))
  175. return remaining / float(label_weight) * 100.0
  176. # Dispatch-toast progress throttling (#1625 follow-up). Mirrors the legacy
  177. # background_dispatch.py upload_progress_callback (200 ms time gate + 256 KB
  178. # byte gate) from before the scheduler unification. Time gate keeps small
  179. # files from going silent (a single 8 KB chunk fires once and that's it);
  180. # byte gate caps the broadcast rate on slow LAN where 200 ms covers many
  181. # chunks. uploaded >= total always emits so the bar closes cleanly even on
  182. # sub-200 ms files.
  183. _DISPATCH_PROGRESS_BYTE_STEP = 256 * 1024
  184. _DISPATCH_PROGRESS_MIN_INTERVAL_SECS = 0.2
  185. # How far back chamber temperature samples are retained. 2h comfortably spans
  186. # any soak a user can configure (capped at 30 min) plus the plate-clearing gap
  187. # before the next print.
  188. _CHAMBER_HISTORY_TTL_SECONDS = 7200
  189. # Fallback for `queue_keep_warm_max_minutes` — how long the bed may be held
  190. # warm on a printer sitting in FINISH before the heaters are shut off. Users
  191. # who clear plates promptly will want far less than this; it is deliberately
  192. # the cautious end, since the cost of it being too short is only a re-soak.
  193. _KEEP_WARM_MAX_MINUTES_DEFAULT = 120
  194. # Max acceptable gap between two consecutive chamber samples before we treat
  195. # the older one as belonging to a separate observation run (printer went
  196. # offline and came back). Above ~30s cadence with a safety margin.
  197. _CHAMBER_SAMPLE_MAX_GAP_SECONDS = 60.0
  198. # How long the chamber must read below target before we accept that it really
  199. # cooled. An enclosed chamber's thermal mass cannot lose and regain several
  200. # degrees quickly: measured on an X1C, falling from ~55°C to below 48°C took
  201. # 23-73 minutes (~0.2 C/min), while the fastest drop ever recorded was
  202. # 27 C/min — impossible for that mass, i.e. a sensor artifact. Brief
  203. # sub-target readings are therefore a door opening or noise, not lost soak,
  204. # and a plate swap (exactly when keep-warm is running) produces one. Six
  205. # minutes clears the longest such artifact observed (~5 min once bracketed by
  206. # its neighbouring samples) and still sits far below the 23-minute floor for
  207. # real cooling.
  208. _CHAMBER_DIP_GRACE_SECONDS = 360.0
  209. # How often the preheat stage re-checks that the item it is heating for still
  210. # wants to be printed. Cancelling only writes `status` to the database — it
  211. # cannot interrupt a coroutine parked in `asyncio.sleep` — so without this the
  212. # heaters run for the rest of max_wait + soak (45 min at the default settings)
  213. # and the printer stays in `busy_printers`, blocking every other queued item
  214. # behind a print that is not happening.
  215. _PREHEAT_CANCEL_CHECK_SECONDS = 10.0
  216. # set_airduct_mode modeId values (bambu_mqtt.py:5937 — 0 cooling, 1 heating).
  217. _AIRDUCT_MODE_COOLING = 0
  218. _AIRDUCT_MODE_HEATING = 1
  219. # How long a queue row may stay 'printing' while its printer sits in a terminal
  220. # state before the scheduler closes it itself (#2829).
  221. #
  222. # A real completion arrives within seconds of the printer going terminal, so
  223. # five minutes is far outside the normal path — this only ever sees a row whose
  224. # completion was refused or never delivered. It is the whole cost of the
  225. # failure to the user, though: a stranded row blocks every later job for that
  226. # printer, so it should not be raised without reason.
  227. _STRANDED_PRINTING_GRACE_SECONDS = 300.0
  228. # gcode_state values that mean the print is over, mapped to the queue status
  229. # they imply. Mirrors the mapping in bambu_mqtt's completion detection
  230. # (FINISH -> completed, FAILED -> failed, anything else terminal -> aborted,
  231. # which the queue calls cancelled) so a recovered row cannot disagree with one
  232. # closed by the normal path.
  233. _TERMINAL_STATE_QUEUE_STATUS = {
  234. "FINISH": "completed",
  235. "FAILED": "failed",
  236. "IDLE": "cancelled",
  237. }
  238. def _terminal_queue_status(state) -> str | None:
  239. """Queue status implied by *state*, or None if the print is not over.
  240. None for a disconnected printer as well as a busy one: a printer we are not
  241. talking to has a stale ``state`` field that proves nothing about what it is
  242. doing now.
  243. """
  244. if state is None or not getattr(state, "connected", False):
  245. return None
  246. return _TERMINAL_STATE_QUEUE_STATUS.get(getattr(state, "state", None))
  247. @dataclass
  248. class _KeepWarmEntry:
  249. """Per-printer keep-warm state.
  250. - ``started``: monotonic time when keep-warm first fired for this printer.
  251. Used by the max-duration timeout.
  252. - ``held_target``: last bed target we successfully published. On release
  253. we only send bed-off when firmware still reports this value, so a user
  254. or subsequent print that changed the target isn't clobbered.
  255. - ``expired``: latched True when the max-duration timeout fires. Prevents
  256. re-engagement (and re-seeding of ``started``) on subsequent ticks. The
  257. release sweep drops the entry entirely once the printer leaves the
  258. candidate set.
  259. """
  260. started: float
  261. held_target: int
  262. expired: bool = False
  263. # Auto-drying re-arm guards (#2770).
  264. #
  265. # An AMS reports HIGHER relative humidity while it is warm than once it has
  266. # cooled: measured on an H2D/AMS 2 Pro at 10-13% cold against 15-20% throughout
  267. # every drying cycle, and the same unit read 16-20% across a full 12 h dry. A
  268. # threshold set inside that band therefore cannot be satisfied while the box is
  269. # hot, and the firmware is free to end a cycle whenever it decides the filament
  270. # is dry — so the next 30 s pass sees dry_time 0 with the reading still above
  271. # the threshold and arms another cycle. The reporter's log has five 12-hour
  272. # cycles armed inside four hours, one of them six seconds after the previous
  273. # ended.
  274. #
  275. # The cooldown stops the six-second re-arm; the unproductive-cycle cap stops the
  276. # loop. Neither ever stops a running cycle — both only gate STARTING one, so a
  277. # manual or firmware-run dry is untouched.
  278. AUTO_DRY_REARM_COOLDOWN_SECONDS = 30 * 60
  279. AUTO_DRY_MAX_UNPRODUCTIVE_CYCLES = 2
  280. # A cycle whose timer stood still (#3478): the firmware took the command and
  281. # left the heater off, so dry_time stays put with dry_status Off. It tells
  282. # nothing about humidity, so it has its own counter; this many in a row
  283. # suspends the unit with a message that says the unit did not heat, not that
  284. # the threshold is too low.
  285. AUTO_DRY_MAX_NEVER_HEATED_CYCLES = 2
  286. def _never_heated(unit_state: dict[str, object]) -> int:
  287. """Cycles in a row whose timer stood still on this unit (#3478)."""
  288. count = unit_state.get("never_heated")
  289. return count if isinstance(count, int) else 0
  290. # Sustained-humidity wait (#2518): an above-threshold streak is only
  291. # "continuous" if some pass observed it within the gap ceiling. A unit that
  292. # goes unobserved longer (print running, printer disconnected, sensor silent)
  293. # restarts its streak rather than inheriting a stale one. The ceiling is
  294. # derived from the scheduler cadence — four missed passes — with this floor so
  295. # a fast-polling configuration does not void streaks on a single hiccup.
  296. AUTO_DRY_SUSTAINED_GAP_FLOOR_SECONDS = 120
  297. # How long a finished scheduled drying row is kept before it is pruned.
  298. SCHEDULED_DRYING_RETENTION_DAYS = 7
  299. # How often that prune actually runs. The check itself is called on every queue
  300. # pass — every 3s while dispatching — and issuing the DELETE is what begins a
  301. # write transaction, which SQLite serialises against every other writer. Rows
  302. # only become prunable a week after they finish, so anything short of hourly is
  303. # paying that cost for nothing.
  304. SCHEDULED_DRYING_PRUNE_INTERVAL_SECONDS = 60 * 60
  305. class _UploadProgressBridge:
  306. """Thread-safe bridge from ``upload_file_async`` to the WS broadcaster.
  307. ``upload_file_async`` runs the FTP transfer in an executor thread and
  308. invokes its ``progress_callback`` from that thread, so the callback
  309. body cannot ``await`` directly. This bridge captures the asyncio loop
  310. at construction (on the scheduler thread) and uses
  311. ``run_coroutine_threadsafe`` to hop back. The byte/time throttle
  312. matches the legacy background_dispatch.py path 1:1 so the toast feels
  313. identical to the pre-#1625 experience.
  314. Failures inside the emit are swallowed — progress is a UX nicety, the
  315. upload itself must not fail because of a WS hiccup.
  316. """
  317. def __init__(self, user_id: int | None, queue_item_id: int):
  318. self._user_id = user_id
  319. self._queue_item_id = queue_item_id
  320. try:
  321. self._loop = asyncio.get_running_loop()
  322. except RuntimeError:
  323. self._loop = None
  324. self._last_emit_bytes = 0
  325. self._last_emit_monotonic = 0.0
  326. self._has_emitted = False
  327. def __call__(self, bytes_transferred: int, total_bytes: int) -> None:
  328. if self._loop is None or total_bytes <= 0:
  329. return
  330. now = time.monotonic()
  331. # Mirrors legacy bg-dispatch: emit if first call OR upload complete
  332. # OR 200 ms elapsed OR ≥256 KB transferred since last emit. Two of
  333. # the four matter most: first-call so the user sees something even
  334. # for sub-chunk-size files; uploaded >= total so the bar locks at
  335. # 100% even when the throttle would otherwise eat it.
  336. should_emit = (
  337. not self._has_emitted
  338. or bytes_transferred >= total_bytes
  339. or now - self._last_emit_monotonic >= _DISPATCH_PROGRESS_MIN_INTERVAL_SECS
  340. or bytes_transferred - self._last_emit_bytes >= _DISPATCH_PROGRESS_BYTE_STEP
  341. )
  342. if not should_emit:
  343. return
  344. self._has_emitted = True
  345. self._last_emit_bytes = bytes_transferred
  346. self._last_emit_monotonic = now
  347. try:
  348. asyncio.run_coroutine_threadsafe(
  349. ws_manager.send_queue_item_upload_progress(
  350. user_id=self._user_id,
  351. queue_item_id=self._queue_item_id,
  352. bytes_transferred=bytes_transferred,
  353. total_bytes=total_bytes,
  354. ),
  355. self._loop,
  356. )
  357. except Exception:
  358. pass # progress is best-effort, never block the upload
  359. # Bambu firmware states that mean the project_file has actually been accepted
  360. # and the printer is now processing / running / paused mid-print. Used by the
  361. # dispatch watchdog (#1370): a transition into one of these states means the
  362. # print landed, anything else (e.g. FINISH -> IDLE after the user dismisses
  363. # a post-print prompt) is NOT a valid "command landed" signal even though the
  364. # state value did change. SLICING is included because some firmwares park
  365. # briefly in SLICING between PREPARE and RUNNING while parsing the g-code.
  366. _ACTIVE_PRINT_STATES: frozenset[str] = frozenset({"PREPARE", "SLICING", "RUNNING", "PAUSE"})
  367. # How many times the start-watchdog may revert an item to 'pending' before it
  368. # gives up and fails the row instead (#2555). Each attempt costs a full 3MF
  369. # re-upload plus the watchdog's wait, so a wedged printer left to retry forever
  370. # both never recovers and starves the other printers of dispatch slots. Three
  371. # is chosen to clear the transient causes the watchdog already recovers from —
  372. # a lost MQTT publish on a half-broken session (#887/#936) is fixed by the
  373. # force-reconnect on the very next attempt — while still bounding the loop.
  374. DISPATCH_MAX_ATTEMPTS = 3
  375. # Upload failures that mean the file never reached the printer's storage: the
  376. # file service refused or never answered (#3210). These put the item back in
  377. # the queue instead of failing it, because nothing about the job is wrong --
  378. # and failing it left the printer idle, so the next pass handed it the next
  379. # item, which failed the same way. One P2S whose file service was out of
  380. # connection slots ate 43 queued jobs in ten minutes that way.
  381. #
  382. # STORAGE (a missing, full or unusable card or USB drive) and AUTH (a rejected
  383. # access code) were left out at first, because they need the user. But
  384. # failing the item leaves the printer just as idle, and #3409's H2D with an
  385. # unusable USB stick ate seven jobs within an hour. Every job sent to that
  386. # printer fails the same way, so they wait too; the retry finds the fix once
  387. # the user has made it.
  388. #
  389. # A 553 can also be this one file: a name the card's filesystem cannot hold.
  390. # That one fails the item, see `_start_print`.
  391. #
  392. # Not here: NOT_FOUND (a path problem, which can be this one file), UNKNOWN,
  393. # and an upload that overran its deadline -- each retry of that costs a whole
  394. # upload that would be just as slow.
  395. _UPLOAD_REQUEUE_KINDS: frozenset[FtpFailureKind] = frozenset(
  396. {
  397. FtpFailureKind.HANDSHAKE,
  398. FtpFailureKind.SESSIONS,
  399. FtpFailureKind.COOLOFF,
  400. FtpFailureKind.TIMEOUT,
  401. FtpFailureKind.NETWORK,
  402. FtpFailureKind.STORAGE,
  403. FtpFailureKind.AUTH,
  404. }
  405. )
  406. def _refused_upload_hold_reason(printer_label: str, kind: FtpFailureKind | None) -> str:
  407. """What a job waiting on a printer that refused its last upload shows.
  408. Storage and the access code need the user to do something, so they say
  409. what, and so does a printer out of FTP sessions: those have been seen to
  410. stay until the printer restarts (#3474). The rest clear on their own.
  411. """
  412. if kind is FtpFailureKind.STORAGE:
  413. return f"{printer_label} refused to store the file — check its SD card or USB drive. {_REFUSED_UPLOAD_TAIL}"
  414. if kind is FtpFailureKind.AUTH:
  415. return (
  416. f"{printer_label} refused the file transfer — check its access code on the Printers page. "
  417. f"{_REFUSED_UPLOAD_TAIL}"
  418. )
  419. if kind is FtpFailureKind.SESSIONS:
  420. return (
  421. f"{printer_label} has no file-transfer sessions left (421 too many connections) — "
  422. f"if this does not clear within a few minutes, restart the printer. {_REFUSED_UPLOAD_TAIL}"
  423. )
  424. return f"{printer_label} is not accepting files — {_REFUSED_UPLOAD_TAIL}"
  425. _REFUSED_UPLOAD_TAIL = "Bambuddy will retry automatically"
  426. def _asks_nothing_new(waiting_reason: str) -> bool:
  427. """A model job's reason made only of busy printers and printers that refused an upload.
  428. The refusal sent its own notification (`_requeue_after_upload_refused`),
  429. so this one has nothing to add.
  430. """
  431. parts = [p.strip() for p in waiting_reason.split(" | ")]
  432. return all(p.startswith("Busy:") or p.endswith(_REFUSED_UPLOAD_TAIL) for p in parts)
  433. # How long a printer whose upload was put back stays out of dispatch (#3210).
  434. # The first window matches the FTP client's own handshake cool-off, so the
  435. # retry lands after the client would talk to the printer again anyway. Each
  436. # further refusal in a row doubles it, up to the cap: #3210's printer refused
  437. # for some 40 hours, and every retry costs a preheat cycle where preheat is on,
  438. # five connection attempts and a page of log. A successful upload resets it.
  439. UPLOAD_FAILURE_BACKOFF_SECONDS = 300
  440. UPLOAD_FAILURE_BACKOFF_MAX_SECONDS = 3600
  441. def _upload_backoff_seconds(refusals: int) -> int:
  442. """Backoff after the *refusals*-th refused upload in a row (1-based)."""
  443. return min(UPLOAD_FAILURE_BACKOFF_SECONDS * 2 ** max(refusals - 1, 0), UPLOAD_FAILURE_BACKOFF_MAX_SECONDS)
  444. @dataclass(slots=True)
  445. class _ModelCandidate:
  446. """One (file, printer model) pair the model-based matcher may try.
  447. Model-based assignment used to have exactly one of these per item, held
  448. directly in the item's own columns. Cross-model queue items (#671) have
  449. several, held in ``print_queue_variants``. Both shapes are normalised into
  450. this so the matching, the cross-model gate and the waiting-reason handling
  451. are written once and an item without variants provably takes the same path
  452. it took before variants existed.
  453. ``variant`` is None for the item's own columns and set for a real variant
  454. row, which is what :meth:`PrintScheduler._resolve_variant` writes onto the
  455. item once that candidate wins.
  456. """
  457. target_model: str | None
  458. sliced_for: str | None
  459. required_filament_types: str | None
  460. filament_overrides: str | None
  461. variant: "PrintQueueVariant | None" = None
  462. # Nozzle diameter (mm) the file was sliced for, None when unknown (#3536).
  463. sliced_nozzle: float | None = None
  464. def _sliced_for_model(archive, library_file) -> str | None:
  465. """Model a 3MF declares it was sliced for, from whichever source holds it."""
  466. if archive is not None:
  467. return archive.sliced_for_model
  468. if library_file is not None and library_file.file_metadata:
  469. return library_file.file_metadata.get("sliced_for_model")
  470. return None
  471. def _sliced_nozzle(archive, library_file) -> float | None:
  472. """Nozzle diameter (mm) a 3MF was sliced for, None when unknown (#3536).
  473. Fail-safe: a missing, unparsable or non-positive value means "unknown", and
  474. unknown never excludes a printer. Not rounded here; the tolerance lives in
  475. ``_nozzle_mismatch_message``.
  476. """
  477. if archive is not None:
  478. raw = archive.nozzle_diameter
  479. elif library_file is not None and library_file.file_metadata:
  480. raw = library_file.file_metadata.get("nozzle_diameter")
  481. else:
  482. return None
  483. try:
  484. value = float(raw)
  485. except (TypeError, ValueError):
  486. return None
  487. return value if value > 0 else None
  488. def _filament_constraints(candidate: _ModelCandidate) -> tuple[list[str] | None, list[dict] | None]:
  489. """The filament a candidate needs, as ``(types, overrides)``.
  490. Both columns are JSON text written by the slicer step. Malformed content is
  491. treated as no constraint rather than as an error: a job whose overrides
  492. cannot be parsed still prints, it just gets no filament-based narrowing.
  493. Overrides carry their own types, so the returned type list is the union of
  494. the two — an override on one slot must not drop the requirements of the
  495. slots it says nothing about.
  496. Shared by the matcher and by the smart-plug wake step so both ask a printer
  497. for the same filament (#2876). Waking a printer the matcher would then
  498. reject on colour is the bug this exists to prevent.
  499. """
  500. required_types = None
  501. if candidate.required_filament_types:
  502. try:
  503. required_types = json.loads(candidate.required_filament_types)
  504. except json.JSONDecodeError:
  505. pass
  506. filament_overrides = None
  507. if candidate.filament_overrides:
  508. try:
  509. filament_overrides = json.loads(candidate.filament_overrides)
  510. except json.JSONDecodeError:
  511. pass
  512. effective_types = required_types
  513. if filament_overrides:
  514. override_types = sorted({o["type"] for o in filament_overrides if "type" in o})
  515. if override_types:
  516. effective_types = sorted(set(required_types or []) | set(override_types))
  517. return effective_types, filament_overrides
  518. def _could_take_printer(item: PrintQueueItem, printer_id: int, printer_model: str | None) -> bool:
  519. """Whether ``item`` was competing for the printer another item just took.
  520. The SJF starvation guard marks a job as jumped when a shorter one lower in
  521. the queue takes a printer it wanted. A pinned job and an "Any <model>" job
  522. compete for the same printer, so the guard has to look across both lanes
  523. (#3200); looking only inside the dispatched item's own lane let a stream of
  524. short model-based jobs hold a longer pinned job back indefinitely.
  525. Location filters are not checked: over-marking only lifts an item that was
  526. going to wait anyway, while under-marking is the starvation this prevents.
  527. """
  528. if item.printer_id is not None:
  529. return item.printer_id == printer_id
  530. if not printer_model:
  531. return False
  532. wanted = (normalize_printer_model(printer_model) or printer_model).lower()
  533. models = [v.target_model for v in item.variants] if item.variants else [item.target_model]
  534. return any(m and (normalize_printer_model(m) or m).lower() == wanted for m in models)
  535. def _candidates_for(item: PrintQueueItem) -> list[_ModelCandidate]:
  536. """Candidate files for ``item``, best first.
  537. An item with no variant rows yields exactly one candidate built from its own
  538. columns — the pre-#671 behaviour, unchanged.
  539. Variants come back least-attempted first, ties broken by the user's
  540. ``position``. On the first pass every count is zero, so this is purely the
  541. user's priority order. After a start-watchdog bounce the printer that failed
  542. drops behind, so the next lap tries the other machine rather than spending the
  543. item's whole retry budget on the one that is wedged. Once every candidate has
  544. been tried equally often they cycle again, which keeps the item-level
  545. ``DISPATCH_MAX_ATTEMPTS`` bound from #2555 intact — a job with alternatives
  546. still gives up, it just does not give up without trying them.
  547. """
  548. if not item.variants:
  549. if not item.archive_id and not item.library_file_id:
  550. # Nothing to print at all. Dispatching would fail deep in the upload
  551. # on "No archive_id or library_file_id"; the caller holds the item
  552. # with an explanation instead.
  553. return []
  554. return [
  555. _ModelCandidate(
  556. target_model=item.target_model,
  557. sliced_for=_sliced_for_model(item.archive, item.library_file),
  558. required_filament_types=item.required_filament_types,
  559. filament_overrides=item.filament_overrides,
  560. sliced_nozzle=_sliced_nozzle(item.archive, item.library_file),
  561. )
  562. ]
  563. # Drop candidates whose file is gone or in the trash. Both are reachable and
  564. # neither is covered by the schema: library deletes are soft (the row lives
  565. # on with ``deleted_at`` set, which no foreign key can express), and SQLite
  566. # ships with ``PRAGMA foreign_keys`` off, so the ON DELETE CASCADE never
  567. # fires there and a hard delete leaves the variant row pointing at nothing.
  568. usable = [v for v in item.variants if v.library_file is not None and v.library_file.deleted_at is None]
  569. ordered = sorted(usable, key=lambda v: (v.attempt_count or 0, v.position, v.id))
  570. return [
  571. _ModelCandidate(
  572. target_model=v.target_model,
  573. sliced_for=_sliced_for_model(None, v.library_file),
  574. required_filament_types=v.required_filament_types,
  575. filament_overrides=v.filament_overrides,
  576. variant=v,
  577. sliced_nozzle=_sliced_nozzle(None, v.library_file),
  578. )
  579. for v in ordered
  580. ]
  581. # Why a job for any printer of a model waits when every free one's last print
  582. # failed. The queue page matches it to offer Resume for the model.
  583. MODEL_GATE_WAIT_REASON = "The last print failed on every free matching printer"
  584. def _collapse_waiting_reasons(per_model: list[tuple[str | None, str]]) -> str | None:
  585. """Fold one waiting reason per candidate into a single line for the item.
  586. A cross-model item produces a reason per candidate, and pasting them
  587. together unlabelled reads as gibberish ("No idle printer; PETG not loaded"
  588. — on which machine?). Each reason is prefixed with its model, except in the
  589. single-candidate case where the item already displays its target model and
  590. the prefix would be noise.
  591. Identical reasons collapse rather than repeat, so three idle-less models
  592. read as one clause.
  593. When *every* candidate is merely busy the parts are joined with the ``" | "``
  594. separator :meth:`PrintScheduler._is_busy_only` already parses, and left
  595. unprefixed. That case must keep testing busy-only: a fleet that is simply
  596. printing needs no user action, and labelling the clauses would turn each pass
  597. over a two-model item into a "job waiting" notification.
  598. """
  599. reasons = [(model, reason) for model, reason in per_model if reason]
  600. if not reasons:
  601. return None
  602. if len(reasons) == 1:
  603. return reasons[0][1]
  604. distinct = list(dict.fromkeys(reason for _model, reason in reasons))
  605. if len(distinct) == 1:
  606. return distinct[0]
  607. if all(PrintScheduler._is_busy_only(reason) for _model, reason in reasons):
  608. return " | ".join(distinct)
  609. return "; ".join(f"{model or 'unassigned'}: {reason}" for model, reason in reasons)
  610. def _candidate_model_label(candidates: list[_ModelCandidate]) -> str | None:
  611. """Human label for the models an item is waiting on ("H2S or H2C").
  612. Notifications take a single target model. For a cross-model item the item's
  613. own ``target_model`` is whichever variant happens to be first, which reads as
  614. a lie once it is the H2C that actually runs — so name all of them.
  615. """
  616. models = list(dict.fromkeys(c.target_model for c in candidates if c.target_model))
  617. if not models:
  618. return None
  619. return " or ".join(models)
  620. def _mapping_is_all_unresolved(mapping: list | None) -> bool:
  621. """True if ``mapping`` is a non-empty list whose every entry is the
  622. unresolved sentinel (-1 / None) — i.e. no required slot ever matched a tray.
  623. Such a mapping is a bug artifact: a frontend status-load race can serialize
  624. ``[-1]`` before the printer's AMS trays are known (#2589). It must be
  625. recomputed from live status at dispatch rather than trusted, otherwise it
  626. reaches the print command and is silently downgraded to external-spool mode.
  627. A partially-resolved mapping (``[-1, -1, 5]`` where slot 3 matched, or a
  628. padding ``-1`` for a slot this plate does not print) is NOT unresolved. An
  629. explicit external selection (``>= 254``) is NOT unresolved either — those
  630. keep their meaning.
  631. """
  632. if not isinstance(mapping, list) or not mapping:
  633. return False
  634. return all(t is None or (isinstance(t, int) and t < 0) for t in mapping)
  635. def _is_tray_id(value: object) -> bool:
  636. """True when ``value`` can be read as a global tray ID.
  637. ``bool`` is a subclass of ``int``, so a hand-written API payload carrying
  638. ``true`` would otherwise be read as tray 1 and judged — or dispatched —
  639. against whatever happens to be loaded there.
  640. """
  641. return isinstance(value, int) and not isinstance(value, bool)
  642. # Prefix of the waiting_reason `_block_on_unmatched_filament` writes when it
  643. # stages an item (#2799). The manual_start branch clears every other reason on
  644. # a staged item (#3074); this one is the reason it was staged, so it stays.
  645. _UNMATCHED_HOLD_PREFIX = "Needs "
  646. # How often an "any model" job held for low filament looks for another printer
  647. # of its model that has enough (#3137). Each look matches its filament on every
  648. # idle printer again, which a 30 s queue pass would otherwise repeat each time.
  649. _SHORT_HOLD_RECHECK_SECONDS = 120
  650. def _is_unmatched_hold_reason(reason: str | None) -> bool:
  651. """True for the reason the unmatched-filament hold wrote when it staged the item."""
  652. return bool(reason) and reason.startswith(_UNMATCHED_HOLD_PREFIX)
  653. def _filament_overrides_of(item: PrintQueueItem) -> list:
  654. """The item's stored filament overrides, or an empty list when there are none or they don't parse."""
  655. try:
  656. overrides = json.loads(item.filament_overrides) if item.filament_overrides else []
  657. except (json.JSONDecodeError, TypeError):
  658. return []
  659. return overrides if isinstance(overrides, list) else []
  660. def _forced_override_requirements(item: PrintQueueItem) -> list[dict]:
  661. """The item's force_color_match overrides as filament requirements (#3280).
  662. What ``_build_override_direct_mapping`` matches when the 3MF cannot be
  663. read, so the checks after it judge the same list.
  664. """
  665. overrides = _filament_overrides_of(item)
  666. return [
  667. {
  668. "slot_id": o.get("slot_id"),
  669. "type": o.get("type", ""),
  670. "color": o.get("color", ""),
  671. "tray_info_idx": o.get("tray_info_idx", ""),
  672. "force_color_match": True,
  673. }
  674. for o in overrides
  675. if isinstance(o, dict) and o.get("force_color_match") and isinstance(o.get("slot_id"), int)
  676. ]
  677. @dataclass(frozen=True)
  678. class FilamentScopeHold:
  679. """``_ensure_ams_mapping``'s answer when the job's creator may not print it as it stands (#3522).
  680. Not a failure: like a printer its creator lost (#1727), the job is held
  681. pending with ``reason`` on its queue row, and starts by itself on the first
  682. pass after the group's rules, the loaded spools or the job allow it.
  683. """
  684. reason: str
  685. def _unresolved_required(required: list[dict], mapping: list) -> list[dict]:
  686. """The requirements in ``required`` that ``mapping`` gives no tray (#2799).
  687. Only slots the plate prints are judged: a ``-1`` anywhere else is padding
  688. for a filament this plate does not use. A slot past the end of the mapping
  689. is unresolved too.
  690. """
  691. unresolved = []
  692. for req in required:
  693. slot_id = req.get("slot_id") or 0
  694. if slot_id <= 0:
  695. continue
  696. tray = mapping[slot_id - 1] if slot_id <= len(mapping) else None
  697. if not _is_tray_id(tray) or tray < 0:
  698. unresolved.append(req)
  699. return unresolved
  700. # Global tray ids at or above this are the external spool(s), not an AMS slot:
  701. # 254 is the deputy feed and 255 the main one. Mirrors the sentinel documented
  702. # on `_mapping_is_all_unresolved`.
  703. _EXTERNAL_TRAY_ID_MIN = 254
  704. def _consumed_mapping_entries(mapping: list | None, required: list[dict] | None) -> list | None:
  705. """The ``mapping`` entries for the slots this plate actually prints.
  706. ``required`` comes from ``extract_filament_requirements``, which drops any
  707. filament with ``used_g <= 0`` — so a slot_id present there is one the plate
  708. consumes, and one absent from it is padding. That distinction is why this
  709. decision lives here and not in the MQTT command builder: a ``-1`` in the
  710. mapping means either "this plate does not print filament N" or "we never
  711. worked out which tray", and only the plate's own filament list separates
  712. them. The builder sees both as the same byte, which is how a plate whose one
  713. printed filament sat on the external spool went out as `use_ams=true` with a
  714. mapping of nothing but -1 and stalled at preheat until the firmware gave up
  715. with 07FF_8012 (#3087).
  716. Returns None whenever the two cannot be lined up — no mapping, no parsed
  717. requirements, or a requirement the mapping is too short to cover — so every
  718. caller falls back to existing behaviour rather than acting on a guess.
  719. """
  720. if not isinstance(mapping, list) or not mapping or not required:
  721. return None
  722. entries = []
  723. for filament in required:
  724. slot_id = filament.get("slot_id")
  725. if not isinstance(slot_id, int) or not 1 <= slot_id <= len(mapping):
  726. # The mapping and the requirements disagree about how many filaments
  727. # the file has. They came from different reads, so judge nothing.
  728. return None
  729. entries.append(mapping[slot_id - 1])
  730. return entries or None
  731. def _plate_uses_only_external(mapping: list | None, required: list[dict] | None, rack: bool = False) -> bool:
  732. """Whether every slot this plate prints is mapped to an external spool (#3087).
  733. Exact, because the plate's own filament list (``required``) tells padding
  734. from a slot that never resolved (see ``_consumed_mapping_entries``). The
  735. dispatch sends it to the command builder, and the filament-permission
  736. check (#3522) judges the feed by it, so both read it the same way.
  737. """
  738. consumed = _consumed_mapping_entries(mapping, required)
  739. return bool(consumed) and all(_is_external_tray(t, rack) for t in consumed or ())
  740. def _is_external_tray(tray_id, rack: bool = False) -> bool:
  741. """True for an explicit external-spool selection (254/255, and 247-253 on a
  742. ``rack`` printer), not for an unresolved slot and not for an AMS tray."""
  743. if tray_id is None:
  744. return False
  745. try:
  746. value = int(tray_id)
  747. except (TypeError, ValueError):
  748. return False
  749. return value >= _EXTERNAL_TRAY_ID_MIN or is_external_tray(value, rack)
  750. def _might_be_dual_nozzle(printer_model: str | None, status) -> bool:
  751. """Whether this printer could have two extruders, judged generously.
  752. On a dual-nozzle printer ``use_ams`` is nozzle routing rather than an
  753. AMS on/off flag — H2D Pro firmware reads it as an extruder index — which is
  754. why the MQTT command builder skips its own use_ams reconcile there. Anything
  755. that might be dual-nozzle therefore keeps whatever ``use_ams`` it arrived
  756. with, external spools or not.
  757. Deliberately over-eager: a wrong "yes" only means this printer keeps the
  758. behaviour it has always had, while a wrong "no" would rewrite a field that
  759. steers which nozzle prints. The model name is the first answer (it is what
  760. the command builder falls back to as well), then the same live evidence the
  761. dispatcher's extruder annotation uses — a second nozzle reporting a
  762. diameter, a populated ``ams_extruder_map``, or more than one external feed,
  763. since a single-nozzle printer has exactly one.
  764. """
  765. if is_dual_nozzle_model(printer_model):
  766. return True
  767. nozzles = getattr(status, "nozzles", None) or []
  768. if len(nozzles) > 1 and getattr(nozzles[1], "nozzle_diameter", ""):
  769. return True
  770. raw = getattr(status, "raw_data", None) or {}
  771. if raw.get("ams_extruder_map"):
  772. return True
  773. vt_trays = raw.get("vt_tray") or []
  774. # Not a nozzle-rack printer's one external input per hotend (H3C, 247-255).
  775. return isinstance(vt_trays, list) and len(vt_trays) > 1 and not TrayLayout.from_units(None, vt_trays).rack
  776. def _int_or(value, default: int) -> int:
  777. """``int(value)``, or ``default`` when the field is missing or junk.
  778. AMS telemetry types its ids inconsistently — `"0"` in one firmware, `0` in
  779. the next — and a tray id that fails to parse must not take the whole
  780. derivation down with it.
  781. """
  782. try:
  783. return int(value)
  784. except (TypeError, ValueError):
  785. return default
  786. # What a queue job's dispatch is worked out from, and an edit can change.
  787. _DISPATCH_INPUTS = (
  788. "printer_id",
  789. "status",
  790. "ams_mapping",
  791. "plate_id",
  792. "filament_overrides",
  793. "skip_filament_check",
  794. # Decides the feed, so it is part of the filament check too (#3522).
  795. "use_ams",
  796. "target_model",
  797. "target_location",
  798. )
  799. async def _edited_since_loaded(db: AsyncSession, item: PrintQueueItem) -> bool:
  800. """Whether ``item`` was changed or removed since ``check_queue`` loaded it.
  801. The pass works on the jobs as loaded at its start, and it may have waited
  802. since: a plug powering a printer on takes minutes. A job edited meanwhile
  803. (moved to another printer, given a mapping or another plate, cancelled)
  804. is left to the next pass, which reads it fresh: deciding from the stale
  805. copy would overwrite the edit, or send the job where it no longer goes.
  806. The query flushes the pass's own pending changes first, so those don't
  807. count as edits.
  808. """
  809. row = (
  810. await db.execute(
  811. select(*(getattr(PrintQueueItem, name) for name in _DISPATCH_INPUTS)).where(PrintQueueItem.id == item.id)
  812. )
  813. ).one_or_none()
  814. if row is not None and all(getattr(item, name) == value for name, value in zip(_DISPATCH_INPUTS, row, strict=True)):
  815. return False
  816. logger.info("Queue item %s changed during this pass; leaving it to the next", item.id)
  817. return True
  818. def _global_tray_id(ams_id: int, tray_id: int, layout: TrayLayout | None = None) -> int:
  819. """Bambu Studio's global tray id (``utils.tray_ids``) of a slot on the printer ``layout`` describes.
  820. Mirrors the calculation in ``_build_loaded_filaments``, which is what
  821. produces the ids stored in ``PrintQueueItem.ams_mapping`` — the two must
  822. agree or a mapping cannot be read back against live tray telemetry.
  823. """
  824. return (layout or TrayLayout()).global_tray_id(ams_id, tray_id)
  825. def _used_global_tray_ids(item: PrintQueueItem | None) -> set[int] | None:
  826. """The global tray ids ``item`` actually prints from, or None if unknown.
  827. ``ams_mapping`` is the array the print command carries: position = filament
  828. slot, value = global tray id, ``-1`` / ``None`` for a slot this plate does
  829. not use. None means "no usable statement" — no mapping, unparseable JSON,
  830. an all-unresolved mapping (the artifact ``_mapping_is_all_unresolved``
  831. documents), or one that resolves to no tray at all. Callers must treat None
  832. as "consider every loaded tray" rather than "consider none": narrowing on
  833. an absent mapping would silently drop requirements the print really has.
  834. """
  835. raw = getattr(item, "ams_mapping", None)
  836. if not raw:
  837. return None
  838. if isinstance(raw, str):
  839. try:
  840. mapping = json.loads(raw)
  841. except (json.JSONDecodeError, TypeError):
  842. return None
  843. else:
  844. mapping = raw
  845. if not isinstance(mapping, list) or _mapping_is_all_unresolved(mapping):
  846. return None
  847. used = {t for t in mapping if isinstance(t, int) and not isinstance(t, bool) and t >= 0}
  848. return used or None
  849. def _mqtt_commands_rejected(status) -> bool:
  850. """True when the printer is currently reporting that it refused a command.
  851. ``HMS_MQTT_VERIFY_FAILED`` means the firmware's authorization check rejected
  852. a control command it could not verify. Queries still answer, so the printer
  853. looks connected and idle while project_file, gcode_line and
  854. ams_change_filament are all dropped — no amount of waiting or re-uploading
  855. changes that (#2732).
  856. Tolerates a missing status and errors without a ``full_code`` (the 8-char
  857. ``print_error`` path builds HMSError differently), so this is safe to call on
  858. every watchdog poll.
  859. """
  860. for err in getattr(status, "hms_errors", None) or []:
  861. if getattr(err, "full_code", "") == HMS_MQTT_VERIFY_FAILED:
  862. return True
  863. return False
  864. def _drying_ams_ids(status) -> list[int]:
  865. """AMS unit ids currently running a drying cycle, per firmware telemetry.
  866. ``dry_time`` is minutes remaining, so >0 is the firmware's own statement that
  867. a cycle is active. Used by the dispatch watchdog to say *why* a print never
  868. started (#2758) — it is a diagnostic, not a gate.
  869. Deliberately not used to block or stop drying before dispatch. This printer
  870. class supports drying concurrently with an active print
  871. (``supports_drying_while_printing``), so drying is not incompatible with
  872. printing in general; what #2758 shows is one X2D refusing to *begin* a print
  873. while two AMS units were drying, one of them without its external PSU. Until
  874. it is known whether the blocker is drying itself or the power budget
  875. (``dry_sf_reason`` 1 / 8), acting on this would tear down drying that the
  876. hardware is perfectly happy to continue.
  877. """
  878. ids: list[int] = []
  879. for unit in (getattr(status, "raw_data", None) or {}).get("ams") or []:
  880. if not isinstance(unit, dict):
  881. continue
  882. try:
  883. if int(unit.get("dry_time") or 0) > 0:
  884. ids.append(int(unit.get("id", 0)))
  885. except (TypeError, ValueError):
  886. continue
  887. return ids
  888. def _parse_diameter(raw) -> float | None:
  889. """``"0.4"`` → ``0.4``; anything unparseable or non-positive → None."""
  890. try:
  891. value = float(raw)
  892. except (TypeError, ValueError):
  893. return None
  894. return value if value > 0 else None
  895. def _nozzle_info_by_id(status) -> dict[int, dict]:
  896. """Index ``PrinterState.nozzle_rack`` by nozzle id.
  897. The field name is historical: on the H2 series ``nozzle_info`` carries an
  898. entry for *every* nozzle the printer knows about — the L/R hotends under
  899. ids 0/1 and, on a rack model, the dock positions under ids 16-21. Only the
  900. latter are the rack proper; :func:`_rack_nozzle_diameters` and
  901. :func:`_installed_nozzle_diameters` each take the half they need.
  902. """
  903. by_id: dict[int, dict] = {}
  904. for entry in getattr(status, "nozzle_rack", None) or []:
  905. if not isinstance(entry, dict):
  906. continue
  907. try:
  908. by_id[int(entry.get("id"))] = entry
  909. except (TypeError, ValueError):
  910. continue
  911. return by_id
  912. # What the firmware puts in a nozzle's serial number when the carriage or dock
  913. # is empty. Measured on an H2C at idle, where the rack-side hotend had parked
  914. # its nozzle back in the rack (#2885).
  915. _EMPTY_NOZZLE_SERIAL = "N/A"
  916. def _nozzle_is_mounted(entry: dict | None) -> bool:
  917. """Whether a ``nozzle_info`` entry describes hardware that is actually there.
  918. A hotend entry is reported whether or not a nozzle is mounted, and an empty
  919. carriage keeps the *last* nozzle's diameter — measured on an H2C at idle,
  920. where id 1 read ``diameter "0.4"`` with ``max_temp 0``, ``serial_number
  921. "N/A"`` and ``wear 0`` because that hotend had parked its nozzle back in
  922. the dock. So the diameter is not a presence signal (#2885).
  923. Emptiness has to be stated, not merely unstated. The serial must be the
  924. firmware's explicit ``"N/A"`` marker *and* the temperature rating must be
  925. absent — an empty serial only means the printer didn't say, and a firmware
  926. that reports neither field normalises to exactly that. Treating "didn't
  927. say" as "empty" would silently switch the #1899 guard off on any such
  928. machine, so everything we cannot positively call empty counts as mounted.
  929. """
  930. if entry is None:
  931. return True
  932. serial = str(entry.get("serial_number") or "").strip().upper()
  933. if serial != _EMPTY_NOZZLE_SERIAL:
  934. return True
  935. try:
  936. max_temp = float(entry.get("max_temp") or 0)
  937. except (TypeError, ValueError):
  938. return True
  939. return max_temp > 0
  940. def _installed_nozzle_diameters(status) -> list[float]:
  941. """Parse the mounted nozzle diameters from a PrinterState (#1899).
  942. Returns the diameters the printer actually reports (e.g. [0.4] single-nozzle,
  943. [0.4, 0.6] dual-nozzle), skipping the empty-string defaults that populate a
  944. NozzleInfo before MQTT fills it in. An empty list means "the printer hasn't
  945. told us its nozzle hardware" — callers must treat that as unknown, not as a
  946. mismatch, so we never block a print on missing data.
  947. A hotend whose ``nozzle_info`` entry says nothing is mounted is skipped even
  948. though ``nozzles`` still carries a diameter for it: that value is stale, and
  949. counting it would let a slice match a nozzle the machine does not have
  950. (#2885). Printers that report no ``nozzle_info`` at all are unaffected.
  951. """
  952. info = _nozzle_info_by_id(status)
  953. diameters: list[float] = []
  954. for index, nozzle in enumerate(getattr(status, "nozzles", None) or []):
  955. raw = getattr(nozzle, "nozzle_diameter", "") or ""
  956. try:
  957. value = float(raw)
  958. except (TypeError, ValueError):
  959. continue
  960. if value > 0 and _nozzle_is_mounted(info.get(index)):
  961. diameters.append(value)
  962. return diameters
  963. def _rack_nozzle_diameters(status) -> list[float]:
  964. """Diameters sitting in the tool-changer rack, nearest dock first (#2885).
  965. Keyed off the nozzle ids themselves rather than the printer model: only a
  966. rack machine ever reports ids 16-21, so there is no model registry to keep
  967. in sync. An empty dock is simply absent from the payload — measured on an
  968. H2C whose R2 (id 17) was empty and unlisted while R1/R3/R4/R5/R6 were all
  969. present — so appearing here already means "a nozzle is in that dock".
  970. ``stat`` is deliberately not interpreted: its values aren't known, and
  971. reading it wrongly could hide a nozzle the printer would happily fetch.
  972. """
  973. diameters: list[float] = []
  974. for nozzle_id, entry in sorted(_nozzle_info_by_id(status).items()):
  975. if nozzle_id not in _RACK_NOZZLE_IDS:
  976. continue
  977. # PrinterState spells it "diameter"; the REST schema renames it to
  978. # "nozzle_diameter". Accept either so a caller holding the serialised
  979. # shape gets the same answer.
  980. value = _parse_diameter(entry.get("diameter") or entry.get("nozzle_diameter"))
  981. if value is not None:
  982. diameters.append(value)
  983. return diameters
  984. def _format_diameters(diameters: list[float]) -> str:
  985. """``[0.4, 0.4, 0.6]`` → ``"0.4mm / 0.6mm"``, in first-seen order.
  986. Deduplicated because a loaded rack holds several nozzles of the same size,
  987. and "0.4mm / 0.4mm / 0.4mm / 0.6mm / 0.2mm" tells the reader nothing the
  988. short form doesn't. Matching still runs over the full list.
  989. """
  990. return " / ".join(f"{d:g}mm" for d in dict.fromkeys(diameters))
  991. def _nozzle_mismatch_message(
  992. sliced_nozzle: float | None,
  993. installed: list[float],
  994. rack: list[float] | None = None,
  995. ) -> str | None:
  996. """Return an actionable error message when the sliced nozzle can't be
  997. printed on any nozzle the machine can reach, else None (#1899).
  998. Fail-safe: returns None whenever we lack the data to judge — no sliced
  999. diameter, or the printer reported no nozzles — so a print is only ever
  1000. blocked on a POSITIVE mismatch. On dual-nozzle printers a match against
  1001. EITHER installed nozzle passes (a 0.6 slice is fine if one hotend is 0.6).
  1002. The 0.05 tolerance absorbs float noise while staying well inside the 0.2
  1003. gap between adjacent nozzle sizes (0.2/0.4/0.6/0.8).
  1004. *rack* holds the diameters parked in a tool-changer dock (H2C). Those count
  1005. as reachable: the printer fetches one as part of starting the print, so a
  1006. slice that matches a docked nozzle is not a mismatch. Without this the
  1007. guard blocked every job whose nozzle happened not to be on a hotend at
  1008. dispatch time — on a rack loaded with 0.2/0.4/0.6 that meant only the
  1009. diameter already mounted could ever print, and the user had to fetch the
  1010. nozzle by hand on the printer's own UI first (#2885).
  1011. """
  1012. reachable = [*installed, *(rack or [])]
  1013. if not sliced_nozzle or not reachable:
  1014. return None
  1015. if any(abs(d - sliced_nozzle) < 0.05 for d in reachable):
  1016. return None
  1017. where = f"{_format_diameters(installed)} installed" if installed else "no nozzle mounted"
  1018. if rack:
  1019. where += f" and {_format_diameters(rack)} in the nozzle rack"
  1020. return (
  1021. f"File sliced for a {sliced_nozzle:g}mm nozzle, but the printer has "
  1022. f"{where}. Re-slice for an available nozzle, or fit the matching "
  1023. f"nozzle before printing."
  1024. )
  1025. def _wrong_nozzle_label(printer_name: str, status, sliced_nozzle: float | None) -> str | None:
  1026. """Label of a printer that cannot print a *sliced_nozzle* file, else None (#3536).
  1027. Used by the "Any <model>" pick, which before #3536 ignored the nozzle and
  1028. handed a 0.6 file to a 0.4 printer that then failed in ``_start_print``.
  1029. None means "fits or unknown" (no status, no sliced diameter, no reported
  1030. nozzles), so a printer is only ever skipped on a POSITIVE mismatch. The
  1031. decision stays in ``_nozzle_mismatch_message`` (same tolerance, same rack
  1032. rule as the start guard); this only formats the name for the wait reason.
  1033. """
  1034. if status is None:
  1035. return None
  1036. installed = _installed_nozzle_diameters(status)
  1037. rack = _rack_nozzle_diameters(status)
  1038. if _nozzle_mismatch_message(sliced_nozzle, installed, rack) is None:
  1039. return None
  1040. where = _format_diameters(installed) if installed else "no nozzle mounted"
  1041. if rack:
  1042. where += f", rack {_format_diameters(rack)}"
  1043. return f"{printer_name} ({where})"
  1044. def _describe_filament(entry: dict, nozzle_key: str) -> str:
  1045. """One-line "PETG #000000 (left nozzle)" for an error message (#2771).
  1046. Shared by the required and loaded sides, which name their extruder
  1047. differently: a 3MF requirement carries ``nozzle_id``, a loaded tray carries
  1048. ``extruder_id``. Both are MQTT extruder ids — 0 is the right/main nozzle,
  1049. 1 the left/deputy — and both are absent on single-nozzle printers, where
  1050. naming a nozzle would be noise.
  1051. """
  1052. parts = [(entry.get("type") or "filament").upper()]
  1053. if entry.get("color"):
  1054. parts.append(str(entry["color"]))
  1055. nozzle = entry.get(nozzle_key)
  1056. if nozzle == 0:
  1057. parts.append("(right nozzle)")
  1058. elif nozzle == 1:
  1059. parts.append("(left nozzle)")
  1060. return " ".join(parts)
  1061. def _unmatched_filament_message(required: list[dict], loaded: list[dict]) -> str:
  1062. """Explain that nothing loaded matches what the file needs (#2771).
  1063. Only ever built for a printer with no AMS, where the loaded list is short
  1064. enough to quote in full and there is no "load another spool and hit Resume"
  1065. recovery — the external spool holder is all there is, so the user needs to
  1066. be told which filament to put on it.
  1067. """
  1068. want = ", ".join(_describe_filament(r, "nozzle_id") for r in required)
  1069. have = ", ".join(_describe_filament(f, "extruder_id") for f in loaded)
  1070. return (
  1071. f"No filament loaded on this printer matches the file. It needs {want}; "
  1072. f"the printer has {have} and no AMS. Load the required filament on the "
  1073. f"external spool holder, or send this job to a printer that has it."
  1074. )
  1075. async def _queued_plate_bed_temperature(item: PrintQueueItem, archive: PrintArchive | None) -> int:
  1076. """Bed temperature for the plate a queued job prints, 0 when unknown.
  1077. The archive stores the highest bed temperature of any filament in the
  1078. project, while the job may print another plate than the one archived: a
  1079. PLA plate of a project that also holds PETG is heated for PLA. Read from
  1080. the archive's file for the job's own plate, falling back to the stored
  1081. value when the file can't be read.
  1082. """
  1083. stored = int(archive.bed_temperature) if archive and archive.bed_temperature else 0
  1084. if archive is None or not archive.file_path:
  1085. return stored
  1086. temperature = await asyncio.to_thread(plate_bed_temperature, settings.base_dir / archive.file_path, item.plate_id)
  1087. return temperature or stored
  1088. def _effective_plate_id(explicit_plate_id: int | None, file_path: Path) -> int:
  1089. """The plate to dispatch, resolved once in ``_start_print`` and reused at
  1090. every call site below it: G-code injection, usage registration, rack-plan
  1091. lookup, slot-extruder lookup, the external-spool check, and the actual
  1092. print command.
  1093. A positive explicit ``plate_id`` on the queue item always wins. A
  1094. non-positive one is treated as "not set" and resolved from the archive,
  1095. the way the rest of the queue code already reads it (``if item.plate_id:``
  1096. in ``api/routes/print_queue.py``): the schemas put no lower bound on the
  1097. field, and passing a 0 straight through would build a print command for
  1098. ``Metadata/plate_0.gcode``, which is the same wedge this function exists
  1099. to prevent. The ``item.plate_id or 1`` this replaced mapped 0 to 1.
  1100. Falling back to a bare ``1`` instead of reading the archive assumes a
  1101. single-plate file's one G-code is numbered 1, which only holds for a
  1102. plate exported on its own: one cut out of a larger project keeps its
  1103. ORIGINAL plate number, so a printer asked to print "plate 1" of a file
  1104. whose only G-code is ``plate_2.gcode`` accepts the command, can't find
  1105. the file, throws an HMS error, and sits wedged in IDLE until
  1106. power-cycled (#2947).
  1107. The call sites agreed on a fallback only by accident before this:
  1108. with G-code injection on, ``inject_gcode_into_3mf`` already falls back to
  1109. the archive's own default plate internally whenever the plate id it's
  1110. handed isn't in the file, so it could silently inject into a different
  1111. plate than the one the print command itself asked for.
  1112. Falls back to 1 when the archive can't be read, holds no G-code member at
  1113. all, or its default member doesn't follow the ``plate_N`` naming
  1114. convention (a slicer that doesn't use it has no number to dispatch).
  1115. An explicit plate the archive doesn't hold is logged and then sent
  1116. anyway. It wedges the printer exactly like #2947 did, but redirecting it
  1117. to a plate that is in the file would print a model nobody asked for,
  1118. which is the worse of the two.
  1119. """
  1120. try:
  1121. with zipfile.ZipFile(file_path, "r") as zf:
  1122. names = zf.namelist()
  1123. except (OSError, zipfile.BadZipFile) as exc:
  1124. logger.warning(
  1125. "Dispatch plate: cannot read %s (%s), so the archive's own plate numbering "
  1126. "is unavailable; dispatching plate %s",
  1127. file_path,
  1128. exc,
  1129. explicit_plate_id if explicit_plate_id is not None and explicit_plate_id > 0 else 1,
  1130. )
  1131. names = None
  1132. if explicit_plate_id is not None and explicit_plate_id > 0:
  1133. if (
  1134. names is not None
  1135. and default_plate_number(names) is not None
  1136. and select_plate_gcode_name(names, explicit_plate_id) is None
  1137. ):
  1138. logger.warning(
  1139. "Dispatch plate: %s was queued for plate %s but holds no G-code for it "
  1140. "(it has %s). Sending plate %s as asked; expect the printer to reject the "
  1141. "file, since printing a different plate would print the wrong model (#2947)",
  1142. file_path,
  1143. explicit_plate_id,
  1144. ", ".join(sorted(n for n in names if n.endswith(".gcode"))),
  1145. explicit_plate_id,
  1146. )
  1147. return explicit_plate_id
  1148. resolved = default_plate_number(names) if names is not None else None
  1149. return resolved if resolved is not None else 1
  1150. def _wear_order(printer: Printer) -> tuple[float, int]:
  1151. """Sort key for printers a model job could run on: fewest print hours, then id (#3480).
  1152. The same hours as maintenance's ``get_printer_total_hours``: the runtime
  1153. counter plus the baseline offset a user set for a second-hand or reset
  1154. machine.
  1155. """
  1156. hours = (printer.runtime_seconds or 0) / 3600.0 + (printer.print_hours_offset or 0.0)
  1157. return hours, printer.id
  1158. def _hours_of(printer: Printer) -> float:
  1159. return (printer.runtime_seconds or 0) / 3600.0 + (printer.print_hours_offset or 0.0)
  1160. def _order_printers(
  1161. printers: list[Printer], order: str, last_finished: dict[int, datetime] | None = None
  1162. ) -> list[Printer]:
  1163. """Sort the printers an "Any model" job could take into the order they are picked (#2915).
  1164. Whatever the strategy, ties fall back to the printer id, so the outcome is
  1165. reproducible.
  1166. * ``fewest_hours`` (the default, #3480): the least worn machine first.
  1167. * ``most_hours``: the most worn first, to use up the machines already near
  1168. replacement.
  1169. * ``least_recently_used``: the printer that finished a print longest ago;
  1170. one that never has goes first.
  1171. * ``priority``: the printers' own ``dispatch_priority``, lowest first; one
  1172. without goes last.
  1173. """
  1174. if order == "most_hours":
  1175. return sorted(printers, key=lambda p: (-_hours_of(p), p.id))
  1176. if order == "least_recently_used":
  1177. finished = last_finished or {}
  1178. never = datetime.min
  1179. return sorted(printers, key=lambda p: (finished.get(p.id, never), p.id))
  1180. if order == "priority":
  1181. return sorted(
  1182. printers,
  1183. key=lambda p: (
  1184. p.dispatch_priority is None,
  1185. p.dispatch_priority if p.dispatch_priority is not None else 0,
  1186. p.id,
  1187. ),
  1188. )
  1189. return sorted(printers, key=_wear_order)
  1190. class PrintScheduler:
  1191. """Background scheduler that processes the print queue."""
  1192. # Built-in drying presets per filament type (from BambuStudio filament profiles)
  1193. # Format: { n3f_temp, n3s_temp, n3f_hours, n3s_hours }
  1194. DEFAULT_DRYING_PRESETS: dict[str, dict[str, int]] = {
  1195. "PLA": {"n3f": 45, "n3s": 45, "n3f_hours": 12, "n3s_hours": 12},
  1196. "PETG": {"n3f": 65, "n3s": 65, "n3f_hours": 12, "n3s_hours": 12},
  1197. "TPU": {"n3f": 65, "n3s": 75, "n3f_hours": 12, "n3s_hours": 18},
  1198. "ABS": {"n3f": 65, "n3s": 80, "n3f_hours": 12, "n3s_hours": 8},
  1199. "ASA": {"n3f": 65, "n3s": 80, "n3f_hours": 12, "n3s_hours": 8},
  1200. "PA": {"n3f": 65, "n3s": 85, "n3f_hours": 12, "n3s_hours": 12},
  1201. "PC": {"n3f": 65, "n3s": 80, "n3f_hours": 12, "n3s_hours": 8},
  1202. "PVA": {"n3f": 65, "n3s": 85, "n3f_hours": 12, "n3s_hours": 18},
  1203. }
  1204. def __init__(self):
  1205. self._running = False
  1206. self._check_interval = 30 # seconds
  1207. # After a pass that actually dispatched something, loop again almost
  1208. # immediately instead of sleeping the full interval (#2555). A dispatch
  1209. # changes printer state — a batch launch fans out over several passes as
  1210. # printers free up, a wedged head-of-line job reverts to pending, an
  1211. # upload slot opens — and the next batch of ready work should not have to
  1212. # wait 30 s behind an idle sleep. When a pass dispatches nothing (all
  1213. # pending items are behind printers that are genuinely busy printing),
  1214. # there is nothing to react to, so we fall back to the normal interval;
  1215. # that also means this can never tight-loop, since fast ticks only
  1216. # continue while dispatches keep happening and the queue is draining.
  1217. self._fast_check_interval = 3 # seconds
  1218. self._power_on_wait_time = 180 # seconds to wait for printer after power on (3 min)
  1219. self._power_on_check_interval = 10 # seconds between connection checks
  1220. # Printers whose class-target power-on failed, mapped to the monotonic
  1221. # time their cool-off expires (#2786).
  1222. #
  1223. # Without this, one printer with an unreachable plug starves every
  1224. # sibling of its model forever: the wake step walks candidates in id
  1225. # order, spends the pass's single attempt on the same broken printer
  1226. # every time, and the healthy one two slots down is never reached. It
  1227. # also costs a full ``_power_on_wait_time`` out of every 30 s pass,
  1228. # which delays the whole queue, not just this job.
  1229. #
  1230. # Entries expire on read rather than being cleared on success: a printer
  1231. # inside its cool-off is skipped before the power-on is reached, so a
  1232. # live entry can never be overwritten by a success anyway. A printer
  1233. # that comes back by any other route stops being a wake candidate the
  1234. # moment it connects.
  1235. self._wake_failures: dict[int, float] = {}
  1236. self._wake_failure_cooloff = 600 # seconds
  1237. # Printers whose last upload never reached them, mapped to the monotonic
  1238. # time they may be dispatched to again (#3210). Same expire-on-read shape
  1239. # as `_wake_failures`. Without it, a printer that cannot take files is
  1240. # idle on every pass, so it is picked on every pass.
  1241. self._upload_backoff: dict[int, float] = {}
  1242. # Refused uploads in a row per printer, which sets the next backoff
  1243. # window. Cleared by a successful upload to that printer.
  1244. self._upload_refusals: dict[int, int] = {}
  1245. # Why each printer in `_upload_backoff` refused its last upload, for
  1246. # the reason its jobs show. Cleared with `_upload_refusals`.
  1247. self._upload_refusal_kind: dict[int, FtpFailureKind] = {}
  1248. # The "job waiting" text last sent for each printer's current run of
  1249. # refusals. A refusal notifies only when its text differs, so a run
  1250. # notifies once, and again only if it turns into something the user
  1251. # has to fix. Cleared with `_upload_refusals`.
  1252. self._upload_refusal_notified: dict[int, str] = {}
  1253. # Track which printers are currently auto-drying (printer_id -> start timestamp)
  1254. self._drying_in_progress: dict[int, float] = {}
  1255. # Per-AMS memory of the auto-drying cycles WE armed, keyed by
  1256. # (printer_id, ams_id) (#2770). Entries only exist between arming a
  1257. # cycle and the humidity finally coming down, so the normal steady
  1258. # state is an empty dict. Fields:
  1259. # running — a cycle we armed is (or should be) on the firmware
  1260. # ended_at — monotonic when we observed that cycle end
  1261. # unproductive — consecutive armed cycles that ended with the reading
  1262. # still above the threshold
  1263. # suspended — we have stopped arming this unit and said so
  1264. self._auto_dry_units: dict[tuple[int, int], dict[str, object]] = {}
  1265. # Sustained-humidity streaks for the ambient-drying wait (#2518). Keyed
  1266. # like _auto_dry_units but deliberately a SEPARATE dict: membership in
  1267. # _auto_dry_units means "Bambuddy armed a cycle on this unit", and the
  1268. # print-takes-priority stop, the manual-cycle adoption guard, and the
  1269. # arming setdefault all act on that meaning -- a unit that is merely
  1270. # waiting out its streak must not become stoppable or judgeable.
  1271. # since -- monotonic stamp when the current continuous above-threshold
  1272. # streak began
  1273. # last -- monotonic stamp of the last pass that observed the streak
  1274. self._auto_dry_above: dict[tuple[int, int], dict[str, float]] = {}
  1275. # Slots already notified as low on filament:
  1276. # {(printer_id, ams_id, tray_id, spool_id)}.
  1277. # Cleared when the slot goes back above its threshold rather than on a
  1278. # timer, mirroring _notified_hms_errors — a spool hovering at the
  1279. # boundary must not produce an alert on every pass, and a slot that has
  1280. # been refilled has to be able to alert again. See #2913.
  1281. #
  1282. # The spool id is part of the key because the slot alone cannot clear
  1283. # itself. Pull a low spool and put a different part-used one in the same
  1284. # slot: the key survives the pass where the slot resolves to nothing --
  1285. # deliberately, so a brief Spoolman outage does not re-alert everything
  1286. # when it returns -- and the replacement never goes above the threshold,
  1287. # so it can never re-arm. Keyed with the spool, the new spool is simply a
  1288. # different key and the stale one is inert.
  1289. self._notified_filament_low: set[tuple[int, int, int, int]] = set()
  1290. # The same, for the MQTT relay's filament/low event: its own record, so
  1291. # one channel having been told never silences the other.
  1292. self._relayed_filament_low: set[tuple[int, int, int, int]] = set()
  1293. # Earliest monotonic time the next low-filament check may run (#2913).
  1294. self._filament_low_next_check: float = 0.0
  1295. # SKU key -> (condition, events told) for a SKU that is alerting (#2955): the
  1296. # condition is "reorder" or "break", the events are those that had a
  1297. # subscriber when it was sent. Absent means not alerting. Cleared when the
  1298. # condition clears, so it can alert again, and when nobody wants either
  1299. # event. Mirrored to one settings row (_STOCK_ALERTS_SETTING_KEY), because the
  1300. # first check runs as soon as Bambuddy starts and a restart would otherwise
  1301. # re-send one message per SKU that is still low.
  1302. self._notified_stock_alerts: dict[stock_forecast.SkuKey, tuple[str, frozenset[str]]] = {}
  1303. # The JSON last read from or written to that row; None until it has been read.
  1304. self._stock_alerts_persisted: str | None = None
  1305. # Earliest monotonic time the next stock forecast check may run (#2955).
  1306. self._stock_forecast_next_check: float = 0.0
  1307. # Printers with a "running" scheduled drying row (#2638). Rebuilt from the
  1308. # DB on every _check_scheduled_dryings call so route-side cancels show up.
  1309. # Auto-drying's stop-all branches must not stop or untrack these printers;
  1310. # both features share _drying_in_progress.
  1311. self._scheduled_drying_printer_ids: set[int] = set()
  1312. # Monotonic stamp of the last scheduled-drying prune. None = never, so
  1313. # the first pass after a restart reaps anything left behind.
  1314. self._last_scheduled_drying_prune: float | None = None
  1315. # Defensive in-memory dispatch hold (#1157): a printer that just received
  1316. # a project_file command must not get a second dispatch until either it
  1317. # transitions out of pre_state OR the hard timeout expires. The H2D Pro
  1318. # can take 80–210 s to flip FINISH→PREPARE after project_file, and
  1319. # during that window the DB busy_printers seed is empirically unreliable
  1320. # (multi-plate batches double-/triple-dispatched onto the same printer
  1321. # 30 s apart). Keyed by printer_id; cleared by the watchdog on success
  1322. # or revert.
  1323. # printer_id -> (monotonic_started_at, pre_state, pre_subtask_id)
  1324. self._dispatch_holds: dict[int, tuple[float, str, str | None]] = {}
  1325. # Minimum cooldown between dispatches to the same printer (covers the
  1326. # H2D's project_file digestion window).
  1327. self._dispatch_min_cooldown = 60.0
  1328. # Hard timeout — drop the hold even if we never observed a transition,
  1329. # so a lost MQTT session can't lock a printer out of the queue forever.
  1330. # Matches the watchdog timeout (90 s) plus a safety margin so the
  1331. # watchdog runs first on the unhappy path.
  1332. self._dispatch_max_hold = 180.0
  1333. # Refillable upload pool (#2602). Items whose FTP upload was launched by
  1334. # an earlier pass and is still running. `_start_print` flips the row
  1335. # pending -> printing only *after* the upload completes, so until then
  1336. # the row stays `pending`: each tick, check_queue excludes these
  1337. # item_ids from re-selection and their printers from new dispatch /
  1338. # auto-drying, and launches only `limit - len(_inflight)` new uploads so
  1339. # freed slots refill on the next fast tick. check_queue is the sole,
  1340. # sequential caller and the prune done-callbacks run in the same
  1341. # event-loop thread, so this dict needs no lock.
  1342. # item_id -> (task, printer_id)
  1343. self._inflight: dict[int, tuple[asyncio.Task, int | None]] = {}
  1344. # The archive each dispatch in flight prints into, held in
  1345. # services/archive_work so a merge leaves it alone until the dispatch
  1346. # ends: item_id -> archive_id. Same threading as above.
  1347. self._held_archives: dict[int, int] = {}
  1348. # Expected prints registered by `_start_print` that have not yet had a
  1349. # print command sent. Populated at registration, dropped once
  1350. # `start_print()` succeeds, and rolled back by `_dispatch_one` on every
  1351. # other exit. Same threading argument as `_inflight` above: one
  1352. # sequential caller, callbacks on the same loop, so no lock.
  1353. # item_id -> (printer_id, remote_filename, archive_id)
  1354. self._unconfirmed_expected_print: dict[int, tuple[int, str, int]] = {}
  1355. # Budget reservations created for a dispatch whose print command has
  1356. # not been confirmed yet. `_dispatch_one` releases these on every
  1357. # unsuccessful exit; a successful start removes the item id and leaves
  1358. # the reservation for finance_billing to consume with the archive.
  1359. self._unconfirmed_budget_reservations: set[int] = set()
  1360. # The print user `_start_print` handed to printer_manager for an item
  1361. # whose print command has not been sent yet (item_id -> printer_id).
  1362. # `_dispatch_one` takes it back on every unsuccessful exit, like the
  1363. # expected print above: it marks the printer's next print as Bambuddy's.
  1364. self._unconfirmed_print_user: dict[int, int] = {}
  1365. # Prints whose command went out but which the printer has not been
  1366. # seen starting yet: item_id -> (printer_id, remote_filename, archive_id),
  1367. # the file and archive None for a library print. The watchdog drops the
  1368. # entry once the print runs, and withdraws what it stands for when the
  1369. # print never starts (see `_forget_unstarted_print`).
  1370. self._sent_unstarted_print: dict[int, tuple[int, str | None, int | None]] = {}
  1371. # Chamber temperature history for smart soak-time reduction.
  1372. # printer_id -> deque of (monotonic_timestamp, celsius) sampled each scheduler tick.
  1373. # Entries older than _chamber_history_ttl are pruned on write.
  1374. self._chamber_history: dict[int, deque[tuple[float, float]]] = {}
  1375. self._chamber_history_ttl = _CHAMBER_HISTORY_TTL_SECONDS
  1376. # Per-printer keep-warm state (see `_KeepWarmEntry` at module top).
  1377. # Populated on engagement in `_apply_keep_warm`, cleared by
  1378. # `_sweep_keep_warm` when the printer leaves the candidate set (or
  1379. # when a gate setting toggles off mid-hold — the release publishes
  1380. # bed → 0 first).
  1381. self._keep_warm: dict[int, _KeepWarmEntry] = {}
  1382. # Preheat rollback registry: printer_id -> subset of
  1383. # {"bed", "chamber", "airduct"} listing which preheat commands
  1384. # actually fired for the in-flight dispatch. `_dispatch_one` unwinds
  1385. # every entry still present at exit unless the print successfully
  1386. # started, so a failed upload / cancel / exception never leaves the
  1387. # printer heating for a job that isn't happening.
  1388. self._preheat_pin: dict[int, set[str]] = {}
  1389. # Bed target (°C) that the pinned "bed" entry above actually set, so the
  1390. # rollback can tell its own target from one someone else has since
  1391. # chosen — the same guard `_release_keep_warm` applies to a keep-warm
  1392. # hold. Written wherever `"bed"` joins the pin, evicted alongside it.
  1393. self._preheat_pin_bed: dict[int, int] = {}
  1394. # Item ids whose in-flight dispatch has been cancelled or deleted while
  1395. # its preheat was still holding at temperature. Set by
  1396. # `notify_dispatch_cancelled` from the queue routes, consumed by
  1397. # `_preheat_sleep`, and cleared when the dispatch exits.
  1398. self._cancelled_dispatches: set[int] = set()
  1399. # printer_id -> monotonic time it was first seen terminal while one of
  1400. # its queue rows was still 'printing'. Reset by any non-terminal
  1401. # observation, so it measures an unbroken run rather than a total.
  1402. # In-memory on purpose: a restart re-arms the grace period, which only
  1403. # delays a recovery that is already the exceptional path (#2829).
  1404. self._terminal_since: dict[int, float] = {}
  1405. # Per-pass memo for `_get_filament_requirements` (#2799 review). Two
  1406. # gates parse the same 3MF per dispatch attempt, and a one-file fan-out
  1407. # across idle printers repeats that for every one of them in a single
  1408. # pass. Cleared at the top of each pass so a re-sliced file is never
  1409. # served from a previous tick.
  1410. self._filament_req_memo: dict[tuple, list[dict] | None] = {}
  1411. # Per-pass memo for `_unmatched_on` (#3280): choosing a printer asks the
  1412. # matcher about every idle candidate, and a pass can ask again for the
  1413. # same job and printer (another variant, a previous-print retry, the
  1414. # low-filament move). Cleared with the requirement memo.
  1415. self._unmatched_memo: dict[tuple, list[str]] = {}
  1416. # Per-pass memo for `_creator_filament_scope` (#3522): choosing a
  1417. # printer for an "any model" job asks the matcher about every idle
  1418. # candidate, and each asks for the creator's filament scope. Cleared
  1419. # with the requirement memo, so a rule changed between passes applies
  1420. # on the next one.
  1421. self._filament_scope_memo: dict[int, FilamentScope] = {}
  1422. # When each "any model" job held for low filament last looked for
  1423. # another printer (#3137), by queue item id.
  1424. self._short_hold_checked_at: dict[int, float] = {}
  1425. async def run(self):
  1426. """Main loop - check queue every interval."""
  1427. self._running = True
  1428. logger.info("Print scheduler started")
  1429. await self._clear_stale_dispatch_claims(at_startup=True)
  1430. while self._running:
  1431. dispatched = False
  1432. try:
  1433. self._sample_chamber_temps()
  1434. # No-op while any upload is in flight; on a quiet tick it releases
  1435. # a claim whose best-effort clear failed (e.g. the database was
  1436. # briefly unreachable), instead of leaving the row wedged until
  1437. # the next restart.
  1438. await self._clear_stale_dispatch_claims()
  1439. await self._close_stranded_printing_items()
  1440. dispatched = await self.check_queue()
  1441. except Exception as e:
  1442. logger.error("Scheduler error: %s", e)
  1443. # Re-check quickly after a productive pass so a draining batch does
  1444. # not stall behind the idle interval; otherwise sleep normally (#2555).
  1445. await asyncio.sleep(self._fast_check_interval if dispatched else self._check_interval)
  1446. async def _close_stranded_printing_items(self) -> None:
  1447. """Close a ``printing`` row the completion event never closed (#2829).
  1448. ``on_print_complete`` refuses to close a row when the completion's
  1449. subtask name disagrees with the file the row was dispatched with, so a
  1450. completion meant for something else (the printer's own
  1451. ``auto_pa_line_calib_mode`` run, say) cannot end someone's job early.
  1452. The refusal has no way back, though: nothing else ever closes the row,
  1453. and ``check_queue`` treats every ``printing`` row as a busy printer, so
  1454. one bad comparison wedges that printer's queue until a human presses
  1455. cancel. That is what #2829's reporters hit, and the guard's own
  1456. docstring already called stranding the worse of the two failures.
  1457. This is the way back. When a row has been ``printing`` while its
  1458. printer sat in a terminal state for the whole grace period, the print
  1459. is over however the event was read, and the row is closed with the
  1460. status the printer's own state implies.
  1461. Deliberately conservative:
  1462. * Only a connected printer counts. A disconnected one has a stale
  1463. ``state`` and proves nothing.
  1464. * The clock is reset by any non-terminal observation, so this cannot
  1465. fire on a printer that is merely between stages.
  1466. * The grace period is far longer than the gap between a printer
  1467. finishing and its completion arriving, so the normal path always
  1468. wins the race and this only ever sees genuine strandings.
  1469. What it does *not* do is replay the completion's side effects --
  1470. notifications, billing, auto-off. It restores the queue, which is the
  1471. harm being undone; the archive was updated by the normal path
  1472. regardless, since only the queue block refuses. A recovery that
  1473. silently re-fired notifications minutes late would be its own bug.
  1474. """
  1475. try:
  1476. async with async_session() as db:
  1477. result = await db.execute(
  1478. select(PrintQueueItem)
  1479. .where(PrintQueueItem.status == "printing")
  1480. .where(PrintQueueItem.printer_id.is_not(None))
  1481. )
  1482. items = list(result.scalars().all())
  1483. if not items:
  1484. self._terminal_since.clear()
  1485. return
  1486. now = time.monotonic()
  1487. seen_printers: set[int] = set()
  1488. closed = False
  1489. for item in items:
  1490. printer_id = item.printer_id
  1491. seen_printers.add(printer_id)
  1492. state = printer_manager.get_status(printer_id)
  1493. status = _terminal_queue_status(state)
  1494. if status is None:
  1495. self._terminal_since.pop(printer_id, None)
  1496. continue
  1497. since = self._terminal_since.setdefault(printer_id, now)
  1498. if now - since < _STRANDED_PRINTING_GRACE_SECONDS:
  1499. continue
  1500. item.status = status
  1501. item.completed_at = datetime.now(timezone.utc)
  1502. closed = True
  1503. # The printer is idle: nothing it prints next is this item's.
  1504. self._forget_unstarted_print(item.id, unless_printing=True)
  1505. logger.warning(
  1506. "Queue item %s was still 'printing' after printer %s reported %s for %.0fs — "
  1507. "closing it as %s. Its completion event was never matched to it, which blocks "
  1508. "every later job for this printer (#2829).",
  1509. item.id,
  1510. printer_id,
  1511. getattr(state, "state", None),
  1512. now - since,
  1513. status,
  1514. )
  1515. if closed:
  1516. await db.commit()
  1517. from backend.app.services.print_batch import refresh_batches_after
  1518. await refresh_batches_after([item.id for item in items if item.status != "printing"])
  1519. for printer_id in list(self._terminal_since):
  1520. if printer_id not in seen_printers:
  1521. del self._terminal_since[printer_id]
  1522. except Exception as e:
  1523. # Best-effort, same as the claim sweep beside it: a recovery path
  1524. # that can itself break the scheduler loop is worse than the strand.
  1525. logger.error("Stranded-item sweep failed: %s", e)
  1526. async def _clear_stale_dispatch_claims(self, *, at_startup: bool = False) -> None:
  1527. """Clear dispatch claims with no live dispatch coroutine behind them (#2615).
  1528. A claim is only ever held by a live dispatch coroutine, so when this
  1529. process has nothing in ``_inflight`` every ``dispatching_at`` in the table
  1530. is stale. At startup that is trivially true — no coroutine survives a
  1531. restart. It is equally true on any later tick where no upload is running,
  1532. which is what makes this safe to repeat rather than only run once.
  1533. Repeating it matters because ``_clear_dispatch_claim`` is best-effort: if
  1534. the database is briefly unreachable at exactly the moment dispatch ends,
  1535. the claim survives and the row is wedged out of the selection query. That
  1536. used to last until the next restart (#2702 follow-up, seen when
  1537. PostgreSQL refused a connection mid-dispatch).
  1538. ``_inflight`` is populated when the task is spawned, before the coroutine
  1539. claims its row, and pruned by a done-callback that cannot run before the
  1540. coroutine's own ``finally`` — so "claim present, nothing in flight" has no
  1541. race window and needs no age threshold. A size-derived upload deadline
  1542. (``max(600s, size/25KB/s)``) has no safe fixed bound anyway.
  1543. """
  1544. if self._inflight:
  1545. return
  1546. try:
  1547. async with async_session() as db:
  1548. res = await db.execute(
  1549. update(PrintQueueItem).where(PrintQueueItem.dispatching_at.is_not(None)).values(dispatching_at=None)
  1550. )
  1551. await db.commit()
  1552. if res.rowcount:
  1553. logger.info(
  1554. "Cleared %d orphaned dispatch claim(s)%s (#2615)",
  1555. res.rowcount,
  1556. " at startup" if at_startup else "",
  1557. )
  1558. except Exception as exc:
  1559. logger.error("Failed to clear orphaned dispatch claims: %s", exc)
  1560. def stop(self):
  1561. """Stop the scheduler."""
  1562. self._running = False
  1563. logger.info("Print scheduler stopped")
  1564. def forget_printer(self, printer_id: int) -> None:
  1565. """Drop what is kept for a deleted printer, so the next printer given its
  1566. id (SQLite reuses it) starts clean: refused uploads and failed power-ons
  1567. would keep it out of dispatch for up to an hour.
  1568. A dispatch already in flight keeps its own state (_inflight, the preheat
  1569. pins, unconfirmed prints): it unwinds that when it ends.
  1570. """
  1571. for keyed in (
  1572. self._upload_backoff,
  1573. self._upload_refusals,
  1574. self._upload_refusal_kind,
  1575. self._upload_refusal_notified,
  1576. self._wake_failures,
  1577. self._drying_in_progress,
  1578. self._dispatch_holds,
  1579. self._chamber_history,
  1580. self._keep_warm,
  1581. self._terminal_since,
  1582. ):
  1583. keyed.pop(printer_id, None)
  1584. for keyed in (self._auto_dry_units, self._auto_dry_above):
  1585. for key in [key for key in keyed if key[0] == printer_id]:
  1586. del keyed[key]
  1587. for item_id in [i for i, sent in self._sent_unstarted_print.items() if sent[0] == printer_id]:
  1588. del self._sent_unstarted_print[item_id]
  1589. def printer_reconnected(self, printer_id: int, ip_address: str | None) -> None:
  1590. """Let a printer that has just reconnected take an upload on the next pass (#3563).
  1591. A printer whose file service stopped answering is often fixed by a
  1592. restart, but its upload backoff ran on for up to an hour, and nothing
  1593. but the timer ended it. A reconnect ends the window. The refusal count
  1594. and the notified text stay: a printer that still refuses gets the next,
  1595. longer window and no second "job waiting", and a successful upload
  1596. resets them as before.
  1597. The FTP client's handshake cool-off for the printer goes too. It dates
  1598. from before the reconnect, and inside it the first retry would fail
  1599. without reaching the printer and count as another refusal.
  1600. """
  1601. if self._upload_backoff.pop(printer_id, None) is not None:
  1602. logger.info("Printer %s reconnected; retrying uploads to it without waiting for the backoff", printer_id)
  1603. if ip_address:
  1604. BambuFTPClient.clear_handshake_cooloff(ip_address)
  1605. async def check_queue(self) -> bool:
  1606. """Check for prints ready to start.
  1607. Returns True if this pass dispatched at least one item, so the caller
  1608. can loop again quickly instead of sleeping the full interval (#2555).
  1609. """
  1610. # Scoped to one pass: a file re-sliced between ticks must be re-read.
  1611. self._filament_req_memo.clear()
  1612. self._unmatched_memo.clear()
  1613. self._filament_scope_memo.clear()
  1614. async with async_session() as db:
  1615. # Check if shortest-job-first scheduling is enabled
  1616. sjf_enabled = await self._get_bool_setting(db, "queue_shortest_first")
  1617. # Get all pending items in the one order the queue page shows them
  1618. # in (#3200). The first eligible item takes a printer, so this order
  1619. # decides who wins a printer that a pinned job and an "Any <model>"
  1620. # job both want. It used to start with ``printer_id``, which made the
  1621. # lane outrank the position: SQLite sorts NULL first, so model-based
  1622. # jobs always won; PostgreSQL sorts it last, so pinned jobs did.
  1623. # Neither is what the user dragged into place.
  1624. if sjf_enabled:
  1625. # SJF: items already jumped get top priority (starvation guard),
  1626. # then sort by print_time ascending. Items with no print time go
  1627. # last, and position breaks ties.
  1628. result = await db.execute(
  1629. select(PrintQueueItem)
  1630. .where(PrintQueueItem.status == "pending")
  1631. # Never re-select a row a dispatch worker has already claimed
  1632. # (#2615) — belt-and-suspenders with the _inflight exclusion
  1633. # below, and the guard that lets an orphaned claim be ignored
  1634. # until startup reconciliation clears it.
  1635. .where(PrintQueueItem.dispatching_at.is_(None))
  1636. # archive/library_file are read by the cross-model gate
  1637. # (#2578); eager-load once per pass instead of a lazy-load
  1638. # (which would raise in async) per item.
  1639. .options(
  1640. selectinload(PrintQueueItem.archive),
  1641. selectinload(PrintQueueItem.library_file),
  1642. # Cross-model candidates (#671), plus each candidate's file
  1643. # for the same cross-model gate. Lazy-loading either would
  1644. # raise in async.
  1645. selectinload(PrintQueueItem.variants).selectinload(PrintQueueVariant.library_file),
  1646. )
  1647. .order_by(
  1648. PrintQueueItem.been_jumped.desc(),
  1649. PrintQueueItem.print_time_seconds.asc().nullslast(),
  1650. PrintQueueItem.position,
  1651. PrintQueueItem.id,
  1652. )
  1653. )
  1654. else:
  1655. result = await db.execute(
  1656. select(PrintQueueItem)
  1657. .where(PrintQueueItem.status == "pending")
  1658. # Skip rows already claimed by a dispatch worker (#2615).
  1659. .where(PrintQueueItem.dispatching_at.is_(None))
  1660. .options(
  1661. selectinload(PrintQueueItem.archive),
  1662. selectinload(PrintQueueItem.library_file),
  1663. # Cross-model candidates (#671), plus each candidate's file
  1664. # for the same cross-model gate. Lazy-loading either would
  1665. # raise in async.
  1666. selectinload(PrintQueueItem.variants).selectinload(PrintQueueVariant.library_file),
  1667. )
  1668. .order_by(PrintQueueItem.position, PrintQueueItem.id)
  1669. )
  1670. items = list(result.scalars().all())
  1671. # Drop rows whose upload is still in flight from an earlier pass
  1672. # (#2602). They stay `pending` until the upload finishes, so without
  1673. # this a fast tick would re-select and re-dispatch the same row.
  1674. # Belt-and-suspenders with the printer exclusion below.
  1675. if self._inflight:
  1676. inflight_ids = set(self._inflight)
  1677. items = [it for it in items if it.id not in inflight_ids]
  1678. # Read plate-clear setting once per queue check. Default MUST be
  1679. # False to match the schema (SettingsSchema.require_plate_clear
  1680. # defaults False) and the frontend (toggle + card badge both treat a
  1681. # missing value as off). When no settings row exists, a True default
  1682. # here re-enabled the plate-clear gate the UI showed as disabled,
  1683. # blocking dispatch to FINISH-state printers forever with no UI path
  1684. # to clear it (#1865).
  1685. require_plate_clear = await self._get_bool_setting(db, "require_plate_clear", default=False)
  1686. # Dispatch and track scheduled drying runs (#2638)
  1687. await self._check_scheduled_dryings(db)
  1688. if not items:
  1689. # No dispatchable pending items — still check auto-drying on idle
  1690. # printers, but keep any printer with an upload still in flight
  1691. # from an earlier pass out of it (#2602): its print is imminent,
  1692. # so it must not be auto-dried in the gap before the row flips to
  1693. # printing. Report the pass as productive while uploads run so the
  1694. # loop stays on the fast interval.
  1695. #
  1696. # Also release any keep-warm holds that got orphaned by the queue
  1697. # emptying — the normal sweep in `_apply_keep_warm` is skipped by
  1698. # this early return, so call it directly with an empty candidate
  1699. # set. Without this, a printer whose queued item was cancelled or
  1700. # deleted would keep its bed at target until the max-duration
  1701. # timeout expired.
  1702. self._sweep_keep_warm(active_candidates=set(), dispatched=set())
  1703. inflight_printers = {pid for (_task, pid) in self._inflight.values() if pid is not None}
  1704. await self._check_auto_drying(db, [], inflight_printers)
  1705. await self._check_filament_low(db)
  1706. await self._check_stock_forecast(db)
  1707. return bool(self._inflight)
  1708. logger.info(
  1709. "Queue check: found %d pending items: %s",
  1710. len(items),
  1711. [(i.id, i.printer_id, i.archive_id, i.library_file_id) for i in items],
  1712. )
  1713. # Forget jobs that left the queue since they were last checked.
  1714. pending_ids = {i.id for i in items}
  1715. self._short_hold_checked_at = {
  1716. item_id: at for item_id, at in self._short_hold_checked_at.items() if item_id in pending_ids
  1717. }
  1718. # Seed busy_printers with printers that already have an item in 'printing'
  1719. # status. _is_printer_idle() alone is not sufficient as a dispatch gate —
  1720. # on H2D / P1 series the MQTT state transition from IDLE to RUNNING can
  1721. # lag several seconds behind the print command, so the next check_queue
  1722. # tick still sees IDLE and would double-dispatch onto the same printer.
  1723. # Without this guard, two pending items targeting the same printer
  1724. # (e.g. a batch with quantity>1) both end up in 'printing' status —
  1725. # surfaced via the "BUG: Multiple queue items" warning in on_print_complete.
  1726. busy_result = await db.execute(
  1727. select(PrintQueueItem.printer_id)
  1728. .where(PrintQueueItem.status == "printing")
  1729. .where(PrintQueueItem.printer_id.is_not(None))
  1730. )
  1731. busy_printers: set[int] = {pid for (pid,) in busy_result.all() if pid is not None}
  1732. # Why each printer left this pass, recorded where the decision is
  1733. # made rather than re-derived when the summary is logged. #3018's
  1734. # bundle shows what the old summary produced: "printer 1 not
  1735. # available -- connected=True, state=IDLE" immediately followed by a
  1736. # dispatch to printer 1. Two things went wrong at once. The line read
  1737. # live state at log time, which by then no longer matched the state
  1738. # the decision was made on; and `busy_printers` holds both printers
  1739. # that cannot take work and printers this pass has claimed for it,
  1740. # which are opposite facts. It is the first line anyone greps for
  1741. # "why did my item not go out", so it has to say which.
  1742. busy_reasons: dict[int, str] = dict.fromkeys(busy_printers, "an item is already printing on it")
  1743. # Printers this pass is dispatching to. They are in busy_printers so
  1744. # nothing else in the pass targets them -- that is a reservation, not
  1745. # an obstruction, and the summary says so.
  1746. claimed_printers: set[int] = set()
  1747. def mark_busy(printer_id: int, reason: str) -> None:
  1748. """Take ``printer_id`` out of this pass, recording why.
  1749. First reason wins: a printer already excluded by a stronger fact
  1750. -- a print running on it -- must not be relabelled by a weaker
  1751. check that ran later and would have excluded it anyway.
  1752. """
  1753. busy_printers.add(printer_id)
  1754. busy_reasons.setdefault(printer_id, reason)
  1755. def claim_printer(printer_id: int) -> None:
  1756. """Reserve ``printer_id`` for an item this pass is dispatching."""
  1757. claimed_printers.add(printer_id)
  1758. mark_busy(printer_id, "selected for dispatch in this pass")
  1759. # The user-facing half of `busy_reasons` (#3074). The same decisions
  1760. # worded for a different audience: the log wants "still inside its
  1761. # post-dispatch hold window", the queue row wants to know the printer
  1762. # is taken and nothing is broken. Only the cases that would read wrong
  1763. # as a plain "Busy" are recorded here; the rest fall back to it.
  1764. item_hold_reasons: dict[int, str] = {}
  1765. # Names and models for the printers this pass may have to write a
  1766. # waiting reason about, read once rather than per skip per tick. The
  1767. # model-based branch already has its names from `_printers_for_model`.
  1768. pinned_printer_ids = {i.printer_id for i in items if i.printer_id}
  1769. pinned_printers: dict[int, tuple[str, str]] = {}
  1770. if pinned_printer_ids:
  1771. pinned_rows = await db.execute(
  1772. select(Printer.id, Printer.name, Printer.model).where(Printer.id.in_(pinned_printer_ids))
  1773. )
  1774. pinned_printers = {pid: (name or "", model or "") for pid, name, model in pinned_rows.all()}
  1775. def printer_label(printer_id: int) -> str:
  1776. """What to call this printer in a queue row."""
  1777. entry = pinned_printers.get(printer_id)
  1778. return (entry[0] if entry else "") or f"printer {printer_id}"
  1779. async def hold_item(item: PrintQueueItem, reason: str | None, *, notify: bool = True) -> None:
  1780. """Record why *item* is not going out, in the words the queue row shows.
  1781. The fixed-printer branch's single writer for ``waiting_reason``
  1782. (#3074). Before this, the sensor interlock was the only thing that
  1783. wrote the field there, so an item pinned to a printer that was
  1784. merely printing sat at `pending` with nothing to show for it —
  1785. indistinguishable from a queue that had stopped working — while
  1786. the same job queued as "Any <model>" explained itself.
  1787. Every exit from that branch now calls this, which is also what
  1788. replaced the interlock's old habit of clearing the field up front:
  1789. a lifted hold cannot leave "Waiting on Enclosure Door" standing,
  1790. because whichever exit runs next overwrites it and the dispatch
  1791. path clears it.
  1792. A notification goes out when this item starts asking for
  1793. something, and only then: the new reason needs the user, and what
  1794. it replaced did not. "What it replaced did not" has to include a
  1795. busy-only reason, not just an empty one. The sequence this branch
  1796. actually produces is a print running (``Busy: X1C-01``) and then
  1797. the plate it left behind (``Waiting for plate confirmation``), and
  1798. testing "was the field empty" would call that no transition at all
  1799. and stay quiet through the one case worth saying out loud.
  1800. The cost is that a printer dropping offline, coming back busy and
  1801. dropping again asks twice rather than once. That is the honest
  1802. reading — it went wrong twice — and the alternative was a rule
  1803. that never fired for the case this was built for.
  1804. *notify* is how a caller opts out. The sensor interlock does: it
  1805. has never sent this notification, and a change about what the
  1806. queue *displays* is not the place to start (#1148).
  1807. """
  1808. if item.waiting_reason == reason:
  1809. return
  1810. # Busy-only and empty are the same thing here: neither is the
  1811. # queue asking the user for anything.
  1812. was_asking = bool(item.waiting_reason) and not self._is_busy_only(item.waiting_reason)
  1813. item.waiting_reason = reason
  1814. await db.commit()
  1815. if not notify or not reason or was_asking or self._is_busy_only(reason):
  1816. return
  1817. try:
  1818. job_name = await self._get_job_name(db, item)
  1819. entry = pinned_printers.get(item.printer_id) if item.printer_id else None
  1820. await notification_service.on_queue_job_waiting(
  1821. job_name=job_name,
  1822. target_model=(entry[1] if entry else "") or "",
  1823. waiting_reason=reason,
  1824. db=db,
  1825. printer_id=item.printer_id,
  1826. )
  1827. except Exception as e:
  1828. # A queue that cannot say why it is waiting is the bug being
  1829. # fixed here; a queue that stops dispatching because a
  1830. # notification provider is down would be a worse one.
  1831. logger.debug("Waiting notification failed for item %s: %s", item.id, e)
  1832. async def hold_for_printer(
  1833. item: PrintQueueItem, printer_id: int, log_reason: str, item_reason: str
  1834. ) -> None:
  1835. """Take *printer_id* out of this pass and tell the item's owner why."""
  1836. mark_busy(printer_id, log_reason)
  1837. item_hold_reasons.setdefault(printer_id, item_reason)
  1838. await hold_item(item, item_reason)
  1839. # Defense-in-depth (#1157): augment busy_printers with any printer
  1840. # still in its post-dispatch hold window. Empirically, the DB seed
  1841. # above can miss in-flight items in a multi-plate batch — same-file
  1842. # plates were being dispatched 30 s apart while the H2D was still
  1843. # digesting the first project_file. The hold is keyed in-memory and
  1844. # released by the watchdog on the success path, so it adds a layer
  1845. # that doesn't depend on DB row visibility or completion-callback
  1846. # timing.
  1847. for held_printer_id in list(self._dispatch_holds.keys()):
  1848. if self._printer_in_dispatch_hold(held_printer_id):
  1849. mark_busy(held_printer_id, "still inside its post-dispatch hold window")
  1850. # Exclude printers whose upload is still in flight from an earlier
  1851. # pass (#2602). The row is `pending` until the upload finishes and
  1852. # the printing-state seed / dispatch hold above only arm once the
  1853. # upload completes, so this is what holds the printer (and, via
  1854. # busy_printers, its auto-drying) out of the pass during the upload.
  1855. for _task, inflight_pid in self._inflight.values():
  1856. if inflight_pid is not None:
  1857. mark_busy(inflight_pid, "an upload to it is still in flight")
  1858. # Snapshot taken here, before the item loop adds anything (#2801).
  1859. #
  1860. # The three sources above all mean the same thing: a print on this
  1861. # printer is running or imminent. Everything the loop adds below
  1862. # means only "the queue could not dispatch to it this pass", which
  1863. # is a different statement -- a printer waiting on a plate-clear
  1864. # acknowledgment, an offline printer, one with no matching file.
  1865. #
  1866. # Auto-drying must only see the first kind. Reading the whole set
  1867. # as "is currently printing" is what put a plate-held printer down
  1868. # the mid-print path: it capped the drying temperature, logged the
  1869. # cycle as (mid-print), and skipped the very gate that was supposed
  1870. # to hold it. The interlock block below already documents the same
  1871. # hazard and works around it by staying out of busy_printers; this
  1872. # generalises that workaround instead of repeating it per case.
  1873. dispatching_printers: set[int] = set(busy_printers)
  1874. # Printers whose last upload never reached them (#3210). After the
  1875. # snapshot above on purpose: such a printer is idle, not about to
  1876. # print, and auto-drying must keep treating it as idle.
  1877. now_mono = time.monotonic()
  1878. # In backoff this pass. Kept apart from busy_printers so keep-warm
  1879. # can leave them out: there is no print to keep a bed warm for.
  1880. # Their holds are written silently: the refusal that put a printer
  1881. # here sent the "job waiting" notification for its run (#3409).
  1882. backoff_printers: set[int] = set()
  1883. for backoff_pid, retry_at in list(self._upload_backoff.items()):
  1884. if now_mono >= retry_at:
  1885. del self._upload_backoff[backoff_pid]
  1886. continue
  1887. backoff_printers.add(backoff_pid)
  1888. mark_busy(
  1889. backoff_pid,
  1890. f"its file service refused the last upload; retrying in {retry_at - now_mono:.0f}s",
  1891. )
  1892. item_hold_reasons.setdefault(
  1893. backoff_pid,
  1894. _refused_upload_hold_reason(printer_label(backoff_pid), self._upload_refusal_kind.get(backoff_pid)),
  1895. )
  1896. # Printers held by a Home Assistant sensor interlock (#1148) — an
  1897. # enclosure door left open, say. The fixed-printer branch turns
  1898. # this into a waiting_reason the user can act on; the model-based
  1899. # branch hides these printers from the matcher so an "Any <model>"
  1900. # job runs on a sibling instead of queueing behind the held one.
  1901. #
  1902. # Deliberately NOT merged into busy_printers, even though that set
  1903. # already means "unavailable this pass". _check_auto_drying reads
  1904. # it as "is currently printing" and would put an idle-but-held
  1905. # printer down the mid-print drying path, which caps the drying
  1906. # temperature and skips the queue-only gating. A held printer is
  1907. # idle; it should dry exactly as it did before.
  1908. #
  1909. # Only sensors we actually read and found alerting appear here; see
  1910. # ha_sensor_manager.blocked_printers. A Home Assistant that is down
  1911. # holds nothing.
  1912. interlocked: dict[int, str] = {}
  1913. try:
  1914. interlocked = await ha_sensor_manager.blocked_printers(db)
  1915. except Exception as e:
  1916. # Never let the interlock stop the queue running. A broken
  1917. # lookup means no holds, not no dispatches.
  1918. logger.warning("Home Assistant interlock check failed: %s", e)
  1919. interlocked = {}
  1920. # Printers a smart plug can bring back, read once for the whole pass
  1921. # (#2786). Used by the model-based branch both to word "Offline" in
  1922. # the waiting reason and to decide what the wake step may switch on.
  1923. wakeable_printer_ids = await self._wakeable_printer_ids(db)
  1924. # At most one power-on per queue check. Each one blocks this loop
  1925. # for the boot wait, so a queue of ten class-targeted jobs must not
  1926. # switch on ten printers inside a single pass — the next pass wakes
  1927. # the next one (#2786).
  1928. power_on_attempted = False
  1929. # Log skip reasons once per queue check (not per item)
  1930. skip_reasons: dict[str, int] = {}
  1931. # Items selected for dispatch in this pass, one per printer. The
  1932. # loop below only *decides* — the uploads happen afterwards, in
  1933. # parallel (#2555). See _dispatch_selected().
  1934. dispatch_ids: list[int] = []
  1935. # Library rows queued with `cleanup_library_after_dispatch` (the
  1936. # printer-card "upload and print" flow) are CONSUMED by the dispatch
  1937. # that prints them: the row is deleted and the 3MF is unlinked from
  1938. # disk. That was safe only because dispatch was serial. Run two of
  1939. # them against the same row at once and the second DELETE matches no
  1940. # row (StaleDataError), and the winner's unlink can pull the file out
  1941. # from under the loser's in-flight upload.
  1942. #
  1943. # Only the cleanup flag mutates the row. An ordinary library print
  1944. # just reads it, so the common fan-out — one file, many printers,
  1945. # which is exactly the reporter's workload — still goes out fully in
  1946. # parallel. Narrow the guard to the mutating case; do not serialise
  1947. # the case the whole fix exists for.
  1948. dispatch_libs: set[int] = set()
  1949. consumed_libs: set[int] = set()
  1950. def _library_row_conflict(candidate: PrintQueueItem) -> bool:
  1951. """True if dispatching `candidate` now would race another item's cleanup."""
  1952. lib_id = candidate.library_file_id
  1953. if lib_id is None:
  1954. return False
  1955. if candidate.cleanup_library_after_dispatch:
  1956. # We would delete a row someone else in this pass is reading.
  1957. return lib_id in dispatch_libs
  1958. # Someone else in this pass will delete the row out from under us.
  1959. return lib_id in consumed_libs
  1960. def _claim_library_row(candidate: PrintQueueItem) -> None:
  1961. lib_id = candidate.library_file_id
  1962. if lib_id is None:
  1963. return
  1964. dispatch_libs.add(lib_id)
  1965. if candidate.cleanup_library_after_dispatch:
  1966. consumed_libs.add(lib_id)
  1967. # Printer scope of each job's creator (#1727), so an "any <model>"
  1968. # job only lands on a printer its creator may use. Loaded lazily,
  1969. # once per creator per pass.
  1970. from backend.app.core.auth import is_auth_enabled
  1971. scope_auth_enabled = await is_auth_enabled(db)
  1972. creator_scopes: dict[int, PrinterScope] = {}
  1973. # Printers a model job passes over while drying blocks the queue,
  1974. # as a pinned job waits for its own. A parked timer never ends,
  1975. # so it does not hold (#2896).
  1976. drying_held: set[int] = (
  1977. {pid for pid in self._drying_in_progress if not self._drying_is_only_parked(pid)}
  1978. if self._drying_in_progress and await self._get_bool_setting(db, "queue_drying_block")
  1979. else set()
  1980. )
  1981. async def _creator_scope(user_id: int | None) -> PrinterScope:
  1982. if not scope_auth_enabled or user_id is None:
  1983. return ALL_PRINTERS
  1984. if user_id not in creator_scopes:
  1985. creator_scopes[user_id] = await resolve_user_id_printer_scope(db, user_id)
  1986. return creator_scopes[user_id]
  1987. for item in items:
  1988. # Check scheduled time first (scheduled_time is stored in UTC from ISO string)
  1989. if item.scheduled_time:
  1990. sched = item.scheduled_time
  1991. if sched.tzinfo is None:
  1992. sched = sched.replace(tzinfo=timezone.utc)
  1993. if sched > datetime.now(timezone.utc):
  1994. # Waiting on the clock, not on a printer.
  1995. await hold_item(item, None)
  1996. skip_reasons["scheduled_future"] = skip_reasons.get("scheduled_future", 0) + 1
  1997. continue
  1998. # Skip items that require manual start, except an "any model"
  1999. # job held for low filament that another idle printer can now
  2000. # run: that one is moved there and carries on below (#3137).
  2001. if item.manual_start and not (
  2002. item.filament_short
  2003. and await self._release_short_model_hold(
  2004. db,
  2005. item,
  2006. busy_printers | interlocked.keys() | drying_held,
  2007. require_plate_clear,
  2008. await _creator_scope(item.created_by_id),
  2009. )
  2010. ):
  2011. # Waiting on the user, not on a printer. Cleared here because
  2012. # this is the last pass that will look at the row: a staged
  2013. # item never reaches the branches below again, so a reason
  2014. # left from before it was staged would stand forever (#3074).
  2015. # The one exception is the unmatched-filament hold, whose
  2016. # reason was written at staging and is why the item waits:
  2017. # it names the filament to load (#2799). Pressing start
  2018. # clears manual_start, and the branches below overwrite it.
  2019. keep = item.waiting_reason if _is_unmatched_hold_reason(item.waiting_reason) else None
  2020. await hold_item(item, keep)
  2021. skip_reasons["manual_start"] = skip_reasons.get("manual_start", 0) + 1
  2022. continue
  2023. if item.printer_id and item.printer_id not in pinned_printers:
  2024. # Moved by the release above to a printer no row named when
  2025. # the labels were read.
  2026. moved_to = await self._get_printer(db, item.printer_id)
  2027. if moved_to is not None:
  2028. pinned_printers[item.printer_id] = (moved_to.name or "", moved_to.model or "")
  2029. if item.printer_id:
  2030. # Its creator may no longer use this printer (#1727): an
  2031. # admin took it away from their group after the job was
  2032. # queued, say to keep it free for a training session. Held,
  2033. # not failed, so it starts if access comes back or the user
  2034. # moves it to a printer they still have.
  2035. if not (await _creator_scope(item.created_by_id)).allows(item.printer_id):
  2036. await hold_item(
  2037. item,
  2038. "Its owner no longer has access to this printer — move it to another printer",
  2039. notify=False,
  2040. )
  2041. skip_reasons["printer_out_of_scope"] = skip_reasons.get("printer_out_of_scope", 0) + 1
  2042. continue
  2043. # Held by a sensor interlock (#1148). Checked before the
  2044. # busy_printers test that would otherwise swallow it
  2045. # silently — "waiting for a printer" and "waiting for you
  2046. # to shut the enclosure" need to read differently, and only
  2047. # one of them is something the user can fix.
  2048. #
  2049. # It used to be the only thing that wrote a waiting_reason on
  2050. # this branch, and it cleared the field up front so a lifted
  2051. # hold could not leave a shut door reading "Waiting on
  2052. # Enclosure Door". `hold_item` carries that guarantee now —
  2053. # every exit below writes — so the clear is gone and the
  2054. # interlock is an ordinary hold like the rest (#3074).
  2055. interlock_reason = interlocked.get(item.printer_id)
  2056. if interlock_reason:
  2057. # Silent, exactly as it has always been. #1148 built this
  2058. # as a hold that shows on the row, never as an alert, and
  2059. # routing it through the shared writer must not quietly
  2060. # turn every open door into a notification.
  2061. await hold_item(item, f"Waiting on {interlock_reason}", notify=False)
  2062. skip_reasons["sensor_interlock"] = skip_reasons.get("sensor_interlock", 0) + 1
  2063. continue
  2064. # Specific printer assignment (existing behavior)
  2065. if item.printer_id in busy_printers:
  2066. # Whatever took the printer out of this pass — a print
  2067. # already running on it, a post-dispatch hold, an upload
  2068. # still in flight, an item ahead of this one in the same
  2069. # pass — reads the same way from the queue: the printer is
  2070. # taken and this item is in line for it. The exceptions
  2071. # that do not (an offline printer, say) recorded their own
  2072. # wording in `item_hold_reasons` when they held it.
  2073. await hold_item(
  2074. item,
  2075. item_hold_reasons.get(item.printer_id) or f"Busy: {printer_label(item.printer_id)}",
  2076. notify=item.printer_id not in backoff_printers,
  2077. )
  2078. continue
  2079. # Check if printer is idle
  2080. printer_idle = self._is_printer_idle(item.printer_id, require_plate_clear)
  2081. printer_connected = printer_manager.is_connected(item.printer_id)
  2082. # If printer not connected, try to power on via smart plug
  2083. if not printer_connected:
  2084. plugs = await self._get_smart_plugs(db, item.printer_id)
  2085. auto_on_plugs = self._auto_on_plugs(plugs)
  2086. if auto_on_plugs:
  2087. logger.info("Printer %s offline, attempting to power on via smart plug(s)", item.printer_id)
  2088. # Power on using the plug that actually feeds the printer, and
  2089. # wait for it to boot on that one only (#2629).
  2090. primary_plug = self._pick_power_plug(auto_on_plugs)
  2091. powered_on = await self._power_on_and_wait(primary_plug, item.printer_id, db)
  2092. if powered_on:
  2093. # Also turn on any remaining auto_on plugs (e.g., filter)
  2094. for extra_plug in [p for p in auto_on_plugs if p.id != primary_plug.id]:
  2095. try:
  2096. service = await smart_plug_manager.get_service_for_plug(extra_plug, db)
  2097. if await service.turn_on(extra_plug):
  2098. await publish_plug_state(
  2099. extra_plug.id,
  2100. extra_plug.name,
  2101. extra_plug.printer_id,
  2102. on=True,
  2103. ha_entity_id=extra_plug.ha_entity_id,
  2104. )
  2105. logger.info(
  2106. "Also powered on plug '%s' for printer %s", extra_plug.name, item.printer_id
  2107. )
  2108. except Exception as e:
  2109. logger.warning("Failed to power on extra plug '%s': %s", extra_plug.name, e)
  2110. printer_connected = True
  2111. printer_idle = self._is_printer_idle(item.printer_id, require_plate_clear)
  2112. else:
  2113. logger.warning("Could not power on printer %s via smart plug", item.printer_id)
  2114. await hold_for_printer(
  2115. item,
  2116. item.printer_id,
  2117. "smart-plug power-on failed",
  2118. f"Offline: {printer_label(item.printer_id)} — the smart plug could not power it on",
  2119. )
  2120. continue
  2121. else:
  2122. # No plug or auto_on disabled. Worded exactly as the
  2123. # model-based branch words it (#2786): this is the one
  2124. # entry on that list the user has to act on, because
  2125. # Bambuddy will never switch this printer on itself.
  2126. await hold_for_printer(
  2127. item,
  2128. item.printer_id,
  2129. "offline, with no smart plug to power it on",
  2130. f"Offline, no Auto On smart plug: {printer_label(item.printer_id)}",
  2131. )
  2132. continue
  2133. # Check if printer is idle (busy with another print)
  2134. if not printer_idle:
  2135. await hold_for_printer(
  2136. item,
  2137. item.printer_id,
  2138. "not idle",
  2139. self._pinned_hold_reason(
  2140. item.printer_id, printer_label(item.printer_id), require_plate_clear
  2141. ),
  2142. )
  2143. continue
  2144. # Drying blocks the queue, if the user asked it to. A hold
  2145. # is a skip like any other, so it belongs here with the
  2146. # rest of the availability checks.
  2147. # A parked timer (#2896) never ends, so it must not hold the
  2148. # queue; it stays tracked so the stop paths still reach it.
  2149. if (
  2150. self._drying_in_progress.get(item.printer_id)
  2151. and not self._drying_is_only_parked(item.printer_id)
  2152. and await self._get_bool_setting(db, "queue_drying_block")
  2153. ):
  2154. # Busy-shaped on purpose: the cycle ends on its own and
  2155. # the job goes out, so there is nothing to alert about.
  2156. await hold_for_printer(
  2157. item,
  2158. item.printer_id,
  2159. "drying, and drying is set to block the queue",
  2160. f"Busy: {printer_label(item.printer_id)} (drying)",
  2161. )
  2162. continue
  2163. # Check condition (previous print success)
  2164. if item.require_previous_success:
  2165. if not await self._check_previous_success(db, item):
  2166. item.status = "skipped"
  2167. item.error_message = "Previous print failed or was aborted"
  2168. item.completed_at = datetime.now(timezone.utc)
  2169. # Not pending any more, so not waiting for anything.
  2170. item.waiting_reason = None
  2171. await db.commit()
  2172. logger.info("Skipped queue item %s - previous print failed", item.id)
  2173. # Send notification
  2174. job_name = await self._get_job_name(db, item)
  2175. printer = await self._get_printer(db, item.printer_id)
  2176. await notification_service.on_queue_job_skipped(
  2177. job_name=job_name,
  2178. printer_id=item.printer_id,
  2179. printer_name=printer.name if printer else "Unknown",
  2180. reason="Previous print failed or was aborted",
  2181. db=db,
  2182. )
  2183. continue
  2184. # Resolve the AMS mapping when it's missing OR unresolved
  2185. # (all -1). A stored all-[-1] mapping is a bug artifact — a
  2186. # frontend status-load race can persist [-1] (#2589) — and
  2187. # must be recomputed from live trays rather than trusted.
  2188. if await _edited_since_loaded(db, item):
  2189. continue
  2190. unmappable = await self._ensure_ams_mapping(db, item.printer_id, item)
  2191. if isinstance(unmappable, FilamentScopeHold):
  2192. # Its creator may not print it as it stands (#3522).
  2193. # Held, not failed, as for a printer they lost above:
  2194. # it starts once the group's rules or the loaded
  2195. # spools allow it.
  2196. if item.waiting_reason != unmappable.reason:
  2197. logger.info(
  2198. "Queue item %s held on printer %s by its creator's filament rules — %s",
  2199. item.id,
  2200. item.printer_id,
  2201. unmappable.reason,
  2202. )
  2203. await hold_item(item, unmappable.reason, notify=False)
  2204. skip_reasons["filament_out_of_scope"] = skip_reasons.get("filament_out_of_scope", 0) + 1
  2205. continue
  2206. if unmappable:
  2207. await self._fail_unmappable_item(db, item, item.printer_id, unmappable)
  2208. continue
  2209. # Filament-deficit pre-dispatch check (#1496). If the
  2210. # assigned spool can't satisfy any required slot grams,
  2211. # promote the item to manual_start so the user must
  2212. # acknowledge via the ▶ button (which re-checks live).
  2213. if await self._block_on_filament_deficit(db, item):
  2214. # Now staged for the user to start by hand, and the row
  2215. # shows the filament-short badge instead. Cleared because
  2216. # a staged item never reaches this branch again.
  2217. await hold_item(item, None)
  2218. continue
  2219. # Unmatched-filament pre-dispatch check (#2799). Hold rather
  2220. # than let the printer pick a substitute for a slot the
  2221. # matcher could not resolve on this printer.
  2222. if await self._block_on_unmatched_filament(db, item):
  2223. continue
  2224. # Hold this item back for the next pass rather than racing
  2225. # another dispatch over the same transient library row. The
  2226. # printer is still marked busy so a later item does not jump
  2227. # its place in this printer's queue.
  2228. if _library_row_conflict(item):
  2229. skip_reasons["library_row_in_use"] = skip_reasons.get("library_row_in_use", 0) + 1
  2230. await hold_for_printer(
  2231. item,
  2232. item.printer_id,
  2233. "holding its place while another item releases a library row",
  2234. f"Busy: {printer_label(item.printer_id)}",
  2235. )
  2236. continue
  2237. # Print takes priority: stop a cycle Bambuddy armed, now
  2238. # that this item is definitely going out.
  2239. #
  2240. # Placement is the whole point (#2801). This used to sit up
  2241. # with the availability checks, inside the not-idle branch
  2242. # -- so it fired only on the passes where the print was NOT
  2243. # going to start, and never on the ones where it was.
  2244. # Drying is not one of the things `_is_printer_idle` looks
  2245. # at, so a stop could never have unblocked that printer
  2246. # anyway; the cycle was spent for nothing, auto-drying
  2247. # re-armed on the next tick, and a plate left
  2248. # unacknowledged turned that into a loop on the scheduler
  2249. # interval. Every skip between there and here -- a failed
  2250. # previous print, an unmappable item, a filament deficit, a
  2251. # contested library row -- is another way to lose a cycle
  2252. # for a print that never happens, which is why this waits
  2253. # until the decision is actually made.
  2254. if self._drying_in_progress.get(
  2255. item.printer_id
  2256. ) and not await self._drying_may_continue_through_print(db, item.printer_id):
  2257. await self._stop_drying(item.printer_id)
  2258. # Queue the dispatch instead of running it here — see
  2259. # _dispatch_selected(). busy_printers still gets the printer
  2260. # immediately, so nothing else in this pass can target it.
  2261. #
  2262. # The reason goes first: this item is not waiting for anything
  2263. # any more, and the model-based branch clears its own at the
  2264. # equivalent moment (#3074).
  2265. await hold_item(item, None)
  2266. _claim_library_row(item)
  2267. dispatch_ids.append(item.id)
  2268. claim_printer(item.printer_id)
  2269. # SJF starvation guard: mark items that were jumped
  2270. if sjf_enabled and item.print_time_seconds is not None:
  2271. pinned_model = pinned_printers.get(item.printer_id, ("", ""))[1]
  2272. for other in items:
  2273. if (
  2274. other.id != item.id
  2275. and other.id not in dispatch_ids
  2276. and other.status == "pending"
  2277. and _could_take_printer(other, item.printer_id, pinned_model)
  2278. and not other.been_jumped
  2279. and other.position < item.position
  2280. and (
  2281. other.print_time_seconds is None
  2282. or other.print_time_seconds > item.print_time_seconds
  2283. )
  2284. ):
  2285. other.been_jumped = True
  2286. await db.commit()
  2287. elif item.target_model or item.variants:
  2288. # Model-based assignment - find any idle printer of matching model.
  2289. # A plain model-based item has exactly one candidate, built from
  2290. # its own columns. A cross-model item (#671) has one per sliced
  2291. # variant and takes the first that matches, walking them in the
  2292. # user's priority order so the pick is reproducible when more
  2293. # than one printer is free in the same pass.
  2294. candidates = _candidates_for(item)
  2295. item_scope = await _creator_scope(item.created_by_id)
  2296. printer_id = None
  2297. chosen: _ModelCandidate | None = None
  2298. per_model_reasons: list[tuple[str | None, str]] = []
  2299. # Candidates that cleared the cross-model gate below. The
  2300. # smart-plug wake step may only consider these — waking a
  2301. # printer for a file that can never legally run on it is
  2302. # worse than not waking at all (#2786).
  2303. wakeable_candidates: list[_ModelCandidate] = []
  2304. # Printers passed over because their last print failed.
  2305. gate_blocked: set[int] = set()
  2306. if not candidates:
  2307. # Every candidate file has been deleted or trashed out from
  2308. # under this item. Hold it with something the user can act
  2309. # on rather than letting it look dispatchable forever.
  2310. per_model_reasons.append(
  2311. (
  2312. item.target_model,
  2313. "Every file for this job has been deleted — add a file back or remove the item",
  2314. )
  2315. )
  2316. for candidate in candidates:
  2317. effective_types, filament_overrides = _filament_constraints(candidate)
  2318. # Cross-model safety gate (#2578): never hand a 3MF sliced
  2319. # for an incompatible model to a printer, no matter how the
  2320. # row got into the DB (old rows, direct API writes). Held
  2321. # as pending with an actionable waiting_reason — the user
  2322. # fixes it by editing the item's target model.
  2323. if not is_gcode_compatible(candidate.sliced_for, candidate.target_model):
  2324. per_model_reasons.append(
  2325. (
  2326. candidate.target_model,
  2327. f"File was sliced for {candidate.sliced_for}, which is not compatible with "
  2328. f"{candidate.target_model} — edit the item and fix its target model",
  2329. )
  2330. )
  2331. skip_reasons["sliced_model_mismatch"] = skip_reasons.get("sliced_model_mismatch", 0) + 1
  2332. continue
  2333. # A file its creator may not print (#3522) waits here,
  2334. # unassigned, with the filament named: no printer of
  2335. # the model could take it, and assigning one only to
  2336. # give it back would do so on every pass. It starts
  2337. # once the group's rules allow it. The check reads the
  2338. # candidate's file off the item; resolving is safe to
  2339. # repeat (see is_short below), and skipped when the
  2340. # creator is unrestricted, so their items are untouched.
  2341. if not (await self._creator_filament_scope(db, item.created_by_id)).is_unrestricted:
  2342. self._resolve_variant(item, candidate)
  2343. if scope_reason := await self._filament_scope_violation(db, item):
  2344. per_model_reasons.append((candidate.target_model, scope_reason))
  2345. skip_reasons["filament_out_of_scope"] = skip_reasons.get("filament_out_of_scope", 0) + 1
  2346. continue
  2347. wakeable_candidates.append(candidate)
  2348. async def is_short(
  2349. pid: int, item: PrintQueueItem = item, candidate: _ModelCandidate = candidate
  2350. ) -> bool:
  2351. # The check reads the file and plate off the item,
  2352. # so it must hold this candidate's. Resolving is
  2353. # safe to repeat, and the winner is resolved again
  2354. # below.
  2355. self._resolve_variant(item, candidate)
  2356. return await self._filament_short_on(db, item, pid)
  2357. async def unmatched_on(
  2358. pid: int, item: PrintQueueItem = item, candidate: _ModelCandidate = candidate
  2359. ) -> list[str]:
  2360. self._resolve_variant(item, candidate)
  2361. return await self._unmatched_on(db, item, pid)
  2362. while True:
  2363. match_id, match_reason = await self._find_idle_printer_for_model(
  2364. db,
  2365. candidate.target_model,
  2366. # Sensor-held printers are unavailable to the
  2367. # matcher but stay out of busy_printers itself
  2368. # (#1148) — see where `interlocked` is built.
  2369. busy_printers | interlocked.keys() | gate_blocked | drying_held,
  2370. effective_types,
  2371. item.target_location,
  2372. filament_overrides=filament_overrides,
  2373. require_plate_clear=require_plate_clear,
  2374. wakeable_ids=wakeable_printer_ids,
  2375. printer_scope=item_scope,
  2376. is_short=is_short,
  2377. drying_ids=drying_held,
  2378. unmatched_on=unmatched_on,
  2379. refused_uploads={pid: self._upload_refusal_kind.get(pid) for pid in backoff_printers},
  2380. sliced_nozzle=candidate.sliced_nozzle,
  2381. )
  2382. # "Only if the previous print succeeded" is about the
  2383. # printer the job lands on: one whose last print failed
  2384. # is passed over for another of the model.
  2385. if (
  2386. match_id
  2387. and item.require_previous_success
  2388. and not await self._check_previous_success(db, item, printer_id=match_id)
  2389. ):
  2390. gate_blocked.add(match_id)
  2391. continue
  2392. break
  2393. if match_id:
  2394. printer_id = match_id
  2395. chosen = candidate
  2396. break
  2397. per_model_reasons.append((candidate.target_model, match_reason or ""))
  2398. # Nothing is available and nothing has been woken this pass:
  2399. # switch one matching printer on. Assignment is left to the
  2400. # next pass, which sees the booted printer's live state
  2401. # instead of guessing at it seconds after connect (#2786).
  2402. if printer_id is None and not power_on_attempted and wakeable_candidates:
  2403. woken_id, attempted_id = await self._wake_printer_for_model(
  2404. db,
  2405. wakeable_candidates,
  2406. item.target_location,
  2407. busy_printers | interlocked.keys(),
  2408. wakeable_printer_ids,
  2409. require_plate_clear,
  2410. printer_scope=item_scope,
  2411. )
  2412. # An attempt spends the pass's one wake whether or not
  2413. # it worked: it has already blocked the queue loop for
  2414. # the boot wait. A failed printer is held out of later
  2415. # passes by its own cool-off, deliberately NOT by
  2416. # busy_printers — it is off, not busy, and labelling it
  2417. # busy would both misdescribe it in every later item's
  2418. # waiting reason and suppress the notification, since
  2419. # an all-busy reason is treated as needing no action.
  2420. power_on_attempted = attempted_id is not None
  2421. if woken_id is not None:
  2422. # Hold this item back rather than dispatching onto a
  2423. # printer whose AMS has not reported yet.
  2424. skip_reasons["powered_on_printer"] = skip_reasons.get("powered_on_printer", 0) + 1
  2425. continue
  2426. if printer_id is None and gate_blocked:
  2427. # The matcher reports the printers passed over as busy;
  2428. # what holds the job is their failed last print.
  2429. per_model_reasons = [(item.target_model, MODEL_GATE_WAIT_REASON)]
  2430. waiting_reason = None if printer_id else _collapse_waiting_reasons(per_model_reasons)
  2431. # Fold the winning variant's file and settings onto the item
  2432. # before anything else looks at them — the guards below and
  2433. # every step of the dispatch read the item's own columns.
  2434. if chosen is not None:
  2435. self._resolve_variant(item, chosen)
  2436. # Sent without the AMS, the job may print from the chosen
  2437. # printer's external spool, so its creator must be allowed
  2438. # that filament (#3522). Judged on the mapping dispatch
  2439. # will compute once it is assigned: a real tray in it
  2440. # turns the AMS back on (#2595), and the spool isn't fed.
  2441. # Checked before assigning: a printer assigned only to be
  2442. # given back would be on every pass. It waits with the
  2443. # filament named instead.
  2444. if (
  2445. printer_id
  2446. and item.use_ams is False
  2447. and not (await self._creator_filament_scope(db, item.created_by_id)).is_unrestricted
  2448. ):
  2449. planned = await self._compute_ams_mapping_for_printer(db, printer_id, item)
  2450. if spool_reason := await self._blocked_external_spools(db, printer_id, item, planned):
  2451. printer_id = None
  2452. waiting_reason = spool_reason
  2453. skip_reasons["filament_out_of_scope"] = skip_reasons.get("filament_out_of_scope", 0) + 1
  2454. # Update waiting_reason if changed and send notification when first waiting
  2455. if item.waiting_reason != waiting_reason:
  2456. was_waiting = item.waiting_reason is not None
  2457. previous_reason = item.waiting_reason
  2458. item.waiting_reason = waiting_reason
  2459. await db.commit()
  2460. # Send waiting notification only when transitioning to waiting state
  2461. # and the reason requires user action (not just "all printers busy")
  2462. # Also when a job that only waited for a busy printer
  2463. # now waits on something the user has to do.
  2464. if (
  2465. waiting_reason
  2466. and (not was_waiting or self._is_busy_only(previous_reason or ""))
  2467. and not _asks_nothing_new(waiting_reason)
  2468. ):
  2469. job_name = await self._get_job_name(db, item)
  2470. await notification_service.on_queue_job_waiting(
  2471. job_name=job_name,
  2472. target_model=_candidate_model_label(candidates) or item.target_model,
  2473. waiting_reason=waiting_reason,
  2474. db=db,
  2475. )
  2476. if printer_id:
  2477. # Before claiming the printer: hold back rather than race
  2478. # another dispatch over the same transient library row.
  2479. # Checked here so a held item does not get a printer
  2480. # assigned and then sit on it. See _library_row_conflict().
  2481. #
  2482. # No busy_printers.add() here, unlike the fixed-printer
  2483. # branch above: that one protects its printer's own queue
  2484. # ordering, but this item was never assigned to `printer_id`
  2485. # — the matcher merely offered it. Marking it busy would
  2486. # strand an idle printer for the rest of the pass.
  2487. if _library_row_conflict(item):
  2488. skip_reasons["library_row_in_use"] = skip_reasons.get("library_row_in_use", 0) + 1
  2489. continue
  2490. if await _edited_since_loaded(db, item):
  2491. continue
  2492. # Assign printer and start - clear waiting reason
  2493. item.printer_id = printer_id
  2494. item.waiting_reason = None
  2495. logger.info("Model-based assignment: queue item %s assigned to printer %s", item.id, printer_id)
  2496. printer = await self._get_printer(db, printer_id)
  2497. # A mapping on this item was not made for the printer just
  2498. # picked: it came with a job moved here from a fixed
  2499. # printer, or from a variant. Its tray IDs can name a
  2500. # tray of the right type in another colour here, which
  2501. # the fit check lets through (#3239). Match afresh, by
  2502. # type and colour.
  2503. if item.ams_mapping:
  2504. logger.info(
  2505. "Queue item %s: dropping stored ams_mapping %s, not made for printer %s",
  2506. item.id,
  2507. item.ams_mapping,
  2508. printer_id,
  2509. )
  2510. item.ams_mapping = None
  2511. # On a nozzle-rack printer its own nozzle mapping likewise
  2512. # (set through the API or an order's plate spec): it names
  2513. # rack positions, and the dispatch sends a stored one as-is
  2514. # instead of resolving this printer's rack. Elsewhere it
  2515. # holds extruder indices, the same on every printer of the
  2516. # model (an H2D VP keeps the slicer's for that reason), and
  2517. # one that came with the chosen variant belongs to that
  2518. # variant's file: both stay.
  2519. if (
  2520. (chosen is None or chosen.variant is None)
  2521. and printer is not None
  2522. and is_nozzle_rack_model(printer.model)
  2523. ):
  2524. item.nozzle_mapping = None
  2525. # Resolve the AMS mapping for the assigned printer. It is
  2526. # always missing here, so this computes it, and it also
  2527. # self-heals a bogus stored [-1] (#2589).
  2528. unmappable = await self._ensure_ams_mapping(db, printer_id, item)
  2529. if isinstance(unmappable, FilamentScopeHold):
  2530. # Its creator may not print it on this printer as
  2531. # it stands (#3522). The checks before choosing it
  2532. # catch nearly all of this; what is left changed in
  2533. # between. Held, not failed, and given back to the
  2534. # pool with its mapping, as the unmatched hold
  2535. # does: "any X1C" means any.
  2536. logger.info(
  2537. "Queue item %s held, released from printer %s by its creator's filament rules — %s",
  2538. item.id,
  2539. printer_id,
  2540. unmappable.reason,
  2541. )
  2542. item.printer_id = None
  2543. drop_printer_resolved(item)
  2544. item.waiting_reason = unmappable.reason
  2545. await db.commit()
  2546. skip_reasons["filament_out_of_scope"] = skip_reasons.get("filament_out_of_scope", 0) + 1
  2547. continue
  2548. if unmappable:
  2549. await self._fail_unmappable_item(db, item, printer_id, unmappable)
  2550. continue
  2551. # Announced only once the mapping and the creator's
  2552. # filament rules let it stay: a job given back above
  2553. # would be assigned, and announced, again on every
  2554. # pass, and the notification has no de-duplication.
  2555. await notification_service.on_queue_job_assigned(
  2556. job_name=await self._get_job_name(db, item),
  2557. printer_id=printer_id,
  2558. printer_name=printer.name if printer else "Unknown",
  2559. target_model=item.target_model,
  2560. db=db,
  2561. )
  2562. # Filament-deficit pre-dispatch check (#1496).
  2563. if await self._block_on_filament_deficit(db, item):
  2564. continue
  2565. # Unmatched-filament pre-dispatch check (#2799). Model-based
  2566. # selection already filters on filament type, so this is a
  2567. # backstop for an AMS that changed between assignment and
  2568. # dispatch, and for a type loaded on the wrong nozzle.
  2569. # The assignment made above is released with the hold —
  2570. # this item asked for a model, not this printer.
  2571. if await self._block_on_unmatched_filament(db, item, release_assignment=True):
  2572. continue
  2573. # As for a pinned job: drying stops once the print is
  2574. # decided, unless it may run through it.
  2575. if self._drying_in_progress.get(
  2576. printer_id
  2577. ) and not await self._drying_may_continue_through_print(db, printer_id):
  2578. await self._stop_drying(printer_id)
  2579. _claim_library_row(item)
  2580. dispatch_ids.append(item.id)
  2581. claim_printer(printer_id)
  2582. # SJF starvation guard: mark items that were jumped
  2583. if sjf_enabled and item.print_time_seconds is not None:
  2584. for other in items:
  2585. if (
  2586. other.id != item.id
  2587. and other.id not in dispatch_ids
  2588. and other.status == "pending"
  2589. and _could_take_printer(other, printer_id, item.target_model)
  2590. and not other.been_jumped
  2591. and other.position < item.position
  2592. and (
  2593. other.print_time_seconds is None
  2594. or other.print_time_seconds > item.print_time_seconds
  2595. )
  2596. ):
  2597. other.been_jumped = True
  2598. await db.commit()
  2599. # Log the decisions BEFORE dispatching. The dispatch below blocks for
  2600. # as long as the slowest upload takes (minutes on a big 3MF), and a
  2601. # skip summary that only lands after the transfers have finished is
  2602. # useless for working out why an item did not go out.
  2603. if skip_reasons:
  2604. logger.info("Queue skip summary: %s", skip_reasons)
  2605. for pid in sorted(busy_printers):
  2606. reason = busy_reasons.get(pid, "no reason recorded")
  2607. if pid in claimed_printers:
  2608. logger.info("Queue: printer %d reserved — %s", pid, reason)
  2609. continue
  2610. # The three live fields stay, because they are what someone
  2611. # reading a bundle wants next -- but they are labelled as read
  2612. # now, not as the state the decision was made on, which is what
  2613. # made the old line contradict itself.
  2614. state = printer_manager.get_status(pid)
  2615. logger.info(
  2616. "Queue: printer %d unavailable — %s (now: connected=%s, state=%s, awaiting_plate_clear=%s)",
  2617. pid,
  2618. reason,
  2619. printer_manager.is_connected(pid),
  2620. state.state if state else "NO_STATUS",
  2621. printer_manager.is_awaiting_plate_clear(pid),
  2622. )
  2623. # Keep-warm is a comfort feature; dispatch is not. It sits between
  2624. # selection and `_launch_uploads`, so anything raising here would
  2625. # discard this tick's selections — computed AMS mappings and all —
  2626. # and, on a persistent fault, stop the queue dispatching entirely.
  2627. # Same reasoning as the deficit check's guard below: never let an
  2628. # auxiliary check wedge the queue. The bed simply stays wherever it
  2629. # was, and the next tick tries again.
  2630. try:
  2631. await self._apply_keep_warm(
  2632. db, items, dispatch_ids, busy_printers - backoff_printers, require_plate_clear
  2633. )
  2634. except Exception as e:
  2635. logger.warning("Keep-warm pass failed, continuing with dispatch: %s", e, exc_info=True)
  2636. # Read the concurrency limit BEFORE the commit below, not inside
  2637. # _dispatch_selected(). A SELECT on this session after the commit
  2638. # implicitly opens a fresh transaction that nothing then closes, and
  2639. # it would stay open for the whole dispatch — minutes of "idle in
  2640. # transaction" on Postgres (pinned MVCC snapshot, vacuum blocked),
  2641. # and on SQLite a pinned WAL read snapshot that stops the WAL being
  2642. # checkpointed while every dispatch is writing to it.
  2643. upload_limit = max(1, await self._get_int_setting(db, "queue_max_concurrent_uploads", default=4))
  2644. # Selection is done; every decision above is recorded on `db`
  2645. # (model-based printer assignment, computed ams_mapping). Flush it
  2646. # before the dispatch tasks open their own sessions, or they will
  2647. # read a row that still says printer_id=None. This also releases the
  2648. # connection back to the pool for the duration of the dispatch.
  2649. await db.commit()
  2650. if dispatch_ids:
  2651. item_printers = {it.id: it.printer_id for it in items}
  2652. self._launch_uploads(dispatch_ids, item_printers, upload_limit)
  2653. # Auto-drying: start drying on idle printers that have no pending queue items
  2654. await self._check_auto_drying(db, items, dispatching_printers)
  2655. # Low filament: alert on assigned spools that have crossed their
  2656. # low-stock threshold (#2913). Runs on both paths out of this method
  2657. # for the same reason auto-drying does — an empty queue does not mean
  2658. # the spools in the printers stopped mattering.
  2659. await self._check_filament_low(db)
  2660. # Stock forecast: alert when a filament SKU reaches its reorder point or
  2661. # is about to run out before a replenishment could arrive (#2955).
  2662. await self._check_stock_forecast(db)
  2663. # Keep the loop on the fast interval while any upload is in flight so
  2664. # a slot freed mid-tick refills within seconds rather than after the
  2665. # 30 s idle sleep (#2602). Selecting anything this pass (launched or
  2666. # deferred because the pool was full) also counts as productive.
  2667. return bool(dispatch_ids) or bool(self._inflight)
  2668. def _launch_uploads(self, item_ids: list[int], item_printers: dict[int, int | None], limit: int) -> None:
  2669. """Launch selected uploads as a refillable pool, capped at ``limit`` (#2602).
  2670. Dispatch used to happen inline in the selection loop: ``await
  2671. _start_print(db, item)`` per item in turn. Since ``_start_print``
  2672. performs the FTP upload, that serialized every printer behind every
  2673. other printer's transfer even though the printers are independent
  2674. machines; #2555 moved it to a parallel ``asyncio.gather()``. But that
  2675. gather was awaited before ``check_queue`` returned, so the run loop
  2676. stayed blocked until the *slowest* upload in the batch finished — a
  2677. 513 s upload left 15 of 16 configured slots idle for 8.5 minutes on a
  2678. 93-printer farm even as other printers came free (#2602).
  2679. Each upload now runs as an independent background task tracked in
  2680. ``self._inflight``. check_queue excludes in-flight item_ids (still
  2681. `pending` until their upload completes) and their printers from the
  2682. next pass's selection, and this method launches at most
  2683. ``limit - len(self._inflight)`` new uploads, so a freed slot refills on
  2684. the next fast tick instead of waiting out the whole batch. The bound
  2685. exists because the printers are independent but the host is not: each
  2686. in-flight upload holds a thread in the FTP pool, a TLS session and a
  2687. file handle.
  2688. The no-overlapping-dispatch invariant the batch-await used to provide
  2689. is now carried by the in-flight exclusion in check_queue. Everything
  2690. else — the pending->printing CAS, the busy-printer guard (#2598), the
  2691. per-printer hold, and each item's independent failure handling — still
  2692. lives in ``_start_print`` and runs per task exactly as before.
  2693. Synchronous on purpose: it registers every launched task into
  2694. ``self._inflight`` before returning, so the next (sequential) tick sees
  2695. an accurate in-flight count with no interleaving await.
  2696. """
  2697. free = limit - len(self._inflight)
  2698. if free <= 0:
  2699. logger.info(
  2700. "Upload pool full (%d/%d in flight) — deferring %d item(s) to a later tick: %s",
  2701. len(self._inflight),
  2702. limit,
  2703. len(item_ids),
  2704. item_ids,
  2705. )
  2706. return
  2707. to_launch = item_ids[:free]
  2708. deferred = item_ids[free:]
  2709. logger.info(
  2710. "Launching %d upload(s) (pool %d/%d in flight)%s",
  2711. len(to_launch),
  2712. len(self._inflight),
  2713. limit,
  2714. f" — deferring {deferred} to a later tick" if deferred else "",
  2715. )
  2716. for item_id in to_launch:
  2717. task = spawn_background_task(
  2718. self._dispatch_one(item_id, item_printers.get(item_id)),
  2719. name=f"queue-upload-{item_id}",
  2720. )
  2721. self._inflight[item_id] = (task, item_printers.get(item_id))
  2722. # Prune on completion so the freed slot is refillable next tick.
  2723. # spawn_background_task already logs any uncaught exception; this
  2724. # only reclaims the pool slot (fires on success, failure, or cancel).
  2725. task.add_done_callback(lambda _t, iid=item_id: self._inflight.pop(iid, None))
  2726. async def _dispatch_one(self, item_id: int, selected_printer_id: int | None = None) -> None:
  2727. """Upload + start one queue item in its own session (pool worker, #2602).
  2728. Its own session: pool workers run concurrently and an AsyncSession is
  2729. not safe to share across tasks; it also keeps a slow upload from pinning
  2730. the scheduler's session (and, on SQLite, its transaction) open for the
  2731. transfer's duration.
  2732. ``selected_printer_id`` is the printer this item was selected for, taken
  2733. from the same snapshot the caller used. It exists so the preheat pin can
  2734. be unwound on the paths that never reach the ``finally`` below — see the
  2735. claim failure a few lines down. Optional so the direct-call tests keep
  2736. working; when it is absent those paths simply behave as they did before.
  2737. """
  2738. async with async_session() as item_db:
  2739. # Claim the row for dispatch BEFORE reading the printer snapshot or
  2740. # touching any slow I/O (#2615). The claim is an atomic CAS on
  2741. # (status='pending', dispatching_at IS NULL); while it's held the edit
  2742. # routes reject reassignment (409), so printer_id can't change out from
  2743. # under the in-flight upload and split the queue row from the
  2744. # archive/expected-print/physical command.
  2745. if not await self._claim_for_dispatch(item_db, item_id, selected_printer_id):
  2746. logger.info(
  2747. "Queue item %s not claimable for dispatch (cancelled, removed, or already claimed) — skipping",
  2748. item_id,
  2749. )
  2750. # This return is outside the try/finally below, so the rollback
  2751. # has to happen here. Selecting this item already handed any
  2752. # keep-warm hold on its printer over to the preheat pin
  2753. # (`_sweep_keep_warm`), on the promise that this dispatch would
  2754. # unwind it. Bailing without doing so leaves the bed hot with
  2755. # nothing tracking it: the keep-warm entry is gone, so the
  2756. # max-duration cap no longer applies, and if this was the
  2757. # printer's last pending item nothing else will ever turn it
  2758. # off. Reachable whenever a cancel or delete lands between
  2759. # selection and the claim.
  2760. if selected_printer_id is not None:
  2761. self._rollback_preheat_pin(item_id, selected_printer_id)
  2762. return
  2763. # Seeded from the caller's snapshot so the `item vanished` return
  2764. # below still unwinds the pin; overwritten with the row's own
  2765. # printer_id as soon as we have it.
  2766. item_printer_id: int | None = selected_printer_id
  2767. try:
  2768. item = await item_db.get(PrintQueueItem, item_id)
  2769. if not item:
  2770. logger.info("Queue item %s vanished after claim — skipping", item_id)
  2771. return
  2772. item_printer_id = item.printer_id
  2773. # The pass judged the job as it loaded it, and the creator
  2774. # could edit the feed (use_ams, the mapping, "Print anyway")
  2775. # until the claim above, after which edits are refused (409).
  2776. # Judged again as claimed, so such an edit can't carry the
  2777. # job onto a filament its creator may not use (#3522). Cheap:
  2778. # the scope and requirement memos are warm, and an
  2779. # unrestricted creator stops at the first lookup. The claim is
  2780. # cleared below, and the next pass holds the job.
  2781. if scope_reason := await self._claimed_scope_violation(item_db, item):
  2782. logger.info(
  2783. "Queue item %s changed before dispatch and its creator's filament rules now hold it — %s",
  2784. item_id,
  2785. scope_reason,
  2786. )
  2787. return
  2788. await self._start_print(item_db, item)
  2789. finally:
  2790. # Undo an expected-print registration whose print command never
  2791. # went out. One choke point covers every way `_start_print` can
  2792. # end without sending: a raised exception (a DB failure mid-
  2793. # dispatch is the reported case), an early return, a cancel
  2794. # winning the #1853 CAS, or `start_print()` returning False.
  2795. # A confirmed send removes the entry itself, so this is a no-op
  2796. # on the happy path.
  2797. self._rollback_unconfirmed_expected_print(item_id)
  2798. # Likewise the print user it set: the printer's next print is
  2799. # not this item's.
  2800. self._rollback_unconfirmed_print_user(item_id)
  2801. # Its archive is no longer held for it (archive_work).
  2802. self._release_held_archive(item_id)
  2803. # Mirror the pre-#1625 background-dispatch lifecycle: a
  2804. # reservation survives only after start_print() accepted the
  2805. # command. Failure, cancellation, deferral, and exceptions all
  2806. # release it here.
  2807. await asyncio.shield(self._release_unconfirmed_budget_reservation(item_id))
  2808. # Unwind preheat state (bed/chamber/airduct) if the
  2809. # dispatch aborted before the print's own gcode took over.
  2810. # `_start_print` clears the pin on successful `start_print()`;
  2811. # anything still present here is by definition an aborted
  2812. # dispatch and gets rolled back so the printer isn't left
  2813. # heating for a job that isn't happening.
  2814. if item_printer_id is not None:
  2815. self._rollback_preheat_pin(item_id, item_printer_id)
  2816. # The cancellation flag only has meaning while this dispatch is
  2817. # running; drop it so the set cannot grow without bound and a
  2818. # re-queued item never inherits a stale cancellation.
  2819. self._cancelled_dispatches.discard(item_id)
  2820. # Release the claim on every exit. Once dispatch has finished the
  2821. # row's status carries the lock (printing/failed/cancelled are all
  2822. # != pending), so the token is only needed for the duration of the
  2823. # upload. A row left pending (e.g. busy-printer deferral) becomes
  2824. # dispatchable again on the next tick.
  2825. await self._clear_dispatch_claim(item_db, item_id)
  2826. async def _requeue_after_upload_refused(
  2827. self,
  2828. db: AsyncSession,
  2829. item: PrintQueueItem,
  2830. printer: Printer,
  2831. error_msg: str,
  2832. toast_uid: int | None,
  2833. kind: FtpFailureKind,
  2834. ) -> None:
  2835. """Put an item back in the queue after its file never reached the printer (#3210).
  2836. The printer goes into `_upload_backoff`, which keeps every item away
  2837. from it until the window ends. The window grows with each refusal in a
  2838. row; see ``_upload_backoff_seconds``.
  2839. An item pinned to this printer keeps it, waits there, and is the one
  2840. that knocks again. An item for "Any <model>" goes back to the pool
  2841. instead, so another printer of the model can take it (#3409). Only the
  2842. assignment goes: the next model assignment drops whatever on the row
  2843. was worked out for one printer, as it does for every model job.
  2844. The first refusal in a run sends the one "Job waiting" notification,
  2845. and a later one only if it asks for something else (a card after a
  2846. dropped connection). It cannot wait for the hold the way it used to: a
  2847. released item is not held on this printer, so nothing would say why
  2848. it stopped. A job cancelled or moved during the upload sends none.
  2849. `dispatch_attempts` is not charged: that budget bounds a printer that
  2850. takes the file and then never starts, which this is not.
  2851. """
  2852. refusals = self._upload_refusals.get(printer.id, 0) + 1
  2853. self._upload_refusals[printer.id] = refusals
  2854. self._upload_refusal_kind[printer.id] = kind
  2855. backoff = _upload_backoff_seconds(refusals)
  2856. self._upload_backoff[printer.id] = time.monotonic() + backoff
  2857. released = False
  2858. if item.target_model:
  2859. # Only while it is still pending on this printer: a cancel, or a
  2860. # move the user made during the upload, stands.
  2861. result = cast(
  2862. CursorResult,
  2863. await db.execute(
  2864. update(PrintQueueItem)
  2865. .where(PrintQueueItem.id == item.id)
  2866. .where(PrintQueueItem.status == "pending")
  2867. .where(PrintQueueItem.printer_id == printer.id)
  2868. .values(printer_id=None, error_message=None)
  2869. .execution_options(synchronize_session=False)
  2870. ),
  2871. )
  2872. await db.commit()
  2873. released = result.rowcount > 0
  2874. elif item.error_message:
  2875. # The row is still `pending` -- it only moves to `printing` after a
  2876. # successful upload -- so there is no status to write back. Writing
  2877. # one anyway would undo a cancel that landed during the upload.
  2878. item.error_message = None
  2879. await db.commit()
  2880. logger.warning(
  2881. "Queue item %s: upload to printer %s (%s) never reached it — %s Kept in the queue%s; "
  2882. "refusal %d in a row, so the printer is out of dispatch for %ds.",
  2883. item.id,
  2884. printer.id,
  2885. printer.name,
  2886. error_msg,
  2887. " and returned to the pool" if released else "",
  2888. refusals,
  2889. backoff,
  2890. )
  2891. reason = _refused_upload_hold_reason(printer.name or f"printer {printer.id}", kind)
  2892. if self._upload_refusal_notified.get(printer.id) != reason and await self._still_waiting_on(
  2893. db, item.id, None if released else printer.id
  2894. ):
  2895. self._upload_refusal_notified[printer.id] = reason
  2896. try:
  2897. await notification_service.on_queue_job_waiting(
  2898. job_name=await self._get_job_name(db, item),
  2899. target_model=item.target_model or printer.model or "",
  2900. waiting_reason=reason,
  2901. db=db,
  2902. printer_id=printer.id,
  2903. )
  2904. except Exception as e:
  2905. # The job is safe in the queue either way.
  2906. logger.debug("Waiting notification failed for item %s: %s", item.id, e)
  2907. try:
  2908. # Closes the dispatch toast for this attempt. Without it the toast
  2909. # keeps spinning on an upload that has ended.
  2910. await ws_manager.send_queue_item_failed(
  2911. user_id=toast_uid,
  2912. queue_item_id=item.id,
  2913. printer_id=printer.id,
  2914. reason="upload_failed",
  2915. )
  2916. except Exception:
  2917. pass # toast is best-effort
  2918. @staticmethod
  2919. async def _still_waiting_on(db: AsyncSession, item_id: int, printer_id: int | None) -> bool:
  2920. """Whether the item is still pending, on *printer_id* (None: unassigned)."""
  2921. row = (
  2922. await db.execute(
  2923. select(PrintQueueItem.status, PrintQueueItem.printer_id).where(PrintQueueItem.id == item_id)
  2924. )
  2925. ).one_or_none()
  2926. return row is not None and row.status == "pending" and row.printer_id == printer_id
  2927. def _rollback_unconfirmed_print_user(self, item_id: int) -> None:
  2928. printer_id = self._unconfirmed_print_user.pop(item_id, None)
  2929. if printer_id is not None:
  2930. printer_manager.clear_current_print_user_for_item(printer_id, item_id)
  2931. def _forget_unstarted_print(self, item_id: int, *, unless_printing: bool, keep_expected: bool = False) -> None:
  2932. """Withdraw the expected print and print user of a sent print that never started.
  2933. Both tell the rest of Bambuddy that the printer's next print is this
  2934. item's: left behind, a print started by hand later is taken for it.
  2935. With ``unless_printing``, a printer that is printing after all keeps
  2936. them, as that print is most likely this one starting late.
  2937. ``keep_expected`` withdraws only the print user, which vouches for any
  2938. print on the printer. The expected print names this item's file, so a
  2939. late start of it is still matched to its archive.
  2940. """
  2941. sent = self._sent_unstarted_print.get(item_id)
  2942. if sent is None:
  2943. return
  2944. printer_id, remote_filename, archive_id = sent
  2945. if unless_printing:
  2946. status = printer_manager.get_status(printer_id)
  2947. if status is not None and status.state in _ACTIVE_PRINT_STATES:
  2948. return
  2949. printer_manager.clear_current_print_user_for_item(printer_id, item_id)
  2950. if keep_expected:
  2951. return
  2952. del self._sent_unstarted_print[item_id]
  2953. if remote_filename is not None and archive_id is not None:
  2954. try:
  2955. from backend.app.main import unregister_expected_print
  2956. unregister_expected_print(printer_id, remote_filename, archive_id)
  2957. except Exception:
  2958. logger.warning("Queue item %s: failed to unregister expected print", item_id)
  2959. def _rollback_unconfirmed_expected_print(self, item_id: int) -> None:
  2960. """Drop an expectation for a print command that was never sent.
  2961. Best-effort and never raises: this runs in the ``finally`` of dispatch,
  2962. where the interesting exception is usually the one already propagating.
  2963. """
  2964. pending = self._unconfirmed_expected_print.pop(item_id, None)
  2965. if pending is None:
  2966. return
  2967. printer_id, remote_filename, archive_id = pending
  2968. try:
  2969. from backend.app.main import unregister_expected_print
  2970. unregister_expected_print(printer_id, remote_filename, archive_id)
  2971. except Exception:
  2972. logger.warning(
  2973. "Queue item %s: failed to unregister expected print (printer=%s, file=%s, archive=%s)",
  2974. item_id,
  2975. printer_id,
  2976. remote_filename,
  2977. archive_id,
  2978. exc_info=True,
  2979. )
  2980. async def _release_unconfirmed_budget_reservation(self, item_id: int) -> None:
  2981. """Release a queue reservation without touching the dispatch session."""
  2982. if item_id not in self._unconfirmed_budget_reservations:
  2983. return
  2984. for attempt in range(1, 4):
  2985. async with async_session() as cleanup_db:
  2986. try:
  2987. await release_budget_reservation(
  2988. cleanup_db,
  2989. source_type="print_queue",
  2990. source_id=item_id,
  2991. status="released",
  2992. )
  2993. await cleanup_db.commit()
  2994. self._unconfirmed_budget_reservations.discard(item_id)
  2995. return
  2996. except Exception as exc:
  2997. try:
  2998. await cleanup_db.rollback()
  2999. except Exception:
  3000. pass
  3001. if attempt == 3:
  3002. logger.error(
  3003. "Queue item %s: failed to release budget reservation after %d attempts: %s",
  3004. item_id,
  3005. attempt,
  3006. exc,
  3007. )
  3008. return
  3009. await asyncio.sleep(0.5 * attempt)
  3010. @staticmethod
  3011. def _reported_bed_target(printer_id: int) -> int | None:
  3012. """The bed target firmware currently reports, or None if it can't be read.
  3013. None means "no evidence", not "zero" — callers must not treat it as a
  3014. temperature. Deliberately total: this feeds cleanup paths that run in a
  3015. ``finally``, where a malformed status must not become the exception the
  3016. caller sees.
  3017. """
  3018. try:
  3019. state = printer_manager.get_status(printer_id)
  3020. if state is None:
  3021. return None
  3022. temps = state.temperatures
  3023. if not isinstance(temps, dict):
  3024. return None
  3025. return int(float(temps.get("bed_target", 0) or 0))
  3026. except (TypeError, ValueError, AttributeError):
  3027. return None
  3028. def _rollback_preheat_pin(self, item_id: int, printer_id: int) -> None:
  3029. """Unwind everything preheat set when dispatch did NOT hand off to a running print.
  3030. Turns the bed heater off, the chamber heater off, and opens the
  3031. airduct flap back to cooling — for whichever of those preheat
  3032. actually applied. `_start_print` clears the pin on successful
  3033. `start_print()`; anything still present when `_dispatch_one` exits
  3034. is by definition an aborted dispatch and gets rolled back here.
  3035. The bed is the one action that can be declined: if firmware has since
  3036. been given a target other than the one we pinned, it belongs to someone
  3037. else and is left alone. See the comment at that branch.
  3038. Also called directly from `_dispatch_one`'s claim-failure return, which
  3039. never reaches the ``finally``.
  3040. Best-effort and never raises — this runs in the ``finally`` of dispatch.
  3041. """
  3042. pin = self._preheat_pin.pop(printer_id, set())
  3043. pinned_bed = self._preheat_pin_bed.pop(printer_id, None)
  3044. if not pin:
  3045. return
  3046. client = printer_manager.get_client(printer_id)
  3047. if client is None:
  3048. logger.info(
  3049. "Dispatch item %s (printer %d): preheat rollback skipped — no client",
  3050. item_id,
  3051. printer_id,
  3052. )
  3053. return
  3054. if "bed" in pin:
  3055. # Only undo our own target. If firmware reports something else, the
  3056. # user or another writer owns the bed now and zeroing it would
  3057. # clobber their choice -- the same guard `_release_keep_warm`
  3058. # applies to a keep-warm hold.
  3059. #
  3060. # Every uncertain case switches the bed off rather than leaving it:
  3061. # no recorded target (a pin written before this bookkeeping, or a
  3062. # setter that raised after pinning) and an unreadable status both
  3063. # fall through. A bed left hot with no owner is the worse failure,
  3064. # and this runs in a `finally` where raising would mask the real
  3065. # exception.
  3066. cur_bed_target = self._reported_bed_target(printer_id) if pinned_bed is not None else None
  3067. if cur_bed_target is not None and cur_bed_target != pinned_bed:
  3068. logger.info(
  3069. "Dispatch item %s (printer %d): rollback skipped bed → 0 (firmware target %d != pinned %d)",
  3070. item_id,
  3071. printer_id,
  3072. cur_bed_target,
  3073. pinned_bed,
  3074. )
  3075. else:
  3076. try:
  3077. client.set_bed_temperature(0)
  3078. except Exception as exc:
  3079. logger.warning("Dispatch item %s: rollback bed → 0 failed: %s", item_id, exc)
  3080. if "chamber" in pin:
  3081. try:
  3082. client.set_chamber_temperature(0)
  3083. except Exception as exc:
  3084. logger.warning("Dispatch item %s: rollback chamber → 0 failed: %s", item_id, exc)
  3085. if "airduct" in pin:
  3086. try:
  3087. client.set_airduct_mode("cooling")
  3088. except Exception as exc:
  3089. logger.warning("Dispatch item %s: rollback airduct → cooling failed: %s", item_id, exc)
  3090. logger.info(
  3091. "Dispatch item %s (printer %d): preheat rollback → %s",
  3092. item_id,
  3093. printer_id,
  3094. sorted(pin),
  3095. )
  3096. async def _claim_for_dispatch(self, db: AsyncSession, item_id: int, printer_id: int | None = None) -> bool:
  3097. """Atomically stamp ``dispatching_at`` on a still-pending, unclaimed row.
  3098. Returns True if this call won the claim, False if the row was already
  3099. claimed, no longer pending (cancelled mid-tick), or removed. The CAS is
  3100. the load-bearing guard against reassign-during-dispatch (#2615).
  3101. ``printer_id`` is the printer the item was selected for: a row moved to
  3102. another printer since is not claimed, and the next pass selects it for
  3103. that one."""
  3104. query = (
  3105. update(PrintQueueItem)
  3106. .where(PrintQueueItem.id == item_id)
  3107. .where(PrintQueueItem.status == "pending")
  3108. .where(PrintQueueItem.dispatching_at.is_(None))
  3109. )
  3110. if printer_id is not None:
  3111. query = query.where(PrintQueueItem.printer_id == printer_id)
  3112. res = await db.execute(query.values(dispatching_at=datetime.now(timezone.utc)))
  3113. await db.commit()
  3114. return res.rowcount > 0
  3115. async def _clear_dispatch_claim(self, db: AsyncSession, item_id: int) -> None:
  3116. """Clear the dispatch claim (#2615). Best-effort: a failure here must not
  3117. mask the dispatch outcome.
  3118. Retried, because the failure mode in practice is transient and narrow: a
  3119. database that is momentarily unreachable — PostgreSQL out of connection
  3120. slots is the observed case — refuses this write for a second or two while
  3121. the dispatch that just ended is still holding the row out of the selection
  3122. query. One attempt was enough to wedge the item; a couple of spaced
  3123. attempts clear it. Each attempt rolls back first, since a failed write
  3124. leaves the session needing it before it can be reused.
  3125. If every attempt fails, ``_clear_stale_dispatch_claims`` picks the row up
  3126. on the next quiet tick.
  3127. """
  3128. for attempt in range(1, 4):
  3129. try:
  3130. await db.execute(update(PrintQueueItem).where(PrintQueueItem.id == item_id).values(dispatching_at=None))
  3131. await db.commit()
  3132. return
  3133. except Exception as exc:
  3134. try:
  3135. await db.rollback()
  3136. except Exception:
  3137. pass
  3138. if attempt == 3:
  3139. logger.warning(
  3140. "Queue item %s: failed to clear dispatch claim after %d attempts: %s "
  3141. "— a later quiet tick will release it",
  3142. item_id,
  3143. attempt,
  3144. exc,
  3145. )
  3146. return
  3147. await asyncio.sleep(0.5 * attempt)
  3148. async def _printers_for_model(
  3149. self,
  3150. db: AsyncSession,
  3151. model: str,
  3152. target_location: str | None = None,
  3153. printer_scope: PrinterScope = ALL_PRINTERS,
  3154. ) -> list[Printer]:
  3155. """Active printers of *model*, optionally narrowed to one location.
  3156. Shared by the matcher and by the smart-plug wake step (#2786) so both
  3157. answer "which printers can this job run on" from one query — a job can
  3158. only be woken onto a printer the matcher would also have considered.
  3159. ``printer_scope`` is the job creator's (#1727).
  3160. In the order of Settings > Queue > printer order (#2915), fewest print
  3161. hours first by default (#3480): both take the first printer that
  3162. fits, so an "Any" job goes to the least worn machine and a farm's jobs
  3163. spread by wear instead of all landing on the lowest id. The hours are
  3164. the maintenance counter's (runtime plus the baseline offset), which
  3165. counts every print, wherever it was started. Ties (printers still at
  3166. zero) go by id, so the order is stable.
  3167. """
  3168. normalized_model = normalize_printer_model(model) or model
  3169. query = (
  3170. select(Printer)
  3171. .where(func.lower(Printer.model) == normalized_model.lower())
  3172. .where(Printer.is_active == True) # noqa: E712
  3173. )
  3174. if target_location:
  3175. query = query.where(Printer.location == target_location)
  3176. if (clause := printer_scope.where_strict(Printer.id)) is not None:
  3177. query = query.where(clause)
  3178. result = await db.execute(query)
  3179. printers = list(result.scalars().all())
  3180. order = await self._get_model_dispatch_order(db)
  3181. last_finished: dict[int, datetime] | None = None
  3182. if order == "least_recently_used" and printers:
  3183. last_finished = await self._last_finished_by_printer(db, [p.id for p in printers])
  3184. return _order_printers(printers, order, last_finished)
  3185. async def _get_model_dispatch_order(self, db: AsyncSession) -> str:
  3186. """The order setting (#2915); anything unknown reads as the default."""
  3187. result = await db.execute(select(Settings).where(Settings.key == "model_dispatch_order"))
  3188. setting = result.scalar_one_or_none()
  3189. value = (setting.value if setting else "") or ""
  3190. return value if value in ("most_hours", "least_recently_used", "priority") else "fewest_hours"
  3191. async def _last_finished_by_printer(self, db: AsyncSession, printer_ids: list[int]) -> dict[int, datetime]:
  3192. """When each printer last finished a print, from the Print Log; none for one that never has."""
  3193. from backend.app.models.print_log import PrintLogEntry
  3194. result = await db.execute(
  3195. select(PrintLogEntry.printer_id, func.max(PrintLogEntry.completed_at))
  3196. .where(PrintLogEntry.printer_id.in_(printer_ids), PrintLogEntry.completed_at.is_not(None))
  3197. .group_by(PrintLogEntry.printer_id)
  3198. )
  3199. return {
  3200. printer_id: finished
  3201. for printer_id, finished in result.all()
  3202. if printer_id is not None and finished is not None
  3203. }
  3204. async def _wakeable_printer_ids(self, db: AsyncSession) -> set[int]:
  3205. """Printer IDs that at least one enabled ``auto_on`` plug can power on.
  3206. Read once per queue check rather than per printer: it decides both
  3207. whether the wake step has anything to do and how an offline printer is
  3208. worded in the waiting reason — "Offline" and "offline with no Auto On
  3209. plug" are different problems, and the second is the one the user has to
  3210. fix themselves (#2786).
  3211. """
  3212. result = await db.execute(
  3213. select(SmartPlug.printer_id)
  3214. .where(SmartPlug.printer_id.is_not(None))
  3215. .where(SmartPlug.enabled == True) # noqa: E712
  3216. .where(SmartPlug.auto_on == True) # noqa: E712
  3217. # Same rule as _auto_on_plugs: an MQTT plug cannot power a printer on (#3583).
  3218. .where(SmartPlug.plug_type != "mqtt")
  3219. )
  3220. return {pid for (pid,) in result.all() if pid is not None}
  3221. def _wake_recently_failed(self, printer_id: int) -> bool:
  3222. """True while this printer's failed power-on is still cooling off (#2786)."""
  3223. deadline = self._wake_failures.get(printer_id)
  3224. if deadline is None:
  3225. return False
  3226. if time.monotonic() >= deadline:
  3227. del self._wake_failures[printer_id]
  3228. return False
  3229. return True
  3230. async def _wake_printer_for_model(
  3231. self,
  3232. db: AsyncSession,
  3233. candidates: list[_ModelCandidate],
  3234. target_location: str | None,
  3235. exclude_ids: set[int],
  3236. wakeable_ids: set[int],
  3237. require_plate_clear: bool,
  3238. printer_scope: PrinterScope = ALL_PRINTERS,
  3239. ) -> tuple[int | None, int | None]:
  3240. """Power on one offline printer a model-based item could run on (#2786).
  3241. The fixed-printer branch has powered a printer on since smart plugs
  3242. existed. The model-based branch never could: its matcher drops an
  3243. offline printer into the "Offline:" waiting reason and nothing looks at
  3244. its plugs, so a class-targeted job with every matching printer switched
  3245. off sat pending forever. The reporter's log is the controlled
  3246. experiment — the same item, same plug, same Auto On setting, dispatched
  3247. the moment they edited it onto a specific printer.
  3248. Returns ``(woken_id, attempted_id)``. ``attempted_id`` is set whenever a
  3249. power-on was actually tried, so the caller can tell "nothing here was
  3250. wakeable" (both None — cheap, other items may still find something)
  3251. from "we tried and it did not come up" (only ``attempted_id`` — the
  3252. boot timeout has already been spent).
  3253. A printer whose last known trays cannot satisfy the job is passed over
  3254. rather than woken (#2876): the colours are readable while it is off, so
  3255. switching a farm on one machine at a time to discover them wakes
  3256. printers that could never have taken the job.
  3257. A printer whose last known nozzle cannot print the file is passed over
  3258. the same way (#3536).
  3259. Deliberately does NOT go on to match the job once a printer is up: AMS
  3260. trays arrive with the first status push after connect, so a filament
  3261. check against a printer that booted seconds ago can reject the printer
  3262. we just woke. The next queue pass matches it with live state.
  3263. At most one printer per pass. Each wake blocks the queue loop for the
  3264. boot wait, and a queue of ten class-targeted jobs must not switch on
  3265. ten printers inside one check.
  3266. """
  3267. for candidate in candidates:
  3268. if not candidate.target_model:
  3269. continue
  3270. required_types, filament_overrides = _filament_constraints(candidate)
  3271. printers = await self._printers_for_model(db, candidate.target_model, target_location, printer_scope)
  3272. # Fewest print hours first, as the matcher picks (#3480).
  3273. for printer in printers:
  3274. if printer.id in exclude_ids or printer.id not in wakeable_ids:
  3275. continue
  3276. if printer_manager.is_connected(printer.id):
  3277. continue
  3278. if self._wake_recently_failed(printer.id):
  3279. # Its plug did not bring it back a moment ago. Move on to a
  3280. # sibling instead of spending this pass — and every pass —
  3281. # on the same printer.
  3282. continue
  3283. if _plate_clear_holds(printer.id, require_plate_clear):
  3284. # Waking this one buys nothing: it would boot into IDLE and
  3285. # then be held by the plate-clear gate, which is exactly
  3286. # what the reporter's log shows happening for 80 minutes
  3287. # after a fixed-printer wake. The flag is Bambuddy-side and
  3288. # persisted, so it is readable while the printer is off.
  3289. logger.info(
  3290. "Not powering on printer %s for a %s job: it is awaiting plate-clear acknowledgment",
  3291. printer.id,
  3292. candidate.target_model,
  3293. )
  3294. continue
  3295. # #3536: the matcher rejects a printer whose nozzle cannot print
  3296. # the file, so waking one only spends this pass's single wake and
  3297. # the boot wait. A powered-off printer keeps its last status, so
  3298. # the nozzle is readable here. Fail-safe: no status or no reported
  3299. # nozzle means "fits" (None) and the printer is woken as before.
  3300. label = _wrong_nozzle_label(
  3301. printer.name, printer_manager.get_status(printer.id), candidate.sliced_nozzle
  3302. )
  3303. if label:
  3304. logger.info(
  3305. "Not powering on printer %s for a %s job: its nozzle cannot print the file (%s)",
  3306. printer.id,
  3307. candidate.target_model,
  3308. label,
  3309. )
  3310. continue
  3311. shortfall = self._cached_filament_shortfall(printer.id, required_types, filament_overrides)
  3312. if shortfall:
  3313. logger.info(
  3314. "Not powering on printer %s for a %s job: last known filament cannot satisfy it (needs %s)",
  3315. printer.id,
  3316. candidate.target_model,
  3317. ", ".join(shortfall),
  3318. )
  3319. continue
  3320. plugs = await self._get_smart_plugs(db, printer.id)
  3321. auto_on_plugs = self._auto_on_plugs(plugs)
  3322. if not auto_on_plugs:
  3323. # wakeable_ids said otherwise — the plug changed under us
  3324. # mid-pass. Nothing to do but move on.
  3325. continue
  3326. logger.info(
  3327. "No %s printer available for a queued job; powering on offline printer %s via smart plug(s)",
  3328. candidate.target_model,
  3329. printer.id,
  3330. )
  3331. primary_plug = self._pick_power_plug(auto_on_plugs)
  3332. if not await self._power_on_and_wait(primary_plug, printer.id, db):
  3333. logger.warning(
  3334. "Could not power on printer %s via smart plug; not trying it again for %ss",
  3335. printer.id,
  3336. self._wake_failure_cooloff,
  3337. )
  3338. self._wake_failures[printer.id] = time.monotonic() + self._wake_failure_cooloff
  3339. return None, printer.id
  3340. for extra_plug in [p for p in auto_on_plugs if p.id != primary_plug.id]:
  3341. try:
  3342. service = await smart_plug_manager.get_service_for_plug(extra_plug, db)
  3343. if await service.turn_on(extra_plug):
  3344. await publish_plug_state(
  3345. extra_plug.id,
  3346. extra_plug.name,
  3347. extra_plug.printer_id,
  3348. on=True,
  3349. ha_entity_id=extra_plug.ha_entity_id,
  3350. )
  3351. logger.info("Also powered on plug '%s' for printer %s", extra_plug.name, printer.id)
  3352. except Exception as e:
  3353. logger.warning("Failed to power on extra plug '%s': %s", extra_plug.name, e)
  3354. return printer.id, printer.id
  3355. return None, None
  3356. async def _find_idle_printer_for_model(
  3357. self,
  3358. db: AsyncSession,
  3359. model: str,
  3360. exclude_ids: set[int],
  3361. required_filament_types: list[str] | None = None,
  3362. target_location: str | None = None,
  3363. filament_overrides: list[dict] | None = None,
  3364. require_plate_clear: bool = True,
  3365. wakeable_ids: set[int] | None = None,
  3366. printer_scope: PrinterScope = ALL_PRINTERS,
  3367. is_short: Callable[[int], Awaitable[bool]] | None = None,
  3368. drying_ids: set[int] | None = None,
  3369. unmatched_on: Callable[[int], Awaitable[list[str]]] | None = None,
  3370. refused_uploads: Mapping[int, FtpFailureKind | None] | None = None,
  3371. sliced_nozzle: float | None = None,
  3372. ) -> tuple[int | None, str | None]:
  3373. """Find an idle, connected printer matching the model with compatible filaments.
  3374. Args:
  3375. db: Database session
  3376. model: Printer model to match (e.g., "X1C", "P1S")
  3377. exclude_ids: Printer IDs to exclude (already busy)
  3378. required_filament_types: Optional list of filament types needed (e.g., ["PLA", "PETG"])
  3379. If provided, only printers with all required types loaded will match.
  3380. target_location: Optional location filter. If provided, only printers in this location are considered.
  3381. filament_overrides: Optional list of override dicts. Each entry may include
  3382. ``force_color_match: true`` to require an exact type+color match
  3383. on the printer for that slot. Without the flag the existing
  3384. colour-preference logic applies.
  3385. wakeable_ids: Printers a smart plug can power on (#2786). Only changes how an
  3386. offline printer is worded: one Bambuddy will switch on reads
  3387. differently from one the user has to go and switch on themselves.
  3388. is_short: Says whether a printer's spools are too light for the job (#3137).
  3389. A printer it flags is passed over while another eligible one
  3390. is not. When every eligible printer is short, the first is
  3391. returned all the same, so the deficit gate holds the job there
  3392. and "Print Anyway" still has a printer to print on.
  3393. drying_ids: Excluded printers held by a drying cycle; named "(drying)"
  3394. in the reason, as a pinned job's wait is.
  3395. unmatched_on: Names the filaments the slot matcher would leave without a
  3396. tray on a printer (#3280). A printer it names any for is
  3397. passed over like one missing a filament type: the checks
  3398. above only ask whether each type and forced colour is
  3399. loaded somewhere, not on the right nozzle or once per slot.
  3400. refused_uploads: Excluded printers out of dispatch after a refused upload,
  3401. with why (#3409). Each is named with the reason its own
  3402. queue rows show, not as "Busy".
  3403. sliced_nozzle: Nozzle diameter the file was sliced for (#3536). A printer
  3404. without that nozzle is passed over; None (unknown) passes
  3405. every printer, as before.
  3406. Returns:
  3407. Tuple of (printer_id, waiting_reason):
  3408. - (printer_id, None) if a matching printer was found
  3409. - (None, reason) if no printer is available, with explanation
  3410. """
  3411. normalized_model = normalize_printer_model(model) or model
  3412. printers = await self._printers_for_model(db, model, target_location, printer_scope)
  3413. location_suffix = f" in {target_location}" if target_location else ""
  3414. if not printers:
  3415. return None, f"No active {normalized_model} printers{location_suffix} configured"
  3416. # Separate force-matched overrides from preference-only overrides
  3417. force_overrides = [o for o in (filament_overrides or []) if o.get("force_color_match")]
  3418. pref_overrides = [o for o in (filament_overrides or []) if not o.get("force_color_match")]
  3419. # Track reasons for skipping printers
  3420. printers_busy = []
  3421. # Printers out of dispatch after a refused upload, worded as the
  3422. # printer's own queue rows word it: "Busy" would read as printing (#3409).
  3423. printers_refused: list[str] = []
  3424. printers_offline = []
  3425. printers_offline_no_plug = []
  3426. printers_missing_filament: list[tuple[str, list[str]]] = []
  3427. printers_wrong_nozzle: list[str] = []
  3428. candidates: list[tuple[int, int]] = [] # (printer_id, color_match_count)
  3429. # The first eligible printer that is short on filament, kept in case
  3430. # every eligible printer is (#3137).
  3431. short_fallback: int | None = None
  3432. for printer in printers:
  3433. # #3536: the pick ignored the nozzle, so a job sliced for 0.6 mm went to a
  3434. # 0.4 mm printer and the #1899 guard then failed it in _start_print instead
  3435. # of trying another printer. Fail-safe: no diameter or no reported nozzle
  3436. # fits (_wrong_nozzle_label returns None). Checked first on purpose: a
  3437. # wrong-nozzle printer that is busy, excluded or offline must not read as
  3438. # "Busy"/"Offline", which would hold the job silently forever.
  3439. wrong_nozzle = _wrong_nozzle_label(printer.name, printer_manager.get_status(printer.id), sliced_nozzle)
  3440. if wrong_nozzle:
  3441. printers_wrong_nozzle.append(wrong_nozzle)
  3442. continue
  3443. if printer.id in exclude_ids and refused_uploads and printer.id in refused_uploads:
  3444. printers_refused.append(_refused_upload_hold_reason(printer.name, refused_uploads[printer.id]))
  3445. continue
  3446. if printer.id in exclude_ids:
  3447. # Printer is already claimed by another job in this scheduling run.
  3448. # For force-color jobs, still check if the color would match — if not,
  3449. # report it as a color mismatch rather than plain "Busy" so the user
  3450. # knows the job needs a filament change, not just to wait for availability.
  3451. if force_overrides and not pref_overrides:
  3452. missing_colors = self._get_missing_force_color_slots(printer.id, force_overrides)
  3453. if missing_colors:
  3454. printers_missing_filament.append((printer.name, missing_colors))
  3455. continue
  3456. printers_busy.append(f"{printer.name} (drying)" if printer.id in (drying_ids or ()) else printer.name)
  3457. continue
  3458. is_connected = printer_manager.is_connected(printer.id)
  3459. is_idle = self._is_printer_idle(printer.id, require_plate_clear) if is_connected else False
  3460. if not is_connected:
  3461. # An offline printer whose last known filament cannot run this
  3462. # job is reported as needing filament rather than as offline
  3463. # (#2876). It is also the printer the smart-plug step will now
  3464. # decline to switch on, and "Offline:" on its own would leave
  3465. # that decision looking like nothing happening at all.
  3466. shortfall = self._cached_filament_shortfall(printer.id, required_filament_types, filament_overrides)
  3467. if shortfall:
  3468. printers_missing_filament.append((printer.name, shortfall))
  3469. elif wakeable_ids is not None and printer.id not in wakeable_ids:
  3470. printers_offline_no_plug.append(printer.name)
  3471. else:
  3472. printers_offline.append(printer.name)
  3473. continue
  3474. if not is_idle:
  3475. # Printer is currently printing. For force-color jobs, check whether the
  3476. # loaded color would satisfy the requirement — if not, surface it as a
  3477. # color-mismatch reason rather than plain "Busy" so the user understands
  3478. # that the job is waiting for a filament change, not just printer availability.
  3479. if force_overrides and not pref_overrides:
  3480. missing_colors = self._get_missing_force_color_slots(printer.id, force_overrides)
  3481. if missing_colors:
  3482. printers_missing_filament.append((printer.name, missing_colors))
  3483. logger.debug(
  3484. "Printer %s (%s) is busy but also has wrong force-color: %s",
  3485. printer.id,
  3486. printer.name,
  3487. missing_colors,
  3488. )
  3489. continue
  3490. printers_busy.append(printer.name)
  3491. continue
  3492. # Validate filament compatibility if required types are specified
  3493. if required_filament_types:
  3494. missing = self._get_missing_filament_types(printer.id, required_filament_types)
  3495. if missing:
  3496. # When force_overrides are present, enrich missing entries with color info
  3497. # so the "Waiting on" message includes "TYPE (color)" instead of just "TYPE"
  3498. if force_overrides:
  3499. force_color_map = {
  3500. (o.get("type") or "").upper(): o.get("color_name") or o.get("color", "?")
  3501. for o in force_overrides
  3502. }
  3503. missing_enriched = [
  3504. f"{t} ({force_color_map[t_upper]})" if (t_upper := t.upper()) in force_color_map else t
  3505. for t in missing
  3506. ]
  3507. printers_missing_filament.append((printer.name, missing_enriched))
  3508. else:
  3509. printers_missing_filament.append((printer.name, missing))
  3510. logger.debug("Skipping printer %s (%s) - missing filaments: %s", printer.id, printer.name, missing)
  3511. continue
  3512. # Force color match: ALL flagged slots must have an exact type+color match
  3513. if force_overrides:
  3514. missing_colors = self._get_missing_force_color_slots(printer.id, force_overrides)
  3515. if missing_colors:
  3516. printers_missing_filament.append((printer.name, missing_colors))
  3517. logger.debug(
  3518. "Skipping printer %s (%s) - missing force-matched colors: %s",
  3519. printer.id,
  3520. printer.name,
  3521. missing_colors,
  3522. )
  3523. continue
  3524. # The matcher has the last word: a printer it cannot map would be
  3525. # held and given back, then chosen again on the next pass (#3280).
  3526. if unmatched_on is not None:
  3527. unmatched = await unmatched_on(printer.id)
  3528. if unmatched:
  3529. printers_missing_filament.append((printer.name, unmatched))
  3530. logger.debug(
  3531. "Skipping printer %s (%s) - the matcher leaves unmapped: %s",
  3532. printer.id,
  3533. printer.name,
  3534. unmatched,
  3535. )
  3536. continue
  3537. # If preference-only overrides exist, rank by color matches (existing behaviour)
  3538. if pref_overrides:
  3539. color_matches = self._count_override_color_matches(printer.id, pref_overrides)
  3540. if color_matches > 0:
  3541. candidates.append((printer.id, color_matches))
  3542. else:
  3543. override_colors = [f"{o.get('type', '?')} ({o.get('color', '?')})" for o in pref_overrides]
  3544. printers_missing_filament.append((printer.name, override_colors))
  3545. logger.debug("Skipping printer %s (%s) - no matching override colors", printer.id, printer.name)
  3546. continue
  3547. else:
  3548. # The first eligible printer wins, which is the one with the
  3549. # fewest print hours (#3480, see _printers_for_model), unless
  3550. # its spools are too light for the job (#3137).
  3551. if is_short is not None and await is_short(printer.id):
  3552. if short_fallback is None:
  3553. short_fallback = printer.id
  3554. logger.info(
  3555. "Model-based assignment: passing over printer %s (%s), not enough filament",
  3556. printer.id,
  3557. printer.name,
  3558. )
  3559. continue
  3560. return printer.id, None
  3561. # If we have candidates from preference override matching, pick the one with most color matches.
  3562. # The sort is stable: among equal matches the fewest print hours still go first (#3480).
  3563. if candidates:
  3564. candidates.sort(key=lambda c: c[1], reverse=True)
  3565. if is_short is not None:
  3566. for candidate_id, _ in candidates:
  3567. if not await is_short(candidate_id):
  3568. return candidate_id, None
  3569. logger.info(
  3570. "Model-based assignment: passing over printer %s, not enough filament",
  3571. candidate_id,
  3572. )
  3573. return candidates[0][0], None
  3574. if short_fallback is not None:
  3575. return short_fallback, None
  3576. # #3536: nothing else is in the way, so the nozzle is the whole reason. Named
  3577. # per printer; when something else also waits, the wrong ones stay out so
  3578. # that reason (e.g. "Busy") is not drowned by a printer that cannot help.
  3579. if printers_wrong_nozzle and not (
  3580. printers_busy
  3581. or printers_refused
  3582. or printers_offline
  3583. or printers_offline_no_plug
  3584. or printers_missing_filament
  3585. ):
  3586. return None, (
  3587. f"No {normalized_model}{location_suffix} has a {sliced_nozzle:g}mm nozzle: "
  3588. f"{', '.join(printers_wrong_nozzle)}"
  3589. )
  3590. # Build waiting reason from what we found
  3591. reasons = []
  3592. if printers_missing_filament:
  3593. # Filament/color mismatch is most actionable - show first
  3594. if force_overrides and not pref_overrides:
  3595. # All mismatches are force-color failures — use descriptive message only;
  3596. # but only if there are no busy printers that DO have the matching color.
  3597. # If a printer has the right color but is busy, surface "Busy" instead so
  3598. # the user knows the job will start automatically once that printer is free.
  3599. # Same for a printer that is merely offline: Bambuddy switches that one on
  3600. # by itself, so the job is not actually waiting on anybody to change a
  3601. # spool (#2876 — offline printers reach this list now that a switched-off
  3602. # printer's own filament is read).
  3603. if not printers_busy and not printers_offline and not printers_refused:
  3604. all_missing = sorted({c for _, cols in printers_missing_filament for c in cols})
  3605. return None, f"No matching material/color. Waiting on {', '.join(all_missing)}"
  3606. # else: fall through — the self-resolving entries are appended below
  3607. else:
  3608. names_and_missing = [
  3609. f"{name} (needs {', '.join(missing)})" for name, missing in printers_missing_filament
  3610. ]
  3611. reasons.append(f"Waiting for filament: {'; '.join(names_and_missing)}")
  3612. reasons.extend(printers_refused)
  3613. if printers_busy:
  3614. reasons.append(f"Busy: {', '.join(printers_busy)}")
  3615. if printers_offline:
  3616. reasons.append(f"Offline: {', '.join(printers_offline)}")
  3617. if printers_offline_no_plug:
  3618. # Named separately because it is the one entry on this list the
  3619. # user has to act on: no enabled Auto On plug means Bambuddy will
  3620. # never power this printer on for the queue (#2786).
  3621. reasons.append(f"Offline, no Auto On smart plug: {', '.join(printers_offline_no_plug)}")
  3622. return None, " | ".join(reasons) if reasons else f"No available {model} printers{location_suffix}"
  3623. @staticmethod
  3624. def _is_busy_only(waiting_reason: str) -> bool:
  3625. """Check if the waiting reason only contains 'Busy' entries.
  3626. When all matching printers are simply busy printing, the queued job
  3627. will start automatically once a printer finishes — no user action
  3628. is required, so we skip the notification.
  3629. """
  3630. parts = [p.strip() for p in waiting_reason.split(" | ")]
  3631. return all(p.startswith("Busy:") for p in parts)
  3632. def _get_missing_force_color_slots(
  3633. self, printer_id: int, force_overrides: list[dict], raw_data: dict | None = None
  3634. ) -> list[str]:
  3635. """Return descriptive strings for force_color_match slots not satisfied by the printer.
  3636. Each entry in ``force_overrides`` must have ``type`` and ``color`` fields and is expected
  3637. to carry ``force_color_match: True``. The printer must have **every** such slot loaded
  3638. with an exact type+color match.
  3639. When both the override and a candidate tray carry a ``tray_info_idx``, they must also
  3640. match on it: Bambu reports every PLA variant as ``tray_type == "PLA"``, so the
  3641. Basic/Matte/Silk distinction lives only in ``tray_info_idx`` (GFA00/GFA01/GFA06/...).
  3642. Without this, a job sliced for PLA Matte matched every white PLA regardless of variant
  3643. (#2650). If either side lacks an idx (custom/third-party spools report a blank one, and
  3644. older 3MFs carry none) we fall back to the historical type+colour behaviour so those
  3645. setups are unaffected.
  3646. Returns:
  3647. List of ``"TYPE (color)"`` strings for unmatched slots (empty list means all match).
  3648. """
  3649. if raw_data is None:
  3650. status = printer_manager.get_status(printer_id)
  3651. if not status:
  3652. return [f"{o.get('type', '?')} ({o.get('color_name') or o.get('color', '?')})" for o in force_overrides]
  3653. raw_data = status.raw_data
  3654. # Build loaded (type, colour, tray_info_idx) triples from AMS and external spool.
  3655. loaded: list[tuple[str, str, str]] = []
  3656. for ams_unit in raw_data.get("ams", []):
  3657. for tray in ams_unit.get("tray", []):
  3658. tray_type = tray.get("tray_type")
  3659. if tray_type:
  3660. color_norm = self._normalize_color_for_compare(self._normalize_color(tray.get("tray_color")))
  3661. loaded.append((canonical_filament_type(tray_type), color_norm, tray.get("tray_info_idx", "") or ""))
  3662. for vt in raw_data.get("vt_tray") or []:
  3663. vt_type = vt.get("tray_type")
  3664. if vt_type:
  3665. color_norm = self._normalize_color_for_compare(self._normalize_color(vt.get("tray_color")))
  3666. loaded.append((canonical_filament_type(vt_type), color_norm, vt.get("tray_info_idx", "") or ""))
  3667. # One spool per slot, the way the matcher assigns them: two slots asking
  3668. # for the same filament need two spools of it (#3280). Slots naming a
  3669. # variant go first and take that variant before a spool naming none, so
  3670. # a slot open to any variant cannot use up the one a later slot needs.
  3671. missing = []
  3672. taken: set[int] = set()
  3673. for o in sorted(force_overrides, key=lambda o: not o.get("tray_info_idx")):
  3674. o_type = canonical_filament_type(o.get("type") or "")
  3675. o_color = (o.get("color") or "").replace("#", "").lower()[:6]
  3676. o_idx = o.get("tray_info_idx") or ""
  3677. fits = [
  3678. i
  3679. for i, (t_type, t_color, t_idx) in enumerate(loaded)
  3680. if i not in taken
  3681. and t_type == o_type
  3682. and t_color == o_color
  3683. and (not o_idx or not t_idx or o_idx == t_idx)
  3684. ]
  3685. pick = next((i for i in fits if o_idx and loaded[i][2] == o_idx), fits[0] if fits else None)
  3686. if pick is not None:
  3687. taken.add(pick)
  3688. else:
  3689. color_label = o.get("color_name") or o.get("color", "?")
  3690. missing.append(f"{o_type} ({color_label})")
  3691. return missing
  3692. def _get_missing_filament_types(
  3693. self, printer_id: int, required_types: list[str], raw_data: dict | None = None
  3694. ) -> list[str]:
  3695. """Get the list of required filament types that are not loaded on the printer.
  3696. Args:
  3697. printer_id: The printer ID
  3698. required_types: List of filament types needed (e.g., ["PLA", "PETG"])
  3699. Returns:
  3700. List of missing filament types (empty if all are loaded)
  3701. """
  3702. if raw_data is None:
  3703. status = printer_manager.get_status(printer_id)
  3704. if not status:
  3705. return required_types # Can't determine, assume all missing
  3706. raw_data = status.raw_data
  3707. # Collect all filament types loaded on this printer (AMS units + external spool)
  3708. # Use canonical types so equivalence groups (e.g. PA-CF/PA12-CF/PAHT-CF) match.
  3709. loaded_types: set[str] = set()
  3710. # Check AMS units (stored in raw_data["ams"])
  3711. ams_data = raw_data.get("ams", [])
  3712. if ams_data:
  3713. for ams_unit in ams_data:
  3714. for tray in ams_unit.get("tray", []):
  3715. tray_type = tray.get("tray_type")
  3716. if tray_type:
  3717. loaded_types.add(canonical_filament_type(tray_type))
  3718. # Check external spool(s) (virtual tray, stored in raw_data["vt_tray"] as list)
  3719. for vt in raw_data.get("vt_tray") or []:
  3720. vt_type = vt.get("tray_type")
  3721. if vt_type:
  3722. loaded_types.add(canonical_filament_type(vt_type))
  3723. # Find which required types are missing (using canonical type for equivalence)
  3724. missing = []
  3725. for req_type in required_types:
  3726. if canonical_filament_type(req_type) not in loaded_types:
  3727. missing.append(req_type)
  3728. return missing
  3729. def _count_override_color_matches(
  3730. self, printer_id: int, overrides: list[dict], raw_data: dict | None = None
  3731. ) -> int:
  3732. """Count how many filament overrides have an exact color match on the printer.
  3733. Used to prefer printers that already have the desired override colors loaded.
  3734. """
  3735. if raw_data is None:
  3736. status = printer_manager.get_status(printer_id)
  3737. if not status:
  3738. return 0
  3739. raw_data = status.raw_data
  3740. # Collect loaded filaments' type+color pairs
  3741. loaded: set[tuple[str, str]] = set()
  3742. for ams_unit in raw_data.get("ams", []):
  3743. for tray in ams_unit.get("tray", []):
  3744. tray_type = tray.get("tray_type")
  3745. # `or ""`, not a dict default: a slot can carry the key with a
  3746. # null value, and this now runs against switched-off printers
  3747. # too, where nobody is watching for the AttributeError.
  3748. tray_color = tray.get("tray_color") or ""
  3749. if tray_type:
  3750. color_norm = tray_color.replace("#", "").lower()[:6]
  3751. loaded.add((canonical_filament_type(tray_type), color_norm))
  3752. for vt in raw_data.get("vt_tray") or []:
  3753. vt_type = vt.get("tray_type")
  3754. if vt_type:
  3755. color_norm = (vt.get("tray_color", "") or "").replace("#", "").lower()[:6]
  3756. loaded.add((canonical_filament_type(vt_type), color_norm))
  3757. # Types compared as the type check compares them: a PA12-CF override
  3758. # is met by PA-CF in its colour.
  3759. matches = 0
  3760. for o in overrides:
  3761. o_type = canonical_filament_type(o.get("type") or "")
  3762. o_color = (o.get("color") or "").replace("#", "").lower()[:6]
  3763. if (o_type, o_color) in loaded:
  3764. matches += 1
  3765. return matches
  3766. @staticmethod
  3767. def _tray_reading(printer_id: int) -> dict:
  3768. """The best tray reading available for a printer that is not printing.
  3769. Live status first: a printer keeps its last status after the power
  3770. goes, because ``mark_power_off`` blanks ``connected`` and ``state`` and
  3771. leaves ``raw_data`` alone. The manager's own record is the fallback,
  3772. for when the client itself has been dropped and taken its status with
  3773. it — which is what every power-on attempt does.
  3774. An empty result means "we have never heard", not "nothing is loaded":
  3775. the two are indistinguishable from here, and only the second would be
  3776. safe to act on.
  3777. """
  3778. status = printer_manager.get_status(printer_id)
  3779. raw = (status.raw_data if status else None) or {}
  3780. for ams_unit in raw.get("ams") or []:
  3781. if any(tray.get("tray_type") for tray in ams_unit.get("tray", [])):
  3782. return raw
  3783. if any(vt.get("tray_type") for vt in raw.get("vt_tray") or []):
  3784. return raw
  3785. return printer_manager.last_known_trays(printer_id)
  3786. def _cached_filament_shortfall(
  3787. self,
  3788. printer_id: int,
  3789. required_types: list[str] | None,
  3790. filament_overrides: list[dict] | None,
  3791. ) -> list[str]:
  3792. """What a switched-off printer's last known filament cannot provide (#2876).
  3793. The smart-plug wake step used to consider only the model, so a job for a
  3794. colour loaded on the last printer in ID order switched on every earlier
  3795. one in turn, evaluated it, rejected it on colour and left it running.
  3796. Bambuddy knew those colours the whole time. This asks the same three
  3797. questions the matcher asks a live printer — required types, forced
  3798. colours, preferred colours — of the trays it last reported, and returns
  3799. the answers in the same shape the "Waiting for filament" reason uses.
  3800. Empty means the printer may still be able to take the job.
  3801. Fails open, and deliberately: with no tray reading (never connected
  3802. since Bambuddy started, or the cache dropped by a reconnect) this
  3803. returns nothing to report and the printer is treated as it was before.
  3804. A farm restarted while its printers were off must not conclude that
  3805. none of them can print.
  3806. """
  3807. if not required_types and not filament_overrides:
  3808. return []
  3809. raw_data = self._tray_reading(printer_id)
  3810. if not raw_data:
  3811. return []
  3812. force_overrides = [o for o in (filament_overrides or []) if o.get("force_color_match")]
  3813. pref_overrides = [o for o in (filament_overrides or []) if not o.get("force_color_match")]
  3814. if required_types:
  3815. missing = self._get_missing_filament_types(printer_id, required_types, raw_data)
  3816. if missing:
  3817. # Same enrichment the live path applies: a bare "PLA" is not
  3818. # much help when what is missing is a particular PLA.
  3819. force_color_map = {
  3820. (o.get("type") or "").upper(): o.get("color_name") or o.get("color", "?") for o in force_overrides
  3821. }
  3822. return [
  3823. f"{t} ({force_color_map[t_upper]})" if (t_upper := t.upper()) in force_color_map else t
  3824. for t in missing
  3825. ]
  3826. if force_overrides:
  3827. missing_colors = self._get_missing_force_color_slots(printer_id, force_overrides, raw_data)
  3828. if missing_colors:
  3829. return missing_colors
  3830. # Preference overrides read as a preference but the matcher treats zero
  3831. # matches as a skip, so a printer with none of the wanted colours is
  3832. # rejected there too. Waking it would only produce that same rejection.
  3833. if pref_overrides and self._count_override_color_matches(printer_id, pref_overrides, raw_data) == 0:
  3834. return [f"{o.get('type', '?')} ({o.get('color_name') or o.get('color', '?')})" for o in pref_overrides]
  3835. return []
  3836. def _resolve_variant(self, item: PrintQueueItem, candidate: _ModelCandidate) -> None:
  3837. """Fold the winning candidate's file and settings onto the queue row (#671).
  3838. This is the whole trick that keeps cross-model items cheap: the many-to-many
  3839. never escapes the selection loop. By the time the pass commits, the row
  3840. looks exactly like an ordinary single-file model-based item, so the upload,
  3841. archive creation, expected-print registration, print history and reprint
  3842. paths need no knowledge that variants exist.
  3843. No-ops for a non-variant candidate, which is already the item's own columns.
  3844. Safe to run and re-run: the item's file columns are only ever *read* when it
  3845. has no variants, so an item that gets resolved and then skipped (library-row
  3846. conflict, previous-print gate) is simply resolved again on the next pass.
  3847. """
  3848. variant = candidate.variant
  3849. if variant is None:
  3850. return
  3851. item.library_file_id = variant.library_file_id
  3852. item.library_file = variant.library_file
  3853. # The dispatcher checks archive_id first and would print that instead of
  3854. # the file we just picked. Creation refuses to combine the two, so this
  3855. # only ever fires on a hand-written row — clear it rather than silently
  3856. # dispatch something the matcher never considered.
  3857. item.archive_id = None
  3858. item.archive = None
  3859. item.target_model = variant.target_model
  3860. item.plate_id = variant.plate_id
  3861. item.ams_mapping = variant.ams_mapping
  3862. item.nozzle_mapping = variant.nozzle_mapping
  3863. item.nozzle_rack_choice = variant.nozzle_rack_choice
  3864. item.filament_overrides = variant.filament_overrides
  3865. item.required_filament_types = variant.required_filament_types
  3866. if variant.print_time_seconds is not None:
  3867. # The row carried the shortest candidate's estimate so SJF could order
  3868. # it before a printer was known; now that one is chosen, record what is
  3869. # actually going to run so history and the ETA agree with reality.
  3870. item.print_time_seconds = variant.print_time_seconds
  3871. async def _creator_filament_scope(self, db: AsyncSession, user_id: int | None) -> FilamentScope:
  3872. """The filament scope of a job's creator (#3522), resolved once per creator per pass.
  3873. Resolving costs a settings read and a user load, and model-based
  3874. printer choice asks once per idle candidate printer, so the answer is
  3875. memoised like the creator's printer scope (#1727). A job with no
  3876. creator needs no lookup at all.
  3877. """
  3878. if user_id is None:
  3879. return ALL_FILAMENTS
  3880. if user_id not in self._filament_scope_memo:
  3881. self._filament_scope_memo[user_id] = await resolve_user_id_filament_scope(db, user_id)
  3882. return self._filament_scope_memo[user_id]
  3883. async def _filament_scope_violation(self, db: AsyncSession, item: PrintQueueItem) -> str | None:
  3884. """Why the job's creator may not print this file (#3522), or None.
  3885. The file prints a filament the creator's groups don't allow. Checked
  3886. first in ``_ensure_ams_mapping``, so "Print anyway"
  3887. (``skip_filament_check``) can't override it: it's a permission, not a
  3888. mismatch the user may accept.
  3889. """
  3890. scope = await self._creator_filament_scope(db, item.created_by_id)
  3891. if scope.is_unrestricted:
  3892. return None
  3893. # A 3MF without slice info is mapped from its force-colour overrides
  3894. # (#3280), so judge those instead.
  3895. required = await self._get_filament_requirements(db, item) or _forced_override_requirements(item)
  3896. if not required:
  3897. return None
  3898. # With the overrides applied as the queue route and the dialog apply
  3899. # them (`judged_requirements`). A slot they leave naming no filament
  3900. # id is the matcher's to fill, and it never picks a tray the creator
  3901. # may not use; one left unmapped waits for its filament, and "Print
  3902. # anyway" can't hand it to a blocked tray either
  3903. # (`_dispatch_scope_violation`).
  3904. blocked = scope.blocked_requirements(judged_requirements(required, _filament_overrides_of(item)))
  3905. if not blocked:
  3906. return None
  3907. return file_hold_message(blocked)
  3908. async def _mapping_uses_blocked_tray(
  3909. self, db: AsyncSession, printer_id: int, item: PrintQueueItem, stored_mapping: list
  3910. ) -> bool:
  3911. """Whether ``stored_mapping`` sends a slot to a tray the creator may not use (#3522).
  3912. The print dialog picks trays itself, without knowing the creator's
  3913. filament scope, so a stored mapping can name a blocked tray that an
  3914. allowed one could replace. The caller drops such a mapping and lets the
  3915. matcher, which skips blocked trays, choose again.
  3916. """
  3917. scope = await self._creator_filament_scope(db, item.created_by_id)
  3918. if scope.is_unrestricted:
  3919. return False
  3920. status = printer_manager.get_status(printer_id)
  3921. if not status:
  3922. return False
  3923. blocked = scope.blocked_tray_ids(self._build_loaded_filaments(status))
  3924. return any(_is_tray_id(tray) and tray in blocked for tray in stored_mapping)
  3925. async def _sent_feeds(
  3926. self, db: AsyncSession, printer_id: int, item: PrintQueueItem, status, mapping: list | None
  3927. ) -> tuple[bool, bool]:
  3928. """Whether the printer would feed (its external spools, the trays ``mapping`` names) (#3522).
  3929. Worked out as the job is sent, not from ``use_ams`` alone. On a
  3930. single-nozzle printer the dispatch turns the AMS off for a plate that
  3931. prints only from the spool holder (#3087), the command builder turns it
  3932. back on for a real tray in the mapping (``reconcile_use_ams``, #2595),
  3933. and a printer that reported no AMS feeds the spool holder (#3356). A
  3934. mapping that couldn't be sent at all is judged both ways.
  3935. On a dual-nozzle printer ``use_ams`` picks the extruder instead, and the
  3936. builder leaves it alone: with it off an external spool may feed, and
  3937. which one is not ours to guess, so the mapping counts as well.
  3938. """
  3939. # On by default: a row not yet flushed reads None.
  3940. use_ams = item.use_ams is not False
  3941. printer = await self._get_printer(db, printer_id)
  3942. # Not quite the builder's test. `start_print` calls a printer
  3943. # dual-nozzle by its model or a reported second extruder
  3944. # (`_is_dual_nozzle`); `_might_be_dual_nozzle` also says yes for a
  3945. # populated `ams_extruder_map` or more than one external feed. Where
  3946. # only this one says yes, the mapping is judged instead of the
  3947. # external spool alone, and a mapping onto a blocked external spool
  3948. # is still caught by its tray id.
  3949. if _might_be_dual_nozzle(printer.model if printer else None, status):
  3950. return not use_ams, True
  3951. if isinstance(mapping, list) and mapping:
  3952. # Each layout as its own step reads it: the dispatch from the live
  3953. # state, the builder also from the model (`tray_layout`).
  3954. rack = TrayLayout.from_state(status).rack
  3955. try:
  3956. only_external = any(_is_external_tray(t, rack) for t in mapping) and _plate_uses_only_external(
  3957. mapping, await self._get_filament_requirements(db, item), rack
  3958. )
  3959. use_ams = reconcile_use_ams(
  3960. mapping,
  3961. use_ams and not only_external,
  3962. TrayLayout.from_state(status, has_ams3h(printer.model if printer else None)),
  3963. plate_uses_only_external=only_external,
  3964. )
  3965. except (TypeError, ValueError):
  3966. return True, True
  3967. if use_ams and reports_no_ams(status.raw_data):
  3968. use_ams = False
  3969. return not use_ams, use_ams
  3970. @staticmethod
  3971. def _blocked_external(scope: FilamentScope, loaded: list[dict], status, mapping: list | None) -> list[dict]:
  3972. """The external spools a job sent with ``mapping`` could feed that ``scope`` doesn't allow (#3522).
  3973. A spool with no filament set can't be judged and is allowed. Every
  3974. external spool counts: which one a dual-nozzle printer feeds is not
  3975. ours to guess. A rack printer (H3C) has one external input per hotend
  3976. and feeds the ones the mapping names, so there only those count; a
  3977. mapping that names none says nothing, and all of them count.
  3978. """
  3979. named: set[int] = set()
  3980. if TrayLayout.from_state(status).rack:
  3981. named = {t for t in mapping or () if _is_tray_id(t) and _is_external_tray(t, True)}
  3982. return [
  3983. spool
  3984. for spool in loaded
  3985. if spool.get("is_external")
  3986. and (not named or spool.get("global_tray_id") in named)
  3987. and not scope.allows(spool.get("type"), spool.get("tray_info_idx"))
  3988. ]
  3989. async def _blocked_external_spools(
  3990. self, db: AsyncSession, printer_id: int, item: PrintQueueItem, mapping: list | None
  3991. ) -> str | None:
  3992. """Why a job sent with ``mapping`` would print from an external spool its creator may not use (#3522), or None.
  3993. The spool is judged like any tray, but only where the printer would
  3994. actually feed it (``_sent_feeds``): a blocked spool on the holder must
  3995. not hold a job whose mapping turns the AMS on. Which external spools
  3996. count is ``_blocked_external``'s to say.
  3997. """
  3998. scope = await self._creator_filament_scope(db, item.created_by_id)
  3999. if scope.is_unrestricted:
  4000. return None
  4001. status = printer_manager.get_status(printer_id)
  4002. if not status:
  4003. return None
  4004. feeds_external, _ = await self._sent_feeds(db, printer_id, item, status, mapping)
  4005. if not feeds_external:
  4006. return None
  4007. blocked = self._blocked_external(scope, self._build_loaded_filaments(status), status, mapping)
  4008. return spool_hold_message(blocked) if blocked else None
  4009. async def _dispatch_scope_violation(self, db: AsyncSession, printer_id: int, item: PrintQueueItem) -> str | None:
  4010. """Why the mapping about to be sent would print a filament its creator may not use (#3522), or None.
  4011. The file check (``_filament_scope_violation``) judges what the file
  4012. asks for; this judges what the printer would feed, as it is sent
  4013. (``_sent_feeds``). That differs where the printer, not Bambuddy, picks
  4014. the spool:
  4015. * The external spool, where the printer would feed it.
  4016. * A mapping that names a blocked tray, where the printer would feed the
  4017. mapping. The matcher never picks one, so this is a backstop for a
  4018. mapping that reached dispatch some other way.
  4019. * A file that lists no filaments, which nothing can map: the printer
  4020. picks from its spools, so a blocked one it could pick holds the job.
  4021. * "Print anyway" with a printed slot left without a tray, while a
  4022. blocked tray is loaded. Without the flag
  4023. ``_block_on_unmatched_filament`` holds that job; with it, the printer
  4024. would choose a feed for the slot, possibly the blocked tray. For a
  4025. restricted creator the flag accepts a mismatch, not that. With
  4026. nothing blocked loaded, every feed it could choose is allowed. A
  4027. tray the printer doesn't report loaded counts as such a slot.
  4028. """
  4029. scope = await self._creator_filament_scope(db, item.created_by_id)
  4030. if scope.is_unrestricted:
  4031. return None
  4032. status = printer_manager.get_status(printer_id)
  4033. if not status:
  4034. return None
  4035. mapping: list | None = None
  4036. if item.ams_mapping:
  4037. try:
  4038. mapping = json.loads(item.ams_mapping)
  4039. except (json.JSONDecodeError, TypeError):
  4040. mapping = None
  4041. if not isinstance(mapping, list):
  4042. mapping = None
  4043. feeds_external, feeds_mapping = await self._sent_feeds(db, printer_id, item, status, mapping)
  4044. loaded = self._build_loaded_filaments(status)
  4045. if feeds_external and (external := self._blocked_external(scope, loaded, status, mapping)):
  4046. return spool_hold_message(external)
  4047. if not feeds_mapping:
  4048. return None
  4049. blocked_ids = scope.blocked_tray_ids(loaded)
  4050. used = {t for t in mapping or [] if _is_tray_id(t) and t in blocked_ids}
  4051. if used:
  4052. return spool_hold_message(tray for tray in loaded if tray["global_tray_id"] in used)
  4053. # What is left is a slot the printer fills by itself, so only a
  4054. # blocked spool it could pick matters: an AMS tray, and an external
  4055. # spool where it feeds those. With the AMS on, a single-nozzle
  4056. # printer never sends such a slot to the spool holder. On a printer
  4057. # that might be dual-nozzle `use_ams` picks the extruder rather than
  4058. # the AMS, so either external spool may feed whatever it says, as it
  4059. # is all an H2D without an AMS has.
  4060. printer = await self._get_printer(db, printer_id)
  4061. externals = feeds_external or _might_be_dual_nozzle(printer.model if printer else None, status)
  4062. pickable = [
  4063. tray
  4064. for tray in loaded
  4065. if tray["global_tray_id"] in blocked_ids and (externals or not tray.get("is_external"))
  4066. ]
  4067. if not pickable:
  4068. return None
  4069. if (
  4070. not (await self._get_filament_requirements(db, item) or _forced_override_requirements(item))
  4071. and (source := await self._source_file_path(db, item)) is not None
  4072. and source.exists()
  4073. ):
  4074. # A file that lists no filaments (no readable slice info) can't be
  4075. # judged from the file, and the matcher has nothing to map: the
  4076. # printer picks its feed. So the spools it could pick from are
  4077. # judged instead. A file that is gone is left to the dispatch,
  4078. # which fails it as missing.
  4079. return unlisted_file_hold_message(pickable)
  4080. if item.skip_filament_check:
  4081. # A tray the printer doesn't report loaded settles nothing: only a
  4082. # hand-written mapping names one, and "Print anyway" keeps it past
  4083. # the fit check (`_stored_mapping_conflict`). The printer would
  4084. # fill that slot itself, so it counts as unresolved.
  4085. known = {tray["global_tray_id"] for tray in loaded}
  4086. if mapping is not None:
  4087. mapping = [t if _is_tray_id(t) and t in known else -1 for t in mapping]
  4088. unresolved = await self._unmatched_requirements(db, item, printer_id, mapping)
  4089. if unresolved:
  4090. return print_anyway_hold_message(unresolved, pickable)
  4091. return None
  4092. async def _claimed_scope_violation(self, db: AsyncSession, item: PrintQueueItem) -> str | None:
  4093. """``_ensure_ams_mapping``'s filament-scope checks, on the row as claimed for dispatch (#3522).
  4094. The creator's rules are read afresh: the pass may have waited minutes
  4095. (a smart plug powering a printer on) since it memoised them, and a rule
  4096. tightened meanwhile must apply to what it dispatches now.
  4097. """
  4098. if item.printer_id is None:
  4099. return None
  4100. if item.created_by_id is not None:
  4101. self._filament_scope_memo.pop(item.created_by_id, None)
  4102. return await self._filament_scope_violation(db, item) or await self._dispatch_scope_violation(
  4103. db, item.printer_id, item
  4104. )
  4105. async def _ensure_ams_mapping(
  4106. self, db: AsyncSession, printer_id: int, item: PrintQueueItem
  4107. ) -> str | FilamentScopeHold | None:
  4108. """Settle the item's AMS mapping for ``printer_id``, then hold it to its creator's filament scope.
  4109. Returns ``_settle_ams_mapping``'s message when the item can't print on
  4110. this printer at all (the caller fails it), or a ``FilamentScopeHold``
  4111. when its creator may not print it as it stands (#3522; the caller
  4112. holds it), or None.
  4113. The file is judged first, so the "Print anyway" short-circuit inside
  4114. can't skip the check. What the printer would feed is judged after the
  4115. mapping is settled, whichever way it was: kept, filled, recomputed or
  4116. left unresolved.
  4117. """
  4118. if violation := await self._filament_scope_violation(db, item):
  4119. return FilamentScopeHold(violation)
  4120. unmappable = await self._settle_ams_mapping(db, printer_id, item)
  4121. if unmappable:
  4122. return unmappable
  4123. if violation := await self._dispatch_scope_violation(db, printer_id, item):
  4124. return FilamentScopeHold(violation)
  4125. return None
  4126. async def _settle_ams_mapping(
  4127. self, db: AsyncSession, printer_id: int, item: PrintQueueItem
  4128. ) -> str | FilamentScopeHold | None:
  4129. """Ensure the queue item carries a usable AMS mapping before dispatch.
  4130. Recomputes from live printer status when the stored mapping is missing OR
  4131. unresolved (all -1). A stored all-[-1] mapping is a bug artifact — a
  4132. frontend status-load race can serialize [-1] before the printer's AMS
  4133. trays are known (#2589) — and must not be trusted: downstream it would be
  4134. silently downgraded to external-spool mode and print against an empty
  4135. feed. A resolved mapping is re-checked against the target printer's
  4136. live trays before it is trusted (#2799) and recomputed when it does not
  4137. fit. When it does fit, its resolved slots are kept as they are and only
  4138. the slots the plate prints that are still unresolved are matched again;
  4139. all of it is left alone when the user has acknowledged it with "Print
  4140. Anyway".
  4141. When recompute cannot resolve it either (no compatible tray loaded), the
  4142. bogus [-1] is cleared to None so it is not later mistaken for an explicit
  4143. external selection. On a printer that reported loaded trays,
  4144. ``_block_on_unmatched_filament`` holds the item rather than let it go out
  4145. mapping-less; where it cannot judge, the print command keeps use_ams=True
  4146. and the firmware surfaces a clear AMS-mapping error instead of silently
  4147. printing to the empty external feed.
  4148. Returns an actionable message when that firmware error is the only
  4149. possible outcome — the matcher ran, matched nothing, and the printer has
  4150. no AMS to load a different spool into (#2771). The caller fails the item
  4151. on it instead of spending an upload on a print that cannot start.
  4152. Returns None everywhere else, including every case where we simply lack
  4153. the data to judge, so dispatch is only ever blocked on a positive
  4154. finding. A stored mapping that uses a tray the job's creator may not
  4155. use is recomputed (#3522), and a job only a spool its creator may not
  4156. use would match, on a printer with no AMS, is held instead of failed
  4157. (``_only_blocked_spools_match``); ``_ensure_ams_mapping`` judges the
  4158. rest of the creator's filament scope around this.
  4159. """
  4160. stored_mapping: list | None = None
  4161. if item.ams_mapping:
  4162. try:
  4163. stored_mapping = json.loads(item.ams_mapping)
  4164. except (json.JSONDecodeError, TypeError):
  4165. stored_mapping = None
  4166. # A mapping that sends a slot to a tray the creator may not use is
  4167. # dropped and recomputed on allowed trays (#3522) -- before the "Print
  4168. # anyway" short-circuit below, which would otherwise keep it.
  4169. if (
  4170. isinstance(stored_mapping, list)
  4171. and not _mapping_is_all_unresolved(stored_mapping)
  4172. and await self._mapping_uses_blocked_tray(db, printer_id, item, stored_mapping)
  4173. ):
  4174. logger.info(
  4175. "Queue item %s: stored ams_mapping %s uses a tray its creator may not use on printer %s; recomputing",
  4176. item.id,
  4177. stored_mapping,
  4178. printer_id,
  4179. )
  4180. item.ams_mapping = None
  4181. stored_mapping = None
  4182. await db.commit()
  4183. # Present and resolved. Global tray IDs only mean something relative to
  4184. # the printer they were resolved against, so "resolved" is not the same
  4185. # as "resolved *here*" — check the mapping still fits the printer that
  4186. # is about to run this item before trusting it (#2799). "Print anyway"
  4187. # is the user overriding exactly this judgement, so it short-circuits.
  4188. if item.ams_mapping and not _mapping_is_all_unresolved(stored_mapping):
  4189. if item.skip_filament_check:
  4190. return None
  4191. conflict = await self._stored_mapping_conflict(db, printer_id, item, stored_mapping)
  4192. if conflict is None:
  4193. # It fits, but a slot the plate prints may still be empty: the
  4194. # spool was missing when it was mapped, and has perhaps been
  4195. # loaded since. Nothing else would ever look at that slot again,
  4196. # so a job held for it would be held again on every Start.
  4197. filled = await self._fill_unresolved_slots(db, printer_id, item, stored_mapping)
  4198. if filled is not None:
  4199. item.ams_mapping = json.dumps(filled)
  4200. logger.info(
  4201. "Queue item %s: filled unresolved slots of %s on printer %s: %s",
  4202. item.id,
  4203. stored_mapping,
  4204. printer_id,
  4205. filled,
  4206. )
  4207. await db.commit()
  4208. return None
  4209. logger.warning(
  4210. "Queue item %s: stored ams_mapping %s does not fit printer %s (%s) — recomputing",
  4211. item.id,
  4212. stored_mapping,
  4213. printer_id,
  4214. conflict,
  4215. )
  4216. # Drop it before recomputing so a failed recompute cannot fall back
  4217. # to the mapping we just rejected.
  4218. item.ams_mapping = None
  4219. stored_mapping = None
  4220. await db.commit()
  4221. computed_mapping = await self._compute_ams_mapping_for_printer(db, printer_id, item)
  4222. if computed_mapping and not _mapping_is_all_unresolved(computed_mapping):
  4223. item.ams_mapping = json.dumps(computed_mapping)
  4224. logger.info(
  4225. "Queue item %s: Computed AMS mapping for printer %s: %s",
  4226. item.id,
  4227. printer_id,
  4228. computed_mapping,
  4229. )
  4230. await db.commit()
  4231. return None
  4232. external_only = await self._external_spool_only_mapping(db, printer_id, item)
  4233. if external_only is not None:
  4234. item.ams_mapping = json.dumps(external_only)
  4235. logger.info(
  4236. "Queue item %s: printer %s has no AMS and its external spool has no filament set; "
  4237. "mapping every filament to it: %s",
  4238. item.id,
  4239. printer_id,
  4240. external_only,
  4241. )
  4242. await db.commit()
  4243. return None
  4244. if _mapping_is_all_unresolved(stored_mapping):
  4245. logger.warning(
  4246. "Queue item %s: stored ams_mapping %s is unresolved and could not be recomputed "
  4247. "from live status on printer %s; clearing it so dispatch does not treat it as external",
  4248. item.id,
  4249. stored_mapping,
  4250. printer_id,
  4251. )
  4252. item.ams_mapping = None
  4253. await db.commit()
  4254. if hold := await self._only_blocked_spools_match(db, printer_id, item):
  4255. return hold
  4256. return await self._unmappable_without_ams_message(db, printer_id, item, computed_mapping)
  4257. async def _only_blocked_spools_match(
  4258. self, db: AsyncSession, printer_id: int, item: PrintQueueItem
  4259. ) -> FilamentScopeHold | None:
  4260. """A hold when, on a printer with no AMS, the job matches only spools its creator may not use (#3522).
  4261. The matcher leaves those spools out, so it resolves nothing, and the
  4262. job would be failed as if nothing matched
  4263. (``_unmappable_without_ams_message``), or held as unmatched. A
  4264. dual-nozzle printer (H2D/H2C) with only external spools reaches here,
  4265. since the single-feed fallback isn't its. Matched again with every
  4266. spool in, a printed slot that lands on a blocked one is a permission
  4267. finding: the job is held, and starts once the group's rules or the
  4268. spools allow it.
  4269. """
  4270. scope = await self._creator_filament_scope(db, item.created_by_id)
  4271. if scope.is_unrestricted:
  4272. return None
  4273. status = printer_manager.get_status(printer_id)
  4274. if status is None or not reports_no_ams(status.raw_data):
  4275. return None
  4276. loaded = self._build_loaded_filaments(status)
  4277. blocked_ids = scope.blocked_tray_ids(loaded)
  4278. if not blocked_ids:
  4279. return None
  4280. required = await self._get_filament_requirements(db, item)
  4281. if required:
  4282. self._apply_filament_overrides(item, required)
  4283. else:
  4284. required = _forced_override_requirements(item)
  4285. if not required:
  4286. return None
  4287. fts_installed = bool(getattr(getattr(status, "fila_switch", None), "installed", False))
  4288. mapping = self._match_filaments_to_slots(required, loaded, fts_installed=fts_installed) or []
  4289. used = {t for t in mapping if _is_tray_id(t) and t in blocked_ids}
  4290. if not used:
  4291. return None
  4292. return FilamentScopeHold(spool_hold_message(tray for tray in loaded if tray["global_tray_id"] in used))
  4293. async def _external_spool_only_mapping(
  4294. self, db: AsyncSession, printer_id: int, item: PrintQueueItem
  4295. ) -> list[int] | None:
  4296. """Every filament on the external spool, for a printer that has nothing else (#3239).
  4297. A printer without an AMS prints from its external spool, and an external
  4298. spool whose filament was never set reports no type, so the matcher has
  4299. nothing to match. Sent without a mapping, the print goes out with the AMS
  4300. on and the firmware rejects it with 0700_8012. A stored ``[254]`` used
  4301. to carry such a job through; a job placed by model or location no longer
  4302. keeps one, and one queued that way never had one.
  4303. Only on a positive report: the printer has said it has no AMS, it has a
  4304. single external feed (a dual-nozzle printer's feeds steer nozzles, which
  4305. is not ours to pick), and no feed has a filament set — with one set, the
  4306. matcher has already given its answer. Not for a job that asked for its
  4307. colours to be matched strictly: a spool without a filament set has no
  4308. colour to check. Returns None otherwise.
  4309. """
  4310. if item.filament_overrides:
  4311. try:
  4312. overrides = json.loads(item.filament_overrides)
  4313. except (json.JSONDecodeError, TypeError):
  4314. return None
  4315. if not isinstance(overrides, list) or any(
  4316. isinstance(o, dict) and o.get("force_color_match") for o in overrides
  4317. ):
  4318. return None
  4319. status = printer_manager.get_status(printer_id)
  4320. if status is None or not isinstance(status.raw_data, dict):
  4321. return None
  4322. ams_units = status.raw_data.get("ams")
  4323. if not isinstance(ams_units, list) or ams_units:
  4324. return None
  4325. vt_trays = status.raw_data.get("vt_tray")
  4326. if not isinstance(vt_trays, list) or len(vt_trays) != 1 or not isinstance(vt_trays[0], dict):
  4327. return None
  4328. if self._build_loaded_filaments(status):
  4329. return None
  4330. printer = await self._get_printer(db, printer_id)
  4331. if _might_be_dual_nozzle(printer.model if printer else None, status):
  4332. return None
  4333. external = _int_or(vt_trays[0].get("id"), _EXTERNAL_TRAY_ID_MIN)
  4334. if external < _EXTERNAL_TRAY_ID_MIN:
  4335. return None
  4336. required = await self._get_filament_requirements(db, item)
  4337. slot_ids = [r.get("slot_id") for r in required or []]
  4338. slot_ids = [s for s in slot_ids if isinstance(s, int) and s > 0]
  4339. if not slot_ids:
  4340. return None
  4341. mapping = [-1] * max(slot_ids)
  4342. for slot_id in slot_ids:
  4343. mapping[slot_id - 1] = external
  4344. return mapping
  4345. async def _fill_unresolved_slots(
  4346. self,
  4347. db: AsyncSession,
  4348. printer_id: int,
  4349. item: PrintQueueItem,
  4350. stored_mapping: list | None,
  4351. ) -> list | None:
  4352. """``stored_mapping`` with its unresolved required slots matched on live trays (#2799).
  4353. Every resolved entry is kept, so a tray the user picked by hand stays
  4354. picked. Only the plate's unresolved slots are matched, and only against
  4355. trays the mapping does not already use: the matcher never gives two
  4356. slots one tray, and filling a gap with a tray the user assigned to
  4357. another slot would print that slot's filament twice.
  4358. Returns None when there is nothing to fill or nothing could be filled.
  4359. """
  4360. if not isinstance(stored_mapping, list) or not stored_mapping:
  4361. return None
  4362. required = await self._get_filament_requirements(db, item)
  4363. if not required:
  4364. return None
  4365. self._apply_filament_overrides(item, required)
  4366. gaps = {req["slot_id"] for req in _unresolved_required(required, stored_mapping)}
  4367. if not gaps:
  4368. return None
  4369. reserved = {t for t in stored_mapping if _is_tray_id(t) and t >= 0}
  4370. computed = await self._compute_ams_mapping_for_printer(
  4371. db, printer_id, item, only_slots=gaps, reserved_trays=reserved
  4372. )
  4373. if not computed:
  4374. return None
  4375. merged = list(stored_mapping) + [-1] * max(0, len(computed) - len(stored_mapping))
  4376. filled = False
  4377. for slot_id in gaps:
  4378. tray = computed[slot_id - 1] if slot_id <= len(computed) else None
  4379. if _is_tray_id(tray) and tray >= 0:
  4380. merged[slot_id - 1] = tray
  4381. filled = True
  4382. return merged if filled else None
  4383. async def missing_filament_for_start(self, db: AsyncSession, item: PrintQueueItem) -> list[str] | None:
  4384. """What a staged item would be held for if it were started now (#2799).
  4385. The Start button asks this before releasing an item, so that a job whose
  4386. filament is still not loaded offers "Print Anyway" instead of being
  4387. released, held again by the scheduler and leaving no way past the hold.
  4388. It reaches the same answer the dispatch path would: a stored mapping
  4389. that fits is kept and only its gaps are matched again, and one that
  4390. does not fit (or is missing) is replaced by a fresh match.
  4391. Returns the missing filaments, described the way the queue row
  4392. describes them, or None when nothing is missing or there is not enough
  4393. evidence to say: no printer yet (a model-based item picks one at
  4394. dispatch), no status, no trays reported, or no readable 3MF.
  4395. """
  4396. if item.skip_filament_check or not item.printer_id:
  4397. return None
  4398. status = printer_manager.get_status(item.printer_id)
  4399. if status is None or not self._build_loaded_filaments(status):
  4400. return None
  4401. # Called from a request, not from a pass: drop whatever the last pass
  4402. # memoised so this reads the file as it is now. A pass running at the
  4403. # same time only loses its cache.
  4404. self._filament_req_memo.clear()
  4405. self._filament_scope_memo.clear()
  4406. mapping: list | None = None
  4407. if item.ams_mapping:
  4408. try:
  4409. mapping = json.loads(item.ams_mapping)
  4410. except (json.JSONDecodeError, TypeError):
  4411. mapping = None
  4412. if not isinstance(mapping, list) or _mapping_is_all_unresolved(mapping):
  4413. mapping = None
  4414. if mapping is not None:
  4415. if await self._stored_mapping_conflict(db, item.printer_id, item, mapping) is None:
  4416. mapping = await self._fill_unresolved_slots(db, item.printer_id, item, mapping) or mapping
  4417. else:
  4418. mapping = None
  4419. if mapping is None:
  4420. mapping = await self._compute_ams_mapping_for_printer(db, item.printer_id, item) or []
  4421. required = await self._get_filament_requirements(db, item)
  4422. if required:
  4423. self._apply_filament_overrides(item, required)
  4424. else:
  4425. # The same fallback the hold uses, or Start would release a job the
  4426. # next pass holds again (#3280).
  4427. required = _forced_override_requirements(item)
  4428. if not required:
  4429. return None
  4430. missing = _unresolved_required(required, mapping)
  4431. return [_describe_filament(req, "nozzle_id") for req in missing] or None
  4432. async def _stored_mapping_conflict(
  4433. self,
  4434. db: AsyncSession,
  4435. printer_id: int,
  4436. item: PrintQueueItem,
  4437. stored_mapping: list | None,
  4438. ) -> str | None:
  4439. """Describe why ``stored_mapping`` cannot be trusted on ``printer_id`` (#2799).
  4440. A mapping is a list of global tray IDs, and those are only meaningful
  4441. relative to the printer they were resolved against — the same rule
  4442. ``print_queue`` documents for tray identity. Two ways a stored mapping
  4443. arrives at a printer it was not resolved for:
  4444. * the print dialog stamps one mapping onto every selected printer, so a
  4445. mapping computed against the first printer is dispatched verbatim to
  4446. the rest, whose AMS slot order differs;
  4447. * a spool is moved between queueing and dispatch.
  4448. Both end the same way: the slot index still resolves, so nothing looks
  4449. wrong, and the printer obeys it — an explicit ``ams_mapping`` bypasses
  4450. the firmware's own type check, so a PETG slot happily prints in ASA.
  4451. Returns a short reason when the mapping names a tray this printer does
  4452. not have loaded, points a slot at a tray holding a different filament
  4453. type, or points it at an external spool this printer reports empty while
  4454. its AMS holds that slot's filament (#3239). Returns None when the
  4455. mapping fits, and — deliberately — whenever we lack the evidence to
  4456. judge, so a recompute only ever follows a positive finding.
  4457. An unresolved (``-1``) required slot is NOT a conflict: it says the
  4458. matcher had nothing, not that the mapping belongs to another printer,
  4459. and destroying a partially hand-resolved mapping over it would lose the
  4460. slots the user did resolve. ``_fill_unresolved_slots`` matches those
  4461. slots again on their own, and ``_block_on_unmatched_filament`` holds the
  4462. item when that finds nothing.
  4463. """
  4464. if not isinstance(stored_mapping, list) or not stored_mapping:
  4465. return None
  4466. status = printer_manager.get_status(printer_id)
  4467. if status is None:
  4468. return None
  4469. loaded = self._build_loaded_filaments(status)
  4470. if not loaded:
  4471. # Nothing reported yet (reconnect, first push pending). Saying
  4472. # "tray not loaded" here would recompute against an AMS we cannot
  4473. # see, which is how #2589 produced a bogus all-[-1] in the first
  4474. # place.
  4475. return None
  4476. by_tray = {f["global_tray_id"]: f for f in loaded}
  4477. layout = TrayLayout.from_state(status)
  4478. # Cheap pass first, on live status alone: every tray the mapping names
  4479. # has to exist here. This catches a foreign mapping without opening the
  4480. # 3MF, which is worth doing because the parse below is neither cached
  4481. # nor free.
  4482. for index, tray in enumerate(stored_mapping):
  4483. if not _is_tray_id(tray) or tray < 0:
  4484. continue
  4485. if tray in by_tray:
  4486. continue
  4487. if tray >= 254 or layout.is_external(tray):
  4488. # An external feed we have not heard about is absence of
  4489. # evidence about *that slot* — the rest of the mapping is still
  4490. # worth judging, so skip it rather than abandoning the pass.
  4491. continue
  4492. return f"slot {index + 1} points at tray {tray}, which this printer does not have loaded"
  4493. # Only now is the 3MF worth opening: the type check needs to know what
  4494. # each slot actually asked for. The external spool is checked the same
  4495. # way as an AMS tray — `_build_loaded_filaments` reports its type, and
  4496. # two printers with different filament in the external feed is the same
  4497. # failure this method exists to catch.
  4498. required = await self._get_filament_requirements(db, item)
  4499. if not required:
  4500. return None
  4501. self._apply_filament_overrides(item, required)
  4502. # External feeds the printer reports and reports as empty. That is
  4503. # evidence, unlike an external feed it says nothing about (#3239).
  4504. empty_external: set[int] = set()
  4505. vt_trays = status.raw_data.get("vt_tray") if isinstance(status.raw_data, dict) else None
  4506. for vt in vt_trays if isinstance(vt_trays, list) else []:
  4507. if isinstance(vt, dict) and not vt.get("tray_type"):
  4508. try:
  4509. empty_external.add(int(vt.get("id", 254)))
  4510. except (TypeError, ValueError):
  4511. continue
  4512. for req in required:
  4513. slot_id = req.get("slot_id") or 0
  4514. if slot_id <= 0:
  4515. continue
  4516. if slot_id > len(stored_mapping):
  4517. return f"slot {slot_id} is not covered by the mapping"
  4518. tray = stored_mapping[slot_id - 1]
  4519. if not _is_tray_id(tray) or tray < 0:
  4520. continue
  4521. loaded_tray = by_tray.get(tray)
  4522. if loaded_tray is None:
  4523. # A mapping made for a printer that feeds this slot from its
  4524. # external spool, sent to one whose external spool is empty and
  4525. # whose AMS holds the filament (#3239). Only then: an external
  4526. # spool can be loaded without its filament set, and on a printer
  4527. # with nothing else to offer that job printed before. The colour
  4528. # has to match too: with only another colour in the AMS, the
  4529. # printer asking for the spool beats printing in that colour.
  4530. want = canonical_filament_type(req.get("type"))
  4531. if (tray >= 254 or layout.is_external(tray)) and tray in empty_external and want:
  4532. ams_tray = next(
  4533. (
  4534. f
  4535. for f in loaded
  4536. if not f.get("is_external")
  4537. and canonical_filament_type(f.get("type")) == want
  4538. and self._colors_are_similar(f.get("color"), req.get("color"))
  4539. ),
  4540. None,
  4541. )
  4542. if ams_tray is not None:
  4543. return (
  4544. f"slot {slot_id} points at the external spool, which is empty, "
  4545. f"while tray {ams_tray['global_tray_id']} holds {ams_tray.get('type')}"
  4546. )
  4547. continue
  4548. want = canonical_filament_type(req.get("type"))
  4549. have = canonical_filament_type(loaded_tray.get("type"))
  4550. if want and have and want != have:
  4551. return f"slot {slot_id} needs {req.get('type')} but tray {tray} holds {loaded_tray.get('type')}"
  4552. # A forced slot fits only a tray the matcher would give it (#3280).
  4553. if req.get("force_color_match"):
  4554. want_idx = req.get("tray_info_idx") or ""
  4555. have_idx = loaded_tray.get("tray_info_idx") or ""
  4556. if self._normalize_color_for_compare(req.get("color")) != self._normalize_color_for_compare(
  4557. loaded_tray.get("color")
  4558. ) or (want_idx and have_idx and want_idx != have_idx):
  4559. return (
  4560. f"slot {slot_id} is forced to {req.get('type')} {req.get('color')} "
  4561. f"but tray {tray} holds {loaded_tray.get('type')} {loaded_tray.get('color')}"
  4562. )
  4563. return None
  4564. async def _unmappable_without_ams_message(
  4565. self,
  4566. db: AsyncSession,
  4567. printer_id: int,
  4568. item: PrintQueueItem,
  4569. computed_mapping: list[int] | None,
  4570. ) -> str | None:
  4571. """Message for a mapping that resolved nothing on an AMS-less printer (#2771).
  4572. A print dispatched with no mapping goes out as ``use_ams: true`` with no
  4573. ``ams_mapping`` and no ``ams_mapping2``, which the firmware rejects with
  4574. 0700_8012 "Failed to get AMS mapping table" — after Bambuddy has already
  4575. uploaded several megabytes and burned its dispatch retries. This method
  4576. speaks only for the AMS-less case: there is nothing to resume into — the
  4577. external spool holder is the whole inventory — so the useful answer is to
  4578. say which filament is missing and stop. With an AMS attached it returns
  4579. None: where live status reported loaded trays,
  4580. ``_block_on_unmatched_filament`` holds the item instead, and where it
  4581. reported none the print goes out and the firmware error is the answer —
  4582. the user can load the right spool and press Resume.
  4583. Fail-safe by construction, mirroring the nozzle-diameter guard (#1899):
  4584. every branch that lacks the evidence to be sure returns None.
  4585. """
  4586. # None means the matcher never ran (no requirements parsed from the 3MF,
  4587. # or nothing loaded at all) rather than "ran and matched nothing". Those
  4588. # dispatch as they always have.
  4589. if not _mapping_is_all_unresolved(computed_mapping):
  4590. return None
  4591. status = printer_manager.get_status(printer_id)
  4592. if status is None:
  4593. return None
  4594. # "No AMS" has to be a fact the printer stated, not the absence of a
  4595. # statement. `raw_data["ams"]` is written only once an AMS push has been
  4596. # handled and is preserved across partial pushes thereafter, so a missing
  4597. # key means we have not heard yet — most likely a reconnect, where the
  4598. # trays of a fully loaded AMS would be invisible for a few seconds. An
  4599. # empty list is the positive report of a printer with no AMS.
  4600. ams_units = status.raw_data.get("ams")
  4601. if not isinstance(ams_units, list) or ams_units:
  4602. return None
  4603. required = await self._get_filament_requirements(db, item)
  4604. loaded = self._build_loaded_filaments(status)
  4605. # A spool its creator may not use isn't one the job could print from,
  4606. # so the message doesn't offer it (#3522).
  4607. scope = await self._creator_filament_scope(db, item.created_by_id)
  4608. if not scope.is_unrestricted:
  4609. blocked_ids = scope.blocked_tray_ids(loaded)
  4610. loaded = [f for f in loaded if f["global_tray_id"] not in blocked_ids]
  4611. if not required or not loaded:
  4612. # Both were non-empty moments ago or the matcher could not have run.
  4613. # If the picture changed under us, say nothing rather than fail an
  4614. # item on stale evidence.
  4615. return None
  4616. self._apply_filament_overrides(item, required)
  4617. if any(r.get("force_color_match") for r in required):
  4618. # A forced colour is a spool the user can load on the external
  4619. # holder, so the unmatched hold stages the job instead (#3280).
  4620. return None
  4621. return _unmatched_filament_message(required, loaded)
  4622. async def _fail_unmappable_item(
  4623. self, db: AsyncSession, item: PrintQueueItem, printer_id: int, message: str
  4624. ) -> None:
  4625. """Fail a queue item whose filament mapping cannot resolve (#2771).
  4626. This replaces a failure, not a success: without it the item is uploaded,
  4627. rejected by the firmware with 0700_8012, retried twice more and failed
  4628. anyway with "never started the print after N dispatch attempts". So this
  4629. applies on the model-based path too. There it is rarely reached: choosing
  4630. the printer already asks the matcher (``_unmatched_on``, #3280) and passes
  4631. over one it cannot map, so an "Any <model>" job tries its siblings and
  4632. otherwise waits; this remains for a match that failed at selection time.
  4633. Never for a filament the job's creator may not use: that is a
  4634. ``FilamentScopeHold``, and the job is held instead (#3522).
  4635. """
  4636. item.status = "failed"
  4637. item.error_message = message
  4638. item.completed_at = datetime.now(timezone.utc)
  4639. item.waiting_reason = None
  4640. await db.commit()
  4641. from backend.app.services.print_batch import refresh_batches_after
  4642. await refresh_batches_after([item.id])
  4643. logger.warning(
  4644. "Queue item %s: no usable AMS mapping on printer %s — %s",
  4645. item.id,
  4646. printer_id,
  4647. message,
  4648. )
  4649. job_name = await self._get_job_name(db, item)
  4650. printer = await self._get_printer(db, printer_id)
  4651. await notification_service.on_queue_job_failed(
  4652. job_name=job_name,
  4653. printer_id=printer_id,
  4654. printer_name=printer.name if printer else "Unknown",
  4655. reason=message,
  4656. db=db,
  4657. )
  4658. try:
  4659. await ws_manager.send_queue_item_failed(
  4660. user_id=item.created_by_id,
  4661. queue_item_id=item.id,
  4662. printer_id=printer_id,
  4663. reason="filament_unmappable",
  4664. )
  4665. except Exception:
  4666. pass
  4667. async def _compute_ams_mapping_for_printer(
  4668. self,
  4669. db: AsyncSession,
  4670. printer_id: int,
  4671. item: PrintQueueItem,
  4672. *,
  4673. only_slots: set[int] | None = None,
  4674. reserved_trays: set[int] | None = None,
  4675. ) -> list[int] | None:
  4676. """Compute AMS mapping for a printer based on filament requirements.
  4677. Called when a queue item has no ams_mapping set — either for model-based
  4678. items after printer assignment, or printer-specific items (e.g. from VP).
  4679. Args:
  4680. db: Database session
  4681. printer_id: The assigned printer ID
  4682. item: The queue item (contains archive_id or library_file_id)
  4683. only_slots: Match only these filament slots; every other slot comes
  4684. back unresolved (-1). Used to fill the gaps of a stored mapping.
  4685. reserved_trays: Global tray IDs to leave out of the match, because
  4686. the stored mapping already gives them to another slot.
  4687. Returns:
  4688. AMS mapping array or None if no mapping needed/possible
  4689. """
  4690. # Get printer status
  4691. status = printer_manager.get_status(printer_id)
  4692. if not status:
  4693. logger.warning("Cannot compute AMS mapping: printer %s status unavailable", printer_id)
  4694. return None
  4695. # Filament Track Switch (FTS): when installed it routes any AMS slot to
  4696. # either extruder, so the per-nozzle hard filter below must NOT apply.
  4697. # Otherwise a print on one nozzle can't use a spool physically loaded in
  4698. # an AMS on the *other* nozzle, and the matcher falls through to a
  4699. # same-type wrong-colour spool on the target nozzle — the H2C + FTS
  4700. # wrong-filament bug (#2186). Mirrors the frontend skip added for #1162.
  4701. fts_installed = bool(getattr(getattr(status, "fila_switch", None), "installed", False))
  4702. # Get filament requirements from source file
  4703. filament_reqs = await self._get_filament_requirements(db, item)
  4704. if not filament_reqs:
  4705. # When the 3MF can't be read but force-color overrides are present, build a
  4706. # direct mapping from the overrides so the printer uses the correct AMS slot.
  4707. if item.filament_overrides:
  4708. try:
  4709. overrides = json.loads(item.filament_overrides)
  4710. force_overrides = [o for o in overrides if o.get("force_color_match")]
  4711. if force_overrides:
  4712. logger.info(
  4713. "Queue item %s: No filament reqs from 3MF; building AMS mapping from %d "
  4714. "force-color override(s)",
  4715. item.id,
  4716. len(force_overrides),
  4717. )
  4718. # Same tray rule as the matcher below (#3522).
  4719. override_scope = await self._creator_filament_scope(db, item.created_by_id)
  4720. blocked_trays = (
  4721. None
  4722. if override_scope.is_unrestricted
  4723. else override_scope.blocked_tray_ids(self._build_loaded_filaments(status))
  4724. )
  4725. return self._build_override_direct_mapping(force_overrides, status, blocked_trays=blocked_trays)
  4726. except (json.JSONDecodeError, KeyError, TypeError) as e:
  4727. logger.warning("Queue item %s: Force-color fallback mapping failed: %s", item.id, e)
  4728. logger.debug("No filament requirements found for queue item %s", item.id)
  4729. return None
  4730. self._apply_filament_overrides(item, filament_reqs)
  4731. if only_slots is not None:
  4732. filament_reqs = [req for req in filament_reqs if req.get("slot_id") in only_slots]
  4733. if not filament_reqs:
  4734. return None
  4735. # Build loaded filaments from printer status
  4736. loaded_filaments = self._build_loaded_filaments(status)
  4737. if reserved_trays:
  4738. loaded_filaments = [f for f in loaded_filaments if f["global_tray_id"] not in reserved_trays]
  4739. # Trays holding a filament the job's creator may not print with
  4740. # (#3522) are never matched: the job takes another tray or waits.
  4741. filament_scope = await self._creator_filament_scope(db, item.created_by_id)
  4742. if not filament_scope.is_unrestricted:
  4743. blocked_trays = filament_scope.blocked_tray_ids(loaded_filaments)
  4744. if blocked_trays:
  4745. loaded_filaments = [f for f in loaded_filaments if f["global_tray_id"] not in blocked_trays]
  4746. if not loaded_filaments:
  4747. logger.debug("No filaments loaded on printer %s", printer_id)
  4748. return None
  4749. # Check if user prefers lowest remaining filament when multiple spools match
  4750. prefer_lowest = await self._get_bool_setting(db, "prefer_lowest_filament")
  4751. # Gate prefer_lowest on the printer's AMS Filament Backup state (#1766).
  4752. # Without backup, the printer will not switch to a second spool when the
  4753. # picked one runs out — so sorting toward the lowest leaves the print
  4754. # at risk of running dry mid-job. None (unknown: no status report with
  4755. # cfg or home_flag yet) preserves today's behaviour intentionally.
  4756. if prefer_lowest and status.ams_filament_backup is False:
  4757. logger.info("[prefer-lowest] skipped (AMS Backup OFF on printer %s)", printer_id)
  4758. prefer_lowest = False
  4759. # When the preference is on, surface Bambuddy's inventory-side
  4760. # remaining for each slot that's bound to a tracked spool, so the
  4761. # sort beats the MQTT-only blind spot (#1508). Skip the lookup
  4762. # entirely when the preference is off — no behaviour change for
  4763. # users who haven't opted in.
  4764. inventory_remain_overrides: dict[int, float] | None = None
  4765. if prefer_lowest:
  4766. inventory_remain_overrides = await self._build_inventory_remain_overrides(db, printer_id, loaded_filaments)
  4767. # Compute mapping: match required filaments to available slots
  4768. return self._match_filaments_to_slots(
  4769. filament_reqs, loaded_filaments, prefer_lowest, inventory_remain_overrides, fts_installed
  4770. )
  4771. def _apply_filament_overrides(self, item: PrintQueueItem, filament_reqs: list[dict]) -> None:
  4772. """Rewrite ``filament_reqs`` in place with the item's per-slot overrides.
  4773. Extracted from ``_compute_ams_mapping_for_printer`` so the unmappable
  4774. diagnosis (#2771) describes the filament the matcher actually looked
  4775. for, not the one the 3MF was sliced with — naming the pre-override
  4776. filament in a user-facing error would send the user to load the wrong
  4777. spool.
  4778. """
  4779. if not item.filament_overrides:
  4780. return
  4781. try:
  4782. overrides = json.loads(item.filament_overrides)
  4783. override_map = {o["slot_id"]: o for o in overrides}
  4784. for req in filament_reqs:
  4785. if req["slot_id"] in override_map:
  4786. override = override_map[req["slot_id"]]
  4787. req["type"] = override["type"]
  4788. req["color"] = override["color"]
  4789. # A manual/preference override SWAPS the slot's filament, so the
  4790. # 3MF's original tray_info_idx now points at the old spool and must
  4791. # be cleared — matching then falls back to type+colour. A
  4792. # force_color_match override is not a swap: it carries the 3MF's
  4793. # intended variant (Basic GFA00 / Matte GFA01 / Silk GFA06), so keep
  4794. # it here too, letting the matcher pin the correct variant slot on a
  4795. # printer holding two same-colour spools of different variants (#2650).
  4796. # A spool that names no variant still counts; one naming another
  4797. # variant does not, as in `_get_missing_force_color_slots` (#3280).
  4798. req["tray_info_idx"] = (
  4799. override.get("tray_info_idx", "") if override.get("force_color_match") else ""
  4800. )
  4801. # The matcher satisfies a forced slot only the way
  4802. # `_get_missing_force_color_slots` does, never by a similar
  4803. # colour or the type alone (#3280).
  4804. req["force_color_match"] = bool(override.get("force_color_match"))
  4805. logger.debug(
  4806. "Queue item %s: Override slot %d -> %s %s",
  4807. item.id,
  4808. req["slot_id"],
  4809. override["type"],
  4810. override["color"],
  4811. )
  4812. except (json.JSONDecodeError, KeyError, TypeError) as e:
  4813. logger.warning("Failed to apply filament overrides for queue item %s: %s", item.id, e)
  4814. def _build_override_direct_mapping(
  4815. self, force_overrides: list[dict], status, *, blocked_trays: set[int] | None = None
  4816. ) -> list[int] | None:
  4817. """Build an AMS mapping directly from force-color overrides without a 3MF.
  4818. Used when ``_get_filament_requirements`` returns nothing (e.g. the 3MF's
  4819. slice_info is missing or unreadable) but ``force_color_match`` overrides
  4820. are present. Each override's ``slot_id``, ``type``, and ``color`` are
  4821. treated as the filament requirement for that slot and matched against the
  4822. current AMS state of the printer.
  4823. ``blocked_trays`` are global tray IDs holding a filament the job's
  4824. creator may not print with (#3522); they are left out of the match.
  4825. Returns the same format as ``_match_filaments_to_slots``, or None when
  4826. the AMS has no loaded filaments (or only blocked ones).
  4827. """
  4828. loaded = self._build_loaded_filaments(status)
  4829. if blocked_trays:
  4830. loaded = [f for f in loaded if f["global_tray_id"] not in blocked_trays]
  4831. if not loaded:
  4832. return None
  4833. reqs = [
  4834. {
  4835. "slot_id": o["slot_id"],
  4836. "type": o.get("type", ""),
  4837. "color": o.get("color", ""),
  4838. # These are all force_color_match overrides, so the idx (when the
  4839. # 3MF carried one) is the intended variant, not a stale swap —
  4840. # keep it so the matcher pins the right variant slot (#2650); a
  4841. # spool naming another variant does not satisfy it (#3280).
  4842. "tray_info_idx": o.get("tray_info_idx", ""),
  4843. "force_color_match": True,
  4844. }
  4845. for o in force_overrides
  4846. ]
  4847. return self._match_filaments_to_slots(reqs, loaded)
  4848. async def _get_filament_requirements(self, db: AsyncSession, item: PrintQueueItem) -> list[dict] | None:
  4849. """Resolve the queue item's source 3MF and parse the per-slot
  4850. filament requirements out of it. Thin DB-resolver wrapper around
  4851. ``filament_requirements.extract_filament_requirements`` so the VP
  4852. queue-mode write path (#1188) can reuse the same parser at upload
  4853. time.
  4854. """
  4855. from backend.app.services.filament_requirements import extract_filament_requirements
  4856. # Callers rewrite these dicts in place via `_apply_filament_overrides`,
  4857. # so every caller gets its own copy — a shared list would leak one
  4858. # item's overrides into the next item that happens to print the same
  4859. # plate.
  4860. memo_key = (item.archive_id, item.library_file_id, item.plate_id)
  4861. if memo_key in self._filament_req_memo:
  4862. cached = self._filament_req_memo[memo_key]
  4863. return [dict(r) for r in cached] if cached else None
  4864. file_path = await self._source_file_path(db, item)
  4865. if not file_path or not file_path.exists():
  4866. self._filament_req_memo[memo_key] = None
  4867. return None
  4868. filaments = extract_filament_requirements(file_path, plate_id=item.plate_id)
  4869. self._filament_req_memo[memo_key] = filaments or None
  4870. return [dict(r) for r in filaments] if filaments else None
  4871. async def _source_file_path(self, db: AsyncSession, item: PrintQueueItem) -> Path | None:
  4872. """Where the queue item's source file should be on disk, or None when its archive or library row is gone."""
  4873. if item.archive_id:
  4874. result = await db.execute(select(PrintArchive).where(PrintArchive.id == item.archive_id))
  4875. archive = result.scalar_one_or_none()
  4876. if archive:
  4877. return settings.base_dir / archive.file_path
  4878. elif item.library_file_id:
  4879. result = await db.execute(LibraryFile.active().where(LibraryFile.id == item.library_file_id))
  4880. library_file = result.scalar_one_or_none()
  4881. if library_file:
  4882. lib_path = Path(library_file.file_path)
  4883. return lib_path if lib_path.is_absolute() else settings.base_dir / library_file.file_path
  4884. return None
  4885. def _build_loaded_filaments(self, status) -> list[dict]:
  4886. """Build list of loaded filaments from printer status.
  4887. Args:
  4888. status: PrinterState from printer_manager
  4889. Returns:
  4890. List of loaded filament dicts with type, color, ams_id, tray_id, global_tray_id
  4891. """
  4892. filaments = []
  4893. # Get ams_extruder_map for dual-nozzle printers (H2D, H2D Pro)
  4894. ams_extruder_map = status.raw_data.get("ams_extruder_map", {})
  4895. # Routes external spools to an extruder (#2771). Mirrors
  4896. # `buildLoadedFilaments` in the frontend.
  4897. vt_trays = status.raw_data.get("vt_tray") or []
  4898. is_dual_nozzle = reports_two_nozzles(status)
  4899. # Parse AMS units from raw_data
  4900. ams_data = status.raw_data.get("ams", [])
  4901. layout = TrayLayout.from_state(status)
  4902. for ams_unit in ams_data:
  4903. ams_id = int(ams_unit.get("id", 0))
  4904. trays = ams_unit.get("tray", [])
  4905. is_ht = is_ams_ht_id(ams_id) # AMS-HT unit ids; a regular AMS may list fewer trays
  4906. for tray in trays:
  4907. tray_type = tray.get("tray_type")
  4908. if tray_type:
  4909. tray_id = int(tray.get("id", 0))
  4910. tray_color = tray.get("tray_color", "")
  4911. # tray_info_idx identifies the specific spool (e.g., "GFA00", "P4d64437")
  4912. tray_info_idx = tray.get("tray_info_idx", "")
  4913. # Normalize color: remove alpha, add hash
  4914. color = self._normalize_color(tray_color)
  4915. # Calculate global tray ID
  4916. # AMS-HT units have IDs starting at 128 with a single tray
  4917. global_tray_id = layout.global_tray_id(ams_id, tray_id)
  4918. filaments.append(
  4919. {
  4920. "type": tray_type,
  4921. "color": color,
  4922. # `color` reads a missing colour as #808080; this
  4923. # says whether the printer reported one at all.
  4924. "color_known": bool(tray_color),
  4925. "tray_info_idx": tray_info_idx,
  4926. "ams_id": ams_id,
  4927. "tray_id": tray_id,
  4928. "is_ht": is_ht,
  4929. "is_external": False,
  4930. "global_tray_id": global_tray_id,
  4931. "extruder_id": ams_extruder_map.get(str(ams_id)),
  4932. "remain": tray.get("remain", -1),
  4933. }
  4934. )
  4935. # Check external spool(s) (vt_tray is a list)
  4936. for idx, vt in enumerate(vt_trays):
  4937. if vt.get("tray_type"):
  4938. color = self._normalize_color(vt.get("tray_color", ""))
  4939. tray_id = int(vt.get("id", 254))
  4940. filaments.append(
  4941. {
  4942. "type": vt["tray_type"],
  4943. "color": color,
  4944. "color_known": bool(vt.get("tray_color")),
  4945. "tray_info_idx": vt.get("tray_info_idx", ""),
  4946. "ams_id": -1,
  4947. "tray_id": idx,
  4948. "is_ht": False,
  4949. "is_external": True,
  4950. "global_tray_id": tray_id,
  4951. # 254 = VIRTUAL_TRAY_DEPUTY_ID feeds extruder 1 (left),
  4952. # 255 = VIRTUAL_TRAY_MAIN_ID feeds extruder 0 (right).
  4953. "extruder_id": (255 - tray_id) if is_dual_nozzle else None,
  4954. "remain": vt.get("remain", -1),
  4955. }
  4956. )
  4957. return filaments
  4958. def _normalize_color(self, color: str | None) -> str:
  4959. """Normalize color to #RRGGBB format."""
  4960. if not color:
  4961. return "#808080"
  4962. hex_color = color.replace("#", "")[:6]
  4963. return f"#{hex_color}"
  4964. def _normalize_color_for_compare(self, color: str | None) -> str:
  4965. """Normalize color for comparison (lowercase, no hash)."""
  4966. if not color:
  4967. return ""
  4968. return color.replace("#", "").lower()[:6]
  4969. def _color_distance(self, color1: str | None, color2: str | None) -> float | None:
  4970. """Perceptual (CIEDE2000) distance, or None when either colour is unusable.
  4971. Ranks the candidates ``_colors_are_similar`` admits (#2804). Eligibility
  4972. stays the per-channel box that shipped — this only decides which of
  4973. several eligible spools is closest, so nothing becomes usable or
  4974. unusable because of it.
  4975. It ranks by how far apart the colours *look*, not how far apart their
  4976. numbers are. RGB distance overweights blue badly enough to invert the
  4977. answer: against a required ``#1E4821`` green, a purple ``#38202F`` is
  4978. the nearer of two eligible spools by RGB and the further by a factor of
  4979. four once measured perceptually.
  4980. Alpha is ignored, deliberately: the alpha a slicer writes for a
  4981. transparent filament is not a colour the user chose, and counting it
  4982. would stop a transparent filament matching itself.
  4983. """
  4984. return perceptual_color_distance(color1, color2)
  4985. def _colors_are_similar(self, color1: str | None, color2: str | None, threshold: int = 40) -> bool:
  4986. """Check if two colors are visually similar within a threshold."""
  4987. hex1 = self._normalize_color_for_compare(color1)
  4988. hex2 = self._normalize_color_for_compare(color2)
  4989. if not hex1 or not hex2 or len(hex1) < 6 or len(hex2) < 6:
  4990. return False
  4991. try:
  4992. r1 = int(hex1[0:2], 16)
  4993. g1 = int(hex1[2:4], 16)
  4994. b1 = int(hex1[4:6], 16)
  4995. r2 = int(hex2[0:2], 16)
  4996. g2 = int(hex2[2:4], 16)
  4997. b2 = int(hex2[4:6], 16)
  4998. return abs(r1 - r2) <= threshold and abs(g1 - g2) <= threshold and abs(b1 - b2) <= threshold
  4999. except ValueError:
  5000. return False
  5001. async def _build_inventory_remain_overrides(
  5002. self, db: AsyncSession, printer_id: int, loaded: list[dict]
  5003. ) -> dict[int, float]:
  5004. """Return ``{global_tray_id: remaining_grams}`` for AMS slots the user
  5005. has bound to an inventory spool — Bambuddy-side or Spoolman-side.
  5006. The MQTT ``remain`` field on a tray is the printer firmware's
  5007. RFID-decremented value, which has two limitations the "Prefer Lowest
  5008. Remaining Filament" feature has been ignoring (#1508):
  5009. - it's only meaningful for Bambu RFID spools; everything else reports
  5010. ``-1`` (then clamped to a sentinel), so multiple non-RFID trays
  5011. compare equal and the sort collapses to AMS-slot order — the user
  5012. who's curating inventory weights gets the lower-slot pick instead
  5013. of the lower-remaining pick;
  5014. - even when set, it's the *printer's* counter, not Bambuddy's
  5015. ``label_weight - weight_used`` (internal mode) or Spoolman's
  5016. ``remaining_weight`` (Spoolman mode) — the two diverge any time the
  5017. user re-spools, swaps cardboard, or runs a print outside Bambuddy.
  5018. When the user has bound a spool to a slot, their own inventory
  5019. tracking is authoritative; this helper surfaces that value so the
  5020. sort can prefer it. Slots without a binding are absent from the
  5021. returned map — the caller then falls back to MQTT ``remain`` for
  5022. those, preserving the pre-#1508 behaviour for un-tracked spools.
  5023. Returns an empty map on any failure (no inventory bindings, DB
  5024. error, Spoolman unreachable). A best-effort lookup; "Prefer Lowest"
  5025. is a preference, not a guarantee.
  5026. """
  5027. if not loaded:
  5028. return {}
  5029. # External / virtual-tray slots are tracked separately from AMS — skip
  5030. # them so a VT-loaded spool doesn't accidentally inherit a tracked
  5031. # AMS binding (the tables use ams_id 254/255 for VT, but the cross
  5032. # match is fiddly and out of scope for this fix).
  5033. tracked_slots = [(f["ams_id"], f["tray_id"], f["global_tray_id"]) for f in loaded if not f.get("is_external")]
  5034. if not tracked_slots:
  5035. return {}
  5036. is_spoolman = await self._is_spoolman_mode(db)
  5037. overrides: dict[int, float] = {}
  5038. if is_spoolman:
  5039. result = await db.execute(
  5040. select(SpoolmanSlotAssignment).where(SpoolmanSlotAssignment.printer_id == printer_id)
  5041. )
  5042. assignments = list(result.scalars().all())
  5043. by_slot = {(a.ams_id, a.tray_id): a.spoolman_spool_id for a in assignments}
  5044. from backend.app.services.filament_deficit import _spoolman_remaining_grams
  5045. for ams_id, tray_id, gtid in tracked_slots:
  5046. spoolman_id = by_slot.get((ams_id, tray_id))
  5047. if spoolman_id is None:
  5048. continue
  5049. grams = await _spoolman_remaining_grams(spoolman_id)
  5050. if grams is not None:
  5051. overrides[gtid] = grams
  5052. return overrides
  5053. # Internal inventory mode (default). selectinload matches the pattern
  5054. # used elsewhere (inventory.py, spoolman.py routes) — a single query
  5055. # plus an eager-loaded relationship rather than an explicit join, so
  5056. # the row-attribute shape is exactly what those routes already rely on.
  5057. result = await db.execute(
  5058. select(SpoolAssignment)
  5059. .options(selectinload(SpoolAssignment.spool))
  5060. .where(SpoolAssignment.printer_id == printer_id)
  5061. )
  5062. assignments = list(result.scalars().all())
  5063. by_slot = {(a.ams_id, a.tray_id): a.spool for a in assignments}
  5064. for ams_id, tray_id, gtid in tracked_slots:
  5065. spool = by_slot.get((ams_id, tray_id))
  5066. if spool is None:
  5067. continue
  5068. label = float(spool.label_weight or 0)
  5069. used = float(spool.weight_used or 0)
  5070. overrides[gtid] = max(0.0, label - used)
  5071. return overrides
  5072. @staticmethod
  5073. async def _is_spoolman_mode(db: AsyncSession) -> bool:
  5074. """Mirror of ``filament_deficit._is_spoolman_mode`` — kept private
  5075. here to avoid making this module import-dependent on that private
  5076. helper's signature."""
  5077. try:
  5078. from backend.app.api.routes.settings import get_setting
  5079. v = await get_setting(db, "spoolman_enabled")
  5080. return bool(v) and v.lower() == "true"
  5081. except Exception:
  5082. return False
  5083. @staticmethod
  5084. def _slot_priority(ams_id: int | None, tray_id: int | None) -> int:
  5085. """Deterministic slot-position tie-breaker for the prefer-lowest sort.
  5086. Three bands, matched to the emission order in ``_build_loaded_filaments``
  5087. so a tied sort produces the same physical-position order the pre-#1508
  5088. stable sort did (preserves the regression-free baseline):
  5089. - Regular AMS (``ams_id`` 0..7): ``ams_id * 4 + tray_id`` → 0..31
  5090. - AMS-HT (``ams_id`` >= 128, single tray): ``1000 + (ams_id - 128) * 4``
  5091. - External / VT (``ams_id`` < 0, or ``None``): ``10_000``
  5092. Banding ensures regular AMS < AMS-HT < external on ties, regardless of
  5093. what the raw ``ams_id`` happens to be (in particular, ``ams_id = -1``
  5094. for VT must NOT sort to a negative number or it would beat AMS slot 0).
  5095. """
  5096. if ams_id is None or ams_id < 0:
  5097. return 10_000
  5098. if ams_id >= 128:
  5099. return 1_000 + (ams_id - 128) * 4 + (tray_id or 0)
  5100. return ams_id * 4 + (tray_id or 0)
  5101. @staticmethod
  5102. def _prefer_lowest_sort_key(f: dict, overrides: dict[int, float] | None) -> tuple[int, float, int]:
  5103. """Sort key for the "Prefer Lowest Remaining Filament" preference.
  5104. Two-tier ordering: inventory-tracked spools always sort BEFORE
  5105. non-tracked spools (the user has told us they care about these
  5106. specifically), then ascending by remaining within each tier, then
  5107. ascending by AMS slot position as the deterministic tie-breaker.
  5108. Tiers are flagged by the first tuple element (0 = inventory-tracked,
  5109. 1 = MQTT-only / unknown). Cross-tier value comparisons never run
  5110. because the tier flag dominates — which is what lets us mix grams
  5111. (inventory) and percent (MQTT) without a unit conversion.
  5112. Within the MQTT tier ``remain = -1`` (unknown) is mapped to 101 so
  5113. spools the printer DOES know something about sort ahead of those
  5114. it knows nothing about — preserves pre-#1508 behaviour for the
  5115. no-inventory-binding case.
  5116. Slot tie-breaker via ``_slot_priority`` so regular AMS < AMS-HT <
  5117. external on ties, matching the legacy emission-order stable sort.
  5118. """
  5119. gtid = f.get("global_tray_id")
  5120. slot_order = PrintScheduler._slot_priority(f.get("ams_id"), f.get("tray_id"))
  5121. if overrides and gtid in overrides:
  5122. return (0, overrides[gtid], slot_order)
  5123. remain = f.get("remain", -1)
  5124. return (1, float(remain) if remain is not None and remain >= 0 else 101.0, slot_order)
  5125. def _match_filaments_to_slots(
  5126. self,
  5127. required: list[dict],
  5128. loaded: list[dict],
  5129. prefer_lowest: bool = False,
  5130. inventory_remain_overrides: dict[int, float] | None = None,
  5131. fts_installed: bool = False,
  5132. ) -> list[int] | None:
  5133. """Match required filaments to loaded filaments and build AMS mapping.
  5134. Priority: unique tray_info_idx match > exact color match > similar color match > type-only match
  5135. The tray_info_idx is a filament type identifier stored in the 3MF file when the user
  5136. slices (e.g., "GFA00" for generic PLA, "P4d64437" for custom presets). If the same
  5137. tray_info_idx appears in only ONE available tray, we use that tray. If multiple trays
  5138. have the same tray_info_idx (e.g., two spools of generic PLA), we fall back to color
  5139. matching among those trays.
  5140. Args:
  5141. required: List of required filaments with slot_id, type, color, tray_info_idx
  5142. loaded: List of loaded filaments with type, color, tray_info_idx, global_tray_id
  5143. Returns:
  5144. AMS mapping array (position = slot_id - 1, value = global_tray_id or -1)
  5145. """
  5146. if not required:
  5147. return None
  5148. # Track used trays to avoid duplicate assignment
  5149. used_tray_ids: set[int] = set()
  5150. comparisons = []
  5151. # Forced slots first: a slot that would take any tray of its type must
  5152. # not take the one tray a forced slot can use (#3280). The mapping is
  5153. # built by slot_id below, so the order only decides who goes first.
  5154. # Among forced slots, those naming a variant first, as
  5155. # `_get_missing_force_color_slots` counts them.
  5156. for req in sorted(
  5157. required,
  5158. key=lambda r: (
  5159. not r.get("force_color_match"),
  5160. bool(r.get("force_color_match")) and not r.get("tray_info_idx"),
  5161. ),
  5162. ):
  5163. req_type = (req.get("type") or "").upper()
  5164. req_color = req.get("color", "")
  5165. req_tray_info_idx = req.get("tray_info_idx", "")
  5166. # Find best match: unique tray_info_idx > exact color > similar color > type-only
  5167. idx_match = None
  5168. exact_match = None
  5169. similar_match = None
  5170. similar_distance = float("inf")
  5171. type_only_match = None
  5172. # Get available trays (not already used)
  5173. available = [f for f in loaded if f["global_tray_id"] not in used_tray_ids]
  5174. # Nozzle-aware filtering: restrict to trays on the correct nozzle.
  5175. # Hard filter — cross-nozzle assignment causes print failures
  5176. # ("position of left hotend is abnormal"), so never fall back.
  5177. # Skipped when an FTS is installed: it routes any AMS slot to either
  5178. # extruder, so restricting to one nozzle would wrongly exclude the
  5179. # correct spool sitting in the other nozzle's AMS (#2186).
  5180. req_nozzle_id = req.get("nozzle_id")
  5181. if req_nozzle_id is not None and not fts_installed:
  5182. available = [f for f in available if f.get("extruder_id") == req_nozzle_id]
  5183. # Sort by remaining filament (ascending) so lowest-remain spool wins .find().
  5184. # Inventory-tracked spools sort before MQTT-only ones (#1508); see
  5185. # _prefer_lowest_sort_key for the full rationale.
  5186. if prefer_lowest:
  5187. available.sort(key=lambda f: self._prefer_lowest_sort_key(f, inventory_remain_overrides))
  5188. # INFO-level decision trace for "Prefer Lowest Filament" #1766.
  5189. # One line per filament req so a bug report can be diagnosed
  5190. # without enabling debug logging: shows what the matcher saw
  5191. # (req shape + sorted candidate trays with their remain values
  5192. # and any inventory override that was applied). Mirrored by
  5193. # the picked-match log at the bottom of the loop.
  5194. logger.info(
  5195. "[prefer-lowest] req slot=%s type=%r color=%r tii=%r nozzle=%s; available (sorted lowest-first): %s",
  5196. req.get("slot_id"),
  5197. req_type,
  5198. req_color,
  5199. req_tray_info_idx,
  5200. req_nozzle_id,
  5201. [
  5202. {
  5203. "gtid": f.get("global_tray_id"),
  5204. "type": f.get("type"),
  5205. "color": f.get("color"),
  5206. "tii": f.get("tray_info_idx"),
  5207. "remain": f.get("remain"),
  5208. "inv_g": (
  5209. inventory_remain_overrides.get(f.get("global_tray_id"))
  5210. if inventory_remain_overrides
  5211. else None
  5212. ),
  5213. }
  5214. for f in available
  5215. ],
  5216. )
  5217. # A forced slot takes only what `_get_missing_force_color_slots`
  5218. # accepts: same type, exact colour, and the same variant when both
  5219. # sides name one -- preferring the tray that does. Nothing looser:
  5220. # the model-based path holds a job the printer cannot satisfy this
  5221. # way, and a printer-pinned job fell through to type_only and
  5222. # printed in whatever colour came first (#3280).
  5223. forced = bool(req.get("force_color_match"))
  5224. if forced:
  5225. want_type = canonical_filament_type(req_type)
  5226. want_color = self._normalize_color_for_compare(req_color)
  5227. candidates = [
  5228. f
  5229. for f in available
  5230. if canonical_filament_type((f.get("type") or "").upper()) == want_type
  5231. and self._normalize_color_for_compare(f.get("color", "")) == want_color
  5232. and (
  5233. not req_tray_info_idx
  5234. or not f.get("tray_info_idx")
  5235. or f.get("tray_info_idx") == req_tray_info_idx
  5236. )
  5237. ]
  5238. exact_match = next(
  5239. (f for f in candidates if req_tray_info_idx and f.get("tray_info_idx") == req_tray_info_idx),
  5240. candidates[0] if candidates else None,
  5241. )
  5242. # Check if tray_info_idx is unique among available trays
  5243. if req_tray_info_idx and not forced:
  5244. idx_matches = [f for f in available if f.get("tray_info_idx") == req_tray_info_idx]
  5245. if len(idx_matches) == 1:
  5246. # Unique tray_info_idx - use it as definitive match
  5247. idx_match = idx_matches[0]
  5248. logger.debug(
  5249. f"Matched filament slot {req.get('slot_id')} by unique tray_info_idx={req_tray_info_idx} "
  5250. f"-> tray {idx_match['global_tray_id']}"
  5251. )
  5252. elif len(idx_matches) > 1:
  5253. # Multiple trays with same tray_info_idx - use color matching among them
  5254. logger.debug(
  5255. f"Non-unique tray_info_idx={req_tray_info_idx} found in {len(idx_matches)} trays, "
  5256. f"using color matching among trays: {[f['global_tray_id'] for f in idx_matches]}"
  5257. )
  5258. if prefer_lowest:
  5259. idx_matches.sort(key=lambda f: self._prefer_lowest_sort_key(f, inventory_remain_overrides))
  5260. # Use color matching within this subset
  5261. for f in idx_matches:
  5262. f_color = f.get("color", "")
  5263. if self._normalize_color_for_compare(f_color) == self._normalize_color_for_compare(req_color):
  5264. if not exact_match:
  5265. exact_match = f
  5266. elif self._colors_are_similar(f_color, req_color):
  5267. distance = self._color_distance(f_color, req_color)
  5268. if distance is not None and distance < similar_distance:
  5269. similar_match = f
  5270. similar_distance = distance
  5271. elif not type_only_match:
  5272. type_only_match = f
  5273. # If no idx_match yet, do standard type/color matching on all available trays
  5274. if not forced and not idx_match and not exact_match and not similar_match and not type_only_match:
  5275. for f in available:
  5276. f_type = (f.get("type") or "").upper()
  5277. if canonical_filament_type(f_type) != canonical_filament_type(req_type):
  5278. continue
  5279. # Type matches - check color
  5280. f_color = f.get("color", "")
  5281. if self._normalize_color_for_compare(f_color) == self._normalize_color_for_compare(req_color):
  5282. if not exact_match:
  5283. exact_match = f
  5284. elif self._colors_are_similar(f_color, req_color):
  5285. # Nearest wins, not first-in-tray-order. `available` is
  5286. # already in the caller's order (slot order, or the
  5287. # prefer-lowest sort), and `<` keeps the earliest of
  5288. # equally close spools — so that order survives as the
  5289. # tie-break (#2804).
  5290. distance = self._color_distance(f_color, req_color)
  5291. if distance is not None and distance < similar_distance:
  5292. similar_match = f
  5293. similar_distance = distance
  5294. elif not type_only_match:
  5295. type_only_match = f
  5296. match = idx_match or exact_match or similar_match or type_only_match
  5297. if match:
  5298. used_tray_ids.add(match["global_tray_id"])
  5299. comparisons.append({"slot_id": req.get("slot_id", 0), "global_tray_id": match["global_tray_id"]})
  5300. else:
  5301. comparisons.append({"slot_id": req.get("slot_id", 0), "global_tray_id": -1})
  5302. # Which bucket won, always — not only under Prefer Lowest (#2804).
  5303. # "Why did it pick that spool" is the question every wrong-filament
  5304. # report starts with, and a `similar_color` win is now a ranked
  5305. # choice among several eligible spools rather than whichever tray
  5306. # came first, so it is worth being able to see after the fact.
  5307. # Pairs with the "available (sorted)" log above when Prefer Lowest
  5308. # is on (#1766).
  5309. if match:
  5310. bucket = (
  5311. "idx"
  5312. if idx_match is not None
  5313. else "exact_color"
  5314. if exact_match is not None
  5315. else "similar_color"
  5316. if similar_match is not None
  5317. else "type_only"
  5318. )
  5319. logger.info(
  5320. "[ams-match] picked gtid=%s via %s for req slot=%s%s",
  5321. match["global_tray_id"],
  5322. bucket,
  5323. req.get("slot_id"),
  5324. f" (deltaE {similar_distance:.2f})" if bucket == "similar_color" else "",
  5325. )
  5326. else:
  5327. logger.info(
  5328. "[ams-match] NO MATCH for req slot=%s (type=%r color=%r tii=%r%s)",
  5329. req.get("slot_id"),
  5330. req_type,
  5331. req_color,
  5332. req_tray_info_idx,
  5333. ", force color match" if forced else "",
  5334. )
  5335. # Build mapping array
  5336. if not comparisons:
  5337. return None
  5338. max_slot_id = max(c["slot_id"] for c in comparisons)
  5339. if max_slot_id <= 0:
  5340. return None
  5341. mapping = [-1] * max_slot_id
  5342. for c in comparisons:
  5343. slot_id = c["slot_id"]
  5344. if slot_id and slot_id > 0:
  5345. mapping[slot_id - 1] = c["global_tray_id"]
  5346. return mapping
  5347. def _mark_printer_dispatched(
  5348. self,
  5349. printer_id: int,
  5350. pre_state: str | None,
  5351. pre_subtask_id: str | None,
  5352. ) -> None:
  5353. """Record that a print command was just sent to ``printer_id``.
  5354. Held until either the watchdog observes a state/subtask transition
  5355. (success path) or the hard timeout expires. See ``_dispatch_holds``.
  5356. """
  5357. if not pre_state:
  5358. # No pre_state means we can't detect a transition — fall back to a
  5359. # pure time-based hold using empty string as a sentinel that won't
  5360. # match any real printer state.
  5361. pre_state = ""
  5362. self._dispatch_holds[printer_id] = (time.monotonic(), pre_state, pre_subtask_id)
  5363. def _release_dispatch_hold(self, printer_id: int) -> None:
  5364. """Drop the dispatch hold for ``printer_id`` (called by the watchdog)."""
  5365. self._dispatch_holds.pop(printer_id, None)
  5366. def _printer_in_dispatch_hold(self, printer_id: int) -> bool:
  5367. """True if ``printer_id`` is still inside its post-dispatch hold window.
  5368. Returns False (and clears the hold) once any of these are true:
  5369. - hard timeout (``_dispatch_max_hold``) has elapsed
  5370. - the printer has transitioned out of pre_state and we're past the
  5371. minimum cooldown
  5372. - the printer's subtask_id has advanced past pre_subtask_id and we're
  5373. past the minimum cooldown
  5374. Otherwise the printer is held — caller should treat it as busy.
  5375. """
  5376. entry = self._dispatch_holds.get(printer_id)
  5377. if not entry:
  5378. return False
  5379. started_at, pre_state, pre_subtask_id = entry
  5380. elapsed = time.monotonic() - started_at
  5381. if elapsed >= self._dispatch_max_hold:
  5382. self._dispatch_holds.pop(printer_id, None)
  5383. return False
  5384. # Without a pre_state we can't detect a transition — fall back to the
  5385. # min cooldown alone, then drop the hold.
  5386. if not pre_state:
  5387. if elapsed >= self._dispatch_min_cooldown:
  5388. self._dispatch_holds.pop(printer_id, None)
  5389. return False
  5390. return True
  5391. status = printer_manager.get_status(printer_id)
  5392. current_state = getattr(status, "state", None) if status else None
  5393. current_subtask_id = getattr(status, "subtask_id", None) if status else None
  5394. transitioned = (current_state is not None and current_state != pre_state) or (
  5395. pre_subtask_id is not None and current_subtask_id is not None and current_subtask_id != pre_subtask_id
  5396. )
  5397. if transitioned and elapsed >= self._dispatch_min_cooldown:
  5398. self._dispatch_holds.pop(printer_id, None)
  5399. return False
  5400. return True
  5401. def _is_printer_idle(
  5402. self, printer_id: int, require_plate_clear: bool = True, *, ignore_plate_clear: bool = False
  5403. ) -> bool:
  5404. """Check if a printer is connected and idle.
  5405. ``require_plate_clear`` is the app setting, which a printer may override
  5406. (#2860). ``ignore_plate_clear`` leaves the gate out whatever either says,
  5407. for work that is not a print, such as drying.
  5408. """
  5409. if not printer_manager.is_connected(printer_id):
  5410. logger.debug("Printer %d: not connected", printer_id)
  5411. return False
  5412. state = printer_manager.get_status(printer_id)
  5413. if not state:
  5414. logger.debug("Printer %d: no status available", printer_id)
  5415. return False
  5416. # Plate-clear gate: if the printer finished/failed a previous print and the user
  5417. # hasn't acknowledged the plate was cleared, the queue must not dispatch the next
  5418. # job — even if the printer currently reports IDLE. After Auto Off cycles the
  5419. # printer, it boots back into IDLE with no memory of the previous finish; without
  5420. # the persisted awaiting flag we'd bypass the confirmation prompt (#961).
  5421. if not ignore_plate_clear and _plate_clear_holds(printer_id, require_plate_clear):
  5422. logger.debug(
  5423. "Printer %d: not idle — awaiting plate-clear acknowledgment (state=%s)",
  5424. printer_id,
  5425. state.state,
  5426. )
  5427. return False
  5428. idle = state.state in ("IDLE", "FINISH", "FAILED")
  5429. if not idle:
  5430. logger.debug("Printer %d: not idle — state=%s", printer_id, state.state)
  5431. return False
  5432. # A print would run from the SD card the firmware is being written to.
  5433. from backend.app.services.firmware_update import upload_in_progress
  5434. if upload_in_progress(printer_id):
  5435. logger.debug("Printer %d: not idle — firmware upload in progress", printer_id)
  5436. return False
  5437. return True
  5438. @staticmethod
  5439. def _pinned_hold_reason(printer_id: int, printer_name: str, require_plate_clear: bool) -> str:
  5440. """Why a connected, non-idle printer cannot take this job, for the queue row (#3074).
  5441. Only ever asked about a printer :meth:`_is_printer_idle` has just refused
  5442. and that the fixed-printer branch has already found connected, so the two
  5443. offline cases answer at their own exits and never arrive here.
  5444. The default is the model-based branch's ``Busy:`` wording, which
  5445. :meth:`_is_busy_only` reads as "resolves itself, stay quiet". That is also
  5446. the right answer for the connected-but-no-telemetry second or two after a
  5447. reconnect: the model-based branch has always reported it that way, and it
  5448. is not something to wake anybody up for.
  5449. A plate nobody has confirmed is the one case here that does not resolve
  5450. itself — somebody has to walk over to the printer — so it is worded as
  5451. itself and allowed to notify.
  5452. """
  5453. if _plate_clear_holds(printer_id, require_plate_clear):
  5454. return f"Waiting for plate confirmation: {printer_name}"
  5455. return f"Busy: {printer_name}"
  5456. async def _get_setting(self, db: AsyncSession, key: str) -> str | None:
  5457. """Read a setting value from the database."""
  5458. result = await db.execute(select(Settings).where(Settings.key == key))
  5459. setting = result.scalar_one_or_none()
  5460. return setting.value if setting else None
  5461. async def _get_bool_setting(self, db: AsyncSession, key: str, default: bool = False) -> bool:
  5462. """Read a boolean setting from the database."""
  5463. result = await db.execute(select(Settings).where(Settings.key == key))
  5464. setting = result.scalar_one_or_none()
  5465. if setting:
  5466. return setting.value.lower() == "true"
  5467. return default
  5468. async def _get_int_setting(self, db: AsyncSession, key: str, default: int) -> int:
  5469. """Read an int setting; falls back to default on missing/unparseable rows."""
  5470. result = await db.execute(select(Settings).where(Settings.key == key))
  5471. setting = result.scalar_one_or_none()
  5472. if setting and setting.value:
  5473. try:
  5474. return int(setting.value)
  5475. except ValueError:
  5476. pass
  5477. return default
  5478. async def _get_drying_presets(self, db: AsyncSession) -> dict[str, dict[str, int]]:
  5479. """Get drying presets (user-configured or built-in defaults)."""
  5480. result = await db.execute(select(Settings).where(Settings.key == "drying_presets"))
  5481. setting = result.scalar_one_or_none()
  5482. if setting and setting.value:
  5483. try:
  5484. presets = json.loads(setting.value)
  5485. if isinstance(presets, dict) and presets:
  5486. return presets
  5487. except json.JSONDecodeError:
  5488. pass
  5489. return self.DEFAULT_DRYING_PRESETS
  5490. async def _get_humidity_thresholds(self, db: AsyncSession) -> dict[str, int]:
  5491. """Per-filament humidity thresholds (#1605).
  5492. Returns the user-configured overrides map keyed by normalized filament
  5493. type (uppercase base, e.g. ``PLA``, ``ASA``) plus a ``default`` key for
  5494. unknown / unmapped types. Empty / unset → empty dict, in which case
  5495. callers fall back to ``ams_humidity_fair``.
  5496. """
  5497. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_thresholds"))
  5498. setting = result.scalar_one_or_none()
  5499. if not setting or not setting.value:
  5500. return {}
  5501. try:
  5502. data = json.loads(setting.value)
  5503. except json.JSONDecodeError:
  5504. return {}
  5505. if not isinstance(data, dict):
  5506. return {}
  5507. out: dict[str, int] = {}
  5508. for key, value in data.items():
  5509. try:
  5510. out[str(key).upper() if key != "default" else "default"] = int(value)
  5511. except (TypeError, ValueError):
  5512. continue
  5513. return out
  5514. # Materials whose AMS spelling differs from the key the tables above use.
  5515. # Bambu labels nylon "PA" while its own composites spell the family out, so
  5516. # PA6, PA11, PA12 and PAHT would otherwise miss a table with a perfectly
  5517. # good PA row (#3067).
  5518. #
  5519. # Mirrors DRYING_MATERIAL_ALIASES in frontend/src/utils/dryingPresets.ts.
  5520. # The drying popover has resolved these correctly since #2774 and the
  5521. # scheduler never did, which is exactly why #3067's reporter could dry a
  5522. # PA6-CF spool by hand while auto-drying skipped it every pass.
  5523. #
  5524. # PPA is here too. Polyphthalamide is a distinct polymer rather than a grade
  5525. # of nylon, so it is the one entry that is a judgement rather than a
  5526. # spelling -- but it is an aromatic polyamide, it absorbs moisture the same
  5527. # way, and PA's row is the hottest the table has. Drying it there is closer
  5528. # to right than not drying it at all, which is what it got before.
  5529. FILAMENT_KEY_ALIASES: dict[str, str] = {
  5530. "NYLON": "PA",
  5531. "PA6": "PA",
  5532. "PA11": "PA",
  5533. "PA12": "PA",
  5534. "PAHT": "PA",
  5535. "PPA": "PA",
  5536. }
  5537. @classmethod
  5538. def _resolve_filament_key(cls, tray_type: str | None, table: Mapping[str, object]) -> str | None:
  5539. """The key in *table* that answers for this tray's material, or None.
  5540. The printer reports the material in ``tray_type``, and it spells filled
  5541. and foamed variants out: PLA-CF, PETG-CF, ABS-GF, PLA-AERO, PA6-CF. The
  5542. tables here are keyed by base material, so matching the raw string alone
  5543. found a row for 8 of the 41 types a printer can report and skipped the
  5544. rest -- silently, because every caller reads "no row" as "nothing to do
  5545. for this tray". Auto-drying therefore ignored every composite spool on
  5546. the install (#3067).
  5547. Exact match first, so a table the user has extended with a row of its
  5548. own -- ``PA6-CF`` at a temperature they picked -- still wins over the
  5549. base material's. Then the suffix is dropped, then the alias map above
  5550. answers for the polyamide spellings.
  5551. Returns None rather than a default: what to do with an unrecognised
  5552. material differs per caller, and only the caller knows whether "no row"
  5553. means skip the tray or fall back to a catch-all. Nothing here invents a
  5554. temperature for a material the table does not list.
  5555. """
  5556. raw = cls._normalize_filament_type(tray_type or "")
  5557. if not raw:
  5558. return None
  5559. for candidate in (raw, raw.split("-")[0]):
  5560. key = cls.FILAMENT_KEY_ALIASES.get(candidate, candidate)
  5561. if key in table:
  5562. return key
  5563. return None
  5564. @staticmethod
  5565. def resolve_humidity_threshold(trays: list[dict], thresholds: dict[str, int], fallback: int) -> int:
  5566. """Resolve the effective humidity threshold for an AMS unit (#1605).
  5567. For mixed filament types loaded into one AMS, returns the most
  5568. restrictive (lowest) threshold across all loaded tray types — matches
  5569. the conservative-params strategy already used for drying temp/hours.
  5570. Empty / unloaded trays contribute no constraint. Unknown types use the
  5571. ``default`` key, falling through to ``fallback`` (= ``ams_humidity_fair``)
  5572. when no per-type map is configured at all.
  5573. """
  5574. default = thresholds.get("default", fallback)
  5575. if not thresholds:
  5576. return fallback
  5577. candidates: list[int] = []
  5578. for tray in trays:
  5579. tray_type = str(tray.get("tray_type") or "").strip()
  5580. if not tray_type:
  5581. continue
  5582. # A composite carries its base material's threshold when it has no
  5583. # row of its own, the same way it takes its drying preset (#3067).
  5584. key = PrintScheduler._resolve_filament_key(tray_type, thresholds)
  5585. candidates.append(thresholds[key] if key is not None else default)
  5586. if not candidates:
  5587. return default
  5588. return min(candidates)
  5589. def _get_conservative_drying_params(
  5590. self, trays: list[dict], module_type: str, presets: dict[str, dict[str, int]]
  5591. ) -> tuple[int, int, str] | None:
  5592. """Get the most conservative drying params for mixed filament types in an AMS unit.
  5593. Returns (temp, duration_hours, filament_type) or None if no drying-eligible filaments.
  5594. """
  5595. temp_key = module_type if module_type in ("n3f", "n3s") else "n3f"
  5596. hours_key = f"{temp_key}_hours"
  5597. min_temp = None
  5598. max_hours = None
  5599. filament_type = ""
  5600. for tray in trays:
  5601. tray_type = tray.get("tray_type", "")
  5602. if not tray_type:
  5603. continue
  5604. # "PLA Basic" -> PLA, and "PA6-CF" -> PA rather than nothing at all,
  5605. # which is what stopped auto-drying on every composite spool (#3067).
  5606. base_type = self._resolve_filament_key(tray_type, presets)
  5607. if base_type is None:
  5608. continue
  5609. # The table is user-editable JSON with no per-row validation, so a
  5610. # row can be present and empty. That has always meant "skip this
  5611. # material", and it has to keep meaning it: the reads below fall
  5612. # back to 55C/12h per missing field, which is a temperature nobody
  5613. # chose and would deform a PLA spool.
  5614. preset = presets[base_type]
  5615. if not preset:
  5616. continue
  5617. temp = preset.get(temp_key, 55)
  5618. hours = preset.get(hours_key, 12)
  5619. # Conservative: lowest temp, longest duration
  5620. if min_temp is None or temp < min_temp:
  5621. min_temp = temp
  5622. if max_hours is None or hours > max_hours:
  5623. max_hours = hours
  5624. if not filament_type:
  5625. filament_type = base_type
  5626. if min_temp is None:
  5627. return None
  5628. return (min_temp, max_hours or 12, filament_type)
  5629. async def _check_filament_low(self, db: AsyncSession) -> None:
  5630. """Alert on AMS slots whose assigned spool has crossed its low-stock threshold (#2913).
  5631. ``on_filament_low`` has had a column, a schema field, a route, a template
  5632. and a UI toggle since the notification system was built, and no caller --
  5633. so the toggle could be switched on and could never fire. This is the
  5634. producer.
  5635. No new setting. ``low_stock_threshold`` (default 20%) with the per-spool
  5636. ``low_stock_threshold_pct`` override is already exactly this decision:
  5637. already configurable, already surfaced, already driving the Inventory
  5638. page's Low Stock count. The AMS ``remain`` percentage is deliberately not
  5639. consulted -- it has been measured up to 56 points out against a scale,
  5640. which is not a number to page someone on.
  5641. Only slots with an assigned spool produce an event. A slot Bambuddy
  5642. cannot resolve to a spool has no remaining weight it can stand behind,
  5643. and guessing one is how the remain percentage would have got in.
  5644. """
  5645. from backend.app.models.spool import Spool
  5646. # Time-gated rather than run on every pass. run() sleeps
  5647. # _fast_check_interval -- 3 seconds -- on any productive pass, and the
  5648. # early-return path counts as productive while an upload is in flight
  5649. # (#2602), so an unthrottled check would re-read the whole spool
  5650. # collection every 3 seconds for the length of a batch drain. In
  5651. # Spoolman mode that is a request storm against a third-party service,
  5652. # which is the thing this producer's design note argues against; and
  5653. # when Spoolman is down, _get_with_retry burns ~16 s inside
  5654. # check_queue holding the scheduler's session, so the outage would
  5655. # throttle the print queue itself. A low spool is not a 3-second
  5656. # concern -- the idle interval is the natural resolution.
  5657. now = time.monotonic()
  5658. if now < self._filament_low_next_check:
  5659. return
  5660. self._filament_low_next_check = now + _FILAMENT_LOW_MIN_INTERVAL
  5661. try:
  5662. # on_filament_low defaults to off on every provider, so on most
  5663. # installs nobody wants this alert. Without this the check would
  5664. # still read every assigned spool each interval -- the whole
  5665. # collection over HTTP in Spoolman mode -- for an event that is
  5666. # switched off. One query against the provider table settles it
  5667. # before any spool work, the same guard the bed-cooled waiter uses.
  5668. # No printer_id: this asks whether any provider wants the event at
  5669. # all; per-printer scoping is applied when the event is sent.
  5670. #
  5671. # While nobody wants it, no pass sees a slot go back above its
  5672. # threshold, so nothing can re-arm; a spool refilled under the same
  5673. # id in that time would stay silenced until a restart. Forgetting
  5674. # what was sent is right anyway: whoever switches the event back on
  5675. # is told about the spools that are low now.
  5676. #
  5677. # The MQTT relay publishes the same event, so a connected relay is
  5678. # reason enough to run the check. Its record is forgotten when the
  5679. # relay is switched off, not when the broker drops: a reconnect must
  5680. # not re-send every low slot.
  5681. notify = bool(await notification_service._get_providers_for_event(db, "on_filament_low", fleet_wide=True))
  5682. relay = mqtt_relay.enabled and mqtt_relay.connected
  5683. if not notify:
  5684. self._notified_filament_low.clear()
  5685. if not mqtt_relay.enabled:
  5686. self._relayed_filament_low.clear()
  5687. if not notify and not relay:
  5688. return
  5689. global_threshold = await self._get_low_stock_threshold(db)
  5690. spoolman_on = await self._get_bool_setting(db, "spoolman_enabled")
  5691. # (printer_id, ams_id, tray_id, spool_id) -> (remaining_pct, threshold, colour name)
  5692. slots: dict[tuple[int, int, int, int], tuple[float, float, str | None]] = {}
  5693. if spoolman_on:
  5694. slots = await self._filament_low_slots_spoolman(db, global_threshold)
  5695. else:
  5696. rows = (
  5697. await db.execute(
  5698. select(SpoolAssignment, Spool)
  5699. .join(Spool, SpoolAssignment.spool_id == Spool.id)
  5700. # An archived spool is not stock. The Inventory page's
  5701. # Low Stock count skips them (InventoryPage.tsx:1089)
  5702. # and get_all_spools without allow_archived already
  5703. # excludes them in Spoolman mode, so without this the
  5704. # internal path is the only one that alerts on them.
  5705. .where(Spool.archived_at.is_(None))
  5706. )
  5707. ).all()
  5708. for assignment, spool in rows:
  5709. pct = _remaining_percent(spool.label_weight, spool.weight_used)
  5710. if pct is None:
  5711. continue
  5712. threshold = float(spool.low_stock_threshold_pct or global_threshold)
  5713. slots[(assignment.printer_id, assignment.ams_id, assignment.tray_id, spool.id)] = (
  5714. pct,
  5715. threshold,
  5716. spool.color_name,
  5717. )
  5718. if not slots:
  5719. # Nothing resolvable this pass. Deliberately not clearing the
  5720. # notified set: a Spoolman that is briefly unreachable would
  5721. # otherwise re-alert on every spool as soon as it came back.
  5722. return
  5723. await self._emit_filament_low(db, slots, notify=notify, relay=relay)
  5724. except Exception as e:
  5725. logger.warning("Low-filament check failed: %s", e, exc_info=True)
  5726. async def _get_low_stock_threshold(self, db: AsyncSession) -> float:
  5727. """The global low-stock percentage, defaulting to the schema's 20.0."""
  5728. raw = (await db.execute(select(Settings).where(Settings.key == "low_stock_threshold"))).scalar_one_or_none()
  5729. if raw is None or raw.value is None:
  5730. return 20.0
  5731. try:
  5732. value = float(raw.value)
  5733. except (TypeError, ValueError):
  5734. return 20.0
  5735. return value if 0 < value <= 100 else 20.0
  5736. async def _filament_low_slots_spoolman(
  5737. self, db: AsyncSession, global_threshold: float
  5738. ) -> dict[tuple[int, int, int, int], tuple[float, float, str | None]]:
  5739. """Resolve Spoolman-mode slots to (remaining %, threshold, colour name).
  5740. Spoolman spools carry no per-spool override -- ``low_stock_threshold_pct``
  5741. is a column on Bambuddy's own spool table and has no Spoolman equivalent,
  5742. so the global threshold is the only one that applies here. That matches
  5743. what the Inventory page already does in this mode.
  5744. One ``get_all_spools`` call covers every slot rather than a request per
  5745. spool. Archived spools do not appear -- ``get_all_spools`` excludes them
  5746. without ``allow_archived`` -- which is the behaviour the internal path
  5747. has to ask for explicitly.
  5748. An unreachable Spoolman returns no slots rather than raising. It is an
  5749. ordinary state for a third-party service, not an error in this pass: the
  5750. caller's "nothing resolvable" branch already leaves the notified set
  5751. alone, which is exactly right here, whereas letting it reach the broad
  5752. ``except`` would write a full traceback every time the check runs.
  5753. """
  5754. from backend.app.api.routes._spoolman_helpers import _map_spoolman_spool
  5755. from backend.app.services.spoolman import SpoolmanUnavailableError, get_spoolman_client
  5756. assignments = (await db.execute(select(SpoolmanSlotAssignment))).scalars().all()
  5757. if not assignments:
  5758. return {}
  5759. client = await get_spoolman_client()
  5760. if client is None:
  5761. return {}
  5762. try:
  5763. all_spools = await client.get_all_spools()
  5764. except SpoolmanUnavailableError as e:
  5765. logger.debug("Low-filament check skipped, Spoolman unreachable: %s", e)
  5766. return {}
  5767. by_id: dict[int, dict] = {}
  5768. for raw in all_spools:
  5769. raw_id = raw.get("id")
  5770. if isinstance(raw_id, int):
  5771. by_id[raw_id] = raw
  5772. slots: dict[tuple[int, int, int, int], tuple[float, float, str | None]] = {}
  5773. for assignment in assignments:
  5774. raw = by_id.get(assignment.spoolman_spool_id)
  5775. if raw is None:
  5776. continue
  5777. try:
  5778. mapped = _map_spoolman_spool(raw)
  5779. except ValueError:
  5780. continue
  5781. pct = _remaining_percent(mapped.get("label_weight"), mapped.get("weight_used"))
  5782. if pct is None:
  5783. continue
  5784. key = (assignment.printer_id, assignment.ams_id, assignment.tray_id, assignment.spoolman_spool_id)
  5785. slots[key] = (pct, global_threshold, mapped.get("color_name"))
  5786. return slots
  5787. async def _emit_filament_low(
  5788. self,
  5789. db: AsyncSession,
  5790. slots: dict[tuple[int, int, int, int], tuple[float, float, str | None]],
  5791. *,
  5792. notify: bool = True,
  5793. relay: bool = False,
  5794. ) -> None:
  5795. """Send one notification and/or MQTT relay event per slot that has newly crossed its threshold."""
  5796. printers: dict[int, Printer] = {}
  5797. for key, (pct, threshold, color) in slots.items():
  5798. printer_id, ams_id, tray_id, spool_id = key
  5799. if pct >= threshold:
  5800. # Back above the line: re-arm rather than expire on a timer, so a
  5801. # refilled slot can alert again and a spool sitting just under the
  5802. # threshold stays quiet.
  5803. self._notified_filament_low.discard(key)
  5804. self._relayed_filament_low.discard(key)
  5805. continue
  5806. send_notification = notify and key not in self._notified_filament_low
  5807. # Only while connected, so a crossing the relay missed reaches it
  5808. # after the reconnect.
  5809. send_relay = relay and key not in self._relayed_filament_low
  5810. if not send_notification and not send_relay:
  5811. continue
  5812. # Marked before sending, which is a choice rather than the only
  5813. # option, and it is the opposite failure mode from the one the
  5814. # debounce argues against: one transient provider failure loses this
  5815. # alert until the spool goes back above the threshold and crosses it
  5816. # again. Marking after a successful send would trade that for
  5817. # re-alerting on every pass while a provider is down -- which is the
  5818. # repetition the whole debounce exists to prevent, and the louder of
  5819. # the two failures. A spool that is low stays low, so the next real
  5820. # signal is not far away; a provider stuck retrying is a signal that
  5821. # never stops.
  5822. if send_notification:
  5823. self._notified_filament_low.add(key)
  5824. if send_relay:
  5825. self._relayed_filament_low.add(key)
  5826. if printer_id not in printers:
  5827. printer = (await db.execute(select(Printer).where(Printer.id == printer_id))).scalar_one_or_none()
  5828. if printer is None:
  5829. continue
  5830. printers[printer_id] = printer
  5831. printer = printers[printer_id]
  5832. slot = _ams_slot_label(ams_id, tray_id, printer_tray_layout(printer_id, printer_manager))
  5833. if send_notification:
  5834. try:
  5835. await notification_service.on_filament_low(
  5836. printer_id,
  5837. printer.name,
  5838. slot,
  5839. int(pct),
  5840. db,
  5841. color=color,
  5842. )
  5843. except Exception as e:
  5844. logger.warning("Low-filament notification failed for slot %s: %s", key, e)
  5845. if send_relay:
  5846. try:
  5847. await mqtt_relay.on_filament_low(
  5848. printer_id, printer.name, printer.serial_number, slot, int(pct), color, spool_id
  5849. )
  5850. except Exception as e:
  5851. logger.warning("Low-filament MQTT relay event failed for slot %s: %s", key, e)
  5852. async def _check_stock_forecast(self, db: AsyncSession) -> None:
  5853. """Alert when a filament SKU reaches its reorder point or is about to break (#2955).
  5854. ``on_stock_reorder_alert`` and ``on_stock_break_alert`` have had a column,
  5855. a schema field, a template and a UI toggle, and nothing that computes the
  5856. condition -- the Forecast panel does it in the browser, so with no page
  5857. open nothing could ever alert. This runs the same arithmetic
  5858. (``stock_forecast``) from the scheduler loop.
  5859. An alert is sent when a SKU *moves into* a condition, once, and again
  5860. after the condition has cleared. A SKU that worsens from reorder to break
  5861. alerts again as a break, because the panel treats the two as exclusive; one
  5862. that eases from break back to reorder does not alert again.
  5863. A SKU with alerts snoozed is treated as not alerting, so un-snoozing it
  5864. while it is still low tells you.
  5865. Time-gated to hourly for the reasons ``_check_filament_low`` is gated: the
  5866. loop runs every 3 s while an upload is in flight, and the whole spool
  5867. collection is read over HTTP in Spoolman mode. Both events default to off
  5868. on every provider, so nothing is read unless a provider wants one.
  5869. """
  5870. now = time.monotonic()
  5871. if now < self._stock_forecast_next_check:
  5872. return
  5873. self._stock_forecast_next_check = now + _STOCK_FORECAST_MIN_INTERVAL
  5874. try:
  5875. if self._stock_alerts_persisted is None:
  5876. await self._load_stock_alerts(db)
  5877. wanted = {
  5878. "reorder": bool(
  5879. await notification_service._get_providers_for_event(db, "on_stock_reorder_alert", fleet_wide=True)
  5880. ),
  5881. "break": bool(
  5882. await notification_service._get_providers_for_event(db, "on_stock_break_alert", fleet_wide=True)
  5883. ),
  5884. }
  5885. if not any(wanted.values()):
  5886. # Nobody wants either event. Forget what was sent, so whoever
  5887. # switches one on is told about the SKUs that are low now.
  5888. self._notified_stock_alerts.clear()
  5889. await self._save_stock_alerts(db)
  5890. return
  5891. forecasts = await self._stock_forecasts(db)
  5892. if forecasts is None:
  5893. # Spoolman unreachable. Deliberately not clearing the notified
  5894. # set: a brief outage would otherwise re-alert every SKU when it
  5895. # came back.
  5896. return
  5897. await self._emit_stock_alerts(db, forecasts, wanted)
  5898. await self._save_stock_alerts(db)
  5899. except Exception as e:
  5900. logger.warning("Stock forecast check failed: %s", e, exc_info=True)
  5901. def _serialize_stock_alerts(self) -> str:
  5902. """The notified set as JSON, in a fixed order so an unchanged set compares equal."""
  5903. rows = sorted([list(key), kind, sorted(events)] for key, (kind, events) in self._notified_stock_alerts.items())
  5904. return json.dumps(rows)
  5905. async def _load_stock_alerts(self, db: AsyncSession) -> None:
  5906. """Read the notified set saved by the last run, once per process.
  5907. A missing or unreadable row is an empty set: the worst that does is one
  5908. repeat of what a restart used to cause every time.
  5909. """
  5910. raw = (await db.execute(select(Settings).where(Settings.key == _STOCK_ALERTS_SETTING_KEY))).scalar_one_or_none()
  5911. loaded: dict[stock_forecast.SkuKey, tuple[str, frozenset[str]]] = {}
  5912. if raw and raw.value:
  5913. try:
  5914. for key, kind, events in json.loads(raw.value):
  5915. if len(key) == 4 and kind in ("reorder", "break"):
  5916. loaded[tuple(str(part) for part in key)] = (kind, frozenset(str(e) for e in events))
  5917. except (ValueError, TypeError):
  5918. logger.warning("Ignoring unreadable %s setting", _STOCK_ALERTS_SETTING_KEY)
  5919. loaded = {}
  5920. self._notified_stock_alerts = loaded
  5921. self._stock_alerts_persisted = self._serialize_stock_alerts()
  5922. async def _save_stock_alerts(self, db: AsyncSession) -> None:
  5923. """Write the notified set to its settings row if it changed since it was last read or written."""
  5924. serialized = self._serialize_stock_alerts()
  5925. if serialized == self._stock_alerts_persisted:
  5926. return
  5927. from backend.app.core.db_dialect import upsert_setting
  5928. await upsert_setting(db, Settings, _STOCK_ALERTS_SETTING_KEY, serialized)
  5929. await db.commit()
  5930. self._stock_alerts_persisted = serialized
  5931. async def _stock_forecasts(self, db: AsyncSession) -> stock_forecast.ForecastMap | None:
  5932. """Forecast every active SKU in whichever inventory mode is on. None if it cannot be read."""
  5933. from backend.app.models.filament_sku_settings import FilamentSkuSettings
  5934. global_lead_time = max(0, await self._get_int_setting(db, "forecast_global_lead_time_days", default=0))
  5935. sku_rows = (await db.execute(select(FilamentSkuSettings))).scalars().all()
  5936. sku_settings = {
  5937. stock_forecast.sku_key(row.material, row.subtype, row.brand, row.color_name): stock_forecast.SkuSettings(
  5938. lead_time_days=row.lead_time_days,
  5939. safety_margin_value=row.safety_margin_value,
  5940. safety_margin_unit=row.safety_margin_unit,
  5941. alerts_snoozed=bool(row.alerts_snoozed),
  5942. )
  5943. for row in sku_rows
  5944. }
  5945. if await self._get_bool_setting(db, "spoolman_enabled"):
  5946. spools = await self._stock_spools_spoolman()
  5947. if spools is None:
  5948. return None
  5949. # The Spoolman spools' own rows (#3407), keyed by Spoolman id. Never
  5950. # the built-in ones: a Spoolman id that matches a local spool id
  5951. # (left over from before Spoolman was switched on) would borrow an
  5952. # unrelated spool's history (#3406).
  5953. history = await self._stock_usage_history(db, spoolman=True)
  5954. history = {
  5955. spool.id: stock_forecast.own_records(spool, history[spool.id])
  5956. for spool in spools
  5957. if spool.id in history
  5958. }
  5959. else:
  5960. spools = await self._stock_spools_internal(db)
  5961. history = await self._stock_usage_history(db)
  5962. return stock_forecast.forecast_all(spools, history, sku_settings, global_lead_time, utcnow_naive())
  5963. async def _stock_spools_internal(self, db: AsyncSession) -> list[stock_forecast.StockSpool]:
  5964. from backend.app.models.spool import Spool
  5965. # Every spool: stock, the recently archived ones that count towards the
  5966. # rate, and the older ones that tell it the SKU was in use before its
  5967. # window (#3407). stock_forecast.forecast_all sorts them.
  5968. rows = (await db.execute(select(Spool))).scalars().all()
  5969. return [
  5970. stock_forecast.StockSpool(
  5971. id=spool.id,
  5972. material=spool.material,
  5973. subtype=spool.subtype,
  5974. brand=spool.brand,
  5975. color_name=spool.color_name,
  5976. label_weight=float(spool.label_weight or 0),
  5977. weight_used=float(spool.weight_used or 0),
  5978. weight_used_baseline=float(spool.weight_used_baseline or 0),
  5979. created_at=spool.created_at,
  5980. archived_at=spool.archived_at,
  5981. last_used_at=spool.last_used,
  5982. )
  5983. for spool in rows
  5984. ]
  5985. async def _stock_usage_history(
  5986. self, db: AsyncSession, *, spoolman: bool = False
  5987. ) -> dict[int, list[stock_forecast.UsageRecord]]:
  5988. """Usage rows by spool id, for one inventory: its own ids, as the forecast
  5989. panel reads them from the usage endpoint."""
  5990. from backend.app.models.spool_usage_history import SpoolUsageHistory
  5991. spool_id = SpoolUsageHistory.spoolman_spool_id if spoolman else SpoolUsageHistory.spool_id
  5992. rows = (
  5993. await db.execute(
  5994. select(spool_id, SpoolUsageHistory.created_at, SpoolUsageHistory.weight_used)
  5995. .where(
  5996. SpoolUsageHistory.spoolman_spool_id.is_not(None)
  5997. if spoolman
  5998. else SpoolUsageHistory.spoolman_spool_id.is_(None)
  5999. )
  6000. .order_by(SpoolUsageHistory.created_at.desc())
  6001. .limit(_STOCK_FORECAST_HISTORY_LIMIT)
  6002. )
  6003. ).all()
  6004. history: dict[int, list[stock_forecast.UsageRecord]] = {}
  6005. for spool_id, created_at, weight_used in rows:
  6006. # A deleted spool's usage forecasts no stock. Skipped here rather than
  6007. # filtered before the limit, so the window matches the forecast panel's.
  6008. if created_at is None or spool_id is None:
  6009. continue
  6010. history.setdefault(spool_id, []).append(stock_forecast.UsageRecord(created_at, float(weight_used or 0)))
  6011. return history
  6012. async def _stock_spools_spoolman(self) -> list[stock_forecast.StockSpool] | None:
  6013. """Spoolman's spools in the forecast's shape, or None when Spoolman cannot be reached.
  6014. Archived ones too: a recently used-up spool counts towards its SKU's
  6015. rate (#3399). ``_map_spoolman_spool`` is the same mapping the Inventory
  6016. page (and so the panel) reads; it dates archiving by the spool's last
  6017. use, as Spoolman records no archive date.
  6018. """
  6019. from backend.app.api.routes._spoolman_helpers import _map_spoolman_spool
  6020. from backend.app.services.spoolman import SpoolmanUnavailableError, get_spoolman_client
  6021. client = await get_spoolman_client()
  6022. if client is None:
  6023. logger.debug("Stock forecast skipped, no Spoolman client (spoolman_enabled without a URL?)")
  6024. return None
  6025. try:
  6026. raw_spools = await client.get_all_spools(allow_archived=True)
  6027. except SpoolmanUnavailableError as e:
  6028. logger.debug("Stock forecast skipped, Spoolman unreachable: %s", e)
  6029. return None
  6030. spools: list[stock_forecast.StockSpool] = []
  6031. for raw in raw_spools:
  6032. try:
  6033. mapped = _map_spoolman_spool(raw)
  6034. except ValueError:
  6035. continue
  6036. created_at = _spoolman_time(mapped.get("created_at"))
  6037. archived_at = None
  6038. if raw.get("archived"):
  6039. # Archived without a date to tell when: not stock, and too old to
  6040. # count towards the rate.
  6041. archived_at = _spoolman_time(mapped.get("archived_at")) or datetime(1970, 1, 1, tzinfo=timezone.utc)
  6042. spools.append(
  6043. stock_forecast.StockSpool(
  6044. id=mapped["id"],
  6045. material=mapped["material"],
  6046. subtype=mapped.get("subtype"),
  6047. brand=mapped.get("brand"),
  6048. color_name=mapped.get("color_name"),
  6049. label_weight=float(mapped.get("label_weight") or 0),
  6050. weight_used=float(mapped.get("weight_used") or 0),
  6051. weight_used_baseline=float(mapped.get("weight_used_baseline") or 0),
  6052. created_at=created_at,
  6053. archived_at=archived_at,
  6054. first_used_at=_spoolman_time(mapped.get("first_used")),
  6055. last_used_at=_spoolman_time(mapped.get("last_used")),
  6056. color_name_is_synthesized=bool(mapped.get("color_name_is_synthesized")),
  6057. )
  6058. )
  6059. return spools
  6060. async def _emit_stock_alerts(
  6061. self,
  6062. db: AsyncSession,
  6063. forecasts: stock_forecast.ForecastMap,
  6064. wanted: dict[str, bool],
  6065. ) -> None:
  6066. """Send the notifications for each SKU that has moved into a stock alert condition.
  6067. A SKU in break has also reached its reorder point, so both events apply to
  6068. it: the break event goes to providers that have it on, and the reorder event
  6069. to the providers that have only that one on. A provider with both on gets the
  6070. break and no second message.
  6071. What is remembered per SKU is the condition and which events had a
  6072. subscriber when it was sent. An event with no subscriber is forgotten, so
  6073. switching it on later reports the SKUs already in that condition.
  6074. """
  6075. active_events = {event for event, on in wanted.items() if on}
  6076. for key, (forecast, spool) in forecasts.items():
  6077. kind: str | None = None
  6078. if not forecast.snoozed:
  6079. if forecast.stock_break_alert:
  6080. kind = "break"
  6081. elif forecast.reorder_alert:
  6082. kind = "reorder"
  6083. applicable = {"break": {"break", "reorder"}, "reorder": {"reorder"}}.get(kind or "", set())
  6084. events = applicable & active_events
  6085. if not events:
  6086. self._notified_stock_alerts.pop(key, None)
  6087. continue
  6088. previous = self._notified_stock_alerts.get(key)
  6089. told = (previous[1] & active_events) if previous else frozenset()
  6090. # Trimmed to the events this condition can send. That is what makes easing from
  6091. # break back to reorder silent (the reorder event was told with the break) and
  6092. # worsening again news again (the break event is no longer remembered).
  6093. self._notified_stock_alerts[key] = (kind, frozenset((told | events) & applicable))
  6094. # Recorded before sending, for the reason _emit_filament_low gives: a
  6095. # provider that is down should lose this alert, not repeat it every hour.
  6096. to_send = events - told
  6097. # Both flags imply a positive rate and so a day count.
  6098. days_left = forecast.days_remaining if forecast.days_remaining is not None else 0
  6099. rate = forecast.daily_rate_g if forecast.daily_rate_g is not None else 0.0
  6100. color = None if spool.color_name_is_synthesized else spool.color_name
  6101. try:
  6102. if "break" in to_send:
  6103. await notification_service.on_stock_break_alert(
  6104. spool.material,
  6105. spool.brand,
  6106. forecast.remaining_g,
  6107. rate,
  6108. days_left,
  6109. forecast.effective_lead_time_days,
  6110. db,
  6111. subtype=spool.subtype,
  6112. color=color,
  6113. )
  6114. if "reorder" in to_send:
  6115. await notification_service.on_stock_reorder_alert(
  6116. spool.material,
  6117. spool.brand,
  6118. forecast.remaining_g,
  6119. rate,
  6120. days_left,
  6121. db,
  6122. subtype=spool.subtype,
  6123. color=color,
  6124. skip_break_subscribers=kind == "break",
  6125. )
  6126. except Exception as e:
  6127. logger.warning("Stock %s notification failed for %s: %s", kind, key, e)
  6128. # A SKU with no spools left is no longer alerting.
  6129. for key in [k for k in self._notified_stock_alerts if k not in forecasts]:
  6130. del self._notified_stock_alerts[key]
  6131. @staticmethod
  6132. async def _creator_printer_scope(db: AsyncSession, user_id: int | None) -> PrinterScope:
  6133. """The printers a job's creator may use (#1727), as the queue pass reads them."""
  6134. from backend.app.core.auth import is_auth_enabled
  6135. if user_id is None or not await is_auth_enabled(db):
  6136. return ALL_PRINTERS
  6137. return await resolve_user_id_printer_scope(db, user_id)
  6138. async def _model_job_printer_ids(
  6139. self,
  6140. db: AsyncSession,
  6141. item: PrintQueueItem,
  6142. scope: PrinterScope,
  6143. cache: dict[tuple[str, str | None, int | None], set[int]],
  6144. ) -> set[int]:
  6145. """Every printer an "Any <model>" job could land on: the active printers
  6146. of each of its candidate models, in its location, that its creator may use.
  6147. ``cache`` holds one lookup per model, location and creator for the pass,
  6148. so a long queue of model jobs does not query once per job.
  6149. A printer whose last known nozzle cannot print the file is left out
  6150. (#3536): the job stays pending there, so it must not hold drying off."""
  6151. printer_ids: set[int] = set()
  6152. for candidate in _candidates_for(item):
  6153. if not candidate.target_model:
  6154. continue
  6155. key = (candidate.target_model, item.target_location, item.created_by_id)
  6156. if key not in cache:
  6157. printers = await self._printers_for_model(db, candidate.target_model, item.target_location, scope)
  6158. cache[key] = {printer.id for printer in printers}
  6159. # #3536: a job with no fitting nozzle now stays pending instead of
  6160. # failing at once, so it must not count for a printer whose last
  6161. # known nozzle cannot print it (it would block or trigger drying
  6162. # there for good). Unknown nozzle fits (fail-safe). The cache stays
  6163. # per model; the filter runs per candidate, not per cache entry.
  6164. printer_ids |= {
  6165. pid
  6166. for pid in cache[key]
  6167. if _wrong_nozzle_label("", printer_manager.get_status(pid), candidate.sliced_nozzle) is None
  6168. }
  6169. return printer_ids
  6170. async def _check_auto_drying(
  6171. self,
  6172. db: AsyncSession,
  6173. queue_items: list[PrintQueueItem],
  6174. dispatching_printers: set[int],
  6175. ):
  6176. """Start drying on idle printers based on humidity.
  6177. Three modes (can all be enabled independently):
  6178. - queue_drying_enabled: Dry between scheduled queue prints
  6179. - ambient_drying_enabled: Dry any idle printer when humidity is high, regardless of queue
  6180. - print_drying_enabled: Also evaluate printers that are currently printing,
  6181. when model+firmware supports "Print While Drying" (gated by
  6182. supports_drying_while_printing). Drying temperature is capped at
  6183. max(40, preset_temp - 5) to protect spools mid-print.
  6184. """
  6185. queue_drying_enabled = await self._get_bool_setting(db, "queue_drying_enabled")
  6186. ambient_drying_enabled = await self._get_bool_setting(db, "ambient_drying_enabled")
  6187. print_drying_enabled = await self._get_bool_setting(db, "print_drying_enabled")
  6188. sustained_minutes = await self._get_int_setting(db, "ambient_drying_sustained_minutes", default=0)
  6189. # The wait belongs to ambient drying: the settings UI only shows it while
  6190. # ambient drying is on, so a value left behind when ambient is turned off
  6191. # must not keep delaying the one path that still reaches the wait gate
  6192. # without it (a mid-print start under print_drying).
  6193. sustained_wait_active = sustained_minutes > 0 and ambient_drying_enabled
  6194. # Clear every streak as soon as the wait is inactive. An early return (or
  6195. # an already-drying unit) may otherwise skip the per-unit cleanup and let
  6196. # a quick toggle-on inherit an old streak.
  6197. if not sustained_wait_active:
  6198. self._auto_dry_above.clear()
  6199. if not queue_drying_enabled and not ambient_drying_enabled:
  6200. # Stop active drying on all printers if both features disabled
  6201. if self._drying_in_progress:
  6202. for pid in list(self._drying_in_progress):
  6203. if pid in self._scheduled_drying_printer_ids:
  6204. continue
  6205. logger.info("Auto-drying: printer %d — stopping, auto-drying disabled", pid)
  6206. await self._stop_drying(pid)
  6207. return
  6208. # Update drying state from printer status (handles backend restart)
  6209. self._sync_drying_state()
  6210. # Find printers with scheduled items (for queue drying mode)
  6211. # An "Any <model>" job counts for every printer it could land on.
  6212. printers_with_scheduled: set[int] = set()
  6213. # Printers with a job that could start now: the ones block mode keeps
  6214. # a new cycle off. A job waiting for its start time is what queue
  6215. # drying dries ahead of.
  6216. printers_with_ready_items: set[int] = set()
  6217. now_utc = datetime.now(timezone.utc)
  6218. creator_scopes: dict[int | None, PrinterScope] = {}
  6219. model_printers: dict[tuple[str, str | None, int | None], set[int]] = {}
  6220. for item in queue_items:
  6221. if item.printer_id:
  6222. item_printers = {item.printer_id}
  6223. elif item.target_model or item.variants:
  6224. if item.created_by_id not in creator_scopes:
  6225. creator_scopes[item.created_by_id] = await self._creator_printer_scope(db, item.created_by_id)
  6226. item_printers = await self._model_job_printer_ids(
  6227. db, item, creator_scopes[item.created_by_id], model_printers
  6228. )
  6229. else:
  6230. continue
  6231. if item.scheduled_time and not item.manual_start:
  6232. printers_with_scheduled |= item_printers
  6233. sched = item.scheduled_time
  6234. if sched is not None and sched.tzinfo is None:
  6235. sched = sched.replace(tzinfo=timezone.utc)
  6236. if sched is None or sched <= now_utc:
  6237. printers_with_ready_items |= item_printers
  6238. # If only queue mode is on and no printers have scheduled items, stop drying
  6239. # (but skip this short-circuit when print_drying_enabled is on — busy printers
  6240. # may still be eligible for mid-print drying regardless of queue state).
  6241. if not ambient_drying_enabled and not printers_with_scheduled and not print_drying_enabled:
  6242. for pid in list(self._drying_in_progress):
  6243. if pid in self._scheduled_drying_printer_ids:
  6244. continue
  6245. logger.info("Auto-drying: printer %d — stopping, no scheduled prints in queue", pid)
  6246. await self._stop_drying(pid)
  6247. return
  6248. # Get humidity threshold (global fallback)
  6249. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  6250. setting = result.scalar_one_or_none()
  6251. global_humidity_threshold = int(setting.value) if setting else 60
  6252. # Per-filament humidity threshold overrides (#1605). Empty → fall back
  6253. # to the global threshold for every AMS unit.
  6254. per_type_thresholds = await self._get_humidity_thresholds(db)
  6255. # Get drying presets
  6256. presets = await self._get_drying_presets(db)
  6257. # Determine if drying should be skipped for printers with pending items
  6258. block_for_drying = await self._get_bool_setting(db, "queue_drying_block")
  6259. # Get all active printers
  6260. all_printers = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  6261. for printer in all_printers.scalars():
  6262. pid = printer.id
  6263. # Resolve model+firmware up front — needed to decide whether this printer
  6264. # qualifies for mid-print drying (busy printer on capable hardware).
  6265. state = printer_manager.get_status(pid)
  6266. if not state:
  6267. logger.debug("Auto-drying: printer %d skipped — no state", pid)
  6268. continue
  6269. model = printer_manager.get_model(pid)
  6270. firmware = state.firmware_version
  6271. # "Mid-print" has to mean the printer is actually printing (#2801).
  6272. # It used to be inferred from the dispatch set, which also holds
  6273. # printers that merely could not be dispatched to -- so a printer
  6274. # sitting in FINISH behind an unacknowledged plate was treated as
  6275. # printing, had its drying temperature capped by the mid-print
  6276. # spool protection, and was logged as (mid-print) while idle.
  6277. is_printing = state.state in _ACTIVE_PRINT_STATES
  6278. mid_print = is_printing and print_drying_enabled and supports_drying_while_printing(model, firmware)
  6279. # A printer whose print is running or imminent is left alone unless
  6280. # it can dry through it. `dispatching_printers` is deliberately the
  6281. # narrow set: running, held post-dispatch, or mid-upload.
  6282. if (is_printing or pid in dispatching_printers) and not mid_print:
  6283. logger.debug("Auto-drying: printer %d skipped — printing or about to", pid)
  6284. continue
  6285. if not mid_print:
  6286. # In queue-only mode, only dry printers that have scheduled prints
  6287. if not ambient_drying_enabled and pid not in printers_with_scheduled:
  6288. if self._drying_in_progress.get(pid) and pid not in self._scheduled_drying_printer_ids:
  6289. logger.info("Auto-drying: printer %d — stopping, no scheduled prints for this printer", pid)
  6290. await self._stop_drying(pid)
  6291. logger.debug("Auto-drying: printer %d skipped — no scheduled prints", pid)
  6292. continue
  6293. # When block mode is on, don't START new drying on printers with
  6294. # a job that could start now: the cycle would hold it. One
  6295. # scheduled for later may pre-dry; if the cycle is still running
  6296. # at its time, the hold applies then. Already-drying printers go
  6297. # through so humidity auto-stop logic still runs.
  6298. if block_for_drying and pid in printers_with_ready_items and not self._drying_in_progress.get(pid):
  6299. logger.debug("Auto-drying: printer %d skipped — has a job ready to start (block mode)", pid)
  6300. continue
  6301. if not printer_manager.is_connected(pid):
  6302. logger.debug("Auto-drying: printer %d skipped — not connected", pid)
  6303. continue
  6304. # Plate-clear is deliberately ignored here (#2801). It answers
  6305. # "is the bed ready for the next job", which says nothing about
  6306. # whether the AMS may heat -- and the gap between a finished print
  6307. # and the acknowledgment is exactly when drying is most useful,
  6308. # because the printer is free and nobody is waiting on it. Leaving
  6309. # the plate unacknowledged is also how people hold the queue by
  6310. # hand, and that hold should not cost them their drying.
  6311. if not mid_print and not self._is_printer_idle(pid, ignore_plate_clear=True):
  6312. logger.debug("Auto-drying: printer %d skipped — not idle", pid)
  6313. continue
  6314. # Check drying capability. For mid-print path, supports_drying_while_printing
  6315. # was already verified when computing mid_print above.
  6316. if not mid_print and not supports_drying(model, firmware):
  6317. logger.debug("Auto-drying: printer %d skipped — model %s does not support drying", pid, model)
  6318. continue
  6319. # Check each AMS unit from raw_data
  6320. ams_list = state.raw_data.get("ams", [])
  6321. logger.debug("Auto-drying: printer %d — checking %d AMS units", pid, len(ams_list))
  6322. for ams_data in ams_list:
  6323. module_type = str(ams_data.get("module_type") or "")
  6324. ams_id = int(ams_data.get("id", 0))
  6325. # Only n3f/n3s support drying
  6326. if module_type not in ("n3f", "n3s"):
  6327. logger.debug("Auto-drying: printer %d AMS %d skipped — module_type=%s", pid, ams_id, module_type)
  6328. continue
  6329. # Resolve per-filament humidity threshold for this AMS unit (#1605).
  6330. # Most-restrictive of all loaded tray types; falls back to the
  6331. # global threshold when no overrides are configured.
  6332. trays = ams_data.get("tray", []) or []
  6333. humidity_threshold = self.resolve_humidity_threshold(
  6334. trays, per_type_thresholds, global_humidity_threshold
  6335. )
  6336. dry_time = int(ams_data.get("dry_time") or 0)
  6337. # Read humidity as a percentage. The 1-5 index is never
  6338. # substituted: it is inverted, and being unable to exceed any
  6339. # threshold it would read as "dry" forever (#3140). ``None``
  6340. # already means "skip this unit" everywhere below.
  6341. humidity_pct = ams_humidity_percent(ams_data)
  6342. humidity = int(round(humidity_pct)) if humidity_pct is not None else None
  6343. unit_key = (pid, ams_id)
  6344. unit_state = self._auto_dry_units.get(unit_key)
  6345. # Already drying — let it run to its configured duration (#1892).
  6346. #
  6347. # We deliberately do NOT stop drying from a humidity re-check here.
  6348. # Relative humidity drops steeply in heated air, so the AMS sensor
  6349. # reads ~15-20% within minutes of the dryer starting even while the
  6350. # filament is still saturated. A humidity-based early-stop therefore
  6351. # always fires at the minimum-time floor, truncating both user-started
  6352. # manual cycles and Bambuddy's own preset-duration dries to ~30 min.
  6353. # The firmware stops when the configured duration elapses; scheduling
  6354. # stops (print takes priority, queue no longer needs drying) are
  6355. # handled separately via _stop_drying().
  6356. #
  6357. # A parked countdown is not a running cycle (#3478): the firmware
  6358. # took the command and left the heater off, so dry_time sits at
  6359. # its full duration and never reaches 0 on its own. Treating it
  6360. # as running blocked auto-drying on that unit for days. A sample
  6361. # the stall detector has not classified yet (its first 150 s)
  6362. # still counts as running.
  6363. parked = is_countdown_parked(ams_data)
  6364. if dry_time > 0 and not parked:
  6365. if pid not in self._drying_in_progress:
  6366. # Drying we didn't start (manual or from before restart) —
  6367. # track it so scheduling stops still apply; never auto-stop it.
  6368. self._drying_in_progress[pid] = time.monotonic()
  6369. if unit_state is not None:
  6370. unit_state["running"] = True
  6371. logger.debug(
  6372. "Auto-drying: printer %d AMS %d — drying (%dm left, humidity %s%%), letting it run",
  6373. pid,
  6374. ams_id,
  6375. dry_time,
  6376. humidity,
  6377. )
  6378. continue
  6379. # Nothing is drying. Close out a cycle we armed ourselves and
  6380. # judge whether it achieved anything (#2770).
  6381. #
  6382. # "Achieved anything" is measured against the LOWEST reading any
  6383. # cycle on this unit has ended at, not against the threshold and
  6384. # not against the previous cycle. A spool that is genuinely wet
  6385. # in a humid room comes down slowly — 40, 37, 35 — and must be
  6386. # allowed to keep going for as long as it is still coming down,
  6387. # however far it still is from the threshold. What must not
  6388. # continue is a cycle that ends exactly where the last one did,
  6389. # which is the reporter's signature: 15, 15, 16, 15, forever.
  6390. # Comparing against the running minimum rather than the previous
  6391. # end is what stops a sensor oscillating by one point between two
  6392. # values from reading as progress every other cycle.
  6393. if unit_state is not None and unit_state.pop("running", False):
  6394. unit_state["ended_at"] = time.monotonic()
  6395. # The streak that armed this cycle is spent; the next one
  6396. # starts fresh (and accumulates through the re-arm cooldown
  6397. # below, so the wait overlaps the cooldown, never stacks on
  6398. # top of it).
  6399. self._auto_dry_above.pop(unit_key, None)
  6400. if parked:
  6401. # The timer stood still (never started, or paused by the
  6402. # power limit or the print), so the cycle says nothing
  6403. # about humidity: count it apart from the unproductive
  6404. # cycles below (#3478).
  6405. never_heated = _never_heated(unit_state) + 1
  6406. unit_state["never_heated"] = never_heated
  6407. logger.warning(
  6408. "Auto-drying: printer %d AMS %d — drying cycle not running: the timer stands "
  6409. "still at %dm with the status Off, %d in a row",
  6410. pid,
  6411. ams_id,
  6412. dry_time,
  6413. never_heated,
  6414. )
  6415. else:
  6416. # A cycle that counted down ran; the parked streak is over.
  6417. unit_state.pop("never_heated", None)
  6418. if not parked and humidity is not None and humidity > humidity_threshold:
  6419. best = unit_state.get("best_end_humidity")
  6420. if isinstance(best, int) and humidity < best:
  6421. # Still coming down. Keep going, however far the
  6422. # threshold still is.
  6423. unproductive = 0
  6424. else:
  6425. unproductive = int(unit_state.get("unproductive", 0)) + 1
  6426. if not isinstance(best, int) or humidity < best:
  6427. unit_state["best_end_humidity"] = humidity
  6428. unit_state["unproductive"] = unproductive
  6429. logger.info(
  6430. "Auto-drying: printer %d AMS %d — cycle ended with humidity still %d%% > "
  6431. "threshold %d%% (best so far %s%%, %d unproductive in a row)",
  6432. pid,
  6433. ams_id,
  6434. humidity,
  6435. humidity_threshold,
  6436. unit_state.get("best_end_humidity"),
  6437. unproductive,
  6438. )
  6439. # A cycle that ended at or below the threshold needs no
  6440. # counter reset here: the branch below drops the whole entry.
  6441. # Humidity below threshold — no need to start drying. This is
  6442. # also the only thing that lifts a suspension: the reading we
  6443. # gave up on has come down, so auto-drying works again and the
  6444. # unit goes back to having no history at all.
  6445. if humidity is None or humidity <= humidity_threshold:
  6446. if unit_state is not None and unit_state.get("suspended"):
  6447. logger.info(
  6448. "Auto-drying: printer %d AMS %d — humidity %s%% is back at or below the %d%% "
  6449. "threshold, resuming automatic drying",
  6450. pid,
  6451. ams_id,
  6452. humidity,
  6453. humidity_threshold,
  6454. )
  6455. # Clear the judgement, keep the clock (#2801). Dropping the
  6456. # whole entry also dropped `ended_at`, and with it the
  6457. # 30-minute cooldown -- so a reading that dips to the
  6458. # threshold as the AMS cools and comes back above it once
  6459. # warm wiped its own history and re-armed immediately. That
  6460. # oscillation is the very thing #2770's cooldown exists to
  6461. # ride out, and it is worst at exactly the margin that makes
  6462. # a unit dry repeatedly: a point or two above the threshold.
  6463. if unit_state is not None:
  6464. unit_state.pop("suspended", None)
  6465. unit_state.pop("unproductive", None)
  6466. unit_state.pop("best_end_humidity", None)
  6467. unit_state.pop("never_heated", None)
  6468. # A real below-threshold reading ends any sustained-wait
  6469. # streak (#2518) -- "continuously above" means exactly that.
  6470. # An absent reading (humidity is None) is no-information and
  6471. # leaves the streak alone; the observation-gap guard handles
  6472. # a prolonged sensor silence.
  6473. if humidity is not None:
  6474. _above = self._auto_dry_above.pop(unit_key, None)
  6475. if _above is not None and sustained_wait_active:
  6476. logger.info(
  6477. "Auto-drying: printer %d AMS %d — humidity fell back to %s%% after "
  6478. "%.0fs of the required %dm above the %d%% threshold; not drying",
  6479. pid,
  6480. ams_id,
  6481. humidity,
  6482. time.monotonic() - _above["since"],
  6483. sustained_minutes,
  6484. humidity_threshold,
  6485. )
  6486. logger.debug(
  6487. "Auto-drying: printer %d AMS %d skipped — humidity %s <= threshold %d",
  6488. pid,
  6489. ams_id,
  6490. humidity,
  6491. humidity_threshold,
  6492. )
  6493. continue
  6494. # Sustained-humidity streak (#2518): updated on every pass that
  6495. # observes the reading above the threshold, BEFORE the
  6496. # suspension/cooldown gates below -- a suspended or cooling-down
  6497. # unit still accumulates streak time, so the wait overlaps those
  6498. # gates instead of stacking after them. Inert when the feature
  6499. # is off: no entries are written, and an entry left over from a
  6500. # toggle-off is dropped so it cannot seed a stale streak later.
  6501. if sustained_wait_active:
  6502. _now = time.monotonic()
  6503. # Four missed scheduler passes, floored: a single slow pass
  6504. # must not void a streak, but the ceiling has to scale with
  6505. # the cadence or a slow loop silently restarts every streak.
  6506. _gap_ceiling = max(4 * self._check_interval, AUTO_DRY_SUSTAINED_GAP_FLOOR_SECONDS)
  6507. _above = self._auto_dry_above.get(unit_key)
  6508. if _above is None:
  6509. self._auto_dry_above[unit_key] = {"since": _now, "last": _now}
  6510. elif _now - _above["last"] > _gap_ceiling:
  6511. # Restart, and say so at the same level as the dip
  6512. # reset: a silent restart voids the streak invisibly,
  6513. # and a user who set a long wait and never gets a dry
  6514. # has no way to see why.
  6515. logger.info(
  6516. "Auto-drying: printer %d AMS %d — sustained-humidity streak restarted after a "
  6517. "%.0fs observation gap (ceiling %ds); the %dm wait starts over",
  6518. pid,
  6519. ams_id,
  6520. _now - _above["last"],
  6521. _gap_ceiling,
  6522. sustained_minutes,
  6523. )
  6524. self._auto_dry_above[unit_key] = {"since": _now, "last": _now}
  6525. else:
  6526. _above["last"] = _now
  6527. else:
  6528. self._auto_dry_above.pop(unit_key, None)
  6529. if unit_state is not None:
  6530. if unit_state.get("suspended"):
  6531. logger.debug(
  6532. "Auto-drying: printer %d AMS %d skipped — suspended until humidity is back at "
  6533. "or below the %d%% threshold",
  6534. pid,
  6535. ams_id,
  6536. humidity_threshold,
  6537. )
  6538. continue
  6539. never_heated = _never_heated(unit_state)
  6540. if never_heated >= AUTO_DRY_MAX_NEVER_HEATED_CYCLES:
  6541. unit_state["suspended"] = True
  6542. logger.warning(
  6543. "Auto-drying: printer %d AMS %d — suspending automatic drying. The printer "
  6544. "accepted %d drying commands in a row, but each time the timer stood still and "
  6545. "the unit did not heat. Clear the timer with Stop on the printer card and check "
  6546. "the AMS power; automatic drying resumes once humidity is back at or below the "
  6547. "%d%% threshold.",
  6548. pid,
  6549. ams_id,
  6550. never_heated,
  6551. humidity_threshold,
  6552. )
  6553. await self._notify_auto_drying_suspended(
  6554. db, printer, ams_id, humidity, humidity_threshold, never_heated, never_heated=True
  6555. )
  6556. continue
  6557. if int(unit_state.get("unproductive", 0)) >= AUTO_DRY_MAX_UNPRODUCTIVE_CYCLES:
  6558. unit_state["suspended"] = True
  6559. logger.warning(
  6560. "Auto-drying: printer %d AMS %d — suspending automatic drying. %d cycles in a "
  6561. "row ended with humidity at %d%%, still above the %d%% threshold. The AMS reads "
  6562. "higher while it is warm, so a threshold in that range can never be reached and "
  6563. "re-arming would loop. Raise the threshold or dry the spools off the printer.",
  6564. pid,
  6565. ams_id,
  6566. int(unit_state.get("unproductive", 0)),
  6567. humidity,
  6568. humidity_threshold,
  6569. )
  6570. await self._notify_auto_drying_suspended(
  6571. db, printer, ams_id, humidity, humidity_threshold, int(unit_state.get("unproductive", 0))
  6572. )
  6573. continue
  6574. ended_at = unit_state.get("ended_at")
  6575. if isinstance(ended_at, float) and time.monotonic() - ended_at < AUTO_DRY_REARM_COOLDOWN_SECONDS:
  6576. logger.debug(
  6577. "Auto-drying: printer %d AMS %d skipped — cooling off for %ds after the last "
  6578. "cycle before the humidity reading is worth acting on",
  6579. pid,
  6580. ams_id,
  6581. AUTO_DRY_REARM_COOLDOWN_SECONDS,
  6582. )
  6583. continue
  6584. # Check cannot-dry reasons (power constraints etc.). Sits
  6585. # ahead of the sustained wait so a unit the firmware refuses
  6586. # to dry never logs a wait it was never going to cash in.
  6587. sf_reasons = ams_data.get("dry_sf_reason", [])
  6588. if sf_reasons:
  6589. logger.debug(
  6590. "Auto-drying: printer %d AMS %d skipped — cannot dry reasons: %s",
  6591. pid,
  6592. ams_id,
  6593. sf_reasons,
  6594. )
  6595. continue
  6596. # Sustained-humidity wait (#2518): ambient-triggered starts
  6597. # wait; only a printer with a scheduled queue item pending
  6598. # keeps the instant behavior, because that drying has a real
  6599. # deadline. Mid-print is deliberately NOT exempt while ambient
  6600. # drying is on: a humidity start on a printer that happens to be
  6601. # printing is as vulnerable to a lid-open spike as one on an idle
  6602. # printer (proven live: a 2-point threshold crossing mid-print
  6603. # bought a parked 12h command). Inactive when ambient drying is
  6604. # off: print_drying then still starts mid-print cycles on its own
  6605. # (#1816, "regardless of queue state"), and those stay instant.
  6606. if sustained_wait_active and pid not in printers_with_scheduled:
  6607. _above = self._auto_dry_above.get(unit_key)
  6608. _waited = time.monotonic() - _above["since"] if _above else 0.0
  6609. if _waited < sustained_minutes * 60:
  6610. logger.debug(
  6611. "Auto-drying: printer %d AMS %d waiting — humidity %s%% above the %d%% "
  6612. "threshold for %.0fs of the required %dm",
  6613. pid,
  6614. ams_id,
  6615. humidity,
  6616. humidity_threshold,
  6617. _waited,
  6618. sustained_minutes,
  6619. )
  6620. continue
  6621. # Get conservative drying params for mixed filaments
  6622. params = self._get_conservative_drying_params(trays, module_type, presets)
  6623. if not params:
  6624. logger.debug(
  6625. "Auto-drying: printer %d AMS %d skipped — no drying-eligible filaments in trays", pid, ams_id
  6626. )
  6627. continue
  6628. temp, duration_hours, filament_type = params
  6629. # Mid-print drying: cap drying temperature to protect spools (Bambu warns
  6630. # "drying temperature must not exceed the filament's softening temperature"
  6631. # for Print While Drying). Floor at 40 degC — below that the dryer is
  6632. # ineffective and firmware will reject anyway.
  6633. if mid_print:
  6634. temp = max(40, temp - 5)
  6635. # Start drying
  6636. logger.info(
  6637. "Auto-drying: printer %d AMS %d — humidity %d%% > threshold %d%%, "
  6638. "starting %s drying at %d°C for %dh%s",
  6639. pid,
  6640. ams_id,
  6641. humidity,
  6642. humidity_threshold,
  6643. filament_type,
  6644. temp,
  6645. duration_hours,
  6646. " (mid-print)" if mid_print else "",
  6647. )
  6648. success = printer_manager.send_drying_command(
  6649. pid, ams_id, temp, duration_hours, mode=1, filament=filament_type
  6650. )
  6651. if success:
  6652. self._drying_in_progress[pid] = time.monotonic()
  6653. armed = self._auto_dry_units.setdefault(
  6654. unit_key, {"unproductive": 0, "suspended": False, "ended_at": None}
  6655. )
  6656. armed["running"] = True
  6657. async def _notify_auto_drying_suspended(
  6658. self,
  6659. db: AsyncSession,
  6660. printer: Printer,
  6661. ams_id: int,
  6662. humidity: int,
  6663. threshold: int,
  6664. cycles: int,
  6665. *,
  6666. never_heated: bool = False,
  6667. ) -> None:
  6668. """Tell the user auto-drying has given up on one AMS unit (#2770).
  6669. ``never_heated`` picks the message for cycles the printer accepted but
  6670. never ran (#3478) instead of cycles that left humidity high.
  6671. Fires once per suspension — the caller sets ``suspended`` before calling
  6672. and every later pass short-circuits on it — because the whole point is
  6673. that Bambuddy has stopped acting. Somebody whose printer sits in another
  6674. building needs that to reach them, and the hourly humidity alarm they
  6675. are already getting says the opposite of what happened here.
  6676. Never raises: a notification provider being down must not stop the
  6677. suspension itself from taking effect.
  6678. """
  6679. ams_label = unit_label(ams_id, printer_tray_layout(printer.id, printer_manager).ams_type(ams_id))
  6680. try:
  6681. await notification_service.on_ams_drying_suspended(
  6682. printer.id,
  6683. printer.name,
  6684. ams_label,
  6685. float(humidity),
  6686. float(threshold),
  6687. cycles,
  6688. db,
  6689. never_heated=never_heated,
  6690. )
  6691. except Exception as e:
  6692. logger.warning("Failed to send auto-drying suspended notification: %s", e)
  6693. def forget_auto_dry_cycle(self, printer_id: int, ams_id: int) -> None:
  6694. """Stop judging the drying cycle currently on this AMS unit (#2770).
  6695. The unproductive-cycle counter exists to notice that *drying* is not
  6696. moving the humidity reading. A cycle that ended because somebody sent a
  6697. stop says nothing about that — it was cut short before it had a chance —
  6698. so counting it would suspend auto-drying for a reason that has nothing to
  6699. do with the loop the counter is there to break.
  6700. Two callers, both of them a stop Bambuddy is responsible for: the
  6701. print-takes-priority stop below, and the manual Stop button. Left
  6702. uncalled, an install that dries between queue jobs would suspend its own
  6703. auto-drying after two prints interrupted a dry — exactly the install
  6704. queue-drying exists for.
  6705. The rest of the unit's history is kept: the cooldown before re-arming
  6706. still applies, and an earlier count still stands.
  6707. """
  6708. state = self._auto_dry_units.get((printer_id, ams_id))
  6709. if state is not None:
  6710. state.pop("running", None)
  6711. state["ended_at"] = time.monotonic()
  6712. # A stopped cycle spends the streak that armed it, same as a completed
  6713. # one (#2518).
  6714. self._auto_dry_above.pop((printer_id, ams_id), None)
  6715. def _sync_drying_state(self):
  6716. """Drop printers from ``_drying_in_progress`` that are no longer drying.
  6717. One direction only: it prunes, it never adds. A printer drying without an
  6718. entry here — because the user started the cycle from Studio, the printer's
  6719. screen or Bambuddy's own manual Dry button, or because Bambuddy restarted
  6720. mid-cycle — stays unknown to the scheduler, so the "print takes priority"
  6721. stop at ``check_queue`` only ever applies to cycles Bambuddy itself began.
  6722. That is deliberate for now rather than an oversight: populating this from
  6723. telemetry would hand the scheduler authority to stop drying a user started
  6724. by hand. It also means the backend-restart case this used to claim to
  6725. handle is not handled.
  6726. """
  6727. to_remove = []
  6728. for pid in self._drying_in_progress:
  6729. state = printer_manager.get_status(pid)
  6730. if not state:
  6731. to_remove.append(pid)
  6732. continue
  6733. # Check if any AMS unit is still drying
  6734. ams_list = state.raw_data.get("ams", [])
  6735. # A parked timer is not drying (#3478): it never ends on its own,
  6736. # so it would keep the printer in here for good.
  6737. any_drying = any(int(a.get("dry_time") or 0) > 0 and not is_countdown_parked(a) for a in ams_list)
  6738. if not any_drying:
  6739. to_remove.append(pid)
  6740. for pid in to_remove:
  6741. self._drying_in_progress.pop(pid, None)
  6742. # A printer that has gone away entirely takes its per-AMS auto-drying
  6743. # history with it (#2770), so a printer deleted and re-added does not
  6744. # inherit a suspension it never earned.
  6745. for key in [k for k in self._auto_dry_units if printer_manager.get_status(k[0]) is None]:
  6746. self._auto_dry_units.pop(key, None)
  6747. # Same for sustained-wait streaks (#2518): a deleted-and-re-added
  6748. # printer starts a fresh wait, and vanished printers do not leak
  6749. # entries.
  6750. for key in [k for k in self._auto_dry_above if printer_manager.get_status(k[0]) is None]:
  6751. self._auto_dry_above.pop(key, None)
  6752. @staticmethod
  6753. def _drying_is_only_parked(printer_id: int) -> bool:
  6754. """True when every AMS unit with a drying timer on this printer is parked.
  6755. A parked timer (#2896: the command was taken but the countdown never
  6756. runs) does not end on its own, so nothing may wait on it. False when no
  6757. unit reports a timer yet -- a command just sent that the firmware has not
  6758. reported back is real drying about to begin.
  6759. """
  6760. state = printer_manager.get_status(printer_id)
  6761. units = [a for a in ((state.raw_data or {}).get("ams") or [] if state else []) if isinstance(a, dict)]
  6762. timed = []
  6763. for unit in units:
  6764. try:
  6765. if int(unit.get("dry_time") or 0) > 0:
  6766. timed.append(unit)
  6767. except (TypeError, ValueError):
  6768. continue
  6769. return bool(timed) and all(is_countdown_parked(unit) for unit in timed)
  6770. async def _drying_may_continue_through_print(self, db: AsyncSession, printer_id: int) -> bool:
  6771. """True when a running cycle can be left alone while the next print runs.
  6772. Some hardware dries happily through a print and #2758 settled that we
  6773. should not tear those cycles down: the X2D there was refusing to
  6774. *start* a job, which is a different problem, and stopping drying before
  6775. every dispatch would throw away cycles the printer was content to run.
  6776. Where the model cannot do it, or the user has not enabled it, the print
  6777. takes priority and the cycle stops -- which is what the queue_drying_block
  6778. setting has always promised in its off position.
  6779. """
  6780. if not await self._get_bool_setting(db, "print_drying_enabled"):
  6781. return False
  6782. status = printer_manager.get_status(printer_id)
  6783. return supports_drying_while_printing(
  6784. printer_manager.get_model(printer_id),
  6785. status.firmware_version if status else None,
  6786. )
  6787. async def _stop_drying(self, printer_id: int):
  6788. """Stop drying cycles Bambuddy armed on a printer (print takes priority).
  6789. Scoped to units in ``_auto_dry_units``. It used to send a stop to every
  6790. AMS reporting ``dry_time > 0``, which meant one auto-dried unit was
  6791. enough to kill a cycle the user had started by hand on a *different*
  6792. unit of the same printer (#2801). That contradicted the contract
  6793. ``_sync_drying_state`` already documents -- the entry gate deliberately
  6794. only knows about cycles Bambuddy began, so the action must not reach
  6795. past them either.
  6796. """
  6797. state = printer_manager.get_status(printer_id)
  6798. if not state:
  6799. self._drying_in_progress.pop(printer_id, None)
  6800. return
  6801. ams_list = state.raw_data.get("ams", [])
  6802. for ams_data in ams_list:
  6803. dry_time = int(ams_data.get("dry_time") or 0)
  6804. if dry_time > 0:
  6805. ams_id = int(ams_data.get("id", 0))
  6806. if (printer_id, ams_id) not in self._auto_dry_units:
  6807. logger.debug(
  6808. "Auto-drying: leaving printer %d AMS %d alone — not a cycle Bambuddy started",
  6809. printer_id,
  6810. ams_id,
  6811. )
  6812. continue
  6813. logger.info(
  6814. "Auto-drying: stopping drying on printer %d AMS %d — print takes priority",
  6815. printer_id,
  6816. ams_id,
  6817. )
  6818. printer_manager.send_drying_command(printer_id, ams_id, 0, 0, mode=0)
  6819. self.forget_auto_dry_cycle(printer_id, ams_id)
  6820. self._drying_in_progress.pop(printer_id, None)
  6821. # Scheduled manual drying (#2638) -----------------------------------
  6822. SCHEDULED_DRYING_GRACE_SECONDS = 120 # firmware needs time to report dry_time
  6823. SCHEDULED_DRYING_COMPLETE_FRACTION = 0.9 # dry_time==0 earlier than this = interrupted
  6824. async def _check_scheduled_dryings(self, db: AsyncSession):
  6825. """Dispatch due scheduled drying runs and track running ones."""
  6826. now = utcnow_naive()
  6827. # Hourly, not every pass: see SCHEDULED_DRYING_PRUNE_INTERVAL_SECONDS.
  6828. # Monotonic, so a clock adjustment cannot park the prune for hours.
  6829. since_prune = time.monotonic()
  6830. if (
  6831. self._last_scheduled_drying_prune is None
  6832. or since_prune - self._last_scheduled_drying_prune >= SCHEDULED_DRYING_PRUNE_INTERVAL_SECONDS
  6833. ):
  6834. self._last_scheduled_drying_prune = since_prune
  6835. await db.execute(
  6836. delete(ScheduledDrying).where(
  6837. ScheduledDrying.status.in_(("completed", "cancelled", "failed")),
  6838. ScheduledDrying.completed_at.is_not(None),
  6839. ScheduledDrying.completed_at < now - timedelta(days=SCHEDULED_DRYING_RETENTION_DAYS),
  6840. )
  6841. )
  6842. # Same order as the list route: with two rows due on one printer the
  6843. # earliest scheduled wins rather than whatever the DB hands back first.
  6844. result = await db.execute(
  6845. select(ScheduledDrying)
  6846. .where(ScheduledDrying.status.in_(("pending", "running")))
  6847. .order_by(ScheduledDrying.start_after.asc().nullsfirst(), ScheduledDrying.id.asc())
  6848. )
  6849. rows = list(result.scalars().all())
  6850. # Rebuild from the DB every tick so route-side cancels and completions
  6851. # show up. Auto-drying's stop-all branches check this set before
  6852. # stopping anything (#2638).
  6853. # Kept from the previous pass so a run that ended between passes — a
  6854. # cancel through the route, say — can still be released below.
  6855. previously_running = self._scheduled_drying_printer_ids
  6856. self._scheduled_drying_printer_ids = {row.printer_id for row in rows if row.status == "running"}
  6857. running_printer_ids = set(self._scheduled_drying_printer_ids)
  6858. # Model and firmware come from the printer row, not the live state.
  6859. printer_ids = {row.printer_id for row in rows}
  6860. printers_by_id: dict[int, Printer] = {}
  6861. if printer_ids:
  6862. printer_rows = await db.execute(select(Printer).where(Printer.id.in_(printer_ids)))
  6863. printers_by_id = {p.id: p for p in printer_rows.scalars()}
  6864. for row in rows:
  6865. if row.status == "running":
  6866. self._update_running_scheduled_drying(row, now)
  6867. continue
  6868. if row.start_after is not None and row.start_after > now:
  6869. continue
  6870. state = printer_manager.get_status(row.printer_id)
  6871. if not state:
  6872. row.waiting_reason = "printer_offline"
  6873. continue
  6874. # Same preflight the immediate endpoint runs. Without it the publish
  6875. # succeeds, the row goes to running, the printer ignores the command
  6876. # and the run silently cancels itself after the grace window.
  6877. printer = printers_by_id.get(row.printer_id)
  6878. unsupported = drying_preflight.check_drying_supported(
  6879. printer.model if printer else None, state.firmware_version
  6880. )
  6881. if unsupported:
  6882. row.status = "failed"
  6883. row.error_message = unsupported
  6884. row.error_code = drying_preflight.DETAIL_CODES.get(unsupported)
  6885. row.completed_at = now
  6886. logger.warning("Scheduled drying %d: %s", row.id, unsupported)
  6887. continue
  6888. if (
  6889. self._drying_in_progress.get(row.printer_id) and not self._drying_is_only_parked(row.printer_id)
  6890. ) or row.printer_id in running_printer_ids:
  6891. row.waiting_reason = "already_drying"
  6892. continue
  6893. if not self._is_printer_idle(row.printer_id, ignore_plate_clear=True):
  6894. row.waiting_reason = "printer_busy"
  6895. continue
  6896. target = drying_preflight.find_ams_unit(state, row.ams_id)
  6897. if target is None:
  6898. row.waiting_reason = "ams_not_found"
  6899. continue
  6900. blocking = drying_preflight.blocking_reason_codes(target)
  6901. if blocking:
  6902. # Keep the power case distinct; it needs the user to act, so the
  6903. # card can say so instead of waiting silently.
  6904. row.waiting_reason = drying_preflight.waiting_reason_for_codes(blocking)
  6905. continue
  6906. filament = drying_preflight.resolve_filament(target, row.filament)
  6907. logger.info(
  6908. "Scheduled drying %d: starting on printer %d AMS %d at %d°C for %dh",
  6909. row.id,
  6910. row.printer_id,
  6911. row.ams_id,
  6912. row.temp,
  6913. row.duration_hours,
  6914. )
  6915. success = printer_manager.send_drying_command(
  6916. row.printer_id,
  6917. row.ams_id,
  6918. row.temp,
  6919. row.duration_hours,
  6920. mode=1,
  6921. filament=filament,
  6922. rotate_tray=row.rotate_tray,
  6923. )
  6924. if success:
  6925. row.status = "running"
  6926. row.started_at = now
  6927. row.waiting_reason = None
  6928. row.filament = filament
  6929. self._drying_in_progress[row.printer_id] = time.monotonic()
  6930. self._scheduled_drying_printer_ids.add(row.printer_id)
  6931. running_printer_ids.add(row.printer_id)
  6932. else:
  6933. row.waiting_reason = "printer_offline"
  6934. # Release the printers whose run has ended. `_drying_in_progress` is
  6935. # shared with auto-drying, which prunes it in `_sync_drying_state()` —
  6936. # but that call sits behind the auto-drying enabled check, and this
  6937. # method is the one writer that runs whether auto-drying is on or not.
  6938. # With it off, nothing would ever drop the entry short of a print being
  6939. # dispatched to the same printer, so the next scheduled run would wait
  6940. # on "already_drying" forever and `queue_drying_block` would hold the
  6941. # printer's prints too. Covers a run that ended during this pass and one
  6942. # cancelled through the route between passes.
  6943. self._scheduled_drying_printer_ids = {row.printer_id for row in rows if row.status == "running"}
  6944. for printer_id in (previously_running | running_printer_ids) - self._scheduled_drying_printer_ids:
  6945. self._drying_in_progress.pop(printer_id, None)
  6946. await db.commit()
  6947. def _update_running_scheduled_drying(self, row: ScheduledDrying, now: datetime):
  6948. """Detect completion or interruption of a running scheduled drying.
  6949. The firmware reports remaining minutes in ams.dry_time; 0 means not
  6950. drying. Within the grace window after start we ignore dry_time==0
  6951. (the status lags the command). After that, dry_time==0 near the end
  6952. of the configured duration means completed. Much earlier means the
  6953. run was stopped: re-queue it if a print preempted the dryer, but a
  6954. stop while the printer is idle was deliberate, so cancel the row
  6955. rather than restart drying the user just stopped.
  6956. """
  6957. if row.started_at is None:
  6958. row.started_at = now
  6959. return
  6960. elapsed = (now - row.started_at).total_seconds()
  6961. if elapsed < self.SCHEDULED_DRYING_GRACE_SECONDS:
  6962. return
  6963. state = printer_manager.get_status(row.printer_id)
  6964. if not state:
  6965. return # offline mid-dry; resolve when it reconnects
  6966. # find_ams_unit, not a local lookup: this runs inside check_queue, so a
  6967. # throw on a malformed id would cost the whole pass including print
  6968. # dispatch, every tick.
  6969. target = drying_preflight.find_ams_unit(state, row.ams_id)
  6970. try:
  6971. dry_time = int(target.get("dry_time") or 0) if target else 0
  6972. except (TypeError, ValueError):
  6973. dry_time = 0
  6974. if dry_time > 0 and is_countdown_parked(target):
  6975. # The printer took the command but the countdown is not running
  6976. # (#2896) and will never reach 0, so the run would stay "running"
  6977. # forever. A print in progress is the likely cause (the power budget
  6978. # is spent), so re-queue it like any interruption; the next start
  6979. # waits for the printer to be idle. Parked on an idle printer is a
  6980. # refusal, not something a retry fixes. The timer itself is left on
  6981. # the printer: it may yet start once power frees up.
  6982. if not self._is_printer_idle(row.printer_id, ignore_plate_clear=True):
  6983. logger.info(
  6984. "Scheduled drying %d: AMS %d countdown is not running during a print; re-queued",
  6985. row.id,
  6986. row.ams_id,
  6987. )
  6988. row.status = "pending"
  6989. row.started_at = None
  6990. row.waiting_reason = "interrupted"
  6991. else:
  6992. logger.warning(
  6993. "Scheduled drying %d: printer accepted the command but AMS %d never started drying",
  6994. row.id,
  6995. row.ams_id,
  6996. )
  6997. row.status = "failed"
  6998. row.error_message = drying_preflight.DID_NOT_START_DETAIL
  6999. row.error_code = drying_preflight.DETAIL_CODES[drying_preflight.DID_NOT_START_DETAIL]
  7000. row.completed_at = now
  7001. return
  7002. if dry_time > 0:
  7003. return
  7004. if elapsed >= row.duration_hours * 3600 * self.SCHEDULED_DRYING_COMPLETE_FRACTION:
  7005. row.status = "completed"
  7006. row.completed_at = now
  7007. elif not self._is_printer_idle(row.printer_id, ignore_plate_clear=True):
  7008. row.status = "pending"
  7009. row.started_at = None
  7010. row.waiting_reason = "interrupted"
  7011. else:
  7012. row.status = "cancelled"
  7013. row.completed_at = now
  7014. async def _get_smart_plugs(self, db: AsyncSession, printer_id: int) -> list[SmartPlug]:
  7015. """Get all smart plugs associated with a printer."""
  7016. result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  7017. return list(result.scalars().all())
  7018. @staticmethod
  7019. def _auto_on_plugs(plugs: list[SmartPlug]) -> list[SmartPlug]:
  7020. """The plugs that may power a printer on: enabled, auto on, and not a
  7021. monitor-only MQTT plug, which Bambuddy cannot switch (#3583). Must
  7022. match the query in :meth:`_wakeable_printer_ids`."""
  7023. return [p for p in plugs if p.auto_on and p.enabled and not is_monitor_only(p)]
  7024. @staticmethod
  7025. def _pick_power_plug(auto_on_plugs: list[SmartPlug]) -> SmartPlug:
  7026. """Pick the plug to power-cycle a printer back online with (#2629).
  7027. Only a plug flagged ``controls_printer_power`` can actually bring the
  7028. printer back; waiting for a boot on an accessory (filter fan, lights)
  7029. just burns the power-on timeout and fails the dispatch. Falls back to
  7030. the first plug when none is flagged, which is the pre-#2629 behaviour.
  7031. Callers must pass a non-empty list.
  7032. """
  7033. for plug in auto_on_plugs:
  7034. if plug.controls_printer_power:
  7035. return plug
  7036. return auto_on_plugs[0]
  7037. # Bundled defaults for preheat_filament_targets (#1468). Values are the
  7038. # chamber-temperature recommendations BambuStudio ships for the matching
  7039. # filament profile; users can override via Settings → Workflow → Preheat
  7040. # card. "default" applies when a loaded tray's normalised type isn't in
  7041. # the map (rare — Bambu RFID-tagged spools always carry a known type).
  7042. DEFAULT_PREHEAT_FILAMENT_TARGETS: dict[str, int] = {
  7043. "PLA": 0,
  7044. "PETG": 0,
  7045. "PETG-CF": 40,
  7046. "ABS": 45,
  7047. "ASA": 45,
  7048. "PA": 50,
  7049. "PA-CF": 55,
  7050. "PC": 50,
  7051. "PC-FR": 50,
  7052. "TPU": 0,
  7053. "PVA": 0,
  7054. "default": 0,
  7055. }
  7056. @classmethod
  7057. def _bundled_preheat_targets(cls) -> dict[str, int]:
  7058. """The bundled map under the same key casing a parsed one gets.
  7059. The constant is declared with a lowercase ``default`` because that is
  7060. the key the Settings editor writes and displays. Every read of the map
  7061. happens after ``str(key).upper()``, so handing the constant back as
  7062. declared broke the contract the parser documents: an install that had
  7063. never touched the setting returned a dict with no ``DEFAULT`` in it,
  7064. and the resolution loop's fallback silently found nothing. It read the
  7065. right number only because the bundled default happens to be 0 -- change
  7066. that constant and every unconfigured install would keep preheating to
  7067. zero with no way to tell why.
  7068. """
  7069. return {key.upper(): value for key, value in cls.DEFAULT_PREHEAT_FILAMENT_TARGETS.items()}
  7070. async def _get_preheat_filament_targets(self, db: AsyncSession) -> dict[str, int]:
  7071. """Parse the user-configured filament→chamber-target map, falling back
  7072. to DEFAULT_PREHEAT_FILAMENT_TARGETS on missing / malformed JSON. Keys
  7073. are uppercased and the 'default' fallback is always present in the
  7074. returned dict so the resolution loop can index it unconditionally."""
  7075. raw = await self._get_setting(db, "preheat_filament_targets")
  7076. if not raw:
  7077. return self._bundled_preheat_targets()
  7078. try:
  7079. parsed = json.loads(raw)
  7080. if not isinstance(parsed, dict):
  7081. raise ValueError("not an object")
  7082. except (json.JSONDecodeError, ValueError) as exc:
  7083. logger.warning("preheat_filament_targets unparseable, using defaults: %s", exc)
  7084. return self._bundled_preheat_targets()
  7085. # Coerce values to int; drop unparseable rows so a stray string
  7086. # doesn't crash the loop.
  7087. out: dict[str, int] = {}
  7088. for key, value in parsed.items():
  7089. try:
  7090. out[str(key).upper()] = int(value)
  7091. except (TypeError, ValueError):
  7092. continue
  7093. if "DEFAULT" not in out:
  7094. out["DEFAULT"] = self.DEFAULT_PREHEAT_FILAMENT_TARGETS["default"]
  7095. return out
  7096. @staticmethod
  7097. def _normalize_filament_type(tray_type: str) -> str:
  7098. """Reduce the printer's tray_type to a preset-lookup key. Mirrors the
  7099. existing drying-preset normalisation (split-at-space, upper-case) so
  7100. the two maps share vocabulary — "PLA Basic" → "PLA", "PA-CF" stays
  7101. "PA-CF" (no space to split on).
  7102. This is the first stage of ``_resolve_filament_key``, which goes on to
  7103. drop the suffix and consult the alias map; on its own it only decides
  7104. what the tray is called, not which row answers for it.
  7105. Indexing the split rather than testing the input: a tray_type of spaces
  7106. is truthy and splits to nothing, so the old ``if tray_type`` guard let
  7107. it through to an IndexError.
  7108. """
  7109. words = (tray_type or "").split()
  7110. return words[0].upper() if words else ""
  7111. def _target_for_tray_type(self, tray_type: str | None, targets: dict[str, int]) -> int:
  7112. """Per-filament chamber target for one tray's reported type, or 0 when
  7113. the tray is empty / RFID-less and reports no type at all.
  7114. A filled or foamed variant wants its base material's chamber when the
  7115. map has no row of its own: ASA-GF is ASA and needs ASA's 45 degrees,
  7116. not the 0 an unknown type falls to. The specific type is still tried
  7117. first, so PETG-CF and PA-CF keep the hotter rows they are listed with
  7118. (#2902).
  7119. That lookup is now the shared one, which adds the polyamide aliases on
  7120. top of the suffix it already dropped -- so PA6-CF reaches PA's row here
  7121. too, rather than the catch-all it was landing on (#3067).
  7122. """
  7123. if not self._normalize_filament_type(tray_type or ""):
  7124. return 0
  7125. key = self._resolve_filament_key(tray_type, targets)
  7126. if key is not None:
  7127. return targets[key]
  7128. # A type the map does not list at all, which is not the same as a tray
  7129. # with nothing in it -- that already returned 0 above.
  7130. return targets.get("DEFAULT", 0)
  7131. def _derive_chamber_target(
  7132. self,
  7133. printer: Printer,
  7134. targets: dict[str, int],
  7135. item: PrintQueueItem | None = None,
  7136. ) -> int:
  7137. """Chamber target for the trays this print actually loads: the max of
  7138. their per-filament targets. Returns 0 when there is nothing to read (no
  7139. status, no AMS telemetry — e.g. external-spool prints) or when every
  7140. tray considered maps to 0, and the chamber phase then short-circuits in
  7141. the main loop.
  7142. ``item`` narrows the scan to the trays named in its ``ams_mapping``.
  7143. Scanning the whole unit instead meant one ASA spool parked in the AMS
  7144. forced a 45°C chamber onto every PLA job sharing it — the full max-wait
  7145. plus soak burned ahead of each upload, on a printer whose chamber never
  7146. reaches the target anyway (#2886). An item with no usable mapping falls
  7147. back to scanning every loaded tray: that is the only signal left, and
  7148. narrowing to nothing would skip preheat on prints that genuinely need
  7149. it.
  7150. Reads from `printer_manager.get_status(...).raw_data['ams']`, which is
  7151. the same source the dispatcher uses for AMS slot mapping. Empty / RFID-
  7152. less slots have empty `tray_type` and contribute nothing. The external
  7153. spool is consulted only when the mapping names it (>= 254); it stays
  7154. out of the unnarrowed scan, so an item without a mapping derives from
  7155. the AMS alone exactly as before.
  7156. """
  7157. state = printer_manager.get_status(printer.id)
  7158. if state is None:
  7159. return 0
  7160. raw_data = state.raw_data or {}
  7161. used = _used_global_tray_ids(item)
  7162. ams_list = raw_data.get("ams")
  7163. # Older Bambu firmware nests AMS as {"ams": {"ams": [...]}} — try both.
  7164. if isinstance(ams_list, dict):
  7165. ams_list = ams_list.get("ams") or []
  7166. if not isinstance(ams_list, list):
  7167. ams_list = []
  7168. best = 0
  7169. layout = TrayLayout.from_state(state)
  7170. for ams in ams_list:
  7171. if not isinstance(ams, dict):
  7172. continue
  7173. ams_id = _int_or(ams.get("id"), 0)
  7174. for tray in ams.get("tray") or []:
  7175. # A non-dict entry has never been seen from real firmware, but
  7176. # `.get` on one raises, and nothing between here and
  7177. # `_dispatch_one`'s try/finally catches it — the item would be
  7178. # left holding its dispatch claim. Preheat is best-effort by
  7179. # contract, so step over it instead.
  7180. if not isinstance(tray, dict):
  7181. continue
  7182. if used is not None and _global_tray_id(ams_id, _int_or(tray.get("id"), 0), layout) not in used:
  7183. continue
  7184. best = max(best, self._target_for_tray_type(tray.get("tray_type"), targets))
  7185. if used is not None and any(t >= _EXTERNAL_TRAY_ID_MIN or layout.is_external(t) for t in used):
  7186. for vt in raw_data.get("vt_tray") or []:
  7187. if not isinstance(vt, dict):
  7188. continue
  7189. # `_build_loaded_filaments` addresses external feeds by the id
  7190. # the firmware reports — 255 main, 254 deputy — defaulting to
  7191. # 254 when the field is absent. Same expression here so the two
  7192. # agree on which entry a mapping's 254/255 refers to.
  7193. if _int_or(vt.get("id"), _EXTERNAL_TRAY_ID_MIN) not in used:
  7194. continue
  7195. best = max(best, self._target_for_tray_type(vt.get("tray_type"), targets))
  7196. return best
  7197. def _release_keep_warm(self, pid: int) -> None:
  7198. """Release keep-warm on a printer that left the candidate set.
  7199. Publishes ``set_bed_temperature(0)`` once — but only if firmware still
  7200. reports the target we set (``entry.held_target``), so a user or
  7201. subsequent print that changed the bed target since is not clobbered.
  7202. Best-effort, never raises.
  7203. The entry is kept, not dropped, when the printer cannot be reached
  7204. right now: a printer that is briefly offline still has a hot bed, and
  7205. holding the entry is what keeps the max-duration timeout applying and
  7206. lets a later tick retry the release. Only a printer that has left the
  7207. manager entirely gives up on that, in ``_sample_chamber_temps``.
  7208. """
  7209. entry = self._keep_warm.get(pid)
  7210. if entry is None:
  7211. return
  7212. state = printer_manager.get_status(pid)
  7213. client = printer_manager.get_client(pid)
  7214. if state is None or client is None:
  7215. logger.debug(
  7216. "Queue: keep-warm release for printer %d deferred — printer unreachable, entry kept",
  7217. pid,
  7218. )
  7219. return
  7220. cur_bed_target = float((state.temperatures or {}).get("bed_target", 0) or 0)
  7221. if int(cur_bed_target) != entry.held_target:
  7222. # Someone else owns the bed now, so there is nothing of ours to
  7223. # undo and nothing left to track.
  7224. logger.info(
  7225. "Queue: keep-warm release for printer %d skipped bed-off (firmware target %d != held %d)",
  7226. pid,
  7227. int(cur_bed_target),
  7228. entry.held_target,
  7229. )
  7230. self._keep_warm.pop(pid, None)
  7231. return
  7232. try:
  7233. client.set_bed_temperature(0)
  7234. logger.info("Queue: keep-warm released for printer %d (bed → 0)", pid)
  7235. self._keep_warm.pop(pid, None)
  7236. except Exception as exc:
  7237. # Keep the entry so the next tick tries again rather than leaving
  7238. # the bed hot with nothing tracking it.
  7239. logger.warning("Queue: keep-warm release for printer %d failed: %s", pid, exc)
  7240. def _sweep_keep_warm(self, active_candidates: set[int], dispatched: set[int]) -> None:
  7241. """Release printers that dropped out of the keep-warm candidate set.
  7242. Called from ``_apply_keep_warm`` on every tick (with the current
  7243. candidate set), and from ``check_queue``'s no-pending-items early
  7244. return (with an empty candidate set) so orphaned holds still get
  7245. released when the queue empties. Also called with an empty candidate
  7246. set when any of the three gate settings toggles off, so a printer
  7247. whose feature was disabled mid-hold gets its bed released.
  7248. Printers being dispatched this tick are excluded from the bed-off
  7249. publish: ``_preheat_and_soak`` owns the bed from that tick on, so a
  7250. transient 0 in between would just churn against preheat. Ownership of
  7251. the hot bed transfers to the preheat rollback pin instead — if the
  7252. dispatch aborts before the print starts (failed upload, cancelled
  7253. item), `_rollback_preheat_pin` turns the bed off; if preheat itself
  7254. skips (e.g. the item has no bed_temperature metadata) the pin entry
  7255. is the ONLY thing standing between an aborted dispatch and a bed
  7256. left hot with no owner. A successful print start clears the pin and
  7257. the print's own gcode takes over, as usual.
  7258. """
  7259. for _pid in list(self._keep_warm):
  7260. if _pid in active_candidates:
  7261. continue
  7262. if _pid in dispatched:
  7263. handed_over = self._keep_warm.pop(_pid, None)
  7264. self._preheat_pin.setdefault(_pid, set()).add("bed")
  7265. if handed_over is not None:
  7266. self._preheat_pin_bed[_pid] = handed_over.held_target
  7267. continue
  7268. self._release_keep_warm(_pid)
  7269. async def _apply_keep_warm(
  7270. self,
  7271. db: AsyncSession,
  7272. items: list[PrintQueueItem],
  7273. dispatch_ids: list[int] | set[int],
  7274. busy_printers: set[int],
  7275. require_plate_clear: bool,
  7276. ) -> None:
  7277. """Hold the bed warm on FINISH printers whose next queued item needs chamber heat.
  7278. When a printer just finished a job (FINISH state) and the next queued
  7279. item needs chamber heating, hold the bed hot so the chamber stays warm
  7280. during the bed-clearing window. The bed is the chamber's heating
  7281. element here, not a print surface — nothing is printing during the
  7282. hold and the dispatched print's own preheat/gcode re-targets the bed —
  7283. so the hold temperature is ``queue_keep_warm_bed_temp`` (default 90°C,
  7284. chosen to sustain chamber warmth and to satisfy bed-threshold-linked
  7285. aftermarket chamber heaters), raised to the item's own parsed
  7286. bed_temperature when that is higher. Items whose archive metadata has
  7287. no bed temperature (e.g. OrcaSlicer gcode.3mf exports) therefore still
  7288. get a hold — chamber need is what gates the feature, not metadata.
  7289. Skips entirely for filaments that map to a 0°C chamber target
  7290. (PLA, PETG, etc.) — read off the trays the next item's ``ams_mapping``
  7291. names, so a hot-chamber spool it never touches does not hold the bed of
  7292. a PLA job (#2886). An item still awaiting its mapping is judged on the
  7293. whole unit, as every item was before. Printers being dispatched this
  7294. cycle are excluded:
  7295. ``_preheat_and_soak`` already handles their bed temperature.
  7296. Bounded by ``queue_keep_warm_max_minutes`` — on timeout the bed is
  7297. released to 0 and the entry is latched ``expired=True`` so
  7298. subsequent ticks neither re-engage nor re-seed the clock. Idempotent
  7299. MQTT: publish is skipped when firmware already has the target.
  7300. The release sweep runs BEFORE the engagement gate so a printer that
  7301. was owned by keep-warm still gets its bed released when any of the
  7302. three gate settings is toggled off mid-hold. The
  7303. ``check_queue`` early-return-when-no-items path also calls
  7304. ``_sweep_keep_warm`` directly to release orphaned holds.
  7305. """
  7306. dispatch_set = set(dispatch_ids)
  7307. dispatched_printers = {it.printer_id for it in items if it.id in dispatch_set and it.printer_id}
  7308. pending_printer_ids = {it.printer_id for it in items if it.printer_id}
  7309. # Only a printer the plate-clear gate applies to waits for its plate to
  7310. # be cleared; the others go straight on to the next job (#2860).
  7311. warm_candidates = {
  7312. pid
  7313. for pid in (pending_printer_ids & busy_printers) - dispatched_printers
  7314. if plate_clear.applies(pid, require_plate_clear)
  7315. }
  7316. keep_warm_enabled = await self._get_bool_setting(db, "queue_keep_bed_warm", default=False)
  7317. preheat_on = await self._get_bool_setting(db, "preheat_enabled", default=False)
  7318. gate_open = keep_warm_enabled and preheat_on and bool(warm_candidates)
  7319. # Release sweep first — must run even when gate_open is False so a
  7320. # printer owned by keep-warm when a gate toggles off gets released.
  7321. self._sweep_keep_warm(
  7322. active_candidates=warm_candidates if gate_open else set(),
  7323. dispatched=dispatched_printers,
  7324. )
  7325. if not gate_open:
  7326. return
  7327. hold_temp = await self._get_int_setting(db, "queue_keep_warm_bed_temp", default=90)
  7328. max_hold_seconds = (
  7329. await self._get_int_setting(db, "queue_keep_warm_max_minutes", default=_KEEP_WARM_MAX_MINUTES_DEFAULT) * 60
  7330. )
  7331. now_mono = time.monotonic()
  7332. filament_targets: dict[str, int] | None = None
  7333. for pid in warm_candidates:
  7334. entry = self._keep_warm.get(pid)
  7335. # Latched-expired: max-duration timeout already fired for this
  7336. # printer. Skip until the release sweep drops the entry (i.e.
  7337. # until the printer leaves the candidate set).
  7338. if entry is not None and entry.expired:
  7339. continue
  7340. # These two guards sit ahead of the max-duration check below, so an
  7341. # engaged hold only ages out while its printer is still reachable
  7342. # and still in FINISH. That is deliberate rather than a hole: with
  7343. # no status or no client there is no M140 to send anyway, and the
  7344. # elapsed check runs off `entry.started` so it fires on the first
  7345. # tick after the printer comes back. Leaving FINISH means the plate
  7346. # was cleared, which drops the printer out of `warm_candidates` and
  7347. # hands it to `_release_keep_warm` instead. The invariant worth
  7348. # preserving if this is ever reordered: every path out of an
  7349. # engaged hold ends in a bed-off, whether by timeout or release.
  7350. state = printer_manager.get_status(pid)
  7351. if state is None or state.state != "FINISH":
  7352. continue
  7353. client = printer_manager.get_client(pid)
  7354. if client is None:
  7355. continue
  7356. next_item = next((it for it in items if it.printer_id == pid), None)
  7357. if next_item is None:
  7358. continue
  7359. # Hold temperature: the configured keep-warm temp, raised to the
  7360. # item's own bed temp when the metadata reports a higher one. A
  7361. # missing bed_temperature (Orca gcode.3mf exports parse without
  7362. # one) does NOT skip the hold — chamber need gates the feature.
  7363. archive = next_item.archive
  7364. item_bed = await _queued_plate_bed_temperature(next_item, archive)
  7365. bed_target = max(item_bed, hold_temp)
  7366. if bed_target <= 0:
  7367. continue
  7368. explicit = getattr(next_item, "preheat_chamber_target_override", None)
  7369. if explicit is not None:
  7370. chamber_needed = int(explicit) > 0
  7371. else:
  7372. if filament_targets is None:
  7373. filament_targets = await self._get_preheat_filament_targets(db)
  7374. printer_obj = await self._get_printer(db, pid)
  7375. chamber_needed = (
  7376. printer_obj is not None
  7377. and self._derive_chamber_target(printer_obj, filament_targets, next_item) > 0
  7378. )
  7379. if not chamber_needed:
  7380. continue
  7381. # Seed the timer on first engagement; keep it on subsequent ticks
  7382. # (never re-seed — that would defeat the max-duration cap).
  7383. if entry is None:
  7384. entry = _KeepWarmEntry(started=now_mono, held_target=bed_target)
  7385. self._keep_warm[pid] = entry
  7386. elapsed = now_mono - entry.started
  7387. if elapsed > max_hold_seconds:
  7388. # Timeout: publish bed → 0 once (if firmware still holds our
  7389. # target) and latch expired. The entry stays until the release
  7390. # sweep drops it, preventing the next tick from re-seeding.
  7391. logger.warning(
  7392. "Queue: keep-warm timeout for printer %d (held for %.0fs) — publishing bed → 0",
  7393. pid,
  7394. elapsed,
  7395. )
  7396. cur_bed_target = float((state.temperatures or {}).get("bed_target", 0) or 0)
  7397. if int(cur_bed_target) == entry.held_target:
  7398. try:
  7399. client.set_bed_temperature(0)
  7400. except Exception as exc:
  7401. logger.warning(
  7402. "Queue: keep-warm timeout bed-off failed for printer %d: %s",
  7403. pid,
  7404. exc,
  7405. )
  7406. else:
  7407. logger.info(
  7408. "Queue: keep-warm timeout for printer %d skipped bed-off (firmware target %d != held %d)",
  7409. pid,
  7410. int(cur_bed_target),
  7411. entry.held_target,
  7412. )
  7413. entry.expired = True
  7414. continue
  7415. # Idempotence: skip publish when firmware already has our target.
  7416. cur_bed_target = float((state.temperatures or {}).get("bed_target", 0) or 0)
  7417. if int(cur_bed_target) == bed_target:
  7418. entry.held_target = bed_target
  7419. continue
  7420. try:
  7421. client.set_bed_temperature(bed_target)
  7422. entry.held_target = bed_target
  7423. logger.info(
  7424. "Queue: keeping bed warm at %d°C for printer %d (FINISH, next item needs chamber heat)",
  7425. bed_target,
  7426. pid,
  7427. )
  7428. except Exception as exc:
  7429. logger.warning("Queue: keep-warm bed command failed for printer %d: %s", pid, exc)
  7430. def _sample_chamber_temps(self) -> None:
  7431. """Record a chamber temperature sample for every connected printer.
  7432. Called once per scheduler tick (every 3–30 s). Entries older than
  7433. _chamber_history_ttl are pruned on each write so the deques stay bounded.
  7434. Also evicts per-printer state whose printer_id is no longer registered
  7435. (e.g. deleted from the DB), so nothing accumulates for gone printers.
  7436. """
  7437. now = time.monotonic()
  7438. cutoff = now - self._chamber_history_ttl
  7439. statuses = printer_manager.get_all_statuses()
  7440. known_pids = set(statuses.keys())
  7441. for pid, status in statuses.items():
  7442. if status is None or not status.connected:
  7443. continue
  7444. temps = status.temperatures or {}
  7445. chamber = temps.get("chamber")
  7446. if chamber is None:
  7447. continue
  7448. hist = self._chamber_history.setdefault(pid, deque())
  7449. hist.append((now, float(chamber)))
  7450. while hist and hist[0][0] < cutoff:
  7451. hist.popleft()
  7452. # Evict state for printers that are no longer registered with the manager.
  7453. # This is the one place a keep-warm entry is dropped without releasing
  7454. # the bed: the printer is gone from the manager, so there is no client
  7455. # left to send M140 to. `_release_keep_warm` deliberately keeps entries
  7456. # for printers that are merely unreachable, which is what makes this
  7457. # the terminal case rather than a silent leak.
  7458. for pid in list(self._chamber_history):
  7459. if pid not in known_pids:
  7460. self._chamber_history.pop(pid, None)
  7461. for pid in list(self._keep_warm):
  7462. if pid not in known_pids:
  7463. logger.info(
  7464. "Queue: dropping keep-warm state for printer %d — no longer registered",
  7465. pid,
  7466. )
  7467. self._keep_warm.pop(pid, None)
  7468. for pid in list(self._preheat_pin):
  7469. if pid not in known_pids:
  7470. self._preheat_pin.pop(pid, None)
  7471. self._preheat_pin_bed.pop(pid, None)
  7472. def _chamber_soak_remaining(
  7473. self,
  7474. printer_id: int,
  7475. chamber_target: float,
  7476. soak_seconds: int,
  7477. tolerance: float = 2.0,
  7478. ) -> int:
  7479. """Return how many seconds of soak time are still needed.
  7480. Credits the time the chamber has already spent at temperature against
  7481. the configured soak. The credit may not start earlier than any of:
  7482. * **The newest sample.** Nothing recent means the printer stopped
  7483. reporting mid-observation and the chamber may have cooled unseen, so
  7484. the full soak is required. (A 2 h history whose last reading is half
  7485. an hour old is not evidence of anything — the measured cooling rate
  7486. is fast enough to cross the threshold in that time.)
  7487. * **The most recent contiguous run of samples.** A gap wider than
  7488. ``_CHAMBER_SAMPLE_MAX_GAP_SECONDS`` is a disconnect, and time on its
  7489. far side is not evidence of temperature.
  7490. * **The end of the most recent real dip below the threshold.**
  7491. A dip only counts as real once it lasts ``_CHAMBER_DIP_GRACE_SECONDS``
  7492. — see that constant for the thermal reasoning. A stray low reading is
  7493. an artifact, and treating it as cooling would discard a soak that
  7494. actually happened.
  7495. Returns ``soak_seconds`` when nothing can be credited (no history,
  7496. stale history, or the chamber is below the threshold right now) and 0
  7497. once the credited time covers the whole soak.
  7498. """
  7499. hist = self._chamber_history.get(printer_id)
  7500. if not hist:
  7501. return soak_seconds
  7502. now = time.monotonic()
  7503. newest_ts, newest_temp = hist[-1]
  7504. if now - newest_ts > _CHAMBER_SAMPLE_MAX_GAP_SECONDS:
  7505. return soak_seconds # stale — no fresh evidence to credit
  7506. threshold = chamber_target - tolerance
  7507. if newest_temp < threshold:
  7508. return soak_seconds # below target right now; nothing is soaked
  7509. samples = list(hist)
  7510. # Earliest point we have unbroken observations for.
  7511. credit_from = samples[-1][0]
  7512. for i in range(len(samples) - 1, 0, -1):
  7513. if samples[i][0] - samples[i - 1][0] > _CHAMBER_SAMPLE_MAX_GAP_SECONDS:
  7514. break
  7515. credit_from = samples[i - 1][0]
  7516. # Pull the credit forward to the end of the last significant dip. Each
  7517. # excursion is measured between the in-range readings that bracket it,
  7518. # so a lone stray sample is charged one sampling interval rather than
  7519. # zero, and the comparison errs towards calling a dip real.
  7520. i = 0
  7521. while i < len(samples):
  7522. if samples[i][1] >= threshold:
  7523. i += 1
  7524. continue
  7525. j = i
  7526. while j < len(samples) and samples[j][1] < threshold:
  7527. j += 1
  7528. # `newest_temp >= threshold` was checked above, so j is in range.
  7529. opened_at = samples[i - 1][0] if i > 0 else samples[i][0]
  7530. if samples[j][0] - opened_at >= _CHAMBER_DIP_GRACE_SECONDS:
  7531. # Credit resumes at the last below-threshold sample rather than
  7532. # the first good one after it, so a recovered dip over-credits
  7533. # by up to one sampling interval — the opposite lean to the
  7534. # bracketing above. Both are bounded by the sample cadence and
  7535. # dwarfed by the grace period, so neither is worth the extra
  7536. # arithmetic to remove.
  7537. credit_from = max(credit_from, samples[j - 1][0])
  7538. i = j
  7539. return max(0, soak_seconds - int(now - credit_from))
  7540. def notify_dispatch_cancelled(self, item_id: int) -> None:
  7541. """Tell an in-flight dispatch that its item no longer wants to print.
  7542. Called by the queue's cancel and delete routes. Those only write to the
  7543. database, which a dispatch coroutine parked in ``asyncio.sleep`` cannot
  7544. observe — so preheat would keep heating for the rest of max_wait + soak
  7545. (45 minutes at the defaults) and keep the printer in ``busy_printers``,
  7546. blocking every other queued item behind a print that is not happening.
  7547. Signalling in memory rather than re-reading the row keeps this off the
  7548. database entirely: no second session, no transaction held across a long
  7549. sleep, and no snapshot staleness deciding whether a print goes ahead.
  7550. Bambuddy serves from a single uvicorn process with one scheduler task,
  7551. so the route and the dispatch always share this object. The flag is
  7552. advisory — dropping it (e.g. after a restart) only costs a wasted
  7553. preheat, never a wrongly-abandoned print.
  7554. Only ids with a dispatch actually in flight are recorded, so the set
  7555. stays bounded by the upload pool rather than growing once per cancelled
  7556. item for the life of the process. Skipping the rest loses nothing: an
  7557. item that is not in flight cannot start heating later either, because
  7558. ``_claim_for_dispatch`` only claims rows that are still ``pending`` and
  7559. the caller has already committed a terminal status (or deleted the row)
  7560. before calling this.
  7561. """
  7562. if item_id in self._inflight:
  7563. self._cancelled_dispatches.add(item_id)
  7564. # A print sent for it earlier that never started is not one any more.
  7565. self._forget_unstarted_print(item_id, unless_printing=False)
  7566. async def _preheat_sleep(self, item_id: int, seconds: float) -> bool:
  7567. """Sleep in slices, returning False as soon as the item stops wanting preheat.
  7568. A single long ``asyncio.sleep`` cannot notice a cancellation that lands
  7569. while it is parked, so the wait is chopped into
  7570. ``_PREHEAT_CANCEL_CHECK_SECONDS`` slices with a check after each.
  7571. """
  7572. remaining = float(seconds)
  7573. while remaining > 0:
  7574. slice_secs = min(_PREHEAT_CANCEL_CHECK_SECONDS, remaining)
  7575. await asyncio.sleep(slice_secs)
  7576. remaining -= slice_secs
  7577. if item_id in self._cancelled_dispatches:
  7578. return False
  7579. return True
  7580. def _preheat_flap_to_cooling(self, item_id: int, printer: Printer) -> None:
  7581. """Put the airduct flap back to cooling for a print that wants no chamber heat.
  7582. The full preheat stage does this as part of its own dispatch: an H2D
  7583. left in heating mode by the ABS job before it would otherwise cook the
  7584. PLA that follows. The skip path never reaches that code, so it calls
  7585. this instead -- one idempotent MQTT command, no waiting, and nothing to
  7586. add to the rollback pin, because a flap set to cooling for a print that
  7587. needs no heat is where it should have been either way.
  7588. Best-effort like everything else in the stage: a refused command logs
  7589. and the dispatch carries on.
  7590. """
  7591. model = printer.model or ""
  7592. if not supports_airduct(model):
  7593. return
  7594. state = printer_manager.get_status(printer.id)
  7595. current = getattr(state, "airduct_mode", None) if state else None
  7596. if current == _AIRDUCT_MODE_COOLING:
  7597. return
  7598. client = printer_manager.get_client(printer.id)
  7599. if client is None:
  7600. return
  7601. try:
  7602. client.set_airduct_mode("cooling")
  7603. except Exception as exc:
  7604. logger.warning("Queue item %s: preheat-skip airduct cooling failed: %s", item_id, exc)
  7605. async def _preheat_and_soak(
  7606. self,
  7607. db: AsyncSession,
  7608. item: PrintQueueItem,
  7609. printer: Printer,
  7610. archive: PrintArchive | None,
  7611. ) -> bool:
  7612. """Run the per-printer preheat + heat-soak stage before FTP upload (#1468).
  7613. Returns True when the dispatch should carry on to the upload — including
  7614. every case where preheat is skipped, since a skipped preheat is not a
  7615. reason to abandon the print. Returns False only when the item stopped
  7616. wanting to be printed while the stage was waiting (cancelled or
  7617. deleted); the caller must then abandon the dispatch, and
  7618. ``_dispatch_one``'s rollback shuts the heaters off on the way out.
  7619. Resolution order:
  7620. 1. `item.preheat_override` — 'off' skips entirely; 'inherit' falls back
  7621. to the global `preheat_enabled` setting; 'on' forces the stage on
  7622. even if the global is off.
  7623. 2. Chamber target — `item.preheat_chamber_target_override` if non-null;
  7624. else max of `preheat_filament_targets[normalize(t.tray_type)]`
  7625. across the trays `item.ams_mapping` names (every loaded slot when
  7626. it names none).
  7627. 3. A target of 0 off the filament map skips the whole stage: the
  7628. materials this print loads want no chamber, so there is nothing to
  7629. soak for and the bed phase would only delay the upload (#3041).
  7630. An explicit 0 typed into the per-item override, or a per-item
  7631. 'on', still runs the bed phase and the soak — both are the user
  7632. asking for a warm bed in so many words.
  7633. 4. Three hardware tiers branch the wait loop:
  7634. - Chamber heater (H2C/H2D/H2DPro/H2S/X2D/X1E via supports_chamber_heater):
  7635. send M141 to the resolved target, then wait for the chamber sensor
  7636. to reach it (or the max-wait timeout to elapse).
  7637. - Chamber sensor only (X1C/P2S via supports_chamber_temp ∧ ¬supports_chamber_heater):
  7638. no M141; the bed is the only heat source, so we wait for the chamber
  7639. sensor to rise via bed radiation OR fall through on timeout.
  7640. - No chamber sensor (P1S/P1P/A1/A1 Mini): no way to verify chamber
  7641. temperature; the function just heats the bed and holds for the
  7642. configured soak duration.
  7643. The bed target comes from the archive's parsed metadata
  7644. (`bed_temperature`); if missing the preheat stage logs and returns
  7645. without dispatching anything, rather than guessing at a default that
  7646. might wreck filament setup.
  7647. Failures are logged but never re-raised — preheat is best-effort. A
  7648. printer that goes offline mid-soak, a refused gcode command, or a
  7649. missing temperature reading must not turn into a failed queue item; the
  7650. normal upload + start path runs immediately after this method returns.
  7651. """
  7652. override = (getattr(item, "preheat_override", None) or "inherit").lower()
  7653. if override == "off":
  7654. return True
  7655. if override == "inherit":
  7656. enabled = await self._get_bool_setting(db, "preheat_enabled", default=False)
  7657. if not enabled:
  7658. return True
  7659. # override == "on" forces the stage on regardless of the global setting.
  7660. max_wait = await self._get_int_setting(db, "preheat_max_wait_seconds", default=900)
  7661. soak_seconds = await self._get_int_setting(db, "preheat_soak_seconds", default=300)
  7662. # Chamber target resolution:
  7663. # 1. Explicit per-item override beats everything (user knows best).
  7664. # 2. Otherwise derive from the filament types this print loads, via
  7665. # the per-filament target map. A PLA-only print derives 0 and the
  7666. # block below skips the stage without the user touching anything,
  7667. # even when an ASA spool is sitting in another slot of the same
  7668. # AMS (#2886).
  7669. explicit_target = getattr(item, "preheat_chamber_target_override", None)
  7670. if explicit_target is not None and explicit_target > 0:
  7671. chamber_target = int(explicit_target)
  7672. chamber_source = "item-override"
  7673. elif explicit_target == 0:
  7674. chamber_target = 0 # explicit 0 means "no chamber, even if filament wants it"
  7675. chamber_source = "item-override-zero"
  7676. else:
  7677. targets = await self._get_preheat_filament_targets(db)
  7678. chamber_target = self._derive_chamber_target(printer, targets, item)
  7679. chamber_source = "filament-map"
  7680. # Nothing to preheat *for*. A zero that came out of the filament map is
  7681. # the map saying this print's materials want no chamber conditioning --
  7682. # PLA, PETG, TPU and PVA all sit at 0 by default. Running the stage
  7683. # anyway heated the bed and then held it for the full soak, which
  7684. # delayed every PLA dispatch by minutes and bought nothing: the print's
  7685. # own G-code sets the bed the moment it starts, so preheating it here
  7686. # only moves that heating ahead of the upload instead of overlapping
  7687. # with it, and the soak has no chamber to condition (#3041).
  7688. #
  7689. # An explicit statement from the user still runs the stage. Forcing the
  7690. # per-item override to 'on', or typing a chamber target of exactly 0,
  7691. # both mean "preheat the bed for this print" -- the second is
  7692. # documented as doing precisely that. Only the automatic path, the
  7693. # global toggle plus the filament map, short-circuits here.
  7694. if chamber_target <= 0 and chamber_source == "filament-map" and override != "on":
  7695. logger.info(
  7696. "Queue item %s: preheat skipped -- the loaded filaments derive no chamber "
  7697. "target, so there is nothing to soak for (override=%s model=%s)",
  7698. item.id,
  7699. override,
  7700. printer.model or "",
  7701. )
  7702. self._preheat_flap_to_cooling(item.id, printer)
  7703. return True
  7704. bed_target = await _queued_plate_bed_temperature(item, archive)
  7705. if bed_target <= 0:
  7706. # No bed temperature in the slicer metadata. When the print needs a
  7707. # hot chamber the bed is simply how we heat it, so fall back to the
  7708. # configured chamber-heating bed temperature rather than skipping
  7709. # the whole stage — otherwise the print starts with a cold chamber,
  7710. # which is exactly what preheat exists to prevent. Without a chamber
  7711. # requirement there is nothing to preheat *for*, so skip as before
  7712. # rather than guess a bed temperature for the print itself.
  7713. if chamber_target <= 0:
  7714. logger.info(
  7715. "Queue item %s: preheat skipped — archive has no bed_temperature metadata and no chamber target",
  7716. item.id,
  7717. )
  7718. return True
  7719. bed_target = await self._get_int_setting(db, "queue_keep_warm_bed_temp", default=90)
  7720. logger.info(
  7721. "Queue item %s: archive has no bed_temperature metadata — heating the bed to "
  7722. "%d°C to drive the chamber to %d°C",
  7723. item.id,
  7724. bed_target,
  7725. chamber_target,
  7726. )
  7727. client = printer_manager.get_client(printer.id)
  7728. if client is None:
  7729. logger.warning("Queue item %s: preheat skipped — printer client unavailable", item.id)
  7730. return True
  7731. model = printer.model or ""
  7732. has_heater = supports_chamber_heater(model)
  7733. has_sensor = supports_chamber_temp(model)
  7734. do_chamber = chamber_target > 0 and (has_heater or has_sensor)
  7735. # Fast path: if the chamber has been continuously above target for at
  7736. # least soak_seconds and the bed is already at temperature, skip the
  7737. # entire preheat stage. Typical case: keep-warm held the bed between
  7738. # consecutive same-material prints and the chamber never dropped.
  7739. if do_chamber and has_sensor and soak_seconds > 0:
  7740. remaining = self._chamber_soak_remaining(printer.id, float(chamber_target), soak_seconds)
  7741. if remaining == 0:
  7742. cur = printer_manager.get_status(printer.id)
  7743. if cur:
  7744. cur_temps = cur.temperatures or {}
  7745. if (
  7746. float(cur_temps.get("bed", 0) or 0) >= bed_target - 2.0
  7747. and float(cur_temps.get("chamber", 0) or 0) >= chamber_target - 2.0
  7748. ):
  7749. logger.info(
  7750. "Queue item %s: preheat skipped — chamber has been above %d°C for ≥%ds "
  7751. "and bed is already at temperature (chamber history fast-path)",
  7752. item.id,
  7753. chamber_target,
  7754. soak_seconds,
  7755. )
  7756. # Still set targets to prevent cooling during the 3MF upload window.
  7757. # Register each successful set in the preheat pin so `_dispatch_one`
  7758. # unwinds them on any non-success exit.
  7759. pin = self._preheat_pin.setdefault(printer.id, set())
  7760. try:
  7761. client.set_bed_temperature(bed_target)
  7762. pin.add("bed")
  7763. self._preheat_pin_bed[printer.id] = bed_target
  7764. except Exception as exc:
  7765. logger.warning("Queue item %s: fast-path bed M140 failed: %s", item.id, exc)
  7766. if supports_airduct(model):
  7767. cur_airduct = getattr(cur, "airduct_mode", None)
  7768. if cur_airduct != _AIRDUCT_MODE_HEATING:
  7769. try:
  7770. client.set_airduct_mode("heating")
  7771. # Only undo what we can see we replaced. `None`
  7772. # means no mode has been observed yet, and
  7773. # rolling that back to cooling would assert a
  7774. # state the printer never reported.
  7775. if cur_airduct == _AIRDUCT_MODE_COOLING:
  7776. pin.add("airduct")
  7777. except Exception as exc:
  7778. logger.warning("Queue item %s: fast-path airduct failed: %s", item.id, exc)
  7779. if has_heater:
  7780. try:
  7781. client.set_chamber_temperature(chamber_target)
  7782. pin.add("chamber")
  7783. except Exception as exc:
  7784. logger.warning("Queue item %s: fast-path chamber M141 failed: %s", item.id, exc)
  7785. return True
  7786. logger.info(
  7787. "Queue item %s: preheat starting — bed=%d°C chamber_target=%d°C (source=%s override=%s "
  7788. "model=%s has_heater=%s has_sensor=%s) max_wait=%ds soak=%ds",
  7789. item.id,
  7790. bed_target,
  7791. chamber_target if do_chamber else 0,
  7792. chamber_source,
  7793. override,
  7794. model,
  7795. has_heater,
  7796. has_sensor,
  7797. max_wait,
  7798. soak_seconds,
  7799. )
  7800. # Preheat rollback registry: everything we set below is recorded here so
  7801. # `_dispatch_one`'s finally clause can unwind the whole heating regime
  7802. # (bed off, chamber off, airduct back to cooling) on any non-success
  7803. # exit. Populated as each command succeeds; consumed and cleared by
  7804. # `_dispatch_one`.
  7805. pin = self._preheat_pin.setdefault(printer.id, set())
  7806. # Dispatch heaters. set_bed_temperature / set_chamber_temperature already
  7807. # cache the target locally so the polling reads below see consistent
  7808. # state (firmware MQTT echoes lag by ~1s).
  7809. try:
  7810. client.set_bed_temperature(bed_target)
  7811. pin.add("bed")
  7812. self._preheat_pin_bed[printer.id] = bed_target
  7813. except Exception as exc:
  7814. logger.warning("Queue item %s: preheat bed M140 failed: %s", item.id, exc)
  7815. return True
  7816. # Airduct mode (#1468 follow-up). Models with the cooling/heating flap
  7817. # (H2C/H2D/H2D Pro/H2S/X2D/P2S) keep the flap whatever the user last
  7818. # left it on, regardless of M141. Default cooling actively vents the
  7819. # chamber, so a `chamber_target > 0` print with the flap stuck in
  7820. # cooling never converges — the heater fights the open exhaust. We
  7821. # flip the flap BEFORE M141 to "heating" when the preheat wants
  7822. # chamber heat, and back to "cooling" when it doesn't (PLA-only print
  7823. # on an H2D that was previously running ABS would otherwise stay in
  7824. # heating mode and overheat PLA). The current-state read keeps the
  7825. # command idempotent — no MQTT chatter when the flap is already where
  7826. # we want it.
  7827. if supports_airduct(model):
  7828. desired_airduct = "heating" if chamber_target > 0 else "cooling"
  7829. desired_id = _AIRDUCT_MODE_HEATING if desired_airduct == "heating" else _AIRDUCT_MODE_COOLING
  7830. current_state = printer_manager.get_status(printer.id)
  7831. current_airduct = getattr(current_state, "airduct_mode", None) if current_state else None
  7832. if current_airduct != desired_id:
  7833. try:
  7834. client.set_airduct_mode(desired_airduct)
  7835. # As in the fast path: only pin a rollback for a flap we
  7836. # saw in cooling. `current_airduct` of None means no mode
  7837. # has been observed, so there is nothing to restore to.
  7838. if desired_airduct == "heating" and current_airduct == _AIRDUCT_MODE_COOLING:
  7839. pin.add("airduct")
  7840. except Exception as exc:
  7841. logger.warning(
  7842. "Queue item %s: preheat airduct %s mode failed: %s",
  7843. item.id,
  7844. desired_airduct,
  7845. exc,
  7846. )
  7847. if do_chamber and has_heater:
  7848. try:
  7849. client.set_chamber_temperature(chamber_target)
  7850. pin.add("chamber")
  7851. except Exception as exc:
  7852. logger.warning("Queue item %s: preheat chamber M141 failed: %s", item.id, exc)
  7853. # Release the pooled DB connection before the (potentially many-minute)
  7854. # heat-soak wait below (#2572). Every setting this method needs is read
  7855. # above; the wait/soak loop only polls printer_manager state and sleeps —
  7856. # it never touches the DB. Without this the caller's transaction sat
  7857. # "idle in transaction" for the whole soak, pinning one pooled connection
  7858. # per preheating printer. expire_on_commit=False keeps item/printer
  7859. # readable afterwards; there are no pending writes to lose here.
  7860. await db.commit()
  7861. # Wait for convergence. Bed warm-up is fast (~5 min from cold); chamber
  7862. # via M141 takes a few minutes; chamber via bed radiation can take 20+.
  7863. # Poll every 3s — frequent enough for responsive logging without
  7864. # spamming the MQTT state stream. The "converged" predicate is:
  7865. # bed reached target (within 2°C tolerance for floating-point + heater hysteresis),
  7866. # AND
  7867. # chamber phase satisfied (no chamber phase, no sensor, or sensor reached target).
  7868. BED_TOLERANCE = 2.0
  7869. CHAMBER_TOLERANCE = 2.0
  7870. POLL_INTERVAL = 3.0
  7871. deadline = asyncio.get_event_loop().time() + max_wait
  7872. while True:
  7873. state = printer_manager.get_status(printer.id)
  7874. if state is None:
  7875. logger.warning("Queue item %s: preheat lost state during wait", item.id)
  7876. break
  7877. temps = state.temperatures or {}
  7878. bed_now = float(temps.get("bed", 0) or 0)
  7879. chamber_now = float(temps.get("chamber", 0) or 0)
  7880. bed_ok = bed_now >= bed_target - BED_TOLERANCE
  7881. if not do_chamber:
  7882. chamber_ok = True # phase disabled or model has neither sensor nor heater
  7883. elif not has_sensor:
  7884. chamber_ok = True # P1S etc — can't read, rely on soak timer only
  7885. else:
  7886. chamber_ok = chamber_now >= chamber_target - CHAMBER_TOLERANCE
  7887. if bed_ok and chamber_ok:
  7888. logger.info(
  7889. "Queue item %s: preheat target reached (bed=%.1f chamber=%.1f) — entering soak",
  7890. item.id,
  7891. bed_now,
  7892. chamber_now,
  7893. )
  7894. break
  7895. if asyncio.get_event_loop().time() >= deadline:
  7896. logger.info(
  7897. "Queue item %s: preheat max_wait reached (bed=%.1f/%d chamber=%.1f/%d) — falling through to soak",
  7898. item.id,
  7899. bed_now,
  7900. bed_target,
  7901. chamber_now,
  7902. chamber_target if do_chamber else 0,
  7903. )
  7904. break
  7905. if not await self._preheat_sleep(item.id, POLL_INTERVAL):
  7906. logger.info(
  7907. "Queue item %s: preheat aborted — item cancelled or deleted while waiting for temperature",
  7908. item.id,
  7909. )
  7910. return False
  7911. if soak_seconds > 0:
  7912. if do_chamber and has_sensor:
  7913. remaining = self._chamber_soak_remaining(printer.id, float(chamber_target), soak_seconds)
  7914. else:
  7915. remaining = soak_seconds # no sensor — can't verify history, run full soak
  7916. if remaining > 0:
  7917. logger.info(
  7918. "Queue item %s: preheat soak — holding for %ds (of %ds configured; chamber "
  7919. "has been above target for ~%ds already)",
  7920. item.id,
  7921. remaining,
  7922. soak_seconds,
  7923. soak_seconds - remaining,
  7924. )
  7925. if not await self._preheat_sleep(item.id, remaining):
  7926. logger.info(
  7927. "Queue item %s: preheat aborted — item cancelled or deleted during soak",
  7928. item.id,
  7929. )
  7930. return False
  7931. else:
  7932. logger.info(
  7933. "Queue item %s: preheat soak skipped — chamber has been above %d°C for ≥%ds",
  7934. item.id,
  7935. chamber_target,
  7936. soak_seconds,
  7937. )
  7938. logger.info("Queue item %s: preheat complete — proceeding to upload", item.id)
  7939. return True
  7940. async def _power_on_and_wait(self, plug: SmartPlug, printer_id: int, db: AsyncSession) -> bool:
  7941. """Turn on smart plug and wait for printer to connect.
  7942. Returns True if printer connected successfully within timeout.
  7943. """
  7944. # Get the appropriate service for the plug type (Tasmota or Home Assistant)
  7945. service = await smart_plug_manager.get_service_for_plug(plug, db)
  7946. # Check current plug state
  7947. status = await service.get_status(plug)
  7948. if not status.get("reachable"):
  7949. logger.warning("Smart plug '%s' is not reachable", plug.name)
  7950. return False
  7951. # Turn on if not already on
  7952. if status.get("state") != "ON":
  7953. success = await service.turn_on(plug)
  7954. if not success:
  7955. logger.warning("Failed to turn on smart plug '%s'", plug.name)
  7956. return False
  7957. await publish_plug_state(plug.id, plug.name, plug.printer_id, on=True, ha_entity_id=plug.ha_entity_id)
  7958. logger.info("Powered on smart plug '%s' for printer %s", plug.name, printer_id)
  7959. # Get printer from database for connection
  7960. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  7961. printer = result.scalar_one_or_none()
  7962. if not printer:
  7963. logger.error("Printer %s not found in database", printer_id)
  7964. return False
  7965. # End the pass's transaction before waiting up to minutes for the boot:
  7966. # held open, it pins a pooled connection idle in transaction
  7967. # (PostgreSQL) and a stale read snapshot (SQLite). Nothing below reads
  7968. # through it.
  7969. await db.commit()
  7970. # Wait for printer to boot (give it some time before trying to connect)
  7971. logger.info("Waiting 30s for printer %s to boot...", printer_id)
  7972. await asyncio.sleep(30)
  7973. # Try to connect to the printer periodically
  7974. elapsed = 30 # Already waited 30s
  7975. while elapsed < self._power_on_wait_time:
  7976. # Try to connect
  7977. logger.info("Attempting to connect to printer %s...", printer_id)
  7978. try:
  7979. connected = await printer_manager.connect_printer(printer)
  7980. if connected:
  7981. logger.info("Printer %s connected after %ss", printer_id, elapsed)
  7982. # Give it a moment to stabilize and get status
  7983. await asyncio.sleep(5)
  7984. return True
  7985. except Exception as e:
  7986. logger.debug("Connection attempt failed: %s", e)
  7987. await asyncio.sleep(self._power_on_check_interval)
  7988. elapsed += self._power_on_check_interval
  7989. logger.debug("Waiting for printer %s to connect... (%ss)", printer_id, elapsed)
  7990. logger.warning("Printer %s did not connect within %ss after power on", printer_id, self._power_on_wait_time)
  7991. return False
  7992. async def _check_previous_success(
  7993. self, db: AsyncSession, item: PrintQueueItem, printer_id: int | None = None
  7994. ) -> bool:
  7995. """Check if the previous print on this printer succeeded.
  7996. ``printer_id`` names the printer to look at when the item has none yet
  7997. (a job for any printer of a model, checked against the one offered).
  7998. A user-cancelled predecessor is treated as neutral — `cancelled` is a
  7999. deliberate action, not a failure, so subsequent items should still
  8000. dispatch (#1667). `skipped` is excluded from the lookback entirely:
  8001. a skip isn't an actual print attempt, so it must not gate downstream
  8002. items — counting it as a failed predecessor was the cascade bug that
  8003. let a single cancellation block 18 items over 3 days for the reporter.
  8004. Only `failed` and `aborted` — real print-attempt failures — block.
  8005. Failures with `gate_acknowledged=True` (set by the per-printer Resume
  8006. action — #1818) are also excluded from the lookback so the user can
  8007. clear the gate after fixing the physical issue without having to
  8008. re-queue every downstream job.
  8009. """
  8010. result = await db.execute(
  8011. select(PrintQueueItem)
  8012. .where(PrintQueueItem.printer_id == (printer_id if printer_id is not None else item.printer_id))
  8013. .where(PrintQueueItem.id != item.id)
  8014. .where(PrintQueueItem.status.in_(["completed", "failed", "cancelled", "aborted"]))
  8015. .where(PrintQueueItem.gate_acknowledged == False) # noqa: E712
  8016. # A row ended without a completion time must not count as the
  8017. # latest: PostgreSQL sorts NULL first in a descending order.
  8018. .order_by(PrintQueueItem.completed_at.desc().nulls_last(), PrintQueueItem.id.desc())
  8019. .limit(1)
  8020. )
  8021. prev_item = result.scalar_one_or_none()
  8022. # If no previous item, assume success (first in queue)
  8023. if not prev_item:
  8024. return True
  8025. return prev_item.status in ("completed", "cancelled")
  8026. def _hold_archive(self, item_id: int, archive_id: int) -> None:
  8027. """Hold the archive a dispatch prints into until it ends (archive_work)."""
  8028. from backend.app.services import archive_work
  8029. self._release_held_archive(item_id)
  8030. archive_work.hold(archive_id)
  8031. self._held_archives[item_id] = archive_id
  8032. def _release_held_archive(self, item_id: int) -> None:
  8033. from backend.app.services import archive_work
  8034. if (archive_id := self._held_archives.pop(item_id, None)) is not None:
  8035. archive_work.release(archive_id)
  8036. @staticmethod
  8037. async def _printing_elsewhere(db: AsyncSession, archive_id: int) -> bool:
  8038. """Whether a print of the archive is running (its run open)."""
  8039. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, RUN_OPEN, PrintLogEntry
  8040. return (
  8041. await db.scalar(
  8042. select(PrintLogEntry.id)
  8043. .where(PrintLogEntry.archive_id == archive_id, PrintLogEntry.status == RUN_OPEN)
  8044. .limit(1)
  8045. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  8046. )
  8047. is not None
  8048. )
  8049. @staticmethod
  8050. async def _vp_prefers_filename(db: AsyncSession, library_file: LibraryFile) -> bool:
  8051. """Whether a virtual printer's upload names its archive after the file.
  8052. Settings > Virtual Printer chooses between the file name and the title
  8053. in the 3MF for the archives of uploads; other files keep the default.
  8054. """
  8055. from backend.app.api.routes.settings import get_setting
  8056. from backend.app.services.virtual_printer.library_store import SOURCE_TYPE
  8057. if library_file.source_type != SOURCE_TYPE:
  8058. return False
  8059. return (await get_setting(db, "virtual_printer_archive_name_source")) == "filename"
  8060. async def _claim_reusable_archive(
  8061. self, db: AsyncSession, item: PrintQueueItem, file_path: Path
  8062. ) -> PrintArchive | None:
  8063. """The archive a library dispatch adds its run to, or None for a new one.
  8064. The same file of the same owner, in the same project, archived from
  8065. this library file or from none (one since deleted, or a print started
  8066. elsewhere), with its copy of the file still on disk. Also while it is
  8067. printing, queued or finishing elsewhere: each print of an archive is
  8068. its own run, with its own start, end, charge, energy, outcome,
  8069. timelapse and photos (archive redesign), as for the copies of a batch.
  8070. """
  8071. from backend.app.services.archive import ArchiveService
  8072. # Off the file as it is now: the stored hash is not refreshed when the
  8073. # file changes on disk (an external folder, overwritten in place).
  8074. content_hash = await asyncio.to_thread(ArchiveService.compute_file_hash, file_path)
  8075. owner = (
  8076. PrintArchive.created_by_id.is_(None)
  8077. if item.created_by_id is None
  8078. else PrintArchive.created_by_id == item.created_by_id
  8079. )
  8080. project = (
  8081. PrintArchive.project_id.is_(None) if item.project_id is None else PrintArchive.project_id == item.project_id
  8082. )
  8083. candidates = (
  8084. await db.scalars(
  8085. select(PrintArchive)
  8086. .where(
  8087. PrintArchive.content_hash == content_hash,
  8088. owner,
  8089. project,
  8090. or_(PrintArchive.library_file_id == item.library_file_id, PrintArchive.library_file_id.is_(None)),
  8091. PrintArchive.deleted_at.is_(None),
  8092. )
  8093. .order_by(PrintArchive.id.desc())
  8094. )
  8095. ).all()
  8096. from backend.app.services import archive_work
  8097. for candidate in candidates:
  8098. # Not one a merge is moving into another archive right now; held
  8099. # from here, with no await between, until the dispatch ends.
  8100. if archive_work.is_merging(candidate.id):
  8101. continue
  8102. if candidate.file_path and (settings.base_dir / candidate.file_path).is_file():
  8103. self._hold_archive(item.id, candidate.id)
  8104. return candidate
  8105. return None
  8106. async def _repoint_siblings_at_archive(
  8107. self,
  8108. db: AsyncSession,
  8109. *,
  8110. consumed_library_file_id: int,
  8111. archive_id: int,
  8112. dispatched_item_id: int,
  8113. ) -> int:
  8114. """Move the other queue items off a library row that is about to be deleted (#2819).
  8115. ``cleanup_library_after_dispatch`` consumes the library row: the printer-card
  8116. upload-and-print flow uploads a transient file, prints it, and deletes it.
  8117. Queue creation happily puts that flag on every copy of a ``quantity > 1``
  8118. request, and ``_clone_queue_item`` copies ``library_file_id`` onto batch
  8119. clones, so the first dispatch could pull the file out from under rows that
  8120. had not run yet. What those rows did next depended on the database, and
  8121. neither answer was right: SQLite ships with ``PRAGMA foreign_keys`` off, so
  8122. the ``ON DELETE CASCADE`` on ``print_queue.library_file_id`` never fired and
  8123. they were left pointing at a row that no longer existed, failing with
  8124. "Library file not found" whenever someone started them -- or sitting
  8125. ``pending`` forever under ``manual_start``. PostgreSQL enforces the same
  8126. constraint, so there the rows were deleted outright and the queued copies
  8127. simply vanished, with no error and no history.
  8128. The archive holds its own copy of the 3MF, so the remaining copies can print
  8129. from it instead. Two things happen here, and both must happen *before* the
  8130. delete -- afterwards there is nothing left to repair on PostgreSQL:
  8131. * every row still naming the file has ``library_file_id`` cleared, which is
  8132. what takes it out of the cascade's reach. That covers rows this cannot
  8133. re-point as well -- a copy already printing from its own archive, and the
  8134. finished ones, which are not spare parts but the record a batch order
  8135. counts its progress from.
  8136. * the rows that still need something to print are pointed at the archive.
  8137. Returns how many items were re-pointed.
  8138. """
  8139. variant_item_ids = (
  8140. (
  8141. await db.execute(
  8142. select(PrintQueueVariant.queue_item_id).where(
  8143. PrintQueueVariant.library_file_id == consumed_library_file_id
  8144. )
  8145. )
  8146. )
  8147. .scalars()
  8148. .all()
  8149. )
  8150. # Candidate rows naming the consumed file have to go rather than be
  8151. # cleared: `library_file_id` is NOT NULL there, so there is no way to keep
  8152. # one out of the cascade. This is what PostgreSQL already does, and
  8153. # _candidates_for skips such a variant on SQLite anyway, so no selection
  8154. # outcome changes -- the two backends simply stop disagreeing about
  8155. # whether the row is still there.
  8156. #
  8157. # It also has to happen before the re-point below: a cross-model item
  8158. # (#671) picks a variant every pass and folds it onto the row, and
  8159. # _resolve_variant clears archive_id as it does so, which would undo the
  8160. # re-point on the very next lap.
  8161. await db.execute(delete(PrintQueueVariant).where(PrintQueueVariant.library_file_id == consumed_library_file_id))
  8162. # An item left with other candidates still has somewhere to go, and those
  8163. # carry their own target model -- pointing it at this archive would print a
  8164. # file the matcher never chose. It is excluded from the re-point and simply
  8165. # re-resolves against what is left.
  8166. surviving_variant_item_ids = set(
  8167. (
  8168. await db.execute(
  8169. select(PrintQueueVariant.queue_item_id).where(
  8170. PrintQueueVariant.queue_item_id.in_(variant_item_ids) if variant_item_ids else false()
  8171. )
  8172. )
  8173. )
  8174. .scalars()
  8175. .all()
  8176. )
  8177. repoint_ids = set(
  8178. (
  8179. await db.execute(
  8180. select(PrintQueueItem.id)
  8181. .where(PrintQueueItem.id != dispatched_item_id)
  8182. .where(PrintQueueItem.archive_id.is_(None))
  8183. # "skipped" belongs here with the two live states: it is not
  8184. # terminal. Clearing a printer's previous-success gate puts
  8185. # every item skipped by it back to "pending"
  8186. # (resume_after_failure), and one restored onto a deleted
  8187. # file is the same orphan by a slower route. "failed",
  8188. # "cancelled", "aborted" and "completed" never return.
  8189. .where(PrintQueueItem.status.in_(("pending", "printing", "skipped")))
  8190. .where(
  8191. PrintQueueItem.id.notin_(surviving_variant_item_ids) if surviving_variant_item_ids else true()
  8192. )
  8193. .where(
  8194. or_(
  8195. PrintQueueItem.library_file_id == consumed_library_file_id,
  8196. PrintQueueItem.id.in_(variant_item_ids) if variant_item_ids else false(),
  8197. )
  8198. )
  8199. )
  8200. )
  8201. .scalars()
  8202. .all()
  8203. )
  8204. if repoint_ids:
  8205. await db.execute(
  8206. update(PrintQueueItem)
  8207. .where(PrintQueueItem.id.in_(repoint_ids))
  8208. .values(
  8209. archive_id=archive_id,
  8210. library_file_id=None,
  8211. # The file this flag named is already consumed. Leaving it set
  8212. # would arm every re-pointed copy to delete whatever library
  8213. # row it is next given.
  8214. cleanup_library_after_dispatch=False,
  8215. )
  8216. )
  8217. logger.info(
  8218. "Queue items %s: re-pointed at archive %s -- library file %s was consumed by item %s",
  8219. sorted(repoint_ids),
  8220. archive_id,
  8221. consumed_library_file_id,
  8222. dispatched_item_id,
  8223. )
  8224. # Everything else that still names the file: taken out of the cascade's
  8225. # reach without touching what it prints. The file is gone either way; what
  8226. # this preserves is the row.
  8227. await db.execute(
  8228. update(PrintQueueItem)
  8229. .where(PrintQueueItem.id != dispatched_item_id)
  8230. .where(PrintQueueItem.library_file_id == consumed_library_file_id)
  8231. .values(library_file_id=None)
  8232. )
  8233. return len(repoint_ids)
  8234. async def _power_off_if_needed(self, db: AsyncSession, item: PrintQueueItem):
  8235. """Schedule power-off if the queue item enabled auto_off_after.
  8236. Delegates to the smart-plug manager so the off honours each plug's
  8237. configured strategy (time delay or temperature threshold), is cancelled
  8238. if the printer starts printing again, and never cuts power on a loaded
  8239. print (#1890). Previously this hardcoded a 50°C / 600s cooldown wait and
  8240. powered off on the timeout regardless of print state.
  8241. """
  8242. if not item.auto_off_after:
  8243. return
  8244. try:
  8245. await smart_plug_manager.schedule_off_after_queue_job(item.printer_id, db)
  8246. except Exception as e:
  8247. logger.warning("Auto-off: Failed to schedule power-off for printer %s: %s", item.printer_id, e)
  8248. async def _get_job_name(self, db: AsyncSession, item: PrintQueueItem) -> str:
  8249. """Get a human-readable name for a queue item."""
  8250. if item.archive_id:
  8251. result = await db.execute(select(PrintArchive).where(PrintArchive.id == item.archive_id))
  8252. archive = result.scalar_one_or_none()
  8253. if archive:
  8254. return archive.filename.replace(".gcode.3mf", "").replace(".3mf", "")
  8255. if item.library_file_id:
  8256. result = await db.execute(LibraryFile.active().where(LibraryFile.id == item.library_file_id))
  8257. library_file = result.scalar_one_or_none()
  8258. if library_file:
  8259. return library_file.filename.replace(".gcode.3mf", "").replace(".3mf", "")
  8260. # A cross-model item (#671) holds no file of its own until a printer is
  8261. # picked, so name it after its first candidate — otherwise every waiting
  8262. # notification for one reads "Job #12". Queried rather than read off
  8263. # item.variants because callers outside the selection loop have not
  8264. # eager-loaded them, and a lazy load raises in async.
  8265. first_variant_name = (
  8266. await db.execute(
  8267. select(LibraryFile.filename)
  8268. .join(PrintQueueVariant, PrintQueueVariant.library_file_id == LibraryFile.id)
  8269. .where(PrintQueueVariant.queue_item_id == item.id)
  8270. .order_by(PrintQueueVariant.position, PrintQueueVariant.id)
  8271. .limit(1)
  8272. )
  8273. ).scalar_one_or_none()
  8274. if first_variant_name:
  8275. return first_variant_name.replace(".gcode.3mf", "").replace(".3mf", "")
  8276. return f"Job #{item.id}"
  8277. async def _get_printer(self, db: AsyncSession, printer_id: int) -> Printer | None:
  8278. """Get printer by ID."""
  8279. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  8280. return result.scalar_one_or_none()
  8281. async def _notify_dispatch_gave_up(
  8282. self,
  8283. queue_item_id: int,
  8284. printer_id: int,
  8285. created_by_id: int | None,
  8286. reason: str = "Printer accepted the file but never started printing",
  8287. ) -> None:
  8288. """Tell the user the queue item was failed after exhausting its dispatch retries.
  8289. Called from the watchdog, which is a background task with no session of
  8290. its own — hence the fresh one here. Best-effort throughout: the row is
  8291. already marked failed and that is the load-bearing part; a notification
  8292. provider being down must not resurrect the retry loop we just stopped.
  8293. ``reason`` defaults to the exhausted-retries wording. The command-rejected
  8294. path passes its own, because "accepted the file but never started" is the
  8295. opposite of what happened there — the printer refused it outright (#2732).
  8296. """
  8297. try:
  8298. async with async_session() as db:
  8299. item = await db.get(PrintQueueItem, queue_item_id)
  8300. if not item:
  8301. return
  8302. job_name = await self._get_job_name(db, item)
  8303. printer = await self._get_printer(db, printer_id)
  8304. await notification_service.on_queue_job_failed(
  8305. job_name=job_name,
  8306. printer_id=printer_id,
  8307. printer_name=printer.name if printer else "Unknown",
  8308. reason=reason,
  8309. db=db,
  8310. )
  8311. except Exception as e:
  8312. logger.warning("Queue item %s: give-up notification failed: %s", queue_item_id, e)
  8313. try:
  8314. await ws_manager.send_queue_item_failed(
  8315. user_id=created_by_id,
  8316. queue_item_id=queue_item_id,
  8317. printer_id=printer_id,
  8318. reason="never_started",
  8319. )
  8320. except Exception:
  8321. pass # toast is best-effort
  8322. async def _filament_short_on(
  8323. self,
  8324. db: AsyncSession,
  8325. item: PrintQueueItem,
  8326. printer_id: int,
  8327. *,
  8328. require_known: bool = False,
  8329. ) -> bool:
  8330. """Would ``item`` be held for a filament deficit on ``printer_id`` (#3137)?
  8331. Asks what ``_block_on_filament_deficit`` would answer once the item is
  8332. assigned there, with the mapping the item would get there, but changes
  8333. nothing. The model-based matcher uses it to prefer a printer whose
  8334. spools can finish the job.
  8335. False whenever the gate would let the item through: "Print Anyway",
  8336. no mapping, or a check that fails. With ``require_known`` all three of
  8337. those, and any printed slot whose remaining amount is unknown, count
  8338. as short instead: releasing a held job starts it with nobody asked, so
  8339. it needs the amount on record, not merely nothing against it.
  8340. """
  8341. if item.skip_filament_check:
  8342. return False
  8343. try:
  8344. mapping = await self._compute_ams_mapping_for_printer(db, printer_id, item)
  8345. if not mapping:
  8346. return require_known
  8347. deficit = await compute_deficit_for_queue_item(
  8348. db, item, printer_id=printer_id, ams_mapping=mapping, require_known=require_known
  8349. )
  8350. except Exception as e:
  8351. logger.warning("Filament deficit check failed for item %s on printer %s: %s", item.id, printer_id, e)
  8352. return require_known
  8353. return bool(deficit)
  8354. async def _release_short_model_hold(
  8355. self,
  8356. db: AsyncSession,
  8357. item: PrintQueueItem,
  8358. unavailable: set[int],
  8359. require_plate_clear: bool,
  8360. printer_scope: PrinterScope,
  8361. ) -> bool:
  8362. """Free an "any model" job held for low filament once a printer can run it (#3137).
  8363. The deficit gate holds a model-based job on the printer the matcher
  8364. gave it, because "Print Anyway" needs a printer to print on. When
  8365. another idle printer of the job's model has the filament, the job is
  8366. moved to that printer and its hold lifted, and the fixed-printer
  8367. branch dispatches it in this same pass, through the same checks a
  8368. held job meets when the user presses Start. It is moved rather than
  8369. put back in the pool because the model-based branch would pick again
  8370. without requiring the amounts on record (below), and could pick a
  8371. different printer. Only the deficit gate sets ``filament_short``, so a
  8372. job held for any other reason stays held.
  8373. The printer it is held on is left out: a spool loaded there is started
  8374. with Start, as before, rather than while someone may still be loading
  8375. the next one. Each job is checked at most every
  8376. ``_SHORT_HOLD_RECHECK_SECONDS``, as every check matches its filament
  8377. on each idle printer again.
  8378. Cross-model jobs (#671) stay held: checking a variant writes its file
  8379. onto the item, which would leave a job that is not moved holding
  8380. another variant's file for the printer it waits on.
  8381. A printer qualifies only on a positive finding: every slot the plate
  8382. prints mapped to a spool whose remaining amount is on record and
  8383. enough. A printer whose spools Bambuddy does not track is not one.
  8384. Returns True when the job was moved. A check that fails keeps the hold
  8385. rather than stopping the queue pass.
  8386. """
  8387. if not item.filament_short or item.skip_filament_check or item.printer_id is None:
  8388. return False
  8389. if not item.target_model or item.variants:
  8390. return False
  8391. now = time.monotonic()
  8392. if now - self._short_hold_checked_at.get(item.id, float("-inf")) < _SHORT_HOLD_RECHECK_SECONDS:
  8393. return False
  8394. self._short_hold_checked_at[item.id] = now
  8395. try:
  8396. candidates = _candidates_for(item)
  8397. if not candidates:
  8398. return False
  8399. effective_types, filament_overrides = _filament_constraints(candidates[0])
  8400. async def is_short(pid: int) -> bool:
  8401. return await self._filament_short_on(db, item, pid, require_known=True)
  8402. async def unmatched_on(pid: int) -> list[str]:
  8403. return await self._unmatched_on(db, item, pid)
  8404. match_id, _ = await self._find_idle_printer_for_model(
  8405. db,
  8406. item.target_model,
  8407. unavailable | {item.printer_id},
  8408. effective_types,
  8409. item.target_location,
  8410. filament_overrides=filament_overrides,
  8411. require_plate_clear=require_plate_clear,
  8412. printer_scope=printer_scope,
  8413. is_short=is_short,
  8414. unmatched_on=unmatched_on,
  8415. sliced_nozzle=candidates[0].sliced_nozzle,
  8416. )
  8417. # Every idle printer short comes back as the first of them.
  8418. if match_id is None or await is_short(match_id):
  8419. return False
  8420. except Exception as e:
  8421. logger.warning("Queue item %s: looking for another printer failed: %s", item.id, e)
  8422. return False
  8423. logger.info(
  8424. "Queue item %s: printer %s has the filament it was short of on printer %s — moving it there",
  8425. item.id,
  8426. match_id,
  8427. item.printer_id,
  8428. )
  8429. self._short_hold_checked_at.pop(item.id, None)
  8430. item.printer_id = match_id
  8431. # Resolved against the printer it leaves (#3239): the tray mapping and
  8432. # the nozzle positions.
  8433. drop_printer_resolved(item)
  8434. item.manual_start = False
  8435. item.filament_short = False
  8436. item.waiting_reason = None
  8437. await db.commit()
  8438. try:
  8439. printer = await self._get_printer(db, match_id)
  8440. await notification_service.on_queue_job_assigned(
  8441. job_name=await self._get_job_name(db, item),
  8442. printer_id=match_id,
  8443. printer_name=printer.name if printer else "Unknown",
  8444. target_model=item.target_model,
  8445. db=db,
  8446. )
  8447. except Exception as e:
  8448. logger.debug("Assignment notification failed for item %s: %s", item.id, e)
  8449. return True
  8450. async def _block_on_filament_deficit(
  8451. self,
  8452. db: AsyncSession,
  8453. item: PrintQueueItem,
  8454. ) -> bool:
  8455. """Promote the item to manual_start when the assigned spool is short (#1496).
  8456. Returns True when this dispatch attempt was blocked, False when the
  8457. item is clear to start. A previously-flagged item whose spool has
  8458. since been swapped to one with enough material clears the flag here
  8459. so the next scheduler tick dispatches it.
  8460. """
  8461. # User has explicitly acknowledged the deficit ("Print Anyway") —
  8462. # don't re-flag, don't even compute. Without this short-circuit the
  8463. # scheduler bounces between "user said anyway" (route clears
  8464. # manual_start) and "scheduler re-blocked" (this method re-flags it
  8465. # on identical spool state) (#1698-followup).
  8466. if item.skip_filament_check:
  8467. # #1762 diagnostic: surface the short-circuit at INFO so a
  8468. # future "Print Anyway didn't work" report (e.g. issue #1762
  8469. # comment 3) has actionable evidence in the support bundle
  8470. # without needing DEBUG enabled.
  8471. logger.info(
  8472. "Queue item %s honouring user's Print Anyway acknowledgement — skipping deficit check",
  8473. item.id,
  8474. )
  8475. return False
  8476. try:
  8477. deficit = await compute_deficit_for_queue_item(db, item)
  8478. except Exception as e:
  8479. # Never let a flaky deficit check wedge the queue — log and let
  8480. # dispatch proceed. The PrintModal-side check still runs on the
  8481. # manual paths.
  8482. logger.warning("Filament deficit check failed for item %s: %s", item.id, e)
  8483. return False
  8484. if deficit:
  8485. item.filament_short = True
  8486. item.manual_start = True
  8487. await db.commit()
  8488. job_name = await self._get_job_name(db, item)
  8489. printer = await self._get_printer(db, item.printer_id) if item.printer_id else None
  8490. logger.info(
  8491. "Queue item %s blocked on filament deficit (%d slot(s)) — promoted to manual_start",
  8492. item.id,
  8493. len(deficit),
  8494. )
  8495. try:
  8496. await notification_service.on_queue_job_waiting(
  8497. job_name=job_name,
  8498. target_model=(printer.model if printer else "") or "",
  8499. waiting_reason="filament_short",
  8500. db=db,
  8501. printer_id=item.printer_id,
  8502. )
  8503. except Exception as e:
  8504. logger.debug("filament_short notification failed for item %s: %s", item.id, e)
  8505. return True
  8506. # No deficit — clear any stale flag from a previous tick.
  8507. if item.filament_short:
  8508. item.filament_short = False
  8509. await db.commit()
  8510. return False
  8511. def _find_color_substitutions(self, required: list[dict], mapping: list, loaded: list[dict]) -> list[dict]:
  8512. """The slots ``mapping`` gives a spool of another colour than they ask for (#3279).
  8513. Compared the way the matcher compares, so the same hex or a similar one
  8514. is a match. Only slots the plate prints, mapped to a tray the printer
  8515. reports, with a colour on both sides: anything else is not a colour the
  8516. job printed in.
  8517. """
  8518. by_tray = {f["global_tray_id"]: f for f in loaded}
  8519. found = []
  8520. for req in required:
  8521. slot_id = req.get("slot_id") or 0
  8522. if slot_id <= 0 or slot_id > len(mapping):
  8523. continue
  8524. tray_id = mapping[slot_id - 1]
  8525. tray = by_tray.get(tray_id) if _is_tray_id(tray_id) else None
  8526. if tray is not None and tray.get("color_known") is False:
  8527. # A spool without a colour set is not a different colour.
  8528. continue
  8529. wanted = self._normalize_color_for_compare(req.get("color"))
  8530. got = self._normalize_color_for_compare(tray.get("color")) if tray else ""
  8531. if not wanted or not got or wanted == got or self._colors_are_similar(wanted, got):
  8532. continue
  8533. found.append(
  8534. {
  8535. "slot_id": slot_id,
  8536. "type": req.get("type") or "",
  8537. "wanted": f"#{wanted.upper()}",
  8538. "got": f"#{got.upper()}",
  8539. "tray": tray_id,
  8540. }
  8541. )
  8542. return found
  8543. async def _color_substitutions_for(self, db: AsyncSession, item: PrintQueueItem) -> str | None:
  8544. """``item``'s colour substitutions as stored on the row, or None (#3279).
  8545. Read from the mapping the job is about to be sent with, whoever made
  8546. it: the matcher's same-material fallback, a preset match in another
  8547. colour, or a slot picked by hand. The file's colour is the one after
  8548. overrides, so a colour swapped on purpose is not one. None whenever
  8549. there is nothing to compare, and on any failure: a missing note must
  8550. never stop a print.
  8551. """
  8552. try:
  8553. mapping = json.loads(item.ams_mapping) if item.ams_mapping else None
  8554. if not isinstance(mapping, list) or not item.printer_id:
  8555. return None
  8556. status = printer_manager.get_status(item.printer_id)
  8557. if status is None:
  8558. return None
  8559. required = await self._get_filament_requirements(db, item)
  8560. if not required:
  8561. return None
  8562. self._apply_filament_overrides(item, required)
  8563. found = self._find_color_substitutions(required, mapping, self._build_loaded_filaments(status))
  8564. except Exception as e:
  8565. logger.debug("Queue item %s: colour comparison failed: %s", item.id, e)
  8566. return None
  8567. if found:
  8568. logger.info("Queue item %s: sent with another colour than the file asks for: %s", item.id, found)
  8569. return json.dumps(found) if found else None
  8570. async def _unmatched_requirements(
  8571. self, db: AsyncSession, item: PrintQueueItem, printer_id: int, mapping: list | None
  8572. ) -> list[dict]:
  8573. """The requirements ``mapping`` leaves without a tray on ``printer_id`` (#2799).
  8574. ``None`` is a mapping that resolved nothing. Only a printer that reported
  8575. loaded trays makes that a finding — see ``_block_on_unmatched_filament``
  8576. on why an empty list is not one.
  8577. """
  8578. if mapping is None:
  8579. status = printer_manager.get_status(printer_id)
  8580. if status is None or not self._build_loaded_filaments(status):
  8581. return []
  8582. # Every required slot reads unresolved against an empty mapping.
  8583. mapping = []
  8584. required = await self._get_filament_requirements(db, item)
  8585. if required:
  8586. self._apply_filament_overrides(item, required)
  8587. else:
  8588. # An unreadable 3MF still has its forced colours to go by: they are
  8589. # what the mapping was built from (#3280).
  8590. required = _forced_override_requirements(item)
  8591. return _unresolved_required(required, mapping) if required else []
  8592. async def _unmatched_on(self, db: AsyncSession, item: PrintQueueItem, printer_id: int) -> list[str]:
  8593. """What ``item`` would be held for on ``printer_id``, as the queue row names it.
  8594. Asks the matcher and the unmatched-filament hold, but changes nothing.
  8595. Choosing a printer for an "any model" job only checked that each type
  8596. and forced colour was loaded somewhere, which a dual-nozzle printer with
  8597. the spool on the other nozzle, or one spool asked for twice, passes. The
  8598. hold then gave the printer back, and the next pass chose it again.
  8599. Empty when the hold would let the item through, including "Print
  8600. Anyway" and a match that fails: the printer is then chosen as before.
  8601. Not "Print Anyway" for a creator whose filaments are restricted, on a
  8602. printer holding a filament they may not use (#3522): dispatch holds
  8603. that job where it can't map it (``_dispatch_scope_violation``), so the
  8604. printer must not be chosen.
  8605. """
  8606. if item.skip_filament_check:
  8607. scope = await self._creator_filament_scope(db, item.created_by_id)
  8608. if scope.is_unrestricted:
  8609. return []
  8610. status = printer_manager.get_status(printer_id)
  8611. if status is None or not scope.blocked_tray_ids(self._build_loaded_filaments(status)):
  8612. return []
  8613. key = (item.id, item.archive_id, item.library_file_id, item.plate_id, item.filament_overrides, printer_id)
  8614. if key in self._unmatched_memo:
  8615. return list(self._unmatched_memo[key])
  8616. try:
  8617. mapping = await self._compute_ams_mapping_for_printer(db, printer_id, item)
  8618. unmatched = await self._unmatched_requirements(db, item, printer_id, mapping)
  8619. except Exception as e:
  8620. logger.warning("Filament match check failed for item %s on printer %s: %s", item.id, printer_id, e)
  8621. return []
  8622. described = [_describe_filament(req, "nozzle_id") for req in unmatched]
  8623. self._unmatched_memo[key] = described
  8624. return list(described)
  8625. async def _block_on_unmatched_filament(
  8626. self,
  8627. db: AsyncSession,
  8628. item: PrintQueueItem,
  8629. release_assignment: bool = False,
  8630. ) -> bool:
  8631. """Promote to manual_start when a slot the plate prints has no tray (#2799).
  8632. The matcher leaves a requirement it cannot satisfy at ``-1`` and dispatch
  8633. goes ahead, letting the printer choose — which is how a job prints in the
  8634. wrong material without anyone being asked. Holding is the same answer the
  8635. deficit gate already gives for "the spool is too light", so it reuses the
  8636. same promote-and-notify machinery.
  8637. Keyed off unresolved slots in the *computed* mapping intersected with the
  8638. plate's own requirement list. Intersecting is what makes ``-1`` safe to
  8639. read: on its own it also pads slots this plate does not print, so the
  8640. array alone would hold perfectly good jobs. Reading the mapping rather
  8641. than the printer's loaded filament types also inherits the matcher's
  8642. per-nozzle restriction for free — a dual-nozzle printer carrying the
  8643. filament on the other nozzle's AMS has it "loaded" but unusable, and a
  8644. type-only scan would wave that through.
  8645. A mapping that resolved *nothing* is the same finding arriving as an
  8646. absence. ``_ensure_ams_mapping`` clears a rejected mapping its recompute
  8647. could not replace, and dispatch then goes out as ``use_ams`` with no
  8648. table at all — several megabytes uploaded for an 0700_8012 rejection.
  8649. That is held too, but only once live status positively reports loaded
  8650. trays: #2589's mapping is bogus precisely because the AMS was not known
  8651. yet, and its empty loaded list is that ignorance rather than a miss.
  8652. ``release_assignment`` is for the model-based path, which commits its
  8653. printer choice before the gates run. Holding an "any P2S" job would
  8654. otherwise pin it to the one P2S that could not run it.
  8655. Returns True when this dispatch attempt was blocked.
  8656. """
  8657. if item.skip_filament_check or not item.printer_id:
  8658. return False
  8659. mapping: list | None = None
  8660. if item.ams_mapping:
  8661. try:
  8662. mapping = json.loads(item.ams_mapping)
  8663. except (json.JSONDecodeError, TypeError):
  8664. return False
  8665. if not isinstance(mapping, list):
  8666. return False
  8667. unmatched = await self._unmatched_requirements(db, item, item.printer_id, mapping)
  8668. if not unmatched:
  8669. # No cleanup needed here: once start is pressed the item is no
  8670. # longer staged, and whichever exit runs next overwrites the reason
  8671. # (`hold_item` on the fixed-printer branch, the assignment on the
  8672. # model-based one), as does dispatch.
  8673. return False
  8674. wanted = ", ".join(_describe_filament(r, "nozzle_id") for r in unmatched)
  8675. held_by = item.printer_id
  8676. item.manual_start = True
  8677. # Human-readable: this renders on the queue row.
  8678. item.waiting_reason = f"{_UNMATCHED_HOLD_PREFIX}{wanted}"
  8679. if release_assignment:
  8680. # "Any P2S" means any, so the job returns to the pool rather than
  8681. # waiting on the printer that could not take it. The mapping goes
  8682. # with the assignment: its tray IDs were resolved against the
  8683. # printer being released and mean nothing on the next one (#2799),
  8684. # and keeping them would hold the job again even on a printer that
  8685. # has the filament, since an unresolved slot is deliberately not a
  8686. # conflict worth recomputing over.
  8687. item.printer_id = None
  8688. drop_printer_resolved(item)
  8689. await db.commit()
  8690. job_name = await self._get_job_name(db, item)
  8691. printer = await self._get_printer(db, held_by)
  8692. logger.info(
  8693. "Queue item %s blocked — printer %s has nothing loaded for %s; promoted to manual_start",
  8694. item.id,
  8695. held_by,
  8696. wanted,
  8697. )
  8698. try:
  8699. await notification_service.on_queue_job_waiting(
  8700. job_name=job_name,
  8701. target_model=(printer.model if printer else "") or "",
  8702. waiting_reason=f"needs {wanted}",
  8703. db=db,
  8704. # Released to the pool: no printer holds it any more.
  8705. printer_id=None if release_assignment else held_by,
  8706. )
  8707. except Exception as e:
  8708. logger.debug("filament_missing notification failed for item %s: %s", item.id, e)
  8709. return True
  8710. @staticmethod
  8711. async def _queue_owner_id(db: AsyncSession, item: PrintQueueItem) -> int | None:
  8712. """The queue item's owner as the row has it now.
  8713. item.created_by_id was read before preheat and upload, which can take
  8714. minutes; deleting the user clears the row, not this session's copy.
  8715. """
  8716. return await db.scalar(select(PrintQueueItem.created_by_id).where(PrintQueueItem.id == item.id))
  8717. async def _propagate_owner_to_printer_manager(self, db: AsyncSession, item: PrintQueueItem) -> None:
  8718. """Hand the queue item's owner to printer_manager so the
  8719. print-complete callback can credit the user in PrintLogEntry (#1670).
  8720. No-ops when the item has no `created_by_id` or the referenced user
  8721. row is missing (e.g. user deleted between queue-add and dispatch —
  8722. in that case the print log row falls back to the existing un-credited
  8723. behaviour rather than crashing the dispatch).
  8724. """
  8725. from backend.app.models.user import User
  8726. # Read from the row, not item.created_by_id or db.get: both can still
  8727. # hold a user deleted during the upload, after delete_user forgot them
  8728. # (on SQLite the id may be someone new's by the time the print ends).
  8729. # Joined with the printer too: one deleted during the upload was
  8730. # forgotten as well, and this would write the owner back under its id.
  8731. owner = (
  8732. await db.execute(
  8733. select(User.id, User.username)
  8734. .join(PrintQueueItem, PrintQueueItem.created_by_id == User.id)
  8735. .join(Printer, Printer.id == PrintQueueItem.printer_id)
  8736. .where(PrintQueueItem.id == item.id)
  8737. )
  8738. ).first()
  8739. if owner:
  8740. printer_manager.set_current_print_user(item.printer_id, owner.id, owner.username, queue_item_id=item.id)
  8741. if item.printer_id is not None:
  8742. self._unconfirmed_print_user[item.id] = item.printer_id
  8743. async def _fail_dispatch(self, db: AsyncSession, item: PrintQueueItem, message: str) -> bool:
  8744. """Mark a dispatching item failed, unless the user cancelled it meanwhile.
  8745. A cancel lands in another session while the upload runs (the edit route
  8746. even tells users to cancel a job that is being sent). An unconditional
  8747. write here would turn that cancellation into a failure, with a failure
  8748. notification, and block the next job that needs the previous print to
  8749. have succeeded. Returns whether the item was marked failed.
  8750. """
  8751. result = cast(
  8752. CursorResult,
  8753. await db.execute(
  8754. update(PrintQueueItem)
  8755. .where(PrintQueueItem.id == item.id)
  8756. .where(PrintQueueItem.status.in_(("pending", "printing")))
  8757. # Failed before the print started: no slot printed in any colour (#3279).
  8758. .values(
  8759. status="failed",
  8760. error_message=message,
  8761. completed_at=datetime.now(timezone.utc),
  8762. color_substitutions=None,
  8763. )
  8764. .execution_options(synchronize_session=False)
  8765. ),
  8766. )
  8767. await db.commit()
  8768. if result.rowcount == 0:
  8769. # Cancelled, or deleted, meanwhile: the row is not refreshed, it may be gone.
  8770. logger.info("Queue item %s: not marked failed (%s), it was cancelled meanwhile", item.id, message)
  8771. return False
  8772. await db.refresh(item)
  8773. from backend.app.services.print_batch import refresh_batches_after
  8774. await refresh_batches_after([item.id])
  8775. return True
  8776. @staticmethod
  8777. async def _defer_until_printer_idle(db: AsyncSession, item: PrintQueueItem) -> None:
  8778. """Put a job whose printer turned busy before the start back in the queue.
  8779. Only if it is still printing: a stop that landed meanwhile stands.
  8780. """
  8781. await db.execute(
  8782. update(PrintQueueItem)
  8783. .where(PrintQueueItem.id == item.id)
  8784. .where(PrintQueueItem.status == "printing")
  8785. .values(status="pending", started_at=None, color_substitutions=None)
  8786. .execution_options(synchronize_session=False)
  8787. )
  8788. await db.commit()
  8789. @staticmethod
  8790. async def _close_cancelled_dispatch_toast(item: PrintQueueItem, toast_uid: int | None) -> None:
  8791. """Close the dispatch toast of an item the user cancelled while it was sent."""
  8792. try:
  8793. await ws_manager.send_queue_item_failed(
  8794. user_id=toast_uid,
  8795. queue_item_id=item.id,
  8796. printer_id=item.printer_id,
  8797. reason="cancelled_mid_dispatch",
  8798. )
  8799. except Exception:
  8800. pass # toast is best-effort
  8801. async def _start_print(self, db: AsyncSession, item: PrintQueueItem):
  8802. """Upload file and start print for a queue item.
  8803. Supports two sources:
  8804. - archive_id: Print from an existing archive
  8805. - library_file_id: Print from a library file (file manager)
  8806. """
  8807. logger.info("Starting queue item %s", item.id)
  8808. # Also covers a reservation left active by a process interruption
  8809. # during an earlier attempt. `_dispatch_one` releases this marker on
  8810. # every exit unless start_print() confirms that the command was sent.
  8811. self._unconfirmed_budget_reservations.add(item.id)
  8812. try:
  8813. from backend.app.models.user import User
  8814. queue_user = await db.get(User, item.created_by_id) if item.created_by_id is not None else None
  8815. # Recompute at the final authorization boundary as well as enqueue
  8816. # time. This covers rows created before the server-side estimate
  8817. # migration and prevents any alternate write path from weakening
  8818. # the budget reservation.
  8819. archive = await db.get(PrintArchive, item.archive_id) if item.archive_id is not None else None
  8820. library_file = await db.get(LibraryFile, item.library_file_id) if item.library_file_id is not None else None
  8821. item.estimated_cost = await estimate_queue_source_cost(
  8822. db,
  8823. archive=archive,
  8824. library_file=library_file,
  8825. plate_id=item.plate_id,
  8826. ams_mapping=item.ams_mapping,
  8827. printer_id=item.printer_id,
  8828. )
  8829. await validate_print_budget(
  8830. db,
  8831. cost_center_id=item.cost_center_id,
  8832. estimated_cost=item.estimated_cost,
  8833. current_user=queue_user,
  8834. exclude_queue_item_id=item.id,
  8835. exclude_reservation_source_type="print_queue",
  8836. exclude_reservation_source_id=item.id,
  8837. )
  8838. budget_reservation = await create_budget_reservation(
  8839. db,
  8840. cost_center_id=item.cost_center_id,
  8841. estimated_cost=item.estimated_cost,
  8842. current_user=queue_user,
  8843. source_type="print_queue",
  8844. source_id=item.id,
  8845. print_archive_id=item.archive_id,
  8846. exclude_queue_item_id=item.id,
  8847. )
  8848. await db.commit()
  8849. except HTTPException as exc:
  8850. if not await self._fail_dispatch(db, item, str(exc.detail)):
  8851. return
  8852. logger.error("Queue item %s: Budget check failed: %s", item.id, item.error_message)
  8853. await self._power_off_if_needed(db, item)
  8854. return
  8855. # Get printer first (needed for both paths)
  8856. result = await db.execute(select(Printer).where(Printer.id == item.printer_id))
  8857. printer = result.scalar_one_or_none()
  8858. if not printer:
  8859. if not await self._fail_dispatch(db, item, "Printer not found"):
  8860. return
  8861. logger.error("Queue item %s: Printer %s not found", item.id, item.printer_id)
  8862. await self._power_off_if_needed(db, item)
  8863. return
  8864. # Check printer is connected
  8865. if not printer_manager.is_connected(item.printer_id):
  8866. if not await self._fail_dispatch(db, item, "Printer not connected"):
  8867. return
  8868. logger.error("Queue item %s: Printer %s not connected", item.id, item.printer_id)
  8869. await self._power_off_if_needed(db, item)
  8870. return
  8871. # Cancel-while-dispatching race (#1853): the scheduler's snapshot of
  8872. # `items` was taken at the top of check_queue, but the user can /cancel
  8873. # any pending row in the gap before we reach this point. Re-read the
  8874. # row and bail out cleanly instead of starting an FTP upload for a row
  8875. # that's already cancelled. The atomic CAS at the pending→printing
  8876. # transition (below, before start_print) is the load-bearing guard;
  8877. # this is the early-exit optimisation that avoids wasted FTP I/O.
  8878. await db.refresh(item)
  8879. if item.status != "pending":
  8880. logger.info(
  8881. "Queue item %s no longer pending (status=%s) — aborting dispatch",
  8882. item.id,
  8883. item.status,
  8884. )
  8885. return
  8886. # Busy-printer guard (#2598). check_queue gates dispatch on
  8887. # _is_printer_idle(), but that treats FINISH as idle and a printer can
  8888. # keep reporting FINISH for tens of seconds *after* it accepted a
  8889. # project_file (see the watchdog's phase-B note). A watchdog revert
  8890. # (#2555) also releases the dispatch hold, so a re-selected item can
  8891. # reach here while its printer has actually started printing. Uploading
  8892. # and dispatching then collides with the live job — the firmware answers
  8893. # 0500_4004 and, on an A1 mini, cancels the running print. Re-check the
  8894. # live state right before the expensive FTP upload: if the printer is
  8895. # busy, leave the item pending and let a later tick dispatch it once the
  8896. # printer is genuinely idle. No wasted upload, no collision.
  8897. pre_dispatch_state = getattr(printer_manager.get_status(item.printer_id), "state", None)
  8898. if pre_dispatch_state in _ACTIVE_PRINT_STATES:
  8899. logger.info(
  8900. "Queue item %s: printer %s is busy (state=%s) — deferring dispatch, "
  8901. "leaving item pending for a later tick (#2598)",
  8902. item.id,
  8903. item.printer_id,
  8904. pre_dispatch_state,
  8905. )
  8906. return
  8907. # Determine source: archive or library file
  8908. archive = None
  8909. library_file = None
  8910. file_path = None
  8911. filename = None
  8912. cleanup_disk_paths: list[Path] = []
  8913. # Set when a dispatch consumes its library file, so the photos can be
  8914. # carried over after the commit that removes the row (#3077).
  8915. consumed_library_file_id: int | None = None
  8916. consumed_photos: list[str] = []
  8917. if item.archive_id:
  8918. # Print from archive
  8919. result = await db.execute(select(PrintArchive).where(PrintArchive.id == item.archive_id))
  8920. archive = result.scalar_one_or_none()
  8921. if archive is not None and archive.deleted_at is not None:
  8922. # Merged into another archive since it was queued (archive
  8923. # redesign P2): print that one, whose file it now holds.
  8924. from backend.app.services.archive_merge import resolve_alias
  8925. if (into := await resolve_alias(db, archive.id)) is not None:
  8926. archive = await db.get(PrintArchive, into)
  8927. if archive is not None:
  8928. item.archive_id = archive.id
  8929. if not archive:
  8930. if not await self._fail_dispatch(db, item, "Archive not found"):
  8931. return
  8932. logger.error("Queue item %s: Archive %s not found", item.id, item.archive_id)
  8933. await self._power_off_if_needed(db, item)
  8934. return
  8935. # Persist the queue item's selected plate onto the archive so Print
  8936. # History can show the actual plate after cancel/fail/complete (#2603).
  8937. # The archive shows its latest run's plate; each run keeps its own
  8938. # on its print log entry.
  8939. if item.plate_id is not None:
  8940. archive.plate_id = item.plate_id
  8941. # Ask-for-outcome opt-in rides from the queue item to the archive
  8942. # the same way (#1898); never cleared here so a reprint of an
  8943. # archive that already asked keeps asking.
  8944. if item.confirm_outcome:
  8945. archive.confirm_requested = True
  8946. file_path = settings.base_dir / archive.file_path
  8947. filename = archive.filename
  8948. elif item.library_file_id:
  8949. # Print from library file (file manager)
  8950. result = await db.execute(LibraryFile.active().where(LibraryFile.id == item.library_file_id))
  8951. library_file = result.scalar_one_or_none()
  8952. if not library_file:
  8953. # "Not found" covers two different situations and only one of
  8954. # them is recoverable, so say which. A trashed file is still
  8955. # there and restoring it makes a re-queued job work; a file that
  8956. # is really gone needs a different one. Neither is knowable from
  8957. # the queue, which is the whole complaint about this message.
  8958. trashed = (
  8959. await db.execute(select(LibraryFile.filename).where(LibraryFile.id == item.library_file_id))
  8960. ).scalar_one_or_none()
  8961. if not await self._fail_dispatch(
  8962. db,
  8963. item,
  8964. (
  8965. f"'{trashed}' is in the library trash — restore it and queue the print again"
  8966. if trashed
  8967. else "Library file not found — it was deleted after this job was queued"
  8968. ),
  8969. ):
  8970. return
  8971. logger.error(
  8972. "Queue item %s: library file %s is %s",
  8973. item.id,
  8974. item.library_file_id,
  8975. "in the trash" if trashed else "gone",
  8976. )
  8977. await self._power_off_if_needed(db, item)
  8978. return
  8979. # Library files store absolute paths
  8980. lib_path = Path(library_file.file_path)
  8981. file_path = lib_path if lib_path.is_absolute() else settings.base_dir / library_file.file_path
  8982. filename = library_file.filename
  8983. # Create archive from library file so usage tracking has access to the 3MF
  8984. queue_item_id = item.id
  8985. # Held separately: a cleanup dispatch clears item.library_file_id
  8986. # below, and the log line at the end of this block reported that
  8987. # cleared field -- so every consumed print logged "from library
  8988. # file None", which is the one case worth being able to trace.
  8989. source_library_file_id = item.library_file_id
  8990. reused_archive = False
  8991. try:
  8992. from backend.app.services.archive import ArchiveService
  8993. # The file printed before: this run joins its archive rather
  8994. # than archiving the same file a second time.
  8995. archive = await self._claim_reusable_archive(db, item, file_path)
  8996. if archive is not None:
  8997. reused_archive = True
  8998. if archive.library_file_id is None:
  8999. archive.library_file_id = item.library_file_id
  9000. # This run's, as a new archive would have them: the
  9001. # archive shows its latest run's plate, and a print
  9002. # without a queue item falls back to the archive's plate
  9003. # and cost centre. Not while another print of it runs: they
  9004. # are that print's (this one's run has its own, from the
  9005. # dispatch), and the archive takes the latest run's plate
  9006. # once none prints (print_log.mirror_latest_run).
  9007. if not await self._printing_elsewhere(db, archive.id):
  9008. if item.plate_id is not None:
  9009. archive.plate_id = item.plate_id
  9010. archive.cost_center_id = item.cost_center_id
  9011. if (slicer_pick := slicer_ams_pick.unpack(item.slicer_ams_mapping)) is not None:
  9012. archive.extra_data = {
  9013. **(archive.extra_data or {}),
  9014. "slicer_ams_mapping": {"mapping": slicer_pick[0], "printer_id": slicer_pick[1]},
  9015. }
  9016. else:
  9017. # A virtual printer's upload: its name setting, and the
  9018. # slicer's AMS pick kept for reprints (#2700). Passed only
  9019. # when set, as for every other file nothing changes.
  9020. vp_extras: dict = {}
  9021. if await self._vp_prefers_filename(db, library_file):
  9022. vp_extras["prefer_filename_for_name"] = True
  9023. if (slicer_pick := slicer_ams_pick.unpack(item.slicer_ams_mapping)) is not None:
  9024. # The printer the slicer resolved it on, not the
  9025. # job's: the job may have been moved since.
  9026. vp_extras["slicer_ams_mapping"], vp_extras["slicer_ams_mapping_printer_id"] = slicer_pick
  9027. archive_service = ArchiveService(db)
  9028. archive = await archive_service.archive_print(
  9029. printer_id=item.printer_id,
  9030. source_file=file_path,
  9031. original_filename=filename,
  9032. created_by_id=item.created_by_id,
  9033. project_id=item.project_id,
  9034. cost_center_id=item.cost_center_id,
  9035. library_file_id=item.library_file_id, # per-file project progress (#1897)
  9036. plate_id=item.plate_id, # selected plate → Print History (#2603)
  9037. **vp_extras,
  9038. )
  9039. if archive:
  9040. self._hold_archive(item.id, archive.id)
  9041. item.archive_id = archive.id
  9042. if item.confirm_outcome:
  9043. archive.confirm_requested = True # ask-for-outcome opt-in (#1898)
  9044. if budget_reservation is not None:
  9045. budget_reservation.print_archive_id = archive.id
  9046. if item.cleanup_library_after_dispatch and not library_file.is_external:
  9047. consumed_library_file_id = library_file.id
  9048. item.library_file_id = None
  9049. cleanup_disk_paths.append(file_path)
  9050. if library_file.thumbnail_path:
  9051. thumb_path = Path(library_file.thumbnail_path)
  9052. if not thumb_path.is_absolute():
  9053. thumb_path = settings.base_dir / library_file.thumbnail_path
  9054. cleanup_disk_paths.append(thumb_path)
  9055. # Before the delete, not after: on PostgreSQL the FK
  9056. # cascade would already have taken these rows (#2819).
  9057. await self._repoint_siblings_at_archive(
  9058. db,
  9059. consumed_library_file_id=consumed_library_file_id,
  9060. archive_id=archive.id,
  9061. dispatched_item_id=item.id,
  9062. )
  9063. # Read while the row is still here; the photos move
  9064. # below, once the delete has actually committed.
  9065. consumed_photos = list(library_file.photos or [])
  9066. # The delete clears the link on the earlier archives of
  9067. # this file (ON DELETE SET NULL; core/sqlite_fk on
  9068. # SQLite). This one is loaded, so set it here too, or it
  9069. # keeps the stale id in the session.
  9070. archive.library_file_id = None
  9071. await db.delete(library_file)
  9072. file_path = settings.base_dir / archive.file_path
  9073. filename = archive.filename
  9074. # Commit, not flush — flush opens the SQLite write
  9075. # transaction (item.archive_id update + library_file
  9076. # delete) and would hold the WAL writer lock through the
  9077. # FTP upload below, causing "database is locked" cascades
  9078. # for sensor history + concurrent cancels (#1853).
  9079. await db.commit()
  9080. logger.info(
  9081. "Queue item %s: %s archive %s from library file %s",
  9082. item.id,
  9083. "Reused" if reused_archive else "Created",
  9084. archive.id,
  9085. source_library_file_id,
  9086. )
  9087. except Exception as e:
  9088. logger.warning(
  9089. "Queue item %s: Failed to create archive from library file: %s",
  9090. queue_item_id,
  9091. e,
  9092. exc_info=True,
  9093. )
  9094. await db.rollback()
  9095. item = await db.get(PrintQueueItem, queue_item_id)
  9096. if item:
  9097. if not await self._fail_dispatch(db, item, "Failed to create archive from library file"):
  9098. return
  9099. await self._power_off_if_needed(db, item)
  9100. return
  9101. if not archive:
  9102. if not await self._fail_dispatch(db, item, "Failed to create archive from library file"):
  9103. return
  9104. logger.error("Queue item %s: Archive creation from library file returned no archive", item.id)
  9105. await self._power_off_if_needed(db, item)
  9106. return
  9107. # The photos follow the file into the archive that replaces it, for
  9108. # the same reason the siblings do (#3077). After the commit above,
  9109. # never before it: that commit can fail ("database is locked",
  9110. # #1853) and roll the library row back, and photos already moved
  9111. # would leave it naming a directory that no longer exists. The
  9112. # file and thumbnail unlinks are deferred for the same reason.
  9113. if consumed_library_file_id is not None and consumed_photos:
  9114. # Held as a plain int, read here while the session is still
  9115. # healthy, because the handler below may not touch an ORM
  9116. # instance at all. The commit it exists for fails inside the
  9117. # FLUSH, not at COMMIT: SQLite takes the write lock at the
  9118. # first DML statement, so a busy writer surfaces as "database
  9119. # is locked" on the UPDATE (#1853). SQLAlchemy rolls that back
  9120. # internally through safe_reraise before re-raising, which
  9121. # expires every loaded instance and leaves the session in
  9122. # pending-rollback state -- so `archive.id` inside the except
  9123. # would itself raise PendingRollbackError and the rollback
  9124. # below would never be reached.
  9125. archive_id = archive.id
  9126. try:
  9127. carried_photos = move_library_photos(
  9128. consumed_library_file_id,
  9129. consumed_photos,
  9130. archive_photos_dir(archive),
  9131. )
  9132. if carried_photos:
  9133. archive.photos = list(archive.photos or []) + carried_photos
  9134. await db.commit()
  9135. except Exception as e:
  9136. # The archive and the delete are already committed; the
  9137. # print goes ahead either way. Worst case the pictures sit
  9138. # unnamed in the archive's own directory.
  9139. #
  9140. # Ints only until the rollback has run, per the note above,
  9141. # which is why this logs queue_item_id and not item.id --
  9142. # the sibling handler forty lines up does the same.
  9143. logger.warning(
  9144. "Queue item %s: failed to carry library photos into archive %s: %s",
  9145. queue_item_id,
  9146. archive_id,
  9147. e,
  9148. )
  9149. await db.rollback()
  9150. # rollback() expires every loaded instance, and in async
  9151. # SQLAlchemy the next plain attribute read is lazy IO
  9152. # outside the greenlet -- MissingGreenlet, which would turn
  9153. # this cosmetic failure into a dispatch crash in exactly the
  9154. # "database is locked" case the block exists for (#1853).
  9155. # The nozzle guard, the upload and the start all keep
  9156. # reading item, archive and printer, so all three go back
  9157. # into the session before falling through.
  9158. item = await db.get(PrintQueueItem, queue_item_id)
  9159. archive = await db.get(PrintArchive, archive_id)
  9160. printer = await db.get(Printer, item.printer_id) if item else None
  9161. if not item or not archive or not printer:
  9162. logger.error(
  9163. "Queue item %s: item, archive %s or printer gone after the photo rollback",
  9164. queue_item_id,
  9165. archive_id,
  9166. )
  9167. return
  9168. else:
  9169. # Neither archive nor library file specified
  9170. if not await self._fail_dispatch(db, item, "No source file specified"):
  9171. return
  9172. logger.error("Queue item %s: No archive_id or library_file_id specified", item.id)
  9173. await self._power_off_if_needed(db, item)
  9174. return
  9175. # Check file exists on disk
  9176. if not file_path.exists():
  9177. if not await self._fail_dispatch(db, item, "Source file not found on disk"):
  9178. return
  9179. logger.error("Queue item %s: File not found: %s", item.id, file_path)
  9180. await self._power_off_if_needed(db, item)
  9181. return
  9182. # Nozzle-diameter mismatch guard (#1899). A file sliced for one nozzle
  9183. # size dispatched to a printer with a different nozzle installed is
  9184. # rejected by the firmware with a cryptic HMS ("Failed to get AMS mapping
  9185. # table" 0700_8012, or "nozzle diameter … not consistent" 0500_4038) that
  9186. # gives the user no idea what went wrong. Catch it here, before we spend
  9187. # time preheating and uploading, and fail with an actionable message.
  9188. # Fail-safe by construction: only a POSITIVE mismatch blocks — when the
  9189. # slice carries no nozzle diameter (archive.nozzle_diameter is None) or
  9190. # the printer hasn't reported its nozzles yet, we fall through and let the
  9191. # print proceed exactly as before. On dual-nozzle printers (H2D) a match
  9192. # against EITHER installed nozzle passes, so a 0.6 slice is fine as long
  9193. # as one of the two hotends is a 0.6.
  9194. #
  9195. # On a tool-changer model (H2C) the nozzles parked in the rack count too
  9196. # (#2885): the printer fetches one as part of starting the print, so the
  9197. # set to test against is "reachable", not "currently mounted". This runs
  9198. # well before the rack picker at the bottom of this method, so without
  9199. # the rack in scope here that picker never got the chance to run.
  9200. sliced_nozzle = archive.nozzle_diameter if archive else None
  9201. if sliced_nozzle:
  9202. nozzle_status = printer_manager.get_status(item.printer_id)
  9203. installed = _installed_nozzle_diameters(nozzle_status)
  9204. rack = _rack_nozzle_diameters(nozzle_status)
  9205. mismatch_msg = _nozzle_mismatch_message(sliced_nozzle, installed, rack)
  9206. if mismatch_msg:
  9207. if not await self._fail_dispatch(db, item, mismatch_msg):
  9208. return
  9209. logger.warning("Queue item %s: nozzle mismatch — %s", item.id, mismatch_msg)
  9210. await notification_service.on_queue_job_failed(
  9211. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  9212. printer_id=printer.id,
  9213. printer_name=printer.name,
  9214. reason=mismatch_msg,
  9215. db=db,
  9216. )
  9217. try:
  9218. await ws_manager.send_queue_item_failed(
  9219. user_id=item.created_by_id,
  9220. queue_item_id=item.id,
  9221. printer_id=item.printer_id,
  9222. reason="nozzle_mismatch",
  9223. )
  9224. except Exception:
  9225. pass
  9226. await self._power_off_if_needed(db, item)
  9227. return
  9228. # Preheat / heat-soak (#1468) — fires before upload so the printer's
  9229. # bed (and chamber, if applicable) is at temperature when the firmware
  9230. # starts the actual print routine. Best-effort: any failure logs and
  9231. # falls through to the normal upload+start path rather than turning a
  9232. # configuration issue into a failed queue item.
  9233. # Returns False only when the item was cancelled or deleted while the
  9234. # stage was holding at temperature. Uploading and starting it anyway
  9235. # would print a job the user has already called off, so abandon the
  9236. # dispatch here; `_dispatch_one`'s finally clause unwinds the heaters.
  9237. if not await self._preheat_and_soak(db, item, printer, archive):
  9238. logger.info("Queue item %s: dispatch abandoned — cancelled during preheat", item.id)
  9239. return
  9240. # See `_effective_plate_id` for why this is resolved once here rather
  9241. # than each site below repeating `item.plate_id or 1`.
  9242. effective_plate_id = _effective_plate_id(item.plate_id, file_path)
  9243. # G-code injection for auto-print systems (#422)
  9244. injected_path = None
  9245. # #2547: tracked separately from `injected_path`, which is also set when
  9246. # only a START snippet was injected. Only an END snippet changes what the
  9247. # camera sees at print completion.
  9248. end_gcode_injected = False
  9249. if item.gcode_injection:
  9250. try:
  9251. snippets_raw = await self._get_setting(db, "gcode_snippets")
  9252. if snippets_raw:
  9253. snippets = json.loads(snippets_raw)
  9254. model_snippets = snippets.get(printer.model, {})
  9255. start_gc = (model_snippets.get("start_gcode") or "").strip()
  9256. end_gc = (model_snippets.get("end_gcode") or "").strip()
  9257. if start_gc or end_gc:
  9258. from backend.app.utils.threemf_tools import inject_gcode_into_3mf
  9259. injected_path = inject_gcode_into_3mf(
  9260. file_path, effective_plate_id, start_gc or None, end_gc or None
  9261. )
  9262. if injected_path:
  9263. file_path = injected_path
  9264. end_gcode_injected = bool(end_gc)
  9265. logger.info("Queue item %s: G-code injected for model %s", item.id, printer.model)
  9266. else:
  9267. logger.warning(
  9268. "Queue item %s: G-code injection returned no result, using original", item.id
  9269. )
  9270. except Exception as e:
  9271. logger.warning("Queue item %s: G-code injection failed, using original: %s", item.id, e)
  9272. # #2547: the finish-photo path can't learn from telemetry that this print
  9273. # ends with user End G-code — which means the plate may be gone by the
  9274. # time FINISH arrives (#1867). Flag it here; `on_print_start` binds it to
  9275. # the print once the printer confirms it running.
  9276. if end_gcode_injected:
  9277. print_dispatch_context.mark_pending(printer.id)
  9278. # Upload to root directory (not /cache/) - the start_print command references
  9279. # files by name only (ftp://{filename}), so they must be in the root
  9280. remote_filename = derive_remote_filename(filename)
  9281. remote_path = f"/{remote_filename}"
  9282. # Get FTP retry settings
  9283. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  9284. logger.info(
  9285. f"Queue item {item.id}: FTP upload starting - printer={printer.name} ({printer.model}), "
  9286. f"ip={printer.ip_address}, file={remote_filename}, local_path={file_path}, "
  9287. f"retry_enabled={ftp_retry_enabled}, retry_count={ftp_retry_count}, timeout={ftp_timeout}"
  9288. )
  9289. # Release the pooled DB connection before the FTP delete/upload (#2572).
  9290. # Every read this method needs (printer, archive/library, preheat) is
  9291. # done, and the library-file branch already committed its archive
  9292. # creation. Without this the transaction opened by the first SELECT above
  9293. # stays "idle in transaction" for the entire upload — multiple seconds
  9294. # for a large 3MF — pinning one pooled connection per in-flight dispatch;
  9295. # a farm dispatching many jobs at once then exhausts the pool. This was
  9296. # correlated to an exact idle-in-transaction session on a 93-printer farm
  9297. # (reporter @Jostxxl). expire_on_commit=False keeps item/printer/archive
  9298. # readable; the status writes below (upload-failure path and the
  9299. # pending->printing CAS) transparently open a fresh transaction.
  9300. # The owner is read here, after preheat, so the commit releases this read too.
  9301. toast_uid = await self._queue_owner_id(db, item)
  9302. await db.commit()
  9303. # Delete existing file if present (avoids 553 error on overwrite)
  9304. try:
  9305. logger.debug("Queue item %s: Deleting existing file %s if present...", item.id, remote_path)
  9306. delete_result = await delete_file_async(
  9307. printer.ip_address,
  9308. printer.access_code,
  9309. remote_path,
  9310. socket_timeout=ftp_timeout,
  9311. printer_model=printer.model,
  9312. # This delete and the upload below are one bounded, user-initiated
  9313. # unit -- at most nine connections -- so neither skips on the
  9314. # handshake cool-off the opportunistic sweeps rely on. In #2898's
  9315. # trace this delete took the TLS failure and armed the cool-off,
  9316. # and the upload's four attempts were then spent against it
  9317. # without a socket being opened.
  9318. respect_handshake_cooloff=False,
  9319. )
  9320. logger.debug("Queue item %s: Delete result: %s", item.id, delete_result)
  9321. except Exception as e:
  9322. logger.debug("Queue item %s: Delete failed (may not exist): %s", item.id, e)
  9323. # Dispatch toast — announce the upload start with the total byte
  9324. # count so the frontend can render an honest progress bar.
  9325. toast_file_name = filename.replace(".gcode.3mf", "").replace(".3mf", "")
  9326. try:
  9327. total_bytes = file_path.stat().st_size
  9328. except OSError:
  9329. total_bytes = 0
  9330. try:
  9331. await ws_manager.send_queue_item_uploading(
  9332. user_id=toast_uid,
  9333. queue_item_id=item.id,
  9334. printer_id=item.printer_id,
  9335. printer_name=printer.name,
  9336. file_name=toast_file_name,
  9337. total_bytes=total_bytes,
  9338. )
  9339. except Exception:
  9340. pass # toast is best-effort
  9341. progress_bridge = _UploadProgressBridge(toast_uid, item.id)
  9342. # A deadline expiry gets its own message: "check your SD card" is the
  9343. # wrong advice for a link that was simply too slow to finish (#2529).
  9344. upload_error: str | None = None
  9345. # Why the upload failed, straight from the client rather than inferred.
  9346. # Owned here, so a background fetch for another print cannot overwrite
  9347. # it between the failure and the sentence built from it (#2899).
  9348. upload_failure = FtpFailureReport()
  9349. try:
  9350. # The printer's MQTT goes quiet while it receives the file; the
  9351. # stale check must not reconnect it under the print command (#3538).
  9352. with printer_manager.uploading(item.printer_id):
  9353. if ftp_retry_enabled:
  9354. uploaded = await with_ftp_retry(
  9355. upload_file_async,
  9356. printer.ip_address,
  9357. printer.access_code,
  9358. file_path,
  9359. remote_path,
  9360. socket_timeout=ftp_timeout,
  9361. printer_model=printer.model,
  9362. progress_callback=progress_bridge,
  9363. respect_handshake_cooloff=False,
  9364. failure=upload_failure,
  9365. max_retries=ftp_retry_count,
  9366. retry_delay=ftp_retry_delay,
  9367. operation_name=f"Upload print to {printer.name}",
  9368. )
  9369. else:
  9370. uploaded = await upload_file_async(
  9371. printer.ip_address,
  9372. printer.access_code,
  9373. file_path,
  9374. remote_path,
  9375. socket_timeout=ftp_timeout,
  9376. printer_model=printer.model,
  9377. progress_callback=progress_bridge,
  9378. respect_handshake_cooloff=False,
  9379. failure=upload_failure,
  9380. )
  9381. except UploadCancelled as e:
  9382. uploaded = False
  9383. upload_error = (
  9384. "Upload was too slow to finish and was cancelled. The printer's connection could not sustain "
  9385. "the transfer — check its Wi-Fi signal, or move it closer to the access point."
  9386. )
  9387. logger.error("Queue item %s: upload deadline exceeded: %s", item.id, e)
  9388. except Exception as e:
  9389. uploaded = False
  9390. logger.error("Queue item %s: FTP error: %s (type: %s)", item.id, e, type(e).__name__)
  9391. # Clean up injected temp file after upload attempt
  9392. if injected_path and injected_path.exists():
  9393. injected_path.unlink(missing_ok=True)
  9394. # Again after the upload, which can take minutes: an owner deleted
  9395. # meanwhile gets no more toasts, and neither does whoever has their id.
  9396. toast_uid = await self._queue_owner_id(db, item)
  9397. # The printer's copy at this path was replaced (or deleted above): the
  9398. # file browser's previews must not show the old one.
  9399. from backend.app.services.printer_file_cache import printer_file_cache
  9400. printer_file_cache.forget(printer.id, remote_path)
  9401. if not uploaded:
  9402. # This used to be one string for every upload failure, telling
  9403. # everyone to check the SD card. The client knows which of seven
  9404. # things went wrong and logs each one differently; it just had no
  9405. # way to say so here, so the card got named even for a TLS
  9406. # handshake that never reached the printer's filesystem (#2899).
  9407. error_msg = upload_error or describe_upload_failure(upload_failure.failure)
  9408. failure = upload_failure.failure
  9409. if failure is not None and failure.kind is FtpFailureKind.STORAGE:
  9410. try:
  9411. validate_print_filename(remote_filename)
  9412. except InvalidFilenameError as e:
  9413. # This 553 is the file's name, which no card can hold, not
  9414. # the printer's storage: it would be refused on every retry
  9415. # and every printer, so this item fails and the queue goes on.
  9416. upload_error = f"The printer could not create the file {remote_filename!r}: {e}. Rename the file."
  9417. error_msg = upload_error
  9418. if upload_error is None and failure is not None and failure.kind in _UPLOAD_REQUEUE_KINDS:
  9419. await self._requeue_after_upload_refused(db, item, printer, error_msg, toast_uid, failure.kind)
  9420. return
  9421. if not await self._fail_dispatch(db, item, error_msg):
  9422. await self._close_cancelled_dispatch_toast(item, toast_uid)
  9423. return
  9424. logger.error(
  9425. f"Queue item {item.id}: FTP upload failed - printer={printer.name}, model={printer.model}, "
  9426. f"ip={printer.ip_address}. Check logs above for storage diagnostics and specific error codes."
  9427. )
  9428. # Send failure notification
  9429. await notification_service.on_queue_job_failed(
  9430. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  9431. printer_id=printer.id,
  9432. printer_name=printer.name,
  9433. # The same sentence the queue shows. A push notification saying
  9434. # something different from the UI is its own small bug (#2899).
  9435. reason=error_msg,
  9436. db=db,
  9437. )
  9438. try:
  9439. await ws_manager.send_queue_item_failed(
  9440. user_id=toast_uid,
  9441. queue_item_id=item.id,
  9442. printer_id=item.printer_id,
  9443. reason="upload_failed",
  9444. )
  9445. except Exception:
  9446. pass
  9447. await self._power_off_if_needed(db, item)
  9448. return
  9449. # The printer took a file, so any run of refused uploads is over: the
  9450. # next refusal starts again at the shortest backoff, and notifies (#3210).
  9451. self._upload_refusals.pop(printer.id, None)
  9452. self._upload_refusal_kind.pop(printer.id, None)
  9453. self._upload_refusal_notified.pop(printer.id, None)
  9454. # Parse AMS mapping if stored
  9455. ams_mapping = None
  9456. if item.ams_mapping:
  9457. try:
  9458. ams_mapping = json.loads(item.ams_mapping)
  9459. except json.JSONDecodeError:
  9460. logger.warning("Queue item %s: Invalid AMS mapping JSON, ignoring", item.id)
  9461. # Register as expected print so we don't create a duplicate archive
  9462. # Only applicable for archive-based prints
  9463. if archive:
  9464. from backend.app.main import register_expected_print
  9465. register_expected_print(
  9466. item.printer_id,
  9467. remote_filename,
  9468. archive.id,
  9469. ams_mapping=ams_mapping,
  9470. # Read after the upload (toast_uid): a user deleted meanwhile gets
  9471. # no print-start email, and neither does whoever has their id.
  9472. created_by_id=toast_uid,
  9473. cost_center_id=item.cost_center_id,
  9474. # The plate actually dispatched, not the queue item's raw
  9475. # column: on None, `register_expected_print` stores nothing,
  9476. # and `extract_filament_usage_from_3mf` then books every
  9477. # filament in the file rather than the one plate that printed.
  9478. plate_id=effective_plate_id,
  9479. )
  9480. # Registration happens before the print command by necessity (the
  9481. # printer can report the print before the send returns), so record
  9482. # what to undo if we never get as far as sending. `_dispatch_one`
  9483. # rolls back anything still pending here on every exit — exception,
  9484. # early return, or cancel winning the CAS below.
  9485. self._unconfirmed_expected_print[item.id] = (item.printer_id, remote_filename, archive.id)
  9486. # Propagate the queue item's owner into printer_manager so the
  9487. # print-complete callback can credit the user in the PrintLogEntry
  9488. # (#1670). `created_by_id` is set either at queue-add time (UI-added
  9489. # items) or when the user clicks the manual-start button.
  9490. await self._propagate_owner_to_printer_manager(db, item)
  9491. # IMPORTANT: Set status to "printing" BEFORE sending the print command.
  9492. # This prevents phantom reprints if the backend crashes/restarts after the
  9493. # print command is sent but before the status update is committed.
  9494. # If we crash after this commit but before start_print(), the item will be
  9495. # in "printing" status without actually printing - but that's safer than
  9496. # accidentally reprinting the same file hours later.
  9497. #
  9498. # Atomic CAS (#1853): a user pressing /cancel mid-dispatch (between the
  9499. # initial pending read at the top of check_queue and this point) flips
  9500. # the row to "cancelled" in a separate session. Without the WHERE
  9501. # status='pending' clause, the unconditional update here would silently
  9502. # overwrite that cancellation and we'd ship the MQTT start_print below
  9503. # — printer obeys, user sees "I pressed cancel and the print started".
  9504. # rowcount==0 means the user won the race; bail out, best-effort delete
  9505. # the file we just uploaded, do NOT send start_print.
  9506. now_utc = datetime.now(timezone.utc)
  9507. billing_run_id = str(uuid.uuid4())
  9508. # Written with the move to printing, so a retried send replaces the
  9509. # last answer and a cancelled one leaves the row alone (#3279).
  9510. color_substitutions = await self._color_substitutions_for(db, item)
  9511. cas = await db.execute(
  9512. update(PrintQueueItem)
  9513. .where(PrintQueueItem.id == item.id)
  9514. .where(PrintQueueItem.status == "pending")
  9515. .values(
  9516. status="printing",
  9517. started_at=now_utc,
  9518. billing_run_id=billing_run_id,
  9519. color_substitutions=color_substitutions,
  9520. )
  9521. )
  9522. await db.commit()
  9523. if cas.rowcount == 0:
  9524. logger.info(
  9525. "Queue item %s no longer pending at print-command time "
  9526. "(cancelled or removed mid-dispatch) — aborting before MQTT send (#1853)",
  9527. item.id,
  9528. )
  9529. try:
  9530. await delete_file_async(
  9531. printer.ip_address,
  9532. printer.access_code,
  9533. remote_path,
  9534. socket_timeout=ftp_timeout,
  9535. printer_model=printer.model,
  9536. )
  9537. except Exception as cleanup_err:
  9538. logger.debug(
  9539. "Queue item %s: best-effort cleanup of uploaded file failed: %s",
  9540. item.id,
  9541. cleanup_err,
  9542. )
  9543. try:
  9544. await ws_manager.send_queue_item_failed(
  9545. user_id=toast_uid,
  9546. queue_item_id=item.id,
  9547. printer_id=item.printer_id,
  9548. reason="cancelled_mid_dispatch",
  9549. )
  9550. except Exception:
  9551. pass
  9552. return
  9553. # Sync the in-memory item so subsequent code that reads item.status /
  9554. # item.started_at sees the values we just persisted.
  9555. item.status = "printing"
  9556. item.started_at = now_utc
  9557. item.billing_run_id = billing_run_id
  9558. if archive is not None:
  9559. archive.billing_run_id = billing_run_id
  9560. # Legacy transaction deletion used an archive-wide skip flag.
  9561. # A newly dispatched run has its own UUID/tombstone, so it must be
  9562. # billable independently of any older deleted run on this archive.
  9563. archive.wallet_charge_skipped = False
  9564. # Persist before MQTT send so completion and restart recovery can
  9565. # always recover the internal billing identity.
  9566. await db.commit()
  9567. for cleanup_path in cleanup_disk_paths:
  9568. try:
  9569. if cleanup_path.exists():
  9570. cleanup_path.unlink()
  9571. except OSError as cleanup_err:
  9572. logger.warning(
  9573. "TRANSIENT_LIBRARY_FILE_ORPHAN %s",
  9574. json.dumps(
  9575. {
  9576. "queue_item_id": item.id,
  9577. "path": str(cleanup_path),
  9578. "error": str(cleanup_err),
  9579. },
  9580. sort_keys=True,
  9581. ),
  9582. )
  9583. # Clear the awaiting-plate-clear flag now that we're starting a new print
  9584. printer_manager.set_awaiting_plate_clear(item.printer_id, False)
  9585. # #1898: with the opt-in default-good setting, moving on to the next
  9586. # print resolves the previous print's unanswered outcome prompt as
  9587. # "good" — this path also covers the camera-based plate detection,
  9588. # which releases the gate by allowing dispatch rather than by an
  9589. # explicit acknowledgment. Rides on the dispatch transaction.
  9590. if await self._get_bool_setting(db, "confirm_default_good_on_plate_clear", default=False):
  9591. from backend.app.services.print_confirmation import resolve_pending_confirmation_as_good
  9592. await resolve_pending_confirmation_as_good(db, item.printer_id)
  9593. logger.info("Queue item %s: Status set to 'printing', sending print command...", item.id)
  9594. # Capture state before dispatch so the watchdog can detect whether the
  9595. # printer actually transitioned (#967). Also capture subtask_id so the
  9596. # watchdog can recognise "command landed but state hasn't flipped yet"
  9597. # on slow H2D transitions (#1078).
  9598. pre_status = printer_manager.get_status(item.printer_id)
  9599. pre_state = getattr(pre_status, "state", None) if pre_status else None
  9600. pre_subtask_id = getattr(pre_status, "subtask_id", None) if pre_status else None
  9601. pre_gcode_file = getattr(pre_status, "gcode_file", None) if pre_status else None
  9602. # #1721: respect the user's explicit timelapse choice. The #1397
  9603. # force-on at dispatch was removed because it caused per-layer nozzle
  9604. # parking on slicer profiles with Timelapse Type = Smooth. Finish-photo
  9605. # capture is now driven by the stg_cur=22 transition in bambu_mqtt.py
  9606. # ("Filament unloading", toolhead parked, bed not yet dropped) with a
  9607. # FINISH-state fallback — no need to force a video.
  9608. effective_timelapse = bool(item.timelapse)
  9609. # Nozzle-rack fallback (#2800). A job that never passed through the
  9610. # Virtual Printer carries no Bambu Studio nozzle pick, and an H2C then
  9611. # dispatches with no nozzle field at all and chooses for itself — which
  9612. # is how a print levelled on one hotend and then printed on another,
  9613. # millimetres above the plate. Derive the per-slot extruder assignment
  9614. # from the file being dispatched.
  9615. #
  9616. # Done here rather than at queue time because this is the first point
  9617. # that knows both the actual printer and the actual file: an item can
  9618. # be created without a printer (model-based assignment), reassigned
  9619. # afterwards, or have its file swapped for a G-code-injected copy just
  9620. # above. Every queue-creation path — the print dialog, a bulk library
  9621. # add, the webhook, a pipeline run — is covered by the one call.
  9622. # Skipped when the item already carries a Bambu Studio capture: that
  9623. # one wins downstream anyway, so reading the 3MF again would be work
  9624. # thrown away on every dispatch.
  9625. # Rack position resolution (#1784), tried before the #2800 fallback
  9626. # below because it can express what that one cannot: a plate wanting a
  9627. # *different* hotend off the rack per filament group. The position per
  9628. # group is the operator's pick — the 3MF states it nowhere, proven by
  9629. # sending one plate twice with different picks and finding the two
  9630. # files identical bar float noise — so it is resolved here against the
  9631. # rack as it stands right now, after the upload, not at queue time.
  9632. resolved_nozzle_mapping = None
  9633. if not item.nozzle_mapping and file_path is not None and is_nozzle_rack_model(printer.model):
  9634. rack_plan = extract_rack_plan_from_3mf(file_path, plate_id=effective_plate_id)
  9635. if rack_plan is not None:
  9636. try:
  9637. stored_choice = json.loads(item.nozzle_rack_choice) if item.nozzle_rack_choice else {}
  9638. except (json.JSONDecodeError, TypeError):
  9639. stored_choice = {}
  9640. logger.warning(
  9641. "Queue item %s: unreadable nozzle_rack_choice %r, assigning rack positions instead",
  9642. item.id,
  9643. item.nozzle_rack_choice,
  9644. )
  9645. # JSON object keys are strings; the groups are ints.
  9646. choice: dict[int, int] = {}
  9647. for key, value in (stored_choice or {}).items():
  9648. try:
  9649. choice[int(key)] = int(value)
  9650. except (TypeError, ValueError):
  9651. continue
  9652. live_rack = getattr(printer_manager.get_status(item.printer_id), "nozzle_rack", None) or []
  9653. resolved_nozzle_mapping, rack_error = resolve_rack_plan_mapping(
  9654. rack_plan.slot_groups, rack_plan.group_dicts(), choice, live_rack
  9655. )
  9656. if resolved_nozzle_mapping is None and choice:
  9657. # An explicit pick that no longer holds stops the print. The
  9658. # operator named a hotend; printing from a different one is
  9659. # how a plate gets levelled on one nozzle and drawn with
  9660. # another, millimetres above the bed. Nothing has been sent
  9661. # to the printer yet, so failing here costs only the upload.
  9662. # The file is already on the SD card by this point, and a
  9663. # 3MF left there is a phantom print waiting to be started
  9664. # from the touchscreen. Same cleanup the start_print
  9665. # failure path below does, for the same reason.
  9666. try:
  9667. await delete_file_async(
  9668. printer.ip_address,
  9669. printer.access_code,
  9670. remote_path,
  9671. printer_model=printer.model,
  9672. )
  9673. except Exception:
  9674. pass # Best-effort — don't fail the error handler
  9675. if not await self._fail_dispatch(
  9676. db,
  9677. item,
  9678. (
  9679. f"Nozzle rack pick no longer fits the printer: {rack_error}. "
  9680. "Edit the item to choose another position."
  9681. ),
  9682. ):
  9683. await self._close_cancelled_dispatch_toast(item, toast_uid)
  9684. return
  9685. logger.warning(
  9686. "Queue item %s: refusing to dispatch to %s — %s (chose %s, rack %s)",
  9687. item.id,
  9688. printer.name,
  9689. rack_error,
  9690. choice,
  9691. [slot.get("id") for slot in live_rack],
  9692. )
  9693. await notification_service.on_queue_job_failed(
  9694. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  9695. printer_id=printer.id,
  9696. printer_name=printer.name,
  9697. reason=item.error_message,
  9698. db=db,
  9699. )
  9700. try:
  9701. await ws_manager.send_queue_item_failed(
  9702. user_id=toast_uid,
  9703. queue_item_id=item.id,
  9704. printer_id=item.printer_id,
  9705. reason="nozzle_rack_pick_stale",
  9706. )
  9707. except Exception:
  9708. pass # Best-effort — don't fail the error handler
  9709. return
  9710. if resolved_nozzle_mapping is None:
  9711. # Nothing was picked and nothing could be assigned. Falls
  9712. # through to the #2800 path, which is what ran before this
  9713. # existed — strictly not worse than today.
  9714. logger.info(
  9715. "Queue item %s: no rack positions assignable (%s); falling back",
  9716. item.id,
  9717. rack_error,
  9718. )
  9719. else:
  9720. logger.info(
  9721. "Queue item %s: rack mapping %s (groups %s, chosen %s)",
  9722. item.id,
  9723. resolved_nozzle_mapping,
  9724. rack_plan.slot_groups,
  9725. choice or "auto",
  9726. )
  9727. nozzle_slot_extruders = None
  9728. if (
  9729. not item.nozzle_mapping
  9730. and resolved_nozzle_mapping is None
  9731. and file_path is not None
  9732. and is_nozzle_rack_model(printer.model)
  9733. ):
  9734. slot_extruders = extract_slot_extruders_from_3mf(file_path, plate_id=effective_plate_id)
  9735. if slot_extruders:
  9736. nozzle_slot_extruders = json.dumps(slot_extruders)
  9737. # Every filament this plate prints is on the external spool -> the print
  9738. # must go out with use_ams=False. The firmware answers use_ams=true plus
  9739. # a mapping it cannot resolve with 07FF_8012 "Failed to get AMS mapping
  9740. # table", which is what held the reporter's P1S at Heatbed preheating
  9741. # for ten minutes before it gave up (#3087). The MQTT command builder
  9742. # already downgrades a mapping that is *only* external ([254]), but a
  9743. # multi-filament project pads the slots this plate does not print with
  9744. # -1 — BambuStudio's own convention — and down there a -1 is
  9745. # indistinguishable from a slot that never resolved, which must never be
  9746. # sent to the spool holder (#2589). Here the plate's filament list says
  9747. # which is which, so the answer is exact rather than a guess.
  9748. #
  9749. # Deliberately narrow: this fires only when every consumed slot is an
  9750. # explicit 254/255. A consumed slot that did not resolve leaves use_ams
  9751. # alone and the firmware still rejects the print, exactly as today. And
  9752. # only for single-nozzle printers, mirroring the builder's own reconcile
  9753. # — on a dual-nozzle machine use_ams is which extruder to feed, not
  9754. # whether to use the AMS, so it is not ours to rewrite.
  9755. effective_use_ams = item.use_ams
  9756. plate_uses_only_external = False
  9757. if (
  9758. ams_mapping
  9759. and file_path is not None
  9760. # Cheap gate before opening the file: with nothing on the spool
  9761. # holder anywhere in the mapping, no subset of it can be all
  9762. # external, so most dispatches never pay for the parse. The
  9763. # isinstance also keeps a malformed stored mapping (a bare number
  9764. # from a hand-edited row) failing where it always failed, in the
  9765. # command builder, rather than here.
  9766. and isinstance(ams_mapping, list)
  9767. and any(_is_external_tray(t, TrayLayout.from_state(pre_status).rack) for t in ams_mapping)
  9768. and not _might_be_dual_nozzle(printer.model, pre_status)
  9769. ):
  9770. from backend.app.services.filament_requirements import extract_filament_requirements
  9771. # Exact for this plate: the command builder must not turn the AMS
  9772. # back on for a real tray another plate of the file uses, whether
  9773. # the job asked for the AMS or already had it off.
  9774. plate_uses_only_external = _plate_uses_only_external(
  9775. ams_mapping,
  9776. extract_filament_requirements(file_path, plate_id=effective_plate_id),
  9777. TrayLayout.from_state(pre_status).rack,
  9778. )
  9779. if plate_uses_only_external and effective_use_ams:
  9780. effective_use_ams = False
  9781. logger.info(
  9782. "Queue item %s: every filament plate %s prints is on the external spool "
  9783. "(mapping %s) — dispatching with use_ams=False (#3087)",
  9784. item.id,
  9785. effective_plate_id,
  9786. ams_mapping,
  9787. )
  9788. # A slot that lost its K-profile selection (a power cycle resets every
  9789. # slot to the default K) would print on the default. The idle check in
  9790. # the status handler normally restores it first, but nothing guarantees
  9791. # it ran before this dispatch (#3219). Unthrottled: once per job.
  9792. # Never allowed to stop the print it is protecting.
  9793. try:
  9794. await kprofile_drift.reapply_lost_kprofiles(item.printer_id, _used_global_tray_ids(item), throttle=False)
  9795. except Exception:
  9796. logger.exception("Queue item %s: K-profile check before dispatch failed", item.id)
  9797. # Start the print with AMS mapping, plate_id and print options.
  9798. # nozzle_mapping rides through verbatim — JSON string captured from
  9799. # Bambu Studio's project_file on VP intake (#1780); the MQTT layer
  9800. # parses + injects it only for dual-nozzle models so a null on every
  9801. # other model is a transparent pass-through. The rack fallback is
  9802. # resolved down there too, where the live rack position is known.
  9803. started = printer_manager.start_print(
  9804. item.printer_id,
  9805. remote_filename,
  9806. plate_id=effective_plate_id,
  9807. ams_mapping=ams_mapping,
  9808. bed_levelling=item.bed_levelling,
  9809. flow_cali=item.flow_cali,
  9810. vibration_cali=item.vibration_cali,
  9811. layer_inspect=item.layer_inspect,
  9812. timelapse=effective_timelapse,
  9813. use_ams=effective_use_ams,
  9814. plate_uses_only_external=plate_uses_only_external,
  9815. nozzle_offset_cali=item.nozzle_offset_cali,
  9816. nozzle_mapping=item.nozzle_mapping
  9817. or (json.dumps(resolved_nozzle_mapping) if resolved_nozzle_mapping else None),
  9818. nozzle_slot_extruders=nozzle_slot_extruders,
  9819. )
  9820. if started:
  9821. # The command is away, so the expectation is now legitimate and must
  9822. # survive. Anything still in this dict when _dispatch_one exits gets
  9823. # rolled back.
  9824. self._unconfirmed_expected_print.pop(item.id, None)
  9825. self._unconfirmed_budget_reservations.discard(item.id)
  9826. self._unconfirmed_print_user.pop(item.id, None)
  9827. if item.printer_id is not None:
  9828. self._sent_unstarted_print[item.id] = (
  9829. item.printer_id,
  9830. remote_filename if archive else None,
  9831. archive.id if archive else None,
  9832. )
  9833. # Handoff to the print's own gcode: keep whatever preheat set (bed
  9834. # target, chamber target, airduct heating) — the gcode owns
  9835. # heater/flap control from here. Clearing the pin prevents
  9836. # `_dispatch_one`'s finally from unwinding a live print.
  9837. self._preheat_pin.pop(item.printer_id, None)
  9838. self._preheat_pin_bed.pop(item.printer_id, None)
  9839. logger.info("Queue item %s: Print started successfully - %s", item.id, filename)
  9840. # No dispatch-toast event here: the legacy bg-dispatch path kept
  9841. # status='processing' from upload start until the printer acked
  9842. # (or timed out). The frontend derives "Awaiting printer…" purely
  9843. # from upload_progress_pct >= 99.9; an explicit 'dispatched' WS
  9844. # event would push the status chip out of 'PROCESSING' prematurely
  9845. # — which is exactly what the screenshot at #1625-followup
  9846. # complained about.
  9847. # Register the local 3MF in the cover-cache so /cover skips FTP
  9848. # (#1166 follow-up). file_path was resolved earlier from either the
  9849. # archive or the library file row.
  9850. if file_path is not None:
  9851. cache_3mf_download(item.printer_id, remote_filename, file_path)
  9852. # Hold the printer against further dispatches until the watchdog
  9853. # confirms the printer transitioned (or until the hard timeout).
  9854. # Prevents multi-plate batches from triple-dispatching onto the
  9855. # same H2D Pro while it digests the first project_file (#1157).
  9856. self._mark_printer_dispatched(item.printer_id, pre_state, pre_subtask_id)
  9857. # Watchdog: if the printer never transitions out of pre_state AND
  9858. # never advances subtask_id, the MQTT publish was accepted locally but
  9859. # didn't reach the printer (half-broken session — same shape as
  9860. # #887/#936). Revert the queue item so the next dispatch can pick it
  9861. # up instead of leaving it stuck in "printing" (#967). subtask_id
  9862. # check avoids false reverts on slow H2D FINISH→PREPARE transitions
  9863. # that would otherwise cause the item to re-dispatch as a reprint
  9864. # of the just-finished job (#1078).
  9865. if pre_state:
  9866. spawn_background_task(
  9867. self._watchdog_print_start(
  9868. item.id,
  9869. item.printer_id,
  9870. pre_state,
  9871. pre_subtask_id,
  9872. pre_gcode_file,
  9873. created_by_id=toast_uid,
  9874. ),
  9875. name=f"watchdog-print-start-{item.id}",
  9876. )
  9877. # Get estimated time for notification.
  9878. #
  9879. # This used to fall back to `library_file.print_time_seconds`, a column
  9880. # LibraryFile does not have — the print time it knows about lives in
  9881. # `file_metadata`. So a library print whose archive carried no parseable
  9882. # print time (a plain .gcode, or a 3MF the parser could not read) raised
  9883. # AttributeError right here, *after* the printer had already been sent
  9884. # the job: the started-notification never fired, and the exception
  9885. # unwound the whole queue pass, so every other printer still waiting to
  9886. # be dispatched on that tick silently missed its turn.
  9887. #
  9888. # The queue item caches the print time at creation ("Cached from
  9889. # archive/library"), which is the value this was reaching for.
  9890. # The item's is its plate's; the archive's adds every plate of the
  9891. # file up (#3375).
  9892. estimated_time = None
  9893. if item.print_time_seconds:
  9894. estimated_time = item.print_time_seconds
  9895. elif archive and archive.print_time_seconds:
  9896. estimated_time = archive.print_time_seconds
  9897. # Send job started notification
  9898. await notification_service.on_queue_job_started(
  9899. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  9900. printer_id=printer.id,
  9901. printer_name=printer.name,
  9902. db=db,
  9903. estimated_time=estimated_time,
  9904. )
  9905. # MQTT relay - publish queue job started
  9906. try:
  9907. await mqtt_relay.on_queue_job_started(
  9908. job_id=item.id,
  9909. filename=filename,
  9910. printer_id=printer.id,
  9911. printer_name=printer.name,
  9912. printer_serial=printer.serial_number,
  9913. )
  9914. except Exception:
  9915. pass # Don't fail if MQTT fails
  9916. else:
  9917. # Clean up uploaded file from SD card to prevent phantom prints
  9918. try:
  9919. await delete_file_async(
  9920. printer.ip_address,
  9921. printer.access_code,
  9922. remote_path,
  9923. printer_model=printer.model,
  9924. )
  9925. except Exception:
  9926. pass # Best-effort — don't fail the error handler
  9927. # Busy-refusal is a deferral, not a failure (#2598). The printer's
  9928. # state can flip from idle to active in the window between the
  9929. # pre-dispatch check above and this publish (the FTP upload takes
  9930. # seconds); start_print() then refuses to send project_file to the
  9931. # now-busy printer and returns False. Failing the item here would be
  9932. # wrong — the printer is fine, it is simply busy — so revert to
  9933. # pending and let a later tick dispatch it once the printer is idle,
  9934. # exactly like the pre-dispatch guard. Only a start_print() False on
  9935. # an idle/unknown printer is a genuine command failure.
  9936. post_dispatch_state = getattr(printer_manager.get_status(item.printer_id), "state", None)
  9937. if post_dispatch_state in _ACTIVE_PRINT_STATES:
  9938. logger.info(
  9939. "Queue item %s: printer %s became busy (state=%s) before the start "
  9940. "command was sent — deferring, reverting item to pending (#2598)",
  9941. item.id,
  9942. item.printer_id,
  9943. post_dispatch_state,
  9944. )
  9945. await self._defer_until_printer_idle(db, item)
  9946. return
  9947. # Print command failed - revert status
  9948. if not await self._fail_dispatch(db, item, "Failed to send print command to printer"):
  9949. await self._close_cancelled_dispatch_toast(item, toast_uid)
  9950. return
  9951. logger.error(
  9952. f"Queue item {item.id}: Failed to start print on {printer.name} ({printer.model}) - "
  9953. f"printer_manager.start_print() returned False. "
  9954. f"This may indicate: printer not connected, MQTT error, unsupported model configuration, or firmware issue. "
  9955. f"Check printer status and backend logs for details."
  9956. )
  9957. # Send failure notification
  9958. await notification_service.on_queue_job_failed(
  9959. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  9960. printer_id=printer.id,
  9961. printer_name=printer.name,
  9962. reason="Failed to send print command to printer - check printer connection and status",
  9963. db=db,
  9964. )
  9965. try:
  9966. await ws_manager.send_queue_item_failed(
  9967. user_id=toast_uid,
  9968. queue_item_id=item.id,
  9969. printer_id=item.printer_id,
  9970. reason="start_command_failed",
  9971. )
  9972. except Exception:
  9973. pass
  9974. await self._power_off_if_needed(db, item)
  9975. @staticmethod
  9976. async def _watchdog_print_start(
  9977. queue_item_id: int,
  9978. printer_id: int,
  9979. pre_state: str,
  9980. pre_subtask_id: str | None = None,
  9981. pre_gcode_file: str | None = None,
  9982. timeout: float = 90.0,
  9983. phase_b_timeout: float = 180.0,
  9984. poll_interval: float = 3.0,
  9985. created_by_id: int | None = None,
  9986. ) -> None:
  9987. """Revert a queue item if the printer never acknowledges the start command.
  9988. Bambuddy optimistically marks the queue item as "printing" right after the
  9989. MQTT project_file publish succeeds locally. The watchdog runs in two phases:
  9990. Phase A (up to ``timeout``): wait for either an active-state transition
  9991. or a ``subtask_id`` advance past ``pre_subtask_id``. State alone is the
  9992. primary signal; subtask_id advance handles the H2D case where state can
  9993. sit at FINISH for ~50 s after the printer accepted ``project_file``
  9994. before flipping to PREPARE (#1078). If neither happens, the MQTT publish
  9995. was lost on a half-broken session (#887/#936) — revert and force
  9996. reconnect (the #967 recovery path).
  9997. Phase B (up to ``phase_b_timeout``, only if Phase A exited on subtask_id
  9998. alone): keep watching for the active-state transition. subtask_id alone
  9999. proves the file landed but not that the printer started — and a printer
  10000. that accepts the command but stays at IDLE/FINISH indefinitely (e.g.
  10001. cloud+LAN re-auth dance after a power cycle on old firmware, #1678)
  10002. used to leave the queue item stuck in 'printing' forever because the
  10003. old watchdog returned success as soon as subtask_id advanced. If Phase
  10004. B times out, revert the queue item so the user can retry without
  10005. restarting Bambuddy. Skip ``force_reconnect`` here: the file landed and
  10006. a forced reconnect mid-parse triggers 0500_4003 (#1150).
  10007. Phase A timeout raised from 45 s → 90 s as belt-and-braces for slow
  10008. transitions that also don't emit an early subtask_id tick.
  10009. Both phases also watch for ``HMS_MQTT_VERIFY_FAILED``. A printer that
  10010. refuses to verify our commands will never start this job or any other,
  10011. so waiting out the full 270 s and re-uploading the 3MF twice more only
  10012. burns an upload slot the rest of the farm is queued behind — that path
  10013. is for a printer that might still come good, which this one cannot
  10014. (#2732). It fails the item on the spot with the actual reason instead.
  10015. """
  10016. last_status = None
  10017. landed_on_subtask = False
  10018. # Latched, not level-tested: state.hms_errors is rebuilt from scratch on
  10019. # every push carrying an `hms` key, so the fault can come and go between
  10020. # 3-second polls. Seeing it once inside the dispatch window is enough.
  10021. command_rejected = False
  10022. # Latched for the same reason as command_rejected: drying can finish, or
  10023. # be stopped by the user, part-way through the dispatch window. Seeing it
  10024. # once is what matters — it is the state the printer was in when it
  10025. # declined to start (#2758).
  10026. drying_ams_ids: list[int] = []
  10027. # The printer answered the start with a refusal: nothing will start,
  10028. # and the command was not lost, so no reconnect either.
  10029. start_refusal: tuple[str, str | None] | None = None
  10030. deadline = time.monotonic() + timeout
  10031. while time.monotonic() < deadline:
  10032. await asyncio.sleep(poll_interval)
  10033. status = printer_manager.get_status(printer_id)
  10034. if not status:
  10035. # Printer disconnected — don't mess with the DB. Drop the
  10036. # in-memory dispatch hold too so a fresh dispatch can retry
  10037. # once the printer comes back; the hard timeout would
  10038. # otherwise hold the printer unnecessarily.
  10039. scheduler._release_dispatch_hold(printer_id)
  10040. return
  10041. last_status = status
  10042. if status.state in _ACTIVE_PRINT_STATES:
  10043. # Printer is actively processing the job — release the
  10044. # post-dispatch hold so the next pending item for this printer
  10045. # can be evaluated normally. We do NOT accept arbitrary state
  10046. # transitions: a printer going FINISH -> IDLE (user dismissed
  10047. # the post-print prompt without accepting our project_file)
  10048. # would otherwise look like "command landed" and leave the
  10049. # queue item stuck in 'printing' forever (#1370).
  10050. scheduler._release_dispatch_hold(printer_id)
  10051. scheduler._sent_unstarted_print.pop(queue_item_id, None)
  10052. try:
  10053. await ws_manager.send_queue_item_acked(
  10054. user_id=created_by_id,
  10055. queue_item_id=queue_item_id,
  10056. printer_id=printer_id,
  10057. )
  10058. except Exception:
  10059. pass
  10060. return
  10061. drying_ams_ids = drying_ams_ids or _drying_ams_ids(status)
  10062. # Checked only after the active-state exit above: a stale HMS left
  10063. # over from an earlier job must never abort a print that is visibly
  10064. # running. An actually-refused command leaves the printer idle, so
  10065. # this ordering costs the detection nothing.
  10066. if _mqtt_commands_rejected(status):
  10067. command_rejected = True
  10068. break
  10069. refusal = getattr(printer_manager.get_client(printer_id), "last_start_refusal", None)
  10070. if isinstance(refusal, tuple):
  10071. start_refusal = refusal
  10072. break
  10073. if pre_subtask_id is not None and status.subtask_id is not None and status.subtask_id != pre_subtask_id:
  10074. # Phase A exit — printer accepted the file (subtask_id flipped
  10075. # to our submission id). Don't return yet: the printer may
  10076. # have accepted the command but never actually start (e.g.
  10077. # cloud+LAN re-auth dance after a power cycle, #1678). Phase
  10078. # B watches for the active-state transition.
  10079. landed_on_subtask = True
  10080. break
  10081. if landed_on_subtask and not command_rejected:
  10082. phase_b_deadline = time.monotonic() + phase_b_timeout
  10083. while time.monotonic() < phase_b_deadline:
  10084. await asyncio.sleep(poll_interval)
  10085. status = printer_manager.get_status(printer_id)
  10086. if not status:
  10087. scheduler._release_dispatch_hold(printer_id)
  10088. return
  10089. last_status = status
  10090. if status.state in _ACTIVE_PRINT_STATES:
  10091. scheduler._release_dispatch_hold(printer_id)
  10092. scheduler._sent_unstarted_print.pop(queue_item_id, None)
  10093. try:
  10094. await ws_manager.send_queue_item_acked(
  10095. user_id=created_by_id,
  10096. queue_item_id=queue_item_id,
  10097. printer_id=printer_id,
  10098. )
  10099. except Exception:
  10100. pass
  10101. return
  10102. drying_ams_ids = drying_ams_ids or _drying_ams_ids(status)
  10103. # Same ordering rule as Phase A: a running print wins over a
  10104. # lingering HMS.
  10105. if _mqtt_commands_rejected(status):
  10106. command_rejected = True
  10107. break
  10108. # No active-state transition. Revert the item so the scheduler can retry.
  10109. # Drop the in-memory hold so the retry isn't blocked by it.
  10110. scheduler._release_dispatch_hold(printer_id)
  10111. # Logged on every failed dispatch window, not just the last one, so a
  10112. # support bundle shows the correlation from the first attempt rather than
  10113. # only after the retry budget is spent (#2758).
  10114. if drying_ams_ids:
  10115. logger.info(
  10116. "Queue item %s: printer %d never started while AMS %s drying — this may be why, see #2758",
  10117. queue_item_id,
  10118. printer_id,
  10119. ", ".join(str(i) for i in drying_ams_ids),
  10120. )
  10121. # Four outcomes from the revert attempt, each routed differently:
  10122. # "reverted": row flipped from printing -> pending, run recovery
  10123. # "gave_up": same, but the retry budget is spent — row failed
  10124. # rather than pending, so it stops going round again
  10125. # "already_moved_on": item.status != 'printing' (completed/cancelled by
  10126. # on_print_complete or user). Skip recovery entirely
  10127. # — the print clearly landed somewhere even if the
  10128. # watchdog didn't see the active-state transition.
  10129. # "revert_failed": SQLite contention exhausted retries. Still run
  10130. # recovery so the MQTT session gets a fresh client_id
  10131. # on the half-broken-session path.
  10132. #
  10133. # The retry budget (#2555): reverting to 'pending' hands the item straight
  10134. # back to the next queue pass, which re-uploads the whole 3MF and waits out
  10135. # the watchdog again. For a printer that is genuinely wedged that loop never
  10136. # ends — the reporter had one printer "since this morning still not launch"
  10137. # — and each lap also consumes an upload slot that the other printers in the
  10138. # farm are waiting on. Retrying is right; retrying forever is not.
  10139. async def _do_revert(db):
  10140. item = await db.get(PrintQueueItem, queue_item_id)
  10141. if not item or item.status != "printing":
  10142. return "already_moved_on"
  10143. item.dispatch_attempts = (item.dispatch_attempts or 0) + 1
  10144. item.started_at = None
  10145. # The print never started: no slot printed in any colour (#3279).
  10146. item.color_substitutions = None
  10147. # Charge the attempt to the candidate that was actually dispatched, so
  10148. # a cross-model item (#671) reaches for its other file next lap instead
  10149. # of retrying the printer that just failed to start. Matched by file
  10150. # because that is what the resolver copied onto the row.
  10151. if item.library_file_id is not None:
  10152. await db.execute(
  10153. update(PrintQueueVariant)
  10154. .where(PrintQueueVariant.queue_item_id == item.id)
  10155. .where(PrintQueueVariant.library_file_id == item.library_file_id)
  10156. .values(attempt_count=PrintQueueVariant.attempt_count + 1)
  10157. )
  10158. if command_rejected:
  10159. # No retry budget for this one: the printer refused to verify the
  10160. # command, and re-uploading the same 3MF to the same printer will
  10161. # be refused the same way. Fail now with the fix rather than after
  10162. # three laps of a message about SD cards (#2732).
  10163. item.status = "failed"
  10164. item.error_message = (
  10165. "The printer rejected the print command: MQTT command verification failed "
  10166. "(HMS 0500-0500-0001-0007). Enable Developer Mode on the printer, restart it, "
  10167. "then start the job again."
  10168. )
  10169. item.completed_at = datetime.now(timezone.utc)
  10170. await db.commit()
  10171. return "command_rejected"
  10172. if item.dispatch_attempts >= DISPATCH_MAX_ATTEMPTS:
  10173. item.status = "failed"
  10174. if drying_ams_ids:
  10175. # #2758: the generic message below sent the reporter looking
  10176. # at the SD card while the actual obstacle — AMS units in a
  10177. # drying cycle — was on screen the whole time. Name what we
  10178. # observed and let the user judge it; Bambuddy does not stop
  10179. # the cycle itself, because on this hardware drying can run
  10180. # alongside a print and stopping it may not be the fix.
  10181. units = ", ".join(f"AMS {i}" for i in drying_ams_ids)
  10182. item.error_message = (
  10183. f"The printer accepted the file but never started printing, after "
  10184. f"{item.dispatch_attempts} attempts. {units} "
  10185. f"{'was' if len(drying_ams_ids) == 1 else 'were'} drying throughout — "
  10186. f"some printers refuse to begin a print while an AMS is in a drying "
  10187. f"cycle, and an AMS drying without its external power supply can also "
  10188. f"leave too little power for the start-of-print calibration. Stop the "
  10189. f"drying, or connect the AMS power supply, and start the job again."
  10190. )
  10191. else:
  10192. item.error_message = (
  10193. f"The printer accepted the file but never started printing, after "
  10194. f"{item.dispatch_attempts} attempts. Check the printer's screen for a "
  10195. f"prompt or error, confirm its SD card is readable, and start the job again."
  10196. )
  10197. item.completed_at = datetime.now(timezone.utc)
  10198. await release_budget_reservation(
  10199. db,
  10200. source_type="print_queue",
  10201. source_id=item.id,
  10202. status="released",
  10203. )
  10204. await db.commit()
  10205. return "gave_up"
  10206. item.status = "pending"
  10207. await db.commit()
  10208. return "reverted"
  10209. try:
  10210. revert_outcome = await run_with_retry(_do_revert, label=f"watchdog revert item={queue_item_id}")
  10211. except Exception as e:
  10212. logger.warning(
  10213. "Queue item %s: failed to revert to 'pending' (printer %d): %s — "
  10214. "scheduler may keep treating this item as in-flight",
  10215. queue_item_id,
  10216. printer_id,
  10217. e,
  10218. )
  10219. revert_outcome = "revert_failed"
  10220. if revert_outcome in ("command_rejected", "gave_up"):
  10221. # The item is failed: its print user and expected print must not
  10222. # pass the printer's next print off as Bambuddy's.
  10223. scheduler._forget_unstarted_print(queue_item_id, unless_printing=True)
  10224. from backend.app.services.print_batch import refresh_batches_after
  10225. await refresh_batches_after([queue_item_id])
  10226. elif revert_outcome == "reverted":
  10227. # Another try claims the printer again. Until then no print on it
  10228. # is vouched for as this item's, except a late start of its file.
  10229. scheduler._forget_unstarted_print(queue_item_id, unless_printing=True, keep_expected=True)
  10230. elif revert_outcome == "already_moved_on":
  10231. scheduler._sent_unstarted_print.pop(queue_item_id, None)
  10232. if revert_outcome == "already_moved_on":
  10233. # Preserves the pre-#1370 early-return: if on_print_complete (or any
  10234. # other path) already moved the item past 'printing', don't run the
  10235. # MQTT session-recovery logic below — a forced reconnect on a healthy
  10236. # session breaks ongoing prints on the same printer.
  10237. return
  10238. total_timeout = timeout + (phase_b_timeout if landed_on_subtask else 0.0)
  10239. if revert_outcome == "command_rejected":
  10240. logger.error(
  10241. "Queue item %s: printer %d reported HMS %s (MQTT command verification "
  10242. "failed) — the print command was rejected, not lost. Failing the item "
  10243. "without retrying; enable Developer Mode on the printer and restart it (#2732)",
  10244. queue_item_id,
  10245. printer_id,
  10246. HMS_MQTT_VERIFY_FAILED,
  10247. )
  10248. await scheduler._notify_dispatch_gave_up(
  10249. queue_item_id,
  10250. printer_id,
  10251. created_by_id,
  10252. reason="Printer rejected the print command (MQTT command verification failed)",
  10253. )
  10254. # Same reasoning as the landed_on_subtask path below: the file is on
  10255. # the printer and a forced reconnect would only add 0500_4003 to a
  10256. # problem that has nothing to do with the MQTT session (#1150).
  10257. return
  10258. if revert_outcome == "gave_up":
  10259. logger.error(
  10260. "Queue item %s: printer %d never started the print after %d dispatch "
  10261. "attempts (last one waited %.0fs) — marking the item failed instead of "
  10262. "re-uploading it again (#2555)",
  10263. queue_item_id,
  10264. printer_id,
  10265. DISPATCH_MAX_ATTEMPTS,
  10266. total_timeout,
  10267. )
  10268. await scheduler._notify_dispatch_gave_up(queue_item_id, printer_id, created_by_id)
  10269. elif revert_outcome == "reverted":
  10270. if start_refusal is not None:
  10271. logger.warning(
  10272. "Queue item %s: printer %d refused the print start (result=%s, reason=%s) — "
  10273. "reverted to 'pending' for retry",
  10274. queue_item_id,
  10275. printer_id,
  10276. start_refusal[0],
  10277. start_refusal[1],
  10278. )
  10279. elif landed_on_subtask:
  10280. logger.warning(
  10281. "Queue item %s: printer %d accepted project_file (subtask_id "
  10282. "advanced) but never transitioned to an active state within "
  10283. "%.0fs — printer wedged post-acceptance; reverted to 'pending' "
  10284. "for retry (#1678)",
  10285. queue_item_id,
  10286. printer_id,
  10287. total_timeout,
  10288. )
  10289. else:
  10290. logger.warning(
  10291. "Queue item %s: printer %d did not respond to print command within "
  10292. "%.0fs (state still %s, subtask_id still %s) — reverted to 'pending' "
  10293. "for retry (#967)",
  10294. queue_item_id,
  10295. printer_id,
  10296. timeout,
  10297. pre_state,
  10298. pre_subtask_id,
  10299. )
  10300. # Phase B was entered iff subtask_id advanced, which means the
  10301. # project_file landed on the printer. A forced reconnect at this point
  10302. # would interrupt the printer's parse and trigger 0500_4003 (#1150) —
  10303. # skip the recovery entirely.
  10304. if landed_on_subtask or start_refusal is not None:
  10305. return
  10306. # Phase A timeout path: if the printer's gcode_file changed since
  10307. # pre-dispatch, the project_file command landed and the printer is
  10308. # parsing — a forced reconnect mid-parse triggers 0500_4003 (#1150).
  10309. # If gcode_file is unchanged, the publish was silently swallowed
  10310. # (#887/#936) and force_reconnect recovery is what we want.
  10311. client = printer_manager.get_client(printer_id)
  10312. current_gcode_file = getattr(last_status, "gcode_file", None) if last_status else None
  10313. publish_landed = current_gcode_file is not None and current_gcode_file != pre_gcode_file
  10314. if publish_landed:
  10315. logger.warning(
  10316. "Queue item %s: gcode_file changed to %r (was %r) — printer "
  10317. "received the command and is parsing slowly. Skipping forced "
  10318. "MQTT reconnect to avoid 0500_4003 mid-parse (#1150).",
  10319. queue_item_id,
  10320. current_gcode_file,
  10321. pre_gcode_file,
  10322. )
  10323. elif client and hasattr(client, "force_reconnect_stale_session"):
  10324. client.force_reconnect_stale_session(
  10325. f"queue print command unacknowledged after {timeout:.0f}s "
  10326. f"(state still {pre_state}, gcode_file {current_gcode_file!r})"
  10327. )
  10328. # Global scheduler instance
  10329. scheduler = PrintScheduler()