main.py 244 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224
  1. import asyncio
  2. import logging
  3. import mimetypes as _mimetypes
  4. import os
  5. import posixpath
  6. import time
  7. from contextlib import asynccontextmanager
  8. from datetime import datetime, timedelta, timezone
  9. from logging.handlers import RotatingFileHandler
  10. from urllib.parse import urlparse
  11. from fastapi import FastAPI
  12. from fastapi.responses import FileResponse
  13. from fastapi.staticfiles import StaticFiles
  14. from sqlalchemy import delete, or_, select, text
  15. from backend.app.api.routes import (
  16. ams_history,
  17. api_keys,
  18. archive_purge,
  19. archives,
  20. auth,
  21. background_dispatch as background_dispatch_routes,
  22. bug_report,
  23. camera,
  24. cloud,
  25. discovery,
  26. external_links,
  27. filaments,
  28. firmware,
  29. github_backup,
  30. groups,
  31. inventory,
  32. kprofiles,
  33. labels,
  34. library,
  35. library_trash,
  36. local_backup,
  37. local_presets,
  38. maintenance,
  39. makerworld,
  40. metrics,
  41. mfa,
  42. notification_templates,
  43. notifications,
  44. obico,
  45. pending_uploads,
  46. print_log,
  47. print_queue,
  48. printers,
  49. projects,
  50. settings as settings_routes,
  51. slice_jobs,
  52. slicer_presets,
  53. smart_plugs,
  54. spoolbuddy,
  55. spoolman,
  56. support,
  57. system,
  58. updates,
  59. user_notifications,
  60. users,
  61. virtual_printers,
  62. webhook,
  63. websocket,
  64. )
  65. from backend.app.api.routes.maintenance import _get_printer_maintenance_internal, ensure_default_types
  66. from backend.app.api.routes.support import init_debug_logging
  67. from backend.app.core.config import APP_VERSION, settings as app_settings
  68. from backend.app.core.database import async_session, engine, init_db
  69. from backend.app.core.websocket import ws_manager
  70. from backend.app.models.smart_plug import SmartPlug
  71. from backend.app.services.archive import ArchiveService, peek_plate_index_in_3mf, swap_plate_suffix
  72. from backend.app.services.archive_purge import archive_purge_service
  73. from backend.app.services.background_dispatch import background_dispatch
  74. from backend.app.services.bambu_ftp import (
  75. FileNotOnPrinterError,
  76. cache_3mf_download,
  77. clear_3mf_cache,
  78. download_file_async,
  79. get_cached_3mf,
  80. get_ftp_retry_settings,
  81. with_ftp_retry,
  82. )
  83. from backend.app.services.bambu_mqtt import PrinterState
  84. from backend.app.services.github_backup import github_backup_service
  85. from backend.app.services.homeassistant import homeassistant_service
  86. from backend.app.services.library_trash import library_trash_service
  87. from backend.app.services.local_backup import local_backup_service
  88. from backend.app.services.mqtt_relay import mqtt_relay
  89. from backend.app.services.mqtt_smart_plug import mqtt_smart_plug_service
  90. from backend.app.services.notification_service import notification_service
  91. from backend.app.services.obico_detection import obico_detection_service
  92. from backend.app.services.print_scheduler import scheduler as print_scheduler
  93. from backend.app.services.printer_manager import (
  94. init_printer_connections,
  95. parse_plate_id,
  96. printer_manager,
  97. printer_state_to_dict,
  98. )
  99. from backend.app.services.smart_plug_manager import smart_plug_manager
  100. from backend.app.services.spool_assignment_notifications import (
  101. notify_missing_spool_assignments_on_print_start,
  102. )
  103. from backend.app.services.spoolman import close_spoolman_client, get_spoolman_client, init_spoolman_client
  104. from backend.app.services.spoolman_tracking import (
  105. cleanup_tracking as _cleanup_spoolman_tracking,
  106. report_usage as _report_spoolman_usage,
  107. store_print_data as _store_spoolman_print_data,
  108. )
  109. from backend.app.services.tasmota import tasmota_service
  110. # =============================================================================
  111. # Dependency Check - runs before other imports to give helpful error messages
  112. # =============================================================================
  113. def _start_error_server(missing_packages: list):
  114. """Start a minimal HTTP server to display dependency errors in browser."""
  115. import os
  116. import signal
  117. from http.server import BaseHTTPRequestHandler, HTTPServer
  118. packages_html = "".join(f"<li><code>{p}</code></li>" for p in missing_packages)
  119. html = f"""<!DOCTYPE html>
  120. <html>
  121. <head>
  122. <title>Bambuddy - Setup Required</title>
  123. <style>
  124. body {{
  125. font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
  126. background: #0f172a; color: #e2e8f0;
  127. display: flex; justify-content: center; align-items: center;
  128. min-height: 100vh; margin: 0; padding: 20px; box-sizing: border-box;
  129. }}
  130. .container {{
  131. background: #1e293b; border-radius: 12px; padding: 40px;
  132. max-width: 600px; text-align: center; box-shadow: 0 4px 20px rgba(0,0,0,0.3);
  133. }}
  134. h1 {{ color: #f87171; margin-bottom: 10px; }}
  135. h2 {{ color: #94a3b8; font-weight: normal; margin-top: 0; }}
  136. .packages {{
  137. background: #0f172a; border-radius: 8px; padding: 20px;
  138. margin: 20px 0; text-align: left;
  139. }}
  140. .packages ul {{ margin: 0; padding-left: 20px; }}
  141. .packages li {{ color: #fbbf24; margin: 8px 0; }}
  142. .command {{
  143. background: #0f172a; border-radius: 8px; padding: 15px 20px;
  144. margin: 15px 0; font-family: monospace; color: #4ade80;
  145. text-align: left; overflow-x: auto;
  146. }}
  147. .note {{ color: #94a3b8; font-size: 14px; margin-top: 20px; }}
  148. </style>
  149. </head>
  150. <body>
  151. <div class="container">
  152. <h1>Setup Required</h1>
  153. <h2>Missing Python packages</h2>
  154. <div class="packages"><ul>{packages_html}</ul></div>
  155. <p>To fix, run this command on your server:</p>
  156. <div class="command">pip install -r requirements.txt</div>
  157. <p>Or if using a virtual environment:</p>
  158. <div class="command">./venv/bin/pip install -r requirements.txt</div>
  159. <p class="note">After installing, restart Bambuddy:<br>
  160. <code>sudo systemctl restart bambuddy</code></p>
  161. </div>
  162. </body>
  163. </html>"""
  164. class ErrorHandler(BaseHTTPRequestHandler):
  165. def do_GET(self):
  166. self.send_response(503)
  167. self.send_header("Content-type", "text/html")
  168. self.end_headers()
  169. self.wfile.write(html.encode())
  170. def log_message(self, format, *args):
  171. print(f"[Error Server] {args[0]}")
  172. port = int(os.environ.get("PORT", 8000))
  173. print(f"\nStarting error server on http://0.0.0.0:{port}")
  174. print("Visit this URL in your browser to see the error details.\n")
  175. server = HTTPServer(("0.0.0.0", port), ErrorHandler) # nosec B104
  176. def shutdown(signum, frame):
  177. print("\nShutting down error server...")
  178. raise SystemExit(0)
  179. signal.signal(signal.SIGTERM, shutdown)
  180. signal.signal(signal.SIGINT, shutdown)
  181. server.serve_forever()
  182. def check_dependencies():
  183. """Check that all required packages are installed."""
  184. missing = []
  185. # Map of import name -> package name (for pip install)
  186. required = {
  187. "jwt": "PyJWT",
  188. "fastapi": "fastapi",
  189. "uvicorn": "uvicorn",
  190. "sqlalchemy": "sqlalchemy",
  191. "aiosqlite": "aiosqlite",
  192. "pydantic": "pydantic",
  193. "paho.mqtt": "paho-mqtt",
  194. }
  195. for module, package in required.items():
  196. try:
  197. __import__(module)
  198. except ImportError:
  199. missing.append(package)
  200. if missing:
  201. print("\n" + "=" * 60)
  202. print("ERROR: Missing required Python packages!")
  203. print("=" * 60)
  204. print(f"\nMissing packages: {', '.join(missing)}")
  205. print("\nTo fix, run:")
  206. print(" pip install -r requirements.txt")
  207. print("\nOr if using a virtual environment:")
  208. print(" ./venv/bin/pip install -r requirements.txt")
  209. print("=" * 60 + "\n")
  210. _start_error_server(missing)
  211. check_dependencies()
  212. # =============================================================================
  213. # Import settings first for logging configuration
  214. # Configure logging based on settings
  215. # DEBUG=true -> DEBUG level, else use LOG_LEVEL setting
  216. log_level_str = "DEBUG" if app_settings.debug else app_settings.log_level.upper()
  217. log_level = getattr(logging, log_level_str, logging.INFO)
  218. # Trace ID column ([-] when no request scope is active — startup, MQTT
  219. # callbacks, scheduled tasks not chained from a request — so the column
  220. # stays visually aligned and missing values are obvious in grep). See
  221. # backend/app/core/trace.py for the ContextVar that feeds this slot.
  222. log_format = "%(asctime)s %(levelname)s [%(name)s] [%(trace_id)s] %(message)s"
  223. # Create root logger
  224. root_logger = logging.getLogger()
  225. root_logger.setLevel(log_level)
  226. # Trace-ID injection: this filter populates record.trace_id from the
  227. # per-request ContextVar so the format string above can reference it.
  228. # Attached to each HANDLER (not the root logger) because Python's
  229. # logging semantics only invoke a logger's filters on records that
  230. # *originated* at that logger — records propagated up from child
  231. # loggers (every named logger in the app) never trigger root's filter.
  232. # Putting it on the handlers means every record any handler emits gets
  233. # trace_id injected just before the formatter runs, regardless of which
  234. # logger created the record. Without this, the formatter raises
  235. # KeyError on every child-logger record and the record is silently
  236. # dropped — which is exactly the "logs/bambuddy.log only shows logs
  237. # partially" bug we hit. See backend/app/core/trace.py for the
  238. # ContextVar the filter reads.
  239. from backend.app.core.trace import TraceIDFilter
  240. _trace_id_filter = TraceIDFilter()
  241. # Console handler - always enabled
  242. console_handler = logging.StreamHandler()
  243. console_handler.setLevel(log_level)
  244. console_handler.setFormatter(logging.Formatter(log_format))
  245. console_handler.addFilter(_trace_id_filter)
  246. root_logger.addHandler(console_handler)
  247. # File handler - only in production or if explicitly enabled
  248. if app_settings.log_to_file:
  249. log_file = app_settings.log_dir / "bambuddy.log"
  250. file_handler = RotatingFileHandler(
  251. log_file,
  252. maxBytes=5 * 1024 * 1024, # 5MB
  253. backupCount=3,
  254. encoding="utf-8",
  255. )
  256. file_handler.setLevel(log_level)
  257. file_handler.setFormatter(logging.Formatter(log_format))
  258. file_handler.addFilter(_trace_id_filter)
  259. root_logger.addHandler(file_handler)
  260. logging.info("Logging to file: %s", log_file)
  261. # Pipe uvicorn's HTTP access log to bambuddy.log too. Uvicorn ships its
  262. # access logger with propagate=False by default, so without this attach
  263. # there is no on-disk record of which endpoint triggered a server-state
  264. # change — the rogue stop_print mystery on 2026-04-26 was untraceable
  265. # for exactly this reason. Filtered to write methods only
  266. # (POST/PUT/PATCH/DELETE) so the high-volume status-poll GETs from the
  267. # frontend don't churn the rotation window faster than it's useful.
  268. from backend.app.core.logging_filters import (
  269. CancelledPoolNoiseFilter,
  270. WriteRequestsOnlyFilter,
  271. )
  272. uvicorn_access_logger = logging.getLogger("uvicorn.access")
  273. uvicorn_access_logger.addHandler(file_handler)
  274. uvicorn_access_logger.addFilter(WriteRequestsOnlyFilter())
  275. # Uvicorn's access logger has propagate=False (its own default), so the
  276. # root-attached TraceIDFilter never sees these records. Attach a
  277. # second instance directly so HTTP access lines carry the same trace
  278. # ID column as the application logs they correlate with.
  279. uvicorn_access_logger.addFilter(TraceIDFilter())
  280. # Drop SQLAlchemy connection-pool log noise that's caused by Starlette's
  281. # BaseHTTPMiddleware cancelling the inner task scope on client
  282. # disconnect (#1112). The cancel-safe `get_db` already prevents the
  283. # underlying transaction leak; this filter only suppresses the residual
  284. # log records that pre-existing pools still emit during their cleanup.
  285. logging.getLogger("sqlalchemy.pool").addFilter(CancelledPoolNoiseFilter())
  286. # Reduce noise from third-party libraries in production
  287. if not app_settings.debug:
  288. logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
  289. logging.getLogger("httpcore").setLevel(logging.WARNING)
  290. logging.getLogger("httpx").setLevel(logging.WARNING)
  291. logging.getLogger("paho.mqtt").setLevel(logging.WARNING)
  292. logging.info("Bambuddy starting - debug=%s, log_level=%s", app_settings.debug, log_level_str)
  293. # Track active prints: {(printer_id, filename): archive_id}
  294. _active_prints: dict[tuple[int, str], int] = {}
  295. # Track expected prints from reprint/scheduled (skip auto-archiving for these)
  296. # {(printer_id, filename): archive_id}
  297. _expected_prints: dict[tuple[int, str], int] = {}
  298. # Track AMS mapping for prints: {archive_id: [global_tray_id_per_slot]}
  299. # Used by usage tracker to map 3MF slots to physical AMS trays
  300. _print_ams_mappings: dict[int, list[int]] = {}
  301. # Track progress milestones for notifications: {printer_id: last_milestone_notified}
  302. # Milestones are 25, 50, 75. Value of 0 means no milestone notified yet for current print.
  303. _last_progress_milestone: dict[int, int] = {}
  304. # Track whether first layer complete notification has been sent for current print
  305. _first_layer_notified: dict[int, bool] = {}
  306. # Track HMS errors that have been notified: {printer_id: set of error codes}
  307. # This prevents sending duplicate notifications for the same error
  308. _notified_hms_errors: dict[int, set[str]] = {}
  309. # Track when HMS errors were last seen: {printer_id: timestamp}
  310. # Used to debounce clearing — prevents flapping errors from re-triggering notifications
  311. _hms_last_seen: dict[int, float] = {}
  312. _HMS_CLEAR_GRACE_SECONDS = 30.0
  313. # Track timelapse file baselines at print start: {printer_id: set of video filenames}
  314. # Used for snapshot-diff detection at print completion
  315. _timelapse_baselines: dict[int, set[str]] = {}
  316. # Track printers waiting for bed to cool after print completion.
  317. # Event-driven: fires when bed_temper arrives via MQTT below threshold.
  318. # {printer_id: {"threshold": float, "filename": str, "registered_at": float}}
  319. _bed_cool_waiters: dict[int, dict] = {}
  320. # Track printers where the user explicitly stopped the print from the queue UI.
  321. # When on_print_complete fires with status "failed" for these printers we treat it
  322. # as "cancelled" (stopped by user) so the correct notification email is sent.
  323. _user_stopped_printers: set[int] = set()
  324. # HMS short-code → human-readable failure reason. Used by _dispatch_archive_update
  325. # when status="failed" to label the print's failure_reason in archives.
  326. #
  327. # Earlier code matched on `module` alone (e.g. "any module 0x0C HMS → Layer shift"),
  328. # which is wrong on two counts:
  329. # 1. Real layer-shift codes live in module 0x03 (see Bambu wiki), not 0x0C.
  330. # 2. Module 0x0C is "Motion Controller" — broad category that also covers cameras
  331. # and visual markers, AND the H2D firmware emits a 0x0C HMS (0C00_001B, not in
  332. # the public wiki) as part of its user-cancel sequence. Matching on the module
  333. # alone caused user-cancellations to be archived as "Layer shift" failures.
  334. # We now match by full short code only — anything not in this map leaves
  335. # failure_reason=None rather than guessing.
  336. _HMS_FAILURE_REASONS: dict[str, str] = {
  337. # Layer shift / step loss
  338. "0300_4057": "Layer shift",
  339. "0300_4068": "Layer shift",
  340. "0300_800C": "Layer shift",
  341. # Filament runout (printer-side & per-AMS-slot)
  342. "0300_8004": "Filament runout",
  343. "0700_8011": "Filament runout",
  344. "0701_8011": "Filament runout",
  345. "0702_8011": "Filament runout",
  346. "0703_8011": "Filament runout",
  347. "0704_8011": "Filament runout",
  348. "0705_8011": "Filament runout",
  349. "0706_8011": "Filament runout",
  350. "0707_8011": "Filament runout",
  351. "07FF_8011": "Filament runout",
  352. # Clogged nozzle / extruder
  353. "0300_4006": "Clogged nozzle",
  354. "0300_8016": "Clogged nozzle",
  355. "0300_801C": "Clogged nozzle",
  356. "0700_8003": "Clogged nozzle",
  357. "0700_8007": "Clogged nozzle",
  358. "0700_8013": "Clogged nozzle",
  359. "0701_8003": "Clogged nozzle",
  360. "0701_8007": "Clogged nozzle",
  361. "0701_8013": "Clogged nozzle",
  362. "0702_8003": "Clogged nozzle",
  363. }
  364. def _hms_short_code(attr: int, code: int | str) -> str:
  365. """Build the canonical "MMMM_CCCC" HMS short code from raw attr/code values."""
  366. if isinstance(code, str):
  367. code_int = int(code.replace("0x", ""), 16) if code else 0
  368. else:
  369. code_int = int(code or 0)
  370. attr_int = int(attr or 0)
  371. return f"{(attr_int >> 16) & 0xFFFF:04X}_{code_int & 0xFFFF:04X}"
  372. def derive_failure_reason(status: str, hms_errors: list[dict] | None) -> str | None:
  373. """Derive a human-readable failure_reason for an archived print.
  374. Returns "User cancelled" for cancelled/aborted prints; for failed prints,
  375. returns the first matching reason from _HMS_FAILURE_REASONS, or None when
  376. no HMS code matches (don't guess — null is honest).
  377. """
  378. if status in ("aborted", "cancelled"):
  379. return "User cancelled"
  380. if status != "failed":
  381. return None
  382. for err in hms_errors or []:
  383. short_code = _hms_short_code(err.get("attr", 0), err.get("code", 0))
  384. if short_code in _HMS_FAILURE_REASONS:
  385. return _HMS_FAILURE_REASONS[short_code]
  386. return None
  387. # Track created_by_id for expected prints so the user email can be sent even when
  388. # the archive itself doesn't have created_by_id set (e.g. library-file-based prints).
  389. # {(printer_id, filename): created_by_id}
  390. _expected_print_creators: dict[tuple[int, str], int] = {}
  391. # Per-printer lock that serialises the spool-assignment side of on_ams_change
  392. # (auto-unlink stale + auto-assign new) when MQTT bursts deliver multiple AMS
  393. # updates for the same printer in quick succession (~30 ms apart, observed in
  394. # the wild on H2D + dual AMS).
  395. #
  396. # Without this serialisation, two concurrent on_ams_change callbacks each read
  397. # "no assignment for (printer, ams, tray)", each call auto_assign_spool, and
  398. # the second commit hits
  399. # IntegrityError: duplicate key value violates unique constraint
  400. # "spool_assignment_printer_id_ams_id_tray_id_key"
  401. # SQLite's WAL serial-write semantics had been silently swallowing the race
  402. # until optional Postgres support landed (asyncpg allows true concurrent
  403. # transactions and surfaces the constraint violation).
  404. #
  405. # Scope is intentionally narrow: only the two DB-mutating blocks (unlink +
  406. # assign) are inside the lock. The Spoolman sync block further down stays
  407. # concurrent because it's network-bound and idempotent.
  408. _ams_assignment_locks: dict[int, asyncio.Lock] = {}
  409. def _get_ams_assignment_lock(printer_id: int) -> asyncio.Lock:
  410. """Return the per-printer assignment lock, creating it on first use."""
  411. lock = _ams_assignment_locks.get(printer_id)
  412. if lock is None:
  413. lock = asyncio.Lock()
  414. _ams_assignment_locks[printer_id] = lock
  415. return lock
  416. # TTL for expected-print entries: evict registrations older than this to prevent
  417. # unbounded growth when a print is registered but never starts (e.g. printer
  418. # disconnect, app restart, print started from the printer panel).
  419. _EXPECTED_PRINT_TTL_SECONDS: int = 2 * 60 * 60 # 2 hours
  420. # Registration timestamps used for TTL eviction: {(printer_id, filename): monotonic_time}
  421. _expected_print_registered_at: dict[tuple[int, str], float] = {}
  422. # Cleanup loop interval
  423. _EXPECTED_PRINT_CLEANUP_INTERVAL: int = 15 * 60 # 15 minutes
  424. _expected_prints_cleanup_task: asyncio.Task | None = None
  425. async def _get_plug_energy(plug, db) -> dict | None:
  426. """Get energy from plug regardless of type (Tasmota, Home Assistant, MQTT, or REST).
  427. For HA plugs, configures the service with current settings from DB.
  428. For MQTT plugs, returns data from the subscription service.
  429. For REST plugs, polls the status URL with JSON path extraction.
  430. """
  431. if plug.plug_type == "homeassistant":
  432. from backend.app.api.routes.settings import get_homeassistant_settings
  433. ha_settings = await get_homeassistant_settings(db)
  434. homeassistant_service.configure(ha_settings["ha_url"], ha_settings["ha_token"])
  435. return await homeassistant_service.get_energy(plug)
  436. elif plug.plug_type == "mqtt":
  437. # MQTT plugs report "today" energy, not lifetime total
  438. # For per-print tracking, we use "today" as the counter (resets at midnight)
  439. mqtt_data = mqtt_relay.smart_plug_service.get_plug_data(plug.id)
  440. if mqtt_data:
  441. return {
  442. "power": mqtt_data.power,
  443. "today": mqtt_data.energy,
  444. "total": mqtt_data.energy, # Use today as total for per-print calculations
  445. }
  446. return None
  447. elif plug.plug_type == "rest":
  448. from backend.app.services.rest_smart_plug import rest_smart_plug_service
  449. return await rest_smart_plug_service.get_energy(plug)
  450. else:
  451. return await tasmota_service.get_energy(plug)
  452. async def _record_energy_start(archive, printer_id: int, db, *, context: str = "") -> bool:
  453. """Capture the smart plug lifetime counter on the archive at print start.
  454. Persists `energy_start_kwh` on the archive row (#941) so per-print energy
  455. tracking survives a backend restart mid-print. The print-end handler reads
  456. this value back from the DB and computes the delta against the current
  457. plug counter.
  458. """
  459. _logger = logging.getLogger(__name__)
  460. try:
  461. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  462. plug = plug_result.scalar_one_or_none()
  463. if not plug:
  464. _logger.info("[ENERGY] No smart plug for printer %s (archive %s)", printer_id, archive.id)
  465. return False
  466. energy = await _get_plug_energy(plug, db)
  467. if not energy or energy.get("total") is None:
  468. _logger.warning("[ENERGY] No 'total' in energy response for archive %s", archive.id)
  469. return False
  470. archive.energy_start_kwh = float(energy["total"])
  471. await db.commit()
  472. _logger.info(
  473. "[ENERGY] Recorded starting energy%s for archive %s: %s kWh",
  474. f" ({context})" if context else "",
  475. archive.id,
  476. energy["total"],
  477. )
  478. return True
  479. except Exception as e:
  480. _logger.warning("[ENERGY] Failed to record starting energy for archive %s: %s", archive.id, e)
  481. return False
  482. def register_expected_print(
  483. printer_id: int,
  484. filename: str,
  485. archive_id: int,
  486. ams_mapping: list[int] | None = None,
  487. created_by_id: int | None = None,
  488. ):
  489. """Register an expected print from reprint/scheduled so we don't create duplicate archives."""
  490. # Store with multiple filename variations to catch different naming patterns
  491. _expected_prints[(printer_id, filename)] = archive_id
  492. # Also store without .3mf extension if present
  493. if filename.endswith(".3mf"):
  494. base = filename[:-4]
  495. _expected_prints[(printer_id, base)] = archive_id
  496. _expected_prints[(printer_id, f"{base}.gcode")] = archive_id
  497. # Store AMS mapping for usage tracking at print completion
  498. if ams_mapping is not None:
  499. _print_ams_mappings[archive_id] = ams_mapping
  500. # Store created_by_id so the user start email can be sent even when the archive
  501. # itself has no created_by_id (e.g. library-file-based queue prints)
  502. if created_by_id is not None:
  503. _expected_print_creators[(printer_id, filename)] = created_by_id
  504. if filename.endswith(".3mf"):
  505. base = filename[:-4]
  506. _expected_print_creators[(printer_id, base)] = created_by_id
  507. _expected_print_creators[(printer_id, f"{base}.gcode")] = created_by_id
  508. # Record registration time for TTL-based eviction
  509. _registered_at = time.monotonic()
  510. _expected_print_registered_at[(printer_id, filename)] = _registered_at
  511. if filename.endswith(".3mf"):
  512. base = filename[:-4]
  513. _expected_print_registered_at[(printer_id, base)] = _registered_at
  514. _expected_print_registered_at[(printer_id, f"{base}.gcode")] = _registered_at
  515. logging.getLogger(__name__).info(
  516. f"Registered expected print: printer={printer_id}, file={filename}, archive={archive_id}, ams_mapping={ams_mapping}"
  517. )
  518. def _get_start_ams_mapping(data: dict, archive_id: int | None) -> list[int] | None:
  519. """Resolve AMS mapping for print start without consuming stored queue/reprint state."""
  520. stored_ams_mapping = data.get("ams_mapping")
  521. if not stored_ams_mapping and archive_id:
  522. stored_ams_mapping = _print_ams_mappings.get(archive_id)
  523. return stored_ams_mapping
  524. def _format_hms_error_summary(hms_errors: list[dict]) -> str | None:
  525. """Build a human-readable failure reason from MQTT hms_errors for PrintQueueItem.error_message.
  526. Each entry has keys: code ('0x4038'), attr (32-bit int), module, severity.
  527. The short code used for the hms_errors.py lookup table is 'MMMM_EEEE' — module
  528. from attr bits 16-31, error from the numeric part of code. Falls back to the raw
  529. short code when no description is on file. Returns None for an empty list so
  530. callers can leave error_message unset.
  531. """
  532. if not hms_errors:
  533. return None
  534. from backend.app.services.hms_errors import get_error_description
  535. parts: list[str] = []
  536. for err in hms_errors:
  537. try:
  538. code_str = str(err.get("code", "")).replace("0x", "")
  539. error_num = int(code_str, 16) if code_str else 0
  540. module_num = (int(err.get("attr", 0)) >> 16) & 0xFFFF
  541. short_code = f"{module_num:04X}_{error_num:04X}"
  542. except (TypeError, ValueError):
  543. continue
  544. description = get_error_description(short_code)
  545. parts.append(f"[{short_code}] {description}" if description else f"[{short_code}]")
  546. return "; ".join(parts) if parts else None
  547. async def _bump_library_file_usage_if_completed(db, item, queue_status: str) -> None:
  548. """Increment LibraryFile.print_count and stamp last_printed_at when a queued
  549. print completes successfully. Gated to status=='completed': failed, cancelled
  550. and aborted prints do not count as usage. Caller is responsible for committing
  551. the session. No-op when the queue item has no linked library file (e.g. reprints
  552. from an archive). See #1008."""
  553. if queue_status != "completed" or item.library_file_id is None:
  554. return
  555. from backend.app.models.library import LibraryFile
  556. lib_file = await db.scalar(select(LibraryFile).where(LibraryFile.id == item.library_file_id))
  557. if lib_file is None:
  558. return
  559. lib_file.print_count = (lib_file.print_count or 0) + 1
  560. lib_file.last_printed_at = datetime.now(timezone.utc)
  561. def mark_printer_stopped_by_user(printer_id: int) -> None:
  562. """Mark that the active print on this printer was stopped by the user from the queue UI.
  563. When on_print_complete fires with status 'failed' for a printer in this set we
  564. reclassify it as 'cancelled' so the correct 'print stopped' notification is sent
  565. rather than a 'print failed' notification.
  566. """
  567. _user_stopped_printers.add(printer_id)
  568. logging.getLogger(__name__).info("Marked printer %s as user-stopped from queue", printer_id)
  569. _last_status_broadcast: dict[int, str] = {}
  570. # Track printers where we've updated nozzle_count
  571. _nozzle_count_updated: set[int] = set()
  572. async def on_printer_status_change(printer_id: int, state: PrinterState):
  573. """Handle printer status changes - broadcast via WebSocket."""
  574. # Only broadcast if something meaningful changed (reduce WebSocket spam)
  575. # Include rounded temperatures to detect meaningful temp changes (within 1 degree)
  576. temps = state.temperatures or {}
  577. nozzle_temp = round(temps.get("nozzle", 0))
  578. bed_temp = round(temps.get("bed", 0))
  579. nozzle_2_temp = round(temps.get("nozzle_2", 0)) if "nozzle_2" in temps else ""
  580. chamber_temp = round(temps.get("chamber", 0)) if "chamber" in temps else ""
  581. # Auto-detect dual-nozzle printers from MQTT temperature data
  582. if "nozzle_2" in temps and printer_id not in _nozzle_count_updated:
  583. _nozzle_count_updated.add(printer_id)
  584. # Update nozzle_count in database
  585. async with async_session() as db:
  586. from backend.app.models.printer import Printer
  587. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  588. printer = result.scalar_one_or_none()
  589. if printer and printer.nozzle_count != 2:
  590. printer.nozzle_count = 2
  591. await db.commit()
  592. logging.getLogger(__name__).info(
  593. f"Auto-detected dual-nozzle printer {printer_id}, updated nozzle_count=2"
  594. )
  595. # Include target temps for heating phase detection
  596. bed_target = round(temps.get("bed_target", 0))
  597. nozzle_target = round(temps.get("nozzle_target", 0))
  598. # Include tray_now and vt_tray hash so external spool changes trigger broadcasts
  599. vt_tray_key = hash(str(state.raw_data.get("vt_tray", []))) if state.raw_data else 0
  600. # Include AMS dry_time and tray state values so drying/slot changes trigger broadcasts
  601. ams_dry_key = tuple(a.get("dry_time", 0) for a in (state.raw_data.get("ams") or [])) if state.raw_data else ()
  602. # Include tray states so load/unload transitions (state 11→10) trigger broadcasts (#784)
  603. ams_tray_key = (
  604. tuple(
  605. (t.get("id"), t.get("tray_type", ""), t.get("state"))
  606. for a in (state.raw_data.get("ams") or [])
  607. for t in a.get("tray", [])
  608. )
  609. if state.raw_data
  610. else ()
  611. )
  612. status_key = (
  613. f"{state.connected}:{state.state}:{state.progress}:{state.layer_num}:"
  614. f"{nozzle_temp}:{bed_temp}:{nozzle_2_temp}:{chamber_temp}:"
  615. f"{state.stg_cur}:{bed_target}:{nozzle_target}:"
  616. f"{state.cooling_fan_speed}:{state.big_fan1_speed}:{state.big_fan2_speed}:"
  617. f"{state.chamber_light}:{state.active_extruder}:{state.tray_now}:{vt_tray_key}:"
  618. f"{ams_dry_key}:{ams_tray_key}:{state.door_open}"
  619. )
  620. # MQTT relay - publish status (before dedup check - always publish to MQTT)
  621. try:
  622. printer_info = printer_manager.get_printer(printer_id)
  623. if printer_info:
  624. await mqtt_relay.on_printer_status(printer_id, state, printer_info.name, printer_info.serial_number)
  625. except Exception:
  626. pass # Don't fail status callback if MQTT fails
  627. if _last_status_broadcast.get(printer_id) == status_key:
  628. return # No change, skip WebSocket broadcast
  629. _last_status_broadcast[printer_id] = status_key
  630. # Check for progress milestone notifications (25%, 50%, 75%)
  631. progress = state.progress or 0
  632. is_printing = state.state in ("RUNNING", "PRINTING")
  633. if is_printing and progress > 0:
  634. # Determine which milestone we've reached
  635. current_milestone = 0
  636. if progress >= 75:
  637. current_milestone = 75
  638. elif progress >= 50:
  639. current_milestone = 50
  640. elif progress >= 25:
  641. current_milestone = 25
  642. last_milestone = _last_progress_milestone.get(printer_id, 0)
  643. # If we've crossed a new milestone, send notification
  644. if current_milestone > last_milestone:
  645. _last_progress_milestone[printer_id] = current_milestone
  646. try:
  647. async with async_session() as db:
  648. from backend.app.models.printer import Printer
  649. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  650. printer = result.scalar_one_or_none()
  651. printer_name = printer.name if printer else f"Printer {printer_id}"
  652. filename = state.subtask_name or state.gcode_file or "Unknown"
  653. # remaining_time is in minutes, convert to seconds for notification
  654. remaining_time_seconds = state.remaining_time * 60 if state.remaining_time else None
  655. # Capture camera snapshot for notification image attachment
  656. image_data = await _capture_snapshot_for_notification(
  657. printer_id, printer, logging.getLogger(__name__)
  658. )
  659. await notification_service.on_print_progress(
  660. printer_id,
  661. printer_name,
  662. filename,
  663. current_milestone,
  664. db,
  665. remaining_time_seconds,
  666. image_data=image_data,
  667. )
  668. except Exception as e:
  669. logging.getLogger(__name__).warning(f"Progress milestone notification failed: {e}")
  670. elif progress < 5:
  671. # Reset milestone tracking when print restarts or new print begins
  672. _last_progress_milestone[printer_id] = 0
  673. _first_layer_notified[printer_id] = False
  674. # HMS error codes that should not trigger notifications even though they
  675. # have known descriptions (e.g. user-initiated actions, not real errors).
  676. _HMS_NOTIFICATION_SUPPRESS = {
  677. "0500_400E", # Printing was cancelled (user action, not an error)
  678. }
  679. # Check for new HMS errors and send notifications
  680. current_hms_errors = getattr(state, "hms_errors", []) or []
  681. if current_hms_errors:
  682. # Build set of current error codes (using attr for uniqueness)
  683. current_error_codes = {f"{e.attr:08x}" for e in current_hms_errors}
  684. previously_notified = _notified_hms_errors.get(printer_id, set())
  685. # Find new errors that haven't been notified yet
  686. new_error_codes = current_error_codes - previously_notified
  687. # Update tracking immediately to prevent duplicate notifications from concurrent callbacks
  688. _notified_hms_errors[printer_id] = current_error_codes
  689. _hms_last_seen[printer_id] = time.time()
  690. if new_error_codes:
  691. # Get the actual new errors for the notification
  692. # Filter to severity >= 2 (skip informational/status messages like H2D sends)
  693. new_errors = [e for e in current_hms_errors if f"{e.attr:08x}" in new_error_codes and e.severity >= 2]
  694. try:
  695. async with async_session() as db:
  696. from backend.app.models.printer import Printer
  697. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  698. printer = result.scalar_one_or_none()
  699. printer_name = printer.name if printer else f"Printer {printer_id}"
  700. # Format error details for notification
  701. # Module 0x07 = AMS/Filament, 0x05 = Nozzle, 0x0C = Motion Controller, etc.
  702. module_names = {
  703. 0x03: "Print/Task",
  704. 0x05: "Nozzle/Extruder",
  705. 0x07: "AMS/Filament",
  706. 0x0C: "Motion Controller",
  707. 0x12: "Chamber",
  708. }
  709. from backend.app.services.hms_errors import get_error_description
  710. # Capture camera snapshot once for all error notifications
  711. error_image_data = await _capture_snapshot_for_notification(
  712. printer_id, printer, logging.getLogger(__name__)
  713. )
  714. sent_count = 0
  715. for error in new_errors:
  716. module_name = module_names.get(error.module, f"Module 0x{error.module:02X}")
  717. # Build short code like "0700_8010"
  718. # Mask to 16 bits to handle printers that send larger values
  719. error_code_int = int(error.code.replace("0x", ""), 16) if error.code else 0
  720. error_code_masked = error_code_int & 0xFFFF
  721. short_code = f"{(error.attr >> 16) & 0xFFFF:04X}_{error_code_masked:04X}"
  722. # Only notify for errors with known descriptions — printers
  723. # send many undocumented/phantom codes that aren't real errors.
  724. description = get_error_description(short_code)
  725. if not description or short_code in _HMS_NOTIFICATION_SUPPRESS:
  726. continue
  727. error_type = f"{module_name} Error"
  728. error_detail = description
  729. await notification_service.on_printer_error(
  730. printer_id, printer_name, error_type, db, error_detail, image_data=error_image_data
  731. )
  732. sent_count += 1
  733. if sent_count:
  734. logging.getLogger(__name__).info(
  735. f"[HMS] Sent notification for {sent_count} error(s) on printer {printer_id}"
  736. )
  737. # Also publish to MQTT relay
  738. printer_info = printer_manager.get_printer(printer_id)
  739. if printer_info:
  740. errors_data = [
  741. {
  742. "code": e.code,
  743. "attr": e.attr,
  744. "module": e.module,
  745. "severity": e.severity,
  746. }
  747. for e in new_errors
  748. ]
  749. await mqtt_relay.on_printer_error(
  750. printer_id, printer_info.name, printer_info.serial_number, errors_data
  751. )
  752. except Exception as e:
  753. logging.getLogger(__name__).warning(f"HMS error notification failed: {e}")
  754. else:
  755. # No HMS errors — only clear tracking after a grace period to prevent
  756. # flapping errors (brief hms:[] gaps) from re-triggering notifications.
  757. # Some HMS codes (e.g. chamber temp regulation during PETG prints) toggle
  758. # on/off every few seconds as conditions fluctuate around thresholds.
  759. if printer_id in _notified_hms_errors:
  760. last_seen = _hms_last_seen.get(printer_id, 0)
  761. if time.time() - last_seen >= _HMS_CLEAR_GRACE_SECONDS:
  762. _notified_hms_errors.pop(printer_id, None)
  763. _hms_last_seen.pop(printer_id, None)
  764. await ws_manager.send_printer_status(
  765. printer_id,
  766. printer_state_to_dict(state, printer_id, printer_manager.get_model(printer_id)),
  767. )
  768. def _is_bambu_uuid(tray_uuid: str) -> bool:
  769. """Check if a tray UUID looks like a valid Bambu Lab RFID UUID (non-empty, non-zero)."""
  770. return bool(tray_uuid) and tray_uuid not in ("", "0" * len(tray_uuid))
  771. async def on_ams_change(printer_id: int, ams_data: list):
  772. """Handle AMS data changes - sync to Spoolman if enabled and auto mode."""
  773. logger = logging.getLogger(__name__)
  774. # Snapshot BEFORE any await: if a print is active, skip weight sync later.
  775. # on_print_complete may pop _active_sessions during our awaits (#880).
  776. from backend.app.services.usage_tracker import _active_sessions
  777. _print_active = printer_id in _active_sessions
  778. # MQTT relay - publish AMS change
  779. try:
  780. printer_info = printer_manager.get_printer(printer_id)
  781. if printer_info:
  782. await mqtt_relay.on_ams_change(printer_id, printer_info.name, printer_info.serial_number, ams_data)
  783. except Exception:
  784. pass # Don't fail AMS callback if MQTT fails
  785. # Broadcast AMS change via WebSocket (bypasses status_key deduplication)
  786. # This ensures frontend gets immediate updates when AMS slots are configured
  787. try:
  788. state = printer_manager.get_status(printer_id)
  789. if state:
  790. logger.info("[Printer %s] Broadcasting AMS change via WebSocket", printer_id)
  791. await ws_manager.send_printer_status(
  792. printer_id,
  793. printer_state_to_dict(state, printer_id, printer_manager.get_model(printer_id)),
  794. )
  795. except Exception as e:
  796. logger.warning("Failed to broadcast AMS change for printer %s: %s", printer_id, e)
  797. from backend.app.utils.color_utils import colors_similar as _colors_similar
  798. # Auto-unlink spool assignments with stale fingerprints
  799. try:
  800. async with async_session() as db:
  801. from sqlalchemy.orm import selectinload
  802. from backend.app.api.routes.inventory import _find_tray_in_ams_data
  803. from backend.app.models.spool import Spool as _Spool
  804. from backend.app.models.spool_assignment import SpoolAssignment as SA
  805. result = await db.execute(
  806. select(SA)
  807. .where(SA.printer_id == printer_id)
  808. .options(selectinload(SA.spool).selectinload(_Spool.k_profiles))
  809. )
  810. stale = []
  811. for assignment in result.scalars().all():
  812. # External spool assignments (ams_id=255) live in vt_tray, not AMS data
  813. if assignment.ams_id == 255:
  814. ps = printer_manager.get_status(printer_id)
  815. vt_tray_raw = ps.raw_data.get("vt_tray", []) if ps else []
  816. ext_id = assignment.tray_id + 254 # 0→254, 1→255
  817. current_tray = None
  818. for vt in vt_tray_raw:
  819. if isinstance(vt, dict) and int(vt.get("id", 254)) == ext_id:
  820. current_tray = vt
  821. break
  822. if not current_tray:
  823. # vt_tray data may not have arrived yet — keep assignment
  824. continue
  825. else:
  826. current_tray = _find_tray_in_ams_data(ams_data, assignment.ams_id, assignment.tray_id)
  827. if not current_tray:
  828. logger.info(
  829. "Auto-unlink: spool %d AMS%d-T%d — tray not found in AMS data (slot empty?)",
  830. assignment.spool_id,
  831. assignment.ams_id,
  832. assignment.tray_id,
  833. )
  834. stale.append(assignment) # Slot empty
  835. elif _is_bambu_uuid(current_tray.get("tray_uuid", "")):
  836. # A Bambu Lab spool is in this slot — check if it's the same spool
  837. # that's currently assigned. If yes, keep the assignment (avoids
  838. # unnecessary unlink/re-assign/ams_filament_setting cycle that clears
  839. # the printer's filament preset on every startup).
  840. tray_uuid = current_tray.get("tray_uuid", "")
  841. tag_uid = current_tray.get("tag_uid", "")
  842. spool = assignment.spool
  843. spool_matches = False
  844. if spool:
  845. if (spool.tray_uuid and spool.tray_uuid.upper() == tray_uuid.upper()) or (
  846. spool.tag_uid
  847. and tag_uid
  848. and tag_uid != "0000000000000000"
  849. and spool.tag_uid.upper() == tag_uid.upper()
  850. ):
  851. spool_matches = True
  852. if spool_matches:
  853. # Same BL spool still in slot — keep assignment, update fingerprint if needed
  854. cur_color = current_tray.get("tray_color", "")
  855. cur_type = current_tray.get("tray_type", "")
  856. fp_color = assignment.fingerprint_color or ""
  857. fp_type = assignment.fingerprint_type or ""
  858. if cur_color.upper() != fp_color.upper() or cur_type.upper() != fp_type.upper():
  859. assignment.fingerprint_color = cur_color
  860. assignment.fingerprint_type = cur_type
  861. logger.debug(
  862. "Auto-unlink: spool %d AMS%d-T%d — same BL spool, updated fingerprint",
  863. assignment.spool_id,
  864. assignment.ams_id,
  865. assignment.tray_id,
  866. )
  867. continue
  868. # Different BL spool or unrecognized — unlink so auto-assign can match
  869. logger.info(
  870. "Auto-unlink: spool %d AMS%d-T%d — different Bambu Lab spool detected (uuid=%s)",
  871. assignment.spool_id,
  872. assignment.ams_id,
  873. assignment.tray_id,
  874. tray_uuid,
  875. )
  876. stale.append(assignment)
  877. else:
  878. cur_color = current_tray.get("tray_color", "")
  879. cur_type = current_tray.get("tray_type", "")
  880. cur_state = current_tray.get("state")
  881. fp_color = assignment.fingerprint_color or ""
  882. fp_type = assignment.fingerprint_type or ""
  883. # SpoolBuddy pre-config replay: fingerprint_type empty means
  884. # the slot was empty when the user pre-assigned via SpoolBuddy
  885. # (the firmware drops ams_filament_setting on empty slots, so
  886. # MQTT was deferred). The moment any filament gets inserted
  887. # — Bambu RFID, 3rd-party, or even an existing-but-now-
  888. # reconfigured spool — fire the deferred configuration.
  889. # The "loaded" signal is `state == 11` (Bambu's "filament fed
  890. # to extruder" code), NOT tray_type — 3rd-party spools without
  891. # readable RFID report state=11 but tray_type="" because the
  892. # AMS sensor reads no filament metadata. Requiring a non-empty
  893. # tray_type would lock out the exact users this feature targets.
  894. if not fp_type.strip() and cur_state == 11 and assignment.spool:
  895. try:
  896. from backend.app.api.routes.inventory import (
  897. apply_spool_to_slot_via_mqtt,
  898. )
  899. await apply_spool_to_slot_via_mqtt(
  900. db=db,
  901. current_user=None,
  902. spool=assignment.spool,
  903. printer_id=printer_id,
  904. ams_id=assignment.ams_id,
  905. tray_id=assignment.tray_id,
  906. current_tray_info_idx=current_tray.get("tray_info_idx", ""),
  907. current_tray_type=cur_type,
  908. )
  909. logger.info(
  910. "SpoolBuddy pre-config applied on insert: spool %d → printer %d AMS%d-T%d",
  911. assignment.spool_id,
  912. printer_id,
  913. assignment.ams_id,
  914. assignment.tray_id,
  915. )
  916. except Exception:
  917. logger.exception(
  918. "Pre-config apply failed for spool %d on printer %d AMS%d-T%d",
  919. assignment.spool_id,
  920. printer_id,
  921. assignment.ams_id,
  922. assignment.tray_id,
  923. )
  924. assignment.fingerprint_color = cur_color
  925. assignment.fingerprint_type = cur_type
  926. continue
  927. if not _colors_similar(cur_color, fp_color) or cur_type.upper() != fp_type.upper():
  928. # Fingerprint mismatch — but check if tray now matches the
  929. # assigned spool (e.g. auto-configure changed the tray).
  930. spool = assignment.spool
  931. if spool:
  932. spool_color = (spool.rgba or "FFFFFFFF").upper()
  933. spool_type = (spool.material or "").upper()
  934. if _colors_similar(cur_color, spool_color) and cur_type.upper() == spool_type:
  935. logger.info(
  936. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch but tray matches spool, updating fp",
  937. assignment.spool_id,
  938. assignment.ams_id,
  939. assignment.tray_id,
  940. )
  941. assignment.fingerprint_color = cur_color
  942. assignment.fingerprint_type = cur_type
  943. continue
  944. logger.info(
  945. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch (cur=%s/%s fp=%s/%s spool=%s/%s)",
  946. assignment.spool_id,
  947. assignment.ams_id,
  948. assignment.tray_id,
  949. cur_color,
  950. cur_type,
  951. fp_color,
  952. fp_type,
  953. spool.rgba if spool else "?",
  954. spool.material if spool else "?",
  955. )
  956. stale.append(assignment) # Spool changed
  957. for a in stale:
  958. await db.delete(a)
  959. if stale:
  960. logger.info("Auto-unlinked %d stale spool assignments for printer %d", len(stale), printer_id)
  961. # Commit any changes (stale deletions and/or fingerprint updates)
  962. await db.commit()
  963. except Exception as e:
  964. logger.warning("Spool assignment cleanup failed: %s", e, exc_info=True)
  965. # Auto-manage inventory spools from AMS tray data (skip if Spoolman manages AMS).
  966. # Serialised per-printer via _ams_assignment_locks: MQTT bursts can deliver
  967. # two AMS pushes ~30 ms apart, and without the lock both callbacks read
  968. # "no existing assignment" for the same (printer, ams, tray) and race to
  969. # INSERT, hitting the spool_assignment_printer_id_ams_id_tray_id_key
  970. # unique constraint on Postgres. SQLite's WAL serialises writes so the
  971. # bug stayed latent there. See _ams_assignment_locks comment for details.
  972. try:
  973. async with _get_ams_assignment_lock(printer_id), async_session() as db:
  974. from backend.app.api.routes.settings import get_setting
  975. from backend.app.models.spool import Spool
  976. from backend.app.models.spool_assignment import SpoolAssignment as SA
  977. from backend.app.services.spool_tag_matcher import (
  978. auto_assign_spool,
  979. create_spool_from_tray,
  980. find_matching_untagged_spool,
  981. get_spool_by_tag,
  982. is_bambu_tag,
  983. is_valid_tag,
  984. link_tag_to_inventory_spool,
  985. )
  986. _spoolman_on = await get_setting(db, "spoolman_enabled")
  987. if not _spoolman_on or _spoolman_on.lower() != "true":
  988. for ams_unit in ams_data:
  989. if not isinstance(ams_unit, dict):
  990. continue
  991. ams_id = int(ams_unit.get("id", 0))
  992. for tray in ams_unit.get("tray", []):
  993. if not isinstance(tray, dict):
  994. continue
  995. tray_id = int(tray.get("id", 0))
  996. tag_uid = tray.get("tag_uid", "")
  997. tray_uuid = tray.get("tray_uuid", "")
  998. tray_info_idx = tray.get("tray_info_idx", "")
  999. if not tray.get("tray_type"):
  1000. continue # Empty slot
  1001. # Check if assignment already exists for this slot
  1002. existing = await db.execute(
  1003. select(SA)
  1004. .options(selectinload(SA.spool).selectinload(Spool.k_profiles))
  1005. .where(SA.printer_id == printer_id, SA.ams_id == ams_id, SA.tray_id == tray_id)
  1006. )
  1007. existing_assignment = existing.scalar_one_or_none()
  1008. if existing_assignment:
  1009. # Sync spool weight_used from AMS remain — only INCREASE, never decrease.
  1010. # The AMS remain% is low-resolution (integer %, i.e. 10g steps for 1kg spool)
  1011. # and must not overwrite precise values from the usage tracker (3MF/G-code).
  1012. # Skip during active prints: the usage tracker handles deduction
  1013. # precisely via 3MF data on print completion. Without this guard the
  1014. # AMS remain% SET and the usage tracker ADD both fire from the same
  1015. # MQTT message, doubling the deduction (#880).
  1016. if _print_active:
  1017. continue
  1018. remain_raw = tray.get("remain")
  1019. if (
  1020. remain_raw is not None
  1021. and existing_assignment.spool
  1022. and not existing_assignment.spool.weight_locked
  1023. ):
  1024. try:
  1025. remain_val = int(remain_raw)
  1026. except (TypeError, ValueError):
  1027. remain_val = -1
  1028. if 1 <= remain_val <= 100:
  1029. lw = existing_assignment.spool.label_weight or 1000
  1030. new_used = round(lw * (100 - remain_val) / 100.0, 1)
  1031. current_used = existing_assignment.spool.weight_used or 0
  1032. if new_used > current_used + 1:
  1033. logger.info(
  1034. "Weight sync: spool %d weight_used %s -> %s (remain=%d)",
  1035. existing_assignment.spool_id,
  1036. current_used,
  1037. new_used,
  1038. remain_val,
  1039. )
  1040. existing_assignment.spool.weight_used = new_used
  1041. await db.commit()
  1042. continue
  1043. if is_bambu_tag(tag_uid, tray_uuid, tray_info_idx):
  1044. # BL spool with RFID tag: auto-match → inventory match → auto-create
  1045. spool = await get_spool_by_tag(db, tag_uid, tray_uuid)
  1046. if not spool:
  1047. # Try matching an untagged inventory spool (same material/color)
  1048. spool = await find_matching_untagged_spool(db, tray)
  1049. if spool:
  1050. await link_tag_to_inventory_spool(db, spool, tray)
  1051. else:
  1052. spool = await create_spool_from_tray(db, tray)
  1053. await auto_assign_spool(
  1054. printer_id,
  1055. ams_id,
  1056. tray_id,
  1057. spool,
  1058. printer_manager,
  1059. db,
  1060. tray_info_idx=tray_info_idx,
  1061. )
  1062. await db.commit()
  1063. await ws_manager.broadcast(
  1064. {
  1065. "type": "spool_auto_assigned",
  1066. "printer_id": printer_id,
  1067. "ams_id": ams_id,
  1068. "tray_id": tray_id,
  1069. "spool_id": spool.id,
  1070. }
  1071. )
  1072. logger.info(
  1073. "RFID auto-assigned spool %d to printer %d AMS%d-T%d",
  1074. spool.id,
  1075. printer_id,
  1076. ams_id,
  1077. tray_id,
  1078. )
  1079. elif is_valid_tag(tag_uid, tray_uuid):
  1080. # Non-BL spool with some tag — let user choose
  1081. await ws_manager.broadcast(
  1082. {
  1083. "type": "unknown_tag",
  1084. "printer_id": printer_id,
  1085. "ams_id": ams_id,
  1086. "tray_id": tray_id,
  1087. "tag_uid": tag_uid,
  1088. "tray_uuid": tray_uuid,
  1089. }
  1090. )
  1091. else:
  1092. # No tag at all — let user choose from inventory
  1093. await ws_manager.broadcast(
  1094. {
  1095. "type": "unknown_tag",
  1096. "printer_id": printer_id,
  1097. "ams_id": ams_id,
  1098. "tray_id": tray_id,
  1099. "tag_uid": "",
  1100. "tray_uuid": "",
  1101. }
  1102. )
  1103. except Exception as e:
  1104. logger.warning("RFID spool auto-assign failed: %s", e, exc_info=True)
  1105. try:
  1106. async with async_session() as db:
  1107. from backend.app.api.routes.settings import get_setting
  1108. from backend.app.models.printer import Printer
  1109. # Check if Spoolman is enabled
  1110. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  1111. if not spoolman_enabled or spoolman_enabled.lower() != "true":
  1112. return
  1113. # Check sync mode
  1114. sync_mode = await get_setting(db, "spoolman_sync_mode")
  1115. if sync_mode and sync_mode != "auto":
  1116. return # Only sync on auto mode
  1117. # Check if weight sync is disabled
  1118. disable_weight_sync_str = await get_setting(db, "spoolman_disable_weight_sync")
  1119. disable_weight_sync = disable_weight_sync_str and disable_weight_sync_str.lower() == "true"
  1120. # Get Spoolman URL
  1121. spoolman_url = await get_setting(db, "spoolman_url")
  1122. if not spoolman_url:
  1123. return
  1124. # Get or create Spoolman client
  1125. client = await get_spoolman_client()
  1126. if not client:
  1127. client = await init_spoolman_client(spoolman_url)
  1128. # Check if Spoolman is reachable
  1129. if not await client.health_check():
  1130. logger.warning("Spoolman not reachable at %s", spoolman_url)
  1131. return
  1132. # Get printer name for location
  1133. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1134. printer = result.scalar_one_or_none()
  1135. printer_name = printer.name if printer else f"Printer {printer_id}"
  1136. # OPTIMIZATION: Fetch all spools once before processing trays
  1137. # This eliminates redundant API calls (one per tray) when syncing multiple trays
  1138. logger.debug("[Printer %s] Fetching spools cache for AMS sync...", printer_id)
  1139. try:
  1140. cached_spools = await client.get_spools()
  1141. logger.debug("[Printer %s] Cached %d spools for batch sync", printer_id, len(cached_spools))
  1142. except Exception as e:
  1143. logger.error(
  1144. "[Printer %s] Failed to fetch spools cache after retries, aborting AMS sync: %s",
  1145. printer_id,
  1146. e,
  1147. )
  1148. return
  1149. # Load inventory weights as fallback (when AMS MQTT data lacks remain values)
  1150. from sqlalchemy.orm import selectinload
  1151. from backend.app.models.spool_assignment import SpoolAssignment
  1152. inventory_weights: dict[tuple[int, int], float] = {}
  1153. try:
  1154. assign_result = await db.execute(
  1155. select(SpoolAssignment)
  1156. .options(selectinload(SpoolAssignment.spool))
  1157. .where(SpoolAssignment.printer_id == printer_id)
  1158. )
  1159. for assignment in assign_result.scalars().all():
  1160. spool = assignment.spool
  1161. if spool and spool.label_weight > 0:
  1162. remaining = max(0.0, spool.label_weight - (spool.weight_used or 0))
  1163. inventory_weights[(assignment.ams_id, assignment.tray_id)] = remaining
  1164. except Exception as e:
  1165. logger.debug("Could not load inventory weights for printer %s: %s", printer_id, e)
  1166. # Sync each AMS tray, tracking UUIDs and spool IDs for cleanup
  1167. synced = 0
  1168. current_tray_uuids: set[str] = set()
  1169. synced_spool_ids: set[int] = set()
  1170. for ams_unit in ams_data:
  1171. ams_id = int(ams_unit.get("id", 0))
  1172. trays = ams_unit.get("tray", [])
  1173. for tray_data in trays:
  1174. tray = client.parse_ams_tray(ams_id, tray_data)
  1175. if not tray:
  1176. continue # Empty tray
  1177. # Track this spool's UUID as currently present in the AMS
  1178. spool_tag = (
  1179. tray.tray_uuid
  1180. if tray.tray_uuid and tray.tray_uuid != "00000000000000000000000000000000"
  1181. else tray.tag_uid
  1182. )
  1183. if spool_tag:
  1184. current_tray_uuids.add(spool_tag.upper())
  1185. try:
  1186. inv_remaining = inventory_weights.get((ams_id, tray.tray_id))
  1187. result = await client.sync_ams_tray(
  1188. tray,
  1189. printer_name,
  1190. disable_weight_sync=disable_weight_sync,
  1191. cached_spools=cached_spools,
  1192. inventory_remaining=inv_remaining,
  1193. )
  1194. if result:
  1195. synced += 1
  1196. if result.get("id"):
  1197. synced_spool_ids.add(result["id"])
  1198. # If a new spool was created, add it to the cache
  1199. # so subsequent trays can find it if they reference the same tag
  1200. # Check if this spool already exists in cache
  1201. spool_exists = any(s.get("id") == result["id"] for s in cached_spools)
  1202. if not spool_exists:
  1203. cached_spools.append(result)
  1204. logger.debug(
  1205. "[Printer %s] Added newly created spool %s to cache",
  1206. printer_id,
  1207. result["id"],
  1208. )
  1209. except Exception as e:
  1210. logger.error("Error syncing AMS %s tray %s: %s", ams_id, tray.tray_id, e)
  1211. if synced > 0:
  1212. logger.info("Auto-synced %s AMS trays to Spoolman for printer %s", synced, printer_id)
  1213. # Clear location for spools no longer in this printer's AMS
  1214. try:
  1215. cleared = await client.clear_location_for_removed_spools(
  1216. printer_name, current_tray_uuids, cached_spools=cached_spools, synced_spool_ids=synced_spool_ids
  1217. )
  1218. if cleared > 0:
  1219. logger.info("Auto-cleared location for %s spools removed from printer %s", cleared, printer_id)
  1220. except Exception as e:
  1221. logger.error("Error clearing locations for removed spools on printer %s: %s", printer_id, e)
  1222. except Exception as e:
  1223. logging.getLogger(__name__).warning(f"Spoolman AMS sync failed: {e}")
  1224. async def _capture_snapshot_for_notification(printer_id: int, printer, logger) -> bytes | None:
  1225. """Capture a camera snapshot for notification image attachment.
  1226. Returns JPEG bytes (max 2.5MB) or None if capture fails or is unavailable.
  1227. Uses: external camera > buffered frame > fresh capture.
  1228. """
  1229. if not printer:
  1230. return None
  1231. try:
  1232. from backend.app.api.routes.settings import get_setting
  1233. async with async_session() as db:
  1234. capture_enabled = await get_setting(db, "capture_finish_photo")
  1235. if capture_enabled is not None and capture_enabled.lower() != "true":
  1236. return None
  1237. # Try external camera first
  1238. if printer.external_camera_enabled and printer.external_camera_url:
  1239. logger.info("[SNAPSHOT] Capturing from external camera for printer %s", printer_id)
  1240. from backend.app.services.external_camera import capture_frame
  1241. frame_data = await capture_frame(
  1242. printer.external_camera_url,
  1243. printer.external_camera_type or "mjpeg",
  1244. snapshot_url=printer.external_camera_snapshot_url,
  1245. )
  1246. if frame_data and len(frame_data) <= 2_500_000:
  1247. logger.info("[SNAPSHOT] External camera frame: %s bytes", len(frame_data))
  1248. return _apply_camera_rotation(frame_data, printer, logger)
  1249. # Try buffered frame from active stream
  1250. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  1251. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  1252. active_chamber = [k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")]
  1253. buffered_frame = get_buffered_frame(printer_id)
  1254. if (active_for_printer or active_chamber) and buffered_frame:
  1255. logger.info("[SNAPSHOT] Using buffered frame for printer %s: %s bytes", printer_id, len(buffered_frame))
  1256. if len(buffered_frame) <= 2_500_000:
  1257. return _apply_camera_rotation(buffered_frame, printer, logger)
  1258. # Fresh capture from printer camera
  1259. logger.info("[SNAPSHOT] Capturing fresh frame for printer %s", printer_id)
  1260. from backend.app.services.camera import capture_camera_frame_bytes
  1261. frame_data = await capture_camera_frame_bytes(
  1262. printer.ip_address, printer.access_code, printer.model, timeout=15
  1263. )
  1264. if frame_data and len(frame_data) <= 2_500_000:
  1265. logger.info("[SNAPSHOT] Fresh camera frame: %s bytes", len(frame_data))
  1266. return _apply_camera_rotation(frame_data, printer, logger)
  1267. except Exception as e:
  1268. logger.warning("[SNAPSHOT] Failed to capture snapshot for printer %s: %s", printer_id, e)
  1269. return None
  1270. def _apply_camera_rotation(image_data: bytes, printer, logger) -> bytes:
  1271. """Apply camera rotation to snapshot image if configured."""
  1272. rotation = getattr(printer, "camera_rotation", 0)
  1273. if not rotation or rotation == 0:
  1274. return image_data
  1275. try:
  1276. from io import BytesIO
  1277. from PIL import Image
  1278. img = Image.open(BytesIO(image_data))
  1279. # PIL rotate is counter-clockwise, so negate for clockwise rotation
  1280. img = img.rotate(-rotation, expand=True)
  1281. buf = BytesIO()
  1282. img.save(buf, format="JPEG", quality=90)
  1283. rotated = buf.getvalue()
  1284. logger.info("[SNAPSHOT] Applied %d° rotation: %s → %s bytes", rotation, len(image_data), len(rotated))
  1285. return rotated
  1286. except Exception as e:
  1287. logger.warning("[SNAPSHOT] Failed to apply rotation: %s", e)
  1288. return image_data
  1289. async def _send_print_start_notification(
  1290. printer_id: int,
  1291. data: dict,
  1292. archive_data: dict | None = None,
  1293. logger=None,
  1294. ):
  1295. """Helper to send print start notification with optional archive data."""
  1296. if logger is None:
  1297. logger = logging.getLogger(__name__)
  1298. try:
  1299. async with async_session() as db:
  1300. from backend.app.models.printer import Printer
  1301. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1302. printer = result.scalar_one_or_none()
  1303. printer_name = printer.name if printer else f"Printer {printer_id}"
  1304. # Capture camera snapshot for notification image attachment
  1305. image_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  1306. if image_data:
  1307. if archive_data is None:
  1308. archive_data = {}
  1309. archive_data["image_data"] = image_data
  1310. await notification_service.on_print_start(printer_id, printer_name, data, db, archive_data=archive_data)
  1311. # Send user-specific email notification for print start
  1312. if archive_data and archive_data.get("created_by_id"):
  1313. await notification_service.send_user_print_email(
  1314. event_type="user_print_start",
  1315. created_by_id=archive_data["created_by_id"],
  1316. printer_name=printer_name,
  1317. filename=data.get("subtask_name") or data.get("filename", "Unknown"),
  1318. db=db,
  1319. )
  1320. except Exception as e:
  1321. logger.warning("Notification on_print_start failed: %s", e)
  1322. async def _dispatch_user_print_email(
  1323. status: str,
  1324. created_by_id: int | None,
  1325. printer_name: str,
  1326. filename: str,
  1327. db,
  1328. ) -> None:
  1329. """Send a user-specific print-completion email based on print status.
  1330. Maps the normalised print status to the correct event type and delegates
  1331. to :meth:`NotificationService.send_user_print_email`. A single helper
  1332. avoids duplicating the ``if status == "completed" / elif "failed" / elif
  1333. "stopped"`` dispatch block at every call site.
  1334. Does nothing if *created_by_id* is ``None``.
  1335. """
  1336. if created_by_id is None:
  1337. return
  1338. if status == "completed":
  1339. event_type = "user_print_complete"
  1340. elif status == "failed":
  1341. event_type = "user_print_failed"
  1342. elif status in ("stopped", "aborted", "cancelled"):
  1343. event_type = "user_print_stopped"
  1344. else:
  1345. return
  1346. await notification_service.send_user_print_email(
  1347. event_type=event_type,
  1348. created_by_id=created_by_id,
  1349. printer_name=printer_name,
  1350. filename=filename,
  1351. db=db,
  1352. )
  1353. def _load_objects_from_archive(archive, printer_id: int, logger) -> None:
  1354. """Extract printable objects from an archive's 3MF file and store in printer state."""
  1355. try:
  1356. from backend.app.services.archive import extract_printable_objects_from_3mf
  1357. file_path = app_settings.base_dir / archive.file_path
  1358. if file_path.is_file() and str(file_path).endswith(".3mf"):
  1359. with open(file_path, "rb") as f:
  1360. threemf_data = f.read()
  1361. # Extract with positions for UI overlay
  1362. printable_objects, bbox_all = extract_printable_objects_from_3mf(threemf_data, include_positions=True)
  1363. if printable_objects:
  1364. client = printer_manager.get_client(printer_id)
  1365. if client:
  1366. client.state.printable_objects = printable_objects
  1367. client.state.printable_objects_bbox_all = bbox_all
  1368. client.state.skipped_objects = []
  1369. logger.info("Loaded %s printable objects for printer %s", len(printable_objects), printer_id)
  1370. except Exception as e:
  1371. logger.debug("Failed to extract printable objects from archive: %s", e)
  1372. async def on_print_start(printer_id: int, data: dict):
  1373. """Handle print start - archive the 3MF file immediately."""
  1374. logger = logging.getLogger(__name__)
  1375. logger.info("[CALLBACK] on_print_start called for printer %s, data keys: %s", printer_id, list(data.keys()))
  1376. # Clear any stale user-stopped flag from previous print cycles
  1377. _user_stopped_printers.discard(printer_id)
  1378. # Cancel any active bed cooldown waiter for this printer
  1379. if _bed_cool_waiters.pop(printer_id, None):
  1380. logger.info("[BED-COOL] Cancelled bed cooldown waiter for printer %s (new print started)", printer_id)
  1381. # Clear cached cover images so the new print's thumbnail is fetched fresh
  1382. from backend.app.api.routes.printers import clear_cover_cache
  1383. clear_cover_cache(printer_id)
  1384. await ws_manager.send_print_start(printer_id, data)
  1385. # Notify when the print-start AMS mapping references tray slots without spool assignments.
  1386. await notify_missing_spool_assignments_on_print_start(printer_id, data, logger)
  1387. # MQTT relay - publish print start
  1388. try:
  1389. printer_info = printer_manager.get_printer(printer_id)
  1390. if printer_info:
  1391. await mqtt_relay.on_print_start(
  1392. printer_id,
  1393. printer_info.name,
  1394. printer_info.serial_number,
  1395. data.get("filename", ""),
  1396. data.get("subtask_name", ""),
  1397. )
  1398. except Exception:
  1399. pass # Don't fail print start callback if MQTT fails
  1400. # Capture AMS tray remain% for filament consumption tracking (skip if Spoolman handles usage)
  1401. try:
  1402. async with async_session() as db:
  1403. from backend.app.api.routes.settings import get_setting
  1404. _spoolman_on = await get_setting(db, "spoolman_enabled")
  1405. if not _spoolman_on or _spoolman_on.lower() != "true":
  1406. from backend.app.services.usage_tracker import on_print_start as usage_on_print_start
  1407. await usage_on_print_start(printer_id, data, printer_manager, db=db)
  1408. except Exception as e:
  1409. logger.warning("Usage tracker on_print_start failed: %s", e)
  1410. # Track if notification was sent (to avoid sending twice)
  1411. notification_sent = False
  1412. # Smart plug automation: turn on plug when print starts
  1413. try:
  1414. async with async_session() as db:
  1415. await smart_plug_manager.on_print_start(printer_id, db)
  1416. except Exception as e:
  1417. logger.warning("Smart plug on_print_start failed: %s", e)
  1418. async with async_session() as db:
  1419. from backend.app.models.printer import Printer
  1420. from backend.app.services.bambu_ftp import list_files_async
  1421. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1422. printer = result.scalar_one_or_none()
  1423. # Plate detection check - pause if objects detected on build plate
  1424. logger.info(
  1425. f"[PLATE CHECK] printer_id={printer_id}, plate_detection_enabled={printer.plate_detection_enabled if printer else 'NO PRINTER'}"
  1426. )
  1427. if printer and printer.plate_detection_enabled:
  1428. logger.info("[PLATE CHECK] ENTERING plate detection code for printer %s", printer_id)
  1429. try:
  1430. from backend.app.services.plate_detection import check_plate_empty
  1431. # Build ROI tuple from printer settings if available
  1432. roi = None
  1433. if all(
  1434. [
  1435. printer.plate_detection_roi_x is not None,
  1436. printer.plate_detection_roi_y is not None,
  1437. printer.plate_detection_roi_w is not None,
  1438. printer.plate_detection_roi_h is not None,
  1439. ]
  1440. ):
  1441. roi = (
  1442. printer.plate_detection_roi_x,
  1443. printer.plate_detection_roi_y,
  1444. printer.plate_detection_roi_w,
  1445. printer.plate_detection_roi_h,
  1446. )
  1447. # Auto-turn on chamber light if it's off for better detection
  1448. light_was_off = False
  1449. client = printer_manager.get_client(printer_id)
  1450. if client and client.state:
  1451. light_was_off = not client.state.chamber_light
  1452. if light_was_off:
  1453. logger.info("[PLATE CHECK] Turning on chamber light for printer %s", printer_id)
  1454. client.set_chamber_light(True)
  1455. # Wait for light to physically turn on and camera to adjust exposure
  1456. await asyncio.sleep(2.5)
  1457. logger.info("[PLATE CHECK] Running plate detection for printer %s", printer_id)
  1458. plate_result = await check_plate_empty(
  1459. printer_id=printer_id,
  1460. ip_address=printer.ip_address,
  1461. access_code=printer.access_code,
  1462. model=printer.model,
  1463. include_debug_image=False,
  1464. external_camera_url=printer.external_camera_url,
  1465. external_camera_type=printer.external_camera_type,
  1466. use_external=printer.external_camera_enabled,
  1467. roi=roi,
  1468. external_camera_snapshot_url=printer.external_camera_snapshot_url,
  1469. )
  1470. # Restore chamber light to original state
  1471. if light_was_off and client:
  1472. logger.info("[PLATE CHECK] Restoring chamber light to off for printer %s", printer_id)
  1473. client.set_chamber_light(False)
  1474. if not plate_result.needs_calibration and not plate_result.is_empty:
  1475. # Objects detected - pause the print!
  1476. logger.warning(
  1477. f"[PLATE CHECK] Objects detected on plate for printer {printer_id}! "
  1478. f"Confidence: {plate_result.confidence:.0%}, Diff: {plate_result.difference_percent:.1f}%"
  1479. )
  1480. client = printer_manager.get_client(printer_id)
  1481. if client:
  1482. client.pause_print()
  1483. logger.info("[PLATE CHECK] Print paused for printer %s", printer_id)
  1484. # Send notification about plate not empty
  1485. await ws_manager.broadcast(
  1486. {
  1487. "type": "plate_not_empty",
  1488. "printer_id": printer_id,
  1489. "printer_name": printer.name,
  1490. "message": f"Objects detected on build plate! Print paused. (Diff: {plate_result.difference_percent:.1f}%)",
  1491. }
  1492. )
  1493. # Also send push notification
  1494. try:
  1495. await notification_service.on_plate_not_empty(
  1496. printer_id=printer_id,
  1497. printer_name=printer.name,
  1498. db=db,
  1499. difference_percent=plate_result.difference_percent,
  1500. )
  1501. except Exception as notif_err:
  1502. logger.warning("[PLATE CHECK] Failed to send notification: %s", notif_err)
  1503. else:
  1504. logger.info("[PLATE CHECK] Plate is empty for printer %s, proceeding with print", printer_id)
  1505. except Exception as plate_err:
  1506. # Don't block print on plate detection errors
  1507. logger.warning("[PLATE CHECK] Plate detection failed for printer %s: %s", printer_id, plate_err)
  1508. if not printer:
  1509. logger.info("[CALLBACK] Skipping archive - printer not found in database")
  1510. if not notification_sent:
  1511. await _send_print_start_notification(printer_id, data, logger=logger)
  1512. return
  1513. if not printer.auto_archive:
  1514. # auto-archive disabled — check if there's an expected print (dispatched
  1515. # by BamBuddy via queue/reprint) that already has an archive to promote.
  1516. # If so, fall through to the expected-print handling below so the archive
  1517. # is tracked in _active_prints and usage tracking works at completion.
  1518. _fn = data.get("filename", "")
  1519. _sn = data.get("subtask_name", "")
  1520. _check_keys: list[tuple[int, str]] = []
  1521. if _sn:
  1522. _check_keys += [
  1523. (printer_id, _sn),
  1524. (printer_id, f"{_sn}.3mf"),
  1525. (printer_id, f"{_sn}.gcode.3mf"),
  1526. ]
  1527. if _fn:
  1528. _base_fn = _fn.split("/")[-1] if "/" in _fn else _fn
  1529. _check_keys.append((printer_id, _base_fn))
  1530. _no_archive_base = _base_fn.replace(".gcode", "").replace(".3mf", "")
  1531. _check_keys += [
  1532. (printer_id, _no_archive_base),
  1533. (printer_id, f"{_no_archive_base}.3mf"),
  1534. ]
  1535. _has_expected = any(k in _expected_prints for k in _check_keys)
  1536. if not _has_expected:
  1537. # No expected print — truly external print (started from slicer/touchscreen)
  1538. logger.info("[CALLBACK] Skipping archive - auto_archive: False, no expected print")
  1539. if not notification_sent:
  1540. _no_archive_creator: int | None = None
  1541. for _key in _check_keys:
  1542. _expected_prints.pop(_key, None)
  1543. _expected_print_registered_at.pop(_key, None)
  1544. popped_creator = _expected_print_creators.pop(_key, None)
  1545. if _no_archive_creator is None:
  1546. _no_archive_creator = popped_creator
  1547. _creator_data = {"created_by_id": _no_archive_creator} if _no_archive_creator else None
  1548. await _send_print_start_notification(printer_id, data, _creator_data, logger)
  1549. return
  1550. else:
  1551. logger.info("[CALLBACK] auto_archive disabled but expected print found — promoting archive")
  1552. # Get the filename and subtask_name
  1553. filename = data.get("filename", "")
  1554. subtask_name = data.get("subtask_name", "")
  1555. # MQTT subtask_id uniquely identifies a print job on the printer. When
  1556. # present, it lets us match an archive across a backend restart (#972):
  1557. # same id → same print → resume the existing row instead of cancelling
  1558. # it and recreating from scratch (which loses started_at). Treat "0"
  1559. # and "" as absent — Bambu reports "0" for non-cloud / local prints.
  1560. raw_mqtt = data.get("raw_data") or {}
  1561. subtask_id = raw_mqtt.get("subtask_id")
  1562. if subtask_id is not None:
  1563. subtask_id = str(subtask_id).strip()
  1564. if subtask_id in ("", "0"):
  1565. subtask_id = None
  1566. logger.info("[CALLBACK] Print start detected - filename: %s, subtask: %s", filename, subtask_name)
  1567. # Skip calibration prints — internal printer files should not be archived
  1568. # Bambu calibration gcode lives under /usr/ (e.g. /usr/etc/print/auto_cali_for_user.gcode)
  1569. if filename and filename.startswith("/usr/"):
  1570. logger.info("[CALLBACK] Skipping archive — internal printer file detected: %s", filename)
  1571. if not notification_sent:
  1572. await _send_print_start_notification(printer_id, data, logger=logger)
  1573. return
  1574. if not filename and not subtask_name:
  1575. # Send notification without archive data (no filename)
  1576. logger.info("[CALLBACK] Skipping archive - no filename or subtask_name")
  1577. if not notification_sent:
  1578. await _send_print_start_notification(printer_id, data, logger=logger)
  1579. return
  1580. # Check if this is an expected print from reprint/scheduled
  1581. # Build list of possible keys to check
  1582. expected_keys = []
  1583. if subtask_name:
  1584. expected_keys.append((printer_id, subtask_name))
  1585. expected_keys.append((printer_id, f"{subtask_name}.3mf"))
  1586. expected_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  1587. if filename:
  1588. fname = filename.split("/")[-1] if "/" in filename else filename
  1589. expected_keys.append((printer_id, fname))
  1590. # Strip extensions to match
  1591. base = fname.replace(".gcode", "").replace(".3mf", "")
  1592. expected_keys.append((printer_id, base))
  1593. expected_keys.append((printer_id, f"{base}.3mf"))
  1594. expected_archive_id = None
  1595. for key in expected_keys:
  1596. expected_archive_id = _expected_prints.pop(key, None)
  1597. _expected_print_registered_at.pop(key, None)
  1598. if expected_archive_id:
  1599. # Clean up other possible keys for this print
  1600. for other_key in expected_keys:
  1601. _expected_prints.pop(other_key, None)
  1602. _expected_print_registered_at.pop(other_key, None)
  1603. break
  1604. if expected_archive_id:
  1605. # This is a reprint/scheduled print - use existing archive, don't create new one
  1606. logger.info("Using expected archive %s for print (skipping duplicate)", expected_archive_id)
  1607. from backend.app.models.archive import PrintArchive
  1608. result = await db.execute(select(PrintArchive).where(PrintArchive.id == expected_archive_id))
  1609. archive = result.scalar_one_or_none()
  1610. if archive:
  1611. # Update archive status to printing
  1612. archive.status = "printing"
  1613. archive.started_at = datetime.now(timezone.utc)
  1614. if subtask_id and not archive.subtask_id:
  1615. archive.subtask_id = subtask_id
  1616. await db.commit()
  1617. # Track as active print
  1618. _active_prints[(printer_id, archive.filename)] = archive.id
  1619. if subtask_name:
  1620. _active_prints[(printer_id, f"{subtask_name}.3mf")] = archive.id
  1621. # Inject ams_mapping into usage tracker session — the session was created
  1622. # before expected-print promotion, so it may have ams_mapping=None when
  1623. # the MQTT request topic subscription failed (common on P1S/A1).
  1624. _stored_map = _print_ams_mappings.get(expected_archive_id)
  1625. if _stored_map:
  1626. try:
  1627. from backend.app.services.usage_tracker import _active_sessions
  1628. _ut_session = _active_sessions.get(printer_id)
  1629. if _ut_session and not _ut_session.ams_mapping:
  1630. _ut_session.ams_mapping = _stored_map
  1631. logger.info("[CALLBACK] Injected ams_mapping into usage tracker session: %s", _stored_map)
  1632. except Exception:
  1633. pass
  1634. # Set up energy tracking (#941: persist start on archive row)
  1635. await _record_energy_start(archive, printer_id, db, context="expected-print")
  1636. await ws_manager.send_archive_updated(
  1637. {
  1638. "id": archive.id,
  1639. "status": "printing",
  1640. }
  1641. )
  1642. # Send notification with archive data (reprint/scheduled)
  1643. if not notification_sent:
  1644. # Use archive's created_by_id; fall back to the creator registered via
  1645. # register_expected_print (handles library-file-based queue items where
  1646. # the freshly-created archive has no created_by_id yet).
  1647. # Pop ALL matching keys so no stale entries remain in the dict.
  1648. fallback_creator = None
  1649. for key in expected_keys:
  1650. popped = _expected_print_creators.pop(key, None)
  1651. if fallback_creator is None:
  1652. fallback_creator = popped
  1653. archive_data = {
  1654. "print_time_seconds": archive.print_time_seconds,
  1655. "created_by_id": archive.created_by_id or fallback_creator,
  1656. }
  1657. await _send_print_start_notification(printer_id, data, archive_data, logger)
  1658. # Extract printable objects from the archived 3MF file
  1659. _load_objects_from_archive(archive, printer_id, logger)
  1660. # Store Spoolman tracking data for per-filament usage reporting
  1661. try:
  1662. await _store_spoolman_print_data(
  1663. printer_id,
  1664. archive.id,
  1665. archive.file_path,
  1666. db,
  1667. printer_manager,
  1668. ams_mapping=_get_start_ams_mapping(data, archive.id),
  1669. )
  1670. except Exception as e:
  1671. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  1672. return # Skip creating a new archive
  1673. # Check if there's already a "printing" archive for this printer/file
  1674. # This prevents duplicates when backend restarts during an active print
  1675. from backend.app.models.archive import PrintArchive
  1676. existing_archive: PrintArchive | None = None
  1677. # Preferred match: subtask_id equality. MQTT reports the same subtask_id
  1678. # across a backend restart for the same print, so this is the most
  1679. # reliable way to reattach. We also accept a previously stale-cancelled
  1680. # archive here so users upgrading mid-print get revived when the row
  1681. # their earlier Bambuddy version wrongly cancelled reappears (#972).
  1682. if subtask_id:
  1683. by_id = await db.execute(
  1684. select(PrintArchive)
  1685. .where(PrintArchive.printer_id == printer_id)
  1686. .where(PrintArchive.subtask_id == subtask_id)
  1687. .where(PrintArchive.status.in_(["printing", "cancelled"]))
  1688. .order_by(PrintArchive.created_at.desc())
  1689. .limit(1)
  1690. )
  1691. candidate = by_id.scalar_one_or_none()
  1692. if candidate and (candidate.status == "printing" or (candidate.failure_reason or "").startswith("Stale")):
  1693. existing_archive = candidate
  1694. # Fallback match: name-based lookup. Kept as-is for prints whose
  1695. # subtask_id is missing ("0" / local / non-cloud prints).
  1696. if existing_archive is None:
  1697. check_name = subtask_name or filename.split("/")[-1].replace(".gcode", "").replace(".3mf", "")
  1698. existing = await db.execute(
  1699. select(PrintArchive)
  1700. .where(PrintArchive.printer_id == printer_id)
  1701. .where(PrintArchive.status == "printing")
  1702. .where(
  1703. or_(
  1704. PrintArchive.print_name == check_name,
  1705. PrintArchive.filename.in_(
  1706. [
  1707. f"{check_name}.3mf",
  1708. f"{check_name}.gcode.3mf",
  1709. ]
  1710. ),
  1711. )
  1712. )
  1713. .order_by(PrintArchive.created_at.desc())
  1714. .limit(1)
  1715. )
  1716. existing_archive = existing.scalar_one_or_none()
  1717. if existing_archive:
  1718. # subtask_id match → always resume, regardless of age. Same print,
  1719. # just a backend restart. Revive if it was previously stale-cancelled.
  1720. subtask_match = bool(subtask_id and existing_archive.subtask_id == subtask_id)
  1721. if subtask_match:
  1722. if existing_archive.status == "cancelled":
  1723. logger.warning(
  1724. "Reviving stale-cancelled archive %s — matching subtask_id %s confirms same print (#972)",
  1725. existing_archive.id,
  1726. subtask_id,
  1727. )
  1728. existing_archive.status = "printing"
  1729. existing_archive.failure_reason = None
  1730. await db.commit()
  1731. else:
  1732. logger.info("Resuming archive %s on subtask_id match (%s)", existing_archive.id, subtask_id)
  1733. _active_prints[(printer_id, existing_archive.filename)] = existing_archive.id
  1734. if existing_archive.energy_start_kwh is None:
  1735. await _record_energy_start(existing_archive, printer_id, db, context="subtask-resume")
  1736. if not notification_sent:
  1737. archive_data = {
  1738. "print_time_seconds": existing_archive.print_time_seconds,
  1739. "created_by_id": existing_archive.created_by_id,
  1740. }
  1741. await _send_print_start_notification(printer_id, data, archive_data, logger)
  1742. _load_objects_from_archive(existing_archive, printer_id, logger)
  1743. return
  1744. # Name-match only: fall back to the legacy 4h staleness heuristic.
  1745. archive_age = datetime.now(timezone.utc) - existing_archive.created_at.replace(tzinfo=timezone.utc)
  1746. if archive_age.total_seconds() > 4 * 60 * 60: # 4 hours
  1747. logger.warning(
  1748. f"Found stale 'printing' archive {existing_archive.id} (age: {archive_age}), "
  1749. f"marking as cancelled and creating new archive"
  1750. )
  1751. existing_archive.status = "cancelled"
  1752. existing_archive.failure_reason = "Stale - print likely cancelled or failed without status update"
  1753. await db.commit()
  1754. # Fall through to create new archive (don't return)
  1755. _existing_archive = None # Clear so we don't use stale archive
  1756. else:
  1757. logger.info(
  1758. f"Skipping duplicate - already have printing archive {existing_archive.id} for {check_name}"
  1759. )
  1760. # Track this as the active print
  1761. _active_prints[(printer_id, existing_archive.filename)] = existing_archive.id
  1762. # Attach subtask_id retroactively so future restarts can resume
  1763. if subtask_id and not existing_archive.subtask_id:
  1764. existing_archive.subtask_id = subtask_id
  1765. await db.commit()
  1766. # Also set up energy tracking if not already tracked (#941: persisted column)
  1767. if existing_archive.energy_start_kwh is None:
  1768. await _record_energy_start(existing_archive, printer_id, db, context="existing-printing")
  1769. # Send notification with archive data (existing archive)
  1770. if not notification_sent:
  1771. archive_data = {
  1772. "print_time_seconds": existing_archive.print_time_seconds,
  1773. "created_by_id": existing_archive.created_by_id,
  1774. }
  1775. await _send_print_start_notification(printer_id, data, archive_data, logger)
  1776. # Extract printable objects from the archived 3MF file
  1777. _load_objects_from_archive(existing_archive, printer_id, logger)
  1778. return
  1779. # Build list of possible 3MF filenames to try
  1780. possible_names = []
  1781. # Bambu printers typically store files as "Name.gcode.3mf"
  1782. # The subtask_name is usually the best source for the filename
  1783. if subtask_name:
  1784. # Try common Bambu naming patterns
  1785. possible_names.append(f"{subtask_name}.gcode.3mf")
  1786. possible_names.append(f"{subtask_name}.3mf")
  1787. # Try original filename with .3mf extension
  1788. if filename:
  1789. # Extract just the filename part, not the full path
  1790. fname = filename.split("/")[-1] if "/" in filename else filename
  1791. if fname.endswith(".3mf"):
  1792. possible_names.append(fname)
  1793. elif fname.endswith(".gcode"):
  1794. base = fname.rsplit(".", 1)[0]
  1795. possible_names.append(f"{base}.gcode.3mf")
  1796. possible_names.append(f"{base}.3mf")
  1797. else:
  1798. possible_names.append(f"{fname}.gcode.3mf")
  1799. possible_names.append(f"{fname}.3mf")
  1800. # Also try with spaces converted to underscores (Bambu Studio may normalize filenames)
  1801. space_variants = []
  1802. for name in possible_names:
  1803. if " " in name:
  1804. space_variants.append(name.replace(" ", "_"))
  1805. possible_names.extend(space_variants)
  1806. # Remove duplicates while preserving order
  1807. seen = set()
  1808. possible_names = [x for x in possible_names if not (x in seen or seen.add(x))]
  1809. logger.info("Trying filenames: %s", possible_names)
  1810. # Try to find and download the 3MF file
  1811. temp_path = None
  1812. downloaded_filename = None
  1813. # Cache check: cover endpoint may have already pulled this 3MF during
  1814. # the print (frontend opens the card and shows the thumbnail) — reuse
  1815. # that file instead of re-downloading 36MB over the same FTP link that
  1816. # just served it (#972). The cache keys on a normalized filename so
  1817. # variants like "X", "X.3mf", "X.gcode.3mf" all collapse to one entry.
  1818. for try_filename in possible_names:
  1819. if not try_filename.endswith(".3mf"):
  1820. continue
  1821. cached = get_cached_3mf(printer_id, try_filename)
  1822. if cached:
  1823. logger.info("Reusing cached 3MF from %s (avoided duplicate FTP)", cached)
  1824. temp_path = cached
  1825. downloaded_filename = try_filename
  1826. break
  1827. # Get FTP retry settings
  1828. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  1829. for try_filename in possible_names if not downloaded_filename else []:
  1830. if not try_filename.endswith(".3mf"):
  1831. continue
  1832. # Root (/) is where BambuStudio/OrcaSlicer uploads land on A1/P1-series
  1833. # printers, so try it first — deferring it to last cost #972's reporter
  1834. # ~48 minutes of retries on /cache//model//data//data/Metadata before
  1835. # landing on the path that actually had the file.
  1836. remote_paths = [
  1837. f"/{try_filename}",
  1838. f"/cache/{try_filename}",
  1839. f"/model/{try_filename}",
  1840. f"/data/{try_filename}",
  1841. f"/data/Metadata/{try_filename}",
  1842. ]
  1843. temp_path = app_settings.archive_dir / "temp" / try_filename
  1844. temp_path.parent.mkdir(parents=True, exist_ok=True)
  1845. for remote_path in remote_paths:
  1846. logger.debug("Trying FTP download: %s", remote_path)
  1847. try:
  1848. if ftp_retry_enabled:
  1849. downloaded = await with_ftp_retry(
  1850. download_file_async,
  1851. printer.ip_address,
  1852. printer.access_code,
  1853. remote_path,
  1854. temp_path,
  1855. timeout=ftp_timeout,
  1856. socket_timeout=ftp_timeout,
  1857. printer_model=printer.model,
  1858. max_retries=ftp_retry_count,
  1859. retry_delay=ftp_retry_delay,
  1860. operation_name=f"Download 3MF from {remote_path}",
  1861. non_retry_exceptions=(FileNotOnPrinterError,),
  1862. )
  1863. else:
  1864. downloaded = await download_file_async(
  1865. printer.ip_address,
  1866. printer.access_code,
  1867. remote_path,
  1868. temp_path,
  1869. timeout=ftp_timeout,
  1870. socket_timeout=ftp_timeout,
  1871. printer_model=printer.model,
  1872. )
  1873. if downloaded:
  1874. downloaded_filename = try_filename
  1875. logger.info("Downloaded: %s", remote_path)
  1876. # Populate shared cache so the cover endpoint (if it
  1877. # runs next) doesn't refetch the same 36MB over FTP.
  1878. cache_3mf_download(printer_id, try_filename, temp_path)
  1879. break
  1880. except FileNotOnPrinterError:
  1881. # 550 — file isn't at this path. Advance to next candidate
  1882. # without burning the retry budget.
  1883. logger.debug("3MF not at %s (550), trying next path", remote_path)
  1884. except Exception as e:
  1885. logger.debug("FTP download failed for %s: %s", remote_path, e)
  1886. if downloaded_filename:
  1887. break
  1888. # If still not found, try listing directories to find matching file
  1889. # Different printer models use different directory structures
  1890. if not downloaded_filename and (filename or subtask_name):
  1891. search_term = (subtask_name or filename).lower().replace(".gcode", "").replace(".3mf", "")
  1892. logger.info("Direct FTP download failed, searching directories for '%s'", search_term)
  1893. search_dirs = ["/cache", "/model", "/data", "/data/Metadata", "/"]
  1894. for search_dir in search_dirs:
  1895. if downloaded_filename:
  1896. break
  1897. try:
  1898. dir_files = await list_files_async(
  1899. printer.ip_address, printer.access_code, search_dir, printer_model=printer.model
  1900. )
  1901. threemf_files = [f.get("name") for f in dir_files if f.get("name", "").endswith(".3mf")]
  1902. if threemf_files:
  1903. logger.info(
  1904. f"Found {len(threemf_files)} 3MF files in {search_dir}: {threemf_files[:5]}{'...' if len(threemf_files) > 5 else ''}"
  1905. )
  1906. for f in dir_files:
  1907. if f.get("is_directory"):
  1908. continue
  1909. fname = f.get("name", "")
  1910. # Normalize both for comparison (spaces and underscores are equivalent)
  1911. fname_normalized = fname.lower().replace(" ", "_")
  1912. search_normalized = search_term.replace(" ", "_")
  1913. if fname.endswith(".3mf") and search_normalized in fname_normalized:
  1914. logger.info("Found matching file in %s: %s", search_dir, fname)
  1915. temp_path = app_settings.archive_dir / "temp" / fname
  1916. temp_path.parent.mkdir(parents=True, exist_ok=True)
  1917. remote_full_path = posixpath.join(search_dir, fname)
  1918. if ftp_retry_enabled:
  1919. downloaded = await with_ftp_retry(
  1920. download_file_async,
  1921. printer.ip_address,
  1922. printer.access_code,
  1923. remote_full_path,
  1924. temp_path,
  1925. timeout=ftp_timeout,
  1926. socket_timeout=ftp_timeout,
  1927. printer_model=printer.model,
  1928. max_retries=ftp_retry_count,
  1929. retry_delay=ftp_retry_delay,
  1930. operation_name=f"Download 3MF from {remote_full_path}",
  1931. )
  1932. else:
  1933. downloaded = await download_file_async(
  1934. printer.ip_address,
  1935. printer.access_code,
  1936. remote_full_path,
  1937. temp_path,
  1938. timeout=ftp_timeout,
  1939. socket_timeout=ftp_timeout,
  1940. printer_model=printer.model,
  1941. )
  1942. if downloaded:
  1943. downloaded_filename = fname
  1944. logger.info("Found and downloaded from %s: %s", search_dir, fname)
  1945. cache_3mf_download(printer_id, fname, temp_path)
  1946. break
  1947. except Exception as e:
  1948. logger.debug("Failed to list %s: %s", search_dir, e)
  1949. # Validate the downloaded 3MF actually matches the plate that's running
  1950. # (#1204): subtask_name lags across consecutive plates of the same model,
  1951. # so the first FTP candidate (built from subtask_name) can land on the
  1952. # previous plate's still-resident upload. Cross-check the slice_info
  1953. # plate index against the plate parsed from gcode_file (always fresh —
  1954. # it's the field whose change triggered this callback).
  1955. if downloaded_filename and temp_path:
  1956. expected_plate = parse_plate_id(filename)
  1957. actual_plate = peek_plate_index_in_3mf(temp_path) if expected_plate is not None else None
  1958. if expected_plate is not None and actual_plate is not None and actual_plate != expected_plate:
  1959. logger.warning(
  1960. "[CALLBACK] 3MF plate mismatch: downloaded %s reports plate %s but printer is "
  1961. "running plate %s — subtask_name=%r appears stale, retrying with corrected name",
  1962. downloaded_filename,
  1963. actual_plate,
  1964. expected_plate,
  1965. subtask_name,
  1966. )
  1967. corrected_subtask = swap_plate_suffix(subtask_name, expected_plate)
  1968. retry_succeeded = False
  1969. if corrected_subtask and corrected_subtask != subtask_name:
  1970. for try_filename in (f"{corrected_subtask}.gcode.3mf", f"{corrected_subtask}.3mf"):
  1971. retry_temp_path = app_settings.archive_dir / "temp" / try_filename
  1972. retry_temp_path.parent.mkdir(parents=True, exist_ok=True)
  1973. for remote_path in (
  1974. f"/{try_filename}",
  1975. f"/cache/{try_filename}",
  1976. f"/model/{try_filename}",
  1977. f"/data/{try_filename}",
  1978. f"/data/Metadata/{try_filename}",
  1979. ):
  1980. try:
  1981. if ftp_retry_enabled:
  1982. downloaded = await with_ftp_retry(
  1983. download_file_async,
  1984. printer.ip_address,
  1985. printer.access_code,
  1986. remote_path,
  1987. retry_temp_path,
  1988. timeout=ftp_timeout,
  1989. socket_timeout=ftp_timeout,
  1990. printer_model=printer.model,
  1991. max_retries=ftp_retry_count,
  1992. retry_delay=ftp_retry_delay,
  1993. operation_name=f"Re-download 3MF from {remote_path}",
  1994. non_retry_exceptions=(FileNotOnPrinterError,),
  1995. )
  1996. else:
  1997. downloaded = await download_file_async(
  1998. printer.ip_address,
  1999. printer.access_code,
  2000. remote_path,
  2001. retry_temp_path,
  2002. timeout=ftp_timeout,
  2003. socket_timeout=ftp_timeout,
  2004. printer_model=printer.model,
  2005. )
  2006. if downloaded and peek_plate_index_in_3mf(retry_temp_path) == expected_plate:
  2007. logger.info(
  2008. "[CALLBACK] Re-download succeeded with corrected name %s "
  2009. "(plate %s) — replacing wrong file",
  2010. try_filename,
  2011. expected_plate,
  2012. )
  2013. try:
  2014. temp_path.unlink(missing_ok=True)
  2015. except OSError:
  2016. pass
  2017. temp_path = retry_temp_path
  2018. downloaded_filename = try_filename
  2019. subtask_name = corrected_subtask
  2020. cache_3mf_download(printer_id, try_filename, temp_path)
  2021. retry_succeeded = True
  2022. break
  2023. elif downloaded:
  2024. # Wrong plate again — discard and keep trying
  2025. try:
  2026. retry_temp_path.unlink(missing_ok=True)
  2027. except OSError:
  2028. pass
  2029. except FileNotOnPrinterError:
  2030. continue
  2031. except Exception as e:
  2032. logger.debug("Re-download failed for %s: %s", remote_path, e)
  2033. if retry_succeeded:
  2034. break
  2035. # If the retry didn't find a matching file, drop the wrong 3MF
  2036. # so the no-3MF fallback below creates an archive whose name
  2037. # at least reflects the right plate.
  2038. if not retry_succeeded:
  2039. logger.warning(
  2040. "[CALLBACK] Could not re-download correct plate %s — falling back to no-3MF archive",
  2041. expected_plate,
  2042. )
  2043. try:
  2044. temp_path.unlink(missing_ok=True)
  2045. except OSError:
  2046. pass
  2047. temp_path = None
  2048. downloaded_filename = None
  2049. # Override the stale subtask_name so the fallback archive's
  2050. # print_name reflects the correct plate. Prefer the swapped
  2051. # name when we have one; otherwise let filename win.
  2052. if corrected_subtask:
  2053. subtask_name = corrected_subtask
  2054. else:
  2055. subtask_name = ""
  2056. if not downloaded_filename or not temp_path:
  2057. logger.warning("Could not find 3MF file for print: %s", filename or subtask_name)
  2058. # Create a fallback archive without 3MF data so the print is still tracked
  2059. # This commonly happens with P1S/A1 printers where FTP has file size limitations
  2060. try:
  2061. from backend.app.models.archive import PrintArchive
  2062. # Derive print name from subtask_name or filename
  2063. print_name = subtask_name or filename
  2064. if print_name:
  2065. # Clean up the name (remove extensions, path parts)
  2066. print_name = print_name.split("/")[-1]
  2067. print_name = print_name.replace(".gcode.3mf", "").replace(".gcode", "").replace(".3mf", "")
  2068. else:
  2069. print_name = "Unknown Print"
  2070. # Recover estimated print time from MQTT (best-effort for notifications)
  2071. fallback_print_time = None
  2072. mqtt_remaining = data.get("remaining_time")
  2073. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  2074. fallback_print_time = int(mqtt_remaining)
  2075. if fallback_print_time is None:
  2076. mc_remaining = (data.get("raw_data") or {}).get("mc_remaining_time")
  2077. if mc_remaining and isinstance(mc_remaining, (int, float)) and mc_remaining > 0:
  2078. fallback_print_time = int(mc_remaining * 60)
  2079. # Create minimal archive entry
  2080. fallback_archive = PrintArchive(
  2081. printer_id=printer_id,
  2082. filename=filename or f"{print_name}.3mf",
  2083. file_path="", # Empty - no 3MF file available
  2084. file_size=0,
  2085. print_name=print_name,
  2086. print_time_seconds=fallback_print_time,
  2087. status="printing",
  2088. started_at=datetime.now(timezone.utc),
  2089. subtask_id=subtask_id,
  2090. extra_data={"no_3mf_available": True, "original_subtask": subtask_name, "_print_data": data},
  2091. )
  2092. db.add(fallback_archive)
  2093. await db.commit()
  2094. await db.refresh(fallback_archive)
  2095. logger.info("Created fallback archive %s for %s (no 3MF available)", fallback_archive.id, print_name)
  2096. # Start timelapse session if external camera is enabled
  2097. if printer.external_camera_enabled and printer.external_camera_url:
  2098. from backend.app.services.layer_timelapse import start_session
  2099. start_session(
  2100. printer_id,
  2101. fallback_archive.id,
  2102. printer.external_camera_url,
  2103. printer.external_camera_type or "mjpeg",
  2104. snapshot_url=printer.external_camera_snapshot_url,
  2105. )
  2106. logger.info("Started layer timelapse for printer %s, archive %s", printer_id, fallback_archive.id)
  2107. # Track as active print
  2108. _active_prints[(printer_id, fallback_archive.filename)] = fallback_archive.id
  2109. if filename:
  2110. _active_prints[(printer_id, filename)] = fallback_archive.id
  2111. if subtask_name:
  2112. _active_prints[(printer_id, f"{subtask_name}.3mf")] = fallback_archive.id
  2113. _active_prints[(printer_id, subtask_name)] = fallback_archive.id
  2114. # Record starting energy if smart plug available (#941: persisted column)
  2115. await _record_energy_start(fallback_archive, printer_id, db, context="fallback")
  2116. # Send WebSocket notification
  2117. await ws_manager.send_archive_created(
  2118. {
  2119. "id": fallback_archive.id,
  2120. "printer_id": fallback_archive.printer_id,
  2121. "filename": fallback_archive.filename,
  2122. "print_name": fallback_archive.print_name,
  2123. "status": fallback_archive.status,
  2124. }
  2125. )
  2126. # MQTT relay - publish archive created
  2127. try:
  2128. await mqtt_relay.on_archive_created(
  2129. archive_id=fallback_archive.id,
  2130. print_name=fallback_archive.print_name,
  2131. printer_name=printer.name,
  2132. status=fallback_archive.status,
  2133. )
  2134. except Exception:
  2135. pass # Don't fail if MQTT fails
  2136. # Store Spoolman tracking data (may not work for fallback since no 3MF)
  2137. try:
  2138. await _store_spoolman_print_data(
  2139. printer_id,
  2140. fallback_archive.id,
  2141. fallback_archive.file_path,
  2142. db,
  2143. printer_manager,
  2144. ams_mapping=_get_start_ams_mapping(data, fallback_archive.id),
  2145. )
  2146. except Exception as e:
  2147. logger.debug("[SPOOLMAN] Could not store tracking for fallback archive: %s", e)
  2148. # Send notification without archive data (file not found)
  2149. if not notification_sent:
  2150. await _send_print_start_notification(printer_id, data, logger=logger)
  2151. return
  2152. except Exception as e:
  2153. logger.error("Failed to create fallback archive: %s", e)
  2154. # Send notification without archive data (file not found)
  2155. if not notification_sent:
  2156. await _send_print_start_notification(printer_id, data, logger=logger)
  2157. return
  2158. try:
  2159. # Archive the file with status "printing"
  2160. service = ArchiveService(db)
  2161. archive = await service.archive_print(
  2162. printer_id=printer_id,
  2163. source_file=temp_path,
  2164. print_data={**data, "status": "printing"},
  2165. subtask_id=subtask_id,
  2166. )
  2167. if archive:
  2168. # Track this active print (use both original filename and downloaded filename)
  2169. _active_prints[(printer_id, downloaded_filename)] = archive.id
  2170. if filename and filename != downloaded_filename:
  2171. _active_prints[(printer_id, filename)] = archive.id
  2172. if subtask_name:
  2173. _active_prints[(printer_id, f"{subtask_name}.3mf")] = archive.id
  2174. logger.info("Created archive %s for %s", archive.id, downloaded_filename)
  2175. # Start timelapse session if external camera is enabled
  2176. if printer.external_camera_enabled and printer.external_camera_url:
  2177. from backend.app.services.layer_timelapse import start_session
  2178. start_session(
  2179. printer_id,
  2180. archive.id,
  2181. printer.external_camera_url,
  2182. printer.external_camera_type or "mjpeg",
  2183. snapshot_url=printer.external_camera_snapshot_url,
  2184. )
  2185. logger.info("Started layer timelapse for printer %s, archive %s", printer_id, archive.id)
  2186. # Record starting energy from smart plug if available (#941: persisted column)
  2187. await _record_energy_start(archive, printer_id, db, context="auto-archive")
  2188. await ws_manager.send_archive_created(
  2189. {
  2190. "id": archive.id,
  2191. "printer_id": archive.printer_id,
  2192. "filename": archive.filename,
  2193. "print_name": archive.print_name,
  2194. "status": archive.status,
  2195. }
  2196. )
  2197. # MQTT relay - publish archive created
  2198. try:
  2199. await mqtt_relay.on_archive_created(
  2200. archive_id=archive.id,
  2201. print_name=archive.print_name,
  2202. printer_name=printer.name,
  2203. status=archive.status,
  2204. )
  2205. except Exception:
  2206. pass # Don't fail if MQTT fails
  2207. # Send notification with archive data (new archive created)
  2208. if not notification_sent:
  2209. archive_data = {
  2210. "print_time_seconds": archive.print_time_seconds,
  2211. "created_by_id": archive.created_by_id,
  2212. }
  2213. await _send_print_start_notification(printer_id, data, archive_data, logger)
  2214. # Extract printable objects for skip object functionality
  2215. try:
  2216. from backend.app.services.archive import extract_printable_objects_from_3mf
  2217. with open(temp_path, "rb") as f:
  2218. threemf_data = f.read()
  2219. # Extract with positions for UI overlay
  2220. printable_objects, bbox_all = extract_printable_objects_from_3mf(
  2221. threemf_data, include_positions=True
  2222. )
  2223. if printable_objects:
  2224. # Store objects in printer state
  2225. client = printer_manager.get_client(printer_id)
  2226. if client:
  2227. client.state.printable_objects = printable_objects
  2228. client.state.printable_objects_bbox_all = bbox_all
  2229. client.state.skipped_objects = [] # Reset skipped objects for new print
  2230. logger.info(
  2231. "Loaded %s printable objects for printer %s", len(printable_objects), printer_id
  2232. )
  2233. except Exception as e:
  2234. logger.debug("Failed to extract printable objects: %s", e)
  2235. # Store Spoolman tracking data for per-filament usage reporting
  2236. try:
  2237. await _store_spoolman_print_data(
  2238. printer_id,
  2239. archive.id,
  2240. archive.file_path,
  2241. db,
  2242. printer_manager,
  2243. ams_mapping=_get_start_ams_mapping(data, archive.id),
  2244. )
  2245. except Exception as e:
  2246. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  2247. # Capture timelapse file baseline for snapshot-diff on completion
  2248. try:
  2249. baseline_files, _ = await _list_timelapse_videos(printer)
  2250. _timelapse_baselines[printer_id] = {f.get("name", "") for f in baseline_files}
  2251. logger.info(
  2252. "[TIMELAPSE] Baseline at print start: %s video files for printer %s",
  2253. len(_timelapse_baselines[printer_id]),
  2254. printer_id,
  2255. )
  2256. except Exception as e:
  2257. logger.warning("[TIMELAPSE] Failed to capture baseline at print start: %s", e)
  2258. finally:
  2259. # Keep temp_path around until print completes so the cover endpoint
  2260. # can reuse it (#972). Cache eviction in on_print_complete deletes
  2261. # the file. If the cache entry was evicted early (file vanished),
  2262. # clean up any stragglers here to avoid leaking disk on retries.
  2263. cached_now = get_cached_3mf(printer_id, downloaded_filename) if downloaded_filename else None
  2264. if temp_path and temp_path.exists() and cached_now != temp_path:
  2265. temp_path.unlink()
  2266. _TIMELAPSE_VIDEO_EXTENSIONS = (".mp4", ".avi")
  2267. async def _list_timelapse_videos(printer) -> tuple[list[dict], str | None]:
  2268. """List video files from printer's timelapse directory.
  2269. Finds MP4 (X1/A1 series) and AVI (P1 series) timelapse files.
  2270. Returns (video_files, found_path) where video_files is a list of file dicts
  2271. and found_path is the directory where they were found, or ([], None).
  2272. """
  2273. from backend.app.services.bambu_ftp import list_files_async
  2274. logger = logging.getLogger(__name__)
  2275. for timelapse_path in ["/timelapse", "/timelapse/video", "/record", "/recording"]:
  2276. try:
  2277. found_files = await list_files_async(
  2278. printer.ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  2279. )
  2280. if found_files:
  2281. video_files = [
  2282. f
  2283. for f in found_files
  2284. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  2285. ]
  2286. if video_files:
  2287. return video_files, timelapse_path
  2288. except Exception as e:
  2289. logger.debug("[TIMELAPSE] Path %s failed: %s", timelapse_path, e)
  2290. continue
  2291. return [], None
  2292. async def _scan_for_timelapse_with_retries(archive_id: int, baseline_names: set[str] | None = None):
  2293. """
  2294. Scan for timelapse with retries using a snapshot-diff approach.
  2295. Instead of picking the "most recent by mtime" (unreliable when the printer
  2296. clock is wrong in LAN-only mode), we snapshot existing MP4 filenames BEFORE
  2297. waiting, then look for any NEW filename that appears after each delay.
  2298. If baseline_names is provided (captured at print start), it is used directly.
  2299. Otherwise falls back to taking a baseline at completion time (best-effort
  2300. for prints started before app restart).
  2301. Falls back to name-matching (print name contained in MP4 filename) if no
  2302. new file appears after all retries.
  2303. """
  2304. from pathlib import Path
  2305. logger = logging.getLogger(__name__)
  2306. # --- Phase 1: Take baseline snapshot of existing timelapse files ---
  2307. try:
  2308. async with async_session() as db:
  2309. from backend.app.models.printer import Printer
  2310. service = ArchiveService(db)
  2311. archive = await service.get_archive(archive_id)
  2312. if not archive:
  2313. logger.warning("[TIMELAPSE] Archive %s not found, aborting", archive_id)
  2314. return
  2315. if archive.timelapse_path:
  2316. logger.info("[TIMELAPSE] Archive %s already has timelapse attached", archive_id)
  2317. return
  2318. if not archive.printer_id:
  2319. logger.warning("[TIMELAPSE] Archive %s has no printer, aborting", archive_id)
  2320. return
  2321. if baseline_names is not None:
  2322. # Use pre-captured baseline from print start (no race condition)
  2323. logger.info(
  2324. "[TIMELAPSE] Using print-start baseline: %s existing video files for archive %s",
  2325. len(baseline_names),
  2326. archive_id,
  2327. )
  2328. else:
  2329. # Fallback: take baseline now (e.g. app restarted mid-print)
  2330. result = await db.execute(select(Printer).where(Printer.id == archive.printer_id))
  2331. printer = result.scalar_one_or_none()
  2332. if not printer:
  2333. logger.warning("[TIMELAPSE] Printer not found for archive %s, aborting", archive_id)
  2334. return
  2335. baseline_files, _ = await _list_timelapse_videos(printer)
  2336. baseline_names = {f.get("name", "") for f in baseline_files}
  2337. logger.info(
  2338. "[TIMELAPSE] Baseline snapshot (fallback): %s existing video files for archive %s",
  2339. len(baseline_names),
  2340. archive_id,
  2341. )
  2342. # Derive base_name for name-matching fallback
  2343. base_name = Path(archive.filename).stem if archive.filename else ""
  2344. if base_name.endswith(".gcode"):
  2345. base_name = base_name[:-6]
  2346. except Exception as e:
  2347. logger.warning("[TIMELAPSE] Failed to take baseline snapshot for archive %s: %s", archive_id, e)
  2348. return
  2349. # --- Phase 2: Retry loop — look for NEW files that weren't in baseline ---
  2350. retry_delays = [5, 10, 20, 30]
  2351. for attempt, delay in enumerate(retry_delays, 1):
  2352. logger.info(
  2353. "[TIMELAPSE] Attempt %s/%s: waiting %ss before scanning for archive %s",
  2354. attempt,
  2355. len(retry_delays),
  2356. delay,
  2357. archive_id,
  2358. )
  2359. await asyncio.sleep(delay)
  2360. try:
  2361. async with async_session() as db:
  2362. from backend.app.models.printer import Printer
  2363. from backend.app.services.bambu_ftp import download_file_bytes_async
  2364. service = ArchiveService(db)
  2365. archive = await service.get_archive(archive_id)
  2366. if not archive:
  2367. logger.warning("[TIMELAPSE] Archive %s not found, stopping retries", archive_id)
  2368. return
  2369. if archive.timelapse_path:
  2370. logger.info("[TIMELAPSE] Archive %s already has timelapse attached, stopping retries", archive_id)
  2371. return
  2372. result = await db.execute(select(Printer).where(Printer.id == archive.printer_id))
  2373. printer = result.scalar_one_or_none()
  2374. if not printer:
  2375. logger.warning("[TIMELAPSE] Printer not found for archive %s, stopping retries", archive_id)
  2376. return
  2377. video_files, found_path = await _list_timelapse_videos(printer)
  2378. if not video_files:
  2379. logger.info("[TIMELAPSE] Attempt %s: No video files found, will retry", attempt)
  2380. continue
  2381. logger.info("[TIMELAPSE] Attempt %s: Found %s video files in %s", attempt, len(video_files), found_path)
  2382. for f in video_files[:5]:
  2383. logger.info("[TIMELAPSE] - %s", f.get("name"))
  2384. # Find files that are NEW (not in baseline snapshot)
  2385. new_files = [f for f in video_files if f.get("name", "") not in baseline_names]
  2386. if new_files:
  2387. # Pick the first new file (there should typically be exactly one)
  2388. target = new_files[0]
  2389. file_name = target.get("name")
  2390. remote_path = target.get("path") or f"/timelapse/{file_name}"
  2391. logger.info(
  2392. "[TIMELAPSE] Attempt %s: New file detected: %s (downloading for archive %s)",
  2393. attempt,
  2394. file_name,
  2395. archive_id,
  2396. )
  2397. timelapse_data = await download_file_bytes_async(
  2398. printer.ip_address, printer.access_code, remote_path, printer_model=printer.model
  2399. )
  2400. if timelapse_data:
  2401. success = await service.attach_timelapse(archive_id, timelapse_data, file_name)
  2402. if success:
  2403. logger.info("[TIMELAPSE] Successfully attached timelapse to archive %s", archive_id)
  2404. await ws_manager.send_archive_updated({"id": archive_id, "timelapse_attached": True})
  2405. return
  2406. else:
  2407. logger.warning("[TIMELAPSE] Failed to attach timelapse to archive %s", archive_id)
  2408. else:
  2409. logger.warning("[TIMELAPSE] Attempt %s: Failed to download new file, will retry", attempt)
  2410. else:
  2411. logger.info("[TIMELAPSE] Attempt %s: No new files since baseline, will retry", attempt)
  2412. except Exception as e:
  2413. logger.warning("[TIMELAPSE] Attempt %s failed with error: %s", attempt, e)
  2414. # --- Phase 3: Fallback — try name matching against all files ---
  2415. if base_name:
  2416. logger.info("[TIMELAPSE] Retries exhausted, trying name-match fallback for '%s'", base_name)
  2417. try:
  2418. async with async_session() as db:
  2419. from backend.app.models.printer import Printer
  2420. from backend.app.services.bambu_ftp import download_file_bytes_async
  2421. service = ArchiveService(db)
  2422. archive = await service.get_archive(archive_id)
  2423. if not archive or archive.timelapse_path:
  2424. return
  2425. result = await db.execute(select(Printer).where(Printer.id == archive.printer_id))
  2426. printer = result.scalar_one_or_none()
  2427. if not printer:
  2428. return
  2429. video_files, found_path = await _list_timelapse_videos(printer)
  2430. for f in video_files:
  2431. fname = f.get("name", "")
  2432. if base_name.lower() in fname.lower():
  2433. remote_path = f.get("path") or f"/timelapse/{fname}"
  2434. logger.info("[TIMELAPSE] Name-match fallback: '%s' matches '%s'", base_name, fname)
  2435. timelapse_data = await download_file_bytes_async(
  2436. printer.ip_address, printer.access_code, remote_path, printer_model=printer.model
  2437. )
  2438. if timelapse_data:
  2439. success = await service.attach_timelapse(archive_id, timelapse_data, fname)
  2440. if success:
  2441. logger.info(
  2442. "[TIMELAPSE] Name-match fallback attached timelapse to archive %s", archive_id
  2443. )
  2444. await ws_manager.send_archive_updated({"id": archive_id, "timelapse_attached": True})
  2445. return
  2446. break # Only try the first name match
  2447. except Exception as e:
  2448. logger.warning("[TIMELAPSE] Name-match fallback failed: %s", e)
  2449. logger.warning("[TIMELAPSE] All attempts exhausted for archive %s, giving up", archive_id)
  2450. async def on_print_complete(printer_id: int, data: dict):
  2451. """Handle print completion - update the archive status."""
  2452. import time
  2453. logger = logging.getLogger(__name__)
  2454. start_time = time.time()
  2455. def log_timing(section: str):
  2456. elapsed = time.time() - start_time
  2457. logger.info("[TIMING] %s: %.3fs elapsed", section, elapsed)
  2458. logger.info("[CALLBACK] on_print_complete started for printer %s", printer_id)
  2459. # Drop the 3MF download cache for this printer (#972). The print is over,
  2460. # nothing else legitimately needs the bytes; keeping them would only risk
  2461. # handing a stale file to the next print if it reuses the same name.
  2462. clear_3mf_cache(printer_id)
  2463. try:
  2464. ws_data = {
  2465. "status": data.get("status"),
  2466. "filename": data.get("filename"),
  2467. "subtask_name": data.get("subtask_name"),
  2468. "timelapse_was_active": data.get("timelapse_was_active"),
  2469. }
  2470. await ws_manager.send_print_complete(printer_id, ws_data)
  2471. log_timing("WebSocket send_print_complete")
  2472. except Exception as e:
  2473. logger.warning("[CALLBACK] WebSocket send_print_complete failed: %s", e)
  2474. # Capture user info before clearing (needed for print log entry)
  2475. _print_user_info = printer_manager.get_current_print_user(printer_id)
  2476. # Clear current print user tracking (Issue #206)
  2477. printer_manager.clear_current_print_user(printer_id)
  2478. # If the user explicitly stopped this print from the queue UI the printer will
  2479. # report "failed" or "aborted" via MQTT. Override that to "cancelled" so the
  2480. # correct "print stopped" notification/email is sent instead of a failure alert.
  2481. _raw_status = data.get("status", "completed")
  2482. if printer_id in _user_stopped_printers and _raw_status in ("failed", "aborted"):
  2483. logger.info(
  2484. "[CALLBACK] Overriding status '%s' -> 'cancelled' for printer %s (print was stopped from queue by user)",
  2485. _raw_status,
  2486. printer_id,
  2487. )
  2488. data = {**data, "status": "cancelled"}
  2489. _user_stopped_printers.discard(printer_id)
  2490. # Raise the plate-clear gate for queued dispatch (#961). Any terminal status
  2491. # may have left material on the bed: a user can cancel ten hours into a
  2492. # twelve-hour print, a printer can self-abort mid-job after a clog, and a
  2493. # touchscreen-stop reports `aborted` rather than `cancelled` because
  2494. # `_user_stopped_printers` is only populated when the user stops via the
  2495. # Bambuddy queue UI. Earlier code raised the flag only for completed/failed,
  2496. # which auto-dispatched the next queued print onto a fouled bed two seconds
  2497. # after a touchscreen-abort (#1171). Persisted to DB so the gate survives
  2498. # Auto Off power cycles and Bambuddy restarts.
  2499. _final_status = data.get("status", "completed")
  2500. if _final_status in ("completed", "failed", "aborted", "cancelled"):
  2501. printer_manager.set_awaiting_plate_clear(printer_id, True)
  2502. # MQTT relay - publish print complete
  2503. try:
  2504. printer_info = printer_manager.get_printer(printer_id)
  2505. if printer_info:
  2506. await mqtt_relay.on_print_complete(
  2507. printer_id,
  2508. printer_info.name,
  2509. printer_info.serial_number,
  2510. data.get("filename", ""),
  2511. data.get("subtask_name", ""),
  2512. data.get("status", "completed"),
  2513. )
  2514. except Exception:
  2515. pass # Don't fail print complete callback if MQTT fails
  2516. filename = data.get("filename", "")
  2517. subtask_name = data.get("subtask_name", "")
  2518. if not filename and not subtask_name:
  2519. logger.warning("Print complete without filename or subtask_name")
  2520. return
  2521. logger.info("Print complete - filename: %s, subtask: %s, status: %s", filename, subtask_name, data.get("status"))
  2522. # Build list of possible keys to try (matching how they were registered in on_print_start)
  2523. possible_keys = []
  2524. # Try subtask_name variations first (most reliable for matching)
  2525. if subtask_name:
  2526. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  2527. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  2528. possible_keys.append((printer_id, subtask_name))
  2529. # Try filename variations
  2530. if filename:
  2531. # Extract just the filename if it's a path
  2532. fname = filename.split("/")[-1] if "/" in filename else filename
  2533. if fname.endswith(".3mf"):
  2534. possible_keys.append((printer_id, fname))
  2535. elif fname.endswith(".gcode"):
  2536. base_name = fname.rsplit(".", 1)[0]
  2537. possible_keys.append((printer_id, f"{base_name}.gcode.3mf"))
  2538. possible_keys.append((printer_id, f"{base_name}.3mf"))
  2539. possible_keys.append((printer_id, fname))
  2540. else:
  2541. possible_keys.append((printer_id, f"{fname}.gcode.3mf"))
  2542. possible_keys.append((printer_id, f"{fname}.3mf"))
  2543. possible_keys.append((printer_id, fname))
  2544. # Also try full path versions
  2545. if filename.endswith(".3mf"):
  2546. possible_keys.append((printer_id, filename))
  2547. elif filename.endswith(".gcode"):
  2548. base_name = filename.rsplit(".", 1)[0]
  2549. possible_keys.append((printer_id, f"{base_name}.3mf"))
  2550. possible_keys.append((printer_id, filename))
  2551. else:
  2552. possible_keys.append((printer_id, f"{filename}.3mf"))
  2553. possible_keys.append((printer_id, filename))
  2554. # Find the archive for this print
  2555. logger.info("Looking for archive in _active_prints, keys to try: %s...", possible_keys[:5])
  2556. logger.info("Current _active_prints: %s", list(_active_prints.keys()))
  2557. archive_id = None
  2558. for key in possible_keys:
  2559. archive_id = _active_prints.pop(key, None)
  2560. if archive_id:
  2561. logger.info("Found archive %s with key %s", archive_id, key)
  2562. # Also clean up any other keys pointing to this archive
  2563. keys_to_remove = [k for k, v in _active_prints.items() if v == archive_id]
  2564. for k in keys_to_remove:
  2565. _active_prints.pop(k, None)
  2566. break
  2567. if not archive_id:
  2568. # Try to find by filename or subtask_name if not tracked (for prints started before app)
  2569. async with async_session() as db:
  2570. from backend.app.models.archive import PrintArchive
  2571. # Try matching by subtask_name (stored as print_name) first
  2572. if subtask_name:
  2573. result = await db.execute(
  2574. select(PrintArchive)
  2575. .where(PrintArchive.printer_id == printer_id)
  2576. .where(PrintArchive.status == "printing")
  2577. .where(
  2578. or_(
  2579. PrintArchive.print_name.ilike(f"%{subtask_name}%"),
  2580. PrintArchive.filename.ilike(f"%{subtask_name}%"),
  2581. )
  2582. )
  2583. .order_by(PrintArchive.created_at.desc())
  2584. .limit(1)
  2585. )
  2586. archive = result.scalar_one_or_none()
  2587. if archive:
  2588. archive_id = archive.id
  2589. logger.info("Found archive %s by subtask_name match: %s", archive_id, subtask_name)
  2590. # Also try by filename
  2591. if not archive_id and filename:
  2592. result = await db.execute(
  2593. select(PrintArchive)
  2594. .where(PrintArchive.printer_id == printer_id)
  2595. .where(PrintArchive.filename == filename)
  2596. .where(PrintArchive.status == "printing")
  2597. .order_by(PrintArchive.created_at.desc())
  2598. .limit(1)
  2599. )
  2600. archive = result.scalar_one_or_none()
  2601. if archive:
  2602. archive_id = archive.id
  2603. # Cleanup: delete uploaded file from printer SD card to prevent phantom prints (Issue #374)
  2604. # The print scheduler uploads files to the SD card root (/). Some printers (e.g. P1S)
  2605. # auto-start files found in root on power cycle, causing ghost prints.
  2606. # Must run before the archive_id early-return so it executes even when archiving is disabled.
  2607. try:
  2608. if subtask_name:
  2609. async with async_session() as db:
  2610. from backend.app.models.printer import Printer
  2611. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2612. printer = result.scalar_one_or_none()
  2613. if printer:
  2614. from backend.app.services.bambu_ftp import delete_file_async
  2615. # Try both .3mf and .gcode extensions — the printer may have either
  2616. for ext in (".3mf", ".gcode"):
  2617. remote_path = f"/{subtask_name}{ext}"
  2618. # Retry up to 3 times — the printer may still lock the filesystem briefly after a print ends
  2619. for attempt in range(1, 4):
  2620. try:
  2621. delete_result = await delete_file_async(
  2622. printer.ip_address,
  2623. printer.access_code,
  2624. remote_path,
  2625. printer_model=printer.model,
  2626. )
  2627. if delete_result:
  2628. logger.info("Deleted %s from printer %s SD card", remote_path, printer.name)
  2629. break
  2630. except Exception as e:
  2631. delete_result = False
  2632. logger.warning(
  2633. "SD card cleanup attempt %d/3 raised for %s: %s",
  2634. attempt,
  2635. remote_path,
  2636. e,
  2637. )
  2638. if not delete_result and attempt < 3:
  2639. await asyncio.sleep(2)
  2640. elif not delete_result:
  2641. logger.warning(
  2642. "SD card cleanup failed after 3 attempts for %s (file may linger on SD card)",
  2643. remote_path,
  2644. )
  2645. except Exception as e:
  2646. logger.warning("SD card file cleanup failed for printer %s: %s", printer_id, e)
  2647. log_timing("SD card cleanup")
  2648. # Update queue item status early — must run before the archive_id early-return
  2649. # so queue items don't get stuck in "printing" when archive lookup fails.
  2650. # Uses run_with_retry to handle SQLite "database is locked" errors (#897).
  2651. queue_item_id = None
  2652. queue_status = None
  2653. queue_auto_off = False
  2654. try:
  2655. from backend.app.core.database import run_with_retry
  2656. from backend.app.models.print_queue import PrintQueueItem
  2657. async def _update_queue_status(db):
  2658. nonlocal queue_item_id, queue_status, queue_auto_off
  2659. result = await db.execute(
  2660. select(PrintQueueItem)
  2661. .where(PrintQueueItem.printer_id == printer_id)
  2662. .where(PrintQueueItem.status == "printing")
  2663. )
  2664. printing_items = list(result.scalars().all())
  2665. if len(printing_items) > 1:
  2666. logger.warning(
  2667. "BUG: Multiple queue items in 'printing' status for printer %s: %s",
  2668. printer_id,
  2669. [(i.id, i.archive_id, i.library_file_id) for i in printing_items],
  2670. )
  2671. item = printing_items[0] if printing_items else None
  2672. if item:
  2673. queue_status = data.get("status", "completed")
  2674. # MQTT sends "aborted" for cancelled prints; normalise to
  2675. # "cancelled" so it matches the queue schema Literal.
  2676. if queue_status == "aborted":
  2677. queue_status = "cancelled"
  2678. item.status = queue_status
  2679. item.completed_at = datetime.now(timezone.utc)
  2680. if queue_status == "failed" and not item.error_message:
  2681. item.error_message = _format_hms_error_summary(data.get("hms_errors") or [])
  2682. # Bump usage counters on the source library file so admins can
  2683. # sort by "last printed" and (eventually) auto-purge stale
  2684. # files — #1008.
  2685. await _bump_library_file_usage_if_completed(db, item, queue_status)
  2686. await db.commit()
  2687. queue_item_id = item.id
  2688. queue_auto_off = item.auto_off_after
  2689. logger.info("Updated queue item %s status to %s", item.id, queue_status)
  2690. await run_with_retry(_update_queue_status, label="queue status update")
  2691. # Post-commit side effects (notifications, MQTT relay, auto-off) use
  2692. # their own sessions and have their own error handling — no retry needed.
  2693. if queue_item_id is not None:
  2694. # MQTT relay - publish queue job completed
  2695. try:
  2696. printer_info = printer_manager.get_printer(printer_id)
  2697. await mqtt_relay.on_queue_job_completed(
  2698. job_id=queue_item_id,
  2699. filename=filename or subtask_name,
  2700. printer_id=printer_id,
  2701. printer_name=printer_info.name if printer_info else "Unknown",
  2702. status=queue_status,
  2703. )
  2704. except Exception:
  2705. pass # Don't fail if MQTT fails
  2706. # Check if queue is now empty and send notification
  2707. try:
  2708. from sqlalchemy import func as sa_func
  2709. async with async_session() as db:
  2710. count_result = await db.execute(
  2711. select(sa_func.count(PrintQueueItem.id)).where(PrintQueueItem.status == "pending")
  2712. )
  2713. pending_count = count_result.scalar() or 0
  2714. if pending_count == 0:
  2715. today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
  2716. completed_result = await db.execute(
  2717. select(sa_func.count(PrintQueueItem.id)).where(
  2718. PrintQueueItem.status.in_(["completed", "failed", "skipped"]),
  2719. PrintQueueItem.completed_at >= today_start,
  2720. )
  2721. )
  2722. completed_count = completed_result.scalar() or 1
  2723. await notification_service.on_queue_completed(
  2724. completed_count=completed_count,
  2725. db=db,
  2726. )
  2727. except Exception:
  2728. pass # Don't fail if notification fails
  2729. # Handle auto_off_after - power off printer if requested (after cooldown)
  2730. if queue_auto_off:
  2731. async with async_session() as db:
  2732. result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  2733. plugs = list(result.scalars().all())
  2734. enabled_plugs = [p for p in plugs if p.enabled]
  2735. if enabled_plugs:
  2736. logger.info("Auto-off requested for printer %s, waiting for cooldown...", printer_id)
  2737. async def cooldown_and_poweroff(pid: int, plug_ids: list[int]):
  2738. # Wait for nozzle to cool down
  2739. await printer_manager.wait_for_cooldown(pid, target_temp=50.0, timeout=600)
  2740. # Re-fetch plugs in new session and turn off each one
  2741. async with async_session() as new_db:
  2742. for plug_id in plug_ids:
  2743. try:
  2744. result = await new_db.execute(select(SmartPlug).where(SmartPlug.id == plug_id))
  2745. p = result.scalar_one_or_none()
  2746. if p and p.enabled:
  2747. service = await smart_plug_manager.get_service_for_plug(p, new_db)
  2748. success = await service.turn_off(p)
  2749. if success:
  2750. logger.info("Powered off printer %s via smart plug '%s'", pid, p.name)
  2751. else:
  2752. logger.warning("Failed to power off plug '%s' for printer %s", p.name, pid)
  2753. except Exception as e:
  2754. logger.warning("Failed to power off plug %s for printer %s: %s", plug_id, pid, e)
  2755. asyncio.create_task(cooldown_and_poweroff(printer_id, [p.id for p in enabled_plugs]))
  2756. except Exception as e:
  2757. logging.getLogger(__name__).warning(f"Queue item update failed: {e}")
  2758. log_timing("Queue item update")
  2759. # Register bed cooldown waiter (event-driven via on_bed_temp_update callback).
  2760. # Must run before archive_id early-return so it fires for all prints (including
  2761. # prints started from BambuStudio/touchscreen that have no archive).
  2762. if data.get("status") == "completed":
  2763. try:
  2764. from backend.app.api.routes.settings import get_setting
  2765. async with async_session() as db:
  2766. threshold_str = await get_setting(db, "bed_cooled_threshold")
  2767. threshold = float(threshold_str) if threshold_str else 35.0
  2768. # Check if any provider has on_bed_cooled enabled (skip registration if none)
  2769. async with async_session() as db:
  2770. providers = await notification_service._get_providers_for_event(db, "on_bed_cooled", printer_id)
  2771. if providers:
  2772. _bed_cool_waiters[printer_id] = {
  2773. "threshold": threshold,
  2774. "filename": filename or subtask_name or "",
  2775. "registered_at": time.time(),
  2776. }
  2777. logger.info(
  2778. "[BED-COOL] Registered waiter for printer %s (threshold: %.0f°C)",
  2779. printer_id,
  2780. threshold,
  2781. )
  2782. else:
  2783. logger.debug("[BED-COOL] No providers enabled for bed_cooled on printer %s", printer_id)
  2784. except Exception as e:
  2785. logger.warning("[BED-COOL] Failed to register waiter: %s", e)
  2786. # --- Track filament consumption (must run before archive_id early-return so usage
  2787. # is recorded even when auto-archive is disabled) ---
  2788. usage_results: list[dict] = []
  2789. # Prefer ams_mapping captured from MQTT request topic (works for all print sources)
  2790. stored_ams_mapping = data.get("ams_mapping")
  2791. # Fallback to _print_ams_mappings for queue/reprint (set before print starts)
  2792. if not stored_ams_mapping and archive_id:
  2793. stored_ams_mapping = _print_ams_mappings.pop(archive_id, None)
  2794. # Internal inventory: track AMS remain% deltas (skip if Spoolman handles usage)
  2795. try:
  2796. async with async_session() as db:
  2797. from backend.app.api.routes.settings import get_setting
  2798. _spoolman_on = await get_setting(db, "spoolman_enabled")
  2799. if not _spoolman_on or _spoolman_on.lower() != "true":
  2800. from backend.app.services.usage_tracker import on_print_complete as usage_on_print_complete
  2801. async with async_session() as db:
  2802. usage_results = await usage_on_print_complete(
  2803. printer_id,
  2804. data,
  2805. printer_manager,
  2806. db,
  2807. archive_id=archive_id,
  2808. ams_mapping=stored_ams_mapping,
  2809. )
  2810. if usage_results:
  2811. await ws_manager.broadcast(
  2812. {
  2813. "type": "spool_usage_logged",
  2814. "printer_id": printer_id,
  2815. "usage": usage_results,
  2816. }
  2817. )
  2818. log_timing("Usage tracker")
  2819. except Exception as e:
  2820. logger.warning("Usage tracker on_print_complete failed: %s", e)
  2821. # Spoolman: report filament usage (requires archive_id for tracking data lookup)
  2822. if archive_id:
  2823. if data.get("status") == "completed":
  2824. try:
  2825. await _report_spoolman_usage(printer_id, archive_id)
  2826. log_timing("Spoolman usage report")
  2827. except Exception as e:
  2828. logger.warning("Spoolman usage reporting failed: %s", e)
  2829. else:
  2830. # Report partial usage if tracking data exists (only stored when weight sync is disabled)
  2831. try:
  2832. async with async_session() as db:
  2833. await _cleanup_spoolman_tracking(
  2834. printer_id,
  2835. archive_id,
  2836. db,
  2837. last_layer_num=data.get("last_layer_num"),
  2838. last_progress=data.get("last_progress"),
  2839. )
  2840. except Exception as e:
  2841. logger.debug("[SPOOLMAN] Cleanup failed: %s", e)
  2842. log_timing("Filament usage tracking")
  2843. if not archive_id:
  2844. logger.warning("Could not find archive for print complete: filename=%s, subtask=%s", filename, subtask_name)
  2845. # Still send print-complete/failed/stopped notifications even without an archive.
  2846. # Try to enrich with queue/library-file data so user-specific emails work too.
  2847. async def _notify_no_archive():
  2848. try:
  2849. async with async_session() as db:
  2850. from backend.app.models.library import LibraryFile
  2851. from backend.app.models.print_queue import PrintQueueItem
  2852. from backend.app.models.printer import Printer
  2853. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2854. printer_obj = result.scalar_one_or_none()
  2855. p_name = printer_obj.name if printer_obj else f"Printer {printer_id}"
  2856. # Try to find the most-recent queue item for this printer so we can
  2857. # recover created_by_id and estimated print time.
  2858. # NOTE: By the time this task runs the queue item status has already
  2859. # been updated to a terminal state (completed/failed/cancelled), so
  2860. # we look for recently-completed items (within the last 5 minutes).
  2861. no_archive_data: dict | None = None
  2862. try:
  2863. cutoff = datetime.now(timezone.utc) - timedelta(minutes=5)
  2864. q_result = await db.execute(
  2865. select(PrintQueueItem)
  2866. .where(PrintQueueItem.printer_id == printer_id)
  2867. .where(PrintQueueItem.status.in_(["completed", "failed", "cancelled"]))
  2868. .where(PrintQueueItem.completed_at >= cutoff)
  2869. .order_by(PrintQueueItem.completed_at.desc())
  2870. .limit(1)
  2871. )
  2872. queue_item = q_result.scalar_one_or_none()
  2873. if queue_item:
  2874. no_archive_data = {"created_by_id": queue_item.created_by_id}
  2875. # Pull estimated time from library file when available
  2876. if queue_item.library_file_id:
  2877. lib_result = await db.execute(
  2878. select(LibraryFile).where(LibraryFile.id == queue_item.library_file_id)
  2879. )
  2880. lib_file = lib_result.scalar_one_or_none()
  2881. if lib_file and lib_file.print_time_seconds:
  2882. no_archive_data["print_time_seconds"] = lib_file.print_time_seconds
  2883. except Exception as lookup_err:
  2884. logger.debug(
  2885. "[NOTIFY-BG] Could not look up queue item for no-archive notification: %s", lookup_err
  2886. )
  2887. # Enrich with usage tracker results (captured in enclosing scope)
  2888. if usage_results:
  2889. if no_archive_data is None:
  2890. no_archive_data = {}
  2891. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  2892. if total_from_usage > 0:
  2893. no_archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  2894. no_archive_data["usage_results"] = usage_results
  2895. # Try MQTT remaining_time for print duration when no queue/library data
  2896. if no_archive_data and not no_archive_data.get("print_time_seconds"):
  2897. mqtt_remaining = data.get("remaining_time")
  2898. if mqtt_remaining and isinstance(mqtt_remaining, (int, float)) and mqtt_remaining > 0:
  2899. no_archive_data["print_time_seconds"] = int(mqtt_remaining)
  2900. ps = data.get("status", "completed")
  2901. logger.info(
  2902. "[NOTIFY-BG] Sending notification without archive: printer=%s, status=%s", printer_id, ps
  2903. )
  2904. await notification_service.on_print_complete(
  2905. printer_id, p_name, ps, data, db, archive_data=no_archive_data
  2906. )
  2907. # Send user-specific email if we have a created_by_id
  2908. if no_archive_data and no_archive_data.get("created_by_id"):
  2909. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  2910. await _dispatch_user_print_email(
  2911. ps,
  2912. no_archive_data["created_by_id"],
  2913. p_name,
  2914. raw_filename,
  2915. db,
  2916. )
  2917. logger.info("[NOTIFY-BG] Completed (no-archive path)")
  2918. except Exception as e:
  2919. logger.warning("[NOTIFY-BG] Failed to send notification without archive: %s", e, exc_info=True)
  2920. task = asyncio.create_task(_notify_no_archive())
  2921. task.add_done_callback(lambda _t: None)
  2922. return
  2923. log_timing("Archive lookup")
  2924. # Update archive status
  2925. logger.info("[ARCHIVE] Updating archive %s status...", archive_id)
  2926. try:
  2927. async with async_session() as db:
  2928. service = ArchiveService(db)
  2929. status = data.get("status", "completed")
  2930. hms_errors = data.get("hms_errors", []) if status == "failed" else None
  2931. if hms_errors:
  2932. logger.info("[ARCHIVE] HMS errors at failure: %s", hms_errors)
  2933. failure_reason = derive_failure_reason(status, hms_errors)
  2934. if failure_reason:
  2935. logger.info("[ARCHIVE] failure_reason=%r (status=%s)", failure_reason, status)
  2936. elif status == "failed" and hms_errors:
  2937. logger.info("[ARCHIVE] HMS errors present but none matched a known failure-reason short code")
  2938. await service.update_archive_status(
  2939. archive_id,
  2940. status=status,
  2941. completed_at=(
  2942. datetime.now(timezone.utc) if status in ("completed", "failed", "aborted", "cancelled") else None
  2943. ),
  2944. failure_reason=failure_reason,
  2945. )
  2946. logger.info(
  2947. "[ARCHIVE] Archive %s status updated to %s, failure_reason=%s", archive_id, status, failure_reason
  2948. )
  2949. await ws_manager.send_archive_updated(
  2950. {
  2951. "id": archive_id,
  2952. "status": status,
  2953. }
  2954. )
  2955. logger.info("[ARCHIVE] WebSocket notification sent for archive %s", archive_id)
  2956. # MQTT relay - publish archive updated
  2957. try:
  2958. await mqtt_relay.on_archive_updated(
  2959. archive_id=archive_id,
  2960. print_name=filename or subtask_name,
  2961. status=status,
  2962. )
  2963. except Exception:
  2964. pass # Don't fail if MQTT fails
  2965. except Exception as e:
  2966. logger.error("[ARCHIVE] Failed to update archive %s status: %s", archive_id, e, exc_info=True)
  2967. # Continue with other operations even if archive update fails
  2968. log_timing("Archive status update")
  2969. # Write independent print log entry (separate table, never touches archives)
  2970. try:
  2971. async with async_session() as db:
  2972. from backend.app.models.archive import PrintArchive
  2973. from backend.app.services.print_log import write_log_entry
  2974. archive = await db.get(PrintArchive, archive_id)
  2975. if archive:
  2976. # Back-fill created_by_id on reprint (#730): reprint reuses the
  2977. # source archive row rather than creating a new one, so an
  2978. # archive that was auto-created from a printer-initiated
  2979. # print (created_by_id=NULL) would otherwise stay unattributed
  2980. # forever. When we have a print-session user AND the archive
  2981. # has no attribution yet, credit the current user. Never
  2982. # overwrite an existing attribution — the original uploader
  2983. # keeps ownership.
  2984. _print_user_id = _print_user_info.get("user_id") if _print_user_info else None
  2985. if archive.created_by_id is None and _print_user_id is not None:
  2986. archive.created_by_id = _print_user_id
  2987. p_info = printer_manager.get_printer(printer_id)
  2988. await write_log_entry(
  2989. db,
  2990. status=data.get("status", "completed"),
  2991. print_name=archive.print_name,
  2992. printer_name=p_info.name if p_info else None,
  2993. printer_id=printer_id,
  2994. started_at=archive.started_at,
  2995. completed_at=archive.completed_at,
  2996. filament_type=archive.filament_type,
  2997. filament_color=archive.filament_color,
  2998. filament_used_grams=archive.filament_used_grams,
  2999. thumbnail_path=archive.thumbnail_path,
  3000. created_by_username=_print_user_info.get("username") if _print_user_info else None,
  3001. )
  3002. await db.commit()
  3003. logger.info("[PRINT_LOG] Log entry written for archive %s", archive_id)
  3004. except Exception as e:
  3005. logger.warning("[PRINT_LOG] Failed to write log entry for archive %s: %s", archive_id, e)
  3006. log_timing("Print log entry")
  3007. # Run slow operations as background tasks to avoid blocking the event loop
  3008. # These operations can take 5-10+ seconds and would freeze the UI if awaited
  3009. async def _background_energy_calculation():
  3010. """Calculate and save energy usage in background.
  3011. Reads the starting kWh from the archive row (#941: persisted so a mid-print
  3012. backend restart no longer loses per-print energy data).
  3013. """
  3014. try:
  3015. logger.info("[ENERGY-BG] Starting energy calculation for archive %s", archive_id)
  3016. async with async_session() as db:
  3017. from backend.app.models.archive import PrintArchive
  3018. archive = await db.get(PrintArchive, archive_id)
  3019. if archive is None:
  3020. logger.warning("[ENERGY-BG] Archive %s no longer exists", archive_id)
  3021. return
  3022. starting_kwh = archive.energy_start_kwh
  3023. if starting_kwh is None:
  3024. logger.info("[ENERGY-BG] No start kWh recorded for archive %s", archive_id)
  3025. return
  3026. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  3027. plug = plug_result.scalar_one_or_none()
  3028. if plug is None:
  3029. logger.info("[ENERGY-BG] No smart plug for printer %s", printer_id)
  3030. return
  3031. energy = await _get_plug_energy(plug, db)
  3032. logger.info("[ENERGY-BG] Energy response: %s", energy)
  3033. if not energy or energy.get("total") is None:
  3034. logger.warning("[ENERGY-BG] No 'total' in energy response")
  3035. return
  3036. energy_used = round(energy["total"] - starting_kwh, 4)
  3037. logger.info("[ENERGY-BG] Per-print energy: %s kWh", energy_used)
  3038. if energy_used < 0:
  3039. logger.warning(
  3040. "[ENERGY-BG] Negative energy delta for archive %s (start=%s, end=%s) — counter reset?",
  3041. archive_id,
  3042. starting_kwh,
  3043. energy["total"],
  3044. )
  3045. return
  3046. from backend.app.api.routes.settings import get_setting
  3047. energy_cost_per_kwh = await get_setting(db, "energy_cost_per_kwh")
  3048. cost_per_kwh = float(energy_cost_per_kwh) if energy_cost_per_kwh else 0.15
  3049. archive.energy_kwh = energy_used
  3050. archive.energy_cost = round(energy_used * cost_per_kwh, 3)
  3051. await db.commit()
  3052. logger.info("[ENERGY-BG] Saved: %s kWh, cost=%s", energy_used, archive.energy_cost)
  3053. except Exception as e:
  3054. logger.warning("[ENERGY-BG] Failed: %s", e)
  3055. async def _background_finish_photo() -> str | None:
  3056. """Capture finish photo in background. Returns photo filename if captured."""
  3057. try:
  3058. logger.info("[PHOTO-BG] Starting finish photo capture for archive %s", archive_id)
  3059. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  3060. async with async_session() as db:
  3061. from backend.app.api.routes.settings import get_setting
  3062. capture_enabled = await get_setting(db, "capture_finish_photo")
  3063. if capture_enabled is None or capture_enabled.lower() == "true":
  3064. from backend.app.models.printer import Printer
  3065. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3066. printer = result.scalar_one_or_none()
  3067. if printer and archive_id:
  3068. from backend.app.models.archive import PrintArchive
  3069. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  3070. archive = result.scalar_one_or_none()
  3071. if archive:
  3072. import uuid
  3073. from datetime import datetime
  3074. from pathlib import Path
  3075. if archive.file_path:
  3076. archive_dir = app_settings.base_dir / Path(archive.file_path).parent
  3077. else:
  3078. logger.warning("[PHOTO-BG] Archive %s has no file_path, using fallback dir", archive_id)
  3079. archive_dir = app_settings.archive_dir / str(archive.id)
  3080. photo_filename = None
  3081. # Check for external camera first
  3082. if printer.external_camera_enabled and printer.external_camera_url:
  3083. logger.info("[PHOTO-BG] Using external camera")
  3084. from backend.app.services.external_camera import capture_frame
  3085. frame_data = await capture_frame(
  3086. printer.external_camera_url,
  3087. printer.external_camera_type or "mjpeg",
  3088. snapshot_url=printer.external_camera_snapshot_url,
  3089. )
  3090. if frame_data:
  3091. photos_dir = archive_dir / "photos"
  3092. photos_dir.mkdir(parents=True, exist_ok=True)
  3093. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  3094. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  3095. photo_path = photos_dir / photo_filename
  3096. await asyncio.to_thread(photo_path.write_bytes, frame_data)
  3097. logger.info("[PHOTO-BG] Saved external camera frame: %s", photo_filename)
  3098. else:
  3099. # Check if camera stream is active - use buffered frame to avoid freeze
  3100. # Check both RTSP streams (_active_streams) and chamber image streams (_active_chamber_streams)
  3101. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  3102. active_chamber_for_printer = [
  3103. k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")
  3104. ]
  3105. buffered_frame = get_buffered_frame(printer_id)
  3106. if (active_for_printer or active_chamber_for_printer) and buffered_frame:
  3107. # Use frame from active stream
  3108. logger.info("[PHOTO-BG] Using buffered frame from active stream")
  3109. photos_dir = archive_dir / "photos"
  3110. photos_dir.mkdir(parents=True, exist_ok=True)
  3111. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  3112. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  3113. photo_path = photos_dir / photo_filename
  3114. await asyncio.to_thread(photo_path.write_bytes, buffered_frame)
  3115. logger.info("[PHOTO-BG] Saved buffered frame: %s", photo_filename)
  3116. else:
  3117. # No active stream - capture new frame
  3118. from backend.app.services.camera import capture_finish_photo
  3119. photo_filename = await capture_finish_photo(
  3120. printer_id=printer_id,
  3121. ip_address=printer.ip_address,
  3122. access_code=printer.access_code,
  3123. model=printer.model,
  3124. archive_dir=archive_dir,
  3125. )
  3126. if photo_filename:
  3127. photos = archive.photos or []
  3128. photos.append(photo_filename)
  3129. archive.photos = photos
  3130. await db.commit()
  3131. logger.info("[PHOTO-BG] Saved: %s", photo_filename)
  3132. return photo_filename
  3133. return None
  3134. except Exception as e:
  3135. logger.warning("[PHOTO-BG] Failed: %s", e)
  3136. return None
  3137. asyncio.create_task(_background_energy_calculation())
  3138. # Photo capture task - result will be used by notifications
  3139. photo_task = asyncio.create_task(_background_finish_photo())
  3140. log_timing("Background tasks scheduled (energy, photo)")
  3141. # Also run smart plug, notifications, and maintenance as background tasks
  3142. print_status = data.get("status", "completed")
  3143. async def _background_smart_plug():
  3144. """Handle smart plug automation in background."""
  3145. try:
  3146. logger.info("[AUTO-OFF-BG] Starting smart plug automation for printer %s", printer_id)
  3147. async with async_session() as db:
  3148. await smart_plug_manager.on_print_complete(printer_id, print_status, db)
  3149. logger.info("[AUTO-OFF-BG] Completed")
  3150. except Exception as e:
  3151. logger.warning("[AUTO-OFF-BG] Failed: %s", e)
  3152. async def _background_notifications(finish_photo_filename: str | None = None):
  3153. """Send print complete notifications in background."""
  3154. try:
  3155. logger.info(
  3156. "[NOTIFY-BG] Starting notifications for printer %s, photo=%s", printer_id, finish_photo_filename
  3157. )
  3158. async with async_session() as db:
  3159. from backend.app.models.archive import PrintArchive
  3160. from backend.app.models.printer import Printer
  3161. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3162. printer = result.scalar_one_or_none()
  3163. printer_name = printer.name if printer else f"Printer {printer_id}"
  3164. archive_data = None
  3165. if archive_id:
  3166. archive_result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  3167. archive = archive_result.scalar_one_or_none()
  3168. if archive:
  3169. # Actual elapsed time from started_at/completed_at when both are
  3170. # populated (every terminal status sets completed_at after #1198).
  3171. # Falls back to None so the notification path can decide whether to
  3172. # render the slicer estimate as a last resort.
  3173. actual_time_seconds = None
  3174. if archive.started_at and archive.completed_at:
  3175. elapsed = (archive.completed_at - archive.started_at).total_seconds()
  3176. if elapsed > 0:
  3177. actual_time_seconds = int(elapsed)
  3178. archive_data = {
  3179. "print_time_seconds": archive.print_time_seconds,
  3180. "actual_time_seconds": actual_time_seconds,
  3181. "actual_filament_grams": archive.filament_used_grams,
  3182. "failure_reason": archive.failure_reason,
  3183. "created_by_id": archive.created_by_id,
  3184. }
  3185. # Scale filament usage for partial prints
  3186. if print_status != "completed" and archive.filament_used_grams:
  3187. progress = data.get("progress") or 0
  3188. scale = max(0.0, min(progress / 100.0, 1.0))
  3189. archive_data["actual_filament_grams"] = round(archive.filament_used_grams * scale, 1)
  3190. archive_data["progress"] = progress
  3191. # Pass per-slot data from archive.extra_data
  3192. if archive.extra_data and archive.extra_data.get("filament_slots"):
  3193. slots = archive.extra_data["filament_slots"]
  3194. if print_status != "completed":
  3195. scale = max(0.0, min((data.get("progress") or 0) / 100.0, 1.0))
  3196. slots = [{**s, "used_g": round(s["used_g"] * scale, 1)} for s in slots]
  3197. archive_data["filament_slots"] = slots
  3198. # Enrich filament_grams from usage_results when archive has no 3MF data
  3199. if not archive_data.get("actual_filament_grams") and usage_results:
  3200. total_from_usage = sum(r.get("weight_used", 0) for r in usage_results)
  3201. if total_from_usage > 0:
  3202. archive_data["actual_filament_grams"] = round(total_from_usage, 1)
  3203. # Pass usage tracker results for AMS slot info in notifications
  3204. if usage_results:
  3205. archive_data["usage_results"] = usage_results
  3206. # Add finish photo URL and image bytes if available
  3207. if finish_photo_filename:
  3208. from backend.app.api.routes.settings import get_setting
  3209. external_url = await get_setting(db, "external_url")
  3210. if external_url:
  3211. external_url = external_url.rstrip("/")
  3212. archive_data["finish_photo_url"] = (
  3213. f"{external_url}/api/v1/archives/{archive_id}/photos/{finish_photo_filename}"
  3214. )
  3215. else:
  3216. # Fallback to relative URL (won't work for external services)
  3217. archive_data["finish_photo_url"] = (
  3218. f"/api/v1/archives/{archive_id}/photos/{finish_photo_filename}"
  3219. )
  3220. # Read finish photo bytes for image attachment (e.g. Pushover)
  3221. try:
  3222. from pathlib import Path
  3223. photo_path = (
  3224. app_settings.base_dir
  3225. / Path(archive.file_path).parent
  3226. / "photos"
  3227. / finish_photo_filename
  3228. )
  3229. if photo_path.exists():
  3230. photo_bytes = await asyncio.to_thread(photo_path.read_bytes)
  3231. if len(photo_bytes) <= 2_500_000:
  3232. archive_data["image_data"] = photo_bytes
  3233. logger.info("[NOTIFY-BG] Loaded finish photo bytes: %s bytes", len(photo_bytes))
  3234. else:
  3235. logger.warning(
  3236. f"[NOTIFY-BG] Finish photo too large for attachment: "
  3237. f"{len(photo_bytes)} bytes"
  3238. )
  3239. except Exception as e:
  3240. logger.warning("[NOTIFY-BG] Failed to read finish photo bytes: %s", e)
  3241. await notification_service.on_print_complete(
  3242. printer_id, printer_name, print_status, data, db, archive_data=archive_data
  3243. )
  3244. # Send user-specific email notification
  3245. if archive_data:
  3246. created_by_id = archive_data.get("created_by_id")
  3247. raw_filename = data.get("subtask_name") or data.get("filename", "Unknown")
  3248. await _dispatch_user_print_email(
  3249. print_status,
  3250. created_by_id,
  3251. printer_name,
  3252. raw_filename,
  3253. db,
  3254. )
  3255. logger.info("[NOTIFY-BG] Completed")
  3256. except Exception as e:
  3257. logger.error("[NOTIFY-BG] Failed: %s", e, exc_info=True)
  3258. async def _background_maintenance_check():
  3259. """Check for maintenance due in background."""
  3260. if print_status != "completed":
  3261. return
  3262. try:
  3263. logger.info("[MAINT-BG] Starting maintenance check for printer %s", printer_id)
  3264. async with async_session() as db:
  3265. from backend.app.models.printer import Printer
  3266. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3267. printer = result.scalar_one_or_none()
  3268. printer_name = printer.name if printer else f"Printer {printer_id}"
  3269. await ensure_default_types(db)
  3270. overview = await _get_printer_maintenance_internal(printer_id, db, commit=True)
  3271. items_needing_attention = [
  3272. {"name": item.maintenance_type_name, "is_due": item.is_due, "is_warning": item.is_warning}
  3273. for item in overview.maintenance_items
  3274. if item.enabled and (item.is_due or item.is_warning)
  3275. ]
  3276. if items_needing_attention:
  3277. await notification_service.on_maintenance_due(printer_id, printer_name, items_needing_attention, db)
  3278. logger.info("[MAINT-BG] Sent notification: %s items need attention", len(items_needing_attention))
  3279. # MQTT relay - publish maintenance alerts
  3280. for item in items_needing_attention:
  3281. try:
  3282. await mqtt_relay.on_maintenance_alert(
  3283. printer_id=printer_id,
  3284. printer_name=printer_name,
  3285. maintenance_type=item["name"],
  3286. current_value=0, # Not easily available here
  3287. threshold=0, # Not easily available here
  3288. )
  3289. except Exception:
  3290. pass # Don't fail if MQTT fails
  3291. else:
  3292. logger.info("[MAINT-BG] Completed (no items need attention)")
  3293. except Exception as e:
  3294. logger.warning("[MAINT-BG] Failed: %s", e)
  3295. asyncio.create_task(_background_smart_plug())
  3296. asyncio.create_task(_background_maintenance_check())
  3297. # Notification task waits for photo capture to complete first (with timeout)
  3298. async def _photo_then_notify():
  3299. """Wait for photo capture, then send notification with photo URL."""
  3300. finish_photo = None
  3301. try:
  3302. finish_photo = await asyncio.wait_for(photo_task, timeout=45)
  3303. logger.info("[PHOTO-NOTIFY] Photo task returned: %s", finish_photo)
  3304. except TimeoutError:
  3305. logger.warning("[PHOTO-NOTIFY] Photo capture timed out after 45s, sending notification without photo")
  3306. except Exception as e:
  3307. logger.warning("[PHOTO-NOTIFY] Photo task failed: %s", e)
  3308. try:
  3309. await _background_notifications(finish_photo)
  3310. except Exception as e:
  3311. logger.error("[PHOTO-NOTIFY] Notification sending failed: %s", e, exc_info=True)
  3312. asyncio.create_task(_photo_then_notify())
  3313. # Stitch external camera layer timelapse if session was active
  3314. print_status = data.get("status", "completed")
  3315. async def _background_layer_timelapse():
  3316. """Stitch layer timelapse and attach to archive."""
  3317. from backend.app.services.layer_timelapse import cancel_session, on_print_complete as tl_complete
  3318. try:
  3319. if print_status == "completed":
  3320. logger.info("[LAYER-TL] Stitching layer timelapse for printer %s", printer_id)
  3321. timelapse_path = await tl_complete(printer_id)
  3322. if timelapse_path and archive_id:
  3323. logger.info("[LAYER-TL] Attaching timelapse %s to archive %s", timelapse_path, archive_id)
  3324. async with async_session() as db:
  3325. service = ArchiveService(db)
  3326. timelapse_data = await asyncio.to_thread(timelapse_path.read_bytes)
  3327. await service.attach_timelapse(archive_id, timelapse_data, "layer_timelapse.mp4")
  3328. # Clean up the temp file
  3329. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  3330. logger.info("[LAYER-TL] Layer timelapse attached successfully")
  3331. elif timelapse_path:
  3332. # Timelapse created but no archive - just clean up
  3333. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  3334. else:
  3335. # Print failed or cancelled - cancel timelapse session
  3336. cancel_session(printer_id)
  3337. logger.info(
  3338. "[LAYER-TL] Cancelled layer timelapse for printer %s (status: %s)", printer_id, print_status
  3339. )
  3340. except Exception as e:
  3341. logger.warning("[LAYER-TL] Failed: %s", e)
  3342. # Try to cancel session on error
  3343. try:
  3344. cancel_session(printer_id)
  3345. except Exception:
  3346. pass # Best-effort timelapse session cancellation on error
  3347. asyncio.create_task(_background_layer_timelapse())
  3348. log_timing("All background tasks scheduled")
  3349. # Auto-scan for timelapse if recording was active during the print
  3350. if archive_id and data.get("timelapse_was_active") and data.get("status") == "completed":
  3351. logger.info("[TIMELAPSE] Timelapse was active during print, scheduling auto-scan for archive %s", archive_id)
  3352. # Schedule timelapse scan as background task with retries
  3353. # The printer needs time to encode the video after print completion
  3354. baseline = _timelapse_baselines.pop(printer_id, None)
  3355. asyncio.create_task(_scan_for_timelapse_with_retries(archive_id, baseline))
  3356. log_timing("Timelapse scan scheduled")
  3357. logger.info("[CALLBACK] on_print_complete finished for printer %s, archive %s", printer_id, archive_id)
  3358. # AMS sensor history recording
  3359. _ams_history_task: asyncio.Task | None = None
  3360. AMS_HISTORY_INTERVAL = 300 # Record every 5 minutes
  3361. AMS_HISTORY_RETENTION_DAYS = 30 # Keep data for 30 days
  3362. _ams_cleanup_counter = 0 # Track recordings to trigger periodic cleanup
  3363. # Track alarm cooldowns (printer_id:ams_id:type -> last_alarm_time)
  3364. _ams_alarm_cooldown: dict[str, datetime] = {}
  3365. AMS_ALARM_COOLDOWN_MINUTES = 60 # Don't send same alarm more than once per hour
  3366. async def record_ams_history():
  3367. """Background task to record AMS humidity and temperature data."""
  3368. logger = logging.getLogger(__name__)
  3369. # Wait a short time for MQTT connections to establish on startup
  3370. await asyncio.sleep(10)
  3371. while True:
  3372. try:
  3373. from backend.app.models.ams_history import AMSSensorHistory
  3374. from backend.app.models.printer import Printer
  3375. from backend.app.models.settings import Settings
  3376. async with async_session() as db:
  3377. # Get all active printers
  3378. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  3379. printers = result.scalars().all()
  3380. # Get alarm thresholds from settings
  3381. humidity_threshold = 60.0 # Default: fair threshold
  3382. temp_threshold = 35.0 # Default: fair threshold
  3383. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  3384. setting = result.scalar_one_or_none()
  3385. if setting:
  3386. try:
  3387. humidity_threshold = float(setting.value)
  3388. except (ValueError, TypeError):
  3389. pass # Keep default threshold if stored value is invalid
  3390. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_fair"))
  3391. setting = result.scalar_one_or_none()
  3392. if setting:
  3393. try:
  3394. temp_threshold = float(setting.value)
  3395. except (ValueError, TypeError):
  3396. pass # Keep default threshold if stored value is invalid
  3397. recorded_count = 0
  3398. for printer in printers:
  3399. # Get current state from printer manager
  3400. state = printer_manager.get_status(printer.id)
  3401. if not state or not state.connected or not state.raw_data:
  3402. continue # Skip disconnected printers - don't use stale data
  3403. raw_data = state.raw_data
  3404. if "ams" not in raw_data or not isinstance(raw_data["ams"], list):
  3405. continue
  3406. # Record data for each AMS unit
  3407. for ams_data in raw_data["ams"]:
  3408. ams_id = int(ams_data.get("id", 0))
  3409. # Get humidity (prefer humidity_raw)
  3410. humidity_raw = ams_data.get("humidity_raw")
  3411. humidity_idx = ams_data.get("humidity")
  3412. humidity = None
  3413. if humidity_raw is not None:
  3414. try:
  3415. humidity = float(humidity_raw)
  3416. except (ValueError, TypeError):
  3417. pass # Skip unparseable humidity; will try fallback
  3418. if humidity is None and humidity_idx is not None:
  3419. try:
  3420. humidity = float(humidity_idx)
  3421. except (ValueError, TypeError):
  3422. pass # Skip unparseable humidity index value
  3423. # Get temperature
  3424. temperature = None
  3425. temp_str = ams_data.get("temp")
  3426. if temp_str is not None:
  3427. try:
  3428. temperature = float(temp_str)
  3429. except (ValueError, TypeError):
  3430. pass # Skip unparseable temperature value
  3431. # Skip if no data
  3432. if humidity is None and temperature is None:
  3433. continue
  3434. # Record the data point
  3435. history = AMSSensorHistory(
  3436. printer_id=printer.id,
  3437. ams_id=ams_id,
  3438. humidity=humidity,
  3439. humidity_raw=float(humidity_raw) if humidity_raw else None,
  3440. temperature=temperature,
  3441. )
  3442. db.add(history)
  3443. recorded_count += 1
  3444. # Generate AMS label and determine if it's AMS-HT (A, B, C, D or HT-A for AMS-Lite/Hub)
  3445. is_ams_ht = ams_id >= 128
  3446. if is_ams_ht:
  3447. ams_label = f"HT-{chr(65 + (ams_id - 128))}"
  3448. else:
  3449. ams_label = f"AMS-{chr(65 + ams_id)}"
  3450. # Check humidity alarm (only if above threshold)
  3451. if humidity is not None and humidity > humidity_threshold:
  3452. cooldown_key = f"{printer.id}:{ams_id}:humidity"
  3453. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  3454. now = datetime.now(timezone.utc)
  3455. if (
  3456. last_alarm is None
  3457. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  3458. ):
  3459. _ams_alarm_cooldown[cooldown_key] = now
  3460. logger.info(
  3461. f"Sending humidity alarm for {printer.name} {ams_label}: {humidity}% > {humidity_threshold}%"
  3462. )
  3463. try:
  3464. # Call different notification method based on AMS type
  3465. if is_ams_ht:
  3466. await notification_service.on_ams_ht_humidity_high(
  3467. printer.id, printer.name, ams_label, humidity, humidity_threshold, db
  3468. )
  3469. else:
  3470. await notification_service.on_ams_humidity_high(
  3471. printer.id, printer.name, ams_label, humidity, humidity_threshold, db
  3472. )
  3473. except Exception as e:
  3474. logger.warning("Failed to send humidity alarm: %s", e)
  3475. # Check temperature alarm (only if above threshold)
  3476. if temperature is not None and temperature > temp_threshold:
  3477. cooldown_key = f"{printer.id}:{ams_id}:temperature"
  3478. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  3479. now = datetime.now(timezone.utc)
  3480. if (
  3481. last_alarm is None
  3482. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  3483. ):
  3484. _ams_alarm_cooldown[cooldown_key] = now
  3485. logger.info(
  3486. f"Sending temperature alarm for {printer.name} {ams_label}: {temperature}°C > {temp_threshold}°C"
  3487. )
  3488. try:
  3489. # Call different notification method based on AMS type
  3490. if is_ams_ht:
  3491. await notification_service.on_ams_ht_temperature_high(
  3492. printer.id, printer.name, ams_label, temperature, temp_threshold, db
  3493. )
  3494. else:
  3495. await notification_service.on_ams_temperature_high(
  3496. printer.id, printer.name, ams_label, temperature, temp_threshold, db
  3497. )
  3498. except Exception as e:
  3499. logger.warning("Failed to send temperature alarm: %s", e)
  3500. await db.commit()
  3501. if recorded_count > 0:
  3502. logger.info("Recorded %s AMS sensor history entries", recorded_count)
  3503. # Periodic cleanup of old data (every ~288 recordings = ~24 hours at 5min interval)
  3504. global _ams_cleanup_counter
  3505. _ams_cleanup_counter += 1
  3506. if _ams_cleanup_counter >= 288:
  3507. _ams_cleanup_counter = 0
  3508. # Get retention days from settings
  3509. from backend.app.models.settings import Settings
  3510. result = await db.execute(select(Settings).where(Settings.key == "ams_history_retention_days"))
  3511. setting = result.scalar_one_or_none()
  3512. retention_days = int(setting.value) if setting else AMS_HISTORY_RETENTION_DAYS
  3513. cutoff = datetime.utcnow() - timedelta(days=retention_days)
  3514. result = await db.execute(delete(AMSSensorHistory).where(AMSSensorHistory.recorded_at < cutoff))
  3515. await db.commit()
  3516. if result.rowcount > 0:
  3517. logger.info(
  3518. f"Cleaned up {result.rowcount} old AMS sensor history entries (older than {retention_days} days)"
  3519. )
  3520. # Wait until next recording interval
  3521. await asyncio.sleep(AMS_HISTORY_INTERVAL)
  3522. except asyncio.CancelledError:
  3523. break
  3524. except Exception as e:
  3525. logger.warning("AMS history recording failed: %s", e)
  3526. await asyncio.sleep(60) # Wait a bit before retrying
  3527. def start_ams_history_recording():
  3528. """Start the AMS history recording background task."""
  3529. global _ams_history_task
  3530. if _ams_history_task is None:
  3531. _ams_history_task = asyncio.create_task(record_ams_history())
  3532. logging.getLogger(__name__).info("AMS history recording started")
  3533. def stop_ams_history_recording():
  3534. """Stop the AMS history recording background task."""
  3535. global _ams_history_task
  3536. if _ams_history_task:
  3537. _ams_history_task.cancel()
  3538. _ams_history_task = None
  3539. logging.getLogger(__name__).info("AMS history recording stopped")
  3540. # Printer runtime tracking
  3541. _runtime_tracking_task: asyncio.Task | None = None
  3542. RUNTIME_TRACKING_INTERVAL = 30 # Update every 30 seconds
  3543. async def track_printer_runtime():
  3544. """Background task to track printer active runtime (RUNNING/PAUSE states)."""
  3545. logger = logging.getLogger(__name__)
  3546. # Wait for MQTT connections to establish on startup
  3547. await asyncio.sleep(15)
  3548. while True:
  3549. try:
  3550. from backend.app.models.printer import Printer
  3551. # Fetch printer IDs in a short-lived read-only session
  3552. async with async_session() as db:
  3553. result = await db.execute(
  3554. select(Printer.id, Printer.name, Printer.runtime_seconds, Printer.last_runtime_update).where(
  3555. Printer.is_active.is_(True)
  3556. )
  3557. )
  3558. printer_rows = result.all()
  3559. now = datetime.now(timezone.utc)
  3560. updated_count = 0
  3561. # Update each printer in its own short session to minimise write-lock
  3562. # hold time and avoid blocking critical commits like queue status
  3563. # updates (#897).
  3564. for pid, pname, runtime_secs, last_update in printer_rows:
  3565. state = printer_manager.get_status(pid)
  3566. if not state:
  3567. logger.debug("[%s] Runtime tracking: no state available", pname)
  3568. continue
  3569. if not state.connected:
  3570. logger.debug("[%s] Runtime tracking: not connected", pname)
  3571. continue
  3572. needs_commit = False
  3573. new_runtime = runtime_secs
  3574. new_last_update = last_update
  3575. if state.state in ("RUNNING", "PAUSE"):
  3576. if last_update:
  3577. lu = last_update if last_update.tzinfo else last_update.replace(tzinfo=timezone.utc)
  3578. elapsed = (now - lu).total_seconds()
  3579. if elapsed > 0:
  3580. new_runtime = runtime_secs + int(elapsed)
  3581. updated_count += 1
  3582. needs_commit = True
  3583. logger.debug(
  3584. f"[{pname}] Runtime tracking: added {int(elapsed)}s, "
  3585. f"total={new_runtime}s ({new_runtime / 3600:.2f}h)"
  3586. )
  3587. else:
  3588. needs_commit = True
  3589. logger.debug("[%s] Runtime tracking: first active detection", pname)
  3590. new_last_update = now
  3591. else:
  3592. if last_update is not None:
  3593. logger.debug(f"[{pname}] Runtime tracking: state={state.state}, clearing last_runtime_update")
  3594. new_last_update = None
  3595. needs_commit = True
  3596. if needs_commit:
  3597. try:
  3598. async with async_session() as db:
  3599. result = await db.execute(select(Printer).where(Printer.id == pid))
  3600. printer = result.scalar_one_or_none()
  3601. if printer:
  3602. printer.runtime_seconds = new_runtime
  3603. printer.last_runtime_update = new_last_update
  3604. await db.commit()
  3605. except Exception as e:
  3606. logger.warning("[%s] Runtime tracking commit failed: %s", pname, e)
  3607. if updated_count > 0:
  3608. logger.debug("Updated runtime for %s printer(s)", updated_count)
  3609. except asyncio.CancelledError:
  3610. logger.info("Runtime tracking cancelled")
  3611. break
  3612. except Exception as e:
  3613. logger.warning("Runtime tracking failed: %s", e)
  3614. await asyncio.sleep(RUNTIME_TRACKING_INTERVAL)
  3615. def start_runtime_tracking():
  3616. """Start the printer runtime tracking background task."""
  3617. global _runtime_tracking_task
  3618. if _runtime_tracking_task is None:
  3619. _runtime_tracking_task = asyncio.create_task(track_printer_runtime())
  3620. logging.getLogger(__name__).info("Printer runtime tracking started")
  3621. def stop_runtime_tracking():
  3622. """Stop the printer runtime tracking background task."""
  3623. global _runtime_tracking_task
  3624. if _runtime_tracking_task:
  3625. _runtime_tracking_task.cancel()
  3626. _runtime_tracking_task = None
  3627. logging.getLogger(__name__).info("Printer runtime tracking stopped")
  3628. # SpoolBuddy device watchdog
  3629. _spoolbuddy_watchdog_task: asyncio.Task | None = None
  3630. SPOOLBUDDY_WATCHDOG_INTERVAL = 15
  3631. async def _spoolbuddy_watchdog_loop():
  3632. """Periodic check for SpoolBuddy devices that have gone offline."""
  3633. from backend.app.api.routes.spoolbuddy import spoolbuddy_watchdog
  3634. while True:
  3635. try:
  3636. await spoolbuddy_watchdog()
  3637. except asyncio.CancelledError:
  3638. break
  3639. except Exception as e:
  3640. logging.getLogger(__name__).warning("SpoolBuddy watchdog failed: %s", e)
  3641. await asyncio.sleep(SPOOLBUDDY_WATCHDOG_INTERVAL)
  3642. def start_spoolbuddy_watchdog():
  3643. global _spoolbuddy_watchdog_task
  3644. if _spoolbuddy_watchdog_task is None:
  3645. _spoolbuddy_watchdog_task = asyncio.create_task(_spoolbuddy_watchdog_loop())
  3646. logging.getLogger(__name__).info("SpoolBuddy watchdog started")
  3647. def stop_spoolbuddy_watchdog():
  3648. global _spoolbuddy_watchdog_task
  3649. if _spoolbuddy_watchdog_task:
  3650. _spoolbuddy_watchdog_task.cancel()
  3651. _spoolbuddy_watchdog_task = None
  3652. logging.getLogger(__name__).info("SpoolBuddy watchdog stopped")
  3653. # Camera stream orphan cleanup
  3654. _camera_cleanup_task: asyncio.Task | None = None
  3655. CAMERA_CLEANUP_INTERVAL = 60
  3656. async def _camera_cleanup_loop():
  3657. """Periodically clean up orphaned ffmpeg processes."""
  3658. from backend.app.api.routes.camera import cleanup_orphaned_streams
  3659. while True:
  3660. try:
  3661. await cleanup_orphaned_streams()
  3662. except asyncio.CancelledError:
  3663. break
  3664. except Exception as e:
  3665. logging.getLogger(__name__).warning("Camera stream cleanup failed: %s", e)
  3666. await asyncio.sleep(CAMERA_CLEANUP_INTERVAL)
  3667. def start_camera_cleanup():
  3668. global _camera_cleanup_task
  3669. if _camera_cleanup_task is None:
  3670. _camera_cleanup_task = asyncio.create_task(_camera_cleanup_loop())
  3671. logging.getLogger(__name__).info("Camera stream cleanup started")
  3672. def stop_camera_cleanup():
  3673. global _camera_cleanup_task
  3674. if _camera_cleanup_task:
  3675. _camera_cleanup_task.cancel()
  3676. _camera_cleanup_task = None
  3677. logging.getLogger(__name__).info("Camera stream cleanup stopped")
  3678. # ---------------------------------------------------------------------------
  3679. # Expected-print TTL eviction
  3680. # ---------------------------------------------------------------------------
  3681. def _evict_stale_expected_prints() -> None:
  3682. """Remove entries from _expected_prints / _expected_print_creators that are
  3683. older than _EXPECTED_PRINT_TTL_SECONDS.
  3684. This prevents unbounded growth when a print is registered (via
  3685. register_expected_print) but on_print_start never fires — e.g. because the
  3686. printer disconnects, the app restarts, or the print is started directly from
  3687. the printer panel without going through the queue.
  3688. """
  3689. # Use monotonic time so the TTL is unaffected by system clock adjustments
  3690. # (e.g. NTP sync, DST changes).
  3691. cutoff = time.monotonic() - _EXPECTED_PRINT_TTL_SECONDS
  3692. stale_keys = [k for k, t in _expected_print_registered_at.items() if t < cutoff]
  3693. if not stale_keys:
  3694. return
  3695. evicted_archive_ids: set[int] = set()
  3696. for key in stale_keys:
  3697. archive_id = _expected_prints.pop(key, None)
  3698. if archive_id is not None:
  3699. evicted_archive_ids.add(archive_id)
  3700. _expected_print_creators.pop(key, None)
  3701. _expected_print_registered_at.pop(key, None)
  3702. # Also clean up _print_ams_mappings for archive_ids that have no remaining
  3703. # live keys in _expected_prints (i.e. all variants were just evicted).
  3704. live_archive_ids = set(_expected_prints.values())
  3705. for archive_id in evicted_archive_ids:
  3706. if archive_id not in live_archive_ids:
  3707. _print_ams_mappings.pop(archive_id, None)
  3708. logging.getLogger(__name__).info(
  3709. "Evicted %d stale expected-print entries (TTL=%ds)", len(stale_keys), _EXPECTED_PRINT_TTL_SECONDS
  3710. )
  3711. async def _expected_prints_cleanup_loop() -> None:
  3712. """Background task: periodically evict stale expected-print entries."""
  3713. while True:
  3714. try:
  3715. _evict_stale_expected_prints()
  3716. except asyncio.CancelledError:
  3717. raise
  3718. except Exception as e:
  3719. logging.getLogger(__name__).warning("Expected prints cleanup failed: %s", e)
  3720. await asyncio.sleep(_EXPECTED_PRINT_CLEANUP_INTERVAL)
  3721. def start_expected_prints_cleanup() -> None:
  3722. global _expected_prints_cleanup_task
  3723. if _expected_prints_cleanup_task is None:
  3724. _expected_prints_cleanup_task = asyncio.create_task(_expected_prints_cleanup_loop())
  3725. logging.getLogger(__name__).info("Expected prints cleanup started")
  3726. def stop_expected_prints_cleanup() -> None:
  3727. global _expected_prints_cleanup_task
  3728. if _expected_prints_cleanup_task:
  3729. _expected_prints_cleanup_task.cancel()
  3730. _expected_prints_cleanup_task = None
  3731. logging.getLogger(__name__).info("Expected prints cleanup stopped")
  3732. # ---------------------------------------------------------------------------
  3733. # L-2: Periodic auth-token cleanup (stale TOTP + expired revoked JTIs)
  3734. # ---------------------------------------------------------------------------
  3735. _auth_cleanup_task: asyncio.Task | None = None
  3736. _AUTH_CLEANUP_INTERVAL = 3600 # seconds (hourly)
  3737. async def _run_auth_cleanup() -> None:
  3738. """Single cleanup pass: remove stale TOTP records, expired revoked JTIs, and old rate-limit events."""
  3739. from backend.app.core.database import async_session
  3740. from backend.app.models.auth_ephemeral import AuthEphemeralToken, AuthRateLimitEvent
  3741. from backend.app.models.user_totp import UserTOTP
  3742. now = datetime.now(timezone.utc)
  3743. # Remove unconfirmed (is_enabled=False) TOTP records older than 1 hour.
  3744. try:
  3745. async with async_session() as db:
  3746. stale_cutoff = now - timedelta(hours=1)
  3747. result = await db.execute(
  3748. select(UserTOTP).where(
  3749. UserTOTP.is_enabled.is_(False),
  3750. UserTOTP.created_at < stale_cutoff,
  3751. )
  3752. )
  3753. stale_records = result.scalars().all()
  3754. if stale_records:
  3755. for rec in stale_records:
  3756. await db.delete(rec)
  3757. await db.commit()
  3758. logging.info("Auth cleanup: removed %d stale unconfirmed TOTP record(s)", len(stale_records))
  3759. except Exception as e:
  3760. logging.warning("Auth cleanup: failed to purge stale TOTP records: %s", e)
  3761. # Remove expired revoked-JTI entries (they are no longer needed once the
  3762. # original token's exp has passed — the token would be rejected by JWT
  3763. # signature verification regardless).
  3764. try:
  3765. async with async_session() as db:
  3766. await db.execute(
  3767. delete(AuthEphemeralToken).where(
  3768. AuthEphemeralToken.token_type == "revoked_jti",
  3769. AuthEphemeralToken.expires_at < now,
  3770. )
  3771. )
  3772. await db.commit()
  3773. except Exception as e:
  3774. logging.warning("Auth cleanup: failed to purge expired revoked JTIs: %s", e)
  3775. # L-R6-B: Purge AuthRateLimitEvent rows older than the lockout window (15 min).
  3776. # Events outside this window can never affect rate-limit decisions — they only
  3777. # consume DB space. Use the same window constant as the rate limiter so the
  3778. # two are always in sync.
  3779. try:
  3780. from backend.app.api.routes.mfa import LOCKOUT_WINDOW
  3781. async with async_session() as db:
  3782. await db.execute(
  3783. delete(AuthRateLimitEvent).where(
  3784. AuthRateLimitEvent.occurred_at < now - LOCKOUT_WINDOW,
  3785. )
  3786. )
  3787. await db.commit()
  3788. except Exception as e:
  3789. logging.warning("Auth cleanup: failed to purge stale rate-limit events: %s", e)
  3790. async def _auth_cleanup_loop() -> None:
  3791. """Periodic background task: run auth cleanup every hour."""
  3792. while True:
  3793. try:
  3794. await _run_auth_cleanup()
  3795. except asyncio.CancelledError:
  3796. break
  3797. except Exception as e:
  3798. logging.warning("Auth cleanup loop error: %s", e)
  3799. await asyncio.sleep(_AUTH_CLEANUP_INTERVAL)
  3800. def start_auth_cleanup() -> None:
  3801. global _auth_cleanup_task
  3802. if _auth_cleanup_task is None:
  3803. _auth_cleanup_task = asyncio.create_task(_auth_cleanup_loop())
  3804. logging.getLogger(__name__).info("Auth periodic cleanup started")
  3805. def stop_auth_cleanup() -> None:
  3806. global _auth_cleanup_task
  3807. if _auth_cleanup_task:
  3808. _auth_cleanup_task.cancel()
  3809. _auth_cleanup_task = None
  3810. logging.getLogger(__name__).info("Auth periodic cleanup stopped")
  3811. @asynccontextmanager
  3812. async def lifespan(app: FastAPI):
  3813. # Startup
  3814. # Install Windows-only asyncio Proactor cleanup-RST filter (#1113) before
  3815. # anything else can spawn tasks that might trip it.
  3816. from backend.app.core.asyncio_handlers import install_proactor_reset_filter
  3817. install_proactor_reset_filter()
  3818. await init_db()
  3819. # Register an app-scoped httpx client for Bambu Cloud services so
  3820. # per-request BambuCloudService instances reuse the same connection pool
  3821. # (important for routes like /cloud/filament-info that chain many
  3822. # get_setting_detail calls). The shared client stores no region/token
  3823. # state, so the per-request ownership pattern that fixed the region-bleed
  3824. # bug is preserved.
  3825. import httpx as _httpx
  3826. from backend.app.services.bambu_cloud import set_shared_http_client
  3827. from backend.app.services.makerworld import (
  3828. set_shared_http_client as set_shared_makerworld_http_client,
  3829. )
  3830. _shared_cloud_http_client = _httpx.AsyncClient(timeout=30.0)
  3831. set_shared_http_client(_shared_cloud_http_client)
  3832. # Reuse the same connection pool for MakerWorld — different host, same
  3833. # keep-alive pool saves a TLS handshake per request.
  3834. set_shared_makerworld_http_client(_shared_cloud_http_client)
  3835. # Fix queue items stuck with invalid "aborted" status (should be "cancelled").
  3836. # This can happen when a print was cancelled mid-print on versions before this fix.
  3837. try:
  3838. async with async_session() as db:
  3839. from backend.app.models.print_queue import PrintQueueItem
  3840. result = await db.execute(select(PrintQueueItem).where(PrintQueueItem.status == "aborted"))
  3841. aborted_items = result.scalars().all()
  3842. if aborted_items:
  3843. for item in aborted_items:
  3844. item.status = "cancelled"
  3845. await db.commit()
  3846. logging.info("Fixed %d queue item(s) with invalid 'aborted' status → 'cancelled'", len(aborted_items))
  3847. except Exception as e:
  3848. logging.warning("Failed to fix aborted queue items: %s", e)
  3849. # Restore debug logging state from previous session
  3850. await init_debug_logging()
  3851. # Set up printer manager callbacks
  3852. loop = asyncio.get_event_loop()
  3853. printer_manager.set_event_loop(loop)
  3854. printer_manager.set_status_change_callback(on_printer_status_change)
  3855. printer_manager.set_print_start_callback(on_print_start)
  3856. printer_manager.set_print_complete_callback(on_print_complete)
  3857. printer_manager.set_ams_change_callback(on_ams_change)
  3858. # Rehydrate persisted awaiting-plate-clear gate (#961) so prompts survive restarts
  3859. await printer_manager.load_awaiting_plate_clear_from_db()
  3860. # Layer change callback for external camera timelapse
  3861. async def on_layer_change(printer_id: int, layer_num: int):
  3862. """Capture timelapse frame on layer change + first layer notification."""
  3863. from backend.app.services.layer_timelapse import on_layer_change as tl_layer_change
  3864. await tl_layer_change(printer_id, layer_num)
  3865. # First layer complete notification (layer_num >= 2 means layer 1 is done)
  3866. if 2 <= layer_num <= 5 and not _first_layer_notified.get(printer_id, False):
  3867. _first_layer_notified[printer_id] = True
  3868. try:
  3869. async with async_session() as db:
  3870. from backend.app.models.printer import Printer
  3871. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  3872. printer = result.scalar_one_or_none()
  3873. if not printer:
  3874. return
  3875. printer_name = printer.name
  3876. client = printer_manager.get_client(printer_id)
  3877. state = client.state if client else None
  3878. filename = (state.subtask_name or state.gcode_file or "Unknown") if state else "Unknown"
  3879. total_layers = state.total_layers if state else 0
  3880. image_data = await _capture_snapshot_for_notification(
  3881. printer_id, printer, logging.getLogger(__name__)
  3882. )
  3883. await notification_service.on_first_layer_complete(
  3884. printer_id, printer_name, filename, total_layers, db, image_data=image_data
  3885. )
  3886. except Exception as e:
  3887. logging.getLogger(__name__).warning("First layer notification failed: %s", e)
  3888. printer_manager.set_layer_change_callback(on_layer_change)
  3889. # Event-driven bed cooldown: fires whenever bed_temper arrives via MQTT
  3890. async def on_bed_temp_update(printer_id: int, bed_temp: float):
  3891. waiter = _bed_cool_waiters.get(printer_id)
  3892. if not waiter:
  3893. return
  3894. threshold = waiter["threshold"]
  3895. if bed_temp > threshold:
  3896. return
  3897. # Bed is at or below threshold — fire notification and remove waiter
  3898. waiter_info = _bed_cool_waiters.pop(printer_id, None)
  3899. if not waiter_info:
  3900. return # Another callback already handled it
  3901. bed_cool_logger = logging.getLogger(__name__)
  3902. bed_cool_logger.info(
  3903. "[BED-COOL] Bed cooled to %.1f°C on printer %s (threshold: %.0f°C)",
  3904. bed_temp,
  3905. printer_id,
  3906. threshold,
  3907. )
  3908. try:
  3909. printer_info = printer_manager.get_printer(printer_id)
  3910. p_name = printer_info.name if printer_info else "Unknown"
  3911. async with async_session() as db:
  3912. await notification_service.on_bed_cooled(
  3913. printer_id=printer_id,
  3914. printer_name=p_name,
  3915. bed_temp=bed_temp,
  3916. threshold=threshold,
  3917. filename=waiter_info["filename"],
  3918. db=db,
  3919. )
  3920. except Exception as e:
  3921. bed_cool_logger.warning("[BED-COOL] Failed to send notification: %s", e)
  3922. printer_manager.set_bed_temp_update_callback(on_bed_temp_update)
  3923. # Initialize MQTT relay from settings
  3924. async with async_session() as db:
  3925. from backend.app.api.routes.settings import get_setting
  3926. mqtt_settings = {
  3927. "mqtt_enabled": (await get_setting(db, "mqtt_enabled") or "false") == "true",
  3928. "mqtt_broker": await get_setting(db, "mqtt_broker") or "",
  3929. "mqtt_port": int(await get_setting(db, "mqtt_port") or "1883"),
  3930. "mqtt_username": await get_setting(db, "mqtt_username") or "",
  3931. "mqtt_password": await get_setting(db, "mqtt_password") or "",
  3932. "mqtt_topic_prefix": await get_setting(db, "mqtt_topic_prefix") or "bambuddy",
  3933. "mqtt_use_tls": (await get_setting(db, "mqtt_use_tls") or "false") == "true",
  3934. }
  3935. await mqtt_relay.configure(mqtt_settings)
  3936. # Restore MQTT smart plug subscriptions
  3937. if mqtt_settings.get("mqtt_enabled"):
  3938. from backend.app.models.smart_plug import SmartPlug
  3939. from backend.app.services.mqtt_smart_plug import subscribe_plug_to_mqtt
  3940. result = await db.execute(select(SmartPlug).where(SmartPlug.plug_type == "mqtt"))
  3941. mqtt_plugs = result.scalars().all()
  3942. restored = 0
  3943. for plug in mqtt_plugs:
  3944. if subscribe_plug_to_mqtt(mqtt_relay.smart_plug_service, plug):
  3945. restored += 1
  3946. if restored:
  3947. logging.info("Restored %s MQTT smart plug subscriptions", restored)
  3948. # Connect to all active printers
  3949. async with async_session() as db:
  3950. await init_printer_connections(db)
  3951. # Auto-connect to Spoolman if enabled
  3952. async with async_session() as db:
  3953. from backend.app.api.routes.settings import get_setting
  3954. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  3955. spoolman_url = await get_setting(db, "spoolman_url")
  3956. if spoolman_enabled and spoolman_enabled.lower() == "true" and spoolman_url:
  3957. try:
  3958. client = await init_spoolman_client(spoolman_url)
  3959. if await client.health_check():
  3960. logging.info("Auto-connected to Spoolman at %s", spoolman_url)
  3961. # Ensure the 'tag' extra field exists for RFID/UUID storage
  3962. await client.ensure_tag_extra_field()
  3963. else:
  3964. logging.warning("Spoolman at %s is not reachable", spoolman_url)
  3965. except Exception as e:
  3966. logging.warning("Failed to auto-connect to Spoolman: %s", e)
  3967. # Start the print scheduler
  3968. asyncio.create_task(print_scheduler.run())
  3969. # Start background dispatch worker for send/start operations
  3970. await background_dispatch.start()
  3971. # Start the smart plug scheduler for time-based on/off
  3972. smart_plug_manager.start_scheduler()
  3973. # Resume any pending auto-offs that were interrupted by restart
  3974. await smart_plug_manager.resume_pending_auto_offs()
  3975. # Start the notification digest scheduler
  3976. notification_service.start_digest_scheduler()
  3977. # Start the GitHub backup scheduler
  3978. await github_backup_service.start_scheduler()
  3979. # Start the local backup scheduler
  3980. await local_backup_service.start_scheduler()
  3981. await obico_detection_service.start()
  3982. # Start the library trash sweeper (#1008)
  3983. await library_trash_service.start_scheduler()
  3984. # Start the archive auto-purge sweeper (#1008 follow-up)
  3985. await archive_purge_service.start_scheduler()
  3986. # Start AMS history recording
  3987. start_ams_history_recording()
  3988. # Start printer runtime tracking
  3989. start_runtime_tracking()
  3990. # Start SpoolBuddy device watchdog
  3991. start_spoolbuddy_watchdog()
  3992. # Start camera stream orphan cleanup
  3993. start_camera_cleanup()
  3994. # Start expected-print TTL eviction (prevents memory leak when prints are
  3995. # registered but on_print_start never fires)
  3996. start_expected_prints_cleanup()
  3997. # L-2: Start periodic auth cleanup (stale TOTP + expired revoked JTIs)
  3998. start_auth_cleanup()
  3999. # Initialize virtual printer manager and sync from DB
  4000. from backend.app.services.virtual_printer import virtual_printer_manager
  4001. virtual_printer_manager.set_session_factory(async_session)
  4002. virtual_printer_manager.set_printer_manager(printer_manager)
  4003. try:
  4004. await virtual_printer_manager.sync_from_db()
  4005. logging.info("Virtual printer manager synced from database")
  4006. except Exception as e:
  4007. logging.warning("Failed to sync virtual printers: %s", e)
  4008. yield
  4009. # Shutdown
  4010. print_scheduler.stop()
  4011. await background_dispatch.stop()
  4012. smart_plug_manager.stop_scheduler()
  4013. notification_service.stop_digest_scheduler()
  4014. github_backup_service.stop_scheduler()
  4015. local_backup_service.stop_scheduler()
  4016. library_trash_service.stop_scheduler()
  4017. archive_purge_service.stop_scheduler()
  4018. obico_detection_service.stop()
  4019. stop_ams_history_recording()
  4020. stop_runtime_tracking()
  4021. stop_spoolbuddy_watchdog()
  4022. stop_camera_cleanup()
  4023. # Tear down all camera fan-out broadcasters (#1089) so subscribers exit
  4024. # cleanly rather than waiting on a queue that nothing will ever fill.
  4025. try:
  4026. from backend.app.services.camera_fanout import shutdown_all_broadcasters
  4027. await shutdown_all_broadcasters()
  4028. except Exception as e:
  4029. logging.warning("Failed to shut down camera broadcasters: %s", e)
  4030. stop_expected_prints_cleanup()
  4031. stop_auth_cleanup()
  4032. printer_manager.disconnect_all()
  4033. await close_spoolman_client()
  4034. # Stop all virtual printer services
  4035. await virtual_printer_manager.stop_all()
  4036. await mqtt_smart_plug_service.disconnect(timeout=2)
  4037. await mqtt_relay.disconnect(timeout=2)
  4038. # Drop the shared Bambu Cloud HTTP client we registered at startup.
  4039. set_shared_http_client(None)
  4040. set_shared_makerworld_http_client(None)
  4041. await _shared_cloud_http_client.aclose()
  4042. # Checkpoint WAL (SQLite only) and close all database connections
  4043. from backend.app.core.db_dialect import is_sqlite
  4044. if is_sqlite():
  4045. try:
  4046. async with engine.begin() as conn:
  4047. await conn.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
  4048. logging.info("WAL checkpoint completed")
  4049. except Exception as e:
  4050. logging.warning("WAL checkpoint failed: %s", e)
  4051. await engine.dispose()
  4052. app = FastAPI(
  4053. title=app_settings.app_name,
  4054. description="Archive and manage Bambu Lab 3MF files",
  4055. version=APP_VERSION,
  4056. lifespan=lifespan,
  4057. )
  4058. # =============================================================================
  4059. # Authentication Middleware - Secures ALL API routes by default
  4060. # =============================================================================
  4061. # Public routes that don't require authentication even when auth is enabled
  4062. PUBLIC_API_ROUTES = {
  4063. # Auth routes needed before/during login
  4064. "/api/v1/auth/status",
  4065. "/api/v1/auth/login",
  4066. "/api/v1/auth/setup", # Needed for initial setup and recovery
  4067. # Advanced auth status needed for login page
  4068. "/api/v1/auth/advanced-auth/status",
  4069. "/api/v1/auth/forgot-password", # Password reset for advanced auth
  4070. "/api/v1/auth/forgot-password/confirm", # Complete password reset with token (H-6)
  4071. # 2FA routes that are called BEFORE a JWT is issued (pre-auth flow)
  4072. "/api/v1/auth/2fa/verify", # Exchange pre_auth_token + 2FA code for JWT
  4073. "/api/v1/auth/2fa/email/send", # Send OTP email (pre_auth_token based)
  4074. # OIDC routes that must be reachable without a JWT
  4075. "/api/v1/auth/oidc/providers", # Public list of enabled providers
  4076. "/api/v1/auth/oidc/callback", # Redirect target from OIDC provider
  4077. "/api/v1/auth/oidc/exchange", # Exchange short-lived OIDC token for JWT
  4078. # Version check for updates (no sensitive data)
  4079. "/api/v1/updates/version",
  4080. # Metrics endpoint handles its own prometheus_token authentication
  4081. "/api/v1/metrics",
  4082. }
  4083. # Route prefixes that are public (for routes with dynamic segments)
  4084. PUBLIC_API_PREFIXES = [
  4085. # WebSocket connections handle their own auth
  4086. "/api/v1/ws",
  4087. # OIDC authorize redirects — include provider_id in path
  4088. "/api/v1/auth/oidc/authorize/",
  4089. ]
  4090. # Route patterns that are public (read-only display data)
  4091. # These are checked with "in path" - needed because browsers load images/videos
  4092. # via <img src> and <video src> which don't include Authorization headers
  4093. PUBLIC_API_PATTERNS = [
  4094. # Thumbnails
  4095. "/thumbnail", # /archives/{id}/thumbnail, /library/files/{id}/thumbnail
  4096. "/plate-thumbnail/", # /archives/{id}/plate-thumbnail/{plate_id}
  4097. # Images and media
  4098. "/photos/", # /archives/{id}/photos/{filename}
  4099. "/project-image/", # /archives/{id}/project-image/{path}
  4100. "/qrcode", # /archives/{id}/qrcode
  4101. "/timelapse", # /archives/{id}/timelapse (video)
  4102. "/cover", # /printers/{id}/cover
  4103. "/icon", # /external-links/{id}/icon
  4104. # Camera (streams loaded via <img> tag)
  4105. "/camera/stream", # /printers/{id}/camera/stream
  4106. "/camera/snapshot", # /printers/{id}/camera/snapshot
  4107. # Slicer token-authenticated downloads — protocol handlers (bambustudioopen://,
  4108. # orcaslicer://) cannot send auth headers. These endpoints validate a short-lived
  4109. # download token in the URL path instead.
  4110. "/dl/", # /archives/{id}/dl/{token}/{filename}, /library/files/{id}/dl/{token}/{filename}
  4111. # Obico ML API fetches JPEG frames by one-shot nonce (issue #172 follow-up).
  4112. # The nonce itself is the credential: 32-byte random, single-use, ~30s TTL.
  4113. "/obico/cached-frame/", # /obico/cached-frame/{nonce}
  4114. ]
  4115. _security_headers_logger = logging.getLogger("backend.app.main.security_headers")
  4116. def _parse_trusted_frame_origins() -> tuple[str, ...]:
  4117. """Parse TRUSTED_FRAME_ORIGINS env var into a validated allowlist (#1191).
  4118. Format: comma-separated list of ``scheme://host[:port]`` origins.
  4119. Used by ``security_headers_middleware`` to relax ``frame-ancestors`` for
  4120. trusted same-LAN deployments (e.g. Home Assistant Webpage panel embedding
  4121. Bambuddy from a different port). Defaults to empty — strict ``'none'``.
  4122. Invalid entries are dropped with a warning rather than failing startup, so
  4123. a typo in one origin doesn't take the whole deployment down.
  4124. """
  4125. raw = os.environ.get("TRUSTED_FRAME_ORIGINS", "").strip()
  4126. if not raw:
  4127. return ()
  4128. valid: list[str] = []
  4129. for item in raw.split(","):
  4130. candidate = item.strip()
  4131. if not candidate:
  4132. continue
  4133. try:
  4134. parsed = urlparse(candidate)
  4135. except ValueError as e:
  4136. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — %s", candidate, e)
  4137. continue
  4138. if parsed.scheme not in ("http", "https"):
  4139. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — must be http(s)", candidate)
  4140. continue
  4141. if not parsed.netloc:
  4142. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — missing host", candidate)
  4143. continue
  4144. if parsed.path and parsed.path != "/":
  4145. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — paths not allowed", candidate)
  4146. continue
  4147. if parsed.query or parsed.fragment:
  4148. _security_headers_logger.warning(
  4149. "TRUSTED_FRAME_ORIGINS: dropping %r — query/fragment not allowed", candidate
  4150. )
  4151. continue
  4152. if "*" in parsed.netloc:
  4153. _security_headers_logger.warning("TRUSTED_FRAME_ORIGINS: dropping %r — wildcards not allowed", candidate)
  4154. continue
  4155. valid.append(f"{parsed.scheme}://{parsed.netloc}")
  4156. if valid:
  4157. _security_headers_logger.info("TRUSTED_FRAME_ORIGINS: %s", ", ".join(valid))
  4158. return tuple(valid)
  4159. _TRUSTED_FRAME_ORIGINS: tuple[str, ...] = _parse_trusted_frame_origins()
  4160. def _frame_ancestors(default_value: str) -> str:
  4161. """Compose the ``frame-ancestors`` CSP directive (#1191).
  4162. ``default_value`` is the strict directive used when the operator has not
  4163. configured ``TRUSTED_FRAME_ORIGINS`` — typically ``'none'`` (catch-all and
  4164. docs) or ``'self'`` (gcode-viewer, served same-origin). When trusted origins
  4165. are configured, ``'self'`` is always included so same-origin embedding never
  4166. breaks even if an operator forgets to add their own origin to the list.
  4167. """
  4168. if _TRUSTED_FRAME_ORIGINS:
  4169. return "frame-ancestors 'self' " + " ".join(_TRUSTED_FRAME_ORIGINS) + ";"
  4170. return f"frame-ancestors {default_value};"
  4171. @app.middleware("http")
  4172. async def security_headers_middleware(request, call_next):
  4173. """Add standard HTTP security headers to every response."""
  4174. response = await call_next(request)
  4175. response.headers["X-Content-Type-Options"] = "nosniff"
  4176. # X-Frame-Options is the legacy cross-origin embedding control. Modern
  4177. # browsers honour CSP frame-ancestors instead, and the legacy
  4178. # `ALLOW-FROM <url>` syntax is deprecated and inconsistent across vendors.
  4179. # When operators have explicitly allowlisted trusted frame origins (#1191
  4180. # — typically Home Assistant on a different port), drop X-Frame-Options
  4181. # and let the CSP-side frame-ancestors directive govern embedding.
  4182. if not _TRUSTED_FRAME_ORIGINS:
  4183. response.headers["X-Frame-Options"] = "SAMEORIGIN"
  4184. response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
  4185. # Content-Security-Policy for the React SPA.
  4186. # Notes:
  4187. # - 'unsafe-inline' for style-src: React and UI libs inject inline styles at runtime.
  4188. # - connect-src ws:/wss:: MQTT/printer WebSocket connections.
  4189. # - img-src data: / blob:: base64 thumbnails and Blob-URL timelapse previews.
  4190. # - media-src blob:: timelapse video player uses Blob URLs.
  4191. # - font-src data:: some icon fonts are embedded as data URIs.
  4192. if request.url.path.startswith("/gcode-viewer"):
  4193. # The gcode viewer is embedded in an iframe served by this same origin,
  4194. # so frame-ancestors must allow 'self'. prettygcode.js also uses eval()
  4195. # internally, so script-src needs 'unsafe-eval'.
  4196. response.headers["Content-Security-Policy"] = (
  4197. "default-src 'self'; "
  4198. "script-src 'self' 'unsafe-eval'; "
  4199. "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; "
  4200. "img-src 'self' data: blob:; "
  4201. "media-src 'self' blob:; "
  4202. "connect-src 'self' ws: wss:; "
  4203. "font-src 'self' data: https://fonts.gstatic.com; "
  4204. "object-src 'none'; "
  4205. "base-uri 'self'; "
  4206. "frame-src 'self' http: https:; " + _frame_ancestors("'self'")
  4207. )
  4208. elif request.url.path in ("/docs", "/redoc", "/docs/oauth2-redirect"):
  4209. # FastAPI's built-in Swagger UI / ReDoc pages load assets from
  4210. # cdn.jsdelivr.net and bootstrap with an inline <script>, so the
  4211. # default CSP would render a blank page.
  4212. response.headers["Content-Security-Policy"] = (
  4213. "default-src 'self'; "
  4214. "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
  4215. "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com; "
  4216. "img-src 'self' data: blob: https://fastapi.tiangolo.com https://cdn.redoc.ly; "
  4217. "connect-src 'self'; "
  4218. "font-src 'self' data: https://fonts.gstatic.com; "
  4219. "worker-src 'self' blob:; "
  4220. "object-src 'none'; "
  4221. "base-uri 'self'; " + _frame_ancestors("'none'")
  4222. )
  4223. else:
  4224. response.headers["Content-Security-Policy"] = (
  4225. "default-src 'self'; "
  4226. "script-src 'self'; "
  4227. "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; "
  4228. "img-src 'self' data: blob:; "
  4229. "media-src 'self' blob:; "
  4230. "connect-src 'self' ws: wss:; "
  4231. "font-src 'self' data: https://fonts.gstatic.com; "
  4232. "object-src 'none'; "
  4233. "base-uri 'self'; "
  4234. "frame-src 'self' http: https:; " + _frame_ancestors("'none'")
  4235. )
  4236. if request.url.scheme == "https":
  4237. response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
  4238. return response
  4239. @app.middleware("http")
  4240. async def auth_middleware(request, call_next):
  4241. """Enforce authentication on all API routes when auth is enabled.
  4242. This middleware provides defense-in-depth by checking auth at the API gateway level,
  4243. regardless of whether individual routes have auth dependencies.
  4244. """
  4245. from starlette.responses import JSONResponse
  4246. path = request.url.path
  4247. # Only apply to API routes
  4248. if not path.startswith("/api/"):
  4249. return await call_next(request)
  4250. # Allow public routes
  4251. if path in PUBLIC_API_ROUTES:
  4252. return await call_next(request)
  4253. # Allow public prefixes
  4254. for prefix in PUBLIC_API_PREFIXES:
  4255. if path.startswith(prefix):
  4256. return await call_next(request)
  4257. # Allow public patterns (read-only display data like thumbnails)
  4258. for pattern in PUBLIC_API_PATTERNS:
  4259. if pattern in path:
  4260. return await call_next(request)
  4261. # Check if auth is enabled
  4262. try:
  4263. async with async_session() as db:
  4264. from backend.app.core.auth import is_auth_enabled
  4265. auth_enabled = await is_auth_enabled(db)
  4266. if not auth_enabled:
  4267. # Auth disabled, allow all requests
  4268. return await call_next(request)
  4269. except Exception:
  4270. # If we can't check auth status, allow request (fail open for DB issues)
  4271. return await call_next(request)
  4272. # Auth is enabled - require valid token
  4273. auth_header = request.headers.get("Authorization")
  4274. x_api_key = request.headers.get("X-API-Key")
  4275. # Check for API key auth first
  4276. if x_api_key or (auth_header and auth_header.startswith("Bearer bb_")):
  4277. # API key authentication - let the request through to be validated by route handler
  4278. # API keys are validated per-route since they have different permission levels
  4279. return await call_next(request)
  4280. # Check for JWT auth
  4281. if not auth_header or not auth_header.startswith("Bearer "):
  4282. return JSONResponse(
  4283. status_code=401,
  4284. content={"detail": "Authentication required"},
  4285. headers={"WWW-Authenticate": "Bearer"},
  4286. )
  4287. # Validate JWT token
  4288. import jwt
  4289. try:
  4290. from backend.app.core.auth import (
  4291. ALGORITHM,
  4292. SECRET_KEY,
  4293. _is_token_fresh,
  4294. get_user_by_username,
  4295. is_jti_revoked,
  4296. )
  4297. token = auth_header.replace("Bearer ", "")
  4298. payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
  4299. username = payload.get("sub")
  4300. if not username:
  4301. raise ValueError("No username in token")
  4302. jti = payload.get("jti")
  4303. if not jti:
  4304. raise ValueError("No jti in token")
  4305. iat = payload.get("iat")
  4306. # Reject revoked tokens (defense-in-depth gateway check)
  4307. if await is_jti_revoked(jti):
  4308. return JSONResponse(
  4309. status_code=401,
  4310. content={"detail": "Token has been revoked"},
  4311. headers={"WWW-Authenticate": "Bearer"},
  4312. )
  4313. # Verify user exists, is active, and token is still fresh (L-R8-A)
  4314. async with async_session() as db:
  4315. user = await get_user_by_username(db, username)
  4316. if not user or not user.is_active:
  4317. return JSONResponse(
  4318. status_code=401,
  4319. content={"detail": "User not found or inactive"},
  4320. headers={"WWW-Authenticate": "Bearer"},
  4321. )
  4322. if not _is_token_fresh(iat, user):
  4323. return JSONResponse(
  4324. status_code=401,
  4325. content={"detail": "Token no longer valid"},
  4326. headers={"WWW-Authenticate": "Bearer"},
  4327. )
  4328. except jwt.ExpiredSignatureError:
  4329. return JSONResponse(
  4330. status_code=401,
  4331. content={"detail": "Token has expired"},
  4332. headers={"WWW-Authenticate": "Bearer"},
  4333. )
  4334. except (jwt.InvalidTokenError, ValueError, Exception):
  4335. return JSONResponse(
  4336. status_code=401,
  4337. content={"detail": "Invalid token"},
  4338. headers={"WWW-Authenticate": "Bearer"},
  4339. )
  4340. return await call_next(request)
  4341. @app.middleware("http")
  4342. async def trace_id_middleware(request, call_next):
  4343. """Stamp every HTTP request with a trace ID and echo it back.
  4344. Decorated AFTER auth_middleware on purpose: Starlette stacks
  4345. @app.middleware decorators LIFO, so the last-decorated runs first
  4346. inbound. Putting the trace stamp last makes it the OUTERMOST layer,
  4347. which means auth-middleware log lines (and every line emitted on the
  4348. way down to and back from the route handler) all carry the same
  4349. trace ID. If we put it before auth, auth's logs would be stamped
  4350. with the *previous* request's ID — useless for correlation.
  4351. Honours an inbound ``X-Trace-Id`` header so callers running their
  4352. own tracing can correlate their span IDs with our log lines, but
  4353. only if the value passes the whitelist gate in
  4354. ``backend.app.core.trace.normalise_inbound_trace_id`` — anything
  4355. rejected (too long, contains control chars, etc.) silently triggers
  4356. a freshly minted server-side ID rather than failing the request.
  4357. The minted (or echoed) ID is set on a ContextVar so that every log
  4358. record emitted during the request — application logs *and* uvicorn's
  4359. access log — carries it via TraceIDFilter, and is also written to
  4360. the ``X-Trace-Id`` response header so clients can pin a server-side
  4361. log search to the exact request they made.
  4362. """
  4363. from backend.app.core.trace import (
  4364. generate_trace_id,
  4365. normalise_inbound_trace_id,
  4366. trace_id_var,
  4367. )
  4368. inbound = normalise_inbound_trace_id(request.headers.get("X-Trace-Id"))
  4369. trace_id = inbound if inbound is not None else generate_trace_id()
  4370. token = trace_id_var.set(trace_id)
  4371. try:
  4372. response = await call_next(request)
  4373. finally:
  4374. # Reset the ContextVar so a record emitted in a totally
  4375. # unrelated background task that just happens to inherit this
  4376. # context doesn't keep referencing this request's ID forever.
  4377. # In practice ContextVar.reset is best-effort under asyncio
  4378. # task-spawn semantics, but the cost is one attribute write so
  4379. # we may as well do it.
  4380. trace_id_var.reset(token)
  4381. response.headers["X-Trace-Id"] = trace_id
  4382. return response
  4383. # API routes
  4384. app.include_router(auth.router, prefix=app_settings.api_prefix)
  4385. app.include_router(mfa.router, prefix=app_settings.api_prefix)
  4386. app.include_router(bug_report.router, prefix=app_settings.api_prefix)
  4387. app.include_router(users.router, prefix=app_settings.api_prefix)
  4388. app.include_router(groups.router, prefix=app_settings.api_prefix)
  4389. app.include_router(printers.router, prefix=app_settings.api_prefix)
  4390. app.include_router(archives.router, prefix=app_settings.api_prefix)
  4391. app.include_router(filaments.router, prefix=app_settings.api_prefix)
  4392. app.include_router(inventory.router, prefix=app_settings.api_prefix)
  4393. app.include_router(labels.router, prefix=app_settings.api_prefix)
  4394. app.include_router(settings_routes.router, prefix=app_settings.api_prefix)
  4395. app.include_router(cloud.router, prefix=app_settings.api_prefix)
  4396. app.include_router(local_presets.router, prefix=app_settings.api_prefix)
  4397. app.include_router(smart_plugs.router, prefix=app_settings.api_prefix)
  4398. app.include_router(print_log.router, prefix=app_settings.api_prefix)
  4399. app.include_router(print_queue.router, prefix=app_settings.api_prefix)
  4400. app.include_router(background_dispatch_routes.router, prefix=app_settings.api_prefix)
  4401. app.include_router(kprofiles.router, prefix=app_settings.api_prefix)
  4402. app.include_router(notifications.router, prefix=app_settings.api_prefix)
  4403. app.include_router(notification_templates.router, prefix=app_settings.api_prefix)
  4404. app.include_router(user_notifications.router, prefix=app_settings.api_prefix)
  4405. app.include_router(spoolman.router, prefix=app_settings.api_prefix)
  4406. app.include_router(updates.router, prefix=app_settings.api_prefix)
  4407. app.include_router(maintenance.router, prefix=app_settings.api_prefix)
  4408. app.include_router(camera.router, prefix=app_settings.api_prefix)
  4409. app.include_router(external_links.router, prefix=app_settings.api_prefix)
  4410. app.include_router(projects.router, prefix=app_settings.api_prefix)
  4411. app.include_router(library.router, prefix=app_settings.api_prefix)
  4412. app.include_router(library_trash.router, prefix=app_settings.api_prefix)
  4413. app.include_router(slice_jobs.router, prefix=app_settings.api_prefix)
  4414. app.include_router(slicer_presets.router, prefix=app_settings.api_prefix)
  4415. app.include_router(archive_purge.router, prefix=app_settings.api_prefix)
  4416. app.include_router(makerworld.router, prefix=app_settings.api_prefix)
  4417. app.include_router(api_keys.router, prefix=app_settings.api_prefix)
  4418. app.include_router(webhook.router, prefix=app_settings.api_prefix)
  4419. app.include_router(ams_history.router, prefix=app_settings.api_prefix)
  4420. app.include_router(system.router, prefix=app_settings.api_prefix)
  4421. app.include_router(support.router, prefix=app_settings.api_prefix)
  4422. app.include_router(websocket.router, prefix=app_settings.api_prefix)
  4423. app.include_router(discovery.router, prefix=app_settings.api_prefix)
  4424. app.include_router(pending_uploads.router, prefix=app_settings.api_prefix)
  4425. app.include_router(firmware.router, prefix=app_settings.api_prefix)
  4426. app.include_router(github_backup.router, prefix=app_settings.api_prefix)
  4427. app.include_router(local_backup.router, prefix=app_settings.api_prefix)
  4428. app.include_router(obico.router, prefix=app_settings.api_prefix)
  4429. app.include_router(metrics.router, prefix=app_settings.api_prefix)
  4430. app.include_router(virtual_printers.router, prefix=app_settings.api_prefix)
  4431. app.include_router(spoolbuddy.router, prefix=app_settings.api_prefix)
  4432. # Serve static files (React build)
  4433. if app_settings.static_dir.exists() and any(app_settings.static_dir.iterdir()):
  4434. app.mount(
  4435. "/assets",
  4436. StaticFiles(directory=app_settings.static_dir / "assets"),
  4437. name="assets",
  4438. )
  4439. if (app_settings.static_dir / "img").exists():
  4440. app.mount(
  4441. "/img",
  4442. StaticFiles(directory=app_settings.static_dir / "img"),
  4443. name="img",
  4444. )
  4445. if (app_settings.static_dir / "icons").exists():
  4446. app.mount(
  4447. "/icons",
  4448. StaticFiles(directory=app_settings.static_dir / "icons"),
  4449. name="icons",
  4450. )
  4451. @app.get("/")
  4452. async def serve_frontend():
  4453. """Serve the React frontend."""
  4454. index_file = app_settings.static_dir / "index.html"
  4455. if index_file.exists():
  4456. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  4457. return {
  4458. "message": "Bambuddy API",
  4459. "docs": "/docs",
  4460. "frontend": "Build and place React app in /static directory",
  4461. }
  4462. # index.html must always be revalidated — Vite emits content-hashed JS/CSS
  4463. # bundles (e.g. `index-JRaF_JhW.js`), so the JS itself is safe to cache
  4464. # forever, but the HTML wrapping it is the only file that knows which hash
  4465. # is current. Without explicit cache-control headers Chromium decides
  4466. # heuristically (typically 10% of the time since Last-Modified) and on
  4467. # long-running kiosks happily serves stale HTML across browser restarts.
  4468. # That stale HTML references an old bundle hash, the old bundle is also
  4469. # in the disk cache, and the user ends up running pre-update JS forever
  4470. # without ever knowing why. ``no-cache`` (revalidate every time, but a
  4471. # 304 is cheap) is the correct setting for an SPA's entry HTML.
  4472. _HTML_CACHE_HEADERS = {"Cache-Control": "no-cache, must-revalidate"}
  4473. @app.get("/health")
  4474. async def health_check():
  4475. """Health check endpoint."""
  4476. return {"status": "healthy"}
  4477. @app.get("/manifest.json")
  4478. async def serve_manifest():
  4479. """Serve PWA manifest."""
  4480. manifest_file = app_settings.static_dir / "manifest.json"
  4481. if manifest_file.exists():
  4482. return FileResponse(manifest_file, media_type="application/manifest+json")
  4483. return {"error": "Manifest not found"}
  4484. @app.get("/sw.js")
  4485. async def serve_service_worker():
  4486. """Serve service worker."""
  4487. sw_file = app_settings.static_dir / "sw.js"
  4488. if sw_file.exists():
  4489. return FileResponse(
  4490. sw_file,
  4491. media_type="application/javascript",
  4492. headers={"Cache-Control": "no-cache, no-store, must-revalidate"},
  4493. )
  4494. return {"error": "Service worker not found"}
  4495. @app.get("/sw-register.js")
  4496. async def serve_sw_register():
  4497. """Serve the service-worker registration bootstrap script.
  4498. Served as a real JS file so the strict `script-src 'self'` CSP covers it
  4499. without needing 'unsafe-inline' or per-build hashes on the inline tag.
  4500. """
  4501. reg_file = app_settings.static_dir / "sw-register.js"
  4502. if reg_file.exists():
  4503. return FileResponse(reg_file, media_type="application/javascript")
  4504. return {"error": "sw-register.js not found"}
  4505. # ── GCode viewer static files ────────────────────────────────────────────────
  4506. # Served via explicit routes so ordering is guaranteed (app.mount() loses
  4507. # to the /{full_path:path} catch-all in some Starlette versions).
  4508. _gcode_viewer_dir = (app_settings.static_dir.parent / "gcode_viewer").resolve()
  4509. def _gcode_viewer_response(rel: str) -> FileResponse:
  4510. from fastapi import HTTPException as _HTTPException
  4511. safe = (_gcode_viewer_dir / rel).resolve()
  4512. if not safe.is_relative_to(_gcode_viewer_dir):
  4513. raise _HTTPException(status_code=403)
  4514. if safe.is_file():
  4515. mt, _ = _mimetypes.guess_type(str(safe))
  4516. return FileResponse(str(safe), media_type=mt or "application/octet-stream")
  4517. raise _HTTPException(status_code=404)
  4518. @app.get("/gcode-viewer/")
  4519. async def serve_gcode_viewer_index() -> FileResponse:
  4520. """Raw PrettyGCode viewer for the iframe. The bare ``/gcode-viewer``
  4521. (no trailing slash) intentionally falls through to the SPA catch-all so a
  4522. full-page reload re-enters the React layout instead of serving the iframe
  4523. contents standalone."""
  4524. return _gcode_viewer_response("index.html")
  4525. @app.get("/gcode-viewer/{file_path:path}")
  4526. async def serve_gcode_viewer_file(file_path: str) -> FileResponse:
  4527. return _gcode_viewer_response(file_path)
  4528. # Catch-all route for React Router (must be last)
  4529. @app.get("/{full_path:path}")
  4530. async def serve_spa(full_path: str):
  4531. """Serve React app for client-side routing."""
  4532. # Don't intercept API routes - raise proper 404 so FastAPI can handle redirects
  4533. if full_path.startswith("api/"):
  4534. from fastapi import HTTPException
  4535. raise HTTPException(status_code=404, detail="Not found")
  4536. index_file = app_settings.static_dir / "index.html"
  4537. if index_file.exists():
  4538. return FileResponse(index_file, headers=_HTML_CACHE_HEADERS)
  4539. return {"error": "Frontend not built"}