main.py 174 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781
  1. import asyncio
  2. import logging
  3. import time
  4. from contextlib import asynccontextmanager
  5. from datetime import datetime, timedelta, timezone
  6. from logging.handlers import RotatingFileHandler
  7. from fastapi import FastAPI
  8. from fastapi.responses import FileResponse
  9. from fastapi.staticfiles import StaticFiles
  10. from sqlalchemy import delete, or_, select, text
  11. from backend.app.api.routes import (
  12. ams_history,
  13. api_keys,
  14. archives,
  15. auth,
  16. background_dispatch as background_dispatch_routes,
  17. bug_report,
  18. camera,
  19. cloud,
  20. discovery,
  21. external_links,
  22. filaments,
  23. firmware,
  24. github_backup,
  25. groups,
  26. inventory,
  27. kprofiles,
  28. library,
  29. local_presets,
  30. maintenance,
  31. metrics,
  32. notification_templates,
  33. notifications,
  34. pending_uploads,
  35. print_log,
  36. print_queue,
  37. printers,
  38. projects,
  39. settings as settings_routes,
  40. smart_plugs,
  41. spoolbuddy,
  42. spoolman,
  43. support,
  44. system,
  45. updates,
  46. users,
  47. virtual_printers,
  48. webhook,
  49. websocket,
  50. )
  51. from backend.app.api.routes.maintenance import _get_printer_maintenance_internal, ensure_default_types
  52. from backend.app.api.routes.support import init_debug_logging
  53. from backend.app.core.config import APP_VERSION, settings as app_settings
  54. from backend.app.core.database import async_session, engine, init_db
  55. from backend.app.core.websocket import ws_manager
  56. from backend.app.models.smart_plug import SmartPlug
  57. from backend.app.services.archive import ArchiveService
  58. from backend.app.services.background_dispatch import background_dispatch
  59. from backend.app.services.bambu_ftp import download_file_async, get_ftp_retry_settings, with_ftp_retry
  60. from backend.app.services.bambu_mqtt import PrinterState
  61. from backend.app.services.github_backup import github_backup_service
  62. from backend.app.services.homeassistant import homeassistant_service
  63. from backend.app.services.mqtt_relay import mqtt_relay
  64. from backend.app.services.mqtt_smart_plug import mqtt_smart_plug_service
  65. from backend.app.services.notification_service import notification_service
  66. from backend.app.services.print_scheduler import scheduler as print_scheduler
  67. from backend.app.services.printer_manager import (
  68. init_printer_connections,
  69. printer_manager,
  70. printer_state_to_dict,
  71. )
  72. from backend.app.services.smart_plug_manager import smart_plug_manager
  73. from backend.app.services.spoolman import close_spoolman_client, get_spoolman_client, init_spoolman_client
  74. from backend.app.services.spoolman_tracking import (
  75. cleanup_tracking as _cleanup_spoolman_tracking,
  76. report_usage as _report_spoolman_usage,
  77. store_print_data as _store_spoolman_print_data,
  78. )
  79. from backend.app.services.tasmota import tasmota_service
  80. # =============================================================================
  81. # Dependency Check - runs before other imports to give helpful error messages
  82. # =============================================================================
  83. def _start_error_server(missing_packages: list):
  84. """Start a minimal HTTP server to display dependency errors in browser."""
  85. import os
  86. import signal
  87. from http.server import BaseHTTPRequestHandler, HTTPServer
  88. packages_html = "".join(f"<li><code>{p}</code></li>" for p in missing_packages)
  89. html = f"""<!DOCTYPE html>
  90. <html>
  91. <head>
  92. <title>Bambuddy - Setup Required</title>
  93. <style>
  94. body {{
  95. font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
  96. background: #0f172a; color: #e2e8f0;
  97. display: flex; justify-content: center; align-items: center;
  98. min-height: 100vh; margin: 0; padding: 20px; box-sizing: border-box;
  99. }}
  100. .container {{
  101. background: #1e293b; border-radius: 12px; padding: 40px;
  102. max-width: 600px; text-align: center; box-shadow: 0 4px 20px rgba(0,0,0,0.3);
  103. }}
  104. h1 {{ color: #f87171; margin-bottom: 10px; }}
  105. h2 {{ color: #94a3b8; font-weight: normal; margin-top: 0; }}
  106. .packages {{
  107. background: #0f172a; border-radius: 8px; padding: 20px;
  108. margin: 20px 0; text-align: left;
  109. }}
  110. .packages ul {{ margin: 0; padding-left: 20px; }}
  111. .packages li {{ color: #fbbf24; margin: 8px 0; }}
  112. .command {{
  113. background: #0f172a; border-radius: 8px; padding: 15px 20px;
  114. margin: 15px 0; font-family: monospace; color: #4ade80;
  115. text-align: left; overflow-x: auto;
  116. }}
  117. .note {{ color: #94a3b8; font-size: 14px; margin-top: 20px; }}
  118. </style>
  119. </head>
  120. <body>
  121. <div class="container">
  122. <h1>Setup Required</h1>
  123. <h2>Missing Python packages</h2>
  124. <div class="packages"><ul>{packages_html}</ul></div>
  125. <p>To fix, run this command on your server:</p>
  126. <div class="command">pip install -r requirements.txt</div>
  127. <p>Or if using a virtual environment:</p>
  128. <div class="command">./venv/bin/pip install -r requirements.txt</div>
  129. <p class="note">After installing, restart Bambuddy:<br>
  130. <code>sudo systemctl restart bambuddy</code></p>
  131. </div>
  132. </body>
  133. </html>"""
  134. class ErrorHandler(BaseHTTPRequestHandler):
  135. def do_GET(self):
  136. self.send_response(503)
  137. self.send_header("Content-type", "text/html")
  138. self.end_headers()
  139. self.wfile.write(html.encode())
  140. def log_message(self, format, *args):
  141. print(f"[Error Server] {args[0]}")
  142. port = int(os.environ.get("PORT", 8000))
  143. print(f"\nStarting error server on http://0.0.0.0:{port}")
  144. print("Visit this URL in your browser to see the error details.\n")
  145. server = HTTPServer(("0.0.0.0", port), ErrorHandler) # nosec B104
  146. def shutdown(signum, frame):
  147. print("\nShutting down error server...")
  148. raise SystemExit(0)
  149. signal.signal(signal.SIGTERM, shutdown)
  150. signal.signal(signal.SIGINT, shutdown)
  151. server.serve_forever()
  152. def check_dependencies():
  153. """Check that all required packages are installed."""
  154. missing = []
  155. # Map of import name -> package name (for pip install)
  156. required = {
  157. "jwt": "PyJWT",
  158. "fastapi": "fastapi",
  159. "uvicorn": "uvicorn",
  160. "sqlalchemy": "sqlalchemy",
  161. "aiosqlite": "aiosqlite",
  162. "pydantic": "pydantic",
  163. "paho.mqtt": "paho-mqtt",
  164. }
  165. for module, package in required.items():
  166. try:
  167. __import__(module)
  168. except ImportError:
  169. missing.append(package)
  170. if missing:
  171. print("\n" + "=" * 60)
  172. print("ERROR: Missing required Python packages!")
  173. print("=" * 60)
  174. print(f"\nMissing packages: {', '.join(missing)}")
  175. print("\nTo fix, run:")
  176. print(" pip install -r requirements.txt")
  177. print("\nOr if using a virtual environment:")
  178. print(" ./venv/bin/pip install -r requirements.txt")
  179. print("=" * 60 + "\n")
  180. _start_error_server(missing)
  181. check_dependencies()
  182. # =============================================================================
  183. # Import settings first for logging configuration
  184. # Configure logging based on settings
  185. # DEBUG=true -> DEBUG level, else use LOG_LEVEL setting
  186. log_level_str = "DEBUG" if app_settings.debug else app_settings.log_level.upper()
  187. log_level = getattr(logging, log_level_str, logging.INFO)
  188. log_format = "%(asctime)s %(levelname)s [%(name)s] %(message)s"
  189. # Create root logger
  190. root_logger = logging.getLogger()
  191. root_logger.setLevel(log_level)
  192. # Console handler - always enabled
  193. console_handler = logging.StreamHandler()
  194. console_handler.setLevel(log_level)
  195. console_handler.setFormatter(logging.Formatter(log_format))
  196. root_logger.addHandler(console_handler)
  197. # File handler - only in production or if explicitly enabled
  198. if app_settings.log_to_file:
  199. log_file = app_settings.log_dir / "bambuddy.log"
  200. file_handler = RotatingFileHandler(
  201. log_file,
  202. maxBytes=5 * 1024 * 1024, # 5MB
  203. backupCount=3,
  204. encoding="utf-8",
  205. )
  206. file_handler.setLevel(log_level)
  207. file_handler.setFormatter(logging.Formatter(log_format))
  208. root_logger.addHandler(file_handler)
  209. logging.info("Logging to file: %s", log_file)
  210. # Reduce noise from third-party libraries in production
  211. if not app_settings.debug:
  212. logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
  213. logging.getLogger("httpcore").setLevel(logging.WARNING)
  214. logging.getLogger("httpx").setLevel(logging.WARNING)
  215. logging.getLogger("paho.mqtt").setLevel(logging.WARNING)
  216. logging.info("Bambuddy starting - debug=%s, log_level=%s", app_settings.debug, log_level_str)
  217. # Track active prints: {(printer_id, filename): archive_id}
  218. _active_prints: dict[tuple[int, str], int] = {}
  219. # Track expected prints from reprint/scheduled (skip auto-archiving for these)
  220. # {(printer_id, filename): archive_id}
  221. _expected_prints: dict[tuple[int, str], int] = {}
  222. # Track starting energy for prints: {archive_id: starting_kwh}
  223. _print_energy_start: dict[int, float] = {}
  224. # Track AMS mapping for prints: {archive_id: [global_tray_id_per_slot]}
  225. # Used by usage tracker to map 3MF slots to physical AMS trays
  226. _print_ams_mappings: dict[int, list[int]] = {}
  227. # Track progress milestones for notifications: {printer_id: last_milestone_notified}
  228. # Milestones are 25, 50, 75. Value of 0 means no milestone notified yet for current print.
  229. _last_progress_milestone: dict[int, int] = {}
  230. # Track whether first layer complete notification has been sent for current print
  231. _first_layer_notified: dict[int, bool] = {}
  232. # Track HMS errors that have been notified: {printer_id: set of error codes}
  233. # This prevents sending duplicate notifications for the same error
  234. _notified_hms_errors: dict[int, set[str]] = {}
  235. # Track when HMS errors were last seen: {printer_id: timestamp}
  236. # Used to debounce clearing — prevents flapping errors from re-triggering notifications
  237. _hms_last_seen: dict[int, float] = {}
  238. _HMS_CLEAR_GRACE_SECONDS = 30.0
  239. # Track timelapse file baselines at print start: {printer_id: set of video filenames}
  240. # Used for snapshot-diff detection at print completion
  241. _timelapse_baselines: dict[int, set[str]] = {}
  242. # Track active bed cooldown monitoring tasks: {printer_id: asyncio.Task}
  243. _bed_cooldown_tasks: dict[int, asyncio.Task] = {}
  244. async def _get_plug_energy(plug, db) -> dict | None:
  245. """Get energy from plug regardless of type (Tasmota, Home Assistant, or MQTT).
  246. For HA plugs, configures the service with current settings from DB.
  247. For MQTT plugs, returns data from the subscription service.
  248. """
  249. if plug.plug_type == "homeassistant":
  250. from backend.app.api.routes.settings import get_homeassistant_settings
  251. ha_settings = await get_homeassistant_settings(db)
  252. homeassistant_service.configure(ha_settings["ha_url"], ha_settings["ha_token"])
  253. return await homeassistant_service.get_energy(plug)
  254. elif plug.plug_type == "mqtt":
  255. # MQTT plugs report "today" energy, not lifetime total
  256. # For per-print tracking, we use "today" as the counter (resets at midnight)
  257. mqtt_data = mqtt_relay.smart_plug_service.get_plug_data(plug.id)
  258. if mqtt_data:
  259. return {
  260. "power": mqtt_data.power,
  261. "today": mqtt_data.energy,
  262. "total": mqtt_data.energy, # Use today as total for per-print calculations
  263. }
  264. return None
  265. else:
  266. return await tasmota_service.get_energy(plug)
  267. def register_expected_print(printer_id: int, filename: str, archive_id: int, ams_mapping: list[int] | None = None):
  268. """Register an expected print from reprint/scheduled so we don't create duplicate archives."""
  269. # Store with multiple filename variations to catch different naming patterns
  270. _expected_prints[(printer_id, filename)] = archive_id
  271. # Also store without .3mf extension if present
  272. if filename.endswith(".3mf"):
  273. base = filename[:-4]
  274. _expected_prints[(printer_id, base)] = archive_id
  275. _expected_prints[(printer_id, f"{base}.gcode")] = archive_id
  276. # Store AMS mapping for usage tracking at print completion
  277. if ams_mapping is not None:
  278. _print_ams_mappings[archive_id] = ams_mapping
  279. logging.getLogger(__name__).info(
  280. f"Registered expected print: printer={printer_id}, file={filename}, archive={archive_id}, ams_mapping={ams_mapping}"
  281. )
  282. def _get_start_ams_mapping(data: dict, archive_id: int | None) -> list[int] | None:
  283. """Resolve AMS mapping for print start without consuming stored queue/reprint state."""
  284. stored_ams_mapping = data.get("ams_mapping")
  285. if not stored_ams_mapping and archive_id:
  286. stored_ams_mapping = _print_ams_mappings.get(archive_id)
  287. return stored_ams_mapping
  288. _last_status_broadcast: dict[int, str] = {}
  289. # Track printers where we've updated nozzle_count
  290. _nozzle_count_updated: set[int] = set()
  291. async def on_printer_status_change(printer_id: int, state: PrinterState):
  292. """Handle printer status changes - broadcast via WebSocket."""
  293. # Only broadcast if something meaningful changed (reduce WebSocket spam)
  294. # Include rounded temperatures to detect meaningful temp changes (within 1 degree)
  295. temps = state.temperatures or {}
  296. nozzle_temp = round(temps.get("nozzle", 0))
  297. bed_temp = round(temps.get("bed", 0))
  298. nozzle_2_temp = round(temps.get("nozzle_2", 0)) if "nozzle_2" in temps else ""
  299. chamber_temp = round(temps.get("chamber", 0)) if "chamber" in temps else ""
  300. # Auto-detect dual-nozzle printers from MQTT temperature data
  301. if "nozzle_2" in temps and printer_id not in _nozzle_count_updated:
  302. _nozzle_count_updated.add(printer_id)
  303. # Update nozzle_count in database
  304. async with async_session() as db:
  305. from backend.app.models.printer import Printer
  306. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  307. printer = result.scalar_one_or_none()
  308. if printer and printer.nozzle_count != 2:
  309. printer.nozzle_count = 2
  310. await db.commit()
  311. logging.getLogger(__name__).info(
  312. f"Auto-detected dual-nozzle printer {printer_id}, updated nozzle_count=2"
  313. )
  314. # Include target temps for heating phase detection
  315. bed_target = round(temps.get("bed_target", 0))
  316. nozzle_target = round(temps.get("nozzle_target", 0))
  317. # Include tray_now and vt_tray hash so external spool changes trigger broadcasts
  318. vt_tray_key = hash(str(state.raw_data.get("vt_tray", []))) if state.raw_data else 0
  319. # Include AMS dry_time values so drying status changes trigger broadcasts
  320. ams_dry_key = tuple(a.get("dry_time", 0) for a in (state.raw_data.get("ams") or [])) if state.raw_data else ()
  321. status_key = (
  322. f"{state.connected}:{state.state}:{state.progress}:{state.layer_num}:"
  323. f"{nozzle_temp}:{bed_temp}:{nozzle_2_temp}:{chamber_temp}:"
  324. f"{state.stg_cur}:{bed_target}:{nozzle_target}:"
  325. f"{state.cooling_fan_speed}:{state.big_fan1_speed}:{state.big_fan2_speed}:"
  326. f"{state.chamber_light}:{state.active_extruder}:{state.tray_now}:{vt_tray_key}:"
  327. f"{ams_dry_key}"
  328. )
  329. # MQTT relay - publish status (before dedup check - always publish to MQTT)
  330. try:
  331. printer_info = printer_manager.get_printer(printer_id)
  332. if printer_info:
  333. await mqtt_relay.on_printer_status(printer_id, state, printer_info.name, printer_info.serial_number)
  334. except Exception:
  335. pass # Don't fail status callback if MQTT fails
  336. if _last_status_broadcast.get(printer_id) == status_key:
  337. return # No change, skip WebSocket broadcast
  338. _last_status_broadcast[printer_id] = status_key
  339. # Check for progress milestone notifications (25%, 50%, 75%)
  340. progress = state.progress or 0
  341. is_printing = state.state in ("RUNNING", "PRINTING")
  342. if is_printing and progress > 0:
  343. # Determine which milestone we've reached
  344. current_milestone = 0
  345. if progress >= 75:
  346. current_milestone = 75
  347. elif progress >= 50:
  348. current_milestone = 50
  349. elif progress >= 25:
  350. current_milestone = 25
  351. last_milestone = _last_progress_milestone.get(printer_id, 0)
  352. # If we've crossed a new milestone, send notification
  353. if current_milestone > last_milestone:
  354. _last_progress_milestone[printer_id] = current_milestone
  355. try:
  356. async with async_session() as db:
  357. from backend.app.models.printer import Printer
  358. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  359. printer = result.scalar_one_or_none()
  360. printer_name = printer.name if printer else f"Printer {printer_id}"
  361. filename = state.subtask_name or state.gcode_file or "Unknown"
  362. # remaining_time is in minutes, convert to seconds for notification
  363. remaining_time_seconds = state.remaining_time * 60 if state.remaining_time else None
  364. # Capture camera snapshot for notification image attachment
  365. image_data = await _capture_snapshot_for_notification(
  366. printer_id, printer, logging.getLogger(__name__)
  367. )
  368. await notification_service.on_print_progress(
  369. printer_id,
  370. printer_name,
  371. filename,
  372. current_milestone,
  373. db,
  374. remaining_time_seconds,
  375. image_data=image_data,
  376. )
  377. except Exception as e:
  378. logging.getLogger(__name__).warning(f"Progress milestone notification failed: {e}")
  379. elif progress < 5:
  380. # Reset milestone tracking when print restarts or new print begins
  381. _last_progress_milestone[printer_id] = 0
  382. _first_layer_notified[printer_id] = False
  383. # HMS error codes that should not trigger notifications.
  384. # These are infrastructure/auth issues, not actionable print errors.
  385. _HMS_NOTIFICATION_SUPPRESS = {
  386. "0500_0007", # MQTT command verification failed (auth/bind issue, not a print error)
  387. "0500_4001", # Failed to connect to Bambu Cloud (network issue)
  388. "0500_400E", # Printing was cancelled (user action, not an error)
  389. }
  390. # Check for new HMS errors and send notifications
  391. current_hms_errors = getattr(state, "hms_errors", []) or []
  392. if current_hms_errors:
  393. # Build set of current error codes (using attr for uniqueness)
  394. current_error_codes = {f"{e.attr:08x}" for e in current_hms_errors}
  395. previously_notified = _notified_hms_errors.get(printer_id, set())
  396. # Find new errors that haven't been notified yet
  397. new_error_codes = current_error_codes - previously_notified
  398. # Update tracking immediately to prevent duplicate notifications from concurrent callbacks
  399. _notified_hms_errors[printer_id] = current_error_codes
  400. _hms_last_seen[printer_id] = time.time()
  401. if new_error_codes:
  402. # Get the actual new errors for the notification
  403. # Filter to severity >= 2 (skip informational/status messages like H2D sends)
  404. new_errors = [e for e in current_hms_errors if f"{e.attr:08x}" in new_error_codes and e.severity >= 2]
  405. try:
  406. async with async_session() as db:
  407. from backend.app.models.printer import Printer
  408. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  409. printer = result.scalar_one_or_none()
  410. printer_name = printer.name if printer else f"Printer {printer_id}"
  411. # Format error details for notification
  412. # Module 0x07 = AMS/Filament, 0x05 = Nozzle, 0x0C = Motion Controller, etc.
  413. module_names = {
  414. 0x03: "Print/Task",
  415. 0x05: "Nozzle/Extruder",
  416. 0x07: "AMS/Filament",
  417. 0x0C: "Motion Controller",
  418. 0x12: "Chamber",
  419. }
  420. from backend.app.services.hms_errors import get_error_description
  421. # Capture camera snapshot once for all error notifications
  422. error_image_data = await _capture_snapshot_for_notification(
  423. printer_id, printer, logging.getLogger(__name__)
  424. )
  425. for error in new_errors:
  426. module_name = module_names.get(error.module, f"Module 0x{error.module:02X}")
  427. # Build short code like "0700_8010"
  428. # Mask to 16 bits to handle printers that send larger values
  429. error_code_int = int(error.code.replace("0x", ""), 16) if error.code else 0
  430. error_code_masked = error_code_int & 0xFFFF
  431. short_code = f"{(error.attr >> 16) & 0xFFFF:04X}_{error_code_masked:04X}"
  432. if short_code in _HMS_NOTIFICATION_SUPPRESS:
  433. continue
  434. error_type = f"{module_name} Error"
  435. # Look up human-readable description
  436. description = get_error_description(short_code)
  437. error_detail = description if description else f"Error code: {short_code}"
  438. await notification_service.on_printer_error(
  439. printer_id, printer_name, error_type, db, error_detail, image_data=error_image_data
  440. )
  441. logging.getLogger(__name__).info(
  442. f"[HMS] Sent notification for {len(new_errors)} new error(s) on printer {printer_id}"
  443. )
  444. # Also publish to MQTT relay
  445. printer_info = printer_manager.get_printer(printer_id)
  446. if printer_info:
  447. errors_data = [
  448. {
  449. "code": e.code,
  450. "attr": e.attr,
  451. "module": e.module,
  452. "severity": e.severity,
  453. }
  454. for e in new_errors
  455. ]
  456. await mqtt_relay.on_printer_error(
  457. printer_id, printer_info.name, printer_info.serial_number, errors_data
  458. )
  459. except Exception as e:
  460. logging.getLogger(__name__).warning(f"HMS error notification failed: {e}")
  461. else:
  462. # No HMS errors — only clear tracking after a grace period to prevent
  463. # flapping errors (brief hms:[] gaps) from re-triggering notifications.
  464. # Some HMS codes (e.g. chamber temp regulation during PETG prints) toggle
  465. # on/off every few seconds as conditions fluctuate around thresholds.
  466. if printer_id in _notified_hms_errors:
  467. last_seen = _hms_last_seen.get(printer_id, 0)
  468. if time.time() - last_seen >= _HMS_CLEAR_GRACE_SECONDS:
  469. _notified_hms_errors.pop(printer_id, None)
  470. _hms_last_seen.pop(printer_id, None)
  471. await ws_manager.send_printer_status(
  472. printer_id,
  473. printer_state_to_dict(state, printer_id, printer_manager.get_model(printer_id)),
  474. )
  475. def _is_bambu_uuid(tray_uuid: str) -> bool:
  476. """Check if a tray UUID looks like a valid Bambu Lab RFID UUID (non-empty, non-zero)."""
  477. return bool(tray_uuid) and tray_uuid not in ("", "0" * len(tray_uuid))
  478. async def on_ams_change(printer_id: int, ams_data: list):
  479. """Handle AMS data changes - sync to Spoolman if enabled and auto mode."""
  480. logger = logging.getLogger(__name__)
  481. # MQTT relay - publish AMS change
  482. try:
  483. printer_info = printer_manager.get_printer(printer_id)
  484. if printer_info:
  485. await mqtt_relay.on_ams_change(printer_id, printer_info.name, printer_info.serial_number, ams_data)
  486. except Exception:
  487. pass # Don't fail AMS callback if MQTT fails
  488. # Broadcast AMS change via WebSocket (bypasses status_key deduplication)
  489. # This ensures frontend gets immediate updates when AMS slots are configured
  490. try:
  491. state = printer_manager.get_status(printer_id)
  492. if state:
  493. logger.info("[Printer %s] Broadcasting AMS change via WebSocket", printer_id)
  494. await ws_manager.send_printer_status(
  495. printer_id,
  496. printer_state_to_dict(state, printer_id, printer_manager.get_model(printer_id)),
  497. )
  498. except Exception as e:
  499. logger.warning("Failed to broadcast AMS change for printer %s: %s", printer_id, e)
  500. from backend.app.utils.color_utils import colors_similar as _colors_similar
  501. # Auto-unlink spool assignments with stale fingerprints
  502. try:
  503. async with async_session() as db:
  504. from sqlalchemy.orm import selectinload
  505. from backend.app.api.routes.inventory import _find_tray_in_ams_data
  506. from backend.app.models.spool_assignment import SpoolAssignment as SA
  507. result = await db.execute(select(SA).where(SA.printer_id == printer_id).options(selectinload(SA.spool)))
  508. stale = []
  509. for assignment in result.scalars().all():
  510. # External spool assignments (ams_id=255) live in vt_tray, not AMS data
  511. if assignment.ams_id == 255:
  512. ps = printer_manager.get_status(printer_id)
  513. vt_tray_raw = ps.raw_data.get("vt_tray", []) if ps else []
  514. ext_id = assignment.tray_id + 254 # 0→254, 1→255
  515. current_tray = None
  516. for vt in vt_tray_raw:
  517. if isinstance(vt, dict) and int(vt.get("id", 254)) == ext_id:
  518. current_tray = vt
  519. break
  520. if not current_tray:
  521. # vt_tray data may not have arrived yet — keep assignment
  522. continue
  523. else:
  524. current_tray = _find_tray_in_ams_data(ams_data, assignment.ams_id, assignment.tray_id)
  525. if not current_tray:
  526. logger.info(
  527. "Auto-unlink: spool %d AMS%d-T%d — tray not found in AMS data (slot empty?)",
  528. assignment.spool_id,
  529. assignment.ams_id,
  530. assignment.tray_id,
  531. )
  532. stale.append(assignment) # Slot empty
  533. elif _is_bambu_uuid(current_tray.get("tray_uuid", "")):
  534. # A Bambu Lab spool is in this slot — check if it's the same spool
  535. # that's currently assigned. If yes, keep the assignment (avoids
  536. # unnecessary unlink/re-assign/ams_filament_setting cycle that clears
  537. # the printer's filament preset on every startup).
  538. tray_uuid = current_tray.get("tray_uuid", "")
  539. tag_uid = current_tray.get("tag_uid", "")
  540. spool = assignment.spool
  541. spool_matches = False
  542. if spool:
  543. if (spool.tray_uuid and spool.tray_uuid.upper() == tray_uuid.upper()) or (
  544. spool.tag_uid
  545. and tag_uid
  546. and tag_uid != "0000000000000000"
  547. and spool.tag_uid.upper() == tag_uid.upper()
  548. ):
  549. spool_matches = True
  550. if spool_matches:
  551. # Same BL spool still in slot — keep assignment, update fingerprint if needed
  552. cur_color = current_tray.get("tray_color", "")
  553. cur_type = current_tray.get("tray_type", "")
  554. fp_color = assignment.fingerprint_color or ""
  555. fp_type = assignment.fingerprint_type or ""
  556. if cur_color.upper() != fp_color.upper() or cur_type.upper() != fp_type.upper():
  557. assignment.fingerprint_color = cur_color
  558. assignment.fingerprint_type = cur_type
  559. logger.debug(
  560. "Auto-unlink: spool %d AMS%d-T%d — same BL spool, updated fingerprint",
  561. assignment.spool_id,
  562. assignment.ams_id,
  563. assignment.tray_id,
  564. )
  565. continue
  566. # Different BL spool or unrecognized — unlink so auto-assign can match
  567. logger.info(
  568. "Auto-unlink: spool %d AMS%d-T%d — different Bambu Lab spool detected (uuid=%s)",
  569. assignment.spool_id,
  570. assignment.ams_id,
  571. assignment.tray_id,
  572. tray_uuid,
  573. )
  574. stale.append(assignment)
  575. else:
  576. cur_color = current_tray.get("tray_color", "")
  577. cur_type = current_tray.get("tray_type", "")
  578. fp_color = assignment.fingerprint_color or ""
  579. fp_type = assignment.fingerprint_type or ""
  580. if not _colors_similar(cur_color, fp_color) or cur_type.upper() != fp_type.upper():
  581. # Fingerprint mismatch — but check if tray now matches the
  582. # assigned spool (e.g. auto-configure changed the tray).
  583. spool = assignment.spool
  584. if spool:
  585. spool_color = (spool.rgba or "FFFFFFFF").upper()
  586. spool_type = (spool.material or "").upper()
  587. if _colors_similar(cur_color, spool_color) and cur_type.upper() == spool_type:
  588. # Tray was reconfigured to match the spool — update fingerprint
  589. logger.info(
  590. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch but tray matches spool, updating fp",
  591. assignment.spool_id,
  592. assignment.ams_id,
  593. assignment.tray_id,
  594. )
  595. assignment.fingerprint_color = cur_color
  596. assignment.fingerprint_type = cur_type
  597. continue
  598. logger.info(
  599. "Auto-unlink: spool %d AMS%d-T%d — fingerprint mismatch (cur=%s/%s fp=%s/%s spool=%s/%s)",
  600. assignment.spool_id,
  601. assignment.ams_id,
  602. assignment.tray_id,
  603. cur_color,
  604. cur_type,
  605. fp_color,
  606. fp_type,
  607. spool.rgba if spool else "?",
  608. spool.material if spool else "?",
  609. )
  610. stale.append(assignment) # Spool changed
  611. for a in stale:
  612. await db.delete(a)
  613. if stale:
  614. logger.info("Auto-unlinked %d stale spool assignments for printer %d", len(stale), printer_id)
  615. # Commit any changes (stale deletions and/or fingerprint updates)
  616. await db.commit()
  617. except Exception as e:
  618. logger.warning("Spool assignment cleanup failed: %s", e, exc_info=True)
  619. # Auto-manage inventory spools from AMS tray data (skip if Spoolman manages AMS)
  620. try:
  621. async with async_session() as db:
  622. from backend.app.api.routes.settings import get_setting
  623. from backend.app.models.spool_assignment import SpoolAssignment as SA
  624. from backend.app.services.spool_tag_matcher import (
  625. auto_assign_spool,
  626. create_spool_from_tray,
  627. get_spool_by_tag,
  628. is_bambu_tag,
  629. is_valid_tag,
  630. )
  631. _spoolman_on = await get_setting(db, "spoolman_enabled")
  632. if not _spoolman_on or _spoolman_on.lower() != "true":
  633. for ams_unit in ams_data:
  634. if not isinstance(ams_unit, dict):
  635. continue
  636. ams_id = int(ams_unit.get("id", 0))
  637. for tray in ams_unit.get("tray", []):
  638. if not isinstance(tray, dict):
  639. continue
  640. tray_id = int(tray.get("id", 0))
  641. tag_uid = tray.get("tag_uid", "")
  642. tray_uuid = tray.get("tray_uuid", "")
  643. tray_info_idx = tray.get("tray_info_idx", "")
  644. if not tray.get("tray_type"):
  645. continue # Empty slot
  646. # Check if assignment already exists for this slot
  647. existing = await db.execute(
  648. select(SA)
  649. .options(selectinload(SA.spool))
  650. .where(SA.printer_id == printer_id, SA.ams_id == ams_id, SA.tray_id == tray_id)
  651. )
  652. existing_assignment = existing.scalar_one_or_none()
  653. if existing_assignment:
  654. # Sync spool weight_used from AMS remain — only INCREASE, never decrease.
  655. # The AMS remain% is low-resolution (integer %, i.e. 10g steps for 1kg spool)
  656. # and must not overwrite precise values from the usage tracker (3MF/G-code).
  657. remain_raw = tray.get("remain")
  658. if (
  659. remain_raw is not None
  660. and existing_assignment.spool
  661. and not existing_assignment.spool.weight_locked
  662. ):
  663. try:
  664. remain_val = int(remain_raw)
  665. except (TypeError, ValueError):
  666. remain_val = -1
  667. if 1 <= remain_val <= 100:
  668. lw = existing_assignment.spool.label_weight or 1000
  669. new_used = round(lw * (100 - remain_val) / 100.0, 1)
  670. current_used = existing_assignment.spool.weight_used or 0
  671. if new_used > current_used + 1:
  672. logger.info(
  673. "Weight sync: spool %d weight_used %s -> %s (remain=%d)",
  674. existing_assignment.spool_id,
  675. current_used,
  676. new_used,
  677. remain_val,
  678. )
  679. existing_assignment.spool.weight_used = new_used
  680. await db.commit()
  681. continue
  682. if is_bambu_tag(tag_uid, tray_uuid, tray_info_idx):
  683. # BL spool with RFID tag: auto-match or auto-create
  684. spool = await get_spool_by_tag(db, tag_uid, tray_uuid)
  685. if not spool:
  686. spool = await create_spool_from_tray(db, tray)
  687. await auto_assign_spool(
  688. printer_id,
  689. ams_id,
  690. tray_id,
  691. spool,
  692. printer_manager,
  693. db,
  694. tray_info_idx=tray_info_idx,
  695. )
  696. await db.commit()
  697. await ws_manager.broadcast(
  698. {
  699. "type": "spool_auto_assigned",
  700. "printer_id": printer_id,
  701. "ams_id": ams_id,
  702. "tray_id": tray_id,
  703. "spool_id": spool.id,
  704. }
  705. )
  706. logger.info(
  707. "RFID auto-assigned spool %d to printer %d AMS%d-T%d",
  708. spool.id,
  709. printer_id,
  710. ams_id,
  711. tray_id,
  712. )
  713. elif is_valid_tag(tag_uid, tray_uuid):
  714. # Non-BL spool with some tag — let user choose
  715. await ws_manager.broadcast(
  716. {
  717. "type": "unknown_tag",
  718. "printer_id": printer_id,
  719. "ams_id": ams_id,
  720. "tray_id": tray_id,
  721. "tag_uid": tag_uid,
  722. "tray_uuid": tray_uuid,
  723. }
  724. )
  725. else:
  726. # No tag at all — let user choose from inventory
  727. await ws_manager.broadcast(
  728. {
  729. "type": "unknown_tag",
  730. "printer_id": printer_id,
  731. "ams_id": ams_id,
  732. "tray_id": tray_id,
  733. "tag_uid": "",
  734. "tray_uuid": "",
  735. }
  736. )
  737. except Exception as e:
  738. logger.warning("RFID spool auto-assign failed: %s", e, exc_info=True)
  739. try:
  740. async with async_session() as db:
  741. from backend.app.api.routes.settings import get_setting
  742. from backend.app.models.printer import Printer
  743. # Check if Spoolman is enabled
  744. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  745. if not spoolman_enabled or spoolman_enabled.lower() != "true":
  746. return
  747. # Check sync mode
  748. sync_mode = await get_setting(db, "spoolman_sync_mode")
  749. if sync_mode and sync_mode != "auto":
  750. return # Only sync on auto mode
  751. # Check if weight sync is disabled
  752. disable_weight_sync_str = await get_setting(db, "spoolman_disable_weight_sync")
  753. disable_weight_sync = disable_weight_sync_str and disable_weight_sync_str.lower() == "true"
  754. # Get Spoolman URL
  755. spoolman_url = await get_setting(db, "spoolman_url")
  756. if not spoolman_url:
  757. return
  758. # Get or create Spoolman client
  759. client = await get_spoolman_client()
  760. if not client:
  761. client = await init_spoolman_client(spoolman_url)
  762. # Check if Spoolman is reachable
  763. if not await client.health_check():
  764. logger.warning("Spoolman not reachable at %s", spoolman_url)
  765. return
  766. # Get printer name for location
  767. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  768. printer = result.scalar_one_or_none()
  769. printer_name = printer.name if printer else f"Printer {printer_id}"
  770. # OPTIMIZATION: Fetch all spools once before processing trays
  771. # This eliminates redundant API calls (one per tray) when syncing multiple trays
  772. logger.debug("[Printer %s] Fetching spools cache for AMS sync...", printer_id)
  773. try:
  774. cached_spools = await client.get_spools()
  775. logger.debug("[Printer %s] Cached %d spools for batch sync", printer_id, len(cached_spools))
  776. except Exception as e:
  777. logger.error(
  778. "[Printer %s] Failed to fetch spools cache after retries, aborting AMS sync: %s",
  779. printer_id,
  780. e,
  781. )
  782. return
  783. # Load inventory weights as fallback (when AMS MQTT data lacks remain values)
  784. from sqlalchemy.orm import selectinload
  785. from backend.app.models.spool_assignment import SpoolAssignment
  786. inventory_weights: dict[tuple[int, int], float] = {}
  787. try:
  788. assign_result = await db.execute(
  789. select(SpoolAssignment)
  790. .options(selectinload(SpoolAssignment.spool))
  791. .where(SpoolAssignment.printer_id == printer_id)
  792. )
  793. for assignment in assign_result.scalars().all():
  794. spool = assignment.spool
  795. if spool and spool.label_weight > 0:
  796. remaining = max(0.0, spool.label_weight - (spool.weight_used or 0))
  797. inventory_weights[(assignment.ams_id, assignment.tray_id)] = remaining
  798. except Exception as e:
  799. logger.debug("Could not load inventory weights for printer %s: %s", printer_id, e)
  800. # Sync each AMS tray
  801. synced = 0
  802. for ams_unit in ams_data:
  803. ams_id = int(ams_unit.get("id", 0))
  804. trays = ams_unit.get("tray", [])
  805. for tray_data in trays:
  806. tray = client.parse_ams_tray(ams_id, tray_data)
  807. if not tray:
  808. continue # Empty tray
  809. try:
  810. inv_remaining = inventory_weights.get((ams_id, tray.tray_id))
  811. result = await client.sync_ams_tray(
  812. tray,
  813. printer_name,
  814. disable_weight_sync=disable_weight_sync,
  815. cached_spools=cached_spools,
  816. inventory_remaining=inv_remaining,
  817. )
  818. if result:
  819. synced += 1
  820. # If a new spool was created, add it to the cache
  821. # so subsequent trays can find it if they reference the same tag
  822. if result.get("id"):
  823. # Check if this spool already exists in cache
  824. spool_exists = any(s.get("id") == result["id"] for s in cached_spools)
  825. if not spool_exists:
  826. cached_spools.append(result)
  827. logger.debug(
  828. "[Printer %s] Added newly created spool %s to cache",
  829. printer_id,
  830. result["id"],
  831. )
  832. except Exception as e:
  833. logger.error("Error syncing AMS %s tray %s: %s", ams_id, tray.tray_id, e)
  834. if synced > 0:
  835. logger.info("Auto-synced %s AMS trays to Spoolman for printer %s", synced, printer_id)
  836. except Exception as e:
  837. logging.getLogger(__name__).warning(f"Spoolman AMS sync failed: {e}")
  838. async def _capture_snapshot_for_notification(printer_id: int, printer, logger) -> bytes | None:
  839. """Capture a camera snapshot for notification image attachment.
  840. Returns JPEG bytes (max 2.5MB) or None if capture fails or is unavailable.
  841. Uses: external camera > buffered frame > fresh capture.
  842. """
  843. if not printer:
  844. return None
  845. try:
  846. from backend.app.api.routes.settings import get_setting
  847. async with async_session() as db:
  848. capture_enabled = await get_setting(db, "capture_finish_photo")
  849. if capture_enabled is not None and capture_enabled.lower() != "true":
  850. return None
  851. # Try external camera first
  852. if printer.external_camera_enabled and printer.external_camera_url:
  853. logger.info("[SNAPSHOT] Capturing from external camera for printer %s", printer_id)
  854. from backend.app.services.external_camera import capture_frame
  855. frame_data = await capture_frame(printer.external_camera_url, printer.external_camera_type or "mjpeg")
  856. if frame_data and len(frame_data) <= 2_500_000:
  857. logger.info("[SNAPSHOT] External camera frame: %s bytes", len(frame_data))
  858. return frame_data
  859. # Try buffered frame from active stream
  860. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  861. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  862. active_chamber = [k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")]
  863. buffered_frame = get_buffered_frame(printer_id)
  864. if (active_for_printer or active_chamber) and buffered_frame:
  865. logger.info("[SNAPSHOT] Using buffered frame for printer %s: %s bytes", printer_id, len(buffered_frame))
  866. if len(buffered_frame) <= 2_500_000:
  867. return buffered_frame
  868. # Fresh capture from printer camera
  869. logger.info("[SNAPSHOT] Capturing fresh frame for printer %s", printer_id)
  870. from backend.app.services.camera import capture_camera_frame_bytes
  871. frame_data = await capture_camera_frame_bytes(
  872. printer.ip_address, printer.access_code, printer.model, timeout=15
  873. )
  874. if frame_data and len(frame_data) <= 2_500_000:
  875. logger.info("[SNAPSHOT] Fresh camera frame: %s bytes", len(frame_data))
  876. return frame_data
  877. except Exception as e:
  878. logger.warning("[SNAPSHOT] Failed to capture snapshot for printer %s: %s", printer_id, e)
  879. return None
  880. async def _send_print_start_notification(
  881. printer_id: int,
  882. data: dict,
  883. archive_data: dict | None = None,
  884. logger=None,
  885. ):
  886. """Helper to send print start notification with optional archive data."""
  887. if logger is None:
  888. logger = logging.getLogger(__name__)
  889. try:
  890. async with async_session() as db:
  891. from backend.app.models.printer import Printer
  892. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  893. printer = result.scalar_one_or_none()
  894. printer_name = printer.name if printer else f"Printer {printer_id}"
  895. # Capture camera snapshot for notification image attachment
  896. image_data = await _capture_snapshot_for_notification(printer_id, printer, logger)
  897. if image_data:
  898. if archive_data is None:
  899. archive_data = {}
  900. archive_data["image_data"] = image_data
  901. await notification_service.on_print_start(printer_id, printer_name, data, db, archive_data=archive_data)
  902. except Exception as e:
  903. logger.warning("Notification on_print_start failed: %s", e)
  904. def _load_objects_from_archive(archive, printer_id: int, logger) -> None:
  905. """Extract printable objects from an archive's 3MF file and store in printer state."""
  906. try:
  907. from backend.app.services.archive import extract_printable_objects_from_3mf
  908. file_path = app_settings.base_dir / archive.file_path
  909. if file_path.is_file() and str(file_path).endswith(".3mf"):
  910. with open(file_path, "rb") as f:
  911. threemf_data = f.read()
  912. # Extract with positions for UI overlay
  913. printable_objects, bbox_all = extract_printable_objects_from_3mf(threemf_data, include_positions=True)
  914. if printable_objects:
  915. client = printer_manager.get_client(printer_id)
  916. if client:
  917. client.state.printable_objects = printable_objects
  918. client.state.printable_objects_bbox_all = bbox_all
  919. client.state.skipped_objects = []
  920. logger.info("Loaded %s printable objects for printer %s", len(printable_objects), printer_id)
  921. except Exception as e:
  922. logger.debug("Failed to extract printable objects from archive: %s", e)
  923. async def on_print_start(printer_id: int, data: dict):
  924. """Handle print start - archive the 3MF file immediately."""
  925. logger = logging.getLogger(__name__)
  926. logger.info("[CALLBACK] on_print_start called for printer %s, data keys: %s", printer_id, list(data.keys()))
  927. # Cancel any active bed cooldown task for this printer
  928. existing_task = _bed_cooldown_tasks.pop(printer_id, None)
  929. if existing_task and not existing_task.done():
  930. existing_task.cancel()
  931. logger.info("[BED-COOL] Cancelled bed cooldown monitor for printer %s (new print started)", printer_id)
  932. # Clear cached cover images so the new print's thumbnail is fetched fresh
  933. from backend.app.api.routes.printers import clear_cover_cache
  934. clear_cover_cache(printer_id)
  935. await ws_manager.send_print_start(printer_id, data)
  936. # MQTT relay - publish print start
  937. try:
  938. printer_info = printer_manager.get_printer(printer_id)
  939. if printer_info:
  940. await mqtt_relay.on_print_start(
  941. printer_id,
  942. printer_info.name,
  943. printer_info.serial_number,
  944. data.get("filename", ""),
  945. data.get("subtask_name", ""),
  946. )
  947. except Exception:
  948. pass # Don't fail print start callback if MQTT fails
  949. # Capture AMS tray remain% for filament consumption tracking (skip if Spoolman handles usage)
  950. try:
  951. async with async_session() as db:
  952. from backend.app.api.routes.settings import get_setting
  953. _spoolman_on = await get_setting(db, "spoolman_enabled")
  954. if not _spoolman_on or _spoolman_on.lower() != "true":
  955. from backend.app.services.usage_tracker import on_print_start as usage_on_print_start
  956. await usage_on_print_start(printer_id, data, printer_manager, db=db)
  957. except Exception as e:
  958. logger.warning("Usage tracker on_print_start failed: %s", e)
  959. # Track if notification was sent (to avoid sending twice)
  960. notification_sent = False
  961. # Smart plug automation: turn on plug when print starts
  962. try:
  963. async with async_session() as db:
  964. await smart_plug_manager.on_print_start(printer_id, db)
  965. except Exception as e:
  966. logger.warning("Smart plug on_print_start failed: %s", e)
  967. async with async_session() as db:
  968. from backend.app.models.printer import Printer
  969. from backend.app.services.bambu_ftp import list_files_async
  970. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  971. printer = result.scalar_one_or_none()
  972. # Plate detection check - pause if objects detected on build plate
  973. logger.info(
  974. f"[PLATE CHECK] printer_id={printer_id}, plate_detection_enabled={printer.plate_detection_enabled if printer else 'NO PRINTER'}"
  975. )
  976. if printer and printer.plate_detection_enabled:
  977. logger.info("[PLATE CHECK] ENTERING plate detection code for printer %s", printer_id)
  978. try:
  979. from backend.app.services.plate_detection import check_plate_empty
  980. # Build ROI tuple from printer settings if available
  981. roi = None
  982. if all(
  983. [
  984. printer.plate_detection_roi_x is not None,
  985. printer.plate_detection_roi_y is not None,
  986. printer.plate_detection_roi_w is not None,
  987. printer.plate_detection_roi_h is not None,
  988. ]
  989. ):
  990. roi = (
  991. printer.plate_detection_roi_x,
  992. printer.plate_detection_roi_y,
  993. printer.plate_detection_roi_w,
  994. printer.plate_detection_roi_h,
  995. )
  996. # Auto-turn on chamber light if it's off for better detection
  997. light_was_off = False
  998. client = printer_manager.get_client(printer_id)
  999. if client and client.state:
  1000. light_was_off = not client.state.chamber_light
  1001. if light_was_off:
  1002. logger.info("[PLATE CHECK] Turning on chamber light for printer %s", printer_id)
  1003. client.set_chamber_light(True)
  1004. # Wait for light to physically turn on and camera to adjust exposure
  1005. await asyncio.sleep(2.5)
  1006. logger.info("[PLATE CHECK] Running plate detection for printer %s", printer_id)
  1007. plate_result = await check_plate_empty(
  1008. printer_id=printer_id,
  1009. ip_address=printer.ip_address,
  1010. access_code=printer.access_code,
  1011. model=printer.model,
  1012. include_debug_image=False,
  1013. external_camera_url=printer.external_camera_url,
  1014. external_camera_type=printer.external_camera_type,
  1015. use_external=printer.external_camera_enabled,
  1016. roi=roi,
  1017. )
  1018. # Restore chamber light to original state
  1019. if light_was_off and client:
  1020. logger.info("[PLATE CHECK] Restoring chamber light to off for printer %s", printer_id)
  1021. client.set_chamber_light(False)
  1022. if not plate_result.needs_calibration and not plate_result.is_empty:
  1023. # Objects detected - pause the print!
  1024. logger.warning(
  1025. f"[PLATE CHECK] Objects detected on plate for printer {printer_id}! "
  1026. f"Confidence: {plate_result.confidence:.0%}, Diff: {plate_result.difference_percent:.1f}%"
  1027. )
  1028. client = printer_manager.get_client(printer_id)
  1029. if client:
  1030. client.pause_print()
  1031. logger.info("[PLATE CHECK] Print paused for printer %s", printer_id)
  1032. # Send notification about plate not empty
  1033. await ws_manager.broadcast(
  1034. {
  1035. "type": "plate_not_empty",
  1036. "printer_id": printer_id,
  1037. "printer_name": printer.name,
  1038. "message": f"Objects detected on build plate! Print paused. (Diff: {plate_result.difference_percent:.1f}%)",
  1039. }
  1040. )
  1041. # Also send push notification
  1042. try:
  1043. await notification_service.on_plate_not_empty(
  1044. printer_id=printer_id,
  1045. printer_name=printer.name,
  1046. db=db,
  1047. difference_percent=plate_result.difference_percent,
  1048. )
  1049. except Exception as notif_err:
  1050. logger.warning("[PLATE CHECK] Failed to send notification: %s", notif_err)
  1051. else:
  1052. logger.info("[PLATE CHECK] Plate is empty for printer %s, proceeding with print", printer_id)
  1053. except Exception as plate_err:
  1054. # Don't block print on plate detection errors
  1055. logger.warning("[PLATE CHECK] Plate detection failed for printer %s: %s", printer_id, plate_err)
  1056. if not printer or not printer.auto_archive:
  1057. # Send notification without archive data (auto-archive disabled)
  1058. logger.info(
  1059. f"[CALLBACK] Skipping archive - printer: {printer is not None}, auto_archive: {printer.auto_archive if printer else 'N/A'}"
  1060. )
  1061. if not notification_sent:
  1062. await _send_print_start_notification(printer_id, data, logger=logger)
  1063. return
  1064. # Get the filename and subtask_name
  1065. filename = data.get("filename", "")
  1066. subtask_name = data.get("subtask_name", "")
  1067. logger.info("[CALLBACK] Print start detected - filename: %s, subtask: %s", filename, subtask_name)
  1068. # Skip calibration prints — internal printer files should not be archived
  1069. # Bambu calibration gcode lives under /usr/ (e.g. /usr/etc/print/auto_cali_for_user.gcode)
  1070. if filename and filename.startswith("/usr/"):
  1071. logger.info("[CALLBACK] Skipping archive — internal printer file detected: %s", filename)
  1072. if not notification_sent:
  1073. await _send_print_start_notification(printer_id, data, logger=logger)
  1074. return
  1075. if not filename and not subtask_name:
  1076. # Send notification without archive data (no filename)
  1077. logger.info("[CALLBACK] Skipping archive - no filename or subtask_name")
  1078. if not notification_sent:
  1079. await _send_print_start_notification(printer_id, data, logger=logger)
  1080. return
  1081. # Check if this is an expected print from reprint/scheduled
  1082. # Build list of possible keys to check
  1083. expected_keys = []
  1084. if subtask_name:
  1085. expected_keys.append((printer_id, subtask_name))
  1086. expected_keys.append((printer_id, f"{subtask_name}.3mf"))
  1087. expected_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  1088. if filename:
  1089. fname = filename.split("/")[-1] if "/" in filename else filename
  1090. expected_keys.append((printer_id, fname))
  1091. # Strip extensions to match
  1092. base = fname.replace(".gcode", "").replace(".3mf", "")
  1093. expected_keys.append((printer_id, base))
  1094. expected_keys.append((printer_id, f"{base}.3mf"))
  1095. expected_archive_id = None
  1096. for key in expected_keys:
  1097. expected_archive_id = _expected_prints.pop(key, None)
  1098. if expected_archive_id:
  1099. # Clean up other possible keys for this print
  1100. for other_key in expected_keys:
  1101. _expected_prints.pop(other_key, None)
  1102. break
  1103. if expected_archive_id:
  1104. # This is a reprint/scheduled print - use existing archive, don't create new one
  1105. logger.info("Using expected archive %s for print (skipping duplicate)", expected_archive_id)
  1106. from backend.app.models.archive import PrintArchive
  1107. result = await db.execute(select(PrintArchive).where(PrintArchive.id == expected_archive_id))
  1108. archive = result.scalar_one_or_none()
  1109. if archive:
  1110. # Update archive status to printing
  1111. archive.status = "printing"
  1112. archive.started_at = datetime.now(timezone.utc)
  1113. await db.commit()
  1114. # Track as active print
  1115. _active_prints[(printer_id, archive.filename)] = archive.id
  1116. if subtask_name:
  1117. _active_prints[(printer_id, f"{subtask_name}.3mf")] = archive.id
  1118. # Set up energy tracking
  1119. try:
  1120. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  1121. plug = plug_result.scalar_one_or_none()
  1122. logger.info(
  1123. f"[ENERGY] Print start - archive {archive.id}, printer {printer_id}, plug found: {plug is not None}"
  1124. )
  1125. if plug:
  1126. energy = await _get_plug_energy(plug, db)
  1127. logger.info("[ENERGY] Energy response from plug: %s", energy)
  1128. if energy and energy.get("total") is not None:
  1129. _print_energy_start[archive.id] = energy["total"]
  1130. logger.info(
  1131. f"[ENERGY] Recorded starting energy for archive {archive.id}: {energy['total']} kWh"
  1132. )
  1133. else:
  1134. logger.warning("[ENERGY] No 'total' in energy response for archive %s", archive.id)
  1135. else:
  1136. logger.info("[ENERGY] No smart plug found for printer %s", printer_id)
  1137. except Exception as e:
  1138. logger.warning("Failed to record starting energy: %s", e)
  1139. await ws_manager.send_archive_updated(
  1140. {
  1141. "id": archive.id,
  1142. "status": "printing",
  1143. }
  1144. )
  1145. # Send notification with archive data (reprint/scheduled)
  1146. if not notification_sent:
  1147. archive_data = {"print_time_seconds": archive.print_time_seconds}
  1148. await _send_print_start_notification(printer_id, data, archive_data, logger)
  1149. # Extract printable objects from the archived 3MF file
  1150. _load_objects_from_archive(archive, printer_id, logger)
  1151. # Store Spoolman tracking data for per-filament usage reporting
  1152. try:
  1153. await _store_spoolman_print_data(
  1154. printer_id,
  1155. archive.id,
  1156. archive.file_path,
  1157. db,
  1158. printer_manager,
  1159. ams_mapping=_get_start_ams_mapping(data, archive.id),
  1160. )
  1161. except Exception as e:
  1162. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  1163. return # Skip creating a new archive
  1164. # Check if there's already a "printing" archive for this printer/file
  1165. # This prevents duplicates when backend restarts during an active print
  1166. from backend.app.models.archive import PrintArchive
  1167. check_name = subtask_name or filename.split("/")[-1].replace(".gcode", "").replace(".3mf", "")
  1168. existing = await db.execute(
  1169. select(PrintArchive)
  1170. .where(PrintArchive.printer_id == printer_id)
  1171. .where(PrintArchive.status == "printing")
  1172. .where(
  1173. or_(
  1174. PrintArchive.print_name == check_name,
  1175. PrintArchive.filename.in_(
  1176. [
  1177. f"{check_name}.3mf",
  1178. f"{check_name}.gcode.3mf",
  1179. ]
  1180. ),
  1181. )
  1182. )
  1183. .order_by(PrintArchive.created_at.desc())
  1184. .limit(1)
  1185. )
  1186. existing_archive = existing.scalar_one_or_none()
  1187. if existing_archive:
  1188. # Check if archive is stale (older than 4 hours) - likely a failed/cancelled print
  1189. # that didn't get properly updated
  1190. archive_age = datetime.now(timezone.utc) - existing_archive.created_at.replace(tzinfo=timezone.utc)
  1191. if archive_age.total_seconds() > 4 * 60 * 60: # 4 hours
  1192. logger.warning(
  1193. f"Found stale 'printing' archive {existing_archive.id} (age: {archive_age}), "
  1194. f"marking as cancelled and creating new archive"
  1195. )
  1196. existing_archive.status = "cancelled"
  1197. existing_archive.failure_reason = "Stale - print likely cancelled or failed without status update"
  1198. await db.commit()
  1199. # Fall through to create new archive (don't return)
  1200. _existing_archive = None # Clear so we don't use stale archive
  1201. else:
  1202. logger.info(
  1203. f"Skipping duplicate - already have printing archive {existing_archive.id} for {check_name}"
  1204. )
  1205. # Track this as the active print
  1206. _active_prints[(printer_id, existing_archive.filename)] = existing_archive.id
  1207. # Also set up energy tracking if not already tracked
  1208. if existing_archive.id not in _print_energy_start:
  1209. try:
  1210. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  1211. plug = plug_result.scalar_one_or_none()
  1212. if plug:
  1213. energy = await _get_plug_energy(plug, db)
  1214. if energy and energy.get("total") is not None:
  1215. _print_energy_start[existing_archive.id] = energy["total"]
  1216. logger.info(
  1217. f"Recorded starting energy for existing archive {existing_archive.id}: {energy['total']} kWh"
  1218. )
  1219. except Exception as e:
  1220. logger.warning("Failed to record starting energy for existing archive: %s", e)
  1221. # Send notification with archive data (existing archive)
  1222. if not notification_sent:
  1223. archive_data = {"print_time_seconds": existing_archive.print_time_seconds}
  1224. await _send_print_start_notification(printer_id, data, archive_data, logger)
  1225. # Extract printable objects from the archived 3MF file
  1226. _load_objects_from_archive(existing_archive, printer_id, logger)
  1227. return
  1228. # Build list of possible 3MF filenames to try
  1229. possible_names = []
  1230. # Bambu printers typically store files as "Name.gcode.3mf"
  1231. # The subtask_name is usually the best source for the filename
  1232. if subtask_name:
  1233. # Try common Bambu naming patterns
  1234. possible_names.append(f"{subtask_name}.gcode.3mf")
  1235. possible_names.append(f"{subtask_name}.3mf")
  1236. # Try original filename with .3mf extension
  1237. if filename:
  1238. # Extract just the filename part, not the full path
  1239. fname = filename.split("/")[-1] if "/" in filename else filename
  1240. if fname.endswith(".3mf"):
  1241. possible_names.append(fname)
  1242. elif fname.endswith(".gcode"):
  1243. base = fname.rsplit(".", 1)[0]
  1244. possible_names.append(f"{base}.gcode.3mf")
  1245. possible_names.append(f"{base}.3mf")
  1246. else:
  1247. possible_names.append(f"{fname}.gcode.3mf")
  1248. possible_names.append(f"{fname}.3mf")
  1249. # Also try with spaces converted to underscores (Bambu Studio may normalize filenames)
  1250. space_variants = []
  1251. for name in possible_names:
  1252. if " " in name:
  1253. space_variants.append(name.replace(" ", "_"))
  1254. possible_names.extend(space_variants)
  1255. # Remove duplicates while preserving order
  1256. seen = set()
  1257. possible_names = [x for x in possible_names if not (x in seen or seen.add(x))]
  1258. logger.info("Trying filenames: %s", possible_names)
  1259. # Try to find and download the 3MF file
  1260. temp_path = None
  1261. downloaded_filename = None
  1262. # Get FTP retry settings
  1263. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  1264. for try_filename in possible_names:
  1265. if not try_filename.endswith(".3mf"):
  1266. continue
  1267. remote_paths = [
  1268. f"/cache/{try_filename}",
  1269. f"/model/{try_filename}",
  1270. f"/data/{try_filename}",
  1271. f"/data/Metadata/{try_filename}",
  1272. f"/{try_filename}",
  1273. ]
  1274. temp_path = app_settings.archive_dir / "temp" / try_filename
  1275. temp_path.parent.mkdir(parents=True, exist_ok=True)
  1276. for remote_path in remote_paths:
  1277. logger.debug("Trying FTP download: %s", remote_path)
  1278. try:
  1279. if ftp_retry_enabled:
  1280. downloaded = await with_ftp_retry(
  1281. download_file_async,
  1282. printer.ip_address,
  1283. printer.access_code,
  1284. remote_path,
  1285. temp_path,
  1286. socket_timeout=ftp_timeout,
  1287. printer_model=printer.model,
  1288. max_retries=ftp_retry_count,
  1289. retry_delay=ftp_retry_delay,
  1290. operation_name=f"Download 3MF from {remote_path}",
  1291. )
  1292. else:
  1293. downloaded = await download_file_async(
  1294. printer.ip_address,
  1295. printer.access_code,
  1296. remote_path,
  1297. temp_path,
  1298. socket_timeout=ftp_timeout,
  1299. printer_model=printer.model,
  1300. )
  1301. if downloaded:
  1302. downloaded_filename = try_filename
  1303. logger.info("Downloaded: %s", remote_path)
  1304. break
  1305. except Exception as e:
  1306. logger.debug("FTP download failed for %s: %s", remote_path, e)
  1307. if downloaded_filename:
  1308. break
  1309. # If still not found, try listing directories to find matching file
  1310. # Different printer models use different directory structures
  1311. if not downloaded_filename and (filename or subtask_name):
  1312. search_term = (subtask_name or filename).lower().replace(".gcode", "").replace(".3mf", "")
  1313. logger.info("Direct FTP download failed, searching directories for '%s'", search_term)
  1314. search_dirs = ["/cache", "/model", "/data", "/data/Metadata", "/"]
  1315. for search_dir in search_dirs:
  1316. if downloaded_filename:
  1317. break
  1318. try:
  1319. dir_files = await list_files_async(
  1320. printer.ip_address, printer.access_code, search_dir, printer_model=printer.model
  1321. )
  1322. threemf_files = [f.get("name") for f in dir_files if f.get("name", "").endswith(".3mf")]
  1323. if threemf_files:
  1324. logger.info(
  1325. f"Found {len(threemf_files)} 3MF files in {search_dir}: {threemf_files[:5]}{'...' if len(threemf_files) > 5 else ''}"
  1326. )
  1327. for f in dir_files:
  1328. if f.get("is_directory"):
  1329. continue
  1330. fname = f.get("name", "")
  1331. # Normalize both for comparison (spaces and underscores are equivalent)
  1332. fname_normalized = fname.lower().replace(" ", "_")
  1333. search_normalized = search_term.replace(" ", "_")
  1334. if fname.endswith(".3mf") and search_normalized in fname_normalized:
  1335. logger.info("Found matching file in %s: %s", search_dir, fname)
  1336. temp_path = app_settings.archive_dir / "temp" / fname
  1337. temp_path.parent.mkdir(parents=True, exist_ok=True)
  1338. if ftp_retry_enabled:
  1339. downloaded = await with_ftp_retry(
  1340. download_file_async,
  1341. printer.ip_address,
  1342. printer.access_code,
  1343. f"{search_dir}/{fname}",
  1344. temp_path,
  1345. socket_timeout=ftp_timeout,
  1346. printer_model=printer.model,
  1347. max_retries=ftp_retry_count,
  1348. retry_delay=ftp_retry_delay,
  1349. operation_name=f"Download 3MF from {search_dir}/{fname}",
  1350. )
  1351. else:
  1352. downloaded = await download_file_async(
  1353. printer.ip_address,
  1354. printer.access_code,
  1355. f"{search_dir}/{fname}",
  1356. temp_path,
  1357. socket_timeout=ftp_timeout,
  1358. printer_model=printer.model,
  1359. )
  1360. if downloaded:
  1361. downloaded_filename = fname
  1362. logger.info("Found and downloaded from %s: %s", search_dir, fname)
  1363. break
  1364. except Exception as e:
  1365. logger.debug("Failed to list %s: %s", search_dir, e)
  1366. if not downloaded_filename or not temp_path:
  1367. logger.warning("Could not find 3MF file for print: %s", filename or subtask_name)
  1368. # Create a fallback archive without 3MF data so the print is still tracked
  1369. # This commonly happens with P1S/A1 printers where FTP has file size limitations
  1370. try:
  1371. from backend.app.models.archive import PrintArchive
  1372. # Derive print name from subtask_name or filename
  1373. print_name = subtask_name or filename
  1374. if print_name:
  1375. # Clean up the name (remove extensions, path parts)
  1376. print_name = print_name.split("/")[-1]
  1377. print_name = print_name.replace(".gcode.3mf", "").replace(".gcode", "").replace(".3mf", "")
  1378. else:
  1379. print_name = "Unknown Print"
  1380. # Create minimal archive entry
  1381. fallback_archive = PrintArchive(
  1382. printer_id=printer_id,
  1383. filename=filename or f"{print_name}.3mf",
  1384. file_path="", # Empty - no 3MF file available
  1385. file_size=0,
  1386. print_name=print_name,
  1387. status="printing",
  1388. started_at=datetime.now(timezone.utc),
  1389. extra_data={"no_3mf_available": True, "original_subtask": subtask_name, "_print_data": data},
  1390. )
  1391. db.add(fallback_archive)
  1392. await db.commit()
  1393. await db.refresh(fallback_archive)
  1394. logger.info("Created fallback archive %s for %s (no 3MF available)", fallback_archive.id, print_name)
  1395. # Start timelapse session if external camera is enabled
  1396. if printer.external_camera_enabled and printer.external_camera_url:
  1397. from backend.app.services.layer_timelapse import start_session
  1398. start_session(
  1399. printer_id,
  1400. fallback_archive.id,
  1401. printer.external_camera_url,
  1402. printer.external_camera_type or "mjpeg",
  1403. )
  1404. logger.info("Started layer timelapse for printer %s, archive %s", printer_id, fallback_archive.id)
  1405. # Track as active print
  1406. _active_prints[(printer_id, fallback_archive.filename)] = fallback_archive.id
  1407. if filename:
  1408. _active_prints[(printer_id, filename)] = fallback_archive.id
  1409. if subtask_name:
  1410. _active_prints[(printer_id, f"{subtask_name}.3mf")] = fallback_archive.id
  1411. _active_prints[(printer_id, subtask_name)] = fallback_archive.id
  1412. # Record starting energy if smart plug available
  1413. try:
  1414. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  1415. plug = plug_result.scalar_one_or_none()
  1416. if plug:
  1417. energy = await _get_plug_energy(plug, db)
  1418. if energy and energy.get("total") is not None:
  1419. _print_energy_start[fallback_archive.id] = energy["total"]
  1420. logger.info(
  1421. f"[ENERGY] Recorded starting energy for fallback archive {fallback_archive.id}: {energy['total']} kWh"
  1422. )
  1423. except Exception as e:
  1424. logger.warning("Failed to record starting energy for fallback: %s", e)
  1425. # Send WebSocket notification
  1426. await ws_manager.send_archive_created(
  1427. {
  1428. "id": fallback_archive.id,
  1429. "printer_id": fallback_archive.printer_id,
  1430. "filename": fallback_archive.filename,
  1431. "print_name": fallback_archive.print_name,
  1432. "status": fallback_archive.status,
  1433. }
  1434. )
  1435. # MQTT relay - publish archive created
  1436. try:
  1437. await mqtt_relay.on_archive_created(
  1438. archive_id=fallback_archive.id,
  1439. print_name=fallback_archive.print_name,
  1440. printer_name=printer.name,
  1441. status=fallback_archive.status,
  1442. )
  1443. except Exception:
  1444. pass # Don't fail if MQTT fails
  1445. # Store Spoolman tracking data (may not work for fallback since no 3MF)
  1446. try:
  1447. await _store_spoolman_print_data(
  1448. printer_id,
  1449. fallback_archive.id,
  1450. fallback_archive.file_path,
  1451. db,
  1452. printer_manager,
  1453. ams_mapping=_get_start_ams_mapping(data, fallback_archive.id),
  1454. )
  1455. except Exception as e:
  1456. logger.debug("[SPOOLMAN] Could not store tracking for fallback archive: %s", e)
  1457. # Send notification without archive data (file not found)
  1458. if not notification_sent:
  1459. await _send_print_start_notification(printer_id, data, logger=logger)
  1460. return
  1461. except Exception as e:
  1462. logger.error("Failed to create fallback archive: %s", e)
  1463. # Send notification without archive data (file not found)
  1464. if not notification_sent:
  1465. await _send_print_start_notification(printer_id, data, logger=logger)
  1466. return
  1467. try:
  1468. # Archive the file with status "printing"
  1469. service = ArchiveService(db)
  1470. archive = await service.archive_print(
  1471. printer_id=printer_id,
  1472. source_file=temp_path,
  1473. print_data={**data, "status": "printing"},
  1474. )
  1475. if archive:
  1476. # Track this active print (use both original filename and downloaded filename)
  1477. _active_prints[(printer_id, downloaded_filename)] = archive.id
  1478. if filename and filename != downloaded_filename:
  1479. _active_prints[(printer_id, filename)] = archive.id
  1480. if subtask_name:
  1481. _active_prints[(printer_id, f"{subtask_name}.3mf")] = archive.id
  1482. logger.info("Created archive %s for %s", archive.id, downloaded_filename)
  1483. # Start timelapse session if external camera is enabled
  1484. if printer.external_camera_enabled and printer.external_camera_url:
  1485. from backend.app.services.layer_timelapse import start_session
  1486. start_session(
  1487. printer_id,
  1488. archive.id,
  1489. printer.external_camera_url,
  1490. printer.external_camera_type or "mjpeg",
  1491. )
  1492. logger.info("Started layer timelapse for printer %s, archive %s", printer_id, archive.id)
  1493. # Record starting energy from smart plug if available
  1494. try:
  1495. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  1496. plug = plug_result.scalar_one_or_none()
  1497. logger.info(
  1498. f"[ENERGY] Auto-archive print start - archive {archive.id}, printer {printer_id}, plug found: {plug is not None}"
  1499. )
  1500. if plug:
  1501. energy = await _get_plug_energy(plug, db)
  1502. logger.info("[ENERGY] Auto-archive energy response: %s", energy)
  1503. if energy and energy.get("total") is not None:
  1504. _print_energy_start[archive.id] = energy["total"]
  1505. logger.info(
  1506. f"[ENERGY] Recorded starting energy for archive {archive.id}: {energy['total']} kWh"
  1507. )
  1508. else:
  1509. logger.warning("[ENERGY] No 'total' in energy response for archive %s", archive.id)
  1510. else:
  1511. logger.info("[ENERGY] No smart plug found for printer %s", printer_id)
  1512. except Exception as e:
  1513. logger.warning("Failed to record starting energy: %s", e)
  1514. await ws_manager.send_archive_created(
  1515. {
  1516. "id": archive.id,
  1517. "printer_id": archive.printer_id,
  1518. "filename": archive.filename,
  1519. "print_name": archive.print_name,
  1520. "status": archive.status,
  1521. }
  1522. )
  1523. # MQTT relay - publish archive created
  1524. try:
  1525. await mqtt_relay.on_archive_created(
  1526. archive_id=archive.id,
  1527. print_name=archive.print_name,
  1528. printer_name=printer.name,
  1529. status=archive.status,
  1530. )
  1531. except Exception:
  1532. pass # Don't fail if MQTT fails
  1533. # Send notification with archive data (new archive created)
  1534. if not notification_sent:
  1535. archive_data = {"print_time_seconds": archive.print_time_seconds}
  1536. await _send_print_start_notification(printer_id, data, archive_data, logger)
  1537. # Extract printable objects for skip object functionality
  1538. try:
  1539. from backend.app.services.archive import extract_printable_objects_from_3mf
  1540. with open(temp_path, "rb") as f:
  1541. threemf_data = f.read()
  1542. # Extract with positions for UI overlay
  1543. printable_objects, bbox_all = extract_printable_objects_from_3mf(
  1544. threemf_data, include_positions=True
  1545. )
  1546. if printable_objects:
  1547. # Store objects in printer state
  1548. client = printer_manager.get_client(printer_id)
  1549. if client:
  1550. client.state.printable_objects = printable_objects
  1551. client.state.printable_objects_bbox_all = bbox_all
  1552. client.state.skipped_objects = [] # Reset skipped objects for new print
  1553. logger.info(
  1554. "Loaded %s printable objects for printer %s", len(printable_objects), printer_id
  1555. )
  1556. except Exception as e:
  1557. logger.debug("Failed to extract printable objects: %s", e)
  1558. # Store Spoolman tracking data for per-filament usage reporting
  1559. try:
  1560. await _store_spoolman_print_data(
  1561. printer_id,
  1562. archive.id,
  1563. archive.file_path,
  1564. db,
  1565. printer_manager,
  1566. ams_mapping=_get_start_ams_mapping(data, archive.id),
  1567. )
  1568. except Exception as e:
  1569. logger.warning("[SPOOLMAN] Failed to store tracking data: %s", e)
  1570. # Capture timelapse file baseline for snapshot-diff on completion
  1571. try:
  1572. baseline_files, _ = await _list_timelapse_videos(printer)
  1573. _timelapse_baselines[printer_id] = {f.get("name", "") for f in baseline_files}
  1574. logger.info(
  1575. "[TIMELAPSE] Baseline at print start: %s video files for printer %s",
  1576. len(_timelapse_baselines[printer_id]),
  1577. printer_id,
  1578. )
  1579. except Exception as e:
  1580. logger.warning("[TIMELAPSE] Failed to capture baseline at print start: %s", e)
  1581. finally:
  1582. if temp_path and temp_path.exists():
  1583. temp_path.unlink()
  1584. _TIMELAPSE_VIDEO_EXTENSIONS = (".mp4", ".avi")
  1585. async def _list_timelapse_videos(printer) -> tuple[list[dict], str | None]:
  1586. """List video files from printer's timelapse directory.
  1587. Finds MP4 (X1/A1 series) and AVI (P1 series) timelapse files.
  1588. Returns (video_files, found_path) where video_files is a list of file dicts
  1589. and found_path is the directory where they were found, or ([], None).
  1590. """
  1591. from backend.app.services.bambu_ftp import list_files_async
  1592. logger = logging.getLogger(__name__)
  1593. for timelapse_path in ["/timelapse", "/timelapse/video", "/record", "/recording"]:
  1594. try:
  1595. found_files = await list_files_async(
  1596. printer.ip_address, printer.access_code, timelapse_path, printer_model=printer.model
  1597. )
  1598. if found_files:
  1599. video_files = [
  1600. f
  1601. for f in found_files
  1602. if not f.get("is_directory") and f.get("name", "").lower().endswith(_TIMELAPSE_VIDEO_EXTENSIONS)
  1603. ]
  1604. if video_files:
  1605. return video_files, timelapse_path
  1606. except Exception as e:
  1607. logger.debug("[TIMELAPSE] Path %s failed: %s", timelapse_path, e)
  1608. continue
  1609. return [], None
  1610. async def _scan_for_timelapse_with_retries(archive_id: int, baseline_names: set[str] | None = None):
  1611. """
  1612. Scan for timelapse with retries using a snapshot-diff approach.
  1613. Instead of picking the "most recent by mtime" (unreliable when the printer
  1614. clock is wrong in LAN-only mode), we snapshot existing MP4 filenames BEFORE
  1615. waiting, then look for any NEW filename that appears after each delay.
  1616. If baseline_names is provided (captured at print start), it is used directly.
  1617. Otherwise falls back to taking a baseline at completion time (best-effort
  1618. for prints started before app restart).
  1619. Falls back to name-matching (print name contained in MP4 filename) if no
  1620. new file appears after all retries.
  1621. """
  1622. from pathlib import Path
  1623. logger = logging.getLogger(__name__)
  1624. # --- Phase 1: Take baseline snapshot of existing timelapse files ---
  1625. try:
  1626. async with async_session() as db:
  1627. from backend.app.models.printer import Printer
  1628. service = ArchiveService(db)
  1629. archive = await service.get_archive(archive_id)
  1630. if not archive:
  1631. logger.warning("[TIMELAPSE] Archive %s not found, aborting", archive_id)
  1632. return
  1633. if archive.timelapse_path:
  1634. logger.info("[TIMELAPSE] Archive %s already has timelapse attached", archive_id)
  1635. return
  1636. if not archive.printer_id:
  1637. logger.warning("[TIMELAPSE] Archive %s has no printer, aborting", archive_id)
  1638. return
  1639. if baseline_names is not None:
  1640. # Use pre-captured baseline from print start (no race condition)
  1641. logger.info(
  1642. "[TIMELAPSE] Using print-start baseline: %s existing video files for archive %s",
  1643. len(baseline_names),
  1644. archive_id,
  1645. )
  1646. else:
  1647. # Fallback: take baseline now (e.g. app restarted mid-print)
  1648. result = await db.execute(select(Printer).where(Printer.id == archive.printer_id))
  1649. printer = result.scalar_one_or_none()
  1650. if not printer:
  1651. logger.warning("[TIMELAPSE] Printer not found for archive %s, aborting", archive_id)
  1652. return
  1653. baseline_files, _ = await _list_timelapse_videos(printer)
  1654. baseline_names = {f.get("name", "") for f in baseline_files}
  1655. logger.info(
  1656. "[TIMELAPSE] Baseline snapshot (fallback): %s existing video files for archive %s",
  1657. len(baseline_names),
  1658. archive_id,
  1659. )
  1660. # Derive base_name for name-matching fallback
  1661. base_name = Path(archive.filename).stem if archive.filename else ""
  1662. if base_name.endswith(".gcode"):
  1663. base_name = base_name[:-6]
  1664. except Exception as e:
  1665. logger.warning("[TIMELAPSE] Failed to take baseline snapshot for archive %s: %s", archive_id, e)
  1666. return
  1667. # --- Phase 2: Retry loop — look for NEW files that weren't in baseline ---
  1668. retry_delays = [5, 10, 20, 30]
  1669. for attempt, delay in enumerate(retry_delays, 1):
  1670. logger.info(
  1671. "[TIMELAPSE] Attempt %s/%s: waiting %ss before scanning for archive %s",
  1672. attempt,
  1673. len(retry_delays),
  1674. delay,
  1675. archive_id,
  1676. )
  1677. await asyncio.sleep(delay)
  1678. try:
  1679. async with async_session() as db:
  1680. from backend.app.models.printer import Printer
  1681. from backend.app.services.bambu_ftp import download_file_bytes_async
  1682. service = ArchiveService(db)
  1683. archive = await service.get_archive(archive_id)
  1684. if not archive:
  1685. logger.warning("[TIMELAPSE] Archive %s not found, stopping retries", archive_id)
  1686. return
  1687. if archive.timelapse_path:
  1688. logger.info("[TIMELAPSE] Archive %s already has timelapse attached, stopping retries", archive_id)
  1689. return
  1690. result = await db.execute(select(Printer).where(Printer.id == archive.printer_id))
  1691. printer = result.scalar_one_or_none()
  1692. if not printer:
  1693. logger.warning("[TIMELAPSE] Printer not found for archive %s, stopping retries", archive_id)
  1694. return
  1695. video_files, found_path = await _list_timelapse_videos(printer)
  1696. if not video_files:
  1697. logger.info("[TIMELAPSE] Attempt %s: No video files found, will retry", attempt)
  1698. continue
  1699. logger.info("[TIMELAPSE] Attempt %s: Found %s video files in %s", attempt, len(video_files), found_path)
  1700. for f in video_files[:5]:
  1701. logger.info("[TIMELAPSE] - %s", f.get("name"))
  1702. # Find files that are NEW (not in baseline snapshot)
  1703. new_files = [f for f in video_files if f.get("name", "") not in baseline_names]
  1704. if new_files:
  1705. # Pick the first new file (there should typically be exactly one)
  1706. target = new_files[0]
  1707. file_name = target.get("name")
  1708. remote_path = target.get("path") or f"/timelapse/{file_name}"
  1709. logger.info(
  1710. "[TIMELAPSE] Attempt %s: New file detected: %s (downloading for archive %s)",
  1711. attempt,
  1712. file_name,
  1713. archive_id,
  1714. )
  1715. timelapse_data = await download_file_bytes_async(
  1716. printer.ip_address, printer.access_code, remote_path, printer_model=printer.model
  1717. )
  1718. if timelapse_data:
  1719. success = await service.attach_timelapse(archive_id, timelapse_data, file_name)
  1720. if success:
  1721. logger.info("[TIMELAPSE] Successfully attached timelapse to archive %s", archive_id)
  1722. await ws_manager.send_archive_updated({"id": archive_id, "timelapse_attached": True})
  1723. return
  1724. else:
  1725. logger.warning("[TIMELAPSE] Failed to attach timelapse to archive %s", archive_id)
  1726. else:
  1727. logger.warning("[TIMELAPSE] Attempt %s: Failed to download new file, will retry", attempt)
  1728. else:
  1729. logger.info("[TIMELAPSE] Attempt %s: No new files since baseline, will retry", attempt)
  1730. except Exception as e:
  1731. logger.warning("[TIMELAPSE] Attempt %s failed with error: %s", attempt, e)
  1732. # --- Phase 3: Fallback — try name matching against all files ---
  1733. if base_name:
  1734. logger.info("[TIMELAPSE] Retries exhausted, trying name-match fallback for '%s'", base_name)
  1735. try:
  1736. async with async_session() as db:
  1737. from backend.app.models.printer import Printer
  1738. from backend.app.services.bambu_ftp import download_file_bytes_async
  1739. service = ArchiveService(db)
  1740. archive = await service.get_archive(archive_id)
  1741. if not archive or archive.timelapse_path:
  1742. return
  1743. result = await db.execute(select(Printer).where(Printer.id == archive.printer_id))
  1744. printer = result.scalar_one_or_none()
  1745. if not printer:
  1746. return
  1747. video_files, found_path = await _list_timelapse_videos(printer)
  1748. for f in video_files:
  1749. fname = f.get("name", "")
  1750. if base_name.lower() in fname.lower():
  1751. remote_path = f.get("path") or f"/timelapse/{fname}"
  1752. logger.info("[TIMELAPSE] Name-match fallback: '%s' matches '%s'", base_name, fname)
  1753. timelapse_data = await download_file_bytes_async(
  1754. printer.ip_address, printer.access_code, remote_path, printer_model=printer.model
  1755. )
  1756. if timelapse_data:
  1757. success = await service.attach_timelapse(archive_id, timelapse_data, fname)
  1758. if success:
  1759. logger.info(
  1760. "[TIMELAPSE] Name-match fallback attached timelapse to archive %s", archive_id
  1761. )
  1762. await ws_manager.send_archive_updated({"id": archive_id, "timelapse_attached": True})
  1763. return
  1764. break # Only try the first name match
  1765. except Exception as e:
  1766. logger.warning("[TIMELAPSE] Name-match fallback failed: %s", e)
  1767. logger.warning("[TIMELAPSE] All attempts exhausted for archive %s, giving up", archive_id)
  1768. async def on_print_complete(printer_id: int, data: dict):
  1769. """Handle print completion - update the archive status."""
  1770. import time
  1771. logger = logging.getLogger(__name__)
  1772. start_time = time.time()
  1773. def log_timing(section: str):
  1774. elapsed = time.time() - start_time
  1775. logger.info("[TIMING] %s: %.3fs elapsed", section, elapsed)
  1776. logger.info("[CALLBACK] on_print_complete started for printer %s", printer_id)
  1777. try:
  1778. ws_data = {
  1779. "status": data.get("status"),
  1780. "filename": data.get("filename"),
  1781. "subtask_name": data.get("subtask_name"),
  1782. "timelapse_was_active": data.get("timelapse_was_active"),
  1783. }
  1784. await ws_manager.send_print_complete(printer_id, ws_data)
  1785. log_timing("WebSocket send_print_complete")
  1786. except Exception as e:
  1787. logger.warning("[CALLBACK] WebSocket send_print_complete failed: %s", e)
  1788. # Capture user info before clearing (needed for print log entry)
  1789. _print_user_info = printer_manager.get_current_print_user(printer_id)
  1790. # Clear current print user tracking (Issue #206)
  1791. printer_manager.clear_current_print_user(printer_id)
  1792. # MQTT relay - publish print complete
  1793. try:
  1794. printer_info = printer_manager.get_printer(printer_id)
  1795. if printer_info:
  1796. await mqtt_relay.on_print_complete(
  1797. printer_id,
  1798. printer_info.name,
  1799. printer_info.serial_number,
  1800. data.get("filename", ""),
  1801. data.get("subtask_name", ""),
  1802. data.get("status", "completed"),
  1803. )
  1804. except Exception:
  1805. pass # Don't fail print complete callback if MQTT fails
  1806. filename = data.get("filename", "")
  1807. subtask_name = data.get("subtask_name", "")
  1808. if not filename and not subtask_name:
  1809. logger.warning("Print complete without filename or subtask_name")
  1810. return
  1811. logger.info("Print complete - filename: %s, subtask: %s, status: %s", filename, subtask_name, data.get("status"))
  1812. # Build list of possible keys to try (matching how they were registered in on_print_start)
  1813. possible_keys = []
  1814. # Try subtask_name variations first (most reliable for matching)
  1815. if subtask_name:
  1816. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  1817. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  1818. possible_keys.append((printer_id, subtask_name))
  1819. # Try filename variations
  1820. if filename:
  1821. # Extract just the filename if it's a path
  1822. fname = filename.split("/")[-1] if "/" in filename else filename
  1823. if fname.endswith(".3mf"):
  1824. possible_keys.append((printer_id, fname))
  1825. elif fname.endswith(".gcode"):
  1826. base_name = fname.rsplit(".", 1)[0]
  1827. possible_keys.append((printer_id, f"{base_name}.gcode.3mf"))
  1828. possible_keys.append((printer_id, f"{base_name}.3mf"))
  1829. possible_keys.append((printer_id, fname))
  1830. else:
  1831. possible_keys.append((printer_id, f"{fname}.gcode.3mf"))
  1832. possible_keys.append((printer_id, f"{fname}.3mf"))
  1833. possible_keys.append((printer_id, fname))
  1834. # Also try full path versions
  1835. if filename.endswith(".3mf"):
  1836. possible_keys.append((printer_id, filename))
  1837. elif filename.endswith(".gcode"):
  1838. base_name = filename.rsplit(".", 1)[0]
  1839. possible_keys.append((printer_id, f"{base_name}.3mf"))
  1840. possible_keys.append((printer_id, filename))
  1841. else:
  1842. possible_keys.append((printer_id, f"{filename}.3mf"))
  1843. possible_keys.append((printer_id, filename))
  1844. # Find the archive for this print
  1845. logger.info("Looking for archive in _active_prints, keys to try: %s...", possible_keys[:5])
  1846. logger.info("Current _active_prints: %s", list(_active_prints.keys()))
  1847. archive_id = None
  1848. for key in possible_keys:
  1849. archive_id = _active_prints.pop(key, None)
  1850. if archive_id:
  1851. logger.info("Found archive %s with key %s", archive_id, key)
  1852. # Also clean up any other keys pointing to this archive
  1853. keys_to_remove = [k for k, v in _active_prints.items() if v == archive_id]
  1854. for k in keys_to_remove:
  1855. _active_prints.pop(k, None)
  1856. break
  1857. if not archive_id:
  1858. # Try to find by filename or subtask_name if not tracked (for prints started before app)
  1859. async with async_session() as db:
  1860. from backend.app.models.archive import PrintArchive
  1861. # Try matching by subtask_name (stored as print_name) first
  1862. if subtask_name:
  1863. result = await db.execute(
  1864. select(PrintArchive)
  1865. .where(PrintArchive.printer_id == printer_id)
  1866. .where(PrintArchive.status == "printing")
  1867. .where(
  1868. or_(
  1869. PrintArchive.print_name.ilike(f"%{subtask_name}%"),
  1870. PrintArchive.filename.ilike(f"%{subtask_name}%"),
  1871. )
  1872. )
  1873. .order_by(PrintArchive.created_at.desc())
  1874. .limit(1)
  1875. )
  1876. archive = result.scalar_one_or_none()
  1877. if archive:
  1878. archive_id = archive.id
  1879. logger.info("Found archive %s by subtask_name match: %s", archive_id, subtask_name)
  1880. # Also try by filename
  1881. if not archive_id and filename:
  1882. result = await db.execute(
  1883. select(PrintArchive)
  1884. .where(PrintArchive.printer_id == printer_id)
  1885. .where(PrintArchive.filename == filename)
  1886. .where(PrintArchive.status == "printing")
  1887. .order_by(PrintArchive.created_at.desc())
  1888. .limit(1)
  1889. )
  1890. archive = result.scalar_one_or_none()
  1891. if archive:
  1892. archive_id = archive.id
  1893. # Cleanup: delete uploaded file from printer SD card to prevent phantom prints (Issue #374)
  1894. # The print scheduler uploads files to the SD card root (/). Some printers (e.g. P1S)
  1895. # auto-start files found in root on power cycle, causing ghost prints.
  1896. # Must run before the archive_id early-return so it executes even when archiving is disabled.
  1897. try:
  1898. if subtask_name:
  1899. async with async_session() as db:
  1900. from backend.app.models.printer import Printer
  1901. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1902. printer = result.scalar_one_or_none()
  1903. if printer:
  1904. from backend.app.services.bambu_ftp import delete_file_async
  1905. # Try both .3mf and .gcode extensions — the printer may have either
  1906. for ext in (".3mf", ".gcode"):
  1907. remote_path = f"/{subtask_name}{ext}"
  1908. # Retry up to 3 times — the printer may still lock the filesystem briefly after a print ends
  1909. for attempt in range(1, 4):
  1910. try:
  1911. delete_result = await delete_file_async(
  1912. printer.ip_address,
  1913. printer.access_code,
  1914. remote_path,
  1915. printer_model=printer.model,
  1916. )
  1917. if delete_result:
  1918. logger.info("Deleted %s from printer %s SD card", remote_path, printer.name)
  1919. break
  1920. except Exception as e:
  1921. delete_result = False
  1922. logger.warning(
  1923. "SD card cleanup attempt %d/3 raised for %s: %s",
  1924. attempt,
  1925. remote_path,
  1926. e,
  1927. )
  1928. if not delete_result and attempt < 3:
  1929. await asyncio.sleep(2)
  1930. elif not delete_result:
  1931. logger.warning(
  1932. "SD card cleanup failed after 3 attempts for %s (file may linger on SD card)",
  1933. remote_path,
  1934. )
  1935. except Exception as e:
  1936. logger.warning("SD card file cleanup failed for printer %s: %s", printer_id, e)
  1937. log_timing("SD card cleanup")
  1938. # Update queue item status early — must run before the archive_id early-return
  1939. # so queue items don't get stuck in "printing" when archive lookup fails.
  1940. try:
  1941. async with async_session() as db:
  1942. from backend.app.models.print_queue import PrintQueueItem
  1943. result = await db.execute(
  1944. select(PrintQueueItem)
  1945. .where(PrintQueueItem.printer_id == printer_id)
  1946. .where(PrintQueueItem.status == "printing")
  1947. )
  1948. printing_items = list(result.scalars().all())
  1949. if len(printing_items) > 1:
  1950. logger.warning(
  1951. "BUG: Multiple queue items in 'printing' status for printer %s: %s",
  1952. printer_id,
  1953. [(i.id, i.archive_id, i.library_file_id) for i in printing_items],
  1954. )
  1955. queue_item = printing_items[0] if printing_items else None
  1956. if queue_item:
  1957. queue_status = data.get("status", "completed")
  1958. # MQTT sends "aborted" for cancelled prints; normalise to
  1959. # "cancelled" so it matches the queue schema Literal.
  1960. if queue_status == "aborted":
  1961. queue_status = "cancelled"
  1962. queue_item.status = queue_status
  1963. queue_item.completed_at = datetime.now(timezone.utc)
  1964. await db.commit()
  1965. logger.info("Updated queue item %s status to %s", queue_item.id, queue_status)
  1966. # MQTT relay - publish queue job completed
  1967. try:
  1968. printer_info = printer_manager.get_printer(printer_id)
  1969. await mqtt_relay.on_queue_job_completed(
  1970. job_id=queue_item.id,
  1971. filename=filename or subtask_name,
  1972. printer_id=printer_id,
  1973. printer_name=printer_info.name if printer_info else "Unknown",
  1974. status=queue_status,
  1975. )
  1976. except Exception:
  1977. pass # Don't fail if MQTT fails
  1978. # Check if queue is now empty and send notification
  1979. try:
  1980. from sqlalchemy import func as sa_func
  1981. count_result = await db.execute(
  1982. select(sa_func.count(PrintQueueItem.id)).where(PrintQueueItem.status == "pending")
  1983. )
  1984. pending_count = count_result.scalar() or 0
  1985. if pending_count == 0:
  1986. today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
  1987. completed_result = await db.execute(
  1988. select(sa_func.count(PrintQueueItem.id)).where(
  1989. PrintQueueItem.status.in_(["completed", "failed", "skipped"]),
  1990. PrintQueueItem.completed_at >= today_start,
  1991. )
  1992. )
  1993. completed_count = completed_result.scalar() or 1
  1994. await notification_service.on_queue_completed(
  1995. completed_count=completed_count,
  1996. db=db,
  1997. )
  1998. except Exception:
  1999. pass # Don't fail if notification fails
  2000. # Handle auto_off_after - power off printer if requested (after cooldown)
  2001. if queue_item.auto_off_after:
  2002. result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  2003. plug = result.scalar_one_or_none()
  2004. if plug and plug.enabled:
  2005. logger.info("Auto-off requested for printer %s, waiting for cooldown...", printer_id)
  2006. async def cooldown_and_poweroff(pid: int, plug_id: int):
  2007. # Wait for nozzle to cool down
  2008. await printer_manager.wait_for_cooldown(pid, target_temp=50.0, timeout=600)
  2009. # Re-fetch plug in new session
  2010. async with async_session() as new_db:
  2011. result = await new_db.execute(select(SmartPlug).where(SmartPlug.id == plug_id))
  2012. p = result.scalar_one_or_none()
  2013. if p and p.enabled:
  2014. success = await tasmota_service.turn_off(p)
  2015. if success:
  2016. logger.info("Powered off printer %s via smart plug '%s'", pid, p.name)
  2017. else:
  2018. logger.warning("Failed to power off printer %s via smart plug", pid)
  2019. asyncio.create_task(cooldown_and_poweroff(printer_id, plug.id))
  2020. except Exception as e:
  2021. logging.getLogger(__name__).warning(f"Queue item update failed: {e}")
  2022. log_timing("Queue item update")
  2023. # Start bed cooldown monitor (polls bed temp until it drops below threshold)
  2024. # Must run before archive_id early-return so it fires for all prints (including
  2025. # prints started from BambuStudio/touchscreen that have no archive).
  2026. async def _background_bed_cooldown():
  2027. """Monitor bed temperature after print and notify when cooled."""
  2028. try:
  2029. from backend.app.api.routes.settings import get_setting
  2030. # Check threshold setting
  2031. async with async_session() as db:
  2032. threshold_str = await get_setting(db, "bed_cooled_threshold")
  2033. threshold = float(threshold_str) if threshold_str else 35.0
  2034. # Check if any provider has on_bed_cooled enabled (early exit if none)
  2035. async with async_session() as db:
  2036. providers = await notification_service._get_providers_for_event(db, "on_bed_cooled", printer_id)
  2037. if not providers:
  2038. logger.debug("[BED-COOL] No providers enabled for bed_cooled on printer %s", printer_id)
  2039. return
  2040. logger.info("[BED-COOL] Monitoring bed temp for printer %s (threshold: %.0f°C)", printer_id, threshold)
  2041. # Request a fresh full status so we get current bed_temper
  2042. printer_manager.request_status_update(printer_id)
  2043. max_polls = 120 # 120 * 15s = 30 min timeout
  2044. for poll_num in range(max_polls):
  2045. await asyncio.sleep(15)
  2046. # Request fresh temperature data every 60s — after print completion,
  2047. # the printer may send partial MQTT updates without bed_temper,
  2048. # leaving the cached value stale at the end-of-print temperature.
  2049. if poll_num % 4 == 0:
  2050. printer_manager.request_status_update(printer_id)
  2051. # Check if printer is still connected
  2052. status = printer_manager.get_status(printer_id)
  2053. if status is None:
  2054. logger.info("[BED-COOL] Printer %s disconnected, stopping monitor", printer_id)
  2055. return
  2056. # Check if a new print started (state == RUNNING)
  2057. if hasattr(status, "state") and status.state == "RUNNING":
  2058. logger.info("[BED-COOL] New print started on printer %s, stopping monitor", printer_id)
  2059. return
  2060. # Get bed temperature
  2061. bed_temp = None
  2062. if hasattr(status, "temperatures") and isinstance(status.temperatures, dict):
  2063. bed_temp = status.temperatures.get("bed")
  2064. if bed_temp is None:
  2065. logger.debug(
  2066. "[BED-COOL] Printer %s: bed temp is None (keys: %s, state: %s)",
  2067. printer_id,
  2068. list(status.temperatures.keys()) if isinstance(status.temperatures, dict) else "N/A",
  2069. status.state if hasattr(status, "state") else "N/A",
  2070. )
  2071. continue
  2072. logger.debug("[BED-COOL] Printer %s: bed=%.1f°C, threshold=%.0f°C", printer_id, bed_temp, threshold)
  2073. if bed_temp <= threshold:
  2074. logger.info(
  2075. "[BED-COOL] Bed cooled to %.1f°C on printer %s (threshold: %.0f°C)",
  2076. bed_temp,
  2077. printer_id,
  2078. threshold,
  2079. )
  2080. printer_info = printer_manager.get_printer(printer_id)
  2081. p_name = printer_info.name if printer_info else "Unknown"
  2082. async with async_session() as db:
  2083. await notification_service.on_bed_cooled(
  2084. printer_id=printer_id,
  2085. printer_name=p_name,
  2086. bed_temp=bed_temp,
  2087. threshold=threshold,
  2088. filename=filename or subtask_name or "",
  2089. db=db,
  2090. )
  2091. return
  2092. logger.info("[BED-COOL] Timeout waiting for bed to cool on printer %s", printer_id)
  2093. except asyncio.CancelledError:
  2094. logger.info("[BED-COOL] Bed cooldown monitor cancelled for printer %s", printer_id)
  2095. except Exception as e:
  2096. logger.warning("[BED-COOL] Failed: %s", e)
  2097. finally:
  2098. _bed_cooldown_tasks.pop(printer_id, None)
  2099. # Only start bed cooldown for completed prints
  2100. if data.get("status") == "completed":
  2101. # Cancel any existing task for this printer
  2102. existing_task = _bed_cooldown_tasks.pop(printer_id, None)
  2103. if existing_task and not existing_task.done():
  2104. existing_task.cancel()
  2105. task = asyncio.create_task(_background_bed_cooldown())
  2106. _bed_cooldown_tasks[printer_id] = task
  2107. if not archive_id:
  2108. logger.warning("Could not find archive for print complete: filename=%s, subtask=%s", filename, subtask_name)
  2109. return
  2110. log_timing("Archive lookup")
  2111. # Update archive status
  2112. logger.info("[ARCHIVE] Updating archive %s status...", archive_id)
  2113. try:
  2114. async with async_session() as db:
  2115. service = ArchiveService(db)
  2116. status = data.get("status", "completed")
  2117. # Auto-detect failure reason
  2118. failure_reason = None
  2119. if status == "aborted":
  2120. failure_reason = "User cancelled"
  2121. logger.info("[ARCHIVE] Print was aborted by user, setting failure_reason='User cancelled'")
  2122. elif status == "failed":
  2123. # Try to determine failure reason from HMS errors
  2124. hms_errors = data.get("hms_errors", [])
  2125. if hms_errors:
  2126. logger.info("[ARCHIVE] HMS errors at failure: %s", hms_errors)
  2127. # Map known HMS error modules to failure reasons
  2128. # Module 0x07 = Filament, 0x0C = MC (Motion Controller), etc.
  2129. for err in hms_errors:
  2130. module = err.get("module", 0)
  2131. if module == 0x07: # Filament module
  2132. failure_reason = "Filament runout"
  2133. break
  2134. elif module == 0x0C: # Motion controller
  2135. failure_reason = "Layer shift"
  2136. break
  2137. elif module == 0x05: # Nozzle/extruder
  2138. failure_reason = "Clogged nozzle"
  2139. break
  2140. if failure_reason:
  2141. logger.info("[ARCHIVE] Detected failure_reason from HMS: %s", failure_reason)
  2142. else:
  2143. logger.info("[ARCHIVE] No HMS errors available to determine failure reason")
  2144. await service.update_archive_status(
  2145. archive_id,
  2146. status=status,
  2147. completed_at=datetime.now(timezone.utc) if status in ("completed", "failed", "aborted") else None,
  2148. failure_reason=failure_reason,
  2149. )
  2150. logger.info(
  2151. "[ARCHIVE] Archive %s status updated to %s, failure_reason=%s", archive_id, status, failure_reason
  2152. )
  2153. await ws_manager.send_archive_updated(
  2154. {
  2155. "id": archive_id,
  2156. "status": status,
  2157. }
  2158. )
  2159. logger.info("[ARCHIVE] WebSocket notification sent for archive %s", archive_id)
  2160. # MQTT relay - publish archive updated
  2161. try:
  2162. await mqtt_relay.on_archive_updated(
  2163. archive_id=archive_id,
  2164. print_name=filename or subtask_name,
  2165. status=status,
  2166. )
  2167. except Exception:
  2168. pass # Don't fail if MQTT fails
  2169. except Exception as e:
  2170. logger.error("[ARCHIVE] Failed to update archive %s status: %s", archive_id, e, exc_info=True)
  2171. # Continue with other operations even if archive update fails
  2172. log_timing("Archive status update")
  2173. # Write independent print log entry (separate table, never touches archives)
  2174. try:
  2175. async with async_session() as db:
  2176. from backend.app.models.archive import PrintArchive
  2177. from backend.app.services.print_log import write_log_entry
  2178. archive = await db.get(PrintArchive, archive_id)
  2179. if archive:
  2180. p_info = printer_manager.get_printer(printer_id)
  2181. await write_log_entry(
  2182. db,
  2183. status=data.get("status", "completed"),
  2184. print_name=archive.print_name,
  2185. printer_name=p_info.name if p_info else None,
  2186. printer_id=printer_id,
  2187. started_at=archive.started_at,
  2188. completed_at=archive.completed_at,
  2189. filament_type=archive.filament_type,
  2190. filament_color=archive.filament_color,
  2191. filament_used_grams=archive.filament_used_grams,
  2192. thumbnail_path=archive.thumbnail_path,
  2193. created_by_username=_print_user_info.get("username") if _print_user_info else None,
  2194. )
  2195. await db.commit()
  2196. logger.info("[PRINT_LOG] Log entry written for archive %s", archive_id)
  2197. except Exception as e:
  2198. logger.warning("[PRINT_LOG] Failed to write log entry for archive %s: %s", archive_id, e)
  2199. log_timing("Print log entry")
  2200. # Track filament consumption from AMS remain% deltas (skip if Spoolman handles usage)
  2201. usage_results: list[dict] = []
  2202. # Prefer ams_mapping captured from MQTT request topic (works for all print sources)
  2203. stored_ams_mapping = data.get("ams_mapping")
  2204. # Fallback to _print_ams_mappings for queue/reprint (set before print starts)
  2205. if not stored_ams_mapping and archive_id:
  2206. stored_ams_mapping = _print_ams_mappings.pop(archive_id, None)
  2207. try:
  2208. async with async_session() as db:
  2209. from backend.app.api.routes.settings import get_setting
  2210. _spoolman_on = await get_setting(db, "spoolman_enabled")
  2211. if not _spoolman_on or _spoolman_on.lower() != "true":
  2212. from backend.app.services.usage_tracker import on_print_complete as usage_on_print_complete
  2213. async with async_session() as db:
  2214. usage_results = await usage_on_print_complete(
  2215. printer_id,
  2216. data,
  2217. printer_manager,
  2218. db,
  2219. archive_id=archive_id,
  2220. ams_mapping=stored_ams_mapping,
  2221. )
  2222. if usage_results:
  2223. await ws_manager.broadcast(
  2224. {
  2225. "type": "spool_usage_logged",
  2226. "printer_id": printer_id,
  2227. "usage": usage_results,
  2228. }
  2229. )
  2230. log_timing("Usage tracker")
  2231. except Exception as e:
  2232. logger.warning("Usage tracker on_print_complete failed: %s", e)
  2233. # Report filament usage to Spoolman if print completed successfully
  2234. if data.get("status") == "completed":
  2235. try:
  2236. await _report_spoolman_usage(printer_id, archive_id)
  2237. log_timing("Spoolman usage report")
  2238. except Exception as e:
  2239. logger.warning("Spoolman usage reporting failed: %s", e)
  2240. else:
  2241. # Report partial usage if tracking data exists (only stored when weight sync is disabled)
  2242. try:
  2243. async with async_session() as db:
  2244. await _cleanup_spoolman_tracking(
  2245. printer_id,
  2246. archive_id,
  2247. db,
  2248. last_layer_num=data.get("last_layer_num"),
  2249. last_progress=data.get("last_progress"),
  2250. )
  2251. except Exception as e:
  2252. logger.debug("[SPOOLMAN] Cleanup failed: %s", e)
  2253. # Run slow operations as background tasks to avoid blocking the event loop
  2254. # These operations can take 5-10+ seconds and would freeze the UI if awaited
  2255. starting_kwh = _print_energy_start.pop(archive_id, None)
  2256. async def _background_energy_calculation():
  2257. """Calculate and save energy usage in background."""
  2258. try:
  2259. logger.info("[ENERGY-BG] Starting energy calculation for archive %s", archive_id)
  2260. async with async_session() as db:
  2261. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  2262. plug = plug_result.scalar_one_or_none()
  2263. if plug:
  2264. energy = await _get_plug_energy(plug, db)
  2265. logger.info("[ENERGY-BG] Energy response: %s", energy)
  2266. energy_used = None
  2267. if starting_kwh is not None and energy and energy.get("total") is not None:
  2268. ending_kwh = energy["total"]
  2269. energy_used = round(ending_kwh - starting_kwh, 4)
  2270. logger.info("[ENERGY-BG] Per-print energy: %s kWh", energy_used)
  2271. if energy_used is not None and energy_used >= 0:
  2272. from backend.app.api.routes.settings import get_setting
  2273. energy_cost_per_kwh = await get_setting(db, "energy_cost_per_kwh")
  2274. cost_per_kwh = float(energy_cost_per_kwh) if energy_cost_per_kwh else 0.15
  2275. energy_cost = round(energy_used * cost_per_kwh, 3)
  2276. from backend.app.models.archive import PrintArchive
  2277. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  2278. archive = result.scalar_one_or_none()
  2279. if archive:
  2280. archive.energy_kwh = energy_used
  2281. archive.energy_cost = energy_cost
  2282. await db.commit()
  2283. logger.info("[ENERGY-BG] Saved: %s kWh, cost=%s", energy_used, energy_cost)
  2284. else:
  2285. logger.info("[ENERGY-BG] No smart plug for printer %s", printer_id)
  2286. except Exception as e:
  2287. logger.warning("[ENERGY-BG] Failed: %s", e)
  2288. async def _background_finish_photo() -> str | None:
  2289. """Capture finish photo in background. Returns photo filename if captured."""
  2290. try:
  2291. logger.info("[PHOTO-BG] Starting finish photo capture for archive %s", archive_id)
  2292. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  2293. async with async_session() as db:
  2294. from backend.app.api.routes.settings import get_setting
  2295. capture_enabled = await get_setting(db, "capture_finish_photo")
  2296. if capture_enabled is None or capture_enabled.lower() == "true":
  2297. from backend.app.models.printer import Printer
  2298. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2299. printer = result.scalar_one_or_none()
  2300. if printer and archive_id:
  2301. from backend.app.models.archive import PrintArchive
  2302. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  2303. archive = result.scalar_one_or_none()
  2304. if archive:
  2305. import uuid
  2306. from datetime import datetime
  2307. from pathlib import Path
  2308. if archive.file_path:
  2309. archive_dir = app_settings.base_dir / Path(archive.file_path).parent
  2310. else:
  2311. logger.warning("[PHOTO-BG] Archive %s has no file_path, using fallback dir", archive_id)
  2312. archive_dir = app_settings.archive_dir / str(archive.id)
  2313. photo_filename = None
  2314. # Check for external camera first
  2315. if printer.external_camera_enabled and printer.external_camera_url:
  2316. logger.info("[PHOTO-BG] Using external camera")
  2317. from backend.app.services.external_camera import capture_frame
  2318. frame_data = await capture_frame(
  2319. printer.external_camera_url, printer.external_camera_type or "mjpeg"
  2320. )
  2321. if frame_data:
  2322. photos_dir = archive_dir / "photos"
  2323. photos_dir.mkdir(parents=True, exist_ok=True)
  2324. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  2325. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  2326. photo_path = photos_dir / photo_filename
  2327. await asyncio.to_thread(photo_path.write_bytes, frame_data)
  2328. logger.info("[PHOTO-BG] Saved external camera frame: %s", photo_filename)
  2329. else:
  2330. # Check if camera stream is active - use buffered frame to avoid freeze
  2331. # Check both RTSP streams (_active_streams) and chamber image streams (_active_chamber_streams)
  2332. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  2333. active_chamber_for_printer = [
  2334. k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")
  2335. ]
  2336. buffered_frame = get_buffered_frame(printer_id)
  2337. if (active_for_printer or active_chamber_for_printer) and buffered_frame:
  2338. # Use frame from active stream
  2339. logger.info("[PHOTO-BG] Using buffered frame from active stream")
  2340. photos_dir = archive_dir / "photos"
  2341. photos_dir.mkdir(parents=True, exist_ok=True)
  2342. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  2343. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  2344. photo_path = photos_dir / photo_filename
  2345. await asyncio.to_thread(photo_path.write_bytes, buffered_frame)
  2346. logger.info("[PHOTO-BG] Saved buffered frame: %s", photo_filename)
  2347. else:
  2348. # No active stream - capture new frame
  2349. from backend.app.services.camera import capture_finish_photo
  2350. photo_filename = await capture_finish_photo(
  2351. printer_id=printer_id,
  2352. ip_address=printer.ip_address,
  2353. access_code=printer.access_code,
  2354. model=printer.model,
  2355. archive_dir=archive_dir,
  2356. )
  2357. if photo_filename:
  2358. photos = archive.photos or []
  2359. photos.append(photo_filename)
  2360. archive.photos = photos
  2361. await db.commit()
  2362. logger.info("[PHOTO-BG] Saved: %s", photo_filename)
  2363. return photo_filename
  2364. return None
  2365. except Exception as e:
  2366. logger.warning("[PHOTO-BG] Failed: %s", e)
  2367. return None
  2368. asyncio.create_task(_background_energy_calculation())
  2369. # Photo capture task - result will be used by notifications
  2370. photo_task = asyncio.create_task(_background_finish_photo())
  2371. log_timing("Background tasks scheduled (energy, photo)")
  2372. # Also run smart plug, notifications, and maintenance as background tasks
  2373. print_status = data.get("status", "completed")
  2374. async def _background_smart_plug():
  2375. """Handle smart plug automation in background."""
  2376. try:
  2377. logger.info("[AUTO-OFF-BG] Starting smart plug automation for printer %s", printer_id)
  2378. async with async_session() as db:
  2379. await smart_plug_manager.on_print_complete(printer_id, print_status, db)
  2380. logger.info("[AUTO-OFF-BG] Completed")
  2381. except Exception as e:
  2382. logger.warning("[AUTO-OFF-BG] Failed: %s", e)
  2383. async def _background_notifications(finish_photo_filename: str | None = None):
  2384. """Send print complete notifications in background."""
  2385. try:
  2386. logger.info(
  2387. "[NOTIFY-BG] Starting notifications for printer %s, photo=%s", printer_id, finish_photo_filename
  2388. )
  2389. async with async_session() as db:
  2390. from backend.app.models.archive import PrintArchive
  2391. from backend.app.models.printer import Printer
  2392. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2393. printer = result.scalar_one_or_none()
  2394. printer_name = printer.name if printer else f"Printer {printer_id}"
  2395. archive_data = None
  2396. if archive_id:
  2397. archive_result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  2398. archive = archive_result.scalar_one_or_none()
  2399. if archive:
  2400. archive_data = {
  2401. "print_time_seconds": archive.print_time_seconds,
  2402. "actual_filament_grams": archive.filament_used_grams,
  2403. "failure_reason": archive.failure_reason,
  2404. }
  2405. # Scale filament usage for partial prints
  2406. if print_status != "completed" and archive.filament_used_grams:
  2407. progress = data.get("progress") or 0
  2408. scale = max(0.0, min(progress / 100.0, 1.0))
  2409. archive_data["actual_filament_grams"] = round(archive.filament_used_grams * scale, 1)
  2410. archive_data["progress"] = progress
  2411. # Pass per-slot data from archive.extra_data
  2412. if archive.extra_data and archive.extra_data.get("filament_slots"):
  2413. slots = archive.extra_data["filament_slots"]
  2414. if print_status != "completed":
  2415. scale = max(0.0, min((data.get("progress") or 0) / 100.0, 1.0))
  2416. slots = [{**s, "used_g": round(s["used_g"] * scale, 1)} for s in slots]
  2417. archive_data["filament_slots"] = slots
  2418. # Pass usage tracker results for AMS slot info in notifications
  2419. if usage_results:
  2420. archive_data["usage_results"] = usage_results
  2421. # Add finish photo URL and image bytes if available
  2422. if finish_photo_filename:
  2423. from backend.app.api.routes.settings import get_setting
  2424. external_url = await get_setting(db, "external_url")
  2425. if external_url:
  2426. external_url = external_url.rstrip("/")
  2427. archive_data["finish_photo_url"] = (
  2428. f"{external_url}/api/v1/archives/{archive_id}/photos/{finish_photo_filename}"
  2429. )
  2430. else:
  2431. # Fallback to relative URL (won't work for external services)
  2432. archive_data["finish_photo_url"] = (
  2433. f"/api/v1/archives/{archive_id}/photos/{finish_photo_filename}"
  2434. )
  2435. # Read finish photo bytes for image attachment (e.g. Pushover)
  2436. try:
  2437. from pathlib import Path
  2438. photo_path = (
  2439. app_settings.base_dir
  2440. / Path(archive.file_path).parent
  2441. / "photos"
  2442. / finish_photo_filename
  2443. )
  2444. if photo_path.exists():
  2445. photo_bytes = await asyncio.to_thread(photo_path.read_bytes)
  2446. if len(photo_bytes) <= 2_500_000:
  2447. archive_data["image_data"] = photo_bytes
  2448. logger.info("[NOTIFY-BG] Loaded finish photo bytes: %s bytes", len(photo_bytes))
  2449. else:
  2450. logger.warning(
  2451. f"[NOTIFY-BG] Finish photo too large for attachment: "
  2452. f"{len(photo_bytes)} bytes"
  2453. )
  2454. except Exception as e:
  2455. logger.warning("[NOTIFY-BG] Failed to read finish photo bytes: %s", e)
  2456. await notification_service.on_print_complete(
  2457. printer_id, printer_name, print_status, data, db, archive_data=archive_data
  2458. )
  2459. logger.info("[NOTIFY-BG] Completed")
  2460. except Exception as e:
  2461. logger.warning("[NOTIFY-BG] Failed: %s", e)
  2462. async def _background_maintenance_check():
  2463. """Check for maintenance due in background."""
  2464. if print_status != "completed":
  2465. return
  2466. try:
  2467. logger.info("[MAINT-BG] Starting maintenance check for printer %s", printer_id)
  2468. async with async_session() as db:
  2469. from backend.app.models.printer import Printer
  2470. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2471. printer = result.scalar_one_or_none()
  2472. printer_name = printer.name if printer else f"Printer {printer_id}"
  2473. await ensure_default_types(db)
  2474. overview = await _get_printer_maintenance_internal(printer_id, db, commit=True)
  2475. items_needing_attention = [
  2476. {"name": item.maintenance_type_name, "is_due": item.is_due, "is_warning": item.is_warning}
  2477. for item in overview.maintenance_items
  2478. if item.enabled and (item.is_due or item.is_warning)
  2479. ]
  2480. if items_needing_attention:
  2481. await notification_service.on_maintenance_due(printer_id, printer_name, items_needing_attention, db)
  2482. logger.info("[MAINT-BG] Sent notification: %s items need attention", len(items_needing_attention))
  2483. # MQTT relay - publish maintenance alerts
  2484. for item in items_needing_attention:
  2485. try:
  2486. await mqtt_relay.on_maintenance_alert(
  2487. printer_id=printer_id,
  2488. printer_name=printer_name,
  2489. maintenance_type=item["name"],
  2490. current_value=0, # Not easily available here
  2491. threshold=0, # Not easily available here
  2492. )
  2493. except Exception:
  2494. pass # Don't fail if MQTT fails
  2495. else:
  2496. logger.info("[MAINT-BG] Completed (no items need attention)")
  2497. except Exception as e:
  2498. logger.warning("[MAINT-BG] Failed: %s", e)
  2499. asyncio.create_task(_background_smart_plug())
  2500. asyncio.create_task(_background_maintenance_check())
  2501. # Notification task waits for photo capture to complete first
  2502. async def _photo_then_notify():
  2503. """Wait for photo capture, then send notification with photo URL."""
  2504. try:
  2505. finish_photo = await photo_task
  2506. logger.info("[PHOTO-NOTIFY] Photo task returned: %s", finish_photo)
  2507. await _background_notifications(finish_photo)
  2508. except Exception as e:
  2509. logger.warning("[PHOTO-NOTIFY] Failed: %s", e)
  2510. # Still try to send notification without photo
  2511. await _background_notifications(None)
  2512. asyncio.create_task(_photo_then_notify())
  2513. # Stitch external camera layer timelapse if session was active
  2514. print_status = data.get("status", "completed")
  2515. async def _background_layer_timelapse():
  2516. """Stitch layer timelapse and attach to archive."""
  2517. from backend.app.services.layer_timelapse import cancel_session, on_print_complete as tl_complete
  2518. try:
  2519. if print_status == "completed":
  2520. logger.info("[LAYER-TL] Stitching layer timelapse for printer %s", printer_id)
  2521. timelapse_path = await tl_complete(printer_id)
  2522. if timelapse_path and archive_id:
  2523. logger.info("[LAYER-TL] Attaching timelapse %s to archive %s", timelapse_path, archive_id)
  2524. async with async_session() as db:
  2525. service = ArchiveService(db)
  2526. timelapse_data = await asyncio.to_thread(timelapse_path.read_bytes)
  2527. await service.attach_timelapse(archive_id, timelapse_data, "layer_timelapse.mp4")
  2528. # Clean up the temp file
  2529. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  2530. logger.info("[LAYER-TL] Layer timelapse attached successfully")
  2531. elif timelapse_path:
  2532. # Timelapse created but no archive - just clean up
  2533. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  2534. else:
  2535. # Print failed or cancelled - cancel timelapse session
  2536. cancel_session(printer_id)
  2537. logger.info(
  2538. "[LAYER-TL] Cancelled layer timelapse for printer %s (status: %s)", printer_id, print_status
  2539. )
  2540. except Exception as e:
  2541. logger.warning("[LAYER-TL] Failed: %s", e)
  2542. # Try to cancel session on error
  2543. try:
  2544. cancel_session(printer_id)
  2545. except Exception:
  2546. pass # Best-effort timelapse session cancellation on error
  2547. asyncio.create_task(_background_layer_timelapse())
  2548. log_timing("All background tasks scheduled")
  2549. # Auto-scan for timelapse if recording was active during the print
  2550. if archive_id and data.get("timelapse_was_active") and data.get("status") == "completed":
  2551. logger.info("[TIMELAPSE] Timelapse was active during print, scheduling auto-scan for archive %s", archive_id)
  2552. # Schedule timelapse scan as background task with retries
  2553. # The printer needs time to encode the video after print completion
  2554. baseline = _timelapse_baselines.pop(printer_id, None)
  2555. asyncio.create_task(_scan_for_timelapse_with_retries(archive_id, baseline))
  2556. log_timing("Timelapse scan scheduled")
  2557. logger.info("[CALLBACK] on_print_complete finished for printer %s, archive %s", printer_id, archive_id)
  2558. # AMS sensor history recording
  2559. _ams_history_task: asyncio.Task | None = None
  2560. AMS_HISTORY_INTERVAL = 300 # Record every 5 minutes
  2561. AMS_HISTORY_RETENTION_DAYS = 30 # Keep data for 30 days
  2562. _ams_cleanup_counter = 0 # Track recordings to trigger periodic cleanup
  2563. # Track alarm cooldowns (printer_id:ams_id:type -> last_alarm_time)
  2564. _ams_alarm_cooldown: dict[str, datetime] = {}
  2565. AMS_ALARM_COOLDOWN_MINUTES = 60 # Don't send same alarm more than once per hour
  2566. async def record_ams_history():
  2567. """Background task to record AMS humidity and temperature data."""
  2568. logger = logging.getLogger(__name__)
  2569. # Wait a short time for MQTT connections to establish on startup
  2570. await asyncio.sleep(10)
  2571. while True:
  2572. try:
  2573. from backend.app.models.ams_history import AMSSensorHistory
  2574. from backend.app.models.printer import Printer
  2575. from backend.app.models.settings import Settings
  2576. async with async_session() as db:
  2577. # Get all active printers
  2578. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  2579. printers = result.scalars().all()
  2580. # Get alarm thresholds from settings
  2581. humidity_threshold = 60.0 # Default: fair threshold
  2582. temp_threshold = 35.0 # Default: fair threshold
  2583. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  2584. setting = result.scalar_one_or_none()
  2585. if setting:
  2586. try:
  2587. humidity_threshold = float(setting.value)
  2588. except (ValueError, TypeError):
  2589. pass # Keep default threshold if stored value is invalid
  2590. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_fair"))
  2591. setting = result.scalar_one_or_none()
  2592. if setting:
  2593. try:
  2594. temp_threshold = float(setting.value)
  2595. except (ValueError, TypeError):
  2596. pass # Keep default threshold if stored value is invalid
  2597. recorded_count = 0
  2598. for printer in printers:
  2599. # Get current state from printer manager
  2600. state = printer_manager.get_status(printer.id)
  2601. if not state or not state.connected or not state.raw_data:
  2602. continue # Skip disconnected printers - don't use stale data
  2603. raw_data = state.raw_data
  2604. if "ams" not in raw_data or not isinstance(raw_data["ams"], list):
  2605. continue
  2606. # Record data for each AMS unit
  2607. for ams_data in raw_data["ams"]:
  2608. ams_id = int(ams_data.get("id", 0))
  2609. # Get humidity (prefer humidity_raw)
  2610. humidity_raw = ams_data.get("humidity_raw")
  2611. humidity_idx = ams_data.get("humidity")
  2612. humidity = None
  2613. if humidity_raw is not None:
  2614. try:
  2615. humidity = float(humidity_raw)
  2616. except (ValueError, TypeError):
  2617. pass # Skip unparseable humidity; will try fallback
  2618. if humidity is None and humidity_idx is not None:
  2619. try:
  2620. humidity = float(humidity_idx)
  2621. except (ValueError, TypeError):
  2622. pass # Skip unparseable humidity index value
  2623. # Get temperature
  2624. temperature = None
  2625. temp_str = ams_data.get("temp")
  2626. if temp_str is not None:
  2627. try:
  2628. temperature = float(temp_str)
  2629. except (ValueError, TypeError):
  2630. pass # Skip unparseable temperature value
  2631. # Skip if no data
  2632. if humidity is None and temperature is None:
  2633. continue
  2634. # Record the data point
  2635. history = AMSSensorHistory(
  2636. printer_id=printer.id,
  2637. ams_id=ams_id,
  2638. humidity=humidity,
  2639. humidity_raw=float(humidity_raw) if humidity_raw else None,
  2640. temperature=temperature,
  2641. )
  2642. db.add(history)
  2643. recorded_count += 1
  2644. # Generate AMS label and determine if it's AMS-HT (A, B, C, D or HT-A for AMS-Lite/Hub)
  2645. is_ams_ht = ams_id >= 128
  2646. if is_ams_ht:
  2647. ams_label = f"HT-{chr(65 + (ams_id - 128))}"
  2648. else:
  2649. ams_label = f"AMS-{chr(65 + ams_id)}"
  2650. # Check humidity alarm (only if above threshold)
  2651. if humidity is not None and humidity > humidity_threshold:
  2652. cooldown_key = f"{printer.id}:{ams_id}:humidity"
  2653. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  2654. now = datetime.now(timezone.utc)
  2655. if (
  2656. last_alarm is None
  2657. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  2658. ):
  2659. _ams_alarm_cooldown[cooldown_key] = now
  2660. logger.info(
  2661. f"Sending humidity alarm for {printer.name} {ams_label}: {humidity}% > {humidity_threshold}%"
  2662. )
  2663. try:
  2664. # Call different notification method based on AMS type
  2665. if is_ams_ht:
  2666. await notification_service.on_ams_ht_humidity_high(
  2667. printer.id, printer.name, ams_label, humidity, humidity_threshold, db
  2668. )
  2669. else:
  2670. await notification_service.on_ams_humidity_high(
  2671. printer.id, printer.name, ams_label, humidity, humidity_threshold, db
  2672. )
  2673. except Exception as e:
  2674. logger.warning("Failed to send humidity alarm: %s", e)
  2675. # Check temperature alarm (only if above threshold)
  2676. if temperature is not None and temperature > temp_threshold:
  2677. cooldown_key = f"{printer.id}:{ams_id}:temperature"
  2678. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  2679. now = datetime.now(timezone.utc)
  2680. if (
  2681. last_alarm is None
  2682. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  2683. ):
  2684. _ams_alarm_cooldown[cooldown_key] = now
  2685. logger.info(
  2686. f"Sending temperature alarm for {printer.name} {ams_label}: {temperature}°C > {temp_threshold}°C"
  2687. )
  2688. try:
  2689. # Call different notification method based on AMS type
  2690. if is_ams_ht:
  2691. await notification_service.on_ams_ht_temperature_high(
  2692. printer.id, printer.name, ams_label, temperature, temp_threshold, db
  2693. )
  2694. else:
  2695. await notification_service.on_ams_temperature_high(
  2696. printer.id, printer.name, ams_label, temperature, temp_threshold, db
  2697. )
  2698. except Exception as e:
  2699. logger.warning("Failed to send temperature alarm: %s", e)
  2700. await db.commit()
  2701. if recorded_count > 0:
  2702. logger.info("Recorded %s AMS sensor history entries", recorded_count)
  2703. # Periodic cleanup of old data (every ~288 recordings = ~24 hours at 5min interval)
  2704. global _ams_cleanup_counter
  2705. _ams_cleanup_counter += 1
  2706. if _ams_cleanup_counter >= 288:
  2707. _ams_cleanup_counter = 0
  2708. # Get retention days from settings
  2709. from backend.app.models.settings import Settings
  2710. result = await db.execute(select(Settings).where(Settings.key == "ams_history_retention_days"))
  2711. setting = result.scalar_one_or_none()
  2712. retention_days = int(setting.value) if setting else AMS_HISTORY_RETENTION_DAYS
  2713. cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
  2714. result = await db.execute(delete(AMSSensorHistory).where(AMSSensorHistory.recorded_at < cutoff))
  2715. await db.commit()
  2716. if result.rowcount > 0:
  2717. logger.info(
  2718. f"Cleaned up {result.rowcount} old AMS sensor history entries (older than {retention_days} days)"
  2719. )
  2720. # Wait until next recording interval
  2721. await asyncio.sleep(AMS_HISTORY_INTERVAL)
  2722. except asyncio.CancelledError:
  2723. break
  2724. except Exception as e:
  2725. logger.warning("AMS history recording failed: %s", e)
  2726. await asyncio.sleep(60) # Wait a bit before retrying
  2727. def start_ams_history_recording():
  2728. """Start the AMS history recording background task."""
  2729. global _ams_history_task
  2730. if _ams_history_task is None:
  2731. _ams_history_task = asyncio.create_task(record_ams_history())
  2732. logging.getLogger(__name__).info("AMS history recording started")
  2733. def stop_ams_history_recording():
  2734. """Stop the AMS history recording background task."""
  2735. global _ams_history_task
  2736. if _ams_history_task:
  2737. _ams_history_task.cancel()
  2738. _ams_history_task = None
  2739. logging.getLogger(__name__).info("AMS history recording stopped")
  2740. # Printer runtime tracking
  2741. _runtime_tracking_task: asyncio.Task | None = None
  2742. RUNTIME_TRACKING_INTERVAL = 30 # Update every 30 seconds
  2743. async def track_printer_runtime():
  2744. """Background task to track printer active runtime (RUNNING/PAUSE states)."""
  2745. logger = logging.getLogger(__name__)
  2746. # Wait for MQTT connections to establish on startup
  2747. await asyncio.sleep(15)
  2748. while True:
  2749. try:
  2750. from backend.app.models.printer import Printer
  2751. async with async_session() as db:
  2752. # Get all active printers
  2753. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  2754. printers = result.scalars().all()
  2755. now = datetime.now(timezone.utc)
  2756. updated_count = 0
  2757. needs_commit = False
  2758. for printer in printers:
  2759. # Get current state from printer manager
  2760. state = printer_manager.get_status(printer.id)
  2761. if not state:
  2762. logger.debug("[%s] Runtime tracking: no state available", printer.name)
  2763. continue
  2764. if not state.connected:
  2765. logger.debug("[%s] Runtime tracking: not connected", printer.name)
  2766. continue
  2767. # Check if printer is in an active state (RUNNING or PAUSE)
  2768. if state.state in ("RUNNING", "PAUSE"):
  2769. # Calculate time since last update
  2770. if printer.last_runtime_update:
  2771. last_update = printer.last_runtime_update
  2772. if last_update.tzinfo is None:
  2773. last_update = last_update.replace(tzinfo=timezone.utc)
  2774. elapsed = (now - last_update).total_seconds()
  2775. if elapsed > 0:
  2776. printer.runtime_seconds += int(elapsed)
  2777. updated_count += 1
  2778. needs_commit = True
  2779. logger.debug(
  2780. f"[{printer.name}] Runtime tracking: added {int(elapsed)}s, "
  2781. f"total={printer.runtime_seconds}s ({printer.runtime_seconds / 3600:.2f}h)"
  2782. )
  2783. else:
  2784. # First time seeing printer active - need to commit to save timestamp
  2785. needs_commit = True
  2786. logger.debug("[%s] Runtime tracking: first active detection", printer.name)
  2787. printer.last_runtime_update = now
  2788. else:
  2789. # Printer is idle/offline - clear last_runtime_update
  2790. if printer.last_runtime_update is not None:
  2791. logger.debug(
  2792. f"[{printer.name}] Runtime tracking: state={state.state}, clearing last_runtime_update"
  2793. )
  2794. printer.last_runtime_update = None
  2795. needs_commit = True
  2796. if needs_commit:
  2797. await db.commit()
  2798. if updated_count > 0:
  2799. logger.debug("Updated runtime for %s printer(s)", updated_count)
  2800. except asyncio.CancelledError:
  2801. logger.info("Runtime tracking cancelled")
  2802. break
  2803. except Exception as e:
  2804. logger.warning("Runtime tracking failed: %s", e)
  2805. await asyncio.sleep(RUNTIME_TRACKING_INTERVAL)
  2806. def start_runtime_tracking():
  2807. """Start the printer runtime tracking background task."""
  2808. global _runtime_tracking_task
  2809. if _runtime_tracking_task is None:
  2810. _runtime_tracking_task = asyncio.create_task(track_printer_runtime())
  2811. logging.getLogger(__name__).info("Printer runtime tracking started")
  2812. def stop_runtime_tracking():
  2813. """Stop the printer runtime tracking background task."""
  2814. global _runtime_tracking_task
  2815. if _runtime_tracking_task:
  2816. _runtime_tracking_task.cancel()
  2817. _runtime_tracking_task = None
  2818. logging.getLogger(__name__).info("Printer runtime tracking stopped")
  2819. # SpoolBuddy device watchdog
  2820. _spoolbuddy_watchdog_task: asyncio.Task | None = None
  2821. SPOOLBUDDY_WATCHDOG_INTERVAL = 15
  2822. async def _spoolbuddy_watchdog_loop():
  2823. """Periodic check for SpoolBuddy devices that have gone offline."""
  2824. from backend.app.api.routes.spoolbuddy import spoolbuddy_watchdog
  2825. while True:
  2826. try:
  2827. await spoolbuddy_watchdog()
  2828. except asyncio.CancelledError:
  2829. break
  2830. except Exception as e:
  2831. logging.getLogger(__name__).warning("SpoolBuddy watchdog failed: %s", e)
  2832. await asyncio.sleep(SPOOLBUDDY_WATCHDOG_INTERVAL)
  2833. def start_spoolbuddy_watchdog():
  2834. global _spoolbuddy_watchdog_task
  2835. if _spoolbuddy_watchdog_task is None:
  2836. _spoolbuddy_watchdog_task = asyncio.create_task(_spoolbuddy_watchdog_loop())
  2837. logging.getLogger(__name__).info("SpoolBuddy watchdog started")
  2838. def stop_spoolbuddy_watchdog():
  2839. global _spoolbuddy_watchdog_task
  2840. if _spoolbuddy_watchdog_task:
  2841. _spoolbuddy_watchdog_task.cancel()
  2842. _spoolbuddy_watchdog_task = None
  2843. logging.getLogger(__name__).info("SpoolBuddy watchdog stopped")
  2844. # Camera stream orphan cleanup
  2845. _camera_cleanup_task: asyncio.Task | None = None
  2846. CAMERA_CLEANUP_INTERVAL = 60
  2847. async def _camera_cleanup_loop():
  2848. """Periodically clean up orphaned ffmpeg processes."""
  2849. from backend.app.api.routes.camera import cleanup_orphaned_streams
  2850. while True:
  2851. try:
  2852. await cleanup_orphaned_streams()
  2853. except asyncio.CancelledError:
  2854. break
  2855. except Exception as e:
  2856. logging.getLogger(__name__).warning("Camera stream cleanup failed: %s", e)
  2857. await asyncio.sleep(CAMERA_CLEANUP_INTERVAL)
  2858. def start_camera_cleanup():
  2859. global _camera_cleanup_task
  2860. if _camera_cleanup_task is None:
  2861. _camera_cleanup_task = asyncio.create_task(_camera_cleanup_loop())
  2862. logging.getLogger(__name__).info("Camera stream cleanup started")
  2863. def stop_camera_cleanup():
  2864. global _camera_cleanup_task
  2865. if _camera_cleanup_task:
  2866. _camera_cleanup_task.cancel()
  2867. _camera_cleanup_task = None
  2868. logging.getLogger(__name__).info("Camera stream cleanup stopped")
  2869. @asynccontextmanager
  2870. async def lifespan(app: FastAPI):
  2871. # Startup
  2872. await init_db()
  2873. # Fix queue items stuck with invalid "aborted" status (should be "cancelled").
  2874. # This can happen when a print was cancelled mid-print on versions before this fix.
  2875. try:
  2876. async with async_session() as db:
  2877. from backend.app.models.print_queue import PrintQueueItem
  2878. result = await db.execute(select(PrintQueueItem).where(PrintQueueItem.status == "aborted"))
  2879. aborted_items = result.scalars().all()
  2880. if aborted_items:
  2881. for item in aborted_items:
  2882. item.status = "cancelled"
  2883. await db.commit()
  2884. logging.info("Fixed %d queue item(s) with invalid 'aborted' status → 'cancelled'", len(aborted_items))
  2885. except Exception as e:
  2886. logging.warning("Failed to fix aborted queue items: %s", e)
  2887. # Restore debug logging state from previous session
  2888. await init_debug_logging()
  2889. # Set up printer manager callbacks
  2890. loop = asyncio.get_event_loop()
  2891. printer_manager.set_event_loop(loop)
  2892. printer_manager.set_status_change_callback(on_printer_status_change)
  2893. printer_manager.set_print_start_callback(on_print_start)
  2894. printer_manager.set_print_complete_callback(on_print_complete)
  2895. printer_manager.set_ams_change_callback(on_ams_change)
  2896. # Layer change callback for external camera timelapse
  2897. async def on_layer_change(printer_id: int, layer_num: int):
  2898. """Capture timelapse frame on layer change + first layer notification."""
  2899. from backend.app.services.layer_timelapse import on_layer_change as tl_layer_change
  2900. await tl_layer_change(printer_id, layer_num)
  2901. # First layer complete notification (layer_num >= 2 means layer 1 is done)
  2902. if 2 <= layer_num <= 5 and not _first_layer_notified.get(printer_id, False):
  2903. _first_layer_notified[printer_id] = True
  2904. try:
  2905. async with async_session() as db:
  2906. from backend.app.models.printer import Printer
  2907. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  2908. printer = result.scalar_one_or_none()
  2909. if not printer:
  2910. return
  2911. printer_name = printer.name
  2912. client = printer_manager.get_client(printer_id)
  2913. state = client.state if client else None
  2914. filename = (state.subtask_name or state.gcode_file or "Unknown") if state else "Unknown"
  2915. total_layers = state.total_layers if state else 0
  2916. image_data = await _capture_snapshot_for_notification(
  2917. printer_id, printer, logging.getLogger(__name__)
  2918. )
  2919. await notification_service.on_first_layer_complete(
  2920. printer_id, printer_name, filename, total_layers, db, image_data=image_data
  2921. )
  2922. except Exception as e:
  2923. logging.getLogger(__name__).warning("First layer notification failed: %s", e)
  2924. printer_manager.set_layer_change_callback(on_layer_change)
  2925. # Initialize MQTT relay from settings
  2926. async with async_session() as db:
  2927. from backend.app.api.routes.settings import get_setting
  2928. mqtt_settings = {
  2929. "mqtt_enabled": (await get_setting(db, "mqtt_enabled") or "false") == "true",
  2930. "mqtt_broker": await get_setting(db, "mqtt_broker") or "",
  2931. "mqtt_port": int(await get_setting(db, "mqtt_port") or "1883"),
  2932. "mqtt_username": await get_setting(db, "mqtt_username") or "",
  2933. "mqtt_password": await get_setting(db, "mqtt_password") or "",
  2934. "mqtt_topic_prefix": await get_setting(db, "mqtt_topic_prefix") or "bambuddy",
  2935. "mqtt_use_tls": (await get_setting(db, "mqtt_use_tls") or "false") == "true",
  2936. }
  2937. await mqtt_relay.configure(mqtt_settings)
  2938. # Restore MQTT smart plug subscriptions
  2939. if mqtt_settings.get("mqtt_enabled"):
  2940. from backend.app.models.smart_plug import SmartPlug
  2941. result = await db.execute(select(SmartPlug).where(SmartPlug.plug_type == "mqtt"))
  2942. mqtt_plugs = result.scalars().all()
  2943. for plug in mqtt_plugs:
  2944. if plug.mqtt_topic:
  2945. mqtt_relay.smart_plug_service.subscribe(
  2946. plug_id=plug.id,
  2947. topic=plug.mqtt_topic,
  2948. power_path=plug.mqtt_power_path,
  2949. energy_path=plug.mqtt_energy_path,
  2950. state_path=plug.mqtt_state_path,
  2951. multiplier=plug.mqtt_multiplier or 1.0,
  2952. )
  2953. if mqtt_plugs:
  2954. logging.info("Restored %s MQTT smart plug subscriptions", len(mqtt_plugs))
  2955. # Connect to all active printers
  2956. async with async_session() as db:
  2957. await init_printer_connections(db)
  2958. # Auto-connect to Spoolman if enabled
  2959. async with async_session() as db:
  2960. from backend.app.api.routes.settings import get_setting
  2961. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  2962. spoolman_url = await get_setting(db, "spoolman_url")
  2963. if spoolman_enabled and spoolman_enabled.lower() == "true" and spoolman_url:
  2964. try:
  2965. client = await init_spoolman_client(spoolman_url)
  2966. if await client.health_check():
  2967. logging.info("Auto-connected to Spoolman at %s", spoolman_url)
  2968. # Ensure the 'tag' extra field exists for RFID/UUID storage
  2969. await client.ensure_tag_extra_field()
  2970. else:
  2971. logging.warning("Spoolman at %s is not reachable", spoolman_url)
  2972. except Exception as e:
  2973. logging.warning("Failed to auto-connect to Spoolman: %s", e)
  2974. # Start the print scheduler
  2975. asyncio.create_task(print_scheduler.run())
  2976. # Start background dispatch worker for send/start operations
  2977. await background_dispatch.start()
  2978. # Start the smart plug scheduler for time-based on/off
  2979. smart_plug_manager.start_scheduler()
  2980. # Resume any pending auto-offs that were interrupted by restart
  2981. await smart_plug_manager.resume_pending_auto_offs()
  2982. # Start the notification digest scheduler
  2983. notification_service.start_digest_scheduler()
  2984. # Start the GitHub backup scheduler
  2985. await github_backup_service.start_scheduler()
  2986. # Start AMS history recording
  2987. start_ams_history_recording()
  2988. # Start printer runtime tracking
  2989. start_runtime_tracking()
  2990. # Start SpoolBuddy device watchdog
  2991. start_spoolbuddy_watchdog()
  2992. # Start camera stream orphan cleanup
  2993. start_camera_cleanup()
  2994. # Initialize virtual printer manager and sync from DB
  2995. from backend.app.services.virtual_printer import virtual_printer_manager
  2996. virtual_printer_manager.set_session_factory(async_session)
  2997. try:
  2998. await virtual_printer_manager.sync_from_db()
  2999. logging.info("Virtual printer manager synced from database")
  3000. except Exception as e:
  3001. logging.warning("Failed to sync virtual printers: %s", e)
  3002. yield
  3003. # Shutdown
  3004. print_scheduler.stop()
  3005. await background_dispatch.stop()
  3006. smart_plug_manager.stop_scheduler()
  3007. notification_service.stop_digest_scheduler()
  3008. github_backup_service.stop_scheduler()
  3009. stop_ams_history_recording()
  3010. stop_runtime_tracking()
  3011. stop_spoolbuddy_watchdog()
  3012. stop_camera_cleanup()
  3013. printer_manager.disconnect_all()
  3014. await close_spoolman_client()
  3015. # Stop all virtual printer services
  3016. await virtual_printer_manager.stop_all()
  3017. await mqtt_smart_plug_service.disconnect(timeout=2)
  3018. await mqtt_relay.disconnect(timeout=2)
  3019. # Checkpoint WAL and close all database connections
  3020. try:
  3021. async with engine.begin() as conn:
  3022. await conn.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
  3023. logging.info("WAL checkpoint completed")
  3024. except Exception as e:
  3025. logging.warning("WAL checkpoint failed: %s", e)
  3026. await engine.dispose()
  3027. app = FastAPI(
  3028. title=app_settings.app_name,
  3029. description="Archive and manage Bambu Lab 3MF files",
  3030. version=APP_VERSION,
  3031. lifespan=lifespan,
  3032. )
  3033. # =============================================================================
  3034. # Authentication Middleware - Secures ALL API routes by default
  3035. # =============================================================================
  3036. # Public routes that don't require authentication even when auth is enabled
  3037. PUBLIC_API_ROUTES = {
  3038. # Auth routes needed before/during login
  3039. "/api/v1/auth/status",
  3040. "/api/v1/auth/login",
  3041. "/api/v1/auth/setup", # Needed for initial setup and recovery
  3042. # Advanced auth status needed for login page
  3043. "/api/v1/auth/advanced-auth/status",
  3044. "/api/v1/auth/forgot-password", # Password reset for advanced auth
  3045. # Version check for updates (no sensitive data)
  3046. "/api/v1/updates/version",
  3047. # Metrics endpoint handles its own prometheus_token authentication
  3048. "/api/v1/metrics",
  3049. }
  3050. # Route prefixes that are public (for routes with dynamic segments)
  3051. PUBLIC_API_PREFIXES = [
  3052. # WebSocket connections handle their own auth
  3053. "/api/v1/ws",
  3054. ]
  3055. # Route patterns that are public (read-only display data)
  3056. # These are checked with "in path" - needed because browsers load images/videos
  3057. # via <img src> and <video src> which don't include Authorization headers
  3058. PUBLIC_API_PATTERNS = [
  3059. # Thumbnails
  3060. "/thumbnail", # /archives/{id}/thumbnail, /library/files/{id}/thumbnail
  3061. "/plate-thumbnail/", # /archives/{id}/plate-thumbnail/{plate_id}
  3062. # Images and media
  3063. "/photos/", # /archives/{id}/photos/{filename}
  3064. "/project-image/", # /archives/{id}/project-image/{path}
  3065. "/qrcode", # /archives/{id}/qrcode
  3066. "/timelapse", # /archives/{id}/timelapse (video)
  3067. "/cover", # /printers/{id}/cover
  3068. "/icon", # /external-links/{id}/icon
  3069. # Camera (streams loaded via <img> tag)
  3070. "/camera/stream", # /printers/{id}/camera/stream
  3071. "/camera/snapshot", # /printers/{id}/camera/snapshot
  3072. # Slicer token-authenticated downloads — protocol handlers (bambustudioopen://,
  3073. # orcaslicer://) cannot send auth headers. These endpoints validate a short-lived
  3074. # download token in the URL path instead.
  3075. "/dl/", # /archives/{id}/dl/{token}/{filename}, /library/files/{id}/dl/{token}/{filename}
  3076. ]
  3077. @app.middleware("http")
  3078. async def auth_middleware(request, call_next):
  3079. """Enforce authentication on all API routes when auth is enabled.
  3080. This middleware provides defense-in-depth by checking auth at the API gateway level,
  3081. regardless of whether individual routes have auth dependencies.
  3082. """
  3083. from starlette.responses import JSONResponse
  3084. path = request.url.path
  3085. # Only apply to API routes
  3086. if not path.startswith("/api/"):
  3087. return await call_next(request)
  3088. # Allow public routes
  3089. if path in PUBLIC_API_ROUTES:
  3090. return await call_next(request)
  3091. # Allow public prefixes
  3092. for prefix in PUBLIC_API_PREFIXES:
  3093. if path.startswith(prefix):
  3094. return await call_next(request)
  3095. # Allow public patterns (read-only display data like thumbnails)
  3096. for pattern in PUBLIC_API_PATTERNS:
  3097. if pattern in path:
  3098. return await call_next(request)
  3099. # Check if auth is enabled
  3100. try:
  3101. async with async_session() as db:
  3102. from backend.app.core.auth import is_auth_enabled
  3103. auth_enabled = await is_auth_enabled(db)
  3104. if not auth_enabled:
  3105. # Auth disabled, allow all requests
  3106. return await call_next(request)
  3107. except Exception:
  3108. # If we can't check auth status, allow request (fail open for DB issues)
  3109. return await call_next(request)
  3110. # Auth is enabled - require valid token
  3111. auth_header = request.headers.get("Authorization")
  3112. x_api_key = request.headers.get("X-API-Key")
  3113. # Check for API key auth first
  3114. if x_api_key or (auth_header and auth_header.startswith("Bearer bb_")):
  3115. # API key authentication - let the request through to be validated by route handler
  3116. # API keys are validated per-route since they have different permission levels
  3117. return await call_next(request)
  3118. # Check for JWT auth
  3119. if not auth_header or not auth_header.startswith("Bearer "):
  3120. return JSONResponse(
  3121. status_code=401,
  3122. content={"detail": "Authentication required"},
  3123. headers={"WWW-Authenticate": "Bearer"},
  3124. )
  3125. # Validate JWT token
  3126. import jwt
  3127. try:
  3128. from backend.app.core.auth import ALGORITHM, SECRET_KEY
  3129. token = auth_header.replace("Bearer ", "")
  3130. payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
  3131. username = payload.get("sub")
  3132. if not username:
  3133. raise ValueError("No username in token")
  3134. # Verify user exists and is active
  3135. async with async_session() as db:
  3136. from backend.app.core.auth import get_user_by_username
  3137. user = await get_user_by_username(db, username)
  3138. if not user or not user.is_active:
  3139. return JSONResponse(
  3140. status_code=401,
  3141. content={"detail": "User not found or inactive"},
  3142. headers={"WWW-Authenticate": "Bearer"},
  3143. )
  3144. except jwt.ExpiredSignatureError:
  3145. return JSONResponse(
  3146. status_code=401,
  3147. content={"detail": "Token has expired"},
  3148. headers={"WWW-Authenticate": "Bearer"},
  3149. )
  3150. except (jwt.InvalidTokenError, ValueError, Exception):
  3151. return JSONResponse(
  3152. status_code=401,
  3153. content={"detail": "Invalid token"},
  3154. headers={"WWW-Authenticate": "Bearer"},
  3155. )
  3156. return await call_next(request)
  3157. # API routes
  3158. app.include_router(auth.router, prefix=app_settings.api_prefix)
  3159. app.include_router(bug_report.router, prefix=app_settings.api_prefix)
  3160. app.include_router(users.router, prefix=app_settings.api_prefix)
  3161. app.include_router(groups.router, prefix=app_settings.api_prefix)
  3162. app.include_router(printers.router, prefix=app_settings.api_prefix)
  3163. app.include_router(archives.router, prefix=app_settings.api_prefix)
  3164. app.include_router(filaments.router, prefix=app_settings.api_prefix)
  3165. app.include_router(inventory.router, prefix=app_settings.api_prefix)
  3166. app.include_router(settings_routes.router, prefix=app_settings.api_prefix)
  3167. app.include_router(cloud.router, prefix=app_settings.api_prefix)
  3168. app.include_router(local_presets.router, prefix=app_settings.api_prefix)
  3169. app.include_router(smart_plugs.router, prefix=app_settings.api_prefix)
  3170. app.include_router(print_log.router, prefix=app_settings.api_prefix)
  3171. app.include_router(print_queue.router, prefix=app_settings.api_prefix)
  3172. app.include_router(background_dispatch_routes.router, prefix=app_settings.api_prefix)
  3173. app.include_router(kprofiles.router, prefix=app_settings.api_prefix)
  3174. app.include_router(notifications.router, prefix=app_settings.api_prefix)
  3175. app.include_router(notification_templates.router, prefix=app_settings.api_prefix)
  3176. app.include_router(spoolman.router, prefix=app_settings.api_prefix)
  3177. app.include_router(updates.router, prefix=app_settings.api_prefix)
  3178. app.include_router(maintenance.router, prefix=app_settings.api_prefix)
  3179. app.include_router(camera.router, prefix=app_settings.api_prefix)
  3180. app.include_router(external_links.router, prefix=app_settings.api_prefix)
  3181. app.include_router(projects.router, prefix=app_settings.api_prefix)
  3182. app.include_router(library.router, prefix=app_settings.api_prefix)
  3183. app.include_router(api_keys.router, prefix=app_settings.api_prefix)
  3184. app.include_router(webhook.router, prefix=app_settings.api_prefix)
  3185. app.include_router(ams_history.router, prefix=app_settings.api_prefix)
  3186. app.include_router(system.router, prefix=app_settings.api_prefix)
  3187. app.include_router(support.router, prefix=app_settings.api_prefix)
  3188. app.include_router(websocket.router, prefix=app_settings.api_prefix)
  3189. app.include_router(discovery.router, prefix=app_settings.api_prefix)
  3190. app.include_router(pending_uploads.router, prefix=app_settings.api_prefix)
  3191. app.include_router(firmware.router, prefix=app_settings.api_prefix)
  3192. app.include_router(github_backup.router, prefix=app_settings.api_prefix)
  3193. app.include_router(metrics.router, prefix=app_settings.api_prefix)
  3194. app.include_router(virtual_printers.router, prefix=app_settings.api_prefix)
  3195. app.include_router(spoolbuddy.router, prefix=app_settings.api_prefix)
  3196. # Serve static files (React build)
  3197. if app_settings.static_dir.exists() and any(app_settings.static_dir.iterdir()):
  3198. app.mount(
  3199. "/assets",
  3200. StaticFiles(directory=app_settings.static_dir / "assets"),
  3201. name="assets",
  3202. )
  3203. if (app_settings.static_dir / "img").exists():
  3204. app.mount(
  3205. "/img",
  3206. StaticFiles(directory=app_settings.static_dir / "img"),
  3207. name="img",
  3208. )
  3209. if (app_settings.static_dir / "icons").exists():
  3210. app.mount(
  3211. "/icons",
  3212. StaticFiles(directory=app_settings.static_dir / "icons"),
  3213. name="icons",
  3214. )
  3215. @app.get("/")
  3216. async def serve_frontend():
  3217. """Serve the React frontend."""
  3218. index_file = app_settings.static_dir / "index.html"
  3219. if index_file.exists():
  3220. return FileResponse(index_file)
  3221. return {
  3222. "message": "Bambuddy API",
  3223. "docs": "/docs",
  3224. "frontend": "Build and place React app in /static directory",
  3225. }
  3226. @app.get("/health")
  3227. async def health_check():
  3228. """Health check endpoint."""
  3229. return {"status": "healthy"}
  3230. @app.get("/manifest.json")
  3231. async def serve_manifest():
  3232. """Serve PWA manifest."""
  3233. manifest_file = app_settings.static_dir / "manifest.json"
  3234. if manifest_file.exists():
  3235. return FileResponse(manifest_file, media_type="application/manifest+json")
  3236. return {"error": "Manifest not found"}
  3237. @app.get("/sw.js")
  3238. async def serve_service_worker():
  3239. """Serve service worker."""
  3240. sw_file = app_settings.static_dir / "sw.js"
  3241. if sw_file.exists():
  3242. return FileResponse(sw_file, media_type="application/javascript")
  3243. return {"error": "Service worker not found"}
  3244. # Catch-all route for React Router (must be last)
  3245. @app.get("/{full_path:path}")
  3246. async def serve_spa(full_path: str):
  3247. """Serve React app for client-side routing."""
  3248. # Don't intercept API routes - raise proper 404 so FastAPI can handle redirects
  3249. if full_path.startswith("api/"):
  3250. from fastapi import HTTPException
  3251. raise HTTPException(status_code=404, detail="Not found")
  3252. index_file = app_settings.static_dir / "index.html"
  3253. if index_file.exists():
  3254. return FileResponse(index_file)
  3255. return {"error": "Frontend not built"}