main.py 120 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646
  1. import asyncio
  2. import logging
  3. from contextlib import asynccontextmanager
  4. from datetime import UTC, datetime, timedelta
  5. from logging.handlers import RotatingFileHandler
  6. # =============================================================================
  7. # Dependency Check - runs before other imports to give helpful error messages
  8. # =============================================================================
  9. def _start_error_server(missing_packages: list):
  10. """Start a minimal HTTP server to display dependency errors in browser."""
  11. import os
  12. import signal
  13. from http.server import BaseHTTPRequestHandler, HTTPServer
  14. packages_html = "".join(f"<li><code>{p}</code></li>" for p in missing_packages)
  15. html = f"""<!DOCTYPE html>
  16. <html>
  17. <head>
  18. <title>Bambuddy - Setup Required</title>
  19. <style>
  20. body {{
  21. font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
  22. background: #0f172a; color: #e2e8f0;
  23. display: flex; justify-content: center; align-items: center;
  24. min-height: 100vh; margin: 0; padding: 20px; box-sizing: border-box;
  25. }}
  26. .container {{
  27. background: #1e293b; border-radius: 12px; padding: 40px;
  28. max-width: 600px; text-align: center; box-shadow: 0 4px 20px rgba(0,0,0,0.3);
  29. }}
  30. h1 {{ color: #f87171; margin-bottom: 10px; }}
  31. h2 {{ color: #94a3b8; font-weight: normal; margin-top: 0; }}
  32. .packages {{
  33. background: #0f172a; border-radius: 8px; padding: 20px;
  34. margin: 20px 0; text-align: left;
  35. }}
  36. .packages ul {{ margin: 0; padding-left: 20px; }}
  37. .packages li {{ color: #fbbf24; margin: 8px 0; }}
  38. .command {{
  39. background: #0f172a; border-radius: 8px; padding: 15px 20px;
  40. margin: 15px 0; font-family: monospace; color: #4ade80;
  41. text-align: left; overflow-x: auto;
  42. }}
  43. .note {{ color: #94a3b8; font-size: 14px; margin-top: 20px; }}
  44. </style>
  45. </head>
  46. <body>
  47. <div class="container">
  48. <h1>Setup Required</h1>
  49. <h2>Missing Python packages</h2>
  50. <div class="packages"><ul>{packages_html}</ul></div>
  51. <p>To fix, run this command on your server:</p>
  52. <div class="command">pip install -r requirements.txt</div>
  53. <p>Or if using a virtual environment:</p>
  54. <div class="command">./venv/bin/pip install -r requirements.txt</div>
  55. <p class="note">After installing, restart Bambuddy:<br>
  56. <code>sudo systemctl restart bambuddy</code></p>
  57. </div>
  58. </body>
  59. </html>"""
  60. class ErrorHandler(BaseHTTPRequestHandler):
  61. def do_GET(self):
  62. self.send_response(503)
  63. self.send_header("Content-type", "text/html")
  64. self.end_headers()
  65. self.wfile.write(html.encode())
  66. def log_message(self, format, *args):
  67. print(f"[Error Server] {args[0]}")
  68. port = int(os.environ.get("PORT", 8000))
  69. print(f"\nStarting error server on http://0.0.0.0:{port}")
  70. print("Visit this URL in your browser to see the error details.\n")
  71. server = HTTPServer(("0.0.0.0", port), ErrorHandler)
  72. def shutdown(signum, frame):
  73. print("\nShutting down error server...")
  74. raise SystemExit(0)
  75. signal.signal(signal.SIGTERM, shutdown)
  76. signal.signal(signal.SIGINT, shutdown)
  77. server.serve_forever()
  78. def check_dependencies():
  79. """Check that all required packages are installed."""
  80. missing = []
  81. # Map of import name -> package name (for pip install)
  82. required = {
  83. "jwt": "PyJWT",
  84. "fastapi": "fastapi",
  85. "uvicorn": "uvicorn",
  86. "sqlalchemy": "sqlalchemy",
  87. "aiosqlite": "aiosqlite",
  88. "pydantic": "pydantic",
  89. "paho.mqtt": "paho-mqtt",
  90. }
  91. for module, package in required.items():
  92. try:
  93. __import__(module)
  94. except ImportError:
  95. missing.append(package)
  96. if missing:
  97. print("\n" + "=" * 60)
  98. print("ERROR: Missing required Python packages!")
  99. print("=" * 60)
  100. print(f"\nMissing packages: {', '.join(missing)}")
  101. print("\nTo fix, run:")
  102. print(" pip install -r requirements.txt")
  103. print("\nOr if using a virtual environment:")
  104. print(" ./venv/bin/pip install -r requirements.txt")
  105. print("=" * 60 + "\n")
  106. _start_error_server(missing)
  107. check_dependencies()
  108. # =============================================================================
  109. from fastapi import FastAPI
  110. # Import settings first for logging configuration
  111. from backend.app.core.config import APP_VERSION, settings as app_settings
  112. # Configure logging based on settings
  113. # DEBUG=true -> DEBUG level, else use LOG_LEVEL setting
  114. log_level_str = "DEBUG" if app_settings.debug else app_settings.log_level.upper()
  115. log_level = getattr(logging, log_level_str, logging.INFO)
  116. log_format = "%(asctime)s %(levelname)s [%(name)s] %(message)s"
  117. # Create root logger
  118. root_logger = logging.getLogger()
  119. root_logger.setLevel(log_level)
  120. # Console handler - always enabled
  121. console_handler = logging.StreamHandler()
  122. console_handler.setLevel(log_level)
  123. console_handler.setFormatter(logging.Formatter(log_format))
  124. root_logger.addHandler(console_handler)
  125. # File handler - only in production or if explicitly enabled
  126. if app_settings.log_to_file:
  127. log_file = app_settings.log_dir / "bambuddy.log"
  128. file_handler = RotatingFileHandler(
  129. log_file,
  130. maxBytes=5 * 1024 * 1024, # 5MB
  131. backupCount=3,
  132. encoding="utf-8",
  133. )
  134. file_handler.setLevel(log_level)
  135. file_handler.setFormatter(logging.Formatter(log_format))
  136. root_logger.addHandler(file_handler)
  137. logging.info(f"Logging to file: {log_file}")
  138. # Reduce noise from third-party libraries in production
  139. if not app_settings.debug:
  140. logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
  141. logging.getLogger("httpcore").setLevel(logging.WARNING)
  142. logging.getLogger("httpx").setLevel(logging.WARNING)
  143. logging.info(f"Bambuddy starting - debug={app_settings.debug}, log_level={log_level_str}")
  144. from fastapi.responses import FileResponse
  145. from fastapi.staticfiles import StaticFiles
  146. from sqlalchemy import delete, or_, select
  147. from backend.app.api.routes import (
  148. ams_history,
  149. api_keys,
  150. archives,
  151. auth,
  152. camera,
  153. cloud,
  154. discovery,
  155. external_links,
  156. filaments,
  157. firmware,
  158. github_backup,
  159. groups,
  160. kprofiles,
  161. library,
  162. maintenance,
  163. metrics,
  164. notification_templates,
  165. notifications,
  166. pending_uploads,
  167. print_queue,
  168. printers,
  169. projects,
  170. settings as settings_routes,
  171. smart_plugs,
  172. spoolman,
  173. support,
  174. system,
  175. updates,
  176. users,
  177. webhook,
  178. websocket,
  179. )
  180. from backend.app.api.routes.maintenance import _get_printer_maintenance_internal, ensure_default_types
  181. from backend.app.api.routes.support import init_debug_logging
  182. from backend.app.core.database import async_session, init_db
  183. from backend.app.core.websocket import ws_manager
  184. from backend.app.models.smart_plug import SmartPlug
  185. from backend.app.services.archive import ArchiveService
  186. from backend.app.services.bambu_ftp import download_file_async, get_ftp_retry_settings, with_ftp_retry
  187. from backend.app.services.bambu_mqtt import PrinterState
  188. from backend.app.services.github_backup import github_backup_service
  189. from backend.app.services.homeassistant import homeassistant_service
  190. from backend.app.services.mqtt_relay import mqtt_relay
  191. from backend.app.services.notification_service import notification_service
  192. from backend.app.services.print_scheduler import scheduler as print_scheduler
  193. from backend.app.services.printer_manager import (
  194. init_printer_connections,
  195. printer_manager,
  196. printer_state_to_dict,
  197. )
  198. from backend.app.services.smart_plug_manager import smart_plug_manager
  199. from backend.app.services.spoolman import close_spoolman_client, get_spoolman_client, init_spoolman_client
  200. from backend.app.services.tasmota import tasmota_service
  201. # Track active prints: {(printer_id, filename): archive_id}
  202. _active_prints: dict[tuple[int, str], int] = {}
  203. # Track expected prints from reprint/scheduled (skip auto-archiving for these)
  204. # {(printer_id, filename): archive_id}
  205. _expected_prints: dict[tuple[int, str], int] = {}
  206. # Track starting energy for prints: {archive_id: starting_kwh}
  207. _print_energy_start: dict[int, float] = {}
  208. # Track reprints to add costs on completion: {archive_id}
  209. _reprint_archives: set[int] = set()
  210. # Track progress milestones for notifications: {printer_id: last_milestone_notified}
  211. # Milestones are 25, 50, 75. Value of 0 means no milestone notified yet for current print.
  212. _last_progress_milestone: dict[int, int] = {}
  213. # Track HMS errors that have been notified: {printer_id: set of error codes}
  214. # This prevents sending duplicate notifications for the same error
  215. _notified_hms_errors: dict[int, set[str]] = {}
  216. async def _get_plug_energy(plug, db) -> dict | None:
  217. """Get energy from plug regardless of type (Tasmota, Home Assistant, or MQTT).
  218. For HA plugs, configures the service with current settings from DB.
  219. For MQTT plugs, returns data from the subscription service.
  220. """
  221. if plug.plug_type == "homeassistant":
  222. from backend.app.api.routes.settings import get_setting
  223. ha_url = await get_setting(db, "ha_url") or ""
  224. ha_token = await get_setting(db, "ha_token") or ""
  225. homeassistant_service.configure(ha_url, ha_token)
  226. return await homeassistant_service.get_energy(plug)
  227. elif plug.plug_type == "mqtt":
  228. # MQTT plugs report "today" energy, not lifetime total
  229. # For per-print tracking, we use "today" as the counter (resets at midnight)
  230. mqtt_data = mqtt_relay.smart_plug_service.get_plug_data(plug.id)
  231. if mqtt_data:
  232. return {
  233. "power": mqtt_data.power,
  234. "today": mqtt_data.energy,
  235. "total": mqtt_data.energy, # Use today as total for per-print calculations
  236. }
  237. return None
  238. else:
  239. return await tasmota_service.get_energy(plug)
  240. def register_expected_print(printer_id: int, filename: str, archive_id: int):
  241. """Register an expected print from reprint/scheduled so we don't create duplicate archives."""
  242. # Store with multiple filename variations to catch different naming patterns
  243. _expected_prints[(printer_id, filename)] = archive_id
  244. # Also store without .3mf extension if present
  245. if filename.endswith(".3mf"):
  246. base = filename[:-4]
  247. _expected_prints[(printer_id, base)] = archive_id
  248. _expected_prints[(printer_id, f"{base}.gcode")] = archive_id
  249. logging.getLogger(__name__).info(
  250. f"Registered expected print: printer={printer_id}, file={filename}, archive={archive_id}"
  251. )
  252. _last_status_broadcast: dict[int, str] = {}
  253. _nozzle_count_updated: set[int] = set() # Track printers where we've updated nozzle_count
  254. async def _report_spoolman_usage(printer_id: int, archive_id: int, logger):
  255. """Report filament usage to Spoolman after print completion.
  256. This finds the spool by RFID tag_uid from current AMS state and reports
  257. the filament_used_grams from the archive metadata.
  258. """
  259. async with async_session() as db:
  260. from backend.app.api.routes.settings import get_setting
  261. from backend.app.models.archive import PrintArchive
  262. # Check if Spoolman is enabled
  263. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  264. if not spoolman_enabled or spoolman_enabled.lower() != "true":
  265. return
  266. # Get Spoolman URL
  267. spoolman_url = await get_setting(db, "spoolman_url")
  268. if not spoolman_url:
  269. return
  270. # Get or create Spoolman client
  271. client = await get_spoolman_client()
  272. if not client:
  273. client = await init_spoolman_client(spoolman_url)
  274. # Check if Spoolman is reachable
  275. if not await client.health_check():
  276. logger.warning("Spoolman not reachable for usage reporting")
  277. return
  278. # Get archive to find filament usage
  279. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  280. archive = result.scalar_one_or_none()
  281. if not archive or not archive.filament_used_grams:
  282. logger.debug(f"No filament usage data for archive {archive_id}")
  283. return
  284. filament_used = archive.filament_used_grams
  285. logger.info(f"[SPOOLMAN] Archive {archive_id} used {filament_used}g of filament")
  286. # Get current AMS state from printer to find the active spool
  287. state = printer_manager.get_status(printer_id)
  288. if not state or not state.raw_data:
  289. logger.debug("No printer state available for usage reporting")
  290. return
  291. ams_data = state.raw_data.get("ams")
  292. if not ams_data:
  293. logger.debug("No AMS data available for usage reporting")
  294. return
  295. # Find spools with RFID tags in Spoolman and report usage
  296. # For now, we report usage to the first spool found with a matching tag
  297. # TODO: In future, track which specific trays were used during the print
  298. spools_updated = 0
  299. for ams_unit in ams_data:
  300. trays = ams_unit.get("tray", [])
  301. for tray_data in trays:
  302. tag_uid = tray_data.get("tag_uid")
  303. if not tag_uid:
  304. continue
  305. # Find spool in Spoolman by tag
  306. spool = await client.find_spool_by_tag(tag_uid)
  307. if spool:
  308. # Report usage to Spoolman
  309. result = await client.use_spool(spool["id"], filament_used)
  310. if result:
  311. logger.info(
  312. f"[SPOOLMAN] Reported {filament_used}g usage to spool {spool['id']} (tag: {tag_uid})"
  313. )
  314. spools_updated += 1
  315. # Only report to one spool for single-material prints
  316. # Multi-material prints would need more sophisticated tracking
  317. return
  318. if spools_updated == 0:
  319. logger.debug(f"No matching Spoolman spools found for printer {printer_id}")
  320. async def on_printer_status_change(printer_id: int, state: PrinterState):
  321. """Handle printer status changes - broadcast via WebSocket."""
  322. # Only broadcast if something meaningful changed (reduce WebSocket spam)
  323. # Include rounded temperatures to detect meaningful temp changes (within 1 degree)
  324. temps = state.temperatures or {}
  325. nozzle_temp = round(temps.get("nozzle", 0))
  326. bed_temp = round(temps.get("bed", 0))
  327. nozzle_2_temp = round(temps.get("nozzle_2", 0)) if "nozzle_2" in temps else ""
  328. chamber_temp = round(temps.get("chamber", 0)) if "chamber" in temps else ""
  329. # Auto-detect dual-nozzle printers from MQTT temperature data
  330. if "nozzle_2" in temps and printer_id not in _nozzle_count_updated:
  331. _nozzle_count_updated.add(printer_id)
  332. # Update nozzle_count in database
  333. async with async_session() as db:
  334. from backend.app.models.printer import Printer
  335. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  336. printer = result.scalar_one_or_none()
  337. if printer and printer.nozzle_count != 2:
  338. printer.nozzle_count = 2
  339. await db.commit()
  340. logging.getLogger(__name__).info(
  341. f"Auto-detected dual-nozzle printer {printer_id}, updated nozzle_count=2"
  342. )
  343. # Include target temps for heating phase detection
  344. bed_target = round(temps.get("bed_target", 0))
  345. nozzle_target = round(temps.get("nozzle_target", 0))
  346. status_key = (
  347. f"{state.connected}:{state.state}:{state.progress}:{state.layer_num}:"
  348. f"{nozzle_temp}:{bed_temp}:{nozzle_2_temp}:{chamber_temp}:"
  349. f"{state.stg_cur}:{bed_target}:{nozzle_target}:"
  350. f"{state.cooling_fan_speed}:{state.big_fan1_speed}:{state.big_fan2_speed}:"
  351. f"{state.chamber_light}:{state.active_extruder}"
  352. )
  353. # MQTT relay - publish status (before dedup check - always publish to MQTT)
  354. try:
  355. printer_info = printer_manager.get_printer(printer_id)
  356. if printer_info:
  357. await mqtt_relay.on_printer_status(printer_id, state, printer_info.name, printer_info.serial_number)
  358. except Exception:
  359. pass # Don't fail status callback if MQTT fails
  360. if _last_status_broadcast.get(printer_id) == status_key:
  361. return # No change, skip WebSocket broadcast
  362. _last_status_broadcast[printer_id] = status_key
  363. # Check for progress milestone notifications (25%, 50%, 75%)
  364. progress = state.progress or 0
  365. is_printing = state.state in ("RUNNING", "PRINTING")
  366. if is_printing and progress > 0:
  367. # Determine which milestone we've reached
  368. current_milestone = 0
  369. if progress >= 75:
  370. current_milestone = 75
  371. elif progress >= 50:
  372. current_milestone = 50
  373. elif progress >= 25:
  374. current_milestone = 25
  375. last_milestone = _last_progress_milestone.get(printer_id, 0)
  376. # If we've crossed a new milestone, send notification
  377. if current_milestone > last_milestone:
  378. _last_progress_milestone[printer_id] = current_milestone
  379. try:
  380. async with async_session() as db:
  381. from backend.app.models.printer import Printer
  382. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  383. printer = result.scalar_one_or_none()
  384. printer_name = printer.name if printer else f"Printer {printer_id}"
  385. filename = state.subtask_name or state.gcode_file or "Unknown"
  386. # remaining_time is in minutes, convert to seconds for notification
  387. remaining_time_seconds = state.remaining_time * 60 if state.remaining_time else None
  388. await notification_service.on_print_progress(
  389. printer_id, printer_name, filename, current_milestone, db, remaining_time_seconds
  390. )
  391. except Exception as e:
  392. logging.getLogger(__name__).warning(f"Progress milestone notification failed: {e}")
  393. elif progress < 5:
  394. # Reset milestone tracking when print restarts or new print begins
  395. _last_progress_milestone[printer_id] = 0
  396. # Check for new HMS errors and send notifications
  397. current_hms_errors = getattr(state, "hms_errors", []) or []
  398. if current_hms_errors:
  399. # Build set of current error codes (using attr for uniqueness)
  400. current_error_codes = {f"{e.attr:08x}" for e in current_hms_errors}
  401. previously_notified = _notified_hms_errors.get(printer_id, set())
  402. # Find new errors that haven't been notified yet
  403. new_error_codes = current_error_codes - previously_notified
  404. if new_error_codes:
  405. # Get the actual new errors for the notification
  406. new_errors = [e for e in current_hms_errors if f"{e.attr:08x}" in new_error_codes]
  407. try:
  408. async with async_session() as db:
  409. from backend.app.models.printer import Printer
  410. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  411. printer = result.scalar_one_or_none()
  412. printer_name = printer.name if printer else f"Printer {printer_id}"
  413. # Format error details for notification
  414. # Module 0x07 = AMS/Filament, 0x05 = Nozzle, 0x0C = Motion Controller, etc.
  415. module_names = {
  416. 0x03: "Print/Task",
  417. 0x05: "Nozzle/Extruder",
  418. 0x07: "AMS/Filament",
  419. 0x0C: "Motion Controller",
  420. 0x12: "Chamber",
  421. }
  422. from backend.app.services.hms_errors import get_error_description
  423. for error in new_errors:
  424. module_name = module_names.get(error.module, f"Module 0x{error.module:02X}")
  425. # Build short code like "0700_8010"
  426. error_code_int = int(error.code.replace("0x", ""), 16) if error.code else 0
  427. short_code = f"{(error.attr >> 16) & 0xFFFF:04X}_{error_code_int:04X}"
  428. error_type = f"{module_name} Error"
  429. # Look up human-readable description
  430. description = get_error_description(short_code)
  431. error_detail = description if description else f"Error code: {short_code}"
  432. await notification_service.on_printer_error(
  433. printer_id, printer_name, error_type, db, error_detail
  434. )
  435. logging.getLogger(__name__).info(
  436. f"[HMS] Sent notification for {len(new_errors)} new error(s) on printer {printer_id}"
  437. )
  438. # Also publish to MQTT relay
  439. printer_info = printer_manager.get_printer(printer_id)
  440. if printer_info:
  441. errors_data = [
  442. {
  443. "code": e.code,
  444. "attr": e.attr,
  445. "module": e.module,
  446. "severity": e.severity,
  447. }
  448. for e in new_errors
  449. ]
  450. await mqtt_relay.on_printer_error(
  451. printer_id, printer_info.name, printer_info.serial_number, errors_data
  452. )
  453. except Exception as e:
  454. logging.getLogger(__name__).warning(f"HMS error notification failed: {e}")
  455. # Update tracking with all current errors
  456. _notified_hms_errors[printer_id] = current_error_codes
  457. else:
  458. # No HMS errors - clear tracking so future errors get notified
  459. if printer_id in _notified_hms_errors:
  460. _notified_hms_errors.pop(printer_id, None)
  461. await ws_manager.send_printer_status(
  462. printer_id,
  463. printer_state_to_dict(state, printer_id, printer_manager.get_model(printer_id)),
  464. )
  465. async def on_ams_change(printer_id: int, ams_data: list):
  466. """Handle AMS data changes - sync to Spoolman if enabled and auto mode."""
  467. import logging
  468. logger = logging.getLogger(__name__)
  469. # MQTT relay - publish AMS change
  470. try:
  471. printer_info = printer_manager.get_printer(printer_id)
  472. if printer_info:
  473. await mqtt_relay.on_ams_change(printer_id, printer_info.name, printer_info.serial_number, ams_data)
  474. except Exception:
  475. pass # Don't fail AMS callback if MQTT fails
  476. # Broadcast AMS change via WebSocket (bypasses status_key deduplication)
  477. # This ensures frontend gets immediate updates when AMS slots are configured
  478. try:
  479. state = printer_manager.get_status(printer_id)
  480. if state:
  481. logger.info(f"[Printer {printer_id}] Broadcasting AMS change via WebSocket")
  482. await ws_manager.send_printer_status(
  483. printer_id,
  484. printer_state_to_dict(state, printer_id, printer_manager.get_model(printer_id)),
  485. )
  486. except Exception as e:
  487. logger.warning(f"Failed to broadcast AMS change for printer {printer_id}: {e}")
  488. try:
  489. async with async_session() as db:
  490. from backend.app.api.routes.settings import get_setting
  491. from backend.app.models.printer import Printer
  492. # Check if Spoolman is enabled
  493. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  494. if not spoolman_enabled or spoolman_enabled.lower() != "true":
  495. return
  496. # Check sync mode
  497. sync_mode = await get_setting(db, "spoolman_sync_mode")
  498. if sync_mode and sync_mode != "auto":
  499. return # Only sync on auto mode
  500. # Get Spoolman URL
  501. spoolman_url = await get_setting(db, "spoolman_url")
  502. if not spoolman_url:
  503. return
  504. # Get or create Spoolman client
  505. client = await get_spoolman_client()
  506. if not client:
  507. client = await init_spoolman_client(spoolman_url)
  508. # Check if Spoolman is reachable
  509. if not await client.health_check():
  510. logger.warning(f"Spoolman not reachable at {spoolman_url}")
  511. return
  512. # Get printer name for location
  513. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  514. printer = result.scalar_one_or_none()
  515. printer_name = printer.name if printer else f"Printer {printer_id}"
  516. # Sync each AMS tray
  517. synced = 0
  518. for ams_unit in ams_data:
  519. ams_id = int(ams_unit.get("id", 0))
  520. trays = ams_unit.get("tray", [])
  521. for tray_data in trays:
  522. tray = client.parse_ams_tray(ams_id, tray_data)
  523. if not tray:
  524. continue # Empty tray
  525. try:
  526. result = await client.sync_ams_tray(tray, printer_name)
  527. if result:
  528. synced += 1
  529. except Exception as e:
  530. logger.error(f"Error syncing AMS {ams_id} tray {tray.tray_id}: {e}")
  531. if synced > 0:
  532. logger.info(f"Auto-synced {synced} AMS trays to Spoolman for printer {printer_id}")
  533. except Exception as e:
  534. import logging
  535. logging.getLogger(__name__).warning(f"Spoolman AMS sync failed: {e}")
  536. async def _send_print_start_notification(
  537. printer_id: int,
  538. data: dict,
  539. archive_data: dict | None = None,
  540. logger=None,
  541. ):
  542. """Helper to send print start notification with optional archive data."""
  543. if logger is None:
  544. import logging
  545. logger = logging.getLogger(__name__)
  546. try:
  547. async with async_session() as db:
  548. from backend.app.models.printer import Printer
  549. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  550. printer = result.scalar_one_or_none()
  551. printer_name = printer.name if printer else f"Printer {printer_id}"
  552. await notification_service.on_print_start(printer_id, printer_name, data, db, archive_data=archive_data)
  553. except Exception as e:
  554. logger.warning(f"Notification on_print_start failed: {e}")
  555. def _load_objects_from_archive(archive, printer_id: int, logger) -> None:
  556. """Extract printable objects from an archive's 3MF file and store in printer state."""
  557. try:
  558. from backend.app.services.archive import extract_printable_objects_from_3mf
  559. file_path = app_settings.base_dir / archive.file_path
  560. if file_path.exists() and str(file_path).endswith(".3mf"):
  561. with open(file_path, "rb") as f:
  562. threemf_data = f.read()
  563. # Extract with positions for UI overlay
  564. printable_objects, bbox_all = extract_printable_objects_from_3mf(threemf_data, include_positions=True)
  565. if printable_objects:
  566. client = printer_manager.get_client(printer_id)
  567. if client:
  568. client.state.printable_objects = printable_objects
  569. client.state.printable_objects_bbox_all = bbox_all
  570. client.state.skipped_objects = []
  571. logger.info(f"Loaded {len(printable_objects)} printable objects for printer {printer_id}")
  572. except Exception as e:
  573. logger.debug(f"Failed to extract printable objects from archive: {e}")
  574. async def on_print_start(printer_id: int, data: dict):
  575. """Handle print start - archive the 3MF file immediately."""
  576. import logging
  577. logger = logging.getLogger(__name__)
  578. logger.info(f"[CALLBACK] on_print_start called for printer {printer_id}, data keys: {list(data.keys())}")
  579. await ws_manager.send_print_start(printer_id, data)
  580. # MQTT relay - publish print start
  581. try:
  582. printer_info = printer_manager.get_printer(printer_id)
  583. if printer_info:
  584. await mqtt_relay.on_print_start(
  585. printer_id,
  586. printer_info.name,
  587. printer_info.serial_number,
  588. data.get("filename", ""),
  589. data.get("subtask_name", ""),
  590. )
  591. except Exception:
  592. pass # Don't fail print start callback if MQTT fails
  593. # Track if notification was sent (to avoid sending twice)
  594. notification_sent = False
  595. # Smart plug automation: turn on plug when print starts
  596. try:
  597. async with async_session() as db:
  598. await smart_plug_manager.on_print_start(printer_id, db)
  599. except Exception as e:
  600. logger.warning(f"Smart plug on_print_start failed: {e}")
  601. async with async_session() as db:
  602. from backend.app.models.printer import Printer
  603. from backend.app.services.bambu_ftp import list_files_async
  604. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  605. printer = result.scalar_one_or_none()
  606. # Plate detection check - pause if objects detected on build plate
  607. if printer and printer.plate_detection_enabled:
  608. try:
  609. from backend.app.services.plate_detection import check_plate_empty
  610. # Build ROI tuple from printer settings if available
  611. roi = None
  612. if all(
  613. [
  614. printer.plate_detection_roi_x is not None,
  615. printer.plate_detection_roi_y is not None,
  616. printer.plate_detection_roi_w is not None,
  617. printer.plate_detection_roi_h is not None,
  618. ]
  619. ):
  620. roi = (
  621. printer.plate_detection_roi_x,
  622. printer.plate_detection_roi_y,
  623. printer.plate_detection_roi_w,
  624. printer.plate_detection_roi_h,
  625. )
  626. # Auto-turn on chamber light if it's off for better detection
  627. light_was_off = False
  628. client = printer_manager.get_client(printer_id)
  629. if client and client.state:
  630. light_was_off = not client.state.chamber_light
  631. if light_was_off:
  632. logger.info(f"[PLATE CHECK] Turning on chamber light for printer {printer_id}")
  633. client.set_chamber_light(True)
  634. # Wait for light to physically turn on and camera to adjust exposure
  635. await asyncio.sleep(2.5)
  636. logger.info(f"[PLATE CHECK] Running plate detection for printer {printer_id}")
  637. plate_result = await check_plate_empty(
  638. printer_id=printer_id,
  639. ip_address=printer.ip_address,
  640. access_code=printer.access_code,
  641. model=printer.model,
  642. include_debug_image=False,
  643. external_camera_url=printer.external_camera_url,
  644. external_camera_type=printer.external_camera_type,
  645. use_external=printer.external_camera_enabled,
  646. roi=roi,
  647. )
  648. # Restore chamber light to original state
  649. if light_was_off and client:
  650. logger.info(f"[PLATE CHECK] Restoring chamber light to off for printer {printer_id}")
  651. client.set_chamber_light(False)
  652. if not plate_result.needs_calibration and not plate_result.is_empty:
  653. # Objects detected - pause the print!
  654. logger.warning(
  655. f"[PLATE CHECK] Objects detected on plate for printer {printer_id}! "
  656. f"Confidence: {plate_result.confidence:.0%}, Diff: {plate_result.difference_percent:.1f}%"
  657. )
  658. client = printer_manager.get_client(printer_id)
  659. if client:
  660. client.pause_print()
  661. logger.info(f"[PLATE CHECK] Print paused for printer {printer_id}")
  662. # Send notification about plate not empty
  663. await ws_manager.broadcast(
  664. {
  665. "type": "plate_not_empty",
  666. "printer_id": printer_id,
  667. "printer_name": printer.name,
  668. "message": f"Objects detected on build plate! Print paused. (Diff: {plate_result.difference_percent:.1f}%)",
  669. }
  670. )
  671. # Also send push notification
  672. try:
  673. await notification_service.on_plate_not_empty(
  674. printer_id=printer_id,
  675. printer_name=printer.name,
  676. db=db,
  677. difference_percent=plate_result.difference_percent,
  678. )
  679. except Exception as notif_err:
  680. logger.warning(f"[PLATE CHECK] Failed to send notification: {notif_err}")
  681. else:
  682. logger.info(f"[PLATE CHECK] Plate is empty for printer {printer_id}, proceeding with print")
  683. except Exception as plate_err:
  684. # Don't block print on plate detection errors
  685. logger.warning(f"[PLATE CHECK] Plate detection failed for printer {printer_id}: {plate_err}")
  686. if not printer or not printer.auto_archive:
  687. # Send notification without archive data (auto-archive disabled)
  688. logger.info(
  689. f"[CALLBACK] Skipping archive - printer: {printer is not None}, auto_archive: {printer.auto_archive if printer else 'N/A'}"
  690. )
  691. if not notification_sent:
  692. await _send_print_start_notification(printer_id, data, logger=logger)
  693. return
  694. # Get the filename and subtask_name
  695. filename = data.get("filename", "")
  696. subtask_name = data.get("subtask_name", "")
  697. logger.info(f"[CALLBACK] Print start detected - filename: {filename}, subtask: {subtask_name}")
  698. if not filename and not subtask_name:
  699. # Send notification without archive data (no filename)
  700. logger.info("[CALLBACK] Skipping archive - no filename or subtask_name")
  701. if not notification_sent:
  702. await _send_print_start_notification(printer_id, data, logger=logger)
  703. return
  704. # Check if this is an expected print from reprint/scheduled
  705. # Build list of possible keys to check
  706. expected_keys = []
  707. if subtask_name:
  708. expected_keys.append((printer_id, subtask_name))
  709. expected_keys.append((printer_id, f"{subtask_name}.3mf"))
  710. expected_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  711. if filename:
  712. fname = filename.split("/")[-1] if "/" in filename else filename
  713. expected_keys.append((printer_id, fname))
  714. # Strip extensions to match
  715. base = fname.replace(".gcode", "").replace(".3mf", "")
  716. expected_keys.append((printer_id, base))
  717. expected_keys.append((printer_id, f"{base}.3mf"))
  718. expected_archive_id = None
  719. for key in expected_keys:
  720. expected_archive_id = _expected_prints.pop(key, None)
  721. if expected_archive_id:
  722. # Clean up other possible keys for this print
  723. for other_key in expected_keys:
  724. _expected_prints.pop(other_key, None)
  725. break
  726. if expected_archive_id:
  727. # This is a reprint/scheduled print - use existing archive, don't create new one
  728. logger.info(f"Using expected archive {expected_archive_id} for print (skipping duplicate)")
  729. from backend.app.models.archive import PrintArchive
  730. result = await db.execute(select(PrintArchive).where(PrintArchive.id == expected_archive_id))
  731. archive = result.scalar_one_or_none()
  732. if archive:
  733. # Update archive status to printing
  734. archive.status = "printing"
  735. archive.started_at = datetime.now()
  736. await db.commit()
  737. # Track as active print
  738. _active_prints[(printer_id, archive.filename)] = archive.id
  739. if subtask_name:
  740. _active_prints[(printer_id, f"{subtask_name}.3mf")] = archive.id
  741. # Mark as reprint so we add cost on completion
  742. _reprint_archives.add(archive.id)
  743. logger.info(f"Marked archive {archive.id} as reprint for cost addition on completion")
  744. # Set up energy tracking
  745. try:
  746. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  747. plug = plug_result.scalar_one_or_none()
  748. logger.info(
  749. f"[ENERGY] Print start - archive {archive.id}, printer {printer_id}, plug found: {plug is not None}"
  750. )
  751. if plug:
  752. energy = await _get_plug_energy(plug, db)
  753. logger.info(f"[ENERGY] Energy response from plug: {energy}")
  754. if energy and energy.get("total") is not None:
  755. _print_energy_start[archive.id] = energy["total"]
  756. logger.info(
  757. f"[ENERGY] Recorded starting energy for archive {archive.id}: {energy['total']} kWh"
  758. )
  759. else:
  760. logger.warning(f"[ENERGY] No 'total' in energy response for archive {archive.id}")
  761. else:
  762. logger.info(f"[ENERGY] No smart plug found for printer {printer_id}")
  763. except Exception as e:
  764. logger.warning(f"Failed to record starting energy: {e}")
  765. await ws_manager.send_archive_updated(
  766. {
  767. "id": archive.id,
  768. "status": "printing",
  769. }
  770. )
  771. # Send notification with archive data (reprint/scheduled)
  772. if not notification_sent:
  773. archive_data = {"print_time_seconds": archive.print_time_seconds}
  774. await _send_print_start_notification(printer_id, data, archive_data, logger)
  775. # Extract printable objects from the archived 3MF file
  776. _load_objects_from_archive(archive, printer_id, logger)
  777. return # Skip creating a new archive
  778. # Check if there's already a "printing" archive for this printer/file
  779. # This prevents duplicates when backend restarts during an active print
  780. from backend.app.models.archive import PrintArchive
  781. check_name = subtask_name or filename.split("/")[-1].replace(".gcode", "").replace(".3mf", "")
  782. existing = await db.execute(
  783. select(PrintArchive)
  784. .where(PrintArchive.printer_id == printer_id)
  785. .where(PrintArchive.status == "printing")
  786. .where(PrintArchive.print_name.ilike(f"%{check_name}%"))
  787. .order_by(PrintArchive.created_at.desc())
  788. .limit(1)
  789. )
  790. existing_archive = existing.scalar_one_or_none()
  791. if existing_archive:
  792. # Check if archive is stale (older than 4 hours) - likely a failed/cancelled print
  793. # that didn't get properly updated
  794. archive_age = datetime.now(UTC) - existing_archive.created_at.replace(tzinfo=UTC)
  795. if archive_age.total_seconds() > 4 * 60 * 60: # 4 hours
  796. logger.warning(
  797. f"Found stale 'printing' archive {existing_archive.id} (age: {archive_age}), "
  798. f"marking as cancelled and creating new archive"
  799. )
  800. existing_archive.status = "cancelled"
  801. existing_archive.failure_reason = "Stale - print likely cancelled or failed without status update"
  802. await db.commit()
  803. # Fall through to create new archive (don't return)
  804. existing_archive = None # Clear so we don't use stale archive
  805. else:
  806. logger.info(
  807. f"Skipping duplicate - already have printing archive {existing_archive.id} for {check_name}"
  808. )
  809. # Track this as the active print
  810. _active_prints[(printer_id, existing_archive.filename)] = existing_archive.id
  811. # Also set up energy tracking if not already tracked
  812. if existing_archive.id not in _print_energy_start:
  813. try:
  814. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  815. plug = plug_result.scalar_one_or_none()
  816. if plug:
  817. energy = await _get_plug_energy(plug, db)
  818. if energy and energy.get("total") is not None:
  819. _print_energy_start[existing_archive.id] = energy["total"]
  820. logger.info(
  821. f"Recorded starting energy for existing archive {existing_archive.id}: {energy['total']} kWh"
  822. )
  823. except Exception as e:
  824. logger.warning(f"Failed to record starting energy for existing archive: {e}")
  825. # Send notification with archive data (existing archive)
  826. if not notification_sent:
  827. archive_data = {"print_time_seconds": existing_archive.print_time_seconds}
  828. await _send_print_start_notification(printer_id, data, archive_data, logger)
  829. # Extract printable objects from the archived 3MF file
  830. _load_objects_from_archive(existing_archive, printer_id, logger)
  831. return
  832. # Build list of possible 3MF filenames to try
  833. possible_names = []
  834. # Bambu printers typically store files as "Name.gcode.3mf"
  835. # The subtask_name is usually the best source for the filename
  836. if subtask_name:
  837. # Try common Bambu naming patterns
  838. possible_names.append(f"{subtask_name}.gcode.3mf")
  839. possible_names.append(f"{subtask_name}.3mf")
  840. # Try original filename with .3mf extension
  841. if filename:
  842. # Extract just the filename part, not the full path
  843. fname = filename.split("/")[-1] if "/" in filename else filename
  844. if fname.endswith(".3mf"):
  845. possible_names.append(fname)
  846. elif fname.endswith(".gcode"):
  847. base = fname.rsplit(".", 1)[0]
  848. possible_names.append(f"{base}.gcode.3mf")
  849. possible_names.append(f"{base}.3mf")
  850. else:
  851. possible_names.append(f"{fname}.gcode.3mf")
  852. possible_names.append(f"{fname}.3mf")
  853. # Also try with spaces converted to underscores (Bambu Studio may normalize filenames)
  854. space_variants = []
  855. for name in possible_names:
  856. if " " in name:
  857. space_variants.append(name.replace(" ", "_"))
  858. possible_names.extend(space_variants)
  859. # Remove duplicates while preserving order
  860. seen = set()
  861. possible_names = [x for x in possible_names if not (x in seen or seen.add(x))]
  862. logger.info(f"Trying filenames: {possible_names}")
  863. # Try to find and download the 3MF file
  864. temp_path = None
  865. downloaded_filename = None
  866. # Get FTP retry settings
  867. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  868. for try_filename in possible_names:
  869. if not try_filename.endswith(".3mf"):
  870. continue
  871. remote_paths = [
  872. f"/cache/{try_filename}",
  873. f"/model/{try_filename}",
  874. f"/data/{try_filename}",
  875. f"/data/Metadata/{try_filename}",
  876. f"/{try_filename}",
  877. ]
  878. temp_path = app_settings.archive_dir / "temp" / try_filename
  879. temp_path.parent.mkdir(parents=True, exist_ok=True)
  880. for remote_path in remote_paths:
  881. logger.debug(f"Trying FTP download: {remote_path}")
  882. try:
  883. if ftp_retry_enabled:
  884. downloaded = await with_ftp_retry(
  885. download_file_async,
  886. printer.ip_address,
  887. printer.access_code,
  888. remote_path,
  889. temp_path,
  890. socket_timeout=ftp_timeout,
  891. printer_model=printer.model,
  892. max_retries=ftp_retry_count,
  893. retry_delay=ftp_retry_delay,
  894. operation_name=f"Download 3MF from {remote_path}",
  895. )
  896. else:
  897. downloaded = await download_file_async(
  898. printer.ip_address,
  899. printer.access_code,
  900. remote_path,
  901. temp_path,
  902. socket_timeout=ftp_timeout,
  903. printer_model=printer.model,
  904. )
  905. if downloaded:
  906. downloaded_filename = try_filename
  907. logger.info(f"Downloaded: {remote_path}")
  908. break
  909. except Exception as e:
  910. logger.debug(f"FTP download failed for {remote_path}: {e}")
  911. if downloaded_filename:
  912. break
  913. # If still not found, try listing directories to find matching file
  914. # Different printer models use different directory structures
  915. if not downloaded_filename and (filename or subtask_name):
  916. search_term = (subtask_name or filename).lower().replace(".gcode", "").replace(".3mf", "")
  917. logger.info(f"Direct FTP download failed, searching directories for '{search_term}'")
  918. search_dirs = ["/cache", "/model", "/data", "/data/Metadata", "/"]
  919. for search_dir in search_dirs:
  920. if downloaded_filename:
  921. break
  922. try:
  923. dir_files = await list_files_async(printer.ip_address, printer.access_code, search_dir)
  924. threemf_files = [f.get("name") for f in dir_files if f.get("name", "").endswith(".3mf")]
  925. if threemf_files:
  926. logger.info(
  927. f"Found {len(threemf_files)} 3MF files in {search_dir}: {threemf_files[:5]}{'...' if len(threemf_files) > 5 else ''}"
  928. )
  929. for f in dir_files:
  930. if f.get("is_directory"):
  931. continue
  932. fname = f.get("name", "")
  933. # Normalize both for comparison (spaces and underscores are equivalent)
  934. fname_normalized = fname.lower().replace(" ", "_")
  935. search_normalized = search_term.replace(" ", "_")
  936. if fname.endswith(".3mf") and search_normalized in fname_normalized:
  937. logger.info(f"Found matching file in {search_dir}: {fname}")
  938. temp_path = app_settings.archive_dir / "temp" / fname
  939. temp_path.parent.mkdir(parents=True, exist_ok=True)
  940. if ftp_retry_enabled:
  941. downloaded = await with_ftp_retry(
  942. download_file_async,
  943. printer.ip_address,
  944. printer.access_code,
  945. f"{search_dir}/{fname}",
  946. temp_path,
  947. max_retries=ftp_retry_count,
  948. retry_delay=ftp_retry_delay,
  949. operation_name=f"Download 3MF from {search_dir}/{fname}",
  950. )
  951. else:
  952. downloaded = await download_file_async(
  953. printer.ip_address,
  954. printer.access_code,
  955. f"{search_dir}/{fname}",
  956. temp_path,
  957. )
  958. if downloaded:
  959. downloaded_filename = fname
  960. logger.info(f"Found and downloaded from {search_dir}: {fname}")
  961. break
  962. except Exception as e:
  963. logger.debug(f"Failed to list {search_dir}: {e}")
  964. if not downloaded_filename or not temp_path:
  965. logger.warning(f"Could not find 3MF file for print: {filename or subtask_name}")
  966. # Create a fallback archive without 3MF data so the print is still tracked
  967. # This commonly happens with P1S/A1 printers where FTP has file size limitations
  968. try:
  969. from backend.app.models.archive import PrintArchive
  970. # Derive print name from subtask_name or filename
  971. print_name = subtask_name or filename
  972. if print_name:
  973. # Clean up the name (remove extensions, path parts)
  974. print_name = print_name.split("/")[-1]
  975. print_name = print_name.replace(".gcode.3mf", "").replace(".gcode", "").replace(".3mf", "")
  976. else:
  977. print_name = "Unknown Print"
  978. # Create minimal archive entry
  979. fallback_archive = PrintArchive(
  980. printer_id=printer_id,
  981. filename=filename or f"{print_name}.3mf",
  982. file_path="", # Empty - no 3MF file available
  983. file_size=0,
  984. print_name=print_name,
  985. status="printing",
  986. started_at=datetime.now(),
  987. extra_data={"no_3mf_available": True, "original_subtask": subtask_name, "_print_data": data},
  988. )
  989. db.add(fallback_archive)
  990. await db.commit()
  991. await db.refresh(fallback_archive)
  992. logger.info(f"Created fallback archive {fallback_archive.id} for {print_name} (no 3MF available)")
  993. # Start timelapse session if external camera is enabled
  994. if printer.external_camera_enabled and printer.external_camera_url:
  995. from backend.app.services.layer_timelapse import start_session
  996. start_session(
  997. printer_id,
  998. fallback_archive.id,
  999. printer.external_camera_url,
  1000. printer.external_camera_type or "mjpeg",
  1001. )
  1002. logger.info(f"Started layer timelapse for printer {printer_id}, archive {fallback_archive.id}")
  1003. # Track as active print
  1004. _active_prints[(printer_id, fallback_archive.filename)] = fallback_archive.id
  1005. if filename:
  1006. _active_prints[(printer_id, filename)] = fallback_archive.id
  1007. if subtask_name:
  1008. _active_prints[(printer_id, f"{subtask_name}.3mf")] = fallback_archive.id
  1009. _active_prints[(printer_id, subtask_name)] = fallback_archive.id
  1010. # Record starting energy if smart plug available
  1011. try:
  1012. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  1013. plug = plug_result.scalar_one_or_none()
  1014. if plug:
  1015. energy = await _get_plug_energy(plug, db)
  1016. if energy and energy.get("total") is not None:
  1017. _print_energy_start[fallback_archive.id] = energy["total"]
  1018. logger.info(
  1019. f"[ENERGY] Recorded starting energy for fallback archive {fallback_archive.id}: {energy['total']} kWh"
  1020. )
  1021. except Exception as e:
  1022. logger.warning(f"Failed to record starting energy for fallback: {e}")
  1023. # Send WebSocket notification
  1024. await ws_manager.send_archive_created(
  1025. {
  1026. "id": fallback_archive.id,
  1027. "printer_id": fallback_archive.printer_id,
  1028. "filename": fallback_archive.filename,
  1029. "print_name": fallback_archive.print_name,
  1030. "status": fallback_archive.status,
  1031. }
  1032. )
  1033. # MQTT relay - publish archive created
  1034. try:
  1035. await mqtt_relay.on_archive_created(
  1036. archive_id=fallback_archive.id,
  1037. print_name=fallback_archive.print_name,
  1038. printer_name=printer.name,
  1039. status=fallback_archive.status,
  1040. )
  1041. except Exception:
  1042. pass # Don't fail if MQTT fails
  1043. # Send notification without archive data (file not found)
  1044. if not notification_sent:
  1045. await _send_print_start_notification(printer_id, data, logger=logger)
  1046. return
  1047. except Exception as e:
  1048. logger.error(f"Failed to create fallback archive: {e}")
  1049. # Send notification without archive data (file not found)
  1050. if not notification_sent:
  1051. await _send_print_start_notification(printer_id, data, logger=logger)
  1052. return
  1053. try:
  1054. # Archive the file with status "printing"
  1055. service = ArchiveService(db)
  1056. archive = await service.archive_print(
  1057. printer_id=printer_id,
  1058. source_file=temp_path,
  1059. print_data={**data, "status": "printing"},
  1060. )
  1061. if archive:
  1062. # Track this active print (use both original filename and downloaded filename)
  1063. _active_prints[(printer_id, downloaded_filename)] = archive.id
  1064. if filename and filename != downloaded_filename:
  1065. _active_prints[(printer_id, filename)] = archive.id
  1066. if subtask_name:
  1067. _active_prints[(printer_id, f"{subtask_name}.3mf")] = archive.id
  1068. logger.info(f"Created archive {archive.id} for {downloaded_filename}")
  1069. # Start timelapse session if external camera is enabled
  1070. if printer.external_camera_enabled and printer.external_camera_url:
  1071. from backend.app.services.layer_timelapse import start_session
  1072. start_session(
  1073. printer_id,
  1074. archive.id,
  1075. printer.external_camera_url,
  1076. printer.external_camera_type or "mjpeg",
  1077. )
  1078. logger.info(f"Started layer timelapse for printer {printer_id}, archive {archive.id}")
  1079. # Record starting energy from smart plug if available
  1080. try:
  1081. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  1082. plug = plug_result.scalar_one_or_none()
  1083. logger.info(
  1084. f"[ENERGY] Auto-archive print start - archive {archive.id}, printer {printer_id}, plug found: {plug is not None}"
  1085. )
  1086. if plug:
  1087. energy = await _get_plug_energy(plug, db)
  1088. logger.info(f"[ENERGY] Auto-archive energy response: {energy}")
  1089. if energy and energy.get("total") is not None:
  1090. _print_energy_start[archive.id] = energy["total"]
  1091. logger.info(
  1092. f"[ENERGY] Recorded starting energy for archive {archive.id}: {energy['total']} kWh"
  1093. )
  1094. else:
  1095. logger.warning(f"[ENERGY] No 'total' in energy response for archive {archive.id}")
  1096. else:
  1097. logger.info(f"[ENERGY] No smart plug found for printer {printer_id}")
  1098. except Exception as e:
  1099. logger.warning(f"Failed to record starting energy: {e}")
  1100. await ws_manager.send_archive_created(
  1101. {
  1102. "id": archive.id,
  1103. "printer_id": archive.printer_id,
  1104. "filename": archive.filename,
  1105. "print_name": archive.print_name,
  1106. "status": archive.status,
  1107. }
  1108. )
  1109. # MQTT relay - publish archive created
  1110. try:
  1111. await mqtt_relay.on_archive_created(
  1112. archive_id=archive.id,
  1113. print_name=archive.print_name,
  1114. printer_name=printer.name,
  1115. status=archive.status,
  1116. )
  1117. except Exception:
  1118. pass # Don't fail if MQTT fails
  1119. # Send notification with archive data (new archive created)
  1120. if not notification_sent:
  1121. archive_data = {"print_time_seconds": archive.print_time_seconds}
  1122. await _send_print_start_notification(printer_id, data, archive_data, logger)
  1123. notification_sent = True
  1124. # Extract printable objects for skip object functionality
  1125. try:
  1126. from backend.app.services.archive import extract_printable_objects_from_3mf
  1127. with open(temp_path, "rb") as f:
  1128. threemf_data = f.read()
  1129. # Extract with positions for UI overlay
  1130. printable_objects, bbox_all = extract_printable_objects_from_3mf(
  1131. threemf_data, include_positions=True
  1132. )
  1133. if printable_objects:
  1134. # Store objects in printer state
  1135. client = printer_manager.get_client(printer_id)
  1136. if client:
  1137. client.state.printable_objects = printable_objects
  1138. client.state.printable_objects_bbox_all = bbox_all
  1139. client.state.skipped_objects = [] # Reset skipped objects for new print
  1140. logger.info(f"Loaded {len(printable_objects)} printable objects for printer {printer_id}")
  1141. except Exception as e:
  1142. logger.debug(f"Failed to extract printable objects: {e}")
  1143. finally:
  1144. if temp_path and temp_path.exists():
  1145. temp_path.unlink()
  1146. async def _scan_for_timelapse_with_retries(archive_id: int):
  1147. """
  1148. Scan for timelapse with retries.
  1149. The printer encodes the timelapse quickly after print completion.
  1150. We just need a short delay then grab the most recent file.
  1151. Since we KNOW timelapse was active (from MQTT ipcam data), the most recent
  1152. file in /timelapse is our target. Retries handle FTP connection issues.
  1153. """
  1154. import logging
  1155. logger = logging.getLogger(__name__)
  1156. # Short delays - printer usually finishes encoding within seconds
  1157. retry_delays = [5, 10, 20]
  1158. for attempt, delay in enumerate(retry_delays, 1):
  1159. logger.info(
  1160. f"[TIMELAPSE] Attempt {attempt}/{len(retry_delays)}: waiting {delay}s before scanning for archive {archive_id}"
  1161. )
  1162. await asyncio.sleep(delay)
  1163. try:
  1164. async with async_session() as db:
  1165. from backend.app.models.printer import Printer
  1166. from backend.app.services.bambu_ftp import download_file_bytes_async, list_files_async
  1167. # Get archive (ArchiveService from module-level import)
  1168. service = ArchiveService(db)
  1169. archive = await service.get_archive(archive_id)
  1170. if not archive:
  1171. logger.warning(f"[TIMELAPSE] Archive {archive_id} not found, stopping retries")
  1172. return
  1173. if archive.timelapse_path:
  1174. logger.info(f"[TIMELAPSE] Archive {archive_id} already has timelapse attached, stopping retries")
  1175. return
  1176. if not archive.printer_id:
  1177. logger.warning(f"[TIMELAPSE] Archive {archive_id} has no printer, stopping retries")
  1178. return
  1179. # Get printer
  1180. result = await db.execute(select(Printer).where(Printer.id == archive.printer_id))
  1181. printer = result.scalar_one_or_none()
  1182. if not printer:
  1183. logger.warning(f"[TIMELAPSE] Printer not found for archive {archive_id}, stopping retries")
  1184. return
  1185. # Scan timelapse directory on printer
  1186. # H2D may store in different locations than X1C
  1187. files = []
  1188. found_path = None
  1189. for timelapse_path in ["/timelapse", "/timelapse/video", "/record", "/recording"]:
  1190. try:
  1191. found_files = await list_files_async(printer.ip_address, printer.access_code, timelapse_path)
  1192. if found_files:
  1193. files = found_files
  1194. found_path = timelapse_path
  1195. logger.info(f"[TIMELAPSE] Attempt {attempt}: Found {len(files)} files in {timelapse_path}")
  1196. break
  1197. except Exception as e:
  1198. logger.debug(f"[TIMELAPSE] Path {timelapse_path} failed: {e}")
  1199. continue
  1200. if not files:
  1201. logger.info(f"[TIMELAPSE] Attempt {attempt}: No timelapse files found on printer, will retry")
  1202. continue
  1203. mp4_files = [f for f in files if not f.get("is_directory") and f.get("name", "").endswith(".mp4")]
  1204. # Log ALL mp4 files found for debugging
  1205. logger.info(f"[TIMELAPSE] Attempt {attempt}: Found {len(mp4_files)} MP4 files in {found_path}")
  1206. for f in mp4_files[:5]: # Log first 5
  1207. logger.info(f"[TIMELAPSE] - {f.get('name')}, mtime={f.get('mtime')}")
  1208. if not mp4_files:
  1209. logger.info(f"[TIMELAPSE] Attempt {attempt}: No MP4 files found, will retry")
  1210. continue
  1211. # Sort by mtime descending to get most recent file
  1212. mp4_files_with_mtime = [f for f in mp4_files if f.get("mtime")]
  1213. if not mp4_files_with_mtime:
  1214. logger.info(f"[TIMELAPSE] Attempt {attempt}: No MP4 files with mtime found, will retry")
  1215. continue
  1216. mp4_files_with_mtime.sort(key=lambda x: x.get("mtime"), reverse=True)
  1217. most_recent = mp4_files_with_mtime[0]
  1218. file_name = most_recent.get("name")
  1219. logger.info(f"[TIMELAPSE] Attempt {attempt}: Most recent file: {file_name}")
  1220. # Since we KNOW timelapse was active (from MQTT), just grab the most recent file
  1221. remote_path = most_recent.get("path") or f"/timelapse/{file_name}"
  1222. logger.info(f"[TIMELAPSE] Downloading {file_name} for archive {archive_id}")
  1223. timelapse_data = await download_file_bytes_async(printer.ip_address, printer.access_code, remote_path)
  1224. if timelapse_data:
  1225. success = await service.attach_timelapse(archive_id, timelapse_data, file_name)
  1226. if success:
  1227. logger.info(f"[TIMELAPSE] Successfully attached timelapse to archive {archive_id}")
  1228. await ws_manager.send_archive_updated({"id": archive_id, "timelapse_attached": True})
  1229. return # Success!
  1230. else:
  1231. logger.warning(f"[TIMELAPSE] Failed to attach timelapse to archive {archive_id}")
  1232. else:
  1233. logger.warning(f"[TIMELAPSE] Attempt {attempt}: Failed to download, will retry")
  1234. except Exception as e:
  1235. logger.warning(f"[TIMELAPSE] Attempt {attempt} failed with error: {e}")
  1236. logger.warning(f"[TIMELAPSE] All {len(retry_delays)} attempts exhausted for archive {archive_id}, giving up")
  1237. async def on_print_complete(printer_id: int, data: dict):
  1238. """Handle print completion - update the archive status."""
  1239. import logging
  1240. import time
  1241. logger = logging.getLogger(__name__)
  1242. start_time = time.time()
  1243. def log_timing(section: str):
  1244. elapsed = time.time() - start_time
  1245. logger.info(f"[TIMING] {section}: {elapsed:.3f}s elapsed")
  1246. logger.info(f"[CALLBACK] on_print_complete started for printer {printer_id}")
  1247. try:
  1248. ws_data = {
  1249. "status": data.get("status"),
  1250. "filename": data.get("filename"),
  1251. "subtask_name": data.get("subtask_name"),
  1252. "timelapse_was_active": data.get("timelapse_was_active"),
  1253. }
  1254. await ws_manager.send_print_complete(printer_id, ws_data)
  1255. log_timing("WebSocket send_print_complete")
  1256. except Exception as e:
  1257. logger.warning(f"[CALLBACK] WebSocket send_print_complete failed: {e}")
  1258. # Clear current print user tracking (Issue #206)
  1259. printer_manager.clear_current_print_user(printer_id)
  1260. # MQTT relay - publish print complete
  1261. try:
  1262. printer_info = printer_manager.get_printer(printer_id)
  1263. if printer_info:
  1264. await mqtt_relay.on_print_complete(
  1265. printer_id,
  1266. printer_info.name,
  1267. printer_info.serial_number,
  1268. data.get("filename", ""),
  1269. data.get("subtask_name", ""),
  1270. data.get("status", "completed"),
  1271. )
  1272. except Exception:
  1273. pass # Don't fail print complete callback if MQTT fails
  1274. filename = data.get("filename", "")
  1275. subtask_name = data.get("subtask_name", "")
  1276. if not filename and not subtask_name:
  1277. logger.warning("Print complete without filename or subtask_name")
  1278. return
  1279. logger.info(f"Print complete - filename: {filename}, subtask: {subtask_name}, status: {data.get('status')}")
  1280. # Build list of possible keys to try (matching how they were registered in on_print_start)
  1281. possible_keys = []
  1282. # Try subtask_name variations first (most reliable for matching)
  1283. if subtask_name:
  1284. possible_keys.append((printer_id, f"{subtask_name}.3mf"))
  1285. possible_keys.append((printer_id, f"{subtask_name}.gcode.3mf"))
  1286. possible_keys.append((printer_id, subtask_name))
  1287. # Try filename variations
  1288. if filename:
  1289. # Extract just the filename if it's a path
  1290. fname = filename.split("/")[-1] if "/" in filename else filename
  1291. if fname.endswith(".3mf"):
  1292. possible_keys.append((printer_id, fname))
  1293. elif fname.endswith(".gcode"):
  1294. base_name = fname.rsplit(".", 1)[0]
  1295. possible_keys.append((printer_id, f"{base_name}.gcode.3mf"))
  1296. possible_keys.append((printer_id, f"{base_name}.3mf"))
  1297. possible_keys.append((printer_id, fname))
  1298. else:
  1299. possible_keys.append((printer_id, f"{fname}.gcode.3mf"))
  1300. possible_keys.append((printer_id, f"{fname}.3mf"))
  1301. possible_keys.append((printer_id, fname))
  1302. # Also try full path versions
  1303. if filename.endswith(".3mf"):
  1304. possible_keys.append((printer_id, filename))
  1305. elif filename.endswith(".gcode"):
  1306. base_name = filename.rsplit(".", 1)[0]
  1307. possible_keys.append((printer_id, f"{base_name}.3mf"))
  1308. possible_keys.append((printer_id, filename))
  1309. else:
  1310. possible_keys.append((printer_id, f"{filename}.3mf"))
  1311. possible_keys.append((printer_id, filename))
  1312. # Find the archive for this print
  1313. logger.info(f"Looking for archive in _active_prints, keys to try: {possible_keys[:5]}...")
  1314. logger.info(f"Current _active_prints: {list(_active_prints.keys())}")
  1315. archive_id = None
  1316. for key in possible_keys:
  1317. archive_id = _active_prints.pop(key, None)
  1318. if archive_id:
  1319. logger.info(f"Found archive {archive_id} with key {key}")
  1320. # Also clean up any other keys pointing to this archive
  1321. keys_to_remove = [k for k, v in _active_prints.items() if v == archive_id]
  1322. for k in keys_to_remove:
  1323. _active_prints.pop(k, None)
  1324. break
  1325. if not archive_id:
  1326. # Try to find by filename or subtask_name if not tracked (for prints started before app)
  1327. async with async_session() as db:
  1328. from backend.app.models.archive import PrintArchive
  1329. # Try matching by subtask_name (stored as print_name) first
  1330. if subtask_name:
  1331. result = await db.execute(
  1332. select(PrintArchive)
  1333. .where(PrintArchive.printer_id == printer_id)
  1334. .where(PrintArchive.status == "printing")
  1335. .where(
  1336. or_(
  1337. PrintArchive.print_name.ilike(f"%{subtask_name}%"),
  1338. PrintArchive.filename.ilike(f"%{subtask_name}%"),
  1339. )
  1340. )
  1341. .order_by(PrintArchive.created_at.desc())
  1342. .limit(1)
  1343. )
  1344. archive = result.scalar_one_or_none()
  1345. if archive:
  1346. archive_id = archive.id
  1347. logger.info(f"Found archive {archive_id} by subtask_name match: {subtask_name}")
  1348. # Also try by filename
  1349. if not archive_id and filename:
  1350. result = await db.execute(
  1351. select(PrintArchive)
  1352. .where(PrintArchive.printer_id == printer_id)
  1353. .where(PrintArchive.filename == filename)
  1354. .where(PrintArchive.status == "printing")
  1355. .order_by(PrintArchive.created_at.desc())
  1356. .limit(1)
  1357. )
  1358. archive = result.scalar_one_or_none()
  1359. if archive:
  1360. archive_id = archive.id
  1361. if not archive_id:
  1362. logger.warning(f"Could not find archive for print complete: filename={filename}, subtask={subtask_name}")
  1363. return
  1364. log_timing("Archive lookup")
  1365. # Update archive status
  1366. logger.info(f"[ARCHIVE] Updating archive {archive_id} status...")
  1367. try:
  1368. async with async_session() as db:
  1369. service = ArchiveService(db)
  1370. status = data.get("status", "completed")
  1371. # Auto-detect failure reason
  1372. failure_reason = None
  1373. if status == "aborted":
  1374. failure_reason = "User cancelled"
  1375. logger.info("[ARCHIVE] Print was aborted by user, setting failure_reason='User cancelled'")
  1376. elif status == "failed":
  1377. # Try to determine failure reason from HMS errors
  1378. hms_errors = data.get("hms_errors", [])
  1379. if hms_errors:
  1380. logger.info(f"[ARCHIVE] HMS errors at failure: {hms_errors}")
  1381. # Map known HMS error modules to failure reasons
  1382. # Module 0x07 = Filament, 0x0C = MC (Motion Controller), etc.
  1383. for err in hms_errors:
  1384. module = err.get("module", 0)
  1385. if module == 0x07: # Filament module
  1386. failure_reason = "Filament runout"
  1387. break
  1388. elif module == 0x0C: # Motion controller
  1389. failure_reason = "Layer shift"
  1390. break
  1391. elif module == 0x05: # Nozzle/extruder
  1392. failure_reason = "Clogged nozzle"
  1393. break
  1394. if failure_reason:
  1395. logger.info(f"[ARCHIVE] Detected failure_reason from HMS: {failure_reason}")
  1396. else:
  1397. logger.info("[ARCHIVE] No HMS errors available to determine failure reason")
  1398. await service.update_archive_status(
  1399. archive_id,
  1400. status=status,
  1401. completed_at=datetime.now() if status in ("completed", "failed", "aborted") else None,
  1402. failure_reason=failure_reason,
  1403. )
  1404. logger.info(f"[ARCHIVE] Archive {archive_id} status updated to {status}, failure_reason={failure_reason}")
  1405. # Add cost for reprints (first prints have cost set in archive_print())
  1406. if status == "completed" and archive_id in _reprint_archives:
  1407. _reprint_archives.discard(archive_id)
  1408. try:
  1409. await service.add_reprint_cost(archive_id)
  1410. logger.info(f"[ARCHIVE] Added reprint cost for archive {archive_id}")
  1411. except Exception as e:
  1412. logger.warning(f"[ARCHIVE] Failed to add reprint cost for archive {archive_id}: {e}")
  1413. await ws_manager.send_archive_updated(
  1414. {
  1415. "id": archive_id,
  1416. "status": status,
  1417. }
  1418. )
  1419. logger.info(f"[ARCHIVE] WebSocket notification sent for archive {archive_id}")
  1420. # MQTT relay - publish archive updated
  1421. try:
  1422. await mqtt_relay.on_archive_updated(
  1423. archive_id=archive_id,
  1424. print_name=filename or subtask_name,
  1425. status=status,
  1426. )
  1427. except Exception:
  1428. pass # Don't fail if MQTT fails
  1429. except Exception as e:
  1430. logger.error(f"[ARCHIVE] Failed to update archive {archive_id} status: {e}", exc_info=True)
  1431. # Continue with other operations even if archive update fails
  1432. log_timing("Archive status update")
  1433. # Report filament usage to Spoolman if print completed successfully
  1434. if data.get("status") == "completed":
  1435. try:
  1436. await _report_spoolman_usage(printer_id, archive_id, logger)
  1437. log_timing("Spoolman usage report")
  1438. except Exception as e:
  1439. logger.warning(f"Spoolman usage reporting failed: {e}")
  1440. # Run slow operations as background tasks to avoid blocking the event loop
  1441. # These operations can take 5-10+ seconds and would freeze the UI if awaited
  1442. starting_kwh = _print_energy_start.pop(archive_id, None)
  1443. async def _background_energy_calculation():
  1444. """Calculate and save energy usage in background."""
  1445. try:
  1446. logger.info(f"[ENERGY-BG] Starting energy calculation for archive {archive_id}")
  1447. async with async_session() as db:
  1448. plug_result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  1449. plug = plug_result.scalar_one_or_none()
  1450. if plug:
  1451. energy = await _get_plug_energy(plug, db)
  1452. logger.info(f"[ENERGY-BG] Energy response: {energy}")
  1453. energy_used = None
  1454. if starting_kwh is not None and energy and energy.get("total") is not None:
  1455. ending_kwh = energy["total"]
  1456. energy_used = round(ending_kwh - starting_kwh, 4)
  1457. logger.info(f"[ENERGY-BG] Per-print energy: {energy_used} kWh")
  1458. if energy_used is not None and energy_used >= 0:
  1459. from backend.app.api.routes.settings import get_setting
  1460. energy_cost_per_kwh = await get_setting(db, "energy_cost_per_kwh")
  1461. cost_per_kwh = float(energy_cost_per_kwh) if energy_cost_per_kwh else 0.15
  1462. energy_cost = round(energy_used * cost_per_kwh, 2)
  1463. from backend.app.models.archive import PrintArchive
  1464. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  1465. archive = result.scalar_one_or_none()
  1466. if archive:
  1467. archive.energy_kwh = energy_used
  1468. archive.energy_cost = energy_cost
  1469. await db.commit()
  1470. logger.info(f"[ENERGY-BG] Saved: {energy_used} kWh, cost={energy_cost}")
  1471. else:
  1472. logger.info(f"[ENERGY-BG] No smart plug for printer {printer_id}")
  1473. except Exception as e:
  1474. logger.warning(f"[ENERGY-BG] Failed: {e}")
  1475. async def _background_finish_photo() -> str | None:
  1476. """Capture finish photo in background. Returns photo filename if captured."""
  1477. try:
  1478. logger.info(f"[PHOTO-BG] Starting finish photo capture for archive {archive_id}")
  1479. from backend.app.api.routes.camera import _active_chamber_streams, _active_streams, get_buffered_frame
  1480. async with async_session() as db:
  1481. from backend.app.api.routes.settings import get_setting
  1482. capture_enabled = await get_setting(db, "capture_finish_photo")
  1483. if capture_enabled is None or capture_enabled.lower() == "true":
  1484. from backend.app.models.printer import Printer
  1485. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1486. printer = result.scalar_one_or_none()
  1487. if printer and archive_id:
  1488. from backend.app.models.archive import PrintArchive
  1489. result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  1490. archive = result.scalar_one_or_none()
  1491. if archive:
  1492. import uuid
  1493. from datetime import datetime
  1494. from pathlib import Path
  1495. archive_dir = app_settings.base_dir / Path(archive.file_path).parent
  1496. photo_filename = None
  1497. # Check for external camera first
  1498. if printer.external_camera_enabled and printer.external_camera_url:
  1499. logger.info("[PHOTO-BG] Using external camera")
  1500. from backend.app.services.external_camera import capture_frame
  1501. frame_data = await capture_frame(
  1502. printer.external_camera_url, printer.external_camera_type or "mjpeg"
  1503. )
  1504. if frame_data:
  1505. photos_dir = archive_dir / "photos"
  1506. photos_dir.mkdir(parents=True, exist_ok=True)
  1507. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  1508. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  1509. photo_path = photos_dir / photo_filename
  1510. await asyncio.to_thread(photo_path.write_bytes, frame_data)
  1511. logger.info(f"[PHOTO-BG] Saved external camera frame: {photo_filename}")
  1512. else:
  1513. # Check if camera stream is active - use buffered frame to avoid freeze
  1514. # Check both RTSP streams (_active_streams) and chamber image streams (_active_chamber_streams)
  1515. active_for_printer = [k for k in _active_streams if k.startswith(f"{printer_id}-")]
  1516. active_chamber_for_printer = [
  1517. k for k in _active_chamber_streams if k.startswith(f"{printer_id}-")
  1518. ]
  1519. buffered_frame = get_buffered_frame(printer_id)
  1520. if (active_for_printer or active_chamber_for_printer) and buffered_frame:
  1521. # Use frame from active stream
  1522. logger.info("[PHOTO-BG] Using buffered frame from active stream")
  1523. photos_dir = archive_dir / "photos"
  1524. photos_dir.mkdir(parents=True, exist_ok=True)
  1525. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  1526. photo_filename = f"finish_{timestamp}_{uuid.uuid4().hex[:8]}.jpg"
  1527. photo_path = photos_dir / photo_filename
  1528. await asyncio.to_thread(photo_path.write_bytes, buffered_frame)
  1529. logger.info(f"[PHOTO-BG] Saved buffered frame: {photo_filename}")
  1530. else:
  1531. # No active stream - capture new frame
  1532. from backend.app.services.camera import capture_finish_photo
  1533. photo_filename = await capture_finish_photo(
  1534. printer_id=printer_id,
  1535. ip_address=printer.ip_address,
  1536. access_code=printer.access_code,
  1537. model=printer.model,
  1538. archive_dir=archive_dir,
  1539. )
  1540. if photo_filename:
  1541. photos = archive.photos or []
  1542. photos.append(photo_filename)
  1543. archive.photos = photos
  1544. await db.commit()
  1545. logger.info(f"[PHOTO-BG] Saved: {photo_filename}")
  1546. return photo_filename
  1547. return None
  1548. except Exception as e:
  1549. logger.warning(f"[PHOTO-BG] Failed: {e}")
  1550. return None
  1551. asyncio.create_task(_background_energy_calculation())
  1552. # Photo capture task - result will be used by notifications
  1553. photo_task = asyncio.create_task(_background_finish_photo())
  1554. log_timing("Background tasks scheduled (energy, photo)")
  1555. # Also run smart plug, notifications, and maintenance as background tasks
  1556. print_status = data.get("status", "completed")
  1557. async def _background_smart_plug():
  1558. """Handle smart plug automation in background."""
  1559. try:
  1560. logger.info(f"[AUTO-OFF-BG] Starting smart plug automation for printer {printer_id}")
  1561. async with async_session() as db:
  1562. await smart_plug_manager.on_print_complete(printer_id, print_status, db)
  1563. logger.info("[AUTO-OFF-BG] Completed")
  1564. except Exception as e:
  1565. logger.warning(f"[AUTO-OFF-BG] Failed: {e}")
  1566. async def _background_notifications(finish_photo_filename: str | None = None):
  1567. """Send print complete notifications in background."""
  1568. try:
  1569. logger.info(f"[NOTIFY-BG] Starting notifications for printer {printer_id}, photo={finish_photo_filename}")
  1570. async with async_session() as db:
  1571. from backend.app.models.archive import PrintArchive
  1572. from backend.app.models.printer import Printer
  1573. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1574. printer = result.scalar_one_or_none()
  1575. printer_name = printer.name if printer else f"Printer {printer_id}"
  1576. archive_data = None
  1577. if archive_id:
  1578. archive_result = await db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  1579. archive = archive_result.scalar_one_or_none()
  1580. if archive:
  1581. archive_data = {
  1582. "print_time_seconds": archive.print_time_seconds,
  1583. "actual_filament_grams": archive.filament_used_grams,
  1584. "failure_reason": archive.failure_reason,
  1585. }
  1586. # Add finish photo URL if available
  1587. if finish_photo_filename:
  1588. from backend.app.api.routes.settings import get_setting
  1589. external_url = await get_setting(db, "external_url")
  1590. if external_url:
  1591. external_url = external_url.rstrip("/")
  1592. archive_data["finish_photo_url"] = (
  1593. f"{external_url}/api/v1/archives/{archive_id}/photos/{finish_photo_filename}"
  1594. )
  1595. else:
  1596. # Fallback to relative URL (won't work for external services)
  1597. archive_data["finish_photo_url"] = (
  1598. f"/api/v1/archives/{archive_id}/photos/{finish_photo_filename}"
  1599. )
  1600. await notification_service.on_print_complete(
  1601. printer_id, printer_name, print_status, data, db, archive_data=archive_data
  1602. )
  1603. logger.info("[NOTIFY-BG] Completed")
  1604. except Exception as e:
  1605. logger.warning(f"[NOTIFY-BG] Failed: {e}")
  1606. async def _background_maintenance_check():
  1607. """Check for maintenance due in background."""
  1608. if print_status != "completed":
  1609. return
  1610. try:
  1611. logger.info(f"[MAINT-BG] Starting maintenance check for printer {printer_id}")
  1612. async with async_session() as db:
  1613. from backend.app.models.printer import Printer
  1614. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1615. printer = result.scalar_one_or_none()
  1616. printer_name = printer.name if printer else f"Printer {printer_id}"
  1617. await ensure_default_types(db)
  1618. overview = await _get_printer_maintenance_internal(printer_id, db, commit=True)
  1619. items_needing_attention = [
  1620. {"name": item.maintenance_type_name, "is_due": item.is_due, "is_warning": item.is_warning}
  1621. for item in overview.maintenance_items
  1622. if item.enabled and (item.is_due or item.is_warning)
  1623. ]
  1624. if items_needing_attention:
  1625. await notification_service.on_maintenance_due(printer_id, printer_name, items_needing_attention, db)
  1626. logger.info(f"[MAINT-BG] Sent notification: {len(items_needing_attention)} items need attention")
  1627. # MQTT relay - publish maintenance alerts
  1628. for item in items_needing_attention:
  1629. try:
  1630. await mqtt_relay.on_maintenance_alert(
  1631. printer_id=printer_id,
  1632. printer_name=printer_name,
  1633. maintenance_type=item["name"],
  1634. current_value=0, # Not easily available here
  1635. threshold=0, # Not easily available here
  1636. )
  1637. except Exception:
  1638. pass # Don't fail if MQTT fails
  1639. else:
  1640. logger.info("[MAINT-BG] Completed (no items need attention)")
  1641. except Exception as e:
  1642. logger.warning(f"[MAINT-BG] Failed: {e}")
  1643. asyncio.create_task(_background_smart_plug())
  1644. asyncio.create_task(_background_maintenance_check())
  1645. # Notification task waits for photo capture to complete first
  1646. async def _photo_then_notify():
  1647. """Wait for photo capture, then send notification with photo URL."""
  1648. try:
  1649. finish_photo = await photo_task
  1650. logger.info(f"[PHOTO-NOTIFY] Photo task returned: {finish_photo}")
  1651. await _background_notifications(finish_photo)
  1652. except Exception as e:
  1653. logger.warning(f"[PHOTO-NOTIFY] Failed: {e}")
  1654. # Still try to send notification without photo
  1655. await _background_notifications(None)
  1656. asyncio.create_task(_photo_then_notify())
  1657. # Stitch external camera layer timelapse if session was active
  1658. print_status = data.get("status", "completed")
  1659. async def _background_layer_timelapse():
  1660. """Stitch layer timelapse and attach to archive."""
  1661. from backend.app.services.layer_timelapse import cancel_session, on_print_complete as tl_complete
  1662. try:
  1663. if print_status == "completed":
  1664. logger.info(f"[LAYER-TL] Stitching layer timelapse for printer {printer_id}")
  1665. timelapse_path = await tl_complete(printer_id)
  1666. if timelapse_path and archive_id:
  1667. logger.info(f"[LAYER-TL] Attaching timelapse {timelapse_path} to archive {archive_id}")
  1668. async with async_session() as db:
  1669. service = ArchiveService(db)
  1670. timelapse_data = await asyncio.to_thread(timelapse_path.read_bytes)
  1671. await service.attach_timelapse(archive_id, timelapse_data, "layer_timelapse.mp4")
  1672. # Clean up the temp file
  1673. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  1674. logger.info("[LAYER-TL] Layer timelapse attached successfully")
  1675. elif timelapse_path:
  1676. # Timelapse created but no archive - just clean up
  1677. await asyncio.to_thread(timelapse_path.unlink, missing_ok=True)
  1678. else:
  1679. # Print failed or cancelled - cancel timelapse session
  1680. cancel_session(printer_id)
  1681. logger.info(f"[LAYER-TL] Cancelled layer timelapse for printer {printer_id} (status: {print_status})")
  1682. except Exception as e:
  1683. logger.warning(f"[LAYER-TL] Failed: {e}")
  1684. # Try to cancel session on error
  1685. try:
  1686. cancel_session(printer_id)
  1687. except Exception:
  1688. pass
  1689. asyncio.create_task(_background_layer_timelapse())
  1690. log_timing("All background tasks scheduled")
  1691. # Auto-scan for timelapse if recording was active during the print
  1692. if archive_id and data.get("timelapse_was_active") and data.get("status") == "completed":
  1693. logger.info(f"[TIMELAPSE] Timelapse was active during print, scheduling auto-scan for archive {archive_id}")
  1694. # Schedule timelapse scan as background task with retries
  1695. # The printer needs time to encode the video after print completion
  1696. asyncio.create_task(_scan_for_timelapse_with_retries(archive_id))
  1697. log_timing("Timelapse scan scheduled")
  1698. # Update queue item if this was a scheduled print
  1699. try:
  1700. async with async_session() as db:
  1701. from backend.app.models.print_queue import PrintQueueItem
  1702. # Note: SmartPlug is already imported at module level (line 56)
  1703. # Do NOT import it here as it would shadow the module-level import
  1704. # and cause "cannot access local variable" errors earlier in this function
  1705. result = await db.execute(
  1706. select(PrintQueueItem)
  1707. .where(PrintQueueItem.printer_id == printer_id)
  1708. .where(PrintQueueItem.status == "printing")
  1709. )
  1710. queue_item = result.scalar_one_or_none()
  1711. if queue_item:
  1712. status = data.get("status", "completed")
  1713. queue_item.status = status
  1714. queue_item.completed_at = datetime.now()
  1715. await db.commit()
  1716. logger.info(f"Updated queue item {queue_item.id} status to {status}")
  1717. # MQTT relay - publish queue job completed
  1718. try:
  1719. printer_info = printer_manager.get_printer(printer_id)
  1720. await mqtt_relay.on_queue_job_completed(
  1721. job_id=queue_item.id,
  1722. filename=filename or subtask_name,
  1723. printer_id=printer_id,
  1724. printer_name=printer_info.name if printer_info else "Unknown",
  1725. status=status,
  1726. )
  1727. except Exception:
  1728. pass # Don't fail if MQTT fails
  1729. # Check if queue is now empty and send notification
  1730. try:
  1731. from sqlalchemy import func
  1732. # Count remaining pending items
  1733. count_result = await db.execute(
  1734. select(func.count(PrintQueueItem.id)).where(PrintQueueItem.status == "pending")
  1735. )
  1736. pending_count = count_result.scalar() or 0
  1737. if pending_count == 0:
  1738. # Count how many completed today (rough approximation)
  1739. today_start = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
  1740. completed_result = await db.execute(
  1741. select(func.count(PrintQueueItem.id)).where(
  1742. PrintQueueItem.status.in_(["completed", "failed", "skipped"]),
  1743. PrintQueueItem.completed_at >= today_start,
  1744. )
  1745. )
  1746. completed_count = completed_result.scalar() or 1
  1747. await notification_service.on_queue_completed(
  1748. completed_count=completed_count,
  1749. db=db,
  1750. )
  1751. except Exception:
  1752. pass # Don't fail if notification fails
  1753. # Handle auto_off_after - power off printer if requested (after cooldown)
  1754. if queue_item.auto_off_after:
  1755. result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  1756. plug = result.scalar_one_or_none()
  1757. if plug and plug.enabled:
  1758. logger.info(f"Auto-off requested for printer {printer_id}, waiting for cooldown...")
  1759. async def cooldown_and_poweroff(pid: int, plug_id: int):
  1760. # Wait for nozzle to cool down
  1761. await printer_manager.wait_for_cooldown(pid, target_temp=50.0, timeout=600)
  1762. # Re-fetch plug in new session
  1763. async with async_session() as new_db:
  1764. result = await new_db.execute(select(SmartPlug).where(SmartPlug.id == plug_id))
  1765. p = result.scalar_one_or_none()
  1766. if p and p.enabled:
  1767. success = await tasmota_service.turn_off(p)
  1768. if success:
  1769. logger.info(f"Powered off printer {pid} via smart plug '{p.name}'")
  1770. else:
  1771. logger.warning(f"Failed to power off printer {pid} via smart plug")
  1772. asyncio.create_task(cooldown_and_poweroff(printer_id, plug.id))
  1773. except Exception as e:
  1774. import logging
  1775. logging.getLogger(__name__).warning(f"Queue item update failed: {e}")
  1776. log_timing("Queue item update")
  1777. logger.info(f"[CALLBACK] on_print_complete finished for printer {printer_id}, archive {archive_id}")
  1778. # AMS sensor history recording
  1779. _ams_history_task: asyncio.Task | None = None
  1780. AMS_HISTORY_INTERVAL = 300 # Record every 5 minutes
  1781. AMS_HISTORY_RETENTION_DAYS = 30 # Keep data for 30 days
  1782. _ams_cleanup_counter = 0 # Track recordings to trigger periodic cleanup
  1783. _ams_alarm_cooldown: dict[str, datetime] = {} # Track alarm cooldowns (printer_id:ams_id:type -> last_alarm_time)
  1784. AMS_ALARM_COOLDOWN_MINUTES = 60 # Don't send same alarm more than once per hour
  1785. async def record_ams_history():
  1786. """Background task to record AMS humidity and temperature data."""
  1787. import logging
  1788. logger = logging.getLogger(__name__)
  1789. # Wait a short time for MQTT connections to establish on startup
  1790. await asyncio.sleep(10)
  1791. while True:
  1792. try:
  1793. from backend.app.models.ams_history import AMSSensorHistory
  1794. from backend.app.models.printer import Printer
  1795. from backend.app.models.settings import Settings
  1796. async with async_session() as db:
  1797. # Get all active printers
  1798. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  1799. printers = result.scalars().all()
  1800. # Get alarm thresholds from settings
  1801. humidity_threshold = 60.0 # Default: fair threshold
  1802. temp_threshold = 35.0 # Default: fair threshold
  1803. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  1804. setting = result.scalar_one_or_none()
  1805. if setting:
  1806. try:
  1807. humidity_threshold = float(setting.value)
  1808. except (ValueError, TypeError):
  1809. pass
  1810. result = await db.execute(select(Settings).where(Settings.key == "ams_temp_fair"))
  1811. setting = result.scalar_one_or_none()
  1812. if setting:
  1813. try:
  1814. temp_threshold = float(setting.value)
  1815. except (ValueError, TypeError):
  1816. pass
  1817. recorded_count = 0
  1818. for printer in printers:
  1819. # Get current state from printer manager
  1820. state = printer_manager.get_status(printer.id)
  1821. if not state or not state.connected or not state.raw_data:
  1822. continue # Skip disconnected printers - don't use stale data
  1823. raw_data = state.raw_data
  1824. if "ams" not in raw_data or not isinstance(raw_data["ams"], list):
  1825. continue
  1826. # Record data for each AMS unit
  1827. for ams_data in raw_data["ams"]:
  1828. ams_id = int(ams_data.get("id", 0))
  1829. # Get humidity (prefer humidity_raw)
  1830. humidity_raw = ams_data.get("humidity_raw")
  1831. humidity_idx = ams_data.get("humidity")
  1832. humidity = None
  1833. if humidity_raw is not None:
  1834. try:
  1835. humidity = float(humidity_raw)
  1836. except (ValueError, TypeError):
  1837. pass
  1838. if humidity is None and humidity_idx is not None:
  1839. try:
  1840. humidity = float(humidity_idx)
  1841. except (ValueError, TypeError):
  1842. pass
  1843. # Get temperature
  1844. temperature = None
  1845. temp_str = ams_data.get("temp")
  1846. if temp_str is not None:
  1847. try:
  1848. temperature = float(temp_str)
  1849. except (ValueError, TypeError):
  1850. pass
  1851. # Skip if no data
  1852. if humidity is None and temperature is None:
  1853. continue
  1854. # Record the data point
  1855. history = AMSSensorHistory(
  1856. printer_id=printer.id,
  1857. ams_id=ams_id,
  1858. humidity=humidity,
  1859. humidity_raw=float(humidity_raw) if humidity_raw else None,
  1860. temperature=temperature,
  1861. )
  1862. db.add(history)
  1863. recorded_count += 1
  1864. # Generate AMS label and determine if it's AMS-HT (A, B, C, D or HT-A for AMS-Lite/Hub)
  1865. is_ams_ht = ams_id >= 128
  1866. if is_ams_ht:
  1867. ams_label = f"HT-{chr(65 + (ams_id - 128))}"
  1868. else:
  1869. ams_label = f"AMS-{chr(65 + ams_id)}"
  1870. # Check humidity alarm (only if above threshold)
  1871. if humidity is not None and humidity > humidity_threshold:
  1872. cooldown_key = f"{printer.id}:{ams_id}:humidity"
  1873. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  1874. now = datetime.now()
  1875. if (
  1876. last_alarm is None
  1877. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  1878. ):
  1879. _ams_alarm_cooldown[cooldown_key] = now
  1880. logger.info(
  1881. f"Sending humidity alarm for {printer.name} {ams_label}: {humidity}% > {humidity_threshold}%"
  1882. )
  1883. try:
  1884. # Call different notification method based on AMS type
  1885. if is_ams_ht:
  1886. await notification_service.on_ams_ht_humidity_high(
  1887. printer.id, printer.name, ams_label, humidity, humidity_threshold, db
  1888. )
  1889. else:
  1890. await notification_service.on_ams_humidity_high(
  1891. printer.id, printer.name, ams_label, humidity, humidity_threshold, db
  1892. )
  1893. except Exception as e:
  1894. logger.warning(f"Failed to send humidity alarm: {e}")
  1895. # Check temperature alarm (only if above threshold)
  1896. if temperature is not None and temperature > temp_threshold:
  1897. cooldown_key = f"{printer.id}:{ams_id}:temperature"
  1898. last_alarm = _ams_alarm_cooldown.get(cooldown_key)
  1899. now = datetime.now()
  1900. if (
  1901. last_alarm is None
  1902. or (now - last_alarm).total_seconds() >= AMS_ALARM_COOLDOWN_MINUTES * 60
  1903. ):
  1904. _ams_alarm_cooldown[cooldown_key] = now
  1905. logger.info(
  1906. f"Sending temperature alarm for {printer.name} {ams_label}: {temperature}°C > {temp_threshold}°C"
  1907. )
  1908. try:
  1909. # Call different notification method based on AMS type
  1910. if is_ams_ht:
  1911. await notification_service.on_ams_ht_temperature_high(
  1912. printer.id, printer.name, ams_label, temperature, temp_threshold, db
  1913. )
  1914. else:
  1915. await notification_service.on_ams_temperature_high(
  1916. printer.id, printer.name, ams_label, temperature, temp_threshold, db
  1917. )
  1918. except Exception as e:
  1919. logger.warning(f"Failed to send temperature alarm: {e}")
  1920. await db.commit()
  1921. if recorded_count > 0:
  1922. logger.info(f"Recorded {recorded_count} AMS sensor history entries")
  1923. # Periodic cleanup of old data (every ~288 recordings = ~24 hours at 5min interval)
  1924. global _ams_cleanup_counter
  1925. _ams_cleanup_counter += 1
  1926. if _ams_cleanup_counter >= 288:
  1927. _ams_cleanup_counter = 0
  1928. # Get retention days from settings
  1929. from backend.app.models.settings import Settings
  1930. result = await db.execute(select(Settings).where(Settings.key == "ams_history_retention_days"))
  1931. setting = result.scalar_one_or_none()
  1932. retention_days = int(setting.value) if setting else AMS_HISTORY_RETENTION_DAYS
  1933. cutoff = datetime.now() - timedelta(days=retention_days)
  1934. result = await db.execute(delete(AMSSensorHistory).where(AMSSensorHistory.recorded_at < cutoff))
  1935. await db.commit()
  1936. if result.rowcount > 0:
  1937. logger.info(
  1938. f"Cleaned up {result.rowcount} old AMS sensor history entries (older than {retention_days} days)"
  1939. )
  1940. # Wait until next recording interval
  1941. await asyncio.sleep(AMS_HISTORY_INTERVAL)
  1942. except asyncio.CancelledError:
  1943. break
  1944. except Exception as e:
  1945. logger.warning(f"AMS history recording failed: {e}")
  1946. await asyncio.sleep(60) # Wait a bit before retrying
  1947. def start_ams_history_recording():
  1948. """Start the AMS history recording background task."""
  1949. global _ams_history_task
  1950. if _ams_history_task is None:
  1951. _ams_history_task = asyncio.create_task(record_ams_history())
  1952. logging.getLogger(__name__).info("AMS history recording started")
  1953. def stop_ams_history_recording():
  1954. """Stop the AMS history recording background task."""
  1955. global _ams_history_task
  1956. if _ams_history_task:
  1957. _ams_history_task.cancel()
  1958. _ams_history_task = None
  1959. logging.getLogger(__name__).info("AMS history recording stopped")
  1960. # Printer runtime tracking
  1961. _runtime_tracking_task: asyncio.Task | None = None
  1962. RUNTIME_TRACKING_INTERVAL = 30 # Update every 30 seconds
  1963. async def track_printer_runtime():
  1964. """Background task to track printer active runtime (RUNNING/PAUSE states)."""
  1965. import logging
  1966. logger = logging.getLogger(__name__)
  1967. # Wait for MQTT connections to establish on startup
  1968. await asyncio.sleep(15)
  1969. while True:
  1970. try:
  1971. from backend.app.models.printer import Printer
  1972. async with async_session() as db:
  1973. # Get all active printers
  1974. result = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  1975. printers = result.scalars().all()
  1976. now = datetime.now()
  1977. updated_count = 0
  1978. needs_commit = False
  1979. for printer in printers:
  1980. # Get current state from printer manager
  1981. state = printer_manager.get_status(printer.id)
  1982. if not state:
  1983. logger.debug(f"[{printer.name}] Runtime tracking: no state available")
  1984. continue
  1985. if not state.connected:
  1986. logger.debug(f"[{printer.name}] Runtime tracking: not connected")
  1987. continue
  1988. # Check if printer is in an active state (RUNNING or PAUSE)
  1989. if state.state in ("RUNNING", "PAUSE"):
  1990. # Calculate time since last update
  1991. if printer.last_runtime_update:
  1992. elapsed = (now - printer.last_runtime_update).total_seconds()
  1993. if elapsed > 0:
  1994. printer.runtime_seconds += int(elapsed)
  1995. updated_count += 1
  1996. needs_commit = True
  1997. logger.debug(
  1998. f"[{printer.name}] Runtime tracking: added {int(elapsed)}s, "
  1999. f"total={printer.runtime_seconds}s ({printer.runtime_seconds / 3600:.2f}h)"
  2000. )
  2001. else:
  2002. # First time seeing printer active - need to commit to save timestamp
  2003. needs_commit = True
  2004. logger.debug(f"[{printer.name}] Runtime tracking: first active detection")
  2005. printer.last_runtime_update = now
  2006. else:
  2007. # Printer is idle/offline - clear last_runtime_update
  2008. if printer.last_runtime_update is not None:
  2009. logger.debug(
  2010. f"[{printer.name}] Runtime tracking: state={state.state}, clearing last_runtime_update"
  2011. )
  2012. printer.last_runtime_update = None
  2013. needs_commit = True
  2014. if needs_commit:
  2015. await db.commit()
  2016. if updated_count > 0:
  2017. logger.debug(f"Updated runtime for {updated_count} printer(s)")
  2018. except asyncio.CancelledError:
  2019. logger.info("Runtime tracking cancelled")
  2020. break
  2021. except Exception as e:
  2022. logger.warning(f"Runtime tracking failed: {e}")
  2023. await asyncio.sleep(RUNTIME_TRACKING_INTERVAL)
  2024. def start_runtime_tracking():
  2025. """Start the printer runtime tracking background task."""
  2026. global _runtime_tracking_task
  2027. if _runtime_tracking_task is None:
  2028. _runtime_tracking_task = asyncio.create_task(track_printer_runtime())
  2029. logging.getLogger(__name__).info("Printer runtime tracking started")
  2030. def stop_runtime_tracking():
  2031. """Stop the printer runtime tracking background task."""
  2032. global _runtime_tracking_task
  2033. if _runtime_tracking_task:
  2034. _runtime_tracking_task.cancel()
  2035. _runtime_tracking_task = None
  2036. logging.getLogger(__name__).info("Printer runtime tracking stopped")
  2037. @asynccontextmanager
  2038. async def lifespan(app: FastAPI):
  2039. # Startup
  2040. await init_db()
  2041. # Restore debug logging state from previous session
  2042. await init_debug_logging()
  2043. # Set up printer manager callbacks
  2044. loop = asyncio.get_event_loop()
  2045. printer_manager.set_event_loop(loop)
  2046. printer_manager.set_status_change_callback(on_printer_status_change)
  2047. printer_manager.set_print_start_callback(on_print_start)
  2048. printer_manager.set_print_complete_callback(on_print_complete)
  2049. printer_manager.set_ams_change_callback(on_ams_change)
  2050. # Layer change callback for external camera timelapse
  2051. async def on_layer_change(printer_id: int, layer_num: int):
  2052. """Capture timelapse frame on layer change."""
  2053. from backend.app.services.layer_timelapse import on_layer_change as tl_layer_change
  2054. await tl_layer_change(printer_id, layer_num)
  2055. printer_manager.set_layer_change_callback(on_layer_change)
  2056. # Initialize MQTT relay from settings
  2057. async with async_session() as db:
  2058. from backend.app.api.routes.settings import get_setting
  2059. mqtt_settings = {
  2060. "mqtt_enabled": (await get_setting(db, "mqtt_enabled") or "false") == "true",
  2061. "mqtt_broker": await get_setting(db, "mqtt_broker") or "",
  2062. "mqtt_port": int(await get_setting(db, "mqtt_port") or "1883"),
  2063. "mqtt_username": await get_setting(db, "mqtt_username") or "",
  2064. "mqtt_password": await get_setting(db, "mqtt_password") or "",
  2065. "mqtt_topic_prefix": await get_setting(db, "mqtt_topic_prefix") or "bambuddy",
  2066. "mqtt_use_tls": (await get_setting(db, "mqtt_use_tls") or "false") == "true",
  2067. }
  2068. await mqtt_relay.configure(mqtt_settings)
  2069. # Restore MQTT smart plug subscriptions
  2070. if mqtt_settings.get("mqtt_enabled"):
  2071. from sqlalchemy import select
  2072. from backend.app.models.smart_plug import SmartPlug
  2073. result = await db.execute(select(SmartPlug).where(SmartPlug.plug_type == "mqtt"))
  2074. mqtt_plugs = result.scalars().all()
  2075. for plug in mqtt_plugs:
  2076. if plug.mqtt_topic:
  2077. mqtt_relay.smart_plug_service.subscribe(
  2078. plug_id=plug.id,
  2079. topic=plug.mqtt_topic,
  2080. power_path=plug.mqtt_power_path,
  2081. energy_path=plug.mqtt_energy_path,
  2082. state_path=plug.mqtt_state_path,
  2083. multiplier=plug.mqtt_multiplier or 1.0,
  2084. )
  2085. if mqtt_plugs:
  2086. logging.info(f"Restored {len(mqtt_plugs)} MQTT smart plug subscriptions")
  2087. # Connect to all active printers
  2088. async with async_session() as db:
  2089. await init_printer_connections(db)
  2090. # Auto-connect to Spoolman if enabled
  2091. async with async_session() as db:
  2092. from backend.app.api.routes.settings import get_setting
  2093. spoolman_enabled = await get_setting(db, "spoolman_enabled")
  2094. spoolman_url = await get_setting(db, "spoolman_url")
  2095. if spoolman_enabled and spoolman_enabled.lower() == "true" and spoolman_url:
  2096. try:
  2097. client = await init_spoolman_client(spoolman_url)
  2098. if await client.health_check():
  2099. logging.info(f"Auto-connected to Spoolman at {spoolman_url}")
  2100. # Ensure the 'tag' extra field exists for RFID/UUID storage
  2101. await client.ensure_tag_extra_field()
  2102. else:
  2103. logging.warning(f"Spoolman at {spoolman_url} is not reachable")
  2104. except Exception as e:
  2105. logging.warning(f"Failed to auto-connect to Spoolman: {e}")
  2106. # Start the print scheduler
  2107. asyncio.create_task(print_scheduler.run())
  2108. # Start the smart plug scheduler for time-based on/off
  2109. smart_plug_manager.start_scheduler()
  2110. # Resume any pending auto-offs that were interrupted by restart
  2111. await smart_plug_manager.resume_pending_auto_offs()
  2112. # Start the notification digest scheduler
  2113. notification_service.start_digest_scheduler()
  2114. # Start the GitHub backup scheduler
  2115. await github_backup_service.start_scheduler()
  2116. # Start AMS history recording
  2117. start_ams_history_recording()
  2118. # Start printer runtime tracking
  2119. start_runtime_tracking()
  2120. # Initialize virtual printer manager
  2121. from backend.app.services.virtual_printer import virtual_printer_manager
  2122. virtual_printer_manager.set_session_factory(async_session)
  2123. # Auto-start virtual printer if enabled
  2124. async with async_session() as db:
  2125. from backend.app.api.routes.settings import get_setting
  2126. vp_enabled = await get_setting(db, "virtual_printer_enabled")
  2127. if vp_enabled and vp_enabled.lower() == "true":
  2128. vp_access_code = await get_setting(db, "virtual_printer_access_code") or ""
  2129. vp_mode = await get_setting(db, "virtual_printer_mode") or "immediate"
  2130. vp_model = await get_setting(db, "virtual_printer_model") or ""
  2131. if vp_access_code:
  2132. try:
  2133. await virtual_printer_manager.configure(
  2134. enabled=True,
  2135. access_code=vp_access_code,
  2136. mode=vp_mode,
  2137. model=vp_model,
  2138. )
  2139. logging.info(f"Virtual printer started (model={vp_model or 'default'})")
  2140. except Exception as e:
  2141. logging.warning(f"Failed to start virtual printer: {e}")
  2142. yield
  2143. # Shutdown
  2144. print_scheduler.stop()
  2145. smart_plug_manager.stop_scheduler()
  2146. notification_service.stop_digest_scheduler()
  2147. github_backup_service.stop_scheduler()
  2148. stop_ams_history_recording()
  2149. stop_runtime_tracking()
  2150. printer_manager.disconnect_all()
  2151. await close_spoolman_client()
  2152. # Stop virtual printer if running
  2153. if virtual_printer_manager.is_enabled:
  2154. await virtual_printer_manager.configure(enabled=False)
  2155. app = FastAPI(
  2156. title=app_settings.app_name,
  2157. description="Archive and manage Bambu Lab 3MF files",
  2158. version=APP_VERSION,
  2159. lifespan=lifespan,
  2160. )
  2161. # API routes
  2162. app.include_router(auth.router, prefix=app_settings.api_prefix)
  2163. app.include_router(users.router, prefix=app_settings.api_prefix)
  2164. app.include_router(groups.router, prefix=app_settings.api_prefix)
  2165. app.include_router(printers.router, prefix=app_settings.api_prefix)
  2166. app.include_router(archives.router, prefix=app_settings.api_prefix)
  2167. app.include_router(filaments.router, prefix=app_settings.api_prefix)
  2168. app.include_router(settings_routes.router, prefix=app_settings.api_prefix)
  2169. app.include_router(cloud.router, prefix=app_settings.api_prefix)
  2170. app.include_router(smart_plugs.router, prefix=app_settings.api_prefix)
  2171. app.include_router(print_queue.router, prefix=app_settings.api_prefix)
  2172. app.include_router(kprofiles.router, prefix=app_settings.api_prefix)
  2173. app.include_router(notifications.router, prefix=app_settings.api_prefix)
  2174. app.include_router(notification_templates.router, prefix=app_settings.api_prefix)
  2175. app.include_router(spoolman.router, prefix=app_settings.api_prefix)
  2176. app.include_router(updates.router, prefix=app_settings.api_prefix)
  2177. app.include_router(maintenance.router, prefix=app_settings.api_prefix)
  2178. app.include_router(camera.router, prefix=app_settings.api_prefix)
  2179. app.include_router(external_links.router, prefix=app_settings.api_prefix)
  2180. app.include_router(projects.router, prefix=app_settings.api_prefix)
  2181. app.include_router(library.router, prefix=app_settings.api_prefix)
  2182. app.include_router(api_keys.router, prefix=app_settings.api_prefix)
  2183. app.include_router(webhook.router, prefix=app_settings.api_prefix)
  2184. app.include_router(ams_history.router, prefix=app_settings.api_prefix)
  2185. app.include_router(system.router, prefix=app_settings.api_prefix)
  2186. app.include_router(support.router, prefix=app_settings.api_prefix)
  2187. app.include_router(websocket.router, prefix=app_settings.api_prefix)
  2188. app.include_router(discovery.router, prefix=app_settings.api_prefix)
  2189. app.include_router(pending_uploads.router, prefix=app_settings.api_prefix)
  2190. app.include_router(firmware.router, prefix=app_settings.api_prefix)
  2191. app.include_router(github_backup.router, prefix=app_settings.api_prefix)
  2192. app.include_router(metrics.router, prefix=app_settings.api_prefix)
  2193. # Serve static files (React build)
  2194. if app_settings.static_dir.exists() and any(app_settings.static_dir.iterdir()):
  2195. app.mount(
  2196. "/assets",
  2197. StaticFiles(directory=app_settings.static_dir / "assets"),
  2198. name="assets",
  2199. )
  2200. if (app_settings.static_dir / "img").exists():
  2201. app.mount(
  2202. "/img",
  2203. StaticFiles(directory=app_settings.static_dir / "img"),
  2204. name="img",
  2205. )
  2206. if (app_settings.static_dir / "icons").exists():
  2207. app.mount(
  2208. "/icons",
  2209. StaticFiles(directory=app_settings.static_dir / "icons"),
  2210. name="icons",
  2211. )
  2212. @app.get("/")
  2213. async def serve_frontend():
  2214. """Serve the React frontend."""
  2215. index_file = app_settings.static_dir / "index.html"
  2216. if index_file.exists():
  2217. return FileResponse(index_file)
  2218. return {
  2219. "message": "Bambuddy API",
  2220. "docs": "/docs",
  2221. "frontend": "Build and place React app in /static directory",
  2222. }
  2223. @app.get("/health")
  2224. async def health_check():
  2225. """Health check endpoint."""
  2226. return {"status": "healthy"}
  2227. @app.get("/manifest.json")
  2228. async def serve_manifest():
  2229. """Serve PWA manifest."""
  2230. manifest_file = app_settings.static_dir / "manifest.json"
  2231. if manifest_file.exists():
  2232. return FileResponse(manifest_file, media_type="application/manifest+json")
  2233. return {"error": "Manifest not found"}
  2234. @app.get("/sw.js")
  2235. async def serve_service_worker():
  2236. """Serve service worker."""
  2237. sw_file = app_settings.static_dir / "sw.js"
  2238. if sw_file.exists():
  2239. return FileResponse(sw_file, media_type="application/javascript")
  2240. return {"error": "Service worker not found"}
  2241. # Catch-all route for React Router (must be last)
  2242. @app.get("/{full_path:path}")
  2243. async def serve_spa(full_path: str):
  2244. """Serve React app for client-side routing."""
  2245. # Don't intercept API routes - raise proper 404 so FastAPI can handle redirects
  2246. if full_path.startswith("api/"):
  2247. from fastapi import HTTPException
  2248. raise HTTPException(status_code=404, detail="Not found")
  2249. index_file = app_settings.static_dir / "index.html"
  2250. if index_file.exists():
  2251. return FileResponse(index_file)
  2252. return {"error": "Frontend not built"}