print_scheduler.py 96 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024
  1. """Print scheduler service - processes the print queue."""
  2. import asyncio
  3. import json
  4. import logging
  5. import time
  6. import zipfile
  7. from datetime import datetime, timezone
  8. from pathlib import Path
  9. import defusedxml.ElementTree as ET
  10. from sqlalchemy import func, select
  11. from sqlalchemy.ext.asyncio import AsyncSession
  12. from backend.app.core.config import settings
  13. from backend.app.core.database import async_session
  14. from backend.app.models.archive import PrintArchive
  15. from backend.app.models.library import LibraryFile
  16. from backend.app.models.print_queue import PrintQueueItem
  17. from backend.app.models.printer import Printer
  18. from backend.app.models.settings import Settings
  19. from backend.app.models.smart_plug import SmartPlug
  20. from backend.app.services.bambu_ftp import delete_file_async, get_ftp_retry_settings, upload_file_async, with_ftp_retry
  21. from backend.app.services.notification_service import notification_service
  22. from backend.app.services.printer_manager import printer_manager, supports_drying
  23. from backend.app.services.smart_plug_manager import smart_plug_manager
  24. from backend.app.utils.printer_models import normalize_printer_model
  25. from backend.app.utils.threemf_tools import extract_nozzle_mapping_from_3mf
  26. logger = logging.getLogger(__name__)
  27. # Filament type equivalence groups — types within the same group are
  28. # interchangeable on the printer side (Bambu Lab firmware treats them as compatible).
  29. _FILAMENT_TYPE_GROUPS: list[list[str]] = [
  30. ["PA-CF", "PA12-CF", "PAHT-CF"],
  31. ]
  32. _FILAMENT_EQUIV_MAP: dict[str, str] = {}
  33. for _group in _FILAMENT_TYPE_GROUPS:
  34. _canonical = _group[0].upper()
  35. for _t in _group:
  36. _FILAMENT_EQUIV_MAP[_t.upper()] = _canonical
  37. def _canonical_filament_type(ftype: str) -> str:
  38. """Return canonical type for equivalence matching."""
  39. upper = ftype.upper()
  40. return _FILAMENT_EQUIV_MAP.get(upper, upper)
  41. class PrintScheduler:
  42. """Background scheduler that processes the print queue."""
  43. # Built-in drying presets per filament type (from BambuStudio filament profiles)
  44. # Format: { n3f_temp, n3s_temp, n3f_hours, n3s_hours }
  45. DEFAULT_DRYING_PRESETS: dict[str, dict[str, int]] = {
  46. "PLA": {"n3f": 45, "n3s": 45, "n3f_hours": 12, "n3s_hours": 12},
  47. "PETG": {"n3f": 65, "n3s": 65, "n3f_hours": 12, "n3s_hours": 12},
  48. "TPU": {"n3f": 65, "n3s": 75, "n3f_hours": 12, "n3s_hours": 18},
  49. "ABS": {"n3f": 65, "n3s": 80, "n3f_hours": 12, "n3s_hours": 8},
  50. "ASA": {"n3f": 65, "n3s": 80, "n3f_hours": 12, "n3s_hours": 8},
  51. "PA": {"n3f": 65, "n3s": 85, "n3f_hours": 12, "n3s_hours": 12},
  52. "PC": {"n3f": 65, "n3s": 80, "n3f_hours": 12, "n3s_hours": 8},
  53. "PVA": {"n3f": 65, "n3s": 85, "n3f_hours": 12, "n3s_hours": 18},
  54. }
  55. def __init__(self):
  56. self._running = False
  57. self._check_interval = 30 # seconds
  58. self._power_on_wait_time = 180 # seconds to wait for printer after power on (3 min)
  59. self._power_on_check_interval = 10 # seconds between connection checks
  60. self._min_drying_seconds = 1800 # 30 minutes minimum before humidity re-check can stop drying
  61. # Track which printers are currently auto-drying (printer_id -> start timestamp)
  62. self._drying_in_progress: dict[int, float] = {}
  63. async def run(self):
  64. """Main loop - check queue every interval."""
  65. self._running = True
  66. logger.info("Print scheduler started")
  67. while self._running:
  68. try:
  69. await self.check_queue()
  70. except Exception as e:
  71. logger.error("Scheduler error: %s", e)
  72. await asyncio.sleep(self._check_interval)
  73. def stop(self):
  74. """Stop the scheduler."""
  75. self._running = False
  76. logger.info("Print scheduler stopped")
  77. async def check_queue(self):
  78. """Check for prints ready to start."""
  79. async with async_session() as db:
  80. # Check if shortest-job-first scheduling is enabled
  81. sjf_enabled = await self._get_bool_setting(db, "queue_shortest_first")
  82. # Get all pending items, ordered by printer and position (or SJF order)
  83. if sjf_enabled:
  84. # SJF: group by printer (and target_model for model-based jobs),
  85. # then items already jumped get top priority (starvation guard),
  86. # then sort by print_time ascending. Items with no print time go last.
  87. result = await db.execute(
  88. select(PrintQueueItem)
  89. .where(PrintQueueItem.status == "pending")
  90. .order_by(
  91. PrintQueueItem.printer_id,
  92. PrintQueueItem.target_model,
  93. PrintQueueItem.been_jumped.desc(),
  94. PrintQueueItem.print_time_seconds.asc().nullslast(),
  95. PrintQueueItem.position,
  96. )
  97. )
  98. else:
  99. result = await db.execute(
  100. select(PrintQueueItem)
  101. .where(PrintQueueItem.status == "pending")
  102. .order_by(PrintQueueItem.printer_id, PrintQueueItem.position)
  103. )
  104. items = list(result.scalars().all())
  105. # Read plate-clear setting once per queue check
  106. require_plate_clear = await self._get_bool_setting(db, "require_plate_clear", default=True)
  107. if not items:
  108. # No pending items — still check auto-drying on idle printers
  109. await self._check_auto_drying(db, [], set(), require_plate_clear=require_plate_clear)
  110. return
  111. logger.info(
  112. "Queue check: found %d pending items: %s",
  113. len(items),
  114. [(i.id, i.printer_id, i.archive_id, i.library_file_id) for i in items],
  115. )
  116. # Seed busy_printers with printers that already have an item in 'printing'
  117. # status. _is_printer_idle() alone is not sufficient as a dispatch gate —
  118. # on H2D / P1 series the MQTT state transition from IDLE to RUNNING can
  119. # lag several seconds behind the print command, so the next check_queue
  120. # tick still sees IDLE and would double-dispatch onto the same printer.
  121. # Without this guard, two pending items targeting the same printer
  122. # (e.g. a batch with quantity>1) both end up in 'printing' status —
  123. # surfaced via the "BUG: Multiple queue items" warning in on_print_complete.
  124. busy_result = await db.execute(
  125. select(PrintQueueItem.printer_id)
  126. .where(PrintQueueItem.status == "printing")
  127. .where(PrintQueueItem.printer_id.is_not(None))
  128. )
  129. busy_printers: set[int] = {pid for (pid,) in busy_result.all() if pid is not None}
  130. # Log skip reasons once per queue check (not per item)
  131. skip_reasons: dict[str, int] = {}
  132. for item in items:
  133. # Check scheduled time first (scheduled_time is stored in UTC from ISO string)
  134. if item.scheduled_time:
  135. sched = item.scheduled_time
  136. if sched.tzinfo is None:
  137. sched = sched.replace(tzinfo=timezone.utc)
  138. if sched > datetime.now(timezone.utc):
  139. skip_reasons["scheduled_future"] = skip_reasons.get("scheduled_future", 0) + 1
  140. continue
  141. # Skip items that require manual start
  142. if item.manual_start:
  143. skip_reasons["manual_start"] = skip_reasons.get("manual_start", 0) + 1
  144. continue
  145. if item.printer_id:
  146. # Specific printer assignment (existing behavior)
  147. if item.printer_id in busy_printers:
  148. continue
  149. # Check if printer is idle
  150. printer_idle = self._is_printer_idle(item.printer_id, require_plate_clear)
  151. printer_connected = printer_manager.is_connected(item.printer_id)
  152. # If printer not connected, try to power on via smart plug
  153. if not printer_connected:
  154. plugs = await self._get_smart_plugs(db, item.printer_id)
  155. auto_on_plugs = [p for p in plugs if p.auto_on and p.enabled]
  156. if auto_on_plugs:
  157. logger.info("Printer %s offline, attempting to power on via smart plug(s)", item.printer_id)
  158. # Power on using the first auto_on plug (the printer power plug)
  159. powered_on = await self._power_on_and_wait(auto_on_plugs[0], item.printer_id, db)
  160. if powered_on:
  161. # Also turn on any remaining auto_on plugs (e.g., filter)
  162. for extra_plug in auto_on_plugs[1:]:
  163. try:
  164. service = await smart_plug_manager.get_service_for_plug(extra_plug, db)
  165. await service.turn_on(extra_plug)
  166. logger.info(
  167. "Also powered on plug '%s' for printer %s", extra_plug.name, item.printer_id
  168. )
  169. except Exception as e:
  170. logger.warning("Failed to power on extra plug '%s': %s", extra_plug.name, e)
  171. printer_connected = True
  172. printer_idle = self._is_printer_idle(item.printer_id, require_plate_clear)
  173. else:
  174. logger.warning("Could not power on printer %s via smart plug", item.printer_id)
  175. busy_printers.add(item.printer_id)
  176. continue
  177. else:
  178. # No plug or auto_on disabled
  179. busy_printers.add(item.printer_id)
  180. continue
  181. # Check if printer is idle (busy with another print)
  182. if not printer_idle:
  183. # If printer is drying (not truly busy), handle based on queue_drying_block
  184. if self._drying_in_progress.get(item.printer_id):
  185. block_for_drying = await self._get_bool_setting(db, "queue_drying_block")
  186. if block_for_drying:
  187. # Drying blocks queue — skip this printer
  188. busy_printers.add(item.printer_id)
  189. continue
  190. else:
  191. # Print takes priority — stop drying
  192. await self._stop_drying(item.printer_id)
  193. # Re-check idle after stopping drying
  194. printer_idle = self._is_printer_idle(item.printer_id, require_plate_clear)
  195. if not printer_idle:
  196. busy_printers.add(item.printer_id)
  197. continue
  198. else:
  199. busy_printers.add(item.printer_id)
  200. continue
  201. # Check condition (previous print success)
  202. if item.require_previous_success:
  203. if not await self._check_previous_success(db, item):
  204. item.status = "skipped"
  205. item.error_message = "Previous print failed or was aborted"
  206. item.completed_at = datetime.now(timezone.utc)
  207. await db.commit()
  208. logger.info("Skipped queue item %s - previous print failed", item.id)
  209. # Send notification
  210. job_name = await self._get_job_name(db, item)
  211. printer = await self._get_printer(db, item.printer_id)
  212. await notification_service.on_queue_job_skipped(
  213. job_name=job_name,
  214. printer_id=item.printer_id,
  215. printer_name=printer.name if printer else "Unknown",
  216. reason="Previous print failed or was aborted",
  217. db=db,
  218. )
  219. continue
  220. # Compute AMS mapping if not already set
  221. if not item.ams_mapping:
  222. computed_mapping = await self._compute_ams_mapping_for_printer(db, item.printer_id, item)
  223. if computed_mapping:
  224. item.ams_mapping = json.dumps(computed_mapping)
  225. logger.info(
  226. f"Queue item {item.id}: Computed AMS mapping for printer {item.printer_id}: {computed_mapping}"
  227. )
  228. await db.commit()
  229. # Start the print
  230. await self._start_print(db, item)
  231. busy_printers.add(item.printer_id)
  232. # SJF starvation guard: mark items that were jumped
  233. if sjf_enabled and item.print_time_seconds is not None:
  234. for other in items:
  235. if (
  236. other.id != item.id
  237. and other.status == "pending"
  238. and other.printer_id == item.printer_id
  239. and not other.been_jumped
  240. and other.position < item.position
  241. and (
  242. other.print_time_seconds is None
  243. or other.print_time_seconds > item.print_time_seconds
  244. )
  245. ):
  246. other.been_jumped = True
  247. await db.commit()
  248. elif item.target_model:
  249. # Model-based assignment - find any idle printer of matching model
  250. # Parse required filament types if present
  251. required_types = None
  252. if item.required_filament_types:
  253. try:
  254. required_types = json.loads(item.required_filament_types)
  255. except json.JSONDecodeError:
  256. pass # Ignore malformed filament types; treat as no constraint
  257. # Parse filament overrides if present
  258. filament_overrides = None
  259. if item.filament_overrides:
  260. try:
  261. filament_overrides = json.loads(item.filament_overrides)
  262. except json.JSONDecodeError:
  263. pass
  264. # If overrides exist, use override types for validation instead
  265. effective_types = required_types
  266. if filament_overrides:
  267. override_types = sorted({o["type"] for o in filament_overrides if "type" in o})
  268. if override_types:
  269. # Merge: keep original types for non-overridden slots, add override types
  270. effective_types = sorted(set(required_types or []) | set(override_types))
  271. printer_id, waiting_reason = await self._find_idle_printer_for_model(
  272. db,
  273. item.target_model,
  274. busy_printers,
  275. effective_types,
  276. item.target_location,
  277. filament_overrides=filament_overrides,
  278. require_plate_clear=require_plate_clear,
  279. )
  280. # Update waiting_reason if changed and send notification when first waiting
  281. if item.waiting_reason != waiting_reason:
  282. was_waiting = item.waiting_reason is not None
  283. item.waiting_reason = waiting_reason
  284. await db.commit()
  285. # Send waiting notification only when transitioning to waiting state
  286. # and the reason requires user action (not just "all printers busy")
  287. if waiting_reason and not was_waiting and not self._is_busy_only(waiting_reason):
  288. job_name = await self._get_job_name(db, item)
  289. await notification_service.on_queue_job_waiting(
  290. job_name=job_name,
  291. target_model=item.target_model,
  292. waiting_reason=waiting_reason,
  293. db=db,
  294. )
  295. if printer_id:
  296. # Check condition (previous print success) before assigning
  297. if item.require_previous_success:
  298. if not await self._check_previous_success(db, item):
  299. item.status = "skipped"
  300. item.error_message = "Previous print failed or was aborted"
  301. item.completed_at = datetime.now(timezone.utc)
  302. await db.commit()
  303. logger.info("Skipped queue item %s - previous print failed", item.id)
  304. # Send notification
  305. job_name = await self._get_job_name(db, item)
  306. printer = await self._get_printer(db, printer_id)
  307. await notification_service.on_queue_job_skipped(
  308. job_name=job_name,
  309. printer_id=printer_id,
  310. printer_name=printer.name if printer else "Unknown",
  311. reason="Previous print failed or was aborted",
  312. db=db,
  313. )
  314. continue
  315. # Assign printer and start - clear waiting reason
  316. item.printer_id = printer_id
  317. item.waiting_reason = None
  318. logger.info("Model-based assignment: queue item %s assigned to printer %s", item.id, printer_id)
  319. # Send assignment notification
  320. job_name = await self._get_job_name(db, item)
  321. printer = await self._get_printer(db, printer_id)
  322. await notification_service.on_queue_job_assigned(
  323. job_name=job_name,
  324. printer_id=printer_id,
  325. printer_name=printer.name if printer else "Unknown",
  326. target_model=item.target_model,
  327. db=db,
  328. )
  329. # Compute AMS mapping for the assigned printer if not already set
  330. # This is critical for model-based jobs where mapping wasn't computed upfront
  331. if not item.ams_mapping:
  332. computed_mapping = await self._compute_ams_mapping_for_printer(db, printer_id, item)
  333. if computed_mapping:
  334. item.ams_mapping = json.dumps(computed_mapping)
  335. logger.info(
  336. f"Queue item {item.id}: Computed AMS mapping for printer {printer_id}: {computed_mapping}"
  337. )
  338. await db.commit()
  339. await self._start_print(db, item)
  340. busy_printers.add(printer_id)
  341. # SJF starvation guard: mark model-based items that were jumped
  342. if sjf_enabled and item.print_time_seconds is not None:
  343. for other in items:
  344. if (
  345. other.id != item.id
  346. and other.status == "pending"
  347. and other.printer_id is None
  348. and other.target_model
  349. and other.target_model.upper() == item.target_model.upper()
  350. and not other.been_jumped
  351. and other.position < item.position
  352. and (
  353. other.print_time_seconds is None
  354. or other.print_time_seconds > item.print_time_seconds
  355. )
  356. ):
  357. other.been_jumped = True
  358. await db.commit()
  359. # Log summary of skip reasons (helps diagnose why queue items aren't starting)
  360. if skip_reasons:
  361. logger.info("Queue skip summary: %s", skip_reasons)
  362. if busy_printers:
  363. # Log why each printer was busy (first time it was checked)
  364. for pid in busy_printers:
  365. state = printer_manager.get_status(pid)
  366. connected = printer_manager.is_connected(pid)
  367. awaiting = printer_manager.is_awaiting_plate_clear(pid)
  368. state_name = state.state if state else "NO_STATUS"
  369. logger.info(
  370. "Queue: printer %d not available — connected=%s, state=%s, awaiting_plate_clear=%s",
  371. pid,
  372. connected,
  373. state_name,
  374. awaiting,
  375. )
  376. # Auto-drying: start drying on idle printers that have no pending queue items
  377. await self._check_auto_drying(db, items, busy_printers, require_plate_clear=require_plate_clear)
  378. async def _find_idle_printer_for_model(
  379. self,
  380. db: AsyncSession,
  381. model: str,
  382. exclude_ids: set[int],
  383. required_filament_types: list[str] | None = None,
  384. target_location: str | None = None,
  385. filament_overrides: list[dict] | None = None,
  386. require_plate_clear: bool = True,
  387. ) -> tuple[int | None, str | None]:
  388. """Find an idle, connected printer matching the model with compatible filaments.
  389. Args:
  390. db: Database session
  391. model: Printer model to match (e.g., "X1C", "P1S")
  392. exclude_ids: Printer IDs to exclude (already busy)
  393. required_filament_types: Optional list of filament types needed (e.g., ["PLA", "PETG"])
  394. If provided, only printers with all required types loaded will match.
  395. target_location: Optional location filter. If provided, only printers in this location are considered.
  396. filament_overrides: Optional list of override dicts. Each entry may include
  397. ``force_color_match: true`` to require an exact type+color match
  398. on the printer for that slot. Without the flag the existing
  399. colour-preference logic applies.
  400. Returns:
  401. Tuple of (printer_id, waiting_reason):
  402. - (printer_id, None) if a matching printer was found
  403. - (None, reason) if no printer is available, with explanation
  404. """
  405. # Normalize model name and use case-insensitive matching
  406. normalized_model = normalize_printer_model(model) or model
  407. query = (
  408. select(Printer)
  409. .where(func.lower(Printer.model) == normalized_model.lower())
  410. .where(Printer.is_active == True) # noqa: E712
  411. )
  412. # Add location filter if specified
  413. if target_location:
  414. query = query.where(Printer.location == target_location)
  415. result = await db.execute(query)
  416. printers = list(result.scalars().all())
  417. location_suffix = f" in {target_location}" if target_location else ""
  418. if not printers:
  419. return None, f"No active {normalized_model} printers{location_suffix} configured"
  420. # Separate force-matched overrides from preference-only overrides
  421. force_overrides = [o for o in (filament_overrides or []) if o.get("force_color_match")]
  422. pref_overrides = [o for o in (filament_overrides or []) if not o.get("force_color_match")]
  423. # Track reasons for skipping printers
  424. printers_busy = []
  425. printers_offline = []
  426. printers_missing_filament: list[tuple[str, list[str]]] = []
  427. candidates: list[tuple[int, int]] = [] # (printer_id, color_match_count)
  428. for printer in printers:
  429. if printer.id in exclude_ids:
  430. # Printer is already claimed by another job in this scheduling run.
  431. # For force-color jobs, still check if the color would match — if not,
  432. # report it as a color mismatch rather than plain "Busy" so the user
  433. # knows the job needs a filament change, not just to wait for availability.
  434. if force_overrides and not pref_overrides:
  435. missing_colors = self._get_missing_force_color_slots(printer.id, force_overrides)
  436. if missing_colors:
  437. printers_missing_filament.append((printer.name, missing_colors))
  438. continue
  439. printers_busy.append(printer.name)
  440. continue
  441. is_connected = printer_manager.is_connected(printer.id)
  442. is_idle = self._is_printer_idle(printer.id, require_plate_clear) if is_connected else False
  443. if not is_connected:
  444. printers_offline.append(printer.name)
  445. continue
  446. if not is_idle:
  447. # Printer is currently printing. For force-color jobs, check whether the
  448. # loaded color would satisfy the requirement — if not, surface it as a
  449. # color-mismatch reason rather than plain "Busy" so the user understands
  450. # that the job is waiting for a filament change, not just printer availability.
  451. if force_overrides and not pref_overrides:
  452. missing_colors = self._get_missing_force_color_slots(printer.id, force_overrides)
  453. if missing_colors:
  454. printers_missing_filament.append((printer.name, missing_colors))
  455. logger.debug(
  456. "Printer %s (%s) is busy but also has wrong force-color: %s",
  457. printer.id,
  458. printer.name,
  459. missing_colors,
  460. )
  461. continue
  462. printers_busy.append(printer.name)
  463. continue
  464. # Validate filament compatibility if required types are specified
  465. if required_filament_types:
  466. missing = self._get_missing_filament_types(printer.id, required_filament_types)
  467. if missing:
  468. # When force_overrides are present, enrich missing entries with color info
  469. # so the "Waiting on" message includes "TYPE (color)" instead of just "TYPE"
  470. if force_overrides:
  471. force_color_map = {
  472. (o.get("type") or "").upper(): o.get("color_name") or o.get("color", "?")
  473. for o in force_overrides
  474. }
  475. missing_enriched = [
  476. f"{t} ({force_color_map[t_upper]})" if (t_upper := t.upper()) in force_color_map else t
  477. for t in missing
  478. ]
  479. printers_missing_filament.append((printer.name, missing_enriched))
  480. else:
  481. printers_missing_filament.append((printer.name, missing))
  482. logger.debug("Skipping printer %s (%s) - missing filaments: %s", printer.id, printer.name, missing)
  483. continue
  484. # Force color match: ALL flagged slots must have an exact type+color match
  485. if force_overrides:
  486. missing_colors = self._get_missing_force_color_slots(printer.id, force_overrides)
  487. if missing_colors:
  488. printers_missing_filament.append((printer.name, missing_colors))
  489. logger.debug(
  490. "Skipping printer %s (%s) - missing force-matched colors: %s",
  491. printer.id,
  492. printer.name,
  493. missing_colors,
  494. )
  495. continue
  496. # If preference-only overrides exist, rank by color matches (existing behaviour)
  497. if pref_overrides:
  498. color_matches = self._count_override_color_matches(printer.id, pref_overrides)
  499. if color_matches > 0:
  500. candidates.append((printer.id, color_matches))
  501. else:
  502. override_colors = [f"{o.get('type', '?')} ({o.get('color', '?')})" for o in pref_overrides]
  503. printers_missing_filament.append((printer.name, override_colors))
  504. logger.debug("Skipping printer %s (%s) - no matching override colors", printer.id, printer.name)
  505. continue
  506. elif force_overrides:
  507. # Passed all force checks — immediately eligible (no preference ordering needed)
  508. return printer.id, None
  509. else:
  510. # No overrides at all - take first available (existing behavior)
  511. return printer.id, None
  512. # If we have candidates from preference override matching, pick the one with most color matches
  513. if candidates:
  514. candidates.sort(key=lambda c: c[1], reverse=True)
  515. return candidates[0][0], None
  516. # Build waiting reason from what we found
  517. reasons = []
  518. if printers_missing_filament:
  519. # Filament/color mismatch is most actionable - show first
  520. if force_overrides and not pref_overrides:
  521. # All mismatches are force-color failures — use descriptive message only;
  522. # but only if there are no busy printers that DO have the matching color.
  523. # If a printer has the right color but is busy, surface "Busy" instead so
  524. # the user knows the job will start automatically once that printer is free.
  525. if not printers_busy:
  526. all_missing = sorted({c for _, cols in printers_missing_filament for c in cols})
  527. return None, f"No matching material/color. Waiting on {', '.join(all_missing)}"
  528. # else: fall through — printers_busy will be appended below
  529. else:
  530. names_and_missing = [
  531. f"{name} (needs {', '.join(missing)})" for name, missing in printers_missing_filament
  532. ]
  533. reasons.append(f"Waiting for filament: {'; '.join(names_and_missing)}")
  534. if printers_busy:
  535. reasons.append(f"Busy: {', '.join(printers_busy)}")
  536. if printers_offline:
  537. reasons.append(f"Offline: {', '.join(printers_offline)}")
  538. return None, " | ".join(reasons) if reasons else f"No available {model} printers{location_suffix}"
  539. @staticmethod
  540. def _is_busy_only(waiting_reason: str) -> bool:
  541. """Check if the waiting reason only contains 'Busy' entries.
  542. When all matching printers are simply busy printing, the queued job
  543. will start automatically once a printer finishes — no user action
  544. is required, so we skip the notification.
  545. """
  546. parts = [p.strip() for p in waiting_reason.split(" | ")]
  547. return all(p.startswith("Busy:") for p in parts)
  548. def _get_missing_force_color_slots(self, printer_id: int, force_overrides: list[dict]) -> list[str]:
  549. """Return descriptive strings for force_color_match slots not satisfied by the printer.
  550. Each entry in ``force_overrides`` must have ``type`` and ``color`` fields and is expected
  551. to carry ``force_color_match: True``. The printer must have **every** such slot loaded
  552. with an exact type+color match.
  553. Returns:
  554. List of ``"TYPE (color)"`` strings for unmatched slots (empty list means all match).
  555. """
  556. status = printer_manager.get_status(printer_id)
  557. if not status:
  558. return [f"{o.get('type', '?')} ({o.get('color_name') or o.get('color', '?')})" for o in force_overrides]
  559. # Build set of loaded type+colour pairs from AMS and external spool
  560. loaded: set[tuple[str, str]] = set()
  561. for ams_unit in status.raw_data.get("ams", []):
  562. for tray in ams_unit.get("tray", []):
  563. tray_type = tray.get("tray_type")
  564. tray_color = tray.get("tray_color", "")
  565. if tray_type:
  566. color_norm = tray_color.replace("#", "").lower()[:6]
  567. loaded.add((_canonical_filament_type(tray_type), color_norm))
  568. for vt in status.raw_data.get("vt_tray") or []:
  569. vt_type = vt.get("tray_type")
  570. if vt_type:
  571. color_norm = (vt.get("tray_color", "") or "").replace("#", "").lower()[:6]
  572. loaded.add((_canonical_filament_type(vt_type), color_norm))
  573. missing = []
  574. for o in force_overrides:
  575. o_type = _canonical_filament_type(o.get("type") or "")
  576. o_color = (o.get("color") or "").replace("#", "").lower()[:6]
  577. if (o_type, o_color) not in loaded:
  578. color_label = o.get("color_name") or o.get("color", "?")
  579. missing.append(f"{o_type} ({color_label})")
  580. return missing
  581. def _get_missing_filament_types(self, printer_id: int, required_types: list[str]) -> list[str]:
  582. """Get the list of required filament types that are not loaded on the printer.
  583. Args:
  584. printer_id: The printer ID
  585. required_types: List of filament types needed (e.g., ["PLA", "PETG"])
  586. Returns:
  587. List of missing filament types (empty if all are loaded)
  588. """
  589. status = printer_manager.get_status(printer_id)
  590. if not status:
  591. return required_types # Can't determine, assume all missing
  592. # Collect all filament types loaded on this printer (AMS units + external spool)
  593. # Use canonical types so equivalence groups (e.g. PA-CF/PA12-CF/PAHT-CF) match.
  594. loaded_types: set[str] = set()
  595. # Check AMS units (stored in raw_data["ams"])
  596. ams_data = status.raw_data.get("ams", [])
  597. if ams_data:
  598. for ams_unit in ams_data:
  599. for tray in ams_unit.get("tray", []):
  600. tray_type = tray.get("tray_type")
  601. if tray_type:
  602. loaded_types.add(_canonical_filament_type(tray_type))
  603. # Check external spool(s) (virtual tray, stored in raw_data["vt_tray"] as list)
  604. for vt in status.raw_data.get("vt_tray") or []:
  605. vt_type = vt.get("tray_type")
  606. if vt_type:
  607. loaded_types.add(_canonical_filament_type(vt_type))
  608. # Find which required types are missing (using canonical type for equivalence)
  609. missing = []
  610. for req_type in required_types:
  611. if _canonical_filament_type(req_type) not in loaded_types:
  612. missing.append(req_type)
  613. return missing
  614. def _count_override_color_matches(self, printer_id: int, overrides: list[dict]) -> int:
  615. """Count how many filament overrides have an exact color match on the printer.
  616. Used to prefer printers that already have the desired override colors loaded.
  617. """
  618. status = printer_manager.get_status(printer_id)
  619. if not status:
  620. return 0
  621. # Collect loaded filaments' type+color pairs
  622. loaded: set[tuple[str, str]] = set()
  623. for ams_unit in status.raw_data.get("ams", []):
  624. for tray in ams_unit.get("tray", []):
  625. tray_type = tray.get("tray_type")
  626. tray_color = tray.get("tray_color", "")
  627. if tray_type:
  628. color_norm = tray_color.replace("#", "").lower()[:6]
  629. loaded.add((tray_type.upper(), color_norm))
  630. for vt in status.raw_data.get("vt_tray") or []:
  631. vt_type = vt.get("tray_type")
  632. if vt_type:
  633. color_norm = (vt.get("tray_color", "") or "").replace("#", "").lower()[:6]
  634. loaded.add((vt_type.upper(), color_norm))
  635. matches = 0
  636. for o in overrides:
  637. o_type = (o.get("type") or "").upper()
  638. o_color = (o.get("color") or "").replace("#", "").lower()[:6]
  639. if (o_type, o_color) in loaded:
  640. matches += 1
  641. return matches
  642. async def _compute_ams_mapping_for_printer(
  643. self, db: AsyncSession, printer_id: int, item: PrintQueueItem
  644. ) -> list[int] | None:
  645. """Compute AMS mapping for a printer based on filament requirements.
  646. Called when a queue item has no ams_mapping set — either for model-based
  647. items after printer assignment, or printer-specific items (e.g. from VP).
  648. Args:
  649. db: Database session
  650. printer_id: The assigned printer ID
  651. item: The queue item (contains archive_id or library_file_id)
  652. Returns:
  653. AMS mapping array or None if no mapping needed/possible
  654. """
  655. # Get printer status
  656. status = printer_manager.get_status(printer_id)
  657. if not status:
  658. logger.warning("Cannot compute AMS mapping: printer %s status unavailable", printer_id)
  659. return None
  660. # Get filament requirements from source file
  661. filament_reqs = await self._get_filament_requirements(db, item)
  662. if not filament_reqs:
  663. logger.debug("No filament requirements found for queue item %s", item.id)
  664. return None
  665. # Apply filament overrides if present
  666. if item.filament_overrides:
  667. try:
  668. overrides = json.loads(item.filament_overrides)
  669. override_map = {o["slot_id"]: o for o in overrides}
  670. for req in filament_reqs:
  671. if req["slot_id"] in override_map:
  672. override = override_map[req["slot_id"]]
  673. req["type"] = override["type"]
  674. req["color"] = override["color"]
  675. # Clear tray_info_idx so matching uses type+color instead of
  676. # the original 3MF's tray_info_idx (which would match the old filament)
  677. req["tray_info_idx"] = ""
  678. logger.debug(
  679. "Queue item %s: Override slot %d -> %s %s",
  680. item.id,
  681. req["slot_id"],
  682. override["type"],
  683. override["color"],
  684. )
  685. except (json.JSONDecodeError, KeyError, TypeError) as e:
  686. logger.warning("Failed to apply filament overrides for queue item %s: %s", item.id, e)
  687. # Build loaded filaments from printer status
  688. loaded_filaments = self._build_loaded_filaments(status)
  689. if not loaded_filaments:
  690. logger.debug("No filaments loaded on printer %s", printer_id)
  691. return None
  692. # Check if user prefers lowest remaining filament when multiple spools match
  693. prefer_lowest = await self._get_bool_setting(db, "prefer_lowest_filament")
  694. # Compute mapping: match required filaments to available slots
  695. return self._match_filaments_to_slots(filament_reqs, loaded_filaments, prefer_lowest)
  696. async def _get_filament_requirements(self, db: AsyncSession, item: PrintQueueItem) -> list[dict] | None:
  697. """Extract filament requirements from the source 3MF file.
  698. Args:
  699. db: Database session
  700. item: Queue item with archive_id or library_file_id
  701. Returns:
  702. List of filament requirement dicts with slot_id, type, color, used_grams
  703. """
  704. file_path: Path | None = None
  705. if item.archive_id:
  706. result = await db.execute(select(PrintArchive).where(PrintArchive.id == item.archive_id))
  707. archive = result.scalar_one_or_none()
  708. if archive:
  709. file_path = settings.base_dir / archive.file_path
  710. elif item.library_file_id:
  711. result = await db.execute(select(LibraryFile).where(LibraryFile.id == item.library_file_id))
  712. library_file = result.scalar_one_or_none()
  713. if library_file:
  714. lib_path = Path(library_file.file_path)
  715. file_path = lib_path if lib_path.is_absolute() else settings.base_dir / library_file.file_path
  716. if not file_path or not file_path.exists():
  717. return None
  718. filaments = []
  719. try:
  720. with zipfile.ZipFile(file_path, "r") as zf:
  721. if "Metadata/slice_info.config" not in zf.namelist():
  722. return None
  723. content = zf.read("Metadata/slice_info.config").decode()
  724. root = ET.fromstring(content)
  725. # Check if plate_id is specified - use that plate's filaments
  726. plate_id = item.plate_id
  727. if plate_id:
  728. for plate_elem in root.findall("./plate"):
  729. plate_index = None
  730. for meta in plate_elem.findall("metadata"):
  731. if meta.get("key") == "index":
  732. plate_index = int(meta.get("value", "0"))
  733. break
  734. if plate_index == plate_id:
  735. for filament_elem in plate_elem.findall("./filament"):
  736. filament_id = filament_elem.get("id")
  737. filament_type = filament_elem.get("type", "")
  738. filament_color = filament_elem.get("color", "")
  739. # tray_info_idx identifies the specific spool selected when slicing
  740. tray_info_idx = filament_elem.get("tray_info_idx", "")
  741. used_g = filament_elem.get("used_g", "0")
  742. try:
  743. used_grams = float(used_g)
  744. if used_grams > 0 and filament_id:
  745. filaments.append(
  746. {
  747. "slot_id": int(filament_id),
  748. "type": filament_type,
  749. "color": filament_color,
  750. "tray_info_idx": tray_info_idx,
  751. "used_grams": round(used_grams, 1),
  752. }
  753. )
  754. except (ValueError, TypeError):
  755. pass # Skip filament entry with unparseable usage data
  756. break
  757. else:
  758. # No plate_id - extract all filaments with used_g > 0
  759. for filament_elem in root.findall("./filament"):
  760. filament_id = filament_elem.get("id")
  761. filament_type = filament_elem.get("type", "")
  762. filament_color = filament_elem.get("color", "")
  763. # tray_info_idx identifies the specific spool selected when slicing
  764. tray_info_idx = filament_elem.get("tray_info_idx", "")
  765. used_g = filament_elem.get("used_g", "0")
  766. try:
  767. used_grams = float(used_g)
  768. if used_grams > 0 and filament_id:
  769. filaments.append(
  770. {
  771. "slot_id": int(filament_id),
  772. "type": filament_type,
  773. "color": filament_color,
  774. "tray_info_idx": tray_info_idx,
  775. "used_grams": round(used_grams, 1),
  776. }
  777. )
  778. except (ValueError, TypeError):
  779. pass # Skip filament entry with unparseable usage data
  780. filaments.sort(key=lambda x: x["slot_id"])
  781. # Enrich with nozzle mapping for dual-nozzle printers
  782. nozzle_mapping = extract_nozzle_mapping_from_3mf(zf)
  783. if nozzle_mapping:
  784. for filament in filaments:
  785. filament["nozzle_id"] = nozzle_mapping.get(filament["slot_id"])
  786. except Exception as e:
  787. logger.warning("Failed to parse filament requirements: %s", e)
  788. return None
  789. return filaments if filaments else None
  790. def _build_loaded_filaments(self, status) -> list[dict]:
  791. """Build list of loaded filaments from printer status.
  792. Args:
  793. status: PrinterState from printer_manager
  794. Returns:
  795. List of loaded filament dicts with type, color, ams_id, tray_id, global_tray_id
  796. """
  797. filaments = []
  798. # Get ams_extruder_map for dual-nozzle printers (H2D, H2D Pro)
  799. ams_extruder_map = status.raw_data.get("ams_extruder_map", {})
  800. # Parse AMS units from raw_data
  801. ams_data = status.raw_data.get("ams", [])
  802. for ams_unit in ams_data:
  803. ams_id = int(ams_unit.get("id", 0))
  804. trays = ams_unit.get("tray", [])
  805. is_ht = len(trays) == 1 # AMS-HT has single tray
  806. for tray in trays:
  807. tray_type = tray.get("tray_type")
  808. if tray_type:
  809. tray_id = int(tray.get("id", 0))
  810. tray_color = tray.get("tray_color", "")
  811. # tray_info_idx identifies the specific spool (e.g., "GFA00", "P4d64437")
  812. tray_info_idx = tray.get("tray_info_idx", "")
  813. # Normalize color: remove alpha, add hash
  814. color = self._normalize_color(tray_color)
  815. # Calculate global tray ID
  816. # AMS-HT units have IDs starting at 128 with a single tray
  817. global_tray_id = ams_id if ams_id >= 128 else ams_id * 4 + tray_id
  818. filaments.append(
  819. {
  820. "type": tray_type,
  821. "color": color,
  822. "tray_info_idx": tray_info_idx,
  823. "ams_id": ams_id,
  824. "tray_id": tray_id,
  825. "is_ht": is_ht,
  826. "is_external": False,
  827. "global_tray_id": global_tray_id,
  828. "extruder_id": ams_extruder_map.get(str(ams_id)),
  829. "remain": tray.get("remain", -1),
  830. }
  831. )
  832. # Check external spool(s) (vt_tray is a list)
  833. for idx, vt in enumerate(status.raw_data.get("vt_tray") or []):
  834. if vt.get("tray_type"):
  835. color = self._normalize_color(vt.get("tray_color", ""))
  836. tray_id = int(vt.get("id", 254))
  837. filaments.append(
  838. {
  839. "type": vt["tray_type"],
  840. "color": color,
  841. "tray_info_idx": vt.get("tray_info_idx", ""),
  842. "ams_id": -1,
  843. "tray_id": idx,
  844. "is_ht": False,
  845. "is_external": True,
  846. "global_tray_id": tray_id,
  847. "extruder_id": (255 - tray_id) if ams_extruder_map else None,
  848. "remain": vt.get("remain", -1),
  849. }
  850. )
  851. return filaments
  852. def _normalize_color(self, color: str | None) -> str:
  853. """Normalize color to #RRGGBB format."""
  854. if not color:
  855. return "#808080"
  856. hex_color = color.replace("#", "")[:6]
  857. return f"#{hex_color}"
  858. def _normalize_color_for_compare(self, color: str | None) -> str:
  859. """Normalize color for comparison (lowercase, no hash)."""
  860. if not color:
  861. return ""
  862. return color.replace("#", "").lower()[:6]
  863. def _colors_are_similar(self, color1: str | None, color2: str | None, threshold: int = 40) -> bool:
  864. """Check if two colors are visually similar within a threshold."""
  865. hex1 = self._normalize_color_for_compare(color1)
  866. hex2 = self._normalize_color_for_compare(color2)
  867. if not hex1 or not hex2 or len(hex1) < 6 or len(hex2) < 6:
  868. return False
  869. try:
  870. r1 = int(hex1[0:2], 16)
  871. g1 = int(hex1[2:4], 16)
  872. b1 = int(hex1[4:6], 16)
  873. r2 = int(hex2[0:2], 16)
  874. g2 = int(hex2[2:4], 16)
  875. b2 = int(hex2[4:6], 16)
  876. return abs(r1 - r2) <= threshold and abs(g1 - g2) <= threshold and abs(b1 - b2) <= threshold
  877. except ValueError:
  878. return False
  879. def _match_filaments_to_slots(
  880. self, required: list[dict], loaded: list[dict], prefer_lowest: bool = False
  881. ) -> list[int] | None:
  882. """Match required filaments to loaded filaments and build AMS mapping.
  883. Priority: unique tray_info_idx match > exact color match > similar color match > type-only match
  884. The tray_info_idx is a filament type identifier stored in the 3MF file when the user
  885. slices (e.g., "GFA00" for generic PLA, "P4d64437" for custom presets). If the same
  886. tray_info_idx appears in only ONE available tray, we use that tray. If multiple trays
  887. have the same tray_info_idx (e.g., two spools of generic PLA), we fall back to color
  888. matching among those trays.
  889. Args:
  890. required: List of required filaments with slot_id, type, color, tray_info_idx
  891. loaded: List of loaded filaments with type, color, tray_info_idx, global_tray_id
  892. Returns:
  893. AMS mapping array (position = slot_id - 1, value = global_tray_id or -1)
  894. """
  895. if not required:
  896. return None
  897. # Track used trays to avoid duplicate assignment
  898. used_tray_ids: set[int] = set()
  899. comparisons = []
  900. for req in required:
  901. req_type = (req.get("type") or "").upper()
  902. req_color = req.get("color", "")
  903. req_tray_info_idx = req.get("tray_info_idx", "")
  904. # Find best match: unique tray_info_idx > exact color > similar color > type-only
  905. idx_match = None
  906. exact_match = None
  907. similar_match = None
  908. type_only_match = None
  909. # Get available trays (not already used)
  910. available = [f for f in loaded if f["global_tray_id"] not in used_tray_ids]
  911. # Nozzle-aware filtering: restrict to trays on the correct nozzle.
  912. # Hard filter — cross-nozzle assignment causes print failures
  913. # ("position of left hotend is abnormal"), so never fall back.
  914. req_nozzle_id = req.get("nozzle_id")
  915. if req_nozzle_id is not None:
  916. available = [f for f in available if f.get("extruder_id") == req_nozzle_id]
  917. # Sort by remaining filament (ascending) so lowest-remain spool wins .find()
  918. if prefer_lowest:
  919. available.sort(key=lambda f: f.get("remain", -1) if f.get("remain", -1) >= 0 else 101)
  920. # Check if tray_info_idx is unique among available trays
  921. if req_tray_info_idx:
  922. idx_matches = [f for f in available if f.get("tray_info_idx") == req_tray_info_idx]
  923. if len(idx_matches) == 1:
  924. # Unique tray_info_idx - use it as definitive match
  925. idx_match = idx_matches[0]
  926. logger.debug(
  927. f"Matched filament slot {req.get('slot_id')} by unique tray_info_idx={req_tray_info_idx} "
  928. f"-> tray {idx_match['global_tray_id']}"
  929. )
  930. elif len(idx_matches) > 1:
  931. # Multiple trays with same tray_info_idx - use color matching among them
  932. logger.debug(
  933. f"Non-unique tray_info_idx={req_tray_info_idx} found in {len(idx_matches)} trays, "
  934. f"using color matching among trays: {[f['global_tray_id'] for f in idx_matches]}"
  935. )
  936. if prefer_lowest:
  937. idx_matches.sort(key=lambda f: f.get("remain", -1) if f.get("remain", -1) >= 0 else 101)
  938. # Use color matching within this subset
  939. for f in idx_matches:
  940. f_color = f.get("color", "")
  941. if self._normalize_color_for_compare(f_color) == self._normalize_color_for_compare(req_color):
  942. if not exact_match:
  943. exact_match = f
  944. elif self._colors_are_similar(f_color, req_color):
  945. if not similar_match:
  946. similar_match = f
  947. elif not type_only_match:
  948. type_only_match = f
  949. # If no idx_match yet, do standard type/color matching on all available trays
  950. if not idx_match and not exact_match and not similar_match and not type_only_match:
  951. for f in available:
  952. f_type = (f.get("type") or "").upper()
  953. if _canonical_filament_type(f_type) != _canonical_filament_type(req_type):
  954. continue
  955. # Type matches - check color
  956. f_color = f.get("color", "")
  957. if self._normalize_color_for_compare(f_color) == self._normalize_color_for_compare(req_color):
  958. if not exact_match:
  959. exact_match = f
  960. elif self._colors_are_similar(f_color, req_color):
  961. if not similar_match:
  962. similar_match = f
  963. elif not type_only_match:
  964. type_only_match = f
  965. match = idx_match or exact_match or similar_match or type_only_match
  966. if match:
  967. used_tray_ids.add(match["global_tray_id"])
  968. comparisons.append({"slot_id": req.get("slot_id", 0), "global_tray_id": match["global_tray_id"]})
  969. else:
  970. comparisons.append({"slot_id": req.get("slot_id", 0), "global_tray_id": -1})
  971. # Build mapping array
  972. if not comparisons:
  973. return None
  974. max_slot_id = max(c["slot_id"] for c in comparisons)
  975. if max_slot_id <= 0:
  976. return None
  977. mapping = [-1] * max_slot_id
  978. for c in comparisons:
  979. slot_id = c["slot_id"]
  980. if slot_id and slot_id > 0:
  981. mapping[slot_id - 1] = c["global_tray_id"]
  982. return mapping
  983. def _is_printer_idle(self, printer_id: int, require_plate_clear: bool = True) -> bool:
  984. """Check if a printer is connected and idle."""
  985. if not printer_manager.is_connected(printer_id):
  986. logger.debug("Printer %d: not connected", printer_id)
  987. return False
  988. state = printer_manager.get_status(printer_id)
  989. if not state:
  990. logger.debug("Printer %d: no status available", printer_id)
  991. return False
  992. # Plate-clear gate: if the printer finished/failed a previous print and the user
  993. # hasn't acknowledged the plate was cleared, the queue must not dispatch the next
  994. # job — even if the printer currently reports IDLE. After Auto Off cycles the
  995. # printer, it boots back into IDLE with no memory of the previous finish; without
  996. # the persisted awaiting flag we'd bypass the confirmation prompt (#961).
  997. if require_plate_clear and printer_manager.is_awaiting_plate_clear(printer_id):
  998. logger.debug(
  999. "Printer %d: not idle — awaiting plate-clear acknowledgment (state=%s)",
  1000. printer_id,
  1001. state.state,
  1002. )
  1003. return False
  1004. idle = state.state in ("IDLE", "FINISH", "FAILED")
  1005. if not idle:
  1006. logger.debug("Printer %d: not idle — state=%s", printer_id, state.state)
  1007. return idle
  1008. async def _get_setting(self, db: AsyncSession, key: str) -> str | None:
  1009. """Read a setting value from the database."""
  1010. result = await db.execute(select(Settings).where(Settings.key == key))
  1011. setting = result.scalar_one_or_none()
  1012. return setting.value if setting else None
  1013. async def _get_bool_setting(self, db: AsyncSession, key: str, default: bool = False) -> bool:
  1014. """Read a boolean setting from the database."""
  1015. result = await db.execute(select(Settings).where(Settings.key == key))
  1016. setting = result.scalar_one_or_none()
  1017. if setting:
  1018. return setting.value.lower() == "true"
  1019. return default
  1020. async def _get_drying_presets(self, db: AsyncSession) -> dict[str, dict[str, int]]:
  1021. """Get drying presets (user-configured or built-in defaults)."""
  1022. result = await db.execute(select(Settings).where(Settings.key == "drying_presets"))
  1023. setting = result.scalar_one_or_none()
  1024. if setting and setting.value:
  1025. try:
  1026. presets = json.loads(setting.value)
  1027. if isinstance(presets, dict) and presets:
  1028. return presets
  1029. except json.JSONDecodeError:
  1030. pass
  1031. return self.DEFAULT_DRYING_PRESETS
  1032. def _get_conservative_drying_params(
  1033. self, trays: list[dict], module_type: str, presets: dict[str, dict[str, int]]
  1034. ) -> tuple[int, int, str] | None:
  1035. """Get the most conservative drying params for mixed filament types in an AMS unit.
  1036. Returns (temp, duration_hours, filament_type) or None if no drying-eligible filaments.
  1037. """
  1038. temp_key = module_type if module_type in ("n3f", "n3s") else "n3f"
  1039. hours_key = f"{temp_key}_hours"
  1040. min_temp = None
  1041. max_hours = None
  1042. filament_type = ""
  1043. for tray in trays:
  1044. tray_type = tray.get("tray_type", "")
  1045. if not tray_type:
  1046. continue
  1047. # Normalize filament type for preset lookup (e.g., "PLA Basic" -> "PLA")
  1048. base_type = tray_type.split()[0].upper()
  1049. preset = presets.get(base_type)
  1050. if not preset:
  1051. continue
  1052. temp = preset.get(temp_key, 55)
  1053. hours = preset.get(hours_key, 12)
  1054. # Conservative: lowest temp, longest duration
  1055. if min_temp is None or temp < min_temp:
  1056. min_temp = temp
  1057. if max_hours is None or hours > max_hours:
  1058. max_hours = hours
  1059. if not filament_type:
  1060. filament_type = base_type
  1061. if min_temp is None:
  1062. return None
  1063. return (min_temp, max_hours or 12, filament_type)
  1064. async def _check_auto_drying(
  1065. self,
  1066. db: AsyncSession,
  1067. queue_items: list[PrintQueueItem],
  1068. busy_printers: set[int],
  1069. *,
  1070. require_plate_clear: bool = True,
  1071. ):
  1072. """Start drying on idle printers based on humidity.
  1073. Two modes (can both be enabled):
  1074. - queue_drying_enabled: Dry between scheduled queue prints
  1075. - ambient_drying_enabled: Dry any idle printer when humidity is high, regardless of queue
  1076. """
  1077. queue_drying_enabled = await self._get_bool_setting(db, "queue_drying_enabled")
  1078. ambient_drying_enabled = await self._get_bool_setting(db, "ambient_drying_enabled")
  1079. if not queue_drying_enabled and not ambient_drying_enabled:
  1080. # Stop active drying on all printers if both features disabled
  1081. if self._drying_in_progress:
  1082. for pid in list(self._drying_in_progress):
  1083. logger.info("Auto-drying: printer %d — stopping, auto-drying disabled", pid)
  1084. await self._stop_drying(pid)
  1085. return
  1086. # Update drying state from printer status (handles backend restart)
  1087. self._sync_drying_state()
  1088. # Find printers with scheduled items (for queue drying mode)
  1089. printers_with_scheduled: set[int] = set()
  1090. printers_with_items: set[int] = set()
  1091. for item in queue_items:
  1092. if item.printer_id:
  1093. printers_with_items.add(item.printer_id)
  1094. if item.scheduled_time and not item.manual_start:
  1095. printers_with_scheduled.add(item.printer_id)
  1096. # If only queue mode is on and no printers have scheduled items, stop drying
  1097. if not ambient_drying_enabled and not printers_with_scheduled:
  1098. for pid in list(self._drying_in_progress):
  1099. logger.info("Auto-drying: printer %d — stopping, no scheduled prints in queue", pid)
  1100. await self._stop_drying(pid)
  1101. return
  1102. # Get humidity threshold
  1103. result = await db.execute(select(Settings).where(Settings.key == "ams_humidity_fair"))
  1104. setting = result.scalar_one_or_none()
  1105. humidity_threshold = int(setting.value) if setting else 60
  1106. # Get drying presets
  1107. presets = await self._get_drying_presets(db)
  1108. # Determine if drying should be skipped for printers with pending items
  1109. block_for_drying = await self._get_bool_setting(db, "queue_drying_block")
  1110. # Get all active printers
  1111. all_printers = await db.execute(select(Printer).where(Printer.is_active.is_(True)))
  1112. for printer in all_printers.scalars():
  1113. pid = printer.id
  1114. if pid in busy_printers:
  1115. logger.debug("Auto-drying: printer %d skipped — busy", pid)
  1116. continue
  1117. # In queue-only mode, only dry printers that have scheduled prints
  1118. if not ambient_drying_enabled and pid not in printers_with_scheduled:
  1119. if self._drying_in_progress.get(pid):
  1120. logger.info("Auto-drying: printer %d — stopping, no scheduled prints for this printer", pid)
  1121. await self._stop_drying(pid)
  1122. logger.debug("Auto-drying: printer %d skipped — no scheduled prints", pid)
  1123. continue
  1124. # When block mode is on, don't START new drying on printers with pending items.
  1125. # But allow already-drying printers through so humidity auto-stop logic still runs.
  1126. if block_for_drying and pid in printers_with_items and not self._drying_in_progress.get(pid):
  1127. logger.debug("Auto-drying: printer %d skipped — has pending items (block mode)", pid)
  1128. continue
  1129. if not printer_manager.is_connected(pid):
  1130. logger.debug("Auto-drying: printer %d skipped — not connected", pid)
  1131. continue
  1132. if not self._is_printer_idle(pid, require_plate_clear):
  1133. logger.debug("Auto-drying: printer %d skipped — not idle", pid)
  1134. continue
  1135. # Check if this printer supports drying
  1136. state = printer_manager.get_status(pid)
  1137. if not state:
  1138. logger.debug("Auto-drying: printer %d skipped — no state", pid)
  1139. continue
  1140. model = printer_manager.get_model(pid)
  1141. firmware = state.firmware_version
  1142. if not supports_drying(model, firmware):
  1143. logger.debug("Auto-drying: printer %d skipped — model %s does not support drying", pid, model)
  1144. continue
  1145. # Check each AMS unit from raw_data
  1146. ams_list = state.raw_data.get("ams", [])
  1147. logger.debug("Auto-drying: printer %d — checking %d AMS units", pid, len(ams_list))
  1148. for ams_data in ams_list:
  1149. module_type = str(ams_data.get("module_type") or "")
  1150. ams_id = int(ams_data.get("id", 0))
  1151. # Only n3f/n3s support drying
  1152. if module_type not in ("n3f", "n3s"):
  1153. logger.debug("Auto-drying: printer %d AMS %d skipped — module_type=%s", pid, ams_id, module_type)
  1154. continue
  1155. dry_time = int(ams_data.get("dry_time") or 0)
  1156. # Read humidity — prefer humidity_raw (actual %) over humidity (index 1-5)
  1157. humidity = None
  1158. h_raw = ams_data.get("humidity_raw")
  1159. if h_raw is not None:
  1160. try:
  1161. humidity = int(h_raw)
  1162. except (ValueError, TypeError):
  1163. pass
  1164. if humidity is None:
  1165. h_idx = ams_data.get("humidity")
  1166. if h_idx is not None:
  1167. try:
  1168. humidity = int(h_idx)
  1169. except (ValueError, TypeError):
  1170. pass
  1171. # Already drying — check if humidity dropped below threshold (with minimum drying time)
  1172. if dry_time > 0:
  1173. if pid not in self._drying_in_progress:
  1174. # Drying we didn't start (manual or from before restart) — track but don't stop
  1175. self._drying_in_progress[pid] = time.monotonic()
  1176. started_at = self._drying_in_progress[pid]
  1177. elapsed = time.monotonic() - started_at
  1178. if humidity is not None and humidity <= humidity_threshold and elapsed >= self._min_drying_seconds:
  1179. logger.info(
  1180. "Auto-drying: printer %d AMS %d — humidity %d%% <= threshold %d%% after %dm, stopping drying",
  1181. pid,
  1182. ams_id,
  1183. humidity,
  1184. humidity_threshold,
  1185. int(elapsed / 60),
  1186. )
  1187. printer_manager.send_drying_command(pid, ams_id, temp=0, duration=0, mode=0)
  1188. else:
  1189. logger.debug(
  1190. "Auto-drying: printer %d AMS %d — drying (%dm left, humidity %s%%, elapsed %dm/%dm min)",
  1191. pid,
  1192. ams_id,
  1193. dry_time,
  1194. humidity,
  1195. int(elapsed / 60),
  1196. self._min_drying_seconds // 60,
  1197. )
  1198. continue
  1199. # Humidity below threshold — no need to start drying
  1200. if humidity is None or humidity <= humidity_threshold:
  1201. logger.debug(
  1202. "Auto-drying: printer %d AMS %d skipped — humidity %s <= threshold %d",
  1203. pid,
  1204. ams_id,
  1205. humidity,
  1206. humidity_threshold,
  1207. )
  1208. continue
  1209. # Check cannot-dry reasons (power constraints etc.)
  1210. sf_reasons = ams_data.get("dry_sf_reason", [])
  1211. if sf_reasons:
  1212. logger.debug(
  1213. "Auto-drying: printer %d AMS %d skipped — cannot dry reasons: %s",
  1214. pid,
  1215. ams_id,
  1216. sf_reasons,
  1217. )
  1218. continue
  1219. # Get conservative drying params for mixed filaments
  1220. trays = ams_data.get("tray", [])
  1221. params = self._get_conservative_drying_params(trays, module_type, presets)
  1222. if not params:
  1223. logger.debug(
  1224. "Auto-drying: printer %d AMS %d skipped — no drying-eligible filaments in trays", pid, ams_id
  1225. )
  1226. continue
  1227. temp, duration_hours, filament_type = params
  1228. # Start drying
  1229. logger.info(
  1230. "Auto-drying: printer %d AMS %d — humidity %d%% > threshold %d%%, "
  1231. "starting %s drying at %d°C for %dh",
  1232. pid,
  1233. ams_id,
  1234. humidity,
  1235. humidity_threshold,
  1236. filament_type,
  1237. temp,
  1238. duration_hours,
  1239. )
  1240. success = printer_manager.send_drying_command(
  1241. pid, ams_id, temp, duration_hours, mode=1, filament=filament_type
  1242. )
  1243. if success:
  1244. self._drying_in_progress[pid] = time.monotonic()
  1245. def _sync_drying_state(self):
  1246. """Sync in-memory drying state with actual printer status.
  1247. Handles backend restart — if a printer is drying but we don't know about it,
  1248. update our state. If we think it's drying but it's not, clear it.
  1249. """
  1250. to_remove = []
  1251. for pid in self._drying_in_progress:
  1252. state = printer_manager.get_status(pid)
  1253. if not state:
  1254. to_remove.append(pid)
  1255. continue
  1256. # Check if any AMS unit is still drying
  1257. ams_list = state.raw_data.get("ams", [])
  1258. any_drying = any(int(a.get("dry_time") or 0) > 0 for a in ams_list)
  1259. if not any_drying:
  1260. to_remove.append(pid)
  1261. for pid in to_remove:
  1262. self._drying_in_progress.pop(pid, None)
  1263. async def _stop_drying(self, printer_id: int):
  1264. """Stop all active drying on a printer (print takes priority)."""
  1265. state = printer_manager.get_status(printer_id)
  1266. if not state:
  1267. self._drying_in_progress.pop(printer_id, None)
  1268. return
  1269. ams_list = state.raw_data.get("ams", [])
  1270. for ams_data in ams_list:
  1271. dry_time = int(ams_data.get("dry_time") or 0)
  1272. if dry_time > 0:
  1273. ams_id = int(ams_data.get("id", 0))
  1274. logger.info(
  1275. "Auto-drying: stopping drying on printer %d AMS %d — print takes priority",
  1276. printer_id,
  1277. ams_id,
  1278. )
  1279. printer_manager.send_drying_command(printer_id, ams_id, 0, 0, mode=0)
  1280. self._drying_in_progress.pop(printer_id, None)
  1281. async def _get_smart_plugs(self, db: AsyncSession, printer_id: int) -> list[SmartPlug]:
  1282. """Get all smart plugs associated with a printer."""
  1283. result = await db.execute(select(SmartPlug).where(SmartPlug.printer_id == printer_id))
  1284. return list(result.scalars().all())
  1285. async def _power_on_and_wait(self, plug: SmartPlug, printer_id: int, db: AsyncSession) -> bool:
  1286. """Turn on smart plug and wait for printer to connect.
  1287. Returns True if printer connected successfully within timeout.
  1288. """
  1289. # Get the appropriate service for the plug type (Tasmota or Home Assistant)
  1290. service = await smart_plug_manager.get_service_for_plug(plug, db)
  1291. # Check current plug state
  1292. status = await service.get_status(plug)
  1293. if not status.get("reachable"):
  1294. logger.warning("Smart plug '%s' is not reachable", plug.name)
  1295. return False
  1296. # Turn on if not already on
  1297. if status.get("state") != "ON":
  1298. success = await service.turn_on(plug)
  1299. if not success:
  1300. logger.warning("Failed to turn on smart plug '%s'", plug.name)
  1301. return False
  1302. logger.info("Powered on smart plug '%s' for printer %s", plug.name, printer_id)
  1303. # Get printer from database for connection
  1304. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1305. printer = result.scalar_one_or_none()
  1306. if not printer:
  1307. logger.error("Printer %s not found in database", printer_id)
  1308. return False
  1309. # Wait for printer to boot (give it some time before trying to connect)
  1310. logger.info("Waiting 30s for printer %s to boot...", printer_id)
  1311. await asyncio.sleep(30)
  1312. # Try to connect to the printer periodically
  1313. elapsed = 30 # Already waited 30s
  1314. while elapsed < self._power_on_wait_time:
  1315. # Try to connect
  1316. logger.info("Attempting to connect to printer %s...", printer_id)
  1317. try:
  1318. connected = await printer_manager.connect_printer(printer)
  1319. if connected:
  1320. logger.info("Printer %s connected after %ss", printer_id, elapsed)
  1321. # Give it a moment to stabilize and get status
  1322. await asyncio.sleep(5)
  1323. return True
  1324. except Exception as e:
  1325. logger.debug("Connection attempt failed: %s", e)
  1326. await asyncio.sleep(self._power_on_check_interval)
  1327. elapsed += self._power_on_check_interval
  1328. logger.debug("Waiting for printer %s to connect... (%ss)", printer_id, elapsed)
  1329. logger.warning("Printer %s did not connect within %ss after power on", printer_id, self._power_on_wait_time)
  1330. return False
  1331. async def _check_previous_success(self, db: AsyncSession, item: PrintQueueItem) -> bool:
  1332. """Check if the previous print on this printer succeeded."""
  1333. # Find the most recent completed queue item for this printer
  1334. result = await db.execute(
  1335. select(PrintQueueItem)
  1336. .where(PrintQueueItem.printer_id == item.printer_id)
  1337. .where(PrintQueueItem.id != item.id)
  1338. .where(PrintQueueItem.status.in_(["completed", "failed", "skipped", "aborted"]))
  1339. .order_by(PrintQueueItem.completed_at.desc())
  1340. .limit(1)
  1341. )
  1342. prev_item = result.scalar_one_or_none()
  1343. # If no previous item, assume success (first in queue)
  1344. if not prev_item:
  1345. return True
  1346. return prev_item.status == "completed"
  1347. async def _power_off_if_needed(self, db: AsyncSession, item: PrintQueueItem):
  1348. """Power off printer if auto_off_after is enabled (waits for cooldown)."""
  1349. if not item.auto_off_after:
  1350. return
  1351. plugs = await self._get_smart_plugs(db, item.printer_id)
  1352. plug_ids = [p.id for p in plugs if p.enabled]
  1353. if plug_ids:
  1354. logger.info("Auto-off: Waiting for printer %s to cool down before power off...", item.printer_id)
  1355. # Wait for cooldown (up to 10 minutes)
  1356. await printer_manager.wait_for_cooldown(item.printer_id, target_temp=50.0, timeout=600)
  1357. # Re-fetch plugs in a fresh session after the long cooldown wait
  1358. async with async_session() as new_db:
  1359. for plug_id in plug_ids:
  1360. try:
  1361. result = await new_db.execute(select(SmartPlug).where(SmartPlug.id == plug_id))
  1362. plug = result.scalar_one_or_none()
  1363. if plug and plug.enabled:
  1364. logger.info("Auto-off: Powering off plug '%s' for printer %s", plug.name, item.printer_id)
  1365. service = await smart_plug_manager.get_service_for_plug(plug, new_db)
  1366. await service.turn_off(plug)
  1367. except Exception as e:
  1368. logger.warning(
  1369. "Auto-off: Failed to power off plug %s for printer %s: %s", plug_id, item.printer_id, e
  1370. )
  1371. async def _get_job_name(self, db: AsyncSession, item: PrintQueueItem) -> str:
  1372. """Get a human-readable name for a queue item."""
  1373. if item.archive_id:
  1374. result = await db.execute(select(PrintArchive).where(PrintArchive.id == item.archive_id))
  1375. archive = result.scalar_one_or_none()
  1376. if archive:
  1377. return archive.filename.replace(".gcode.3mf", "").replace(".3mf", "")
  1378. if item.library_file_id:
  1379. result = await db.execute(select(LibraryFile).where(LibraryFile.id == item.library_file_id))
  1380. library_file = result.scalar_one_or_none()
  1381. if library_file:
  1382. return library_file.filename.replace(".gcode.3mf", "").replace(".3mf", "")
  1383. return f"Job #{item.id}"
  1384. async def _get_printer(self, db: AsyncSession, printer_id: int) -> Printer | None:
  1385. """Get printer by ID."""
  1386. result = await db.execute(select(Printer).where(Printer.id == printer_id))
  1387. return result.scalar_one_or_none()
  1388. async def _start_print(self, db: AsyncSession, item: PrintQueueItem):
  1389. """Upload file and start print for a queue item.
  1390. Supports two sources:
  1391. - archive_id: Print from an existing archive
  1392. - library_file_id: Print from a library file (file manager)
  1393. """
  1394. logger.info("Starting queue item %s", item.id)
  1395. # Get printer first (needed for both paths)
  1396. result = await db.execute(select(Printer).where(Printer.id == item.printer_id))
  1397. printer = result.scalar_one_or_none()
  1398. if not printer:
  1399. item.status = "failed"
  1400. item.error_message = "Printer not found"
  1401. item.completed_at = datetime.now(timezone.utc)
  1402. await db.commit()
  1403. logger.error("Queue item %s: Printer %s not found", item.id, item.printer_id)
  1404. await self._power_off_if_needed(db, item)
  1405. return
  1406. # Check printer is connected
  1407. if not printer_manager.is_connected(item.printer_id):
  1408. item.status = "failed"
  1409. item.error_message = "Printer not connected"
  1410. item.completed_at = datetime.now(timezone.utc)
  1411. await db.commit()
  1412. logger.error("Queue item %s: Printer %s not connected", item.id, item.printer_id)
  1413. await self._power_off_if_needed(db, item)
  1414. return
  1415. # Determine source: archive or library file
  1416. archive = None
  1417. library_file = None
  1418. file_path = None
  1419. filename = None
  1420. if item.archive_id:
  1421. # Print from archive
  1422. result = await db.execute(select(PrintArchive).where(PrintArchive.id == item.archive_id))
  1423. archive = result.scalar_one_or_none()
  1424. if not archive:
  1425. item.status = "failed"
  1426. item.error_message = "Archive not found"
  1427. item.completed_at = datetime.now(timezone.utc)
  1428. await db.commit()
  1429. logger.error("Queue item %s: Archive %s not found", item.id, item.archive_id)
  1430. await self._power_off_if_needed(db, item)
  1431. return
  1432. file_path = settings.base_dir / archive.file_path
  1433. filename = archive.filename
  1434. elif item.library_file_id:
  1435. # Print from library file (file manager)
  1436. result = await db.execute(select(LibraryFile).where(LibraryFile.id == item.library_file_id))
  1437. library_file = result.scalar_one_or_none()
  1438. if not library_file:
  1439. item.status = "failed"
  1440. item.error_message = "Library file not found"
  1441. item.completed_at = datetime.now(timezone.utc)
  1442. await db.commit()
  1443. logger.error("Queue item %s: Library file %s not found", item.id, item.library_file_id)
  1444. await self._power_off_if_needed(db, item)
  1445. return
  1446. # Library files store absolute paths
  1447. lib_path = Path(library_file.file_path)
  1448. file_path = lib_path if lib_path.is_absolute() else settings.base_dir / library_file.file_path
  1449. filename = library_file.filename
  1450. # Create archive from library file so usage tracking has access to the 3MF
  1451. try:
  1452. from backend.app.services.archive import ArchiveService
  1453. archive_service = ArchiveService(db)
  1454. archive = await archive_service.archive_print(
  1455. printer_id=item.printer_id,
  1456. source_file=file_path,
  1457. original_filename=filename,
  1458. created_by_id=item.created_by_id,
  1459. project_id=item.project_id,
  1460. )
  1461. if archive:
  1462. item.archive_id = archive.id
  1463. await db.flush()
  1464. logger.info(
  1465. "Queue item %s: Created archive %s from library file %s",
  1466. item.id,
  1467. archive.id,
  1468. item.library_file_id,
  1469. )
  1470. except Exception as e:
  1471. logger.warning("Queue item %s: Failed to create archive from library file: %s", item.id, e)
  1472. else:
  1473. # Neither archive nor library file specified
  1474. item.status = "failed"
  1475. item.error_message = "No source file specified"
  1476. item.completed_at = datetime.now(timezone.utc)
  1477. await db.commit()
  1478. logger.error("Queue item %s: No archive_id or library_file_id specified", item.id)
  1479. await self._power_off_if_needed(db, item)
  1480. return
  1481. # Check file exists on disk
  1482. if not file_path.exists():
  1483. item.status = "failed"
  1484. item.error_message = "Source file not found on disk"
  1485. item.completed_at = datetime.now(timezone.utc)
  1486. await db.commit()
  1487. logger.error("Queue item %s: File not found: %s", item.id, file_path)
  1488. await self._power_off_if_needed(db, item)
  1489. return
  1490. # G-code injection for auto-print systems (#422)
  1491. injected_path = None
  1492. if item.gcode_injection:
  1493. try:
  1494. snippets_raw = await self._get_setting(db, "gcode_snippets")
  1495. if snippets_raw:
  1496. snippets = json.loads(snippets_raw)
  1497. model_snippets = snippets.get(printer.model, {})
  1498. start_gc = (model_snippets.get("start_gcode") or "").strip()
  1499. end_gc = (model_snippets.get("end_gcode") or "").strip()
  1500. if start_gc or end_gc:
  1501. from backend.app.utils.threemf_tools import inject_gcode_into_3mf
  1502. injected_path = inject_gcode_into_3mf(
  1503. file_path, item.plate_id or 1, start_gc or None, end_gc or None
  1504. )
  1505. if injected_path:
  1506. file_path = injected_path
  1507. logger.info("Queue item %s: G-code injected for model %s", item.id, printer.model)
  1508. else:
  1509. logger.warning(
  1510. "Queue item %s: G-code injection returned no result, using original", item.id
  1511. )
  1512. except Exception as e:
  1513. logger.warning("Queue item %s: G-code injection failed, using original: %s", item.id, e)
  1514. # Upload file to printer via FTP
  1515. # Use a clean filename to avoid issues with double extensions like .gcode.3mf
  1516. base_name = filename
  1517. if base_name.endswith(".gcode.3mf"):
  1518. base_name = base_name[:-10] # Remove .gcode.3mf
  1519. elif base_name.endswith(".3mf"):
  1520. base_name = base_name[:-4] # Remove .3mf
  1521. remote_filename = f"{base_name}.3mf"
  1522. # Sanitize: firmware parses ftp://{filename} as a URL, spaces break it
  1523. remote_filename = remote_filename.replace(" ", "_")
  1524. # Upload to root directory (not /cache/) - the start_print command references
  1525. # files by name only (ftp://{filename}), so they must be in the root
  1526. remote_path = f"/{remote_filename}"
  1527. # Get FTP retry settings
  1528. ftp_retry_enabled, ftp_retry_count, ftp_retry_delay, ftp_timeout = await get_ftp_retry_settings()
  1529. logger.info(
  1530. f"Queue item {item.id}: FTP upload starting - printer={printer.name} ({printer.model}), "
  1531. f"ip={printer.ip_address}, file={remote_filename}, local_path={file_path}, "
  1532. f"retry_enabled={ftp_retry_enabled}, retry_count={ftp_retry_count}, timeout={ftp_timeout}"
  1533. )
  1534. # Delete existing file if present (avoids 553 error on overwrite)
  1535. try:
  1536. logger.debug("Queue item %s: Deleting existing file %s if present...", item.id, remote_path)
  1537. delete_result = await delete_file_async(
  1538. printer.ip_address,
  1539. printer.access_code,
  1540. remote_path,
  1541. socket_timeout=ftp_timeout,
  1542. printer_model=printer.model,
  1543. )
  1544. logger.debug("Queue item %s: Delete result: %s", item.id, delete_result)
  1545. except Exception as e:
  1546. logger.debug("Queue item %s: Delete failed (may not exist): %s", item.id, e)
  1547. try:
  1548. if ftp_retry_enabled:
  1549. uploaded = await with_ftp_retry(
  1550. upload_file_async,
  1551. printer.ip_address,
  1552. printer.access_code,
  1553. file_path,
  1554. remote_path,
  1555. socket_timeout=ftp_timeout,
  1556. printer_model=printer.model,
  1557. max_retries=ftp_retry_count,
  1558. retry_delay=ftp_retry_delay,
  1559. operation_name=f"Upload print to {printer.name}",
  1560. )
  1561. else:
  1562. uploaded = await upload_file_async(
  1563. printer.ip_address,
  1564. printer.access_code,
  1565. file_path,
  1566. remote_path,
  1567. socket_timeout=ftp_timeout,
  1568. printer_model=printer.model,
  1569. )
  1570. except Exception as e:
  1571. uploaded = False
  1572. logger.error("Queue item %s: FTP error: %s (type: %s)", item.id, e, type(e).__name__)
  1573. # Clean up injected temp file after upload attempt
  1574. if injected_path and injected_path.exists():
  1575. injected_path.unlink(missing_ok=True)
  1576. if not uploaded:
  1577. error_msg = (
  1578. "Failed to upload file to printer. Check if SD card is inserted and properly formatted (FAT32/exFAT). "
  1579. "See server logs for detailed diagnostics."
  1580. )
  1581. item.status = "failed"
  1582. item.error_message = error_msg
  1583. item.completed_at = datetime.now(timezone.utc)
  1584. await db.commit()
  1585. logger.error(
  1586. f"Queue item {item.id}: FTP upload failed - printer={printer.name}, model={printer.model}, "
  1587. f"ip={printer.ip_address}. Check logs above for storage diagnostics and specific error codes."
  1588. )
  1589. # Send failure notification
  1590. await notification_service.on_queue_job_failed(
  1591. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  1592. printer_id=printer.id,
  1593. printer_name=printer.name,
  1594. reason="Failed to upload file to printer",
  1595. db=db,
  1596. )
  1597. await self._power_off_if_needed(db, item)
  1598. return
  1599. # Parse AMS mapping if stored
  1600. ams_mapping = None
  1601. if item.ams_mapping:
  1602. try:
  1603. ams_mapping = json.loads(item.ams_mapping)
  1604. except json.JSONDecodeError:
  1605. logger.warning("Queue item %s: Invalid AMS mapping JSON, ignoring", item.id)
  1606. # Register as expected print so we don't create a duplicate archive
  1607. # Only applicable for archive-based prints
  1608. if archive:
  1609. from backend.app.main import register_expected_print
  1610. register_expected_print(
  1611. item.printer_id,
  1612. remote_filename,
  1613. archive.id,
  1614. ams_mapping=ams_mapping,
  1615. created_by_id=item.created_by_id,
  1616. )
  1617. # IMPORTANT: Set status to "printing" BEFORE sending the print command.
  1618. # This prevents phantom reprints if the backend crashes/restarts after the
  1619. # print command is sent but before the status update is committed.
  1620. # If we crash after this commit but before start_print(), the item will be
  1621. # in "printing" status without actually printing - but that's safer than
  1622. # accidentally reprinting the same file hours later.
  1623. item.status = "printing"
  1624. item.started_at = datetime.now(timezone.utc)
  1625. await db.commit()
  1626. # Clear the awaiting-plate-clear flag now that we're starting a new print
  1627. printer_manager.set_awaiting_plate_clear(item.printer_id, False)
  1628. logger.info("Queue item %s: Status set to 'printing', sending print command...", item.id)
  1629. # Capture state before dispatch so the watchdog can detect whether the
  1630. # printer actually transitioned (#967). Also capture subtask_id so the
  1631. # watchdog can recognise "command landed but state hasn't flipped yet"
  1632. # on slow H2D transitions (#1078).
  1633. pre_status = printer_manager.get_status(item.printer_id)
  1634. pre_state = getattr(pre_status, "state", None) if pre_status else None
  1635. pre_subtask_id = getattr(pre_status, "subtask_id", None) if pre_status else None
  1636. # Start the print with AMS mapping, plate_id and print options
  1637. started = printer_manager.start_print(
  1638. item.printer_id,
  1639. remote_filename,
  1640. plate_id=item.plate_id or 1,
  1641. ams_mapping=ams_mapping,
  1642. bed_levelling=item.bed_levelling,
  1643. flow_cali=item.flow_cali,
  1644. vibration_cali=item.vibration_cali,
  1645. layer_inspect=item.layer_inspect,
  1646. timelapse=item.timelapse,
  1647. use_ams=item.use_ams,
  1648. )
  1649. if started:
  1650. logger.info("Queue item %s: Print started successfully - %s", item.id, filename)
  1651. # Watchdog: if the printer never transitions out of pre_state AND
  1652. # never advances subtask_id, the MQTT publish was accepted locally but
  1653. # didn't reach the printer (half-broken session — same shape as
  1654. # #887/#936). Revert the queue item so the next dispatch can pick it
  1655. # up instead of leaving it stuck in "printing" (#967). subtask_id
  1656. # check avoids false reverts on slow H2D FINISH→PREPARE transitions
  1657. # that would otherwise cause the item to re-dispatch as a reprint
  1658. # of the just-finished job (#1078).
  1659. if pre_state:
  1660. asyncio.create_task(
  1661. self._watchdog_print_start(
  1662. item.id,
  1663. item.printer_id,
  1664. pre_state,
  1665. pre_subtask_id,
  1666. )
  1667. )
  1668. # Get estimated time for notification
  1669. estimated_time = None
  1670. if archive and archive.print_time_seconds:
  1671. estimated_time = archive.print_time_seconds
  1672. elif library_file and library_file.print_time_seconds:
  1673. estimated_time = library_file.print_time_seconds
  1674. # Send job started notification
  1675. await notification_service.on_queue_job_started(
  1676. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  1677. printer_id=printer.id,
  1678. printer_name=printer.name,
  1679. db=db,
  1680. estimated_time=estimated_time,
  1681. )
  1682. # MQTT relay - publish queue job started
  1683. try:
  1684. from backend.app.services.mqtt_relay import mqtt_relay
  1685. await mqtt_relay.on_queue_job_started(
  1686. job_id=item.id,
  1687. filename=filename,
  1688. printer_id=printer.id,
  1689. printer_name=printer.name,
  1690. printer_serial=printer.serial_number,
  1691. )
  1692. except Exception:
  1693. pass # Don't fail if MQTT fails
  1694. else:
  1695. # Clean up uploaded file from SD card to prevent phantom prints
  1696. try:
  1697. await delete_file_async(
  1698. printer.ip_address,
  1699. printer.access_code,
  1700. remote_path,
  1701. printer_model=printer.model,
  1702. )
  1703. except Exception:
  1704. pass # Best-effort — don't fail the error handler
  1705. # Print command failed - revert status
  1706. item.status = "failed"
  1707. item.error_message = "Failed to send print command to printer"
  1708. item.completed_at = datetime.now(timezone.utc)
  1709. await db.commit()
  1710. logger.error(
  1711. f"Queue item {item.id}: Failed to start print on {printer.name} ({printer.model}) - "
  1712. f"printer_manager.start_print() returned False. "
  1713. f"This may indicate: printer not connected, MQTT error, unsupported model configuration, or firmware issue. "
  1714. f"Check printer status and backend logs for details."
  1715. )
  1716. # Send failure notification
  1717. await notification_service.on_queue_job_failed(
  1718. job_name=filename.replace(".gcode.3mf", "").replace(".3mf", ""),
  1719. printer_id=printer.id,
  1720. printer_name=printer.name,
  1721. reason="Failed to send print command to printer - check printer connection and status",
  1722. db=db,
  1723. )
  1724. await self._power_off_if_needed(db, item)
  1725. @staticmethod
  1726. async def _watchdog_print_start(
  1727. queue_item_id: int,
  1728. printer_id: int,
  1729. pre_state: str,
  1730. pre_subtask_id: str | None = None,
  1731. timeout: float = 90.0,
  1732. poll_interval: float = 3.0,
  1733. ) -> None:
  1734. """Revert a queue item if the printer never acknowledges the start command.
  1735. Bambuddy optimistically marks the queue item as "printing" right after the
  1736. MQTT project_file publish succeeds locally. If the printer drops/ignores the
  1737. command (half-broken MQTT session — #887/#936), the state never transitions
  1738. and the item would otherwise stay stuck in "printing" forever (#967).
  1739. Exit paths (printer picked up the job — no revert):
  1740. - gcode_state changed from pre_state, OR
  1741. - subtask_id advanced past pre_subtask_id — the printer echoes our
  1742. per-dispatch identity back on push_status, so a subtask_id change is
  1743. a definitive "command landed" signal even while state is still FINISH.
  1744. H2D can sit at FINISH for ~50 s after accepting project_file before
  1745. transitioning to PREPARE, which used to trip the state-only watchdog
  1746. and caused the scheduler to revert + re-dispatch the item; the next
  1747. successful dispatch then looked like a reprint of the just-finished
  1748. job (#1078).
  1749. Timeout raised from 45 s → 90 s as belt-and-braces for slow transitions
  1750. that also don't emit an early subtask_id tick.
  1751. """
  1752. deadline = time.monotonic() + timeout
  1753. while time.monotonic() < deadline:
  1754. await asyncio.sleep(poll_interval)
  1755. status = printer_manager.get_status(printer_id)
  1756. if not status:
  1757. return # Printer disconnected — don't mess with the DB
  1758. if status.state != pre_state:
  1759. return # Printer picked up the job (state transition)
  1760. if pre_subtask_id is not None and status.subtask_id is not None and status.subtask_id != pre_subtask_id:
  1761. return # Printer picked up the job (subtask_id advanced)
  1762. # No transition. Revert the item so the scheduler can retry.
  1763. async with async_session() as db:
  1764. item = await db.get(PrintQueueItem, queue_item_id)
  1765. if not item or item.status != "printing":
  1766. return # Already moved on (completed/cancelled/etc.)
  1767. item.status = "pending"
  1768. item.started_at = None
  1769. await db.commit()
  1770. logger.warning(
  1771. "Queue item %s: printer %d did not respond to print command within "
  1772. "%.0fs (state still %s, subtask_id still %s) — reverted to 'pending' "
  1773. "for retry (#967)",
  1774. queue_item_id,
  1775. printer_id,
  1776. timeout,
  1777. pre_state,
  1778. pre_subtask_id,
  1779. )
  1780. # Same half-broken-session recovery as background_dispatch: force the
  1781. # MQTT client to reconnect so the next dispatch lands without a power cycle.
  1782. client = printer_manager.get_client(printer_id)
  1783. if client and hasattr(client, "force_reconnect_stale_session"):
  1784. client.force_reconnect_stale_session(
  1785. f"queue print command unacknowledged after {timeout:.0f}s (state still {pre_state})"
  1786. )
  1787. # Global scheduler instance
  1788. scheduler = PrintScheduler()