archive.py 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996
  1. import hashlib
  2. import json
  3. import logging
  4. import re
  5. import shutil
  6. import zipfile
  7. from datetime import datetime
  8. from pathlib import Path
  9. from xml.etree import ElementTree as ET
  10. from sqlalchemy import and_, or_, select
  11. from sqlalchemy.ext.asyncio import AsyncSession
  12. from backend.app.core.config import settings
  13. from backend.app.models.archive import PrintArchive
  14. from backend.app.models.filament import Filament
  15. from backend.app.models.printer import Printer
  16. logger = logging.getLogger(__name__)
  17. class ThreeMFParser:
  18. """Parser for Bambu Lab 3MF files."""
  19. def __init__(self, file_path: Path, plate_number: int | None = None):
  20. self.file_path = file_path
  21. self.plate_number = plate_number # Which plate was printed (1, 2, 3, etc.)
  22. self.metadata: dict = {}
  23. def parse(self) -> dict:
  24. """Extract metadata from 3MF file."""
  25. try:
  26. with zipfile.ZipFile(self.file_path, "r") as zf:
  27. self._parse_slice_info(zf)
  28. self._parse_project_settings(zf)
  29. self._parse_gcode_header(zf)
  30. self._parse_3dmodel(zf)
  31. self._extract_thumbnail(zf)
  32. # ALWAYS prefer slice_info values - they contain ONLY filaments actually used in print
  33. # project_settings contains ALL configured filaments (AMS slots), not just used ones
  34. if self.metadata.get("_slice_filament_type"):
  35. self.metadata["filament_type"] = self.metadata["_slice_filament_type"]
  36. if self.metadata.get("_slice_filament_color"):
  37. self.metadata["filament_color"] = self.metadata["_slice_filament_color"]
  38. # Clean up internal keys
  39. self.metadata.pop("_slice_filament_type", None)
  40. self.metadata.pop("_slice_filament_color", None)
  41. except Exception:
  42. pass
  43. return self.metadata
  44. def _parse_slice_info(self, zf: zipfile.ZipFile):
  45. """Parse slice_info.config for print settings and printable objects."""
  46. try:
  47. if "Metadata/slice_info.config" in zf.namelist():
  48. content = zf.read("Metadata/slice_info.config").decode()
  49. root = ET.fromstring(content)
  50. # Get the correct plate's metadata (use plate_number if specified)
  51. if self.plate_number:
  52. plate = root.find(f".//plate[@plate_idx='{self.plate_number}']")
  53. if plate is None:
  54. # Fallback to first plate if specific plate not found
  55. plate = root.find(".//plate")
  56. else:
  57. plate = root.find(".//plate")
  58. if plate is not None:
  59. # Get prediction and weight from metadata elements
  60. for meta in plate.findall("metadata"):
  61. key = meta.get("key")
  62. value = meta.get("value")
  63. if key == "prediction" and value:
  64. self.metadata["print_time_seconds"] = int(value)
  65. elif key == "weight" and value:
  66. self.metadata["filament_used_grams"] = float(value)
  67. # Extract printable objects for skip object functionality
  68. # Objects are stored as <object identify_id="123" name="Part1" skipped="false" />
  69. printable_objects = {}
  70. for obj in plate.findall("object"):
  71. identify_id = obj.get("identify_id")
  72. name = obj.get("name")
  73. skipped = obj.get("skipped", "false")
  74. # Only include objects that are not pre-skipped
  75. if identify_id and name and skipped.lower() != "true":
  76. try:
  77. printable_objects[int(identify_id)] = name
  78. except ValueError:
  79. pass
  80. if printable_objects:
  81. self.metadata["printable_objects"] = printable_objects
  82. # Get filament info from filaments ACTUALLY USED in the print
  83. # slice_info has <filament id="1" type="PLA" color="#FFFFFF" used_g="100" />
  84. # Only include filaments where used_g > 0
  85. filaments = root.findall(".//filament")
  86. if filaments:
  87. # Collect unique filament types and colors for filaments that are actually used
  88. types = []
  89. colors = []
  90. for f in filaments:
  91. # Check if this filament is actually used in the print
  92. used_g = f.get("used_g", "0")
  93. try:
  94. used_amount = float(used_g)
  95. except (ValueError, TypeError):
  96. used_amount = 0
  97. # Only include if used_g > 0 (filament is actually consumed)
  98. if used_amount > 0:
  99. ftype = f.get("type")
  100. fcolor = f.get("color")
  101. if ftype and ftype not in types:
  102. types.append(ftype)
  103. if fcolor and fcolor not in colors:
  104. colors.append(fcolor)
  105. if types:
  106. self.metadata["_slice_filament_type"] = ", ".join(types)
  107. if colors:
  108. self.metadata["_slice_filament_color"] = ",".join(colors)
  109. except Exception:
  110. pass
  111. def _parse_project_settings(self, zf: zipfile.ZipFile):
  112. """Parse project settings for print configuration."""
  113. try:
  114. if "Metadata/project_settings.config" in zf.namelist():
  115. content = zf.read("Metadata/project_settings.config").decode()
  116. try:
  117. data = json.loads(content)
  118. self._extract_filament_info(data)
  119. self._extract_print_settings(data)
  120. except json.JSONDecodeError:
  121. pass
  122. except Exception:
  123. pass
  124. def _parse_gcode_header(self, zf: zipfile.ZipFile):
  125. """Parse G-code file header for total layer count."""
  126. import re
  127. try:
  128. # Look for plate_1.gcode or similar
  129. gcode_files = [f for f in zf.namelist() if f.endswith(".gcode")]
  130. if not gcode_files:
  131. return
  132. # Read first 2KB of G-code (header contains the layer count)
  133. gcode_path = gcode_files[0]
  134. with zf.open(gcode_path) as f:
  135. header = f.read(2048).decode("utf-8", errors="ignore")
  136. # Look for "; total layer number: XX" pattern
  137. match = re.search(r";\s*total\s+layer\s+number[:\s]+(\d+)", header, re.IGNORECASE)
  138. if match:
  139. self.metadata["total_layers"] = int(match.group(1))
  140. except Exception:
  141. pass
  142. def _extract_filament_info(self, data: dict):
  143. """Extract filament info, preferring non-support filaments."""
  144. try:
  145. filament_types = data.get("filament_type", [])
  146. filament_colors = data.get("filament_colour", [])
  147. filament_is_support = data.get("filament_is_support", [])
  148. if not filament_types:
  149. return
  150. # Collect all non-support filaments
  151. non_support_types = []
  152. non_support_colors = []
  153. for i, ftype in enumerate(filament_types):
  154. is_support = filament_is_support[i] if i < len(filament_is_support) else "0"
  155. if is_support == "0":
  156. if ftype and ftype not in non_support_types:
  157. non_support_types.append(ftype)
  158. if i < len(filament_colors) and filament_colors[i]:
  159. color = filament_colors[i]
  160. if color not in non_support_colors:
  161. non_support_colors.append(color)
  162. # Fallback to first filament if all are support
  163. if not non_support_types and filament_types:
  164. non_support_types = [filament_types[0]]
  165. if not non_support_colors and filament_colors:
  166. non_support_colors = [filament_colors[0]]
  167. # Store filament type(s)
  168. if non_support_types:
  169. self.metadata["filament_type"] = ", ".join(non_support_types)
  170. # Store all colors as comma-separated (for multi-color display)
  171. if non_support_colors:
  172. self.metadata["filament_color"] = ",".join(non_support_colors)
  173. except Exception:
  174. pass
  175. def _extract_print_settings(self, data: dict):
  176. """Extract print settings from JSON config."""
  177. try:
  178. # Layer height - usually an array, get first value
  179. if "layer_height" in data:
  180. val = data["layer_height"]
  181. if isinstance(val, list) and val:
  182. self.metadata["layer_height"] = float(val[0])
  183. elif isinstance(val, (int, float, str)):
  184. self.metadata["layer_height"] = float(val)
  185. # Nozzle diameter
  186. if "nozzle_diameter" in data:
  187. val = data["nozzle_diameter"]
  188. if isinstance(val, list) and val:
  189. self.metadata["nozzle_diameter"] = float(val[0])
  190. elif isinstance(val, (int, float, str)):
  191. self.metadata["nozzle_diameter"] = float(val)
  192. # Bed temperature - first layer or regular
  193. for key in ["bed_temperature_initial_layer", "bed_temperature"]:
  194. if key in data:
  195. val = data[key]
  196. if isinstance(val, list) and val:
  197. self.metadata["bed_temperature"] = int(float(val[0]))
  198. elif isinstance(val, (int, float, str)):
  199. self.metadata["bed_temperature"] = int(float(val))
  200. break
  201. # Nozzle temperature
  202. for key in ["nozzle_temperature_initial_layer", "nozzle_temperature"]:
  203. if key in data:
  204. val = data[key]
  205. if isinstance(val, list) and val:
  206. self.metadata["nozzle_temperature"] = int(float(val[0]))
  207. elif isinstance(val, (int, float, str)):
  208. self.metadata["nozzle_temperature"] = int(float(val))
  209. break
  210. except Exception:
  211. pass
  212. def _extract_settings_from_content(self, content: str):
  213. """Extract print settings from config content."""
  214. settings_map = {
  215. "layer_height": ("layer_height", float),
  216. "nozzle_diameter": ("nozzle_diameter", float),
  217. "bed_temperature": ("bed_temperature", int),
  218. "nozzle_temperature": ("nozzle_temperature", int),
  219. }
  220. for key, (search_key, converter) in settings_map.items():
  221. if key not in self.metadata:
  222. try:
  223. # Try JSON format
  224. if f'"{search_key}"' in content:
  225. start = content.find(f'"{search_key}"')
  226. value_start = content.find(":", start) + 1
  227. value_end = content.find(",", value_start)
  228. if value_end == -1:
  229. value_end = content.find("}", value_start)
  230. value = content[value_start:value_end].strip().strip('"')
  231. self.metadata[key] = converter(value)
  232. except Exception:
  233. pass
  234. def _parse_3dmodel(self, zf: zipfile.ZipFile):
  235. """Parse 3D/3dmodel.model for MakerWorld metadata."""
  236. import re
  237. try:
  238. model_path = "3D/3dmodel.model"
  239. if model_path not in zf.namelist():
  240. return
  241. content = zf.read(model_path).decode("utf-8", errors="ignore")
  242. # Parse XML metadata elements
  243. # MakerWorld adds metadata like: <metadata name="Designer">username</metadata>
  244. metadata_pattern = r'<metadata\s+name="([^"]+)"[^>]*>([^<]*)</metadata>'
  245. matches = re.findall(metadata_pattern, content)
  246. makerworld_fields = {}
  247. for name, value in matches:
  248. makerworld_fields[name] = value.strip()
  249. # Check for direct MakerWorld URL in content
  250. url_pattern = r'https?://makerworld\.com/[^\s<>"\']+/models/(\d+)'
  251. url_match = re.search(url_pattern, content)
  252. if url_match:
  253. self.metadata["makerworld_url"] = url_match.group(0)
  254. self.metadata["makerworld_model_id"] = url_match.group(1)
  255. # Extract model ID from DSM reference in image URLs
  256. # Format: https://makerworld.bblmw.com/makerworld/model/DSM00000001275614/...
  257. # The numeric part (1275614) is the MakerWorld model ID
  258. if "makerworld_url" not in self.metadata:
  259. dsm_pattern = r"DSM0+(\d+)"
  260. dsm_match = re.search(dsm_pattern, content)
  261. if dsm_match:
  262. model_id = dsm_match.group(1)
  263. self.metadata["makerworld_url"] = f"https://makerworld.com/en/models/{model_id}"
  264. self.metadata["makerworld_model_id"] = model_id
  265. # Store designer info
  266. if "Designer" in makerworld_fields:
  267. self.metadata["designer"] = makerworld_fields["Designer"]
  268. if "Title" in makerworld_fields:
  269. self.metadata["print_name"] = makerworld_fields["Title"]
  270. except Exception:
  271. pass
  272. def _extract_thumbnail(self, zf: zipfile.ZipFile):
  273. """Extract thumbnail image from 3MF.
  274. If a plate_number was specified, try to use that plate's thumbnail first.
  275. """
  276. thumbnail_paths = []
  277. # If a specific plate was printed, try that thumbnail first
  278. if self.plate_number:
  279. thumbnail_paths.append(f"Metadata/plate_{self.plate_number}.png")
  280. # Fallback to default paths
  281. thumbnail_paths.extend(
  282. [
  283. "Metadata/plate_1.png",
  284. "Metadata/thumbnail.png",
  285. "Metadata/model_thumbnail.png",
  286. ]
  287. )
  288. for thumb_path in thumbnail_paths:
  289. if thumb_path in zf.namelist():
  290. self.metadata["_thumbnail_data"] = zf.read(thumb_path)
  291. self.metadata["_thumbnail_ext"] = ".png"
  292. break
  293. def extract_printable_objects_from_3mf(
  294. data: bytes, plate_number: int | None = None, include_positions: bool = False
  295. ) -> dict[int, str] | dict[int, dict] | tuple[dict[int, dict], list | None]:
  296. """Extract printable objects from 3MF file bytes.
  297. This is a lightweight function used during print start to get the list
  298. of objects that can be skipped.
  299. Args:
  300. data: Raw bytes of the 3MF file
  301. plate_number: Which plate was printed (1-based), or None for first plate
  302. include_positions: If True, return tuple of (objects dict, bbox_all)
  303. Returns:
  304. If include_positions=False: Dictionary mapping identify_id (int) to object name (str)
  305. If include_positions=True: Tuple of (dict mapping identify_id to {name, x, y}, bbox_all list or None)
  306. """
  307. import json
  308. from io import BytesIO
  309. printable_objects: dict = {}
  310. bbox_all: list | None = None
  311. try:
  312. with zipfile.ZipFile(BytesIO(data), "r") as zf:
  313. if "Metadata/slice_info.config" not in zf.namelist():
  314. return printable_objects
  315. content = zf.read("Metadata/slice_info.config").decode()
  316. root = ET.fromstring(content)
  317. # Find the correct plate
  318. if plate_number:
  319. plate = root.find(f".//plate[@plate_idx='{plate_number}']")
  320. if plate is None:
  321. plate = root.find(".//plate")
  322. else:
  323. plate = root.find(".//plate")
  324. if plate is None:
  325. return printable_objects
  326. # Get actual plate index from metadata (sliced files only have one plate)
  327. plate_idx = plate_number or 1
  328. for meta in plate.findall("metadata"):
  329. if meta.get("key") == "index":
  330. try:
  331. plate_idx = int(meta.get("value", "1"))
  332. except ValueError:
  333. pass
  334. break
  335. # Load position data from plate_N.json if we need positions
  336. # Build a lookup by name since bbox_objects.id != slice_info identify_id
  337. bbox_by_name: dict[str, list] = {}
  338. if include_positions:
  339. plate_json_path = f"Metadata/plate_{plate_idx}.json"
  340. if plate_json_path in zf.namelist():
  341. try:
  342. plate_json = json.loads(zf.read(plate_json_path).decode())
  343. # Get bbox_all - the bounding box of all objects (used for image bounds)
  344. bbox_all = plate_json.get("bbox_all")
  345. for bbox_obj in plate_json.get("bbox_objects", []):
  346. obj_name = bbox_obj.get("name")
  347. bbox = bbox_obj.get("bbox", [])
  348. if obj_name and len(bbox) >= 4:
  349. bbox_by_name[obj_name] = bbox
  350. except (json.JSONDecodeError, KeyError):
  351. pass
  352. # Extract objects from slice_info.config
  353. for obj in plate.findall("object"):
  354. identify_id = obj.get("identify_id")
  355. name = obj.get("name")
  356. skipped = obj.get("skipped", "false")
  357. if identify_id and name and skipped.lower() != "true":
  358. try:
  359. obj_id = int(identify_id)
  360. if include_positions:
  361. x, y = None, None
  362. # Match by name to get bbox coordinates
  363. bbox = bbox_by_name.get(name)
  364. if bbox:
  365. # Calculate center from bbox [x_min, y_min, x_max, y_max]
  366. x = (bbox[0] + bbox[2]) / 2
  367. y = (bbox[1] + bbox[3]) / 2
  368. printable_objects[obj_id] = {"name": name, "x": x, "y": y}
  369. else:
  370. printable_objects[obj_id] = name
  371. except ValueError:
  372. pass
  373. except Exception:
  374. pass
  375. if include_positions:
  376. return printable_objects, bbox_all
  377. return printable_objects
  378. class ProjectPageParser:
  379. """Parser for extracting project page data from Bambu Lab 3MF files."""
  380. def __init__(self, file_path: Path):
  381. self.file_path = file_path
  382. def parse(self, archive_id: int) -> dict:
  383. """Extract project page metadata and images from 3MF file."""
  384. import html
  385. import re
  386. result = {
  387. "title": None,
  388. "description": None,
  389. "designer": None,
  390. "designer_user_id": None,
  391. "license": None,
  392. "copyright": None,
  393. "creation_date": None,
  394. "modification_date": None,
  395. "origin": None,
  396. "profile_title": None,
  397. "profile_description": None,
  398. "profile_cover": None,
  399. "profile_user_id": None,
  400. "profile_user_name": None,
  401. "design_model_id": None,
  402. "design_profile_id": None,
  403. "design_region": None,
  404. "model_pictures": [],
  405. "profile_pictures": [],
  406. "thumbnails": [],
  407. }
  408. try:
  409. with zipfile.ZipFile(self.file_path, "r") as zf:
  410. # Parse 3D/3dmodel.model for metadata
  411. model_path = "3D/3dmodel.model"
  412. if model_path in zf.namelist():
  413. content = zf.read(model_path).decode("utf-8", errors="ignore")
  414. # Extract metadata elements using regex
  415. # Format: <metadata name="Key">Value</metadata> or <metadata name="Key" />
  416. metadata_pattern = r'<metadata\s+name="([^"]+)"[^>]*>([^<]*)</metadata>'
  417. matches = re.findall(metadata_pattern, content)
  418. field_mapping = {
  419. "Title": "title",
  420. "Description": "description",
  421. "Designer": "designer",
  422. "DesignerUserId": "designer_user_id",
  423. "License": "license",
  424. "Copyright": "copyright",
  425. "CreationDate": "creation_date",
  426. "ModificationDate": "modification_date",
  427. "Origin": "origin",
  428. "ProfileTitle": "profile_title",
  429. "ProfileDescription": "profile_description",
  430. "ProfileCover": "profile_cover",
  431. "ProfileUserId": "profile_user_id",
  432. "ProfileUserName": "profile_user_name",
  433. "DesignModelId": "design_model_id",
  434. "DesignProfileId": "design_profile_id",
  435. "DesignRegion": "design_region",
  436. }
  437. for name, value in matches:
  438. if name in field_mapping:
  439. # Decode HTML entities multiple times (content is often triple-encoded)
  440. decoded = value.strip()
  441. prev = None
  442. while prev != decoded:
  443. prev = decoded
  444. decoded = html.unescape(decoded)
  445. # Normalize non-breaking spaces to regular spaces
  446. decoded = decoded.replace("\xa0", " ")
  447. result[field_mapping[name]] = decoded if decoded else None
  448. # List images in Auxiliaries folder
  449. from urllib.parse import quote
  450. for name in zf.namelist():
  451. if name.startswith("Auxiliaries/Model Pictures/"):
  452. filename = name.split("/")[-1]
  453. if filename:
  454. result["model_pictures"].append(
  455. {
  456. "name": filename,
  457. "path": name,
  458. "url": f"/api/v1/archives/{archive_id}/project-image/{quote(name, safe='')}",
  459. }
  460. )
  461. elif name.startswith("Auxiliaries/Profile Pictures/"):
  462. filename = name.split("/")[-1]
  463. if filename:
  464. result["profile_pictures"].append(
  465. {
  466. "name": filename,
  467. "path": name,
  468. "url": f"/api/v1/archives/{archive_id}/project-image/{quote(name, safe='')}",
  469. }
  470. )
  471. elif name.startswith("Auxiliaries/.thumbnails/"):
  472. filename = name.split("/")[-1]
  473. if filename:
  474. result["thumbnails"].append(
  475. {
  476. "name": filename,
  477. "path": name,
  478. "url": f"/api/v1/archives/{archive_id}/project-image/{quote(name, safe='')}",
  479. }
  480. )
  481. except Exception as e:
  482. result["_error"] = str(e)
  483. return result
  484. def get_image(self, image_path: str) -> tuple[bytes, str] | None:
  485. """Extract an image from the 3MF file.
  486. Returns tuple of (image_data, content_type) or None if not found.
  487. """
  488. try:
  489. with zipfile.ZipFile(self.file_path, "r") as zf:
  490. if image_path in zf.namelist():
  491. data = zf.read(image_path)
  492. # Determine content type from extension
  493. ext = image_path.lower().split(".")[-1]
  494. content_types = {
  495. "png": "image/png",
  496. "jpg": "image/jpeg",
  497. "jpeg": "image/jpeg",
  498. "webp": "image/webp",
  499. "gif": "image/gif",
  500. }
  501. content_type = content_types.get(ext, "application/octet-stream")
  502. return (data, content_type)
  503. except Exception:
  504. pass
  505. return None
  506. def update_metadata(self, updates: dict) -> bool:
  507. """Update project page metadata in the 3MF file.
  508. Args:
  509. updates: Dict with fields to update (title, description, designer, etc.)
  510. Returns:
  511. True if successful, False otherwise.
  512. """
  513. import html
  514. import re
  515. import tempfile
  516. try:
  517. # Read the 3MF file
  518. with zipfile.ZipFile(self.file_path, "r") as zf_read:
  519. # Find and read the 3dmodel.model file
  520. model_path = "3D/3dmodel.model"
  521. if model_path not in zf_read.namelist():
  522. return False
  523. content = zf_read.read(model_path).decode("utf-8")
  524. # Update metadata fields
  525. field_mapping = {
  526. "title": "Title",
  527. "description": "Description",
  528. "designer": "Designer",
  529. "license": "License",
  530. "copyright": "Copyright",
  531. "profile_title": "ProfileTitle",
  532. "profile_description": "ProfileDescription",
  533. }
  534. for field, xml_name in field_mapping.items():
  535. if field in updates and updates[field] is not None:
  536. new_value = html.escape(updates[field])
  537. # Replace existing metadata or we'd need to add it
  538. pattern = rf'(<metadata\s+name="{xml_name}"[^>]*>)[^<]*(</metadata>)'
  539. replacement = rf"\g<1>{new_value}\g<2>"
  540. content = re.sub(pattern, replacement, content)
  541. # Write to a temporary file first
  542. with tempfile.NamedTemporaryFile(delete=False, suffix=".3mf") as tmp:
  543. tmp_path = Path(tmp.name)
  544. # Create new zip with updated content
  545. with zipfile.ZipFile(tmp_path, "w", zipfile.ZIP_DEFLATED) as zf_write:
  546. for item in zf_read.namelist():
  547. if item == model_path:
  548. zf_write.writestr(item, content.encode("utf-8"))
  549. else:
  550. zf_write.writestr(item, zf_read.read(item))
  551. # Replace original file with updated one
  552. shutil.move(tmp_path, self.file_path)
  553. return True
  554. except Exception:
  555. # Clean up temp file if it exists
  556. if "tmp_path" in locals() and tmp_path.exists():
  557. tmp_path.unlink()
  558. return False
  559. class ArchiveService:
  560. """Service for archiving print jobs."""
  561. def __init__(self, db: AsyncSession):
  562. self.db = db
  563. @staticmethod
  564. def compute_file_hash(file_path: Path) -> str:
  565. """Compute SHA256 hash of a file for duplicate detection."""
  566. sha256 = hashlib.sha256()
  567. with open(file_path, "rb") as f:
  568. # Read in chunks to handle large files
  569. for chunk in iter(lambda: f.read(8192), b""):
  570. sha256.update(chunk)
  571. return sha256.hexdigest()
  572. async def get_duplicate_hashes(self) -> set[str]:
  573. """Get all content hashes that appear more than once.
  574. Returns a set of hashes that have duplicates.
  575. """
  576. from sqlalchemy import func
  577. result = await self.db.execute(
  578. select(PrintArchive.content_hash)
  579. .where(PrintArchive.content_hash.isnot(None))
  580. .group_by(PrintArchive.content_hash)
  581. .having(func.count(PrintArchive.id) > 1)
  582. )
  583. return {row[0] for row in result.all()}
  584. async def find_duplicates(
  585. self,
  586. archive_id: int,
  587. content_hash: str | None = None,
  588. print_name: str | None = None,
  589. makerworld_model_id: str | None = None,
  590. ) -> list[dict]:
  591. """Find duplicate archives based on hash or name matching.
  592. Returns list of dicts with id, print_name, created_at, match_type.
  593. """
  594. duplicates = []
  595. # First, find exact matches by content hash
  596. if content_hash:
  597. result = await self.db.execute(
  598. select(PrintArchive)
  599. .where(
  600. and_(
  601. PrintArchive.content_hash == content_hash,
  602. PrintArchive.id != archive_id,
  603. )
  604. )
  605. .order_by(PrintArchive.created_at.desc())
  606. .limit(10)
  607. )
  608. for archive in result.scalars().all():
  609. duplicates.append(
  610. {
  611. "id": archive.id,
  612. "print_name": archive.print_name,
  613. "created_at": archive.created_at,
  614. "match_type": "exact",
  615. }
  616. )
  617. # Then, find similar matches by print name or MakerWorld ID
  618. if print_name or makerworld_model_id:
  619. conditions = [PrintArchive.id != archive_id]
  620. name_conditions = []
  621. if print_name:
  622. # Match if print names are similar (ignoring case)
  623. name_conditions.append(PrintArchive.print_name.ilike(print_name))
  624. if makerworld_model_id:
  625. # Match by MakerWorld model ID stored in extra_data
  626. # Use json_extract for SQLite compatibility (astext is PostgreSQL-only)
  627. from sqlalchemy import func
  628. name_conditions.append(
  629. func.json_extract(PrintArchive.extra_data, "$.makerworld_model_id") == str(makerworld_model_id)
  630. )
  631. if name_conditions:
  632. conditions.append(or_(*name_conditions))
  633. result = await self.db.execute(
  634. select(PrintArchive).where(and_(*conditions)).order_by(PrintArchive.created_at.desc()).limit(10)
  635. )
  636. for archive in result.scalars().all():
  637. # Don't add if already in duplicates (exact match)
  638. if not any(d["id"] == archive.id for d in duplicates):
  639. duplicates.append(
  640. {
  641. "id": archive.id,
  642. "print_name": archive.print_name,
  643. "created_at": archive.created_at,
  644. "match_type": "similar",
  645. }
  646. )
  647. return duplicates
  648. async def archive_print(
  649. self,
  650. printer_id: int | None,
  651. source_file: Path,
  652. print_data: dict | None = None,
  653. ) -> PrintArchive | None:
  654. """Archive a 3MF file with metadata."""
  655. # Verify printer exists if specified
  656. if printer_id is not None:
  657. result = await self.db.execute(select(Printer).where(Printer.id == printer_id))
  658. printer = result.scalar_one_or_none()
  659. if not printer:
  660. return None
  661. # Create archive directory structure
  662. timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
  663. archive_name = f"{timestamp}_{source_file.stem}"
  664. # Use "unassigned" folder for archives without a printer
  665. printer_folder = str(printer_id) if printer_id is not None else "unassigned"
  666. archive_dir = settings.archive_dir / printer_folder / archive_name
  667. archive_dir.mkdir(parents=True, exist_ok=True)
  668. # Copy 3MF file
  669. dest_file = archive_dir / source_file.name
  670. shutil.copy2(source_file, dest_file)
  671. # Compute content hash for duplicate detection
  672. content_hash = self.compute_file_hash(dest_file)
  673. # Extract plate number from filename (e.g., "plate_5" from "/data/Metadata/plate_5.gcode")
  674. plate_number = None
  675. if print_data:
  676. filename = print_data.get("filename", "")
  677. match = re.search(r"plate_(\d+)", filename)
  678. if match:
  679. plate_number = int(match.group(1))
  680. # Parse 3MF metadata
  681. parser = ThreeMFParser(dest_file, plate_number=plate_number)
  682. metadata = parser.parse()
  683. # Save thumbnail if present
  684. thumbnail_path = None
  685. if "_thumbnail_data" in metadata:
  686. thumb_file = archive_dir / f"thumbnail{metadata['_thumbnail_ext']}"
  687. thumb_file.write_bytes(metadata["_thumbnail_data"])
  688. thumbnail_path = str(thumb_file.relative_to(settings.base_dir))
  689. del metadata["_thumbnail_data"]
  690. del metadata["_thumbnail_ext"]
  691. # Merge with print data from MQTT
  692. if print_data:
  693. metadata["_print_data"] = print_data
  694. # Determine status and timestamps
  695. status = print_data.get("status", "completed") if print_data else "archived"
  696. started_at = datetime.now() if status == "printing" else None
  697. completed_at = datetime.now() if status in ("completed", "failed", "archived") else None
  698. # Calculate cost based on filament usage and type
  699. cost = None
  700. filament_grams = metadata.get("filament_used_grams")
  701. filament_type = metadata.get("filament_type")
  702. if filament_grams and filament_type:
  703. # For multi-material prints, use the first filament type for cost calculation
  704. primary_type = filament_type.split(",")[0].strip()
  705. # Look up filament cost_per_kg from database
  706. filament_result = await self.db.execute(select(Filament).where(Filament.type == primary_type).limit(1))
  707. filament = filament_result.scalar_one_or_none()
  708. if filament:
  709. cost = round((filament_grams / 1000) * filament.cost_per_kg, 2)
  710. else:
  711. # Default cost_per_kg if filament type not found
  712. default_cost_per_kg = 25.0
  713. cost = round((filament_grams / 1000) * default_cost_per_kg, 2)
  714. # Create archive record
  715. archive = PrintArchive(
  716. printer_id=printer_id,
  717. filename=source_file.name,
  718. file_path=str(dest_file.relative_to(settings.base_dir)),
  719. file_size=dest_file.stat().st_size,
  720. content_hash=content_hash,
  721. thumbnail_path=thumbnail_path,
  722. print_name=metadata.get("print_name") or source_file.stem,
  723. print_time_seconds=metadata.get("print_time_seconds"),
  724. filament_used_grams=metadata.get("filament_used_grams"),
  725. filament_type=metadata.get("filament_type"),
  726. filament_color=metadata.get("filament_color"),
  727. layer_height=metadata.get("layer_height"),
  728. total_layers=metadata.get("total_layers"),
  729. nozzle_diameter=metadata.get("nozzle_diameter"),
  730. bed_temperature=metadata.get("bed_temperature"),
  731. nozzle_temperature=metadata.get("nozzle_temperature"),
  732. makerworld_url=metadata.get("makerworld_url"),
  733. designer=metadata.get("designer"),
  734. status=status,
  735. started_at=started_at,
  736. completed_at=completed_at,
  737. cost=cost,
  738. extra_data=metadata,
  739. )
  740. self.db.add(archive)
  741. await self.db.commit()
  742. await self.db.refresh(archive)
  743. return archive
  744. async def get_archive(self, archive_id: int) -> PrintArchive | None:
  745. """Get an archive by ID."""
  746. result = await self.db.execute(select(PrintArchive).where(PrintArchive.id == archive_id))
  747. return result.scalar_one_or_none()
  748. async def update_archive_status(
  749. self,
  750. archive_id: int,
  751. status: str,
  752. completed_at: datetime | None = None,
  753. failure_reason: str | None = None,
  754. ) -> bool:
  755. """Update the status of an archive."""
  756. archive = await self.get_archive(archive_id)
  757. if not archive:
  758. return False
  759. archive.status = status
  760. if completed_at:
  761. archive.completed_at = completed_at
  762. if failure_reason:
  763. archive.failure_reason = failure_reason
  764. await self.db.commit()
  765. return True
  766. async def list_archives(
  767. self,
  768. printer_id: int | None = None,
  769. project_id: int | None = None,
  770. limit: int = 50,
  771. offset: int = 0,
  772. ) -> list[PrintArchive]:
  773. """List archives with optional filtering."""
  774. from sqlalchemy.orm import selectinload
  775. query = (
  776. select(PrintArchive).options(selectinload(PrintArchive.project)).order_by(PrintArchive.created_at.desc())
  777. )
  778. if printer_id:
  779. query = query.where(PrintArchive.printer_id == printer_id)
  780. if project_id:
  781. query = query.where(PrintArchive.project_id == project_id)
  782. query = query.limit(limit).offset(offset)
  783. result = await self.db.execute(query)
  784. return list(result.scalars().all())
  785. async def delete_archive(self, archive_id: int) -> bool:
  786. """Delete an archive and its files."""
  787. archive = await self.get_archive(archive_id)
  788. if not archive:
  789. return False
  790. # Delete files - with CRITICAL safety checks to prevent accidental deletion
  791. # of parent directories (e.g., /opt) if file_path is empty/malformed
  792. if archive.file_path and archive.file_path.strip():
  793. file_path = settings.base_dir / archive.file_path
  794. if file_path.exists():
  795. archive_dir = file_path.parent
  796. # Safety check 1: archive_dir must be inside archive_dir
  797. try:
  798. archive_dir.resolve().relative_to(settings.archive_dir.resolve())
  799. except ValueError:
  800. logger.error(
  801. f"SECURITY: Refusing to delete archive {archive_id} - "
  802. f"path {archive_dir} is outside archive directory {settings.archive_dir}"
  803. )
  804. # Still delete the database record, just not the files
  805. await self.db.delete(archive)
  806. await self.db.commit()
  807. return True
  808. # Safety check 2: archive_dir must be at least 1 level deep inside archive_dir
  809. # (should be archive_dir/uuid/file.3mf, so parent should be archive_dir/uuid)
  810. try:
  811. relative_path = archive_dir.resolve().relative_to(settings.archive_dir.resolve())
  812. if len(relative_path.parts) < 1:
  813. logger.error(
  814. f"SECURITY: Refusing to delete archive {archive_id} - "
  815. f"path {archive_dir} is not deep enough inside archive directory"
  816. )
  817. await self.db.delete(archive)
  818. await self.db.commit()
  819. return True
  820. except ValueError:
  821. pass # Already handled above
  822. shutil.rmtree(archive_dir, ignore_errors=True)
  823. else:
  824. logger.error(
  825. f"SECURITY: Refusing to delete files for archive {archive_id} - "
  826. f"file_path is empty or invalid: '{archive.file_path}'"
  827. )
  828. # Delete database record
  829. await self.db.delete(archive)
  830. await self.db.commit()
  831. return True
  832. async def attach_timelapse(
  833. self,
  834. archive_id: int,
  835. timelapse_data: bytes,
  836. filename: str = "timelapse.mp4",
  837. ) -> bool:
  838. """Attach a timelapse video to an archive."""
  839. import asyncio
  840. archive = await self.get_archive(archive_id)
  841. if not archive:
  842. return False
  843. # Get archive directory
  844. file_path = settings.base_dir / archive.file_path
  845. archive_dir = file_path.parent
  846. # Save timelapse - use thread pool to avoid blocking event loop
  847. # (timelapse files can be 100MB+, sync write blocks for seconds)
  848. timelapse_file = archive_dir / filename
  849. await asyncio.to_thread(timelapse_file.write_bytes, timelapse_data)
  850. # Update archive record
  851. archive.timelapse_path = str(timelapse_file.relative_to(settings.base_dir))
  852. await self.db.commit()
  853. return True