archive_gcode_hash.py 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134
  1. """The G-code fingerprint of an archive's file (#1634).
  2. ``content_hash`` is the SHA256 of the archived file's bytes. A slicer that sends
  3. the same slice again writes a new 3MF around it (zip entry times, metadata), so
  4. the two files differ in every byte but their G-code. ``gcode_hash`` is the
  5. SHA256 of the G-code alone: every ``Metadata/plate_N.gcode`` of a 3MF in plate
  6. order, each with its name and size, or the bytes of a plain ``.gcode`` file.
  7. A print sent from the slicer whose G-code matches an archive adds a run to that
  8. archive instead of archiving the file again (``main._claim_slicer_resend_archive``).
  9. A file sliced again with other settings has other G-code and stays separate.
  10. OrcaSlicer writes the slicing time into the G-code header, so with it only the
  11. same slice sent again matches, not the same model sliced again.
  12. ``NO_GCODE`` ("") marks an archive read without G-code (no file, not a sliced
  13. file, unreadable), so it is read once; NULL is not read yet. Reading unzips the
  14. file, which takes seconds for a large 3MF: callers on the event loop go through
  15. ``asyncio.to_thread`` (``refresh_gcode_hash_async``).
  16. """
  17. from __future__ import annotations
  18. import asyncio
  19. import hashlib
  20. import logging
  21. import re
  22. import zipfile
  23. from pathlib import Path
  24. from typing import Any
  25. from sqlalchemy import select, update
  26. from backend.app.core.config import settings
  27. from backend.app.models.archive import PrintArchive
  28. logger = logging.getLogger(__name__)
  29. NO_GCODE = ""
  30. _PLATE_GCODE = re.compile(r"^Metadata/plate_(\d+)\.gcode$")
  31. _CHUNK = 1 << 20
  32. def gcode_fingerprint(path: Path) -> str | None:
  33. """SHA256 of the G-code in the file at ``path``, or None when it holds none
  34. or cannot be read."""
  35. try:
  36. if zipfile.is_zipfile(path):
  37. with zipfile.ZipFile(path) as zf:
  38. plates = sorted(
  39. (int(m.group(1)), info)
  40. for info in zf.infolist()
  41. if (m := _PLATE_GCODE.match(info.filename)) and not info.is_dir()
  42. )
  43. if not plates:
  44. return None
  45. digest = hashlib.sha256()
  46. for _number, info in plates:
  47. # Name and length first: two files splitting the same bytes
  48. # over other plates are not one.
  49. digest.update(f"{info.filename}\0{info.file_size}\0".encode())
  50. with zf.open(info) as handle:
  51. for chunk in iter(lambda: handle.read(_CHUNK), b""):
  52. digest.update(chunk)
  53. return digest.hexdigest()
  54. if path.suffix.lower() == ".gcode":
  55. digest = hashlib.sha256()
  56. with path.open("rb") as handle:
  57. for chunk in iter(lambda: handle.read(_CHUNK), b""):
  58. digest.update(chunk)
  59. return digest.hexdigest()
  60. except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile, RuntimeError, ValueError) as exc:
  61. logger.debug("G-code fingerprint of %s failed: %s", path, exc)
  62. return None
  63. def read_gcode_hash(archive: Any) -> str | None:
  64. """The G-code fingerprint of the archive's file on disk: ``NO_GCODE`` for a
  65. file without readable G-code, None when there is no file to read."""
  66. if not archive.file_path:
  67. return None
  68. path = settings.base_dir / archive.file_path # SEC-PATH-OK: archive.file_path is DB-stored, internally generated
  69. if not path.is_file():
  70. return None
  71. return gcode_fingerprint(path) or NO_GCODE
  72. def refresh_gcode_hash(archive: PrintArchive) -> None:
  73. """Read the G-code fingerprint of the archive's file into it: when the
  74. archive is made, or given a file. Does not commit."""
  75. fingerprint = read_gcode_hash(archive)
  76. if fingerprint is not None:
  77. archive.gcode_hash = fingerprint
  78. async def refresh_gcode_hash_async(archive: PrintArchive) -> None:
  79. """``refresh_gcode_hash`` with the read off the event loop."""
  80. fingerprint = await asyncio.to_thread(read_gcode_hash, archive)
  81. if fingerprint is not None:
  82. archive.gcode_hash = fingerprint
  83. async def fill_gcode_hashes(session_factory, batch: int = 100) -> int:
  84. """Give every archive not yet read its G-code fingerprint, from its file.
  85. An archive whose file is missing or unreadable gets ``NO_GCODE``: it is
  86. read once. Returns how many were read.
  87. Each is written only while it is still unread: an archive given its file
  88. meanwhile (``refresh_gcode_hash``) keeps what that read."""
  89. filled = 0
  90. last_id = 0
  91. while True:
  92. async with session_factory() as db:
  93. rows = (
  94. await db.execute(
  95. select(PrintArchive.id, PrintArchive.file_path)
  96. .where(PrintArchive.gcode_hash.is_(None), PrintArchive.id > last_id)
  97. .order_by(PrintArchive.id)
  98. .limit(batch)
  99. )
  100. ).all()
  101. if not rows:
  102. return filled
  103. for row in rows:
  104. fingerprint = await asyncio.to_thread(read_gcode_hash, row)
  105. written = await db.execute(
  106. update(PrintArchive)
  107. .where(PrintArchive.id == row.id, PrintArchive.gcode_hash.is_(None))
  108. .values(gcode_hash=fingerprint if fingerprint is not None else NO_GCODE)
  109. .execution_options(synchronize_session=False)
  110. )
  111. filled += written.rowcount or 0
  112. last_id = rows[-1].id
  113. await db.commit()