Browse Source

fix(library): defer external-scan STL thumbnails + Path coerce (#1299)

  External scan hung on a 1200-subdir NAS because (1) every STL crashed
  with TypeError ('str / str') inside generate_stl_thumbnail and (2)
  thumbnail generation ran synchronously per file, so the FE timed out
  before db.commit() and nothing was persisted.

  stl_thumbnail.py now coerces inputs to Path defensively, and
  scan_external_folder defers STL thumbnail generation to a background
  asyncio task that opens its own session and processes each file
  post-commit. Subdirs appear in the sidebar immediately; thumbnails
  backfill over the next seconds/minutes.
maziggy 2 weeks ago
parent
commit
db308aa80b

File diff suppressed because it is too large
+ 0 - 0
CHANGELOG.md


+ 68 - 10
backend/app/api/routes/library.py

@@ -1,5 +1,6 @@
 """API routes for File Manager (Library) functionality."""
 
+import asyncio
 import base64
 import binascii
 import contextlib
@@ -27,7 +28,7 @@ from backend.app.core.auth import (
     require_permission_if_auth_enabled,
 )
 from backend.app.core.config import settings as app_settings
-from backend.app.core.database import get_db
+from backend.app.core.database import async_session, get_db
 from backend.app.core.permissions import Permission
 from backend.app.models.archive import PrintArchive
 from backend.app.models.library import LibraryFile, LibraryFolder
@@ -512,6 +513,55 @@ def create_image_thumbnail(file_path: Path, thumbnails_dir: Path, max_size: int
 IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".tif"}
 
 
+async def _backfill_external_stl_thumbnails(folder_ids: list[int]) -> None:
+    """Generate STL thumbnails for an external folder tree in the background.
+
+    Spawned via ``asyncio.create_task`` from ``scan_external_folder`` so the
+    HTTP request can return as soon as the filesystem walk + folder/file rows
+    are committed. Thumbnails for thousands of STL files would otherwise hold
+    the request open for many minutes (each file triggers a ``trimesh.load``
+    + matplotlib render, ~1-5s each) and the FE modal times out before the
+    final ``db.commit()`` runs — causing the original symptom in #1299 where
+    subdirectories never showed up because nothing got committed.
+
+    Opens its own session because the request session is closed by the time
+    this task starts running. Commits per-file so a worker restart mid-run
+    only loses the in-flight file. Caps STL load to a single file at a time
+    to avoid memory pressure on systems with many huge STLs.
+    """
+    if not folder_ids:
+        return
+    thumbnails_dir = get_library_thumbnails_dir()
+    async with async_session() as db:
+        result = await db.execute(
+            LibraryFile.active().where(
+                LibraryFile.folder_id.in_(folder_ids),
+                LibraryFile.file_type == "stl",
+                LibraryFile.thumbnail_path.is_(None),
+            )
+        )
+        stl_files = result.scalars().all()
+        if not stl_files:
+            return
+        logger.info(
+            "Backfilling STL thumbnails: %d file(s) across %d folder(s)",
+            len(stl_files),
+            len(folder_ids),
+        )
+        for stl_file in stl_files:
+            abs_path = to_absolute_path(stl_file.file_path)
+            if not abs_path or not abs_path.exists():
+                continue
+            try:
+                thumb_path = generate_stl_thumbnail(abs_path, thumbnails_dir)
+            except Exception as exc:  # noqa: BLE001 — never let one bad STL kill the rest
+                logger.debug("STL thumbnail backfill skipped %s: %s", abs_path, exc)
+                continue
+            if thumb_path:
+                stl_file.thumbnail_path = to_relative_path(Path(thumb_path))
+                await db.commit()
+
+
 # ============ Folder Endpoints ============
 
 
@@ -1249,15 +1299,10 @@ async def scan_external_folder(
                 except Exception as e:
                     logger.debug("Failed to extract metadata from external 3mf %s: %s", filepath, e)
 
-            # Generate thumbnail for STL files
-            if file_type == "stl" and thumbnail_path is None:
-                try:
-                    thumb_dir = get_library_thumbnails_dir()
-                    thumb_result = generate_stl_thumbnail(str(filepath), str(thumb_dir))
-                    if thumb_result:
-                        thumbnail_path = to_relative_path(Path(thumb_result))
-                except Exception as e:
-                    logger.debug("Failed to generate STL thumbnail for external %s: %s", filepath, e)
+            # STL thumbnails are deferred to a background task spawned after
+            # the scan's db.commit() — see _backfill_external_stl_thumbnails.
+            # Doing them inline would block the HTTP request for minutes on a
+            # large NAS mount (#1299).
 
             # Extract gcode thumbnail
             if file_type == "gcode" and thumbnail_path is None:
@@ -1330,6 +1375,19 @@ async def scan_external_folder(
 
     await db.commit()
 
+    # Spawn STL thumbnail backfill in the background — the scan endpoint
+    # returns immediately so the FE modal closes and subdirectories are
+    # visible right away; thumbnails fill in over the following seconds /
+    # minutes as the task processes each STL file. Survives FE refresh —
+    # the task lives in the FastAPI event loop, not the request scope.
+    # folder_cache.values() covers the root + every pre-existing subfolder
+    # + every subfolder created during this scan. all_folder_ids on its own
+    # would miss the newly-created ones (it's snapshotted before the walk).
+    asyncio.create_task(
+        _backfill_external_stl_thumbnails(list(set(folder_cache.values()))),
+        name=f"stl-backfill-folder-{folder_id}",
+    )
+
     return {"status": "success", "added": added, "removed": removed}
 
 

+ 6 - 0
backend/app/services/stl_thumbnail.py

@@ -32,6 +32,12 @@ def generate_stl_thumbnail(
     Returns:
         Path to the generated thumbnail, or None on failure
     """
+    # Callers historically pass either Path or str; coerce so the `thumbnails_dir
+    # / thumb_filename` join at the end of this function can't fail with the
+    # str-divided-by-str TypeError (see #1299).
+    stl_path = Path(stl_path)
+    thumbnails_dir = Path(thumbnails_dir)
+
     try:
         import matplotlib
         import trimesh

+ 29 - 0
backend/tests/unit/services/test_stl_thumbnail.py

@@ -180,6 +180,35 @@ endsolid cube"""
             result = generate_stl_thumbnail(stl_path, thumbnails_dir)
             assert result is None
 
+    @pytest.mark.skipif(
+        not _check_trimesh_available(),
+        reason="trimesh not installed",
+    )
+    def test_string_arguments_accepted_without_typeerror(self):
+        """Regression for #1299: external-scan path passed both args as str.
+
+        Before the fix, the function did ``thumbnails_dir / thumb_filename`` on
+        a ``str`` and raised ``TypeError: unsupported operand type(s) for /:
+        'str' and 'str'`` for every STL on an external folder scan. The fix
+        coerces both args to ``Path`` at entry. This test passes string args
+        and asserts the function either succeeds or returns ``None`` — but
+        never raises the TypeError.
+        """
+        from backend.app.services.stl_thumbnail import generate_stl_thumbnail
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            stl_path = Path(tmpdir) / "cube.stl"
+            # Minimal valid binary STL: header (80 bytes) + tri count (0)
+            stl_path.write_bytes(b"\x00" * 80 + (0).to_bytes(4, "little"))
+
+            # str args — the exact shape the external-scan call site used.
+            result = generate_stl_thumbnail(str(stl_path), str(tmpdir))
+
+            # Zero-triangle mesh either yields no thumbnail or fails the
+            # downstream render — both are acceptable; what's NOT acceptable
+            # is a TypeError leaking out, which is what the str/str bug did.
+            assert result is None or Path(result).exists()
+
 
 class TestStlThumbnailConstants:
     """Tests for STL thumbnail service constants."""

Some files were not shown because too many files changed in this diff