فهرست منبع

Backup: make and restore a backup without freezing the server
- Unpacking a backup, copying the data folders back and writing a
backup ZIP ran on the event loop; with a large archive folder the
server answered nothing for over 30 s and the loop watchdog dumped
every thread's stack during a restore that then succeeded
- These steps and the SQLite database copy now run in a worker
thread; the code is unchanged, only moved into helpers

maziggy 12 ساعت پیش
والد
کامیت
b668344822
2فایلهای تغییر یافته به همراه289 افزوده شده و 80 حذف شده
  1. 123 80
      backend/app/api/routes/settings.py
  2. 166 0
      backend/tests/integration/test_backup_restore_off_event_loop.py

+ 123 - 80
backend/app/api/routes/settings.py

@@ -1,3 +1,4 @@
+import asyncio
 import io
 import json
 import logging
@@ -913,6 +914,120 @@ async def get_homeassistant_settings(db: AsyncSession) -> dict:
     }
 
 
+# The file steps of a backup and a restore run in a worker thread. On the
+# event loop they froze the whole server for as long as they took -- no page,
+# no websocket, no printer message -- and with a large archive folder that was
+# over half a minute, long enough for the loop watchdog to dump every thread.
+
+
+def _copy_backup_dirs(dirs_to_backup: list[tuple[str, Path]], temp_path: Path) -> None:
+    """Copy each data directory that has anything in it into the backup folder."""
+    import shutil
+
+    for name, src_dir in dirs_to_backup:
+        if src_dir.exists() and any(src_dir.iterdir()):
+            try:
+                shutil.copytree(
+                    src_dir, temp_path / name
+                )  # SEC-PATH-OK: name iterates the dirs_to_backup tuple of constant strings ("archive", "virtual_printer", ...)
+            except shutil.Error as e:
+                logger.warning("Some files in %s could not be copied: %s", name, e)
+            except PermissionError as e:
+                logger.warning("Permission denied copying %s: %s", name, e)
+
+
+def _write_backup_zip(temp_path: Path, zip_file: Path) -> None:
+    """Zip every file under temp_path into zip_file, paths relative to temp_path."""
+    with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) as zf:
+        for file_path in temp_path.rglob("*"):
+            if file_path.is_file():
+                arcname = file_path.relative_to(temp_path)
+                zf.write(file_path, arcname)
+
+
+def _extract_backup_zip(content: bytes, temp_path: Path) -> None:
+    """Extract a backup ZIP into temp_path, refusing any entry that escapes it.
+
+    Raises HTTPException(400) for an unsafe path or a file that is not a ZIP.
+    """
+    from fastapi import HTTPException
+
+    try:
+        with zipfile.ZipFile(io.BytesIO(content), "r") as zf:
+            for name in zf.namelist():
+                # Reject path-traversal payloads: any entry whose resolved
+                # path escapes temp_path would allow writing arbitrary files
+                # on the host (ZipSlip / CVE-2006-5456).
+                dest = (
+                    temp_path / name
+                ).resolve()  # SEC-PATH-OK: is_relative_to containment check below before extractall
+                # is_relative_to (Python 3.9+) covers both relative
+                # path-traversal (../etc/passwd) and absolute-path overrides
+                # (/etc/passwd) — str.startswith was vulnerable to
+                # prefix-collision attacks (e.g. /tmp/abc_evil/file passing
+                # a /tmp/abc prefix check).
+                if not dest.is_relative_to(temp_path.resolve()):
+                    raise HTTPException(400, f"Invalid backup: unsafe path in ZIP: {name!r}")
+            zf.extractall(temp_path)
+    except zipfile.BadZipFile:
+        raise HTTPException(400, "Invalid backup file: not a valid ZIP")
+
+
+def _restore_data_dirs(dirs_to_restore: list[tuple[str, Path]], temp_path: Path) -> list[str]:
+    """Replace each data directory's contents with the backup's.
+
+    Clears the contents, not the directory itself, which may be a Docker
+    mount. Returns the names of the directories that could not be restored.
+    """
+    import shutil
+
+    skipped_dirs = []
+    for name, dest_dir in dirs_to_restore:
+        src_dir = (
+            temp_path / name
+        )  # SEC-PATH-OK: name iterates the dirs_to_restore tuple of constant strings ("archive", "virtual_printer", ...)
+        if src_dir.exists():
+            logger.info("Restoring %s directory...", name)
+            try:
+                if dest_dir.exists():
+                    for item in dest_dir.iterdir():
+                        try:
+                            if item.is_dir():
+                                shutil.rmtree(item)
+                            else:
+                                item.unlink()
+                        except OSError as e:
+                            logger.warning("Could not delete %s: %s", item, e)
+                else:
+                    dest_dir.mkdir(parents=True, exist_ok=True)
+                # Copy contents from backup
+                for item in src_dir.iterdir():
+                    dest_item = dest_dir / item.name
+                    if item.is_dir():
+                        shutil.copytree(item, dest_item)
+                    else:
+                        shutil.copy2(item, dest_item)
+            except OSError as e:
+                logger.warning("Could not restore %s directory: %s", name, e)
+                skipped_dirs.append(name)
+    return skipped_dirs
+
+
+def _restore_sqlite_db(backup_db: Path, db_path: Path) -> None:
+    """Copy the backup into the live SQLite database with the online backup API."""
+    import sqlite3
+
+    src_conn = sqlite3.connect(str(backup_db))
+    try:
+        dst_conn = sqlite3.connect(str(db_path))
+        try:
+            src_conn.backup(dst_conn)
+        finally:
+            dst_conn.close()
+    finally:
+        src_conn.close()
+
+
 async def create_backup_zip(output_path: Path | None = None) -> tuple[Path, str]:
     """Create a complete backup ZIP (database + all data directories).
 
@@ -942,8 +1057,9 @@ async def create_backup_zip(output_path: Path | None = None) -> tuple[Path, str]
             async with engine.begin() as conn:
                 await conn.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
 
-            # Copy database file
-            shutil.copy2(db_path, temp_path / "bambuddy.db")
+            # Copy database file. Off the event loop, like every file step
+            # below: a large file blocked the whole server while it copied.
+            await asyncio.to_thread(shutil.copy2, db_path, temp_path / "bambuddy.db")
         else:
             # PostgreSQL: export to a portable SQLite file via SQLAlchemy.
             # This makes backups restorable on both SQLite and Postgres installs.
@@ -1007,16 +1123,7 @@ async def create_backup_zip(output_path: Path | None = None) -> tuple[Path, str]
             ("overlay-branding", base_dir / "overlay-branding"),
         ]
 
-        for name, src_dir in dirs_to_backup:
-            if src_dir.exists() and any(src_dir.iterdir()):
-                try:
-                    shutil.copytree(
-                        src_dir, temp_path / name
-                    )  # SEC-PATH-OK: name iterates the dirs_to_backup tuple of constant strings ("archive", "virtual_printer", ...)
-                except shutil.Error as e:
-                    logger.warning("Some files in %s could not be copied: %s", name, e)
-                except PermissionError as e:
-                    logger.warning("Permission denied copying %s: %s", name, e)
+        await asyncio.to_thread(_copy_backup_dirs, dirs_to_backup, temp_path)
 
         # Say which version made this, so a restore that cannot import it can
         # name the versions rather than a list of columns. Backups from before
@@ -1060,11 +1167,7 @@ async def create_backup_zip(output_path: Path | None = None) -> tuple[Path, str]
             os.close(fd)
             zip_file = Path(tmp)
 
-        with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) as zf:
-            for file_path in temp_path.rglob("*"):
-                if file_path.is_file():
-                    arcname = file_path.relative_to(temp_path)
-                    zf.write(file_path, arcname)
+        await asyncio.to_thread(_write_backup_zip, temp_path, zip_file)
 
     return zip_file, filename
 
@@ -1519,7 +1622,6 @@ async def restore_backup(
     Replaces the database and all data directories from the backup ZIP.
     Requires a restart after restore.
     """
-    import shutil
     import tempfile
 
     from fastapi import HTTPException
@@ -1540,25 +1642,7 @@ async def restore_backup(
         if not file.filename or not file.filename.endswith(".zip"):
             raise HTTPException(400, "Invalid backup file: must be a .zip file")
 
-        try:
-            with zipfile.ZipFile(io.BytesIO(content), "r") as zf:
-                for name in zf.namelist():
-                    # Reject path-traversal payloads: any entry whose resolved
-                    # path escapes temp_path would allow writing arbitrary files
-                    # on the host (ZipSlip / CVE-2006-5456).
-                    dest = (
-                        temp_path / name
-                    ).resolve()  # SEC-PATH-OK: is_relative_to containment check below before extractall
-                    # is_relative_to (Python 3.9+) covers both relative
-                    # path-traversal (../etc/passwd) and absolute-path overrides
-                    # (/etc/passwd) — str.startswith was vulnerable to
-                    # prefix-collision attacks (e.g. /tmp/abc_evil/file passing
-                    # a /tmp/abc prefix check).
-                    if not dest.is_relative_to(temp_path.resolve()):
-                        raise HTTPException(400, f"Invalid backup: unsafe path in ZIP: {name!r}")
-                zf.extractall(temp_path)
-        except zipfile.BadZipFile:
-            raise HTTPException(400, "Invalid backup file: not a valid ZIP")
+        await asyncio.to_thread(_extract_backup_zip, content, temp_path)
 
         # 2. Validate backup
         backup_db = temp_path / "bambuddy.db"
@@ -1590,8 +1674,6 @@ async def restore_backup(
                 raise HTTPException(400, f"Invalid backup: bambuddy.db is not readable ({exc})") from exc
 
         try:
-            import asyncio
-
             # 3. Stop the virtual printers if running (releases file locks).
             # stop_all, not the legacy configure(): that ignores its arguments
             # and re-syncs from the database not yet replaced, so their FTP,
@@ -1710,17 +1792,7 @@ async def restore_backup(
                 # own WAL — so concurrent open sessions see their own
                 # snapshot until they close (transaction isolation) but
                 # can't corrupt the restored state.
-                import sqlite3
-
-                src_conn = sqlite3.connect(str(backup_db))
-                try:
-                    dst_conn = sqlite3.connect(str(db_path))
-                    try:
-                        src_conn.backup(dst_conn)
-                    finally:
-                        dst_conn.close()
-                finally:
-                    src_conn.close()
+                await asyncio.to_thread(_restore_sqlite_db, backup_db, db_path)
             else:
                 # Import SQLite backup into PostgreSQL
                 logger.info("Importing SQLite backup into PostgreSQL...")
@@ -1737,36 +1809,7 @@ async def restore_backup(
                 ("overlay-branding", base_dir / "overlay-branding"),
             ]
 
-            skipped_dirs = []
-            for name, dest_dir in dirs_to_restore:
-                src_dir = (
-                    temp_path / name
-                )  # SEC-PATH-OK: name iterates the dirs_to_restore tuple of constant strings ("archive", "virtual_printer", ...)
-                if src_dir.exists():
-                    logger.info("Restoring %s directory...", name)
-                    try:
-                        # Clear destination contents (not the dir itself - may be Docker mount)
-                        if dest_dir.exists():
-                            for item in dest_dir.iterdir():
-                                try:
-                                    if item.is_dir():
-                                        shutil.rmtree(item)
-                                    else:
-                                        item.unlink()
-                                except OSError as e:
-                                    logger.warning("Could not delete %s: %s", item, e)
-                        else:
-                            dest_dir.mkdir(parents=True, exist_ok=True)
-                        # Copy contents from backup
-                        for item in src_dir.iterdir():
-                            dest_item = dest_dir / item.name
-                            if item.is_dir():
-                                shutil.copytree(item, dest_item)
-                            else:
-                                shutil.copy2(item, dest_item)
-                    except OSError as e:
-                        logger.warning("Could not restore %s directory: %s", name, e)
-                        skipped_dirs.append(name)
+            skipped_dirs = await asyncio.to_thread(_restore_data_dirs, dirs_to_restore, temp_path)
 
             # 7. Reset the encryption singleton so the migration that runs
             # inside init_db() picks up the restored key file (if a new one

+ 166 - 0
backend/tests/integration/test_backup_restore_off_event_loop.py

@@ -0,0 +1,166 @@
+"""Backup and restore do their file work off the event loop.
+
+Unzipping a backup, copying the data directories back and writing a backup ZIP
+ran on the event loop. With a large archive folder that froze the whole server
+for over half a minute -- no page, no websocket, no printer message -- and the
+loop watchdog dumped every thread's stack in the middle of a restore that then
+succeeded.
+
+Each test makes one step slow and counts how often a ticker on the loop gets to
+run meanwhile. On the loop, the step starves it; in a worker thread it ticks on.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import io
+import sqlite3
+import time
+import zipfile
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from backend.app.api.routes import settings as settings_routes
+from backend.app.core.config import settings as app_settings
+
+STEP_SECONDS = 0.5
+TICK_SECONDS = 0.02
+# Off the loop the ticker runs ~25 times during the step; on it, not at all.
+MIN_TICKS = 8
+
+
+class _Ticker:
+    """Counts loop ticks, and how many happened while the slow step ran."""
+
+    def __init__(self):
+        self.ticks = 0
+        self.during_step: list[int] = []
+
+    def slow(self, fn):
+        def wrapper(*args, **kwargs):
+            before = self.ticks
+            time.sleep(STEP_SECONDS)
+            self.during_step.append(self.ticks - before)
+            return fn(*args, **kwargs)
+
+        return wrapper
+
+    async def run_while(self, coro):
+        done = False
+
+        async def tick():
+            while not done:
+                await asyncio.sleep(TICK_SECONDS)
+                self.ticks += 1
+
+        task = asyncio.create_task(tick())
+        try:
+            return await coro
+        finally:
+            done = True
+            await task
+
+
+def _data_dir(monkeypatch, tmp_path):
+    db_path = tmp_path / "bambuddy.db"
+    with sqlite3.connect(db_path) as db:
+        db.execute("CREATE TABLE backup_marker (id INTEGER PRIMARY KEY)")
+    (tmp_path / "archive" / "1").mkdir(parents=True)
+    (tmp_path / "archive" / "1" / "print.3mf").write_bytes(b"x" * 1024)
+    monkeypatch.setenv("DATA_DIR", str(tmp_path))
+    monkeypatch.setattr(app_settings, "base_dir", tmp_path)
+    monkeypatch.setattr(app_settings, "database_url", f"sqlite+aiosqlite:///{db_path}")
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+@pytest.mark.sqlite_only  # copies the app's own SQLite database file
+@pytest.mark.parametrize("step", ["_copy_backup_dirs", "_write_backup_zip", "shutil.copy2"])
+async def test_a_slow_backup_step_does_not_block_the_loop(async_client, monkeypatch, tmp_path, step):
+    import shutil
+
+    _data_dir(monkeypatch, tmp_path)
+    ticker = _Ticker()
+    if step == "shutil.copy2":
+        monkeypatch.setattr(shutil, "copy2", ticker.slow(shutil.copy2))
+    else:
+        monkeypatch.setattr(settings_routes, step, ticker.slow(getattr(settings_routes, step)))
+
+    out = tmp_path / "out"
+    out.mkdir()
+    zip_path, _ = await ticker.run_while(settings_routes.create_backup_zip(output_path=out))
+
+    with zipfile.ZipFile(zip_path) as zf:
+        names = zf.namelist()
+    assert "bambuddy.db" in names
+    assert "archive/1/print.3mf" in names
+    assert ticker.during_step, f"{step} was not called"
+    assert min(ticker.during_step) >= MIN_TICKS, f"{step} blocked the event loop: {ticker.during_step}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+@pytest.mark.sqlite_only  # restores into the app's own SQLite database file
+@pytest.mark.parametrize("step", ["_extract_backup_zip", "_restore_sqlite_db", "_restore_data_dirs"])
+async def test_a_slow_restore_step_does_not_block_the_loop(async_client, monkeypatch, tmp_path, step):
+    _data_dir(monkeypatch, tmp_path)
+    out = tmp_path / "out"
+    out.mkdir()
+    zip_path, _ = await settings_routes.create_backup_zip(output_path=out)
+    payload = zip_path.read_bytes()
+    # The restore must put the archive back, so take it away first.
+    (tmp_path / "archive" / "1" / "print.3mf").unlink()
+
+    ticker = _Ticker()
+    monkeypatch.setattr(settings_routes, step, ticker.slow(getattr(settings_routes, step)))
+    with (
+        patch("backend.app.core.database.close_all_connections", new_callable=AsyncMock),
+        patch("backend.app.core.database.reinitialize_database", new_callable=AsyncMock),
+        patch("backend.app.core.database.init_db", new_callable=AsyncMock),
+        patch("backend.app.core.database.invalidate_all_sessions", new_callable=AsyncMock),
+        patch("backend.app.services.backup_paths.convert_windows_paths", new_callable=AsyncMock),
+        patch("backend.app.services.print_scheduler.scheduler.stop"),
+        patch("backend.app.services.smart_plug_manager.smart_plug_manager.stop_scheduler"),
+        patch("backend.app.services.notification_service.notification_service.stop_digest_scheduler"),
+    ):
+        response = await ticker.run_while(
+            async_client.post(
+                "/api/v1/settings/restore",
+                files={"file": ("backup.zip", payload, "application/zip")},
+            )
+        )
+
+    assert response.status_code == 200, response.text
+    assert (tmp_path / "archive" / "1" / "print.3mf").read_bytes() == b"x" * 1024
+    assert ticker.during_step, f"{step} was not called"
+    assert min(ticker.during_step) >= MIN_TICKS, f"{step} blocked the event loop: {ticker.during_step}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_an_unsafe_path_is_still_refused(async_client):
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w") as zf:
+        zf.writestr("../evil.txt", "x")
+        zf.writestr("bambuddy.db", "x")
+
+    response = await async_client.post(
+        "/api/v1/settings/restore",
+        files={"file": ("backup.zip", buffer.getvalue(), "application/zip")},
+    )
+
+    assert response.status_code == 400
+    assert "unsafe path" in response.json()["detail"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_a_file_that_is_not_a_zip_is_still_refused(async_client):
+    response = await async_client.post(
+        "/api/v1/settings/restore",
+        files={"file": ("backup.zip", b"not a zip", "application/zip")},
+    )
+
+    assert response.status_code == 400
+    assert "not a valid ZIP" in response.json()["detail"]