github_restore.py 62 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384
  1. """Restore Bambuddy data from a Git provider backup (issue #2656).
  2. The backup side (``github_backup.py``) is push-only: it collects a handful of
  3. JSON documents and commits them. This module is the read side — it walks the
  4. backup repository's history, lets a caller inspect what a given commit contains,
  5. and applies selected categories back into the local database (or, for
  6. K-profiles, back onto the printers).
  7. Design notes worth knowing before editing:
  8. * **A restore never reuses the backup's primary keys.** ``spool.id`` and
  9. ``print_archives.id`` are bare autoincrement columns, so the ids in a backup
  10. taken weeks ago very likely belong to unrelated rows today. Rows are matched
  11. on natural keys instead, inserted without an explicit id, and an
  12. ``old_id -> new_id`` map is threaded through so foreign keys in dependent
  13. tables (spool usage history) still line up.
  14. The printer-side ``cali_idx`` behaves the same way and gets the same
  15. treatment. Editing a K-profile in Bambuddy is a delete-then-add on a
  16. single-nozzle printer, which re-keys it, and ``extrusion_cali_set`` aimed at a
  17. slot that no longer exists is silently dropped — so the live index is read
  18. back and matched before writing, never taken from the backup.
  19. * **Categories are applied archives -> spools -> settings -> kprofiles.**
  20. Archives first because spool usage history references ``archive_id``;
  21. K-profiles last because they leave the database and talk to hardware.
  22. * **Cloud profiles are not restorable.** The backup collector never actually
  23. writes ``cloud_profiles/*.json``, and the preset list it would write carries
  24. no setting payload. Tracked separately from #2656.
  25. """
  26. import asyncio
  27. import json
  28. import logging
  29. import os
  30. import re
  31. from dataclasses import dataclass
  32. from datetime import datetime, timezone
  33. import httpx
  34. from sqlalchemy import select
  35. from sqlalchemy.ext.asyncio import AsyncSession
  36. from backend.app.core.database import async_session
  37. from backend.app.models.archive import PrintArchive
  38. from backend.app.models.github_backup import GitHubBackupConfig, GitHubBackupLog
  39. from backend.app.models.printer import Printer
  40. from backend.app.models.project import Project
  41. from backend.app.models.settings import Settings
  42. from backend.app.models.spool import Spool
  43. from backend.app.models.spool_usage_history import SpoolUsageHistory
  44. from backend.app.models.user import User
  45. from backend.app.schemas.github_backup import RestoreCategory
  46. from backend.app.services.git_providers.factory import get_provider_backend
  47. from backend.app.services.printer_manager import printer_manager
  48. logger = logging.getLogger(__name__)
  49. METADATA_PATH = "backup_metadata.json"
  50. SETTINGS_PATH = "settings/app_settings.json"
  51. SPOOLS_PATH = "spools/inventory.json"
  52. SPOOL_USAGE_PATH = "spools/usage_history.json"
  53. ARCHIVES_PATH = "archives/print_history.json"
  54. # kprofiles/{printer_serial}/{nozzle_diameter}.json
  55. _KPROFILE_PATH_RE = re.compile(r"^kprofiles/([^/]+)/([^/]+)\.json$")
  56. # Settings keys the backup collector already refuses to write. Applied again on
  57. # the read side because a backup taken before that denylist existed can still
  58. # contain them, and a restore must not resurrect a stale credential.
  59. _SENSITIVE_SETTING_KEYS = {"bambu_cloud_token", "auth_secret_key"}
  60. # Belt-and-braces for the same reason: any key that looks like a secret is
  61. # skipped even if it isn't in the explicit denylist above.
  62. _SECRET_KEY_HINTS = ("token", "secret", "password", "access_code", "api_key", "passphrase")
  63. # Settings the MQTT relay reads only when it is (re)configured, so restoring the
  64. # rows is not enough on its own. Mirrors the set the settings PUT handler
  65. # watches. mqtt_password is in here for the configure() payload's sake — the
  66. # credential blocklist means a restore never writes it.
  67. _MQTT_SETTING_KEYS = {
  68. "mqtt_enabled",
  69. "mqtt_broker",
  70. "mqtt_port",
  71. "mqtt_username",
  72. "mqtt_password",
  73. "mqtt_topic_prefix",
  74. "mqtt_use_tls",
  75. }
  76. # Keys that decide *who can reach the instance* rather than how it behaves. The
  77. # backup collector writes them like any other Settings row, so a backup taken
  78. # before auth was turned on carries auth_enabled=false — and a restore reaches
  79. # the table directly, so honouring them would:
  80. #
  81. # * disable authentication outright. ``set_auth_enabled`` pairs its write with
  82. # ``invalidate_auth_enabled_cache()``; we cannot, so the 30 s TTL in
  83. # core.auth is the only thing between the write and an open instance. That
  84. # cache is built to fail closed — writing the stored value behind its back
  85. # is what would make it fail open.
  86. # * bypass the lockout refusals ``update_settings`` enforces (a
  87. # ``local_login_enabled=false`` with no enabled OIDC provider, or with no
  88. # OIDC link on the caller, is a 400 there — #1589).
  89. # * cross a permission boundary: /github-backup/restore is gated on
  90. # GITHUB_RESTORE alone, so this would be a way to rewrite auth config
  91. # without SETTINGS_UPDATE.
  92. #
  93. # Auth is reconfigured through the auth UI, which has the guards. Restoring it
  94. # from a snapshot has no safe reading.
  95. _PROTECTED_SETTING_KEYS = {
  96. "auth_enabled",
  97. "advanced_auth_enabled",
  98. "local_login_enabled",
  99. "setup_completed",
  100. }
  101. # Nozzle diameters the backup collector iterates. A path outside this set means
  102. # the backup was written by a newer version, so accept it rather than dropping
  103. # data, but keep the list for validation messages.
  104. _KNOWN_NOZZLES = {"0.2", "0.4", "0.6", "0.8"}
  105. def _parse_dt(value) -> datetime | None:
  106. """Best-effort parse of a datetime the backup wrote via ``str(...)``."""
  107. if not value or not isinstance(value, str):
  108. return None
  109. try:
  110. return datetime.fromisoformat(value)
  111. except ValueError:
  112. return None
  113. def _is_blocked_setting_key(key: str) -> bool:
  114. lowered = key.lower()
  115. return key in _SENSITIVE_SETTING_KEYS or any(hint in lowered for hint in _SECRET_KEY_HINTS)
  116. def _is_protected_setting_key(key: str) -> bool:
  117. return key in _PROTECTED_SETTING_KEYS
  118. # There used to be an ``_is_skipped_setting_key`` here, the union of the two
  119. # predicates above, shared by the preview and the restore so neither could drift
  120. # from the other. It is gone because a name is no longer enough to decide: the
  121. # third refusal below depends on the payload's *other* values and on local
  122. # database state. ``_plan_settings`` is the shared classifier now, and it covers
  123. # all three reasons.
  124. # Toggles whose *safety* depends on a companion credential that the blocklist
  125. # above refuses to restore. Writing the toggle alone is not a partial restore,
  126. # it is a downgrade:
  127. #
  128. # * prometheus_enabled with no token opens /api/v1/metrics. The route is on
  129. # PUBLIC_API_ROUTES and its own gate is ``if token:`` (api/routes/metrics.py),
  130. # so an empty or absent token means no authentication at all — a full,
  131. # unauthenticated dump of the instance to anyone who can reach the port. On
  132. # an instance that never enabled Prometheus there is no token row, so
  133. # overwrite-off alone is enough to do it.
  134. # * the other four switch an integration on with no way to authenticate to it,
  135. # which breaks the login path (LDAP) or the connection (MQTT, HA).
  136. #
  137. # virtual_printer_enabled is largely vestigial post-migration — core/database.py
  138. # copies the rows into the virtual_printers table — but it is the same shape, and
  139. # refusing a vestigial toggle is a harmless no-op.
  140. _COMPANION_CREDENTIALS = {
  141. "prometheus_enabled": "prometheus_token",
  142. "ldap_enabled": "ldap_bind_password",
  143. "mqtt_enabled": "mqtt_password",
  144. "ha_enabled": "ha_token",
  145. "virtual_printer_enabled": "virtual_printer_access_code",
  146. }
  147. # Companion credentials a reader takes from the environment rather than from a
  148. # Settings row. ha_token is the only one: get_homeassistant_settings prefers
  149. # HA_TOKEN over the row, and auto-enables ha_enabled when HA_URL and HA_TOKEN are
  150. # both set, so an env-configured instance has a usable credential and no row.
  151. _COMPANION_CREDENTIAL_ENV = {"ha_token": "HA_TOKEN"}
  152. def _setting_value_is_true(value: object) -> bool:
  153. """True if a settings *payload* value would be stored as "on".
  154. Deliberately as narrow as ``api.routes.settings.setting_is_true``: a restore
  155. writes ``str(value)`` verbatim and no reader in the codebase treats "1",
  156. "on" or "yes" as on, so restoring one of those cannot switch anything on.
  157. Bool-tolerant because a backup's JSON can carry a real boolean.
  158. """
  159. if isinstance(value, bool):
  160. return value
  161. if value is None:
  162. return False
  163. return str(value).strip().lower() == "true"
  164. def _is_usable_credential(value: object) -> bool:
  165. """True if a credential value is present and not blank.
  166. A present-but-*blank* ``prometheus_token`` row counts as unusable, because an
  167. empty token is exactly the ``if token:`` hole the companion rule exists to
  168. stop a restore from opening.
  169. """
  170. return value is not None and bool(str(value).strip())
  171. @dataclass(frozen=True)
  172. class _SettingsPlan:
  173. """Which keys of a settings payload will not be written, and why.
  174. Built once, before anything is added to the session, and shared by the
  175. preview and the restore so the two cannot disagree about what a commit will
  176. change. The companion bucket is why this needs a session at all: unlike the
  177. two name-based buckets it depends on local database state.
  178. The three buckets are disjoint — a key is classified once, in order.
  179. """
  180. blocked: tuple[str, ...] = ()
  181. protected: tuple[str, ...] = ()
  182. companion: tuple[str, ...] = ()
  183. @property
  184. def refused(self) -> frozenset[str]:
  185. return frozenset(self.blocked) | frozenset(self.protected) | frozenset(self.companion)
  186. @property
  187. def refused_count(self) -> int:
  188. return len(self.blocked) + len(self.protected) + len(self.companion)
  189. class _CategoryTally:
  190. """Mutable accumulator matching ``GitHubRestoreCategoryResult``."""
  191. def __init__(self) -> None:
  192. self.restored = 0
  193. self.skipped = 0
  194. self.failed = 0
  195. self.notes: list[str] = []
  196. def note(self, message: str) -> None:
  197. # Notes are surfaced verbatim in the UI, so keep the list bounded rather
  198. # than emitting one line per row for a large backup.
  199. if message not in self.notes and len(self.notes) < 20:
  200. self.notes.append(message)
  201. def as_dict(self) -> dict:
  202. return {"restored": self.restored, "skipped": self.skipped, "failed": self.failed, "notes": self.notes}
  203. class GitHubRestoreService:
  204. """Reads a backup repository and applies selected categories locally."""
  205. def __init__(self) -> None:
  206. self._running_restore: bool = False
  207. self._progress: str | None = None
  208. self._http_client: httpx.AsyncClient | None = None
  209. # Guards the check-then-set on ``_running_restore``. Without it two
  210. # concurrent POSTs can both observe False before either sets it.
  211. self._lock = asyncio.Lock()
  212. async def _get_client(self) -> httpx.AsyncClient:
  213. if self._http_client is None or self._http_client.is_closed:
  214. self._http_client = httpx.AsyncClient(timeout=60.0)
  215. return self._http_client
  216. @property
  217. def is_running(self) -> bool:
  218. return self._running_restore
  219. @property
  220. def progress(self) -> str | None:
  221. return self._progress
  222. # --- Repository reads --------------------------------------------------
  223. async def list_commits(self, config: GitHubBackupConfig, limit: int = 20) -> dict:
  224. """List recent commits on the configured branch."""
  225. backend = get_provider_backend(config.provider)
  226. client = await self._get_client()
  227. result = await backend.list_commits(
  228. repo_url=config.repository_url,
  229. token=config.access_token,
  230. branch=config.branch,
  231. client=client,
  232. limit=limit,
  233. )
  234. result["branch"] = config.branch
  235. return result
  236. async def _resolve_ref(self, config: GitHubBackupConfig, ref: str) -> tuple[str | None, str]:
  237. """Turn ``HEAD`` into a concrete commit SHA.
  238. Done once up front so a preview and the restore that follows it act on
  239. the same commit even if a scheduled backup lands in between.
  240. """
  241. if ref and ref.upper() != "HEAD":
  242. return ref, ""
  243. result = await self.list_commits(config, limit=1)
  244. if not result.get("success"):
  245. return None, result.get("message") or "Could not read the backup repository"
  246. commits = result.get("commits") or []
  247. if not commits:
  248. return None, f"Branch '{config.branch}' has no commits to restore from"
  249. return commits[0]["sha"], ""
  250. def _category_paths(self, category: RestoreCategory, available: list[str]) -> list[str]:
  251. """Return the paths in ``available`` that belong to ``category``."""
  252. if category == RestoreCategory.SETTINGS:
  253. return [p for p in (SETTINGS_PATH,) if p in available]
  254. if category == RestoreCategory.SPOOLS:
  255. return [p for p in (SPOOLS_PATH, SPOOL_USAGE_PATH) if p in available]
  256. if category == RestoreCategory.ARCHIVES:
  257. return [p for p in (ARCHIVES_PATH,) if p in available]
  258. if category == RestoreCategory.KPROFILES:
  259. return sorted(p for p in available if _KPROFILE_PATH_RE.match(p))
  260. return []
  261. @staticmethod
  262. def _parse_json_files(raw: dict[str, str]) -> tuple[dict[str, object], list[str]]:
  263. """Parse each fetched file, collecting paths that failed to parse."""
  264. parsed: dict[str, object] = {}
  265. bad: list[str] = []
  266. for path, text in raw.items():
  267. try:
  268. parsed[path] = json.loads(text)
  269. except (ValueError, TypeError):
  270. bad.append(path)
  271. return parsed, bad
  272. @staticmethod
  273. async def _plan_settings(db: AsyncSession, values: dict) -> _SettingsPlan:
  274. """Classify every key of a settings payload into its refusal bucket.
  275. Keys with an unusable name land in no bucket: they are the restore's
  276. ``failed``, not a refusal, and the preview counts them because the run
  277. will still report on them.
  278. Reads local state, so it must run before anything is added to the
  279. session — otherwise "does this instance already have a credential" would
  280. see the restore's own writes.
  281. """
  282. blocked: list[str] = []
  283. protected: list[str] = []
  284. # Toggle -> credential for the pairs that survived the payload-only
  285. # conditions and still need local state to judge.
  286. candidates: dict[str, str] = {}
  287. for key, value in values.items():
  288. if not isinstance(key, str) or not key:
  289. continue
  290. if _is_blocked_setting_key(key):
  291. blocked.append(key)
  292. continue
  293. if _is_protected_setting_key(key):
  294. protected.append(key)
  295. continue
  296. credential = _COMPANION_CREDENTIALS.get(key)
  297. if credential is None:
  298. continue
  299. # Turning something *off* is always safe to write.
  300. if not _setting_value_is_true(value):
  301. continue
  302. # Expressed as the predicate rather than assumed, so the map cannot
  303. # go quietly inert if _SECRET_KEY_HINTS is ever edited: a credential
  304. # the restore is willing to write travels with its toggle.
  305. if not _is_blocked_setting_key(credential):
  306. continue
  307. # The backup itself carried no credential here. An anonymous MQTT
  308. # broker and an anonymous LDAP bind are both legitimate configs
  309. # (mqtt_relay.py and ldap_service.py pass empty credentials straight
  310. # through), so refusing this toggle would be a false positive — the
  311. # restore is not producing anything weaker than the backup.
  312. if not _is_usable_credential(values.get(credential)):
  313. continue
  314. candidates[key] = credential
  315. if not candidates:
  316. return _SettingsPlan(blocked=tuple(blocked), protected=tuple(protected))
  317. # One SELECT covering both halves of every candidate pair.
  318. wanted = set(candidates) | set(candidates.values())
  319. rows = await db.execute(select(Settings).where(Settings.key.in_(wanted)))
  320. local = {row.key: row.value for row in rows.scalars().all()}
  321. companion: list[str] = []
  322. for toggle, credential in candidates.items():
  323. if _is_usable_credential(local.get(credential)):
  324. continue
  325. env_name = _COMPANION_CREDENTIAL_ENV.get(credential)
  326. if env_name and _is_usable_credential(os.environ.get(env_name)):
  327. continue
  328. # Already on locally with no credential: the exposure pre-dates this
  329. # restore, so refusing changes nothing and "left switched off" would
  330. # be a lie.
  331. if _setting_value_is_true(local.get(toggle)):
  332. continue
  333. companion.append(toggle)
  334. return _SettingsPlan(
  335. blocked=tuple(blocked),
  336. protected=tuple(protected),
  337. companion=tuple(companion),
  338. )
  339. async def preview(self, db: AsyncSession, config: GitHubBackupConfig, ref: str = "HEAD") -> dict:
  340. """Report which categories a commit contains, and how much is in each.
  341. Takes a session because the settings count depends on local state — see
  342. ``_plan_settings``. ``ref`` stays keyword-friendly for callers.
  343. """
  344. resolved, error = await self._resolve_ref(config, ref)
  345. if resolved is None:
  346. return {"success": False, "message": error, "ref": ref, "categories": []}
  347. backend = get_provider_backend(config.provider)
  348. client = await self._get_client()
  349. tree = await backend.list_tree(
  350. repo_url=config.repository_url, token=config.access_token, ref=resolved, client=client
  351. )
  352. if not tree.get("success"):
  353. return {"success": False, "message": tree.get("message") or "Could not list the commit", "ref": resolved}
  354. available: list[str] = tree.get("paths") or []
  355. # One batched read covers metadata plus every category payload.
  356. wanted = [METADATA_PATH] if METADATA_PATH in available else []
  357. for category in RestoreCategory:
  358. wanted.extend(self._category_paths(category, available))
  359. fetched = await backend.fetch_files(
  360. repo_url=config.repository_url, token=config.access_token, ref=resolved, paths=wanted, client=client
  361. )
  362. if not fetched.get("success"):
  363. return {
  364. "success": False,
  365. "message": fetched.get("message") or "Could not read the commit contents",
  366. "ref": resolved,
  367. }
  368. parsed, bad_paths = self._parse_json_files(fetched.get("files") or {})
  369. metadata = parsed.get(METADATA_PATH)
  370. metadata_version = metadata.get("version") if isinstance(metadata, dict) else None
  371. categories = []
  372. for category in RestoreCategory:
  373. paths = self._category_paths(category, available)
  374. if not paths:
  375. categories.append(
  376. {
  377. "category": category,
  378. "available": False,
  379. "item_count": 0,
  380. "detail": "Not present in this backup commit",
  381. }
  382. )
  383. continue
  384. unreadable = [p for p in paths if p in bad_paths]
  385. if unreadable:
  386. categories.append(
  387. {
  388. "category": category,
  389. "available": False,
  390. "item_count": 0,
  391. "detail": f"Unreadable JSON: {', '.join(unreadable)}",
  392. }
  393. )
  394. continue
  395. count, detail = await self._count_items(db, category, parsed)
  396. categories.append({"category": category, "available": True, "item_count": count, "detail": detail})
  397. commit_info = None
  398. commits = (await self.list_commits(config, limit=20)).get("commits") or []
  399. for entry in commits:
  400. if entry["sha"] == resolved:
  401. commit_info = entry
  402. break
  403. return {
  404. "success": True,
  405. "message": "OK",
  406. "ref": resolved,
  407. "commit": commit_info,
  408. "metadata_version": metadata_version,
  409. "categories": categories,
  410. }
  411. async def _count_items(self, db: AsyncSession, category: RestoreCategory, parsed: dict) -> tuple[int, str | None]:
  412. """Count restorable items for ``category`` and describe any caveat."""
  413. if category == RestoreCategory.SETTINGS:
  414. payload = parsed.get(SETTINGS_PATH)
  415. values = payload.get("settings") if isinstance(payload, dict) else None
  416. if not isinstance(values, dict):
  417. return 0, "No settings in payload"
  418. # Every refusal is subtracted so the count matches what the restore
  419. # actually writes. The wording calls out the credential ones (what a
  420. # user might expect to come back) and the companion ones (a
  421. # behaviour change worth explaining before it happens); the auth
  422. # policy keys stay unmentioned on purpose.
  423. plan = await self._plan_settings(db, values)
  424. detail = None
  425. if plan.companion:
  426. detail = (
  427. f"{len(plan.blocked)} credential-like key(s) will be skipped, and "
  428. f"{len(plan.companion)} switch(es) that depend on them will be left off"
  429. )
  430. elif plan.blocked:
  431. detail = f"{len(plan.blocked)} credential-like keys will be skipped"
  432. return len(values) - plan.refused_count, detail
  433. if category == RestoreCategory.SPOOLS:
  434. payload = parsed.get(SPOOLS_PATH)
  435. spools = payload.get("spools") if isinstance(payload, dict) else None
  436. usage_payload = parsed.get(SPOOL_USAGE_PATH)
  437. usage = usage_payload.get("usage_history") if isinstance(usage_payload, dict) else None
  438. count = len(spools) if isinstance(spools, list) else 0
  439. detail = f"plus {len(usage)} usage records" if isinstance(usage, list) and usage else None
  440. return count, detail
  441. if category == RestoreCategory.ARCHIVES:
  442. payload = parsed.get(ARCHIVES_PATH)
  443. archives = payload.get("archives") if isinstance(payload, dict) else None
  444. count = len(archives) if isinstance(archives, list) else 0
  445. return count, "Metadata only — 3MF files and thumbnails are not in a Git backup"
  446. if category == RestoreCategory.KPROFILES:
  447. total = 0
  448. serials = set()
  449. for path, payload in parsed.items():
  450. match = _KPROFILE_PATH_RE.match(path)
  451. if not match or not isinstance(payload, dict):
  452. continue
  453. serials.add(match.group(1))
  454. profiles = payload.get("profiles")
  455. if isinstance(profiles, list):
  456. total += len(profiles)
  457. detail = f"across {len(serials)} printer(s)" if serials else None
  458. return total, detail
  459. return 0, None
  460. # --- Restore -----------------------------------------------------------
  461. async def run_restore(
  462. self,
  463. config_id: int,
  464. ref: str,
  465. categories: list[RestoreCategory],
  466. overwrite_existing: bool = False,
  467. ) -> dict:
  468. """Apply selected categories from one backup commit."""
  469. # Import locally to avoid a module-level cycle: the backup service takes
  470. # the mirror-image lock against us.
  471. from backend.app.services.github_backup import github_backup_service
  472. # The lock serialises two concurrent restores; the backup side has no
  473. # lock of its own, and relies on this region staying await-free after the
  474. # acquisition. Both flags are plain bools on one event loop, so with no
  475. # suspension point between the two reads and the write, the loop cannot
  476. # slip github_backup.run_backup's mirror-image check in between. Adding an
  477. # `await` below the acquisition and above `self._running_restore = True`
  478. # would let a backup and a restore run at once.
  479. async with self._lock:
  480. if self._running_restore:
  481. return {"success": False, "message": "A restore is already running", "results": {}}
  482. if github_backup_service.is_running:
  483. return {
  484. "success": False,
  485. "message": "A backup is currently running. Wait for it to finish before restoring.",
  486. "results": {},
  487. }
  488. self._running_restore = True
  489. log_id = None
  490. try:
  491. async with async_session() as db:
  492. result = await db.execute(select(GitHubBackupConfig).where(GitHubBackupConfig.id == config_id))
  493. config = result.scalar_one_or_none()
  494. if not config:
  495. return {"success": False, "message": "Configuration not found", "results": {}}
  496. self._progress = "Resolving commit..."
  497. resolved, error = await self._resolve_ref(config, ref)
  498. if resolved is None:
  499. return {"success": False, "message": error, "results": {}}
  500. log = GitHubBackupLog(config_id=config_id, status="running", trigger="restore", commit_sha=resolved)
  501. db.add(log)
  502. await db.commit()
  503. await db.refresh(log)
  504. log_id = log.id
  505. try:
  506. payload, error = await self._read_categories(config, resolved, categories)
  507. if error:
  508. raise RuntimeError(error)
  509. settings_keys_written: set[str] = set()
  510. results = await self._apply(db, payload, categories, overwrite_existing, settings_keys_written)
  511. await db.commit()
  512. # After the commit: this reconnects the relay, which is not
  513. # something to do on values that could still roll back.
  514. settings_tally = results.get(RestoreCategory.SETTINGS.value)
  515. if settings_tally is not None:
  516. self._progress = "Reconnecting the MQTT relay..."
  517. await self._reconfigure_mqtt_relay(db, settings_keys_written, settings_tally)
  518. total_restored = sum(tally.restored for tally in results.values())
  519. any_failed = any(tally.failed for tally in results.values())
  520. log.status = "failed" if any_failed and total_restored == 0 else "success"
  521. log.completed_at = datetime.now(timezone.utc)
  522. log.files_changed = total_restored
  523. if any_failed:
  524. log.error_message = "Some items could not be restored — see the restore result for detail"
  525. await db.commit()
  526. return {
  527. "success": True,
  528. "message": f"Restored {total_restored} item(s) from {resolved[:7]}",
  529. "log_id": log_id,
  530. "ref": resolved,
  531. "results": {name: tally.as_dict() for name, tally in results.items()},
  532. }
  533. except Exception as e:
  534. # Rolls back whatever is still uncommitted. That is every
  535. # database category unless K-profiles were also selected, in
  536. # which case _apply has already committed them before talking
  537. # to the printers — see the comment there.
  538. logger.exception("Restore failed for config %s ref %s", config_id, resolved)
  539. await db.rollback()
  540. log.status = "failed"
  541. log.completed_at = datetime.now(timezone.utc)
  542. log.error_message = str(e)[:1000]
  543. await db.commit()
  544. return {"success": False, "message": str(e), "log_id": log_id, "ref": resolved, "results": {}}
  545. finally:
  546. self._running_restore = False
  547. self._progress = None
  548. async def _read_categories(
  549. self, config: GitHubBackupConfig, ref: str, categories: list[RestoreCategory]
  550. ) -> tuple[dict, str]:
  551. """Fetch and parse just the files the requested categories need."""
  552. backend = get_provider_backend(config.provider)
  553. client = await self._get_client()
  554. self._progress = "Listing backup contents..."
  555. tree = await backend.list_tree(
  556. repo_url=config.repository_url, token=config.access_token, ref=ref, client=client
  557. )
  558. if not tree.get("success"):
  559. return {}, tree.get("message") or "Could not list the commit"
  560. available: list[str] = tree.get("paths") or []
  561. wanted: list[str] = []
  562. for category in categories:
  563. wanted.extend(self._category_paths(category, available))
  564. if not wanted:
  565. return {}, "None of the selected categories are present in that commit"
  566. self._progress = "Downloading backup files..."
  567. fetched = await backend.fetch_files(
  568. repo_url=config.repository_url, token=config.access_token, ref=ref, paths=wanted, client=client
  569. )
  570. if not fetched.get("success"):
  571. return {}, fetched.get("message") or "Could not read the commit contents"
  572. parsed, bad = self._parse_json_files(fetched.get("files") or {})
  573. if bad:
  574. return {}, f"Backup contains unreadable JSON: {', '.join(sorted(bad))}"
  575. return parsed, ""
  576. async def _apply(
  577. self,
  578. db: AsyncSession,
  579. payload: dict,
  580. categories: list[RestoreCategory],
  581. overwrite: bool,
  582. settings_keys_written: set[str] | None = None,
  583. ) -> dict[str, _CategoryTally]:
  584. """Apply categories in dependency order and return per-category tallies.
  585. ``settings_keys_written``, if given, collects the setting keys actually
  586. written, for the caller's post-commit side effects (see
  587. ``_reconfigure_mqtt_relay``).
  588. """
  589. results: dict[str, _CategoryTally] = {}
  590. archive_id_map: dict[int, int] = {}
  591. # Archives first: spool usage history references archive_id.
  592. if RestoreCategory.ARCHIVES in categories:
  593. self._progress = "Restoring print archives..."
  594. tally = _CategoryTally()
  595. await self._restore_archives(db, payload.get(ARCHIVES_PATH), overwrite, tally, archive_id_map)
  596. results[RestoreCategory.ARCHIVES.value] = tally
  597. if RestoreCategory.SPOOLS in categories:
  598. self._progress = "Restoring spool inventory..."
  599. tally = _CategoryTally()
  600. await self._restore_spools(
  601. db,
  602. payload.get(SPOOLS_PATH),
  603. payload.get(SPOOL_USAGE_PATH),
  604. overwrite,
  605. tally,
  606. archive_id_map,
  607. )
  608. results[RestoreCategory.SPOOLS.value] = tally
  609. if RestoreCategory.SETTINGS in categories:
  610. self._progress = "Restoring app settings..."
  611. tally = _CategoryTally()
  612. await self._restore_settings(
  613. db, payload.get(SETTINGS_PATH), overwrite, tally, keys_written=settings_keys_written
  614. )
  615. results[RestoreCategory.SETTINGS.value] = tally
  616. # Last, because it leaves the database and publishes over MQTT.
  617. if RestoreCategory.KPROFILES in categories:
  618. # Commit the database categories FIRST, and not just for tidiness.
  619. # Everything above has already autoflushed its INSERTs, so SQLite is
  620. # holding the single write transaction — and _restore_kprofiles then
  621. # awaits get_kprofiles per printer per nozzle, which is
  622. # timeout=5.0 * max_retries=3, i.e. up to ~15 s each against an
  623. # unresponsive printer. busy_timeout is 15 s (core/database.py), so a
  624. # farm with a couple of sulking printers would hold the writer past
  625. # it and every concurrent writer in the app would fail with
  626. # "database is locked".
  627. #
  628. # The cost is that a K-profile failure no longer rolls back the
  629. # categories that already succeeded. That is the correct trade
  630. # anyway: extrusion_cali_set has left for the printer by then and
  631. # cannot be rolled back either, so a rollback would only have made
  632. # the database disagree with the hardware.
  633. await db.commit()
  634. self._progress = "Sending K-profiles to printers..."
  635. tally = _CategoryTally()
  636. await self._restore_kprofiles(db, payload, tally)
  637. results[RestoreCategory.KPROFILES.value] = tally
  638. return results
  639. # --- Per-category appliers --------------------------------------------
  640. async def _restore_archives(
  641. self,
  642. db: AsyncSession,
  643. payload,
  644. overwrite: bool,
  645. tally: _CategoryTally,
  646. id_map: dict[int, int],
  647. ) -> None:
  648. archives = payload.get("archives") if isinstance(payload, dict) else None
  649. if not isinstance(archives, list):
  650. tally.note("No archive data in this backup")
  651. return
  652. valid_printers = set((await db.execute(select(Printer.id))).scalars().all())
  653. valid_projects = set((await db.execute(select(Project.id))).scalars().all())
  654. # Ownership decides visibility, not just attribution: an archive with a
  655. # NULL created_by_id is a 404 to every caller without archives:read_all
  656. # (_ensure_archive_visible fails closed on it) and never appears in the
  657. # ownership-scoped list queries. Hoisted like the two above.
  658. #
  659. # Note this is the one place a raw backup id is reused, against the
  660. # module's own rule at the top of the file. Users have no natural key the
  661. # backup carries today, and the id is validated rather than trusted, so a
  662. # *stale* id clears instead of pointing somewhere wrong. What it cannot
  663. # catch is a live id belonging to a different person on a different
  664. # instance. Collecting username and resolving on that would close it;
  665. # raised with the maintainer rather than decided here.
  666. valid_users = set((await db.execute(select(User.id))).scalars().all())
  667. # Only metadata is backed up, never the 3MF/thumbnail bytes, and
  668. # print_archives.file_path is NOT NULL — so inserted rows get an empty
  669. # path and are history-only. Say so once rather than per row.
  670. warned_files = False
  671. for entry in archives:
  672. if not isinstance(entry, dict):
  673. tally.failed += 1
  674. continue
  675. old_id = entry.get("id") if isinstance(entry.get("id"), int) else None
  676. started_at = _parse_dt(entry.get("started_at"))
  677. existing = await self._find_archive(db, entry, started_at)
  678. fields = {
  679. "print_name": entry.get("print_name"),
  680. "print_time_seconds": entry.get("print_time_seconds"),
  681. "filament_used_grams": entry.get("filament_used_grams"),
  682. "filament_type": entry.get("filament_type"),
  683. "filament_color": entry.get("filament_color"),
  684. "layer_height": entry.get("layer_height"),
  685. "total_layers": entry.get("total_layers"),
  686. "nozzle_diameter": entry.get("nozzle_diameter"),
  687. "bed_temperature": entry.get("bed_temperature"),
  688. "nozzle_temperature": entry.get("nozzle_temperature"),
  689. "sliced_for_model": entry.get("sliced_for_model"),
  690. "status": entry.get("status") or "completed",
  691. "started_at": started_at,
  692. "completed_at": _parse_dt(entry.get("completed_at")),
  693. "makerworld_url": entry.get("makerworld_url"),
  694. "designer": entry.get("designer"),
  695. "external_url": entry.get("external_url"),
  696. "is_favorite": bool(entry.get("is_favorite")),
  697. "tags": entry.get("tags"),
  698. "notes": entry.get("notes"),
  699. "cost": entry.get("cost"),
  700. "failure_reason": entry.get("failure_reason"),
  701. "quantity": entry.get("quantity") or 1,
  702. "energy_kwh": entry.get("energy_kwh"),
  703. "energy_cost": entry.get("energy_cost"),
  704. # A soft-deleted archive is still in the backup (its row is kept
  705. # so stats keep counting it), so carry the flag across or the
  706. # restore turns something the user deleted back into a visible
  707. # archive. Backups written before this key existed have no
  708. # deleted_at, and those rows can only come back live.
  709. "deleted_at": _parse_dt(entry.get("deleted_at")),
  710. }
  711. printer_id = entry.get("printer_id")
  712. if printer_id is not None and printer_id not in valid_printers:
  713. tally.note("Some archives referenced printers that no longer exist — link cleared")
  714. printer_id = None
  715. project_id = entry.get("project_id")
  716. if project_id is not None and project_id not in valid_projects:
  717. tally.note("Some archives referenced projects that no longer exist — link cleared")
  718. project_id = None
  719. created_by_id = entry.get("created_by_id")
  720. if created_by_id is not None and created_by_id not in valid_users:
  721. # Coerced rather than failing the row: the archive is still worth
  722. # having, and an admin can reassign it. Said out loud because a
  723. # cleared owner is not silent-safe — the archive becomes visible
  724. # only to archives:read_all until someone does.
  725. tally.note(
  726. "Some archives referenced users that no longer exist — owner cleared, so they are "
  727. "visible only to users with the archives:read_all permission until an admin reassigns them"
  728. )
  729. created_by_id = None
  730. fields["printer_id"] = printer_id
  731. fields["project_id"] = project_id
  732. fields["created_by_id"] = created_by_id
  733. if existing is not None:
  734. if old_id is not None:
  735. id_map[old_id] = existing.id
  736. if not overwrite:
  737. tally.skipped += 1
  738. continue
  739. # Overwrite means "make the local row match the backup", which
  740. # includes un-deleting one the user deleted after the backup was
  741. # taken. Legitimate, but not obvious from a restored/skipped
  742. # count, so say it.
  743. if existing.deleted_at is not None and fields["deleted_at"] is None:
  744. tally.note("Archive(s) deleted since the backup are visible again — overwrite was on")
  745. for key, value in fields.items():
  746. setattr(existing, key, value)
  747. tally.restored += 1
  748. continue
  749. if not warned_files:
  750. tally.note(
  751. "Restored archives carry metadata only — the 3MF and thumbnail files are not in a Git backup"
  752. )
  753. warned_files = True
  754. row = PrintArchive(
  755. filename=entry.get("filename") or "restored-from-backup",
  756. file_path="",
  757. file_size=entry.get("file_size") or 0,
  758. content_hash=entry.get("content_hash"),
  759. **fields,
  760. )
  761. created_at = _parse_dt(entry.get("created_at"))
  762. if created_at is not None:
  763. row.created_at = created_at
  764. db.add(row)
  765. await db.flush()
  766. if old_id is not None:
  767. id_map[old_id] = row.id
  768. tally.restored += 1
  769. async def _find_archive(self, db: AsyncSession, entry: dict, started_at: datetime | None) -> PrintArchive | None:
  770. """Match a backed-up archive to a local row by natural key.
  771. ``started_at`` is nullable and genuinely NULL for a whole class of rows —
  772. the re-slice path in ``library.py`` constructs ``PrintArchive`` without
  773. one — so it cannot be *required* by the key. It narrows the match instead:
  774. a backed-up row with no ``started_at`` matches a local row that has none
  775. either. Requiring it meant those archives never matched, so each restore
  776. re-inserted them as duplicates and overwrite mode could never update them.
  777. ``content_hash`` identifies the sliced file on its own, which is why it is
  778. the branch allowed to run without a ``started_at``; ``filename`` is too
  779. weak for that (re-slices share it) and still requires one. Two backed-up
  780. rows sharing a hash *and* having no ``started_at`` are indistinguishable
  781. in the backup, so they collapse onto one local row — better than
  782. duplicating both on every restore.
  783. Soft-deleted rows are matched deliberately: there is no ``deleted_at``
  784. filter here because the row still exists, and matching it is what stops a
  785. restore inserting a live duplicate of an archive the user has deleted.
  786. """
  787. started_predicate = PrintArchive.started_at == started_at if started_at else PrintArchive.started_at.is_(None)
  788. content_hash = entry.get("content_hash")
  789. if content_hash:
  790. result = await db.execute(
  791. select(PrintArchive).where(PrintArchive.content_hash == content_hash, started_predicate)
  792. )
  793. row = result.scalars().first()
  794. if row is not None:
  795. return row
  796. filename = entry.get("filename")
  797. if filename and started_at:
  798. result = await db.execute(select(PrintArchive).where(PrintArchive.filename == filename, started_predicate))
  799. return result.scalars().first()
  800. return None
  801. async def _restore_spools(
  802. self,
  803. db: AsyncSession,
  804. inventory,
  805. usage_payload,
  806. overwrite: bool,
  807. tally: _CategoryTally,
  808. archive_id_map: dict[int, int],
  809. ) -> None:
  810. spools = inventory.get("spools") if isinstance(inventory, dict) else None
  811. if not isinstance(spools, list):
  812. tally.note("No spool data in this backup")
  813. return
  814. spool_id_map: dict[int, int] = {}
  815. for entry in spools:
  816. if not isinstance(entry, dict):
  817. tally.failed += 1
  818. continue
  819. old_id = entry.get("id") if isinstance(entry.get("id"), int) else None
  820. existing = await self._find_spool(db, entry)
  821. fields = {
  822. "material": entry.get("material") or "PLA",
  823. "subtype": entry.get("subtype"),
  824. "color_name": entry.get("color_name"),
  825. "rgba": entry.get("rgba"),
  826. "brand": entry.get("brand"),
  827. "label_weight": entry.get("label_weight") or 1000,
  828. "core_weight": entry.get("core_weight") or 250,
  829. "weight_used": entry.get("weight_used") or 0,
  830. "weight_locked": bool(entry.get("weight_locked")),
  831. "slicer_filament": entry.get("slicer_filament"),
  832. "slicer_filament_name": entry.get("slicer_filament_name"),
  833. "nozzle_temp_min": entry.get("nozzle_temp_min"),
  834. "nozzle_temp_max": entry.get("nozzle_temp_max"),
  835. "note": entry.get("note"),
  836. "cost_per_kg": entry.get("cost_per_kg"),
  837. "tag_uid": entry.get("tag_uid"),
  838. "tray_uuid": entry.get("tray_uuid"),
  839. "data_origin": entry.get("data_origin"),
  840. "tag_type": entry.get("tag_type"),
  841. "archived_at": _parse_dt(entry.get("archived_at")),
  842. }
  843. if existing is not None:
  844. if old_id is not None:
  845. spool_id_map[old_id] = existing.id
  846. if not overwrite:
  847. tally.skipped += 1
  848. continue
  849. for key, value in fields.items():
  850. setattr(existing, key, value)
  851. tally.restored += 1
  852. continue
  853. row = Spool(**fields)
  854. # Carry the original created_at across. Without it the row would be
  855. # stamped "now", and the composite fallback in _find_spool (which
  856. # keys on created_at) would miss on a second restore and insert a
  857. # duplicate instead of matching.
  858. created_at = _parse_dt(entry.get("created_at"))
  859. if created_at is not None:
  860. row.created_at = created_at
  861. db.add(row)
  862. await db.flush()
  863. if old_id is not None:
  864. spool_id_map[old_id] = row.id
  865. tally.restored += 1
  866. await self._restore_spool_usage(db, usage_payload, tally, spool_id_map, archive_id_map)
  867. async def _find_spool(self, db: AsyncSession, entry: dict) -> Spool | None:
  868. """Match a backed-up spool to a local row.
  869. Physical identity first (an RFID/Bambu tag is the spool), then a
  870. descriptive composite including ``created_at`` so two otherwise
  871. identical spools added at different times stay distinct.
  872. """
  873. tag_uid = entry.get("tag_uid")
  874. if tag_uid:
  875. result = await db.execute(select(Spool).where(Spool.tag_uid == tag_uid))
  876. row = result.scalars().first()
  877. if row is not None:
  878. return row
  879. tray_uuid = entry.get("tray_uuid")
  880. if tray_uuid:
  881. result = await db.execute(select(Spool).where(Spool.tray_uuid == tray_uuid))
  882. row = result.scalars().first()
  883. if row is not None:
  884. return row
  885. created_at = _parse_dt(entry.get("created_at"))
  886. if created_at is None:
  887. return None
  888. result = await db.execute(
  889. select(Spool).where(
  890. Spool.created_at == created_at,
  891. Spool.material == (entry.get("material") or "PLA"),
  892. Spool.brand == entry.get("brand"),
  893. Spool.subtype == entry.get("subtype"),
  894. Spool.color_name == entry.get("color_name"),
  895. )
  896. )
  897. return result.scalars().first()
  898. async def _restore_spool_usage(
  899. self,
  900. db: AsyncSession,
  901. usage_payload,
  902. tally: _CategoryTally,
  903. spool_id_map: dict[int, int],
  904. archive_id_map: dict[int, int],
  905. ) -> None:
  906. usage = usage_payload.get("usage_history") if isinstance(usage_payload, dict) else None
  907. if not isinstance(usage, list) or not usage:
  908. return
  909. valid_printers = set((await db.execute(select(Printer.id))).scalars().all())
  910. unresolved = 0
  911. unlinked_archives = 0
  912. for entry in usage:
  913. if not isinstance(entry, dict):
  914. tally.failed += 1
  915. continue
  916. old_spool_id = entry.get("spool_id")
  917. spool_id = spool_id_map.get(old_spool_id) if isinstance(old_spool_id, int) else None
  918. if spool_id is None:
  919. # The parent spool never made it into the map: the backup's spool
  920. # list didn't include it, or its entry carried no integer id. A
  921. # spool that was merely *skipped* (matched locally, overwrite off)
  922. # is mapped a few lines up in _restore_spools, so it never lands
  923. # here — which is why the note below offers no remedy.
  924. unresolved += 1
  925. tally.skipped += 1
  926. continue
  927. created_at = _parse_dt(entry.get("created_at"))
  928. # Usage history has no natural key of its own, so dedupe on the
  929. # tuple that makes a consumption event unique in practice.
  930. existing = await db.execute(
  931. select(SpoolUsageHistory).where(
  932. SpoolUsageHistory.spool_id == spool_id,
  933. SpoolUsageHistory.created_at == created_at,
  934. SpoolUsageHistory.weight_used == (entry.get("weight_used") or 0),
  935. SpoolUsageHistory.print_name == entry.get("print_name"),
  936. )
  937. )
  938. if existing.scalars().first() is not None:
  939. tally.skipped += 1
  940. continue
  941. printer_id = entry.get("printer_id")
  942. if printer_id is not None and printer_id not in valid_printers:
  943. printer_id = None
  944. old_archive_id = entry.get("archive_id")
  945. archive_id = archive_id_map.get(old_archive_id) if isinstance(old_archive_id, int) else None
  946. if archive_id is None and isinstance(old_archive_id, int):
  947. # Restoring spools without archives leaves archive_id_map empty,
  948. # so every "this print consumed that spool" link is dropped — the
  949. # local archive may well exist, but its payload wasn't fetched,
  950. # so there is no natural key here to match it on. Nor is it
  951. # repairable by a later archives-only restore: the dedupe key
  952. # above doesn't include archive_id, so these rows are recognised
  953. # as already-present and skipped. Worth telling the user while
  954. # they can still redo the run with both categories ticked.
  955. unlinked_archives += 1
  956. row = SpoolUsageHistory(
  957. spool_id=spool_id,
  958. printer_id=printer_id,
  959. print_name=entry.get("print_name"),
  960. archive_id=archive_id,
  961. weight_used=entry.get("weight_used") or 0,
  962. percent_used=entry.get("percent_used") or 0,
  963. status=entry.get("status") or "completed",
  964. cost=entry.get("cost"),
  965. )
  966. if created_at is not None:
  967. row.created_at = created_at
  968. db.add(row)
  969. tally.restored += 1
  970. if unresolved:
  971. tally.note(
  972. f"{unresolved} usage record(s) skipped — their spool is not in this backup's "
  973. "spool list, so there is nothing to attach them to."
  974. )
  975. if unlinked_archives:
  976. tally.note(
  977. f"{unlinked_archives} usage record(s) restored without their print-history link — "
  978. "select Print archives alongside Spool inventory to keep it."
  979. )
  980. async def _restore_settings(
  981. self,
  982. db: AsyncSession,
  983. payload,
  984. overwrite: bool,
  985. tally: _CategoryTally,
  986. keys_written: set[str] | None = None,
  987. ) -> None:
  988. values = payload.get("settings") if isinstance(payload, dict) else None
  989. if not isinstance(values, dict):
  990. tally.note("No settings data in this backup")
  991. return
  992. # Planned before the first write, so the companion rule reads genuinely
  993. # pre-restore local state, and so the preview and this run classify the
  994. # payload identically.
  995. plan = await self._plan_settings(db, values)
  996. refused = plan.refused
  997. for key, value in values.items():
  998. if not isinstance(key, str) or not key:
  999. tally.failed += 1
  1000. continue
  1001. if key in refused:
  1002. # Refusals are reported in the notes and nowhere else. They are
  1003. # already outside the preview's item count, and the preview is
  1004. # the number the user was shown, so counting them here would
  1005. # make restored + skipped + failed exceed it. The two skips
  1006. # below stay counted because they depend on this run's flags,
  1007. # which the preview cannot see.
  1008. continue
  1009. if value is None:
  1010. tally.skipped += 1
  1011. continue
  1012. result = await db.execute(select(Settings).where(Settings.key == key))
  1013. existing = result.scalar_one_or_none()
  1014. if existing is not None:
  1015. if not overwrite:
  1016. tally.skipped += 1
  1017. continue
  1018. existing.value = str(value)
  1019. tally.restored += 1
  1020. if keys_written is not None:
  1021. keys_written.add(key)
  1022. continue
  1023. db.add(Settings(key=key, value=str(value)))
  1024. tally.restored += 1
  1025. if keys_written is not None:
  1026. keys_written.add(key)
  1027. if plan.blocked:
  1028. tally.note(f"{len(plan.blocked)} credential-like key(s) skipped — re-enter secrets manually")
  1029. if plan.protected:
  1030. tally.note(
  1031. f"{len(plan.protected)} authentication setting(s) skipped — change those in Settings > "
  1032. "Authentication so the lockout checks still run"
  1033. )
  1034. if plan.companion:
  1035. tally.note(
  1036. f"{', '.join(sorted(plan.companion))} left switched off — the credential each one needs "
  1037. "cannot be restored from a backup and this instance has none stored, so switching them "
  1038. "on would leave the integration unauthenticated"
  1039. )
  1040. async def _reconfigure_mqtt_relay(self, db: AsyncSession, keys_written: set[str], tally: _CategoryTally) -> None:
  1041. """Push restored mqtt_* settings into the live relay.
  1042. The relay reads its broker config once, at configure() time — the
  1043. settings PUT handler reconfigures it for exactly this reason
  1044. (api/routes/settings.py). Writing the rows alone left the relay on the
  1045. pre-restore broker until the next backend restart while the UI showed
  1046. the restored values, which is the one way a restore could look applied
  1047. and not be.
  1048. Called after the commit, never before: configure() tears the connection
  1049. down and rebuilds it, so it must not run against values a later failure
  1050. could roll back. Only mqtt_password can't come back this way (the
  1051. credential blocklist skips it) — the row already in the database is
  1052. reused, so an unchanged broker keeps working.
  1053. """
  1054. if not _MQTT_SETTING_KEYS & keys_written:
  1055. return
  1056. try:
  1057. from backend.app.services.mqtt_relay import mqtt_relay
  1058. rows = await db.execute(select(Settings).where(Settings.key.in_(_MQTT_SETTING_KEYS)))
  1059. stored = {s.key: s.value for s in rows.scalars().all()}
  1060. # Same shape and defaults the settings PUT handler builds.
  1061. await mqtt_relay.configure(
  1062. {
  1063. "mqtt_enabled": (stored.get("mqtt_enabled") or "false") == "true",
  1064. "mqtt_broker": stored.get("mqtt_broker") or "",
  1065. "mqtt_port": int(stored.get("mqtt_port") or "1883"),
  1066. "mqtt_username": stored.get("mqtt_username") or "",
  1067. "mqtt_password": stored.get("mqtt_password") or "",
  1068. "mqtt_topic_prefix": stored.get("mqtt_topic_prefix") or "bambuddy",
  1069. "mqtt_use_tls": (stored.get("mqtt_use_tls") or "false") == "true",
  1070. }
  1071. )
  1072. except Exception:
  1073. # Same call is best-effort in the settings PUT handler: the rows are
  1074. # committed either way, and a broker that refuses the new config
  1075. # must not turn a successful restore into a failed one. Noted rather
  1076. # than swallowed silently, so the user knows to restart.
  1077. logger.warning("Could not reconfigure the MQTT relay after a settings restore", exc_info=True)
  1078. tally.note("MQTT settings restored, but the relay could not be reconnected — restart Bambuddy")
  1079. async def _restore_kprofiles(self, db: AsyncSession, payload: dict, tally: _CategoryTally) -> None:
  1080. by_serial: dict[str, list[tuple[str, dict]]] = {}
  1081. for path, content in payload.items():
  1082. match = _KPROFILE_PATH_RE.match(path)
  1083. if not match or not isinstance(content, dict):
  1084. continue
  1085. by_serial.setdefault(match.group(1), []).append((match.group(2), content))
  1086. if not by_serial:
  1087. tally.note("No K-profile data in this backup")
  1088. return
  1089. result = await db.execute(select(Printer))
  1090. printers = {p.serial_number: p for p in result.scalars().all() if p.serial_number}
  1091. # Overwrite is not offered for K-profiles: extrusion_cali_set replaces
  1092. # the profile occupying a slot, so writing is always an overwrite on the
  1093. # printer side.
  1094. tally.note("K-profiles always overwrite the matching slot on the printer")
  1095. tally.note("The printer's acknowledgement is not reliable — verify the profiles on the printer")
  1096. for serial, entries in sorted(by_serial.items()):
  1097. profile_total = sum(len(c.get("profiles") or []) for _, c in entries)
  1098. printer = printers.get(serial)
  1099. if printer is None:
  1100. tally.skipped += profile_total
  1101. tally.note(f"No printer with serial {serial} — skipped")
  1102. continue
  1103. client = printer_manager.get_client(printer.id)
  1104. if not client or not client.state.connected:
  1105. tally.skipped += profile_total
  1106. tally.note(f"{printer.name} ({serial}) is not connected — skipped")
  1107. continue
  1108. for nozzle, content in sorted(entries):
  1109. profiles = content.get("profiles")
  1110. if not isinstance(profiles, list) or not profiles:
  1111. continue
  1112. if nozzle not in _KNOWN_NOZZLES:
  1113. tally.note(f"Unexpected nozzle diameter {nozzle} for {serial} — sent as-is")
  1114. # The backup's slot_id is a cali_idx, and cali_idx is as
  1115. # unstable as the autoincrement ids we already refuse to reuse
  1116. # for spools and archives: editing a profile in Bambuddy is a
  1117. # delete-then-add on a single-nozzle printer, which re-keys it.
  1118. # Addressing extrusion_cali_set at a slot that no longer exists
  1119. # is a silent no-op — the printer drops it and we would still
  1120. # report the profile restored. So resolve the live index first.
  1121. current = await self._current_kprofile_index(client, nozzle, serial)
  1122. profile_dicts = []
  1123. unmatched = 0
  1124. for p in profiles:
  1125. if not isinstance(p, dict):
  1126. continue
  1127. match = self._match_kprofile(p, current)
  1128. if match is None:
  1129. unmatched += 1
  1130. profile_dicts.append(
  1131. {
  1132. "filament_id": p.get("filament_id", ""),
  1133. "name": p.get("name", ""),
  1134. "k_value": p.get("k_value", "0.020000"),
  1135. "nozzle_id": p.get("nozzle_id"),
  1136. "extruder_id": p.get("extruder_id", 0),
  1137. # Prefer the live setting_id when we matched: it is
  1138. # what the printer currently associates with the slot.
  1139. "setting_id": (match.setting_id if match else None) or p.get("setting_id"),
  1140. # cali_idx -1 tells the printer to add a new profile
  1141. # rather than address a slot that isn't there.
  1142. "cali_idx": match.slot_id if match else -1,
  1143. # Only consulted for the generated-setting_id
  1144. # fallback; cali_idx above takes precedence.
  1145. "slot_id": 0,
  1146. }
  1147. )
  1148. if not profile_dicts:
  1149. continue
  1150. if unmatched:
  1151. tally.note(
  1152. f"{unmatched} profile(s) for {nozzle} had no counterpart on {printer.name} "
  1153. "— added as new profiles"
  1154. )
  1155. try:
  1156. sent = client.set_kprofiles_batch(profile_dicts, nozzle)
  1157. except Exception as e:
  1158. logger.warning("K-profile restore failed for %s nozzle %s: %s", serial, nozzle, e)
  1159. sent = False
  1160. if sent:
  1161. tally.restored += len(profile_dicts)
  1162. else:
  1163. tally.failed += len(profile_dicts)
  1164. tally.note(f"Failed to send {nozzle} profiles to {printer.name} ({serial})")
  1165. @staticmethod
  1166. async def _current_kprofile_index(client, nozzle: str, serial: str) -> list:
  1167. """Read the printer's live profiles for one nozzle.
  1168. Best-effort: a read failure degrades to "nothing matched", which makes
  1169. every profile an add rather than aborting the restore.
  1170. """
  1171. try:
  1172. return list(await client.get_kprofiles(nozzle_diameter=nozzle) or [])
  1173. except Exception as e:
  1174. logger.warning("Could not read live K-profiles for %s nozzle %s: %s", serial, nozzle, e)
  1175. return []
  1176. @staticmethod
  1177. def _match_kprofile(entry: dict, current: list):
  1178. """Find the live profile a backed-up entry corresponds to.
  1179. ``setting_id`` is the filament preset the profile was calibrated for and
  1180. is the strongest signal; a delete-then-add edit regenerates it, so fall
  1181. back to the display name, which Bambuddy's own editor preserves.
  1182. Both are scoped by ``filament_id`` — the same preset on a different
  1183. filament is a different profile.
  1184. """
  1185. filament_id = entry.get("filament_id")
  1186. if not filament_id:
  1187. return None
  1188. candidates = [c for c in current if c.filament_id == filament_id]
  1189. if not candidates:
  1190. return None
  1191. setting_id = entry.get("setting_id")
  1192. if setting_id:
  1193. for c in candidates:
  1194. if c.setting_id == setting_id:
  1195. return c
  1196. name = entry.get("name")
  1197. if name:
  1198. for c in candidates:
  1199. if c.name == name:
  1200. return c
  1201. # Exactly one profile for this filament and no better discriminator:
  1202. # treat it as the same profile rather than duplicating it.
  1203. return candidates[0] if len(candidates) == 1 else None
  1204. # Singleton instance
  1205. github_restore_service = GitHubRestoreService()