github_restore.py 129 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758
  1. """Restore Bambuddy data from a Git provider backup (issue #2656).
  2. The backup side (``github_backup.py``) is push-only: it collects a handful of
  3. JSON documents and commits them. This module is the read side — it walks the
  4. backup repository's history, lets a caller inspect what a given commit contains,
  5. and applies selected categories back into the local database (or, for
  6. K-profiles, back onto the printers).
  7. Design notes worth knowing before editing:
  8. * **A restore never reuses the backup's primary keys.** ``spool.id`` and
  9. ``print_archives.id`` are bare autoincrement columns, so the ids in a backup
  10. taken weeks ago very likely belong to unrelated rows today. Rows are matched
  11. on natural keys instead, inserted without an explicit id, and an
  12. ``old_id -> new_id`` map is threaded through so foreign keys in dependent
  13. tables (spool usage history) still line up.
  14. The printer-side ``cali_idx`` behaves the same way and gets the same
  15. treatment. Editing a K-profile in Bambuddy is a delete-then-add on a
  16. single-nozzle printer, which re-keys it, and ``extrusion_cali_set`` aimed at a
  17. slot that no longer exists is silently dropped — so the live index is read
  18. back and matched before writing, never taken from the backup.
  19. * **Categories are applied archives -> spools -> settings -> kprofiles.**
  20. Archives first because spool usage history references ``archive_id``;
  21. K-profiles last because they leave the database and talk to hardware.
  22. * **Cloud profiles are not restorable.** Restoring a preset means writing to a
  23. Bambu or Orca Cloud account, which is a different operation from everything
  24. else here — every other category lands in the local database or, for
  25. K-profiles, on a printer the instance already owns. Tracked separately from
  26. #2656. (The collector does write ``cloud_profiles/*.json`` as of #2717; the
  27. earlier claim that it did not is no longer true.)
  28. """
  29. import asyncio
  30. import json
  31. import logging
  32. import math
  33. import os
  34. import re
  35. from collections.abc import Iterable
  36. from contextvars import ContextVar
  37. from dataclasses import dataclass, field as dataclasses_field
  38. from datetime import datetime, timezone
  39. import httpx
  40. from pydantic import BaseModel
  41. from sqlalchemy import select
  42. from sqlalchemy.ext.asyncio import AsyncSession
  43. from backend.app.api.routes._spoolman_helpers import assert_safe_spoolman_url
  44. from backend.app.core.database import async_session
  45. from backend.app.core.printer_scope import drop_printers_from_obico
  46. from backend.app.models.archive import PrintArchive
  47. from backend.app.models.archive_alias import ArchiveAlias
  48. from backend.app.models.github_backup import GitHubBackupConfig, GitHubBackupLog
  49. from backend.app.models.print_log import INCLUDE_OPEN_RUNS, RUN_OPEN, PrintLogEntry
  50. from backend.app.models.printer import Printer
  51. from backend.app.models.project import Project
  52. from backend.app.models.settings import Settings
  53. from backend.app.models.spool import Spool
  54. from backend.app.models.spool_usage_history import SpoolUsageHistory
  55. from backend.app.models.user import User
  56. from backend.app.schemas.auth import SMTPSettings
  57. from backend.app.schemas.github_backup import RestoreCategory
  58. from backend.app.schemas.settings import ADMIN_ONLY_SETTING_KEYS, AppSettings, AppSettingsUpdate
  59. from backend.app.services.git_providers.factory import get_provider_backend
  60. from backend.app.services.model_providers.manyfold.config import normalize_url
  61. from backend.app.services.printer_manager import printer_manager
  62. from backend.app.services.secret_fields import SETTINGS_SECRETS, SMTP_DESTINATIONS, is_secret_setting_key
  63. logger = logging.getLogger(__name__)
  64. METADATA_PATH = "backup_metadata.json"
  65. SETTINGS_PATH = "settings/app_settings.json"
  66. SPOOLS_PATH = "spools/inventory.json"
  67. SPOOL_USAGE_PATH = "spools/usage_history.json"
  68. ARCHIVES_PATH = "archives/print_history.json"
  69. # Each print of an archive and which archive went into which (archive
  70. # redesign P3). Backups written before carry neither.
  71. RUNS_PATH = "archives/print_runs.json"
  72. ALIASES_PATH = "archives/aliases.json"
  73. # kprofiles/{printer_serial}/{nozzle_diameter}.json
  74. _KPROFILE_PATH_RE = re.compile(r"^kprofiles/([^/]+)/([^/]+)\.json$")
  75. # Settings keys holding a credential are refused by name (is_secret_setting_key,
  76. # shared with the backup collector). A current backup carries them only as the
  77. # placeholder; one taken before that can still hold the values, and a restore
  78. # must not resurrect a stale credential. _COMPANION_CREDENTIALS sits downstream
  79. # of that: it withholds a toggle when the credential it needs was refused, so
  80. # narrowing the predicate would both write a stale credential and quietly make
  81. # that rule inert.
  82. # Settings the MQTT relay reads only when it is (re)configured, so restoring the
  83. # rows is not enough on its own. Mirrors the set the settings PUT handler
  84. # watches. mqtt_password is in here for the configure() payload's sake — the
  85. # credential blocklist means a restore never writes it.
  86. _MQTT_SETTING_KEYS = {
  87. "mqtt_enabled",
  88. "mqtt_broker",
  89. "mqtt_port",
  90. "mqtt_username",
  91. "mqtt_password",
  92. "mqtt_topic_prefix",
  93. "mqtt_use_tls",
  94. "panda_breath_enabled",
  95. }
  96. # Keys that decide *who can reach the instance* rather than how it behaves. The
  97. # backup collector writes them like any other Settings row, so a backup taken
  98. # before auth was turned on carries auth_enabled=false — and a restore reaches
  99. # the table directly, so honouring them would:
  100. #
  101. # * disable authentication outright. ``set_auth_enabled`` pairs its write with
  102. # ``invalidate_auth_enabled_cache()``; we cannot, so the 30 s TTL in
  103. # core.auth is the only thing between the write and an open instance. That
  104. # cache is built to fail closed — writing the stored value behind its back
  105. # is what would make it fail open.
  106. # * bypass the lockout refusals ``update_settings`` enforces (a
  107. # ``local_login_enabled=false`` with no enabled OIDC provider, or with no
  108. # OIDC link on the caller, is a 400 there — #1589).
  109. # * cross a permission boundary: a restore would be a way to rewrite auth
  110. # config without SETTINGS_UPDATE. (The endpoint gates each category on the
  111. # permission owning its rows now, but that is settings:update — still not
  112. # the auth UI's own guards, which is what these keys actually need.)
  113. #
  114. # Auth is reconfigured through the auth UI, which has the guards. Restoring it
  115. # from a snapshot has no safe reading.
  116. _PROTECTED_SETTING_KEYS = {
  117. "auth_enabled",
  118. "advanced_auth_enabled",
  119. "local_login_enabled",
  120. "setup_completed",
  121. }
  122. # The LDAP family, refused for the same reason and by prefix rather than by
  123. # name, so a key added to the schema later is refused by default.
  124. #
  125. # These are not "how the instance behaves" settings — together they name *which
  126. # directory server decides who you are*. auth.py reads them live from this table
  127. # on every login (see the ldap_keys list in _get_ldap_settings), so a restore
  128. # that writes them substitutes the authentication source wholesale:
  129. # ldap_server_url points at another directory, ldap_auto_provision creates a
  130. # local account for whoever it vouches for, and ldap_default_group decides what
  131. # that account gets — Administrators, if the backup says so.
  132. #
  133. # The companion rule does NOT cover this, which is the trap. ldap_enabled is
  134. # paired with ldap_bind_password there, but an *anonymous* bind is a working
  135. # config, so a backup that simply omits the password skips the refusal at the
  136. # _COMPANION_EXPOSURE_TOGGLES check and the toggle is written. Omitting a
  137. # credential is exactly what an attacker authoring this file would do — they own
  138. # the directory being pointed at, so they need no bind credential from us.
  139. _PROTECTED_SETTING_PREFIXES = ("ldap_",)
  140. # Restoring as a non-admin: the settings only an admin may change through the
  141. # settings page (ADMIN_ONLY_SETTING_KEYS) and the mail server's, which only an
  142. # admin may set (routes/auth.update_smtp_settings), are kept as they are here.
  143. # A restore is otherwise a way around those checks. Set by preview() and
  144. # run_restore() for the call; an admin's restore is unchanged.
  145. _restoring_as_admin: ContextVar[bool] = ContextVar("restoring_as_admin", default=True)
  146. def _admin_only_for_restore(key: str) -> bool:
  147. return key in ADMIN_ONLY_SETTING_KEYS or key in _EXTRA_RESTORABLE_KEYS
  148. # Nozzle diameters the backup collector iterates. A path outside this set means
  149. # the backup was written by a newer version, so accept it rather than dropping
  150. # data, but keep the list for validation messages.
  151. _KNOWN_NOZZLES = {"0.2", "0.4", "0.6", "0.8"}
  152. def _parse_dt(value) -> datetime | None:
  153. """Best-effort parse of a datetime the backup wrote via ``str(...)``.
  154. Normalised to naive UTC, because that is what every ``DateTime`` column
  155. here holds: the models write ``datetime.now(timezone.utc)`` into naive
  156. columns and both dialects drop the offset on the way in. Carrying an aware
  157. value through would store the wrong wall clock, and comparing one against a
  158. value read back out of a naive column raises ``TypeError``. The collector
  159. only ever writes naive strings, so this is a guard on hand-edited or
  160. foreign backups rather than a path Bambuddy takes itself.
  161. """
  162. if not value or not isinstance(value, str):
  163. return None
  164. try:
  165. parsed = datetime.fromisoformat(value)
  166. except ValueError:
  167. return None
  168. if parsed.tzinfo is not None:
  169. parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None)
  170. return parsed
  171. def _created_at_matches(row, created_at: datetime | None) -> bool:
  172. """Does ``row.created_at`` equal a timestamp read out of a backup?
  173. Compared in Python, not in SQL, and that is the whole point. Every
  174. ``created_at`` these callers dedupe on is ``server_default=func.now()``, so
  175. SQLite fills it from ``CURRENT_TIMESTAMP``, which has second precision and
  176. stores ``'2026-08-02 11:28:41'``. SQLAlchemy binds a Python datetime as
  177. ``'2026-08-02 11:28:41.000000'``, and SQLite compares the two as strings —
  178. so ``Model.created_at == created_at`` never matches a row the application
  179. itself created, not even when handed that row's own value straight back.
  180. Every dedupe keyed on it misses, and the restore inserts a duplicate of
  181. everything instead of recognising what is already there.
  182. Reading the candidates back and comparing the parsed datetimes sidesteps
  183. the bind format entirely, and is equally correct on PostgreSQL (where the
  184. column keeps microseconds and the SQL comparison happened to work).
  185. """
  186. return created_at is not None and row.created_at == created_at
  187. def _is_blocked_setting_key(key: str) -> bool:
  188. return is_secret_setting_key(key)
  189. def _is_protected_setting_key(key: str) -> bool:
  190. # Lowered for the prefix test for the same reason _is_blocked_setting_key
  191. # lowers: the key comes from the backup's JSON, not from our own writer, so
  192. # its casing is whatever the file says. An exact-match name stays exact —
  193. # those four are ours and are only ever written lowercase.
  194. return key in _PROTECTED_SETTING_KEYS or key.lower().startswith(_PROTECTED_SETTING_PREFIXES)
  195. # A user's email code sign-in switch. Per user, and it decides how that user
  196. # signs in: never taken from a backup, whatever its value.
  197. _TWO_FACTOR_KEY = re.compile(r"^user_\d+_email_2fa_enabled$")
  198. # Settings kept outside AppSettings that a restore writes, each checked as the
  199. # editor that owns it checks it: (model, field) for a field of that editor's
  200. # request model. Anything else outside AppSettings is runtime state, a login's
  201. # account state, automation that needs its own permission (the purge
  202. # schedules), a migration flag or unknown, and is not restored.
  203. _EXTRA_RESTORABLE_KEYS: dict[str, tuple[type[BaseModel], str]] = {
  204. key: (SMTPSettings, key)
  205. for key in (
  206. "smtp_host",
  207. "smtp_port",
  208. "smtp_security",
  209. "smtp_auth_enabled",
  210. "smtp_from_email",
  211. "smtp_from_name",
  212. "smtp_username",
  213. "smtp_use_tls",
  214. )
  215. }
  216. # Settings that decide where Bambuddy sends something: the links in its emails
  217. # (password reset and welcome links carry a sign-in token), the mail server
  218. # itself, the full database backups, printer status, camera snapshots, model
  219. # files, inventory. A backup naming another address would send it there, so a
  220. # value that differs from this instance's is kept as it is here.
  221. _ADDRESS_KEYS = (
  222. "external_url",
  223. "smtp_host",
  224. "smtp_port",
  225. "smtp_security",
  226. "smtp_use_tls",
  227. "local_backup_path",
  228. "mqtt_broker",
  229. "mqtt_port",
  230. "mqtt_use_tls",
  231. "ha_url",
  232. "obico_ml_url",
  233. "bambuddy_internal_url",
  234. "orcaslicer_api_url",
  235. "bambu_studio_api_url",
  236. "spoolman_url",
  237. "manyfold_url",
  238. )
  239. # Restored without a model check: the Manyfold editor checks the URL with
  240. # normalize_url and takes the client id as text.
  241. _MANYFOLD_URL_KEY = "manyfold_url"
  242. _MANYFOLD_CLIENT_ID_KEY = "manyfold_client_id"
  243. # The settings whose saved null (the text "None") the settings reader reads as
  244. # null. Anywhere else it is refused, as the editor refuses a null.
  245. _READS_A_SAVED_NULL = frozenset({"ams_temp_alarm", "default_printer_id", "open_in_slicer"})
  246. # Settings that no longer exist. Older backups still hold them; a restore drops
  247. # them without listing them as skipped.
  248. _RETIRED_SETTING_KEYS = frozenset({"library_archive_mode"})
  249. def is_restorable_setting_key(key: str) -> bool:
  250. """Whether a restore writes this settings key at all (with a value its
  251. editor accepts). The Git backup holds only these, so nothing a restore
  252. would refuse as unknown leaves the instance."""
  253. return (
  254. key in AppSettingsUpdate.model_fields
  255. or key in _EXTRA_RESTORABLE_KEYS
  256. or key in (_MANYFOLD_URL_KEY, _MANYFOLD_CLIENT_ID_KEY)
  257. )
  258. def _stored_form(value: object) -> str:
  259. """A checked value as its editor stores it."""
  260. if isinstance(value, bool):
  261. return "true" if value else "false"
  262. if value is None:
  263. return "None"
  264. return str(value)
  265. def _checked_setting(key: str, value: object) -> tuple[bool | None, str]:
  266. """A restored value checked as its editor checks it.
  267. ``(True, stored form)`` if the editor would accept it, ``(False, "")`` if
  268. not, ``(None, "")`` if no editor writes the key, so it is not restored.
  269. The stored form is what the editor would write: a value the check only
  270. coerced ("587.0", "yes") is written as the editor writes it.
  271. """
  272. candidate = None if value == "None" and key in _READS_A_SAVED_NULL else value
  273. try:
  274. if key in AppSettingsUpdate.model_fields:
  275. checked = getattr(AppSettingsUpdate.model_validate({key: candidate}), key)
  276. # Checked where it is used rather than where it is saved: checked
  277. # here as well, before it is written.
  278. if key == "spoolman_url" and checked:
  279. assert_safe_spoolman_url(str(checked))
  280. return True, _stored_form(checked)
  281. if key in _EXTRA_RESTORABLE_KEYS:
  282. model, field = _EXTRA_RESTORABLE_KEYS[key]
  283. instance = model.__pydantic_validator__.validate_assignment(model.model_construct(), field, candidate)
  284. return True, _stored_form(getattr(instance, field))
  285. if key == _MANYFOLD_URL_KEY:
  286. return True, normalize_url(str(candidate)) if candidate else ""
  287. except ValueError:
  288. return False, ""
  289. if key == _MANYFOLD_CLIENT_ID_KEY:
  290. return True, str(value)
  291. return None, ""
  292. # There used to be an ``_is_skipped_setting_key`` here, the union of the two
  293. # predicates above, shared by the preview and the restore so neither could drift
  294. # from the other. It is gone because a name is no longer enough to decide: the
  295. # third refusal below depends on the payload's *other* values and on local
  296. # database state. ``_plan_settings`` is the shared classifier now, and it covers
  297. # all three reasons.
  298. # Toggles whose *safety* depends on a companion credential that the blocklist
  299. # above refuses to restore. Writing the toggle alone is not a partial restore,
  300. # it is a downgrade:
  301. #
  302. # * prometheus_enabled with no token opens /api/v1/metrics. The route is on
  303. # PUBLIC_API_ROUTES and its own gate is ``if token:`` (api/routes/metrics.py),
  304. # so an empty or absent token means no authentication at all — a full,
  305. # unauthenticated dump of the instance to anyone who can reach the port. On
  306. # an instance that never enabled Prometheus there is no token row, so
  307. # overwrite-off alone is enough to do it.
  308. # * the other four switch an integration on with no way to authenticate to it,
  309. # which breaks the login path (LDAP) or the connection (MQTT, HA).
  310. #
  311. # ldap_enabled is deliberately NOT here. It was, paired with
  312. # ldap_bind_password — but this rule judges availability ("will the integration
  313. # work?"), and that is the wrong question for an authentication source. An
  314. # anonymous bind is a working config, so the pair let a backup omit the password
  315. # and have the toggle written; the whole LDAP family is refused by prefix above
  316. # instead. _is_protected_setting_key runs first in _plan_settings, so leaving the
  317. # entry here would be dead code that reads like coverage.
  318. _COMPANION_CREDENTIALS = {
  319. "prometheus_enabled": "prometheus_token",
  320. "mqtt_enabled": "mqtt_password",
  321. "ha_enabled": "ha_token",
  322. }
  323. # Companion credentials a reader takes from the environment rather than from a
  324. # Settings row. ha_token is the only one: get_homeassistant_settings prefers
  325. # HA_TOKEN over the row, and auto-enables ha_enabled when HA_URL and HA_TOKEN are
  326. # both set, so an env-configured instance has a usable credential and no row.
  327. _COMPANION_CREDENTIAL_ENV = {"ha_token": "HA_TOKEN"}
  328. # The pairs above divide into two classes, because "did the *backup* carry a
  329. # usable credential?" does not mean the same thing for both.
  330. #
  331. # For the availability pairs it is the condition that stops the rule
  332. # over-refusing. An anonymous MQTT broker and an anonymous LDAP bind are working
  333. # configs, so a backup with an empty credential is describing something that
  334. # works, and refusing its toggle would be a false positive. Those pairs only
  335. # matter when the restore would produce a config weaker than *both* the backup
  336. # and the local instance.
  337. #
  338. # For the exposure pair it does not transfer. An empty prometheus_token removes
  339. # /api/v1/metrics' only gate (the route is on PUBLIC_API_ROUTES and its own
  340. # check is ``if token:``), so the exposure is a property of the toggle itself,
  341. # not of a downgrade relative to the backup: a backup taken on an instance that
  342. # enabled Prometheus *without* a token — the field is optional and defaults to
  343. # "" — is the more likely source of one, not the less. So an exposure toggle
  344. # skips this condition and is judged on local state alone.
  345. _COMPANION_EXPOSURE_TOGGLES = frozenset({"prometheus_enabled"})
  346. # Setting -> the credentials it decides the destination of. smtp_use_tls
  347. # decides the SMTP security mode when smtp_security is not set.
  348. _DESTINATION_SECRETS: dict[str, tuple[str, ...]] = {}
  349. for _secret, _fields in {
  350. **SETTINGS_SECRETS,
  351. "smtp_password": (*SMTP_DESTINATIONS, "smtp_use_tls"),
  352. "manyfold_client_secret": ("manyfold_url",),
  353. }.items():
  354. for _field in _fields:
  355. _DESTINATION_SECRETS[_field] = (*_DESTINATION_SECRETS.get(_field, ()), _secret)
  356. def _setting_value_is_true(value: object) -> bool:
  357. """True if a settings *payload* value would be stored as "on".
  358. Deliberately as narrow as ``api.routes.settings.setting_is_true``: a restore
  359. writes ``str(value)`` verbatim and no reader in the codebase treats "1",
  360. "on" or "yes" as on, so restoring one of those cannot switch anything on.
  361. Bool-tolerant because a backup's JSON can carry a real boolean.
  362. """
  363. if isinstance(value, bool):
  364. return value
  365. if value is None:
  366. return False
  367. return str(value).strip().lower() == "true"
  368. def _as_setting_text(value: object) -> str:
  369. """A setting as its row stores it, so a backup value compares with a local one."""
  370. if value is None:
  371. return ""
  372. if isinstance(value, bool) or str(value).lower() in ("true", "false"):
  373. return str(value).lower()
  374. return str(value)
  375. def _is_usable_credential(value: object) -> bool:
  376. """True if a credential value is present and not blank.
  377. A present-but-*blank* ``prometheus_token`` row counts as unusable, because an
  378. empty token is exactly the ``if token:`` hole the companion rule exists to
  379. stop a restore from opening.
  380. """
  381. return value is not None and bool(str(value).strip())
  382. @dataclass(frozen=True)
  383. class _SettingsPlan:
  384. """Which keys of a settings payload will not be written, and why.
  385. Built once, before anything is added to the session, and shared by the
  386. preview and the restore so the two cannot disagree about what a commit will
  387. change. The companion bucket is why this needs a session at all: unlike the
  388. two name-based buckets it depends on local database state.
  389. The buckets are disjoint — a key is classified once, in order. The
  390. destination bucket depends on local state as well.
  391. """
  392. blocked: tuple[str, ...] = ()
  393. protected: tuple[str, ...] = ()
  394. companion: tuple[str, ...] = ()
  395. destination: tuple[str, ...] = ()
  396. two_factor: tuple[str, ...] = ()
  397. invalid: tuple[str, ...] = ()
  398. unknown: tuple[str, ...] = ()
  399. # Bambuddy's own one-shot flags (a leading underscore): refused without a
  400. # word, as every backup carries them.
  401. internal: tuple[str, ...] = ()
  402. address: tuple[str, ...] = ()
  403. # Each key to write, in the form its editor stores.
  404. stored: dict[str, str] = dataclasses_field(default_factory=dict)
  405. @property
  406. def _buckets(self) -> tuple[tuple[str, ...], ...]:
  407. return (
  408. self.blocked,
  409. self.protected,
  410. self.companion,
  411. self.destination,
  412. self.two_factor,
  413. self.invalid,
  414. self.unknown,
  415. self.internal,
  416. self.address,
  417. )
  418. @property
  419. def refused(self) -> frozenset[str]:
  420. return frozenset(key for bucket in self._buckets for key in bucket)
  421. @property
  422. def refused_count(self) -> int:
  423. return sum(len(bucket) for bucket in self._buckets)
  424. @dataclass(frozen=True)
  425. class _Detail:
  426. """A preview caveat, as a translation code plus its English rendering.
  427. Same contract as a note: the client translates ``code`` with ``params`` and
  428. falls back to ``message``.
  429. """
  430. code: str
  431. message: str
  432. params: dict[str, str | int] = dataclasses_field(default_factory=dict)
  433. # The archive editor's own bound on grams (ArchiveUpdate).
  434. _MAX_RESTORED_GRAMS = 100_000
  435. # What a 32-bit integer column takes (PostgreSQL rejects more).
  436. _MAX_RESTORED_INT = 2**31 - 1
  437. def _restored_number(value, *, maximum: float | None = None, integer: bool = False):
  438. """A number from a backup as the database should hold it, or None.
  439. Only a finite real number of at least 0 (and at most ``maximum``) passes:
  440. a damaged or hand-edited backup must not put text, negative or absurd
  441. values into the filament and cost totals.
  442. """
  443. if isinstance(value, bool) or not isinstance(value, (int, float)):
  444. return None
  445. if integer and maximum is None:
  446. maximum = _MAX_RESTORED_INT
  447. try:
  448. if not math.isfinite(value) or value < 0 or (maximum is not None and value > maximum):
  449. return None
  450. return int(value) if integer else float(value)
  451. except OverflowError: # an integer too large for a float
  452. return None
  453. def _runs_of(payload) -> list:
  454. """The prints in a backup's print_runs.json, or none."""
  455. runs = payload.get("runs") if isinstance(payload, dict) else None
  456. return runs if isinstance(runs, list) else []
  457. def _aliases_of(payload) -> dict[int, int]:
  458. """merged archive id -> the archive it went into, from aliases.json."""
  459. aliases = payload.get("aliases") if isinstance(payload, dict) else None
  460. found: dict[int, int] = {}
  461. for alias in aliases if isinstance(aliases, list) else []:
  462. if not isinstance(alias, dict):
  463. continue
  464. merged, into = alias.get("merged_archive_id"), alias.get("archive_id")
  465. if isinstance(merged, int) and isinstance(into, int) and merged != into:
  466. found[merged] = into
  467. return found
  468. # A print's outcome verdict, as the confirm flow writes it.
  469. _RUN_VERDICTS = ("good", "reject")
  470. class _CategoryTally:
  471. """Mutable accumulator matching ``GitHubRestoreCategoryResult``."""
  472. def __init__(self) -> None:
  473. self.restored = 0
  474. self.skipped = 0
  475. self.failed = 0
  476. self.notes: list[dict] = []
  477. def note(self, code: str, message: str, **params) -> None:
  478. """Record a note as a translation code, its params and an English fallback.
  479. Deduped on ``(code, params)`` rather than on the rendered text, which is
  480. the same thing today but keeps two notes that differ only in a printer
  481. name from collapsing into one. Bounded for the reason it always was: the
  482. UI renders every note, so a large backup must not emit one per row.
  483. """
  484. if any(existing["code"] == code and existing["params"] == params for existing in self.notes):
  485. return
  486. if len(self.notes) >= 20:
  487. return
  488. self.notes.append({"code": code, "params": params, "message": message})
  489. def as_dict(self) -> dict:
  490. return {"restored": self.restored, "skipped": self.skipped, "failed": self.failed, "notes": self.notes}
  491. class GitHubRestoreService:
  492. """Reads a backup repository and applies selected categories locally."""
  493. def __init__(self) -> None:
  494. self._running_restore: bool = False
  495. self._progress: str | None = None
  496. self._http_client: httpx.AsyncClient | None = None
  497. # Guards the check-then-set on ``_running_restore``. Without it two
  498. # concurrent POSTs can both observe False before either sets it.
  499. self._lock = asyncio.Lock()
  500. async def _get_client(self) -> httpx.AsyncClient:
  501. if self._http_client is None or self._http_client.is_closed:
  502. self._http_client = httpx.AsyncClient(timeout=60.0)
  503. return self._http_client
  504. @property
  505. def is_running(self) -> bool:
  506. return self._running_restore
  507. @property
  508. def progress(self) -> str | None:
  509. return self._progress
  510. # --- Repository reads --------------------------------------------------
  511. async def list_commits(self, config: GitHubBackupConfig, limit: int = 20) -> dict:
  512. """List recent commits on the configured branch."""
  513. backend = get_provider_backend(config.provider)
  514. client = await self._get_client()
  515. result = await backend.list_commits(
  516. repo_url=config.repository_url,
  517. token=config.access_token,
  518. branch=config.branch,
  519. client=client,
  520. limit=limit,
  521. )
  522. result["branch"] = config.branch
  523. return result
  524. async def _resolve_ref(self, config: GitHubBackupConfig, ref: str) -> tuple[str | None, str, dict | None]:
  525. """Turn ``HEAD`` into a concrete commit SHA.
  526. Done once up front so a preview and the restore that follows it act on
  527. the same commit even if a scheduled backup lands in between.
  528. The third element is the commit entry, when resolving already fetched
  529. one. ``preview`` displays it, and taking it from here means the ``HEAD``
  530. case — by far the common one — costs one ``list_commits`` call rather
  531. than two.
  532. """
  533. if ref and ref.upper() != "HEAD":
  534. return ref, "", None
  535. result = await self.list_commits(config, limit=1)
  536. if not result.get("success"):
  537. return None, result.get("message") or "Could not read the backup repository", None
  538. commits = result.get("commits") or []
  539. if not commits:
  540. return None, f"Branch '{config.branch}' has no commits to restore from", None
  541. return commits[0]["sha"], "", commits[0]
  542. async def _describe_commit(self, config: GitHubBackupConfig, resolved: str) -> dict | None:
  543. """Find the display metadata for one commit SHA.
  544. Two things used to leave ``commit: null`` in a preview, and the second is
  545. the one that bit in practice:
  546. * the commit is older than the 20 the picker lists, so it is not in the
  547. scan at all — that is what ``get_commit`` is for;
  548. * ``REF_PATTERN`` accepts a 7-character ref while providers return the
  549. full 40, so an exact ``==`` never matched an abbreviated SHA *even when
  550. the commit was in the window*. Hence the prefix comparison.
  551. Best-effort throughout: this is a subject line and a date, so a failure
  552. returns None and the preview renders without them rather than failing.
  553. """
  554. commits = (await self.list_commits(config, limit=20)).get("commits") or []
  555. for entry in commits:
  556. sha = entry.get("sha") or ""
  557. if sha == resolved or sha.startswith(resolved) or resolved.startswith(sha):
  558. return entry
  559. backend = get_provider_backend(config.provider)
  560. client = await self._get_client()
  561. result = await backend.get_commit(
  562. repo_url=config.repository_url, token=config.access_token, ref=resolved, client=client
  563. )
  564. return result.get("commit") if result.get("success") else None
  565. def _category_paths(self, category: RestoreCategory, available: list[str]) -> list[str]:
  566. """Return the paths in ``available`` that belong to ``category``."""
  567. if category == RestoreCategory.SETTINGS:
  568. return [p for p in (SETTINGS_PATH,) if p in available]
  569. if category == RestoreCategory.SPOOLS:
  570. return [p for p in (SPOOLS_PATH, SPOOL_USAGE_PATH) if p in available]
  571. if category == RestoreCategory.ARCHIVES:
  572. return [p for p in (ARCHIVES_PATH, RUNS_PATH, ALIASES_PATH) if p in available]
  573. if category == RestoreCategory.KPROFILES:
  574. return sorted(p for p in available if _KPROFILE_PATH_RE.match(p))
  575. return []
  576. @staticmethod
  577. def _parse_json_files(raw: dict[str, str]) -> tuple[dict[str, object], list[str]]:
  578. """Parse each fetched file, collecting paths that failed to parse."""
  579. parsed: dict[str, object] = {}
  580. bad: list[str] = []
  581. for path, text in raw.items():
  582. try:
  583. parsed[path] = json.loads(text)
  584. except (ValueError, TypeError):
  585. bad.append(path)
  586. return parsed, bad
  587. @staticmethod
  588. async def _plan_settings(db: AsyncSession, values: dict) -> _SettingsPlan:
  589. """Classify every key of a settings payload into its refusal bucket.
  590. Keys with an unusable name land in no bucket: they are the restore's
  591. ``failed``, not a refusal, and the preview counts them because the run
  592. will still report on them.
  593. Reads local state, so it must run before anything is added to the
  594. session — otherwise "does this instance already have a credential" would
  595. see the restore's own writes.
  596. """
  597. blocked: list[str] = []
  598. protected: list[str] = []
  599. two_factor: list[str] = []
  600. invalid: list[str] = []
  601. unknown: list[str] = []
  602. internal: list[str] = []
  603. stored: dict[str, str] = {}
  604. # Toggle -> credential for the pairs that survived the payload-only
  605. # conditions and still need local state to judge.
  606. candidates: dict[str, str] = {}
  607. for key, value in values.items():
  608. if not isinstance(key, str) or not key:
  609. continue
  610. if _is_blocked_setting_key(key):
  611. blocked.append(key)
  612. continue
  613. if _is_protected_setting_key(key) or (not _restoring_as_admin.get() and _admin_only_for_restore(key)):
  614. protected.append(key)
  615. continue
  616. if _TWO_FACTOR_KEY.match(key):
  617. two_factor.append(key)
  618. continue
  619. if key.startswith("_"):
  620. internal.append(key)
  621. continue
  622. if key in _RETIRED_SETTING_KEYS:
  623. continue
  624. # A null is skipped by the restore itself, not refused.
  625. if value is not None:
  626. valid, form = _checked_setting(key, value)
  627. if valid is None:
  628. unknown.append(key)
  629. continue
  630. if not valid:
  631. invalid.append(key)
  632. continue
  633. stored[key] = form
  634. credential = _COMPANION_CREDENTIALS.get(key)
  635. if credential is None:
  636. continue
  637. # Turning something *off* is always safe to write.
  638. # Judged as written: a spelling the check reads as on is stored as on.
  639. if not _setting_value_is_true(stored.get(key, value)):
  640. continue
  641. # Expressed as the predicate rather than assumed, so the map cannot
  642. # go quietly inert if the secret hints are ever edited: a credential
  643. # the restore is willing to write travels with its toggle.
  644. if not _is_blocked_setting_key(credential):
  645. continue
  646. # The backup itself carried no credential here. For an availability
  647. # pair that describes a working config — an anonymous MQTT broker and
  648. # an anonymous LDAP bind both are (mqtt_relay.py and ldap_service.py
  649. # pass empty credentials straight through) — so refusing the toggle
  650. # would be a false positive. For an exposure pair a blank credential
  651. # is the hole itself, so the condition is skipped and only local
  652. # state decides. See _COMPANION_EXPOSURE_TOGGLES.
  653. if key not in _COMPANION_EXPOSURE_TOGGLES and not _is_usable_credential(values.get(credential)):
  654. continue
  655. candidates[key] = credential
  656. # Settings that decide where a credential is sent, which the payload
  657. # sets. Judged against local state below.
  658. destinations = {
  659. key: secrets
  660. for key, value in values.items()
  661. if value is not None
  662. and (secrets := _DESTINATION_SECRETS.get(key))
  663. and key not in protected
  664. and key not in invalid
  665. and key not in unknown
  666. and key not in internal
  667. }
  668. addresses = [key for key in _ADDRESS_KEYS if key in stored]
  669. if not candidates and not destinations and not addresses:
  670. return _SettingsPlan(
  671. blocked=tuple(blocked),
  672. protected=tuple(protected),
  673. two_factor=tuple(two_factor),
  674. invalid=tuple(invalid),
  675. unknown=tuple(unknown),
  676. internal=tuple(internal),
  677. stored=stored,
  678. )
  679. # One SELECT covering both halves of every candidate pair.
  680. wanted = set(candidates) | set(candidates.values()) | set(destinations)
  681. wanted |= {secret for secrets in destinations.values() for secret in secrets}
  682. wanted |= set(addresses)
  683. rows = await db.execute(select(Settings).where(Settings.key.in_(wanted)))
  684. local = {row.key: row.value for row in rows.scalars().all()}
  685. companion: list[str] = []
  686. for toggle, credential in candidates.items():
  687. if _is_usable_credential(local.get(credential)):
  688. continue
  689. env_name = _COMPANION_CREDENTIAL_ENV.get(credential)
  690. if env_name and _is_usable_credential(os.environ.get(env_name)):
  691. continue
  692. # Already on locally with no credential: the exposure pre-dates this
  693. # restore, so refusing changes nothing and "left switched off" would
  694. # be a lie.
  695. if _setting_value_is_true(local.get(toggle)):
  696. continue
  697. companion.append(toggle)
  698. # A credential kept here does not follow a destination the backup
  699. # changes, as the settings editor refuses the same change without the
  700. # credential entered again. A token from the environment is the
  701. # deployment's own and may follow, as it may there.
  702. destination: list[str] = []
  703. for key, secrets in destinations.items():
  704. kept = [
  705. secret
  706. for secret in secrets
  707. if _is_usable_credential(local.get(secret))
  708. and not (
  709. (env_name := _COMPANION_CREDENTIAL_ENV.get(secret))
  710. and _is_usable_credential(os.environ.get(env_name))
  711. )
  712. ]
  713. if not kept:
  714. continue
  715. if key in local:
  716. current = local[key]
  717. elif key in AppSettings.model_fields:
  718. current = AppSettings.model_fields[key].get_default(call_default_factory=True)
  719. else:
  720. # No row and no schema default: nothing to compare with, so
  721. # counted as a change.
  722. destination.append(key)
  723. continue
  724. if _as_setting_text(stored.get(key, values[key])) != _as_setting_text(current):
  725. destination.append(key)
  726. # Compared as written with this instance's value: its row, else the
  727. # schema default, else nothing.
  728. address = [
  729. key
  730. for key in addresses
  731. if key not in destination
  732. and _as_setting_text(stored[key])
  733. != _as_setting_text(
  734. local[key]
  735. if key in local
  736. else AppSettings.model_fields[key].get_default(call_default_factory=True)
  737. if key in AppSettings.model_fields
  738. else None
  739. )
  740. ]
  741. for key in address:
  742. del stored[key]
  743. return _SettingsPlan(
  744. blocked=tuple(blocked),
  745. protected=tuple(protected),
  746. companion=tuple(companion),
  747. address=tuple(address),
  748. destination=tuple(destination),
  749. two_factor=tuple(two_factor),
  750. invalid=tuple(invalid),
  751. unknown=tuple(unknown),
  752. internal=tuple(internal),
  753. stored=stored,
  754. )
  755. async def preview(
  756. self, db: AsyncSession, config: GitHubBackupConfig, ref: str = "HEAD", *, as_admin: bool = True
  757. ) -> dict:
  758. """Report which categories a commit contains, and how much is in each.
  759. Takes a session because the settings count depends on local state — see
  760. ``_plan_settings``. ``ref`` stays keyword-friendly for callers.
  761. ``as_admin`` False counts as a non-admin's restore would write.
  762. """
  763. token = _restoring_as_admin.set(as_admin)
  764. try:
  765. return await self._preview(db, config, ref)
  766. finally:
  767. _restoring_as_admin.reset(token)
  768. async def _preview(self, db: AsyncSession, config: GitHubBackupConfig, ref: str) -> dict:
  769. resolved, error, commit_info = await self._resolve_ref(config, ref)
  770. if resolved is None:
  771. return {"success": False, "message": error, "ref": ref, "categories": []}
  772. backend = get_provider_backend(config.provider)
  773. client = await self._get_client()
  774. tree = await backend.list_tree(
  775. repo_url=config.repository_url, token=config.access_token, ref=resolved, client=client
  776. )
  777. if not tree.get("success"):
  778. return {"success": False, "message": tree.get("message") or "Could not list the commit", "ref": resolved}
  779. available: list[str] = tree.get("paths") or []
  780. # One batched read covers metadata plus every category payload.
  781. wanted = [METADATA_PATH] if METADATA_PATH in available else []
  782. for category in RestoreCategory:
  783. wanted.extend(self._category_paths(category, available))
  784. fetched = await backend.fetch_files(
  785. repo_url=config.repository_url,
  786. token=config.access_token,
  787. ref=resolved,
  788. paths=wanted,
  789. client=client,
  790. # The listing above already built this map; without it the GitHub
  791. # family would GET the same recursive tree a second time.
  792. blob_shas=tree.get("blob_shas") or None,
  793. )
  794. if not fetched.get("success"):
  795. return {
  796. "success": False,
  797. "message": fetched.get("message") or "Could not read the commit contents",
  798. "ref": resolved,
  799. }
  800. parsed, bad_paths = self._parse_json_files(fetched.get("files") or {})
  801. metadata = parsed.get(METADATA_PATH)
  802. metadata_version = metadata.get("version") if isinstance(metadata, dict) else None
  803. categories = []
  804. for category in RestoreCategory:
  805. paths = self._category_paths(category, available)
  806. if not paths:
  807. categories.append(
  808. self._category_entry(category, False, 0, _Detail("notPresent", "Not present in this backup commit"))
  809. )
  810. continue
  811. unreadable = [p for p in paths if p in bad_paths]
  812. if unreadable:
  813. joined = ", ".join(unreadable)
  814. categories.append(
  815. self._category_entry(
  816. category,
  817. False,
  818. 0,
  819. _Detail("unreadableJson", f"Unreadable JSON: {joined}", {"paths": joined}),
  820. )
  821. )
  822. continue
  823. count, detail = await self._count_items(db, category, parsed)
  824. categories.append(self._category_entry(category, True, count, detail))
  825. if commit_info is None:
  826. commit_info = await self._describe_commit(config, resolved)
  827. return {
  828. "success": True,
  829. "message": "OK",
  830. "ref": resolved,
  831. "commit": commit_info,
  832. "metadata_version": metadata_version,
  833. "categories": categories,
  834. }
  835. @staticmethod
  836. def _category_entry(category: RestoreCategory, available: bool, item_count: int, detail: _Detail | None) -> dict:
  837. """Shape one ``GitHubRestorePreviewCategory``, translated detail included."""
  838. return {
  839. "category": category,
  840. "available": available,
  841. "item_count": item_count,
  842. "detail": detail.message if detail else None,
  843. "detail_code": detail.code if detail else None,
  844. "detail_params": detail.params if detail else {},
  845. }
  846. async def _count_items(
  847. self, db: AsyncSession, category: RestoreCategory, parsed: dict
  848. ) -> tuple[int, _Detail | None]:
  849. """Count restorable items for ``category`` and describe any caveat."""
  850. if category == RestoreCategory.SETTINGS:
  851. payload = parsed.get(SETTINGS_PATH)
  852. values = payload.get("settings") if isinstance(payload, dict) else None
  853. if not isinstance(values, dict):
  854. return 0, _Detail("settingsNoPayload", "No settings in payload")
  855. # Every refusal is subtracted so the count matches what the restore
  856. # actually writes. The wording calls out the credential ones (what a
  857. # user might expect to come back) and the companion ones (a
  858. # behaviour change worth explaining before it happens); the auth
  859. # policy keys stay unmentioned on purpose.
  860. plan = await self._plan_settings(db, values)
  861. detail = None
  862. not_restorable = len(plan.two_factor) + len(plan.invalid) + len(plan.unknown) + len(plan.address)
  863. if plan.destination:
  864. detail = _Detail(
  865. "settingsDestinationWillKeep",
  866. f"{len(plan.destination)} setting(s) will be left as they are - they would send a credential "
  867. "stored here somewhere else; credentials are never restored",
  868. {"destination": len(plan.destination)},
  869. )
  870. elif not_restorable:
  871. detail = _Detail(
  872. "settingsNotRestorableWillSkip",
  873. f"{not_restorable} setting(s) will be skipped - values Settings would not accept, or keys a "
  874. "backup does not restore",
  875. {"count": not_restorable},
  876. )
  877. elif plan.companion and not plan.blocked:
  878. # An exposure toggle becomes a candidate whether or not the
  879. # backup carried its credential, so this commit can refuse a
  880. # switch without having a single credential-like key to skip —
  881. # "0 credential-like key(s) will be skipped" would read as noise.
  882. detail = _Detail(
  883. "settingsCompanionOnlyWillSkip",
  884. f"{len(plan.companion)} switch(es) will be left off — the credential each one needs "
  885. "cannot be restored from a backup",
  886. {"companion": len(plan.companion)},
  887. )
  888. elif plan.companion:
  889. detail = _Detail(
  890. "settingsCompanionWillSkip",
  891. f"{len(plan.blocked)} credential-like key(s) will be skipped, and "
  892. f"{len(plan.companion)} switch(es) that depend on them will be left off",
  893. {"count": len(plan.blocked), "companion": len(plan.companion)},
  894. )
  895. elif plan.blocked:
  896. detail = _Detail(
  897. "settingsCredentialsWillSkip",
  898. f"{len(plan.blocked)} credential-like keys will be skipped",
  899. {"count": len(plan.blocked)},
  900. )
  901. return len(values) - plan.refused_count, detail
  902. if category == RestoreCategory.SPOOLS:
  903. payload = parsed.get(SPOOLS_PATH)
  904. spools = payload.get("spools") if isinstance(payload, dict) else None
  905. usage_payload = parsed.get(SPOOL_USAGE_PATH)
  906. usage = usage_payload.get("usage_history") if isinstance(usage_payload, dict) else None
  907. # Usage records are counted here, not just described in the detail:
  908. # _restore_spool_usage increments this category's tally, so counting
  909. # only the spools broke restored + skipped + failed == item_count —
  910. # the invariant the settings count is careful to hold. The detail
  911. # breaks the total down rather than adding to it.
  912. count = len(spools) if isinstance(spools, list) else 0
  913. detail = None
  914. if isinstance(usage, list) and usage:
  915. count += len(usage)
  916. detail = _Detail("spoolsUsageCount", f"including {len(usage)} usage records", {"count": len(usage)})
  917. return count, detail
  918. if category == RestoreCategory.ARCHIVES:
  919. payload = parsed.get(ARCHIVES_PATH)
  920. archives = payload.get("archives") if isinstance(payload, dict) else None
  921. count = len(archives) if isinstance(archives, list) else 0
  922. # Prints are tallied with the archives, so counted here too.
  923. runs = _runs_of(parsed.get(RUNS_PATH))
  924. if runs:
  925. return count + len(runs), _Detail(
  926. "archivesRunsCount",
  927. f"including {len(runs)} print(s) - metadata only, 3MF files and thumbnails are not in a Git backup",
  928. {"count": len(runs)},
  929. )
  930. return count, _Detail(
  931. "archivesMetadataOnly", "Metadata only — 3MF files and thumbnails are not in a Git backup"
  932. )
  933. if category == RestoreCategory.KPROFILES:
  934. total = 0
  935. serials = set()
  936. for path, payload in parsed.items():
  937. match = _KPROFILE_PATH_RE.match(path)
  938. if not match or not isinstance(payload, dict):
  939. continue
  940. serials.add(match.group(1))
  941. profiles = payload.get("profiles")
  942. if isinstance(profiles, list):
  943. total += len(profiles)
  944. detail = None
  945. if serials:
  946. detail = _Detail("kprofilesPrinterCount", f"across {len(serials)} printer(s)", {"count": len(serials)})
  947. return total, detail
  948. return 0, None
  949. # --- Restore -----------------------------------------------------------
  950. async def run_restore(
  951. self,
  952. config_id: int,
  953. ref: str,
  954. categories: list[RestoreCategory],
  955. overwrite_existing: bool = False,
  956. *,
  957. as_admin: bool = True,
  958. ) -> dict:
  959. """Apply selected categories from one backup commit.
  960. ``as_admin`` False keeps the admin-only settings as they are here
  961. (see ``_restoring_as_admin``).
  962. """
  963. token = _restoring_as_admin.set(as_admin)
  964. try:
  965. return await self._run_restore(config_id, ref, categories, overwrite_existing)
  966. finally:
  967. _restoring_as_admin.reset(token)
  968. async def _run_restore(
  969. self,
  970. config_id: int,
  971. ref: str,
  972. categories: list[RestoreCategory],
  973. overwrite_existing: bool,
  974. ) -> dict:
  975. # Import locally to avoid a module-level cycle: the backup service takes
  976. # the mirror-image lock against us.
  977. from backend.app.services.github_backup import github_backup_service
  978. # The lock serialises two concurrent restores; the backup side has no
  979. # lock of its own, and relies on this region staying await-free after the
  980. # acquisition. Both flags are plain bools on one event loop, so with no
  981. # suspension point between the two reads and the write, the loop cannot
  982. # slip github_backup.run_backup's mirror-image check in between. Adding an
  983. # `await` below the acquisition and above `self._running_restore = True`
  984. # would let a backup and a restore run at once.
  985. async with self._lock:
  986. if self._running_restore:
  987. return {"success": False, "message": "A restore is already running", "results": {}}
  988. if github_backup_service.is_running:
  989. return {
  990. "success": False,
  991. "message": "A backup is currently running. Wait for it to finish before restoring.",
  992. "results": {},
  993. }
  994. self._running_restore = True
  995. log_id = None
  996. try:
  997. async with async_session() as db:
  998. result = await db.execute(select(GitHubBackupConfig).where(GitHubBackupConfig.id == config_id))
  999. config = result.scalar_one_or_none()
  1000. if not config:
  1001. return {"success": False, "message": "Configuration not found", "results": {}}
  1002. self._progress = "Resolving commit..."
  1003. resolved, error, _ = await self._resolve_ref(config, ref)
  1004. if resolved is None:
  1005. return {"success": False, "message": error, "results": {}}
  1006. log = GitHubBackupLog(config_id=config_id, status="running", trigger="restore", commit_sha=resolved)
  1007. db.add(log)
  1008. await db.commit()
  1009. await db.refresh(log)
  1010. log_id = log.id
  1011. # Owned here rather than by _apply so the failure path can see
  1012. # the categories that were already committed when the raise
  1013. # happened. _apply records a tally only after its category's
  1014. # commit, so every entry present is on disk.
  1015. results: dict[str, _CategoryTally] = {}
  1016. try:
  1017. payload, error = await self._read_categories(config, resolved, categories)
  1018. if error:
  1019. raise RuntimeError(error)
  1020. settings_keys_written: set[str] = set()
  1021. await self._apply(
  1022. db, payload, categories, overwrite_existing, settings_keys_written, results=results
  1023. )
  1024. await db.commit()
  1025. # After the commit: this reconnects the relay, which is not
  1026. # something to do on values that could still roll back.
  1027. settings_tally = results.get(RestoreCategory.SETTINGS.value)
  1028. if settings_tally is not None:
  1029. self._progress = "Reconnecting the MQTT relay..."
  1030. await self._reconfigure_mqtt_relay(db, settings_keys_written, settings_tally)
  1031. total_restored = sum(tally.restored for tally in results.values())
  1032. any_failed = any(tally.failed for tally in results.values())
  1033. log.status = "failed" if any_failed and total_restored == 0 else "success"
  1034. log.completed_at = datetime.now(timezone.utc)
  1035. log.files_changed = total_restored
  1036. if any_failed:
  1037. log.error_message = "Some items could not be restored — see the restore result for detail"
  1038. await db.commit()
  1039. return {
  1040. "success": True,
  1041. "message": f"Restored {total_restored} item(s) from {resolved[:7]}",
  1042. "log_id": log_id,
  1043. "ref": resolved,
  1044. "results": {name: tally.as_dict() for name, tally in results.items()},
  1045. }
  1046. except Exception as e:
  1047. # Rolls back the category that was mid-flight. Every category
  1048. # already in ``results`` committed as it finished (see
  1049. # _apply), so those rows survive this — and reporting an
  1050. # empty result over them would tell the user nothing was
  1051. # restored while their archives and spools are on disk.
  1052. logger.exception("Restore failed for config %s ref %s", config_id, resolved)
  1053. await db.rollback()
  1054. committed = sum(tally.restored for tally in results.values())
  1055. log.status = "failed"
  1056. log.completed_at = datetime.now(timezone.utc)
  1057. log.files_changed = committed
  1058. log.error_message = str(e)[:1000]
  1059. await db.commit()
  1060. return {
  1061. "success": False,
  1062. "message": str(e),
  1063. "log_id": log_id,
  1064. "ref": resolved,
  1065. "results": {name: tally.as_dict() for name, tally in results.items()},
  1066. }
  1067. finally:
  1068. self._running_restore = False
  1069. self._progress = None
  1070. async def _read_categories(
  1071. self, config: GitHubBackupConfig, ref: str, categories: list[RestoreCategory]
  1072. ) -> tuple[dict, str]:
  1073. """Fetch and parse just the files the requested categories need."""
  1074. backend = get_provider_backend(config.provider)
  1075. client = await self._get_client()
  1076. self._progress = "Listing backup contents..."
  1077. tree = await backend.list_tree(
  1078. repo_url=config.repository_url, token=config.access_token, ref=ref, client=client
  1079. )
  1080. if not tree.get("success"):
  1081. return {}, tree.get("message") or "Could not list the commit"
  1082. available: list[str] = tree.get("paths") or []
  1083. wanted: list[str] = []
  1084. for category in categories:
  1085. wanted.extend(self._category_paths(category, available))
  1086. if not wanted:
  1087. return {}, "None of the selected categories are present in that commit"
  1088. self._progress = "Downloading backup files..."
  1089. fetched = await backend.fetch_files(
  1090. repo_url=config.repository_url,
  1091. token=config.access_token,
  1092. ref=ref,
  1093. paths=wanted,
  1094. client=client,
  1095. blob_shas=tree.get("blob_shas") or None,
  1096. )
  1097. if not fetched.get("success"):
  1098. return {}, fetched.get("message") or "Could not read the commit contents"
  1099. parsed, bad = self._parse_json_files(fetched.get("files") or {})
  1100. if bad:
  1101. return {}, f"Backup contains unreadable JSON: {', '.join(sorted(bad))}"
  1102. return parsed, ""
  1103. async def _apply(
  1104. self,
  1105. db: AsyncSession,
  1106. payload: dict,
  1107. categories: list[RestoreCategory],
  1108. overwrite: bool,
  1109. settings_keys_written: set[str] | None = None,
  1110. results: dict[str, _CategoryTally] | None = None,
  1111. ) -> dict[str, _CategoryTally]:
  1112. """Apply categories in dependency order and return per-category tallies.
  1113. ``settings_keys_written``, if given, collects the setting keys actually
  1114. written, for the caller's post-commit side effects (see
  1115. ``_reconfigure_mqtt_relay``).
  1116. ``results``, if given, is the caller's own dict rather than a fresh one.
  1117. Each category is committed before it is recorded there, so on a raise
  1118. the caller can report exactly what is already on disk — see the
  1119. per-category commit below.
  1120. """
  1121. results = {} if results is None else results
  1122. archive_id_map: dict[int, int] = {}
  1123. # Every database category commits before the next one starts, and only
  1124. # then is its tally recorded. Two reasons:
  1125. #
  1126. # * SQLite has one writer. Each category is a long run of one SELECT per
  1127. # row or per key — _find_archive, _find_spool, the usage dedupe,
  1128. # _restore_settings — interleaved with autoflushed INSERTs, all inside
  1129. # the open write transaction. A few thousand archives plus a full
  1130. # usage history plausibly passes the 15 s busy_timeout
  1131. # (core/database.py), at which point every concurrent writer in the
  1132. # app fails with "database is locked". This is the same hold the
  1133. # K-profile phase had, arriving by volume rather than by awaiting a
  1134. # sulking printer.
  1135. # * The ordering tolerates it: the only cross-category state is
  1136. # archive_id_map and spool_id_map, both plain dicts in memory, and
  1137. # the session is expire_on_commit=False so nothing reloads.
  1138. #
  1139. # The cost is that a later failure no longer rolls back an earlier
  1140. # category — which is why the tally is recorded after the commit, so
  1141. # run_restore's failure path reports the rows that really landed instead
  1142. # of claiming nothing was restored.
  1143. # Archives first: spool usage history references archive_id.
  1144. if RestoreCategory.ARCHIVES in categories:
  1145. self._progress = "Restoring print archives..."
  1146. tally = _CategoryTally()
  1147. await self._restore_archives(
  1148. db,
  1149. payload.get(ARCHIVES_PATH),
  1150. overwrite,
  1151. tally,
  1152. archive_id_map,
  1153. runs_payload=payload.get(RUNS_PATH),
  1154. aliases_payload=payload.get(ALIASES_PATH),
  1155. )
  1156. await self._restore_runs(db, payload.get(RUNS_PATH), overwrite, tally, archive_id_map)
  1157. await db.commit()
  1158. results[RestoreCategory.ARCHIVES.value] = tally
  1159. if RestoreCategory.SPOOLS in categories:
  1160. self._progress = "Restoring spool inventory..."
  1161. tally = _CategoryTally()
  1162. await self._restore_spools(
  1163. db,
  1164. payload.get(SPOOLS_PATH),
  1165. payload.get(SPOOL_USAGE_PATH),
  1166. overwrite,
  1167. tally,
  1168. archive_id_map,
  1169. )
  1170. await db.commit()
  1171. results[RestoreCategory.SPOOLS.value] = tally
  1172. if RestoreCategory.SETTINGS in categories:
  1173. self._progress = "Restoring app settings..."
  1174. tally = _CategoryTally()
  1175. await self._restore_settings(
  1176. db, payload.get(SETTINGS_PATH), overwrite, tally, keys_written=settings_keys_written
  1177. )
  1178. await db.commit()
  1179. results[RestoreCategory.SETTINGS.value] = tally
  1180. # Last, because it leaves the database and publishes over MQTT.
  1181. if RestoreCategory.KPROFILES in categories:
  1182. # The database categories are already committed by the loop above,
  1183. # and that is load-bearing here rather than tidiness:
  1184. # _restore_kprofiles awaits get_kprofiles per printer per nozzle,
  1185. # which is timeout=5.0 * max_retries=3, i.e. up to ~15 s each against
  1186. # an unresponsive printer. Holding SQLite's writer across that would
  1187. # pass the 15 s busy_timeout on a farm with a couple of sulking
  1188. # printers.
  1189. #
  1190. # The cost is that a K-profile failure no longer rolls back the
  1191. # categories that already succeeded. That is the correct trade
  1192. # anyway: extrusion_cali_set has left for the printer by then and
  1193. # cannot be rolled back either, so a rollback would only have made
  1194. # the database disagree with the hardware.
  1195. self._progress = "Sending K-profiles to printers..."
  1196. tally = _CategoryTally()
  1197. try:
  1198. await self._restore_kprofiles(db, payload, tally)
  1199. except Exception as e:
  1200. # Everything above is committed and cannot be un-committed, so
  1201. # letting this reach run_restore's handler would report
  1202. # "nothing was restored" over durable archive, spool and
  1203. # settings rows — and skip the post-commit MQTT reconfigure,
  1204. # leaving the relay on the pre-restore broker. The K-profile
  1205. # phase is the last thing that runs, so containing it here is
  1206. # what keeps the result honest about what actually landed.
  1207. logger.exception("The K-profile step failed after the database categories were committed")
  1208. # Discards the phase's own read transaction. The rows above went
  1209. # in at the commit two statements up; this only stops a session
  1210. # left in a failed state by a database error from turning the
  1211. # caller's commit into that same false report.
  1212. await db.rollback()
  1213. outstanding = self._kprofile_profile_count(
  1214. content for path, content in payload.items() if _KPROFILE_PATH_RE.match(path)
  1215. )
  1216. outstanding -= tally.restored + tally.skipped + tally.failed
  1217. tally.failed += max(outstanding, 0)
  1218. tally.note(
  1219. "kprofilesStepFailed",
  1220. f"The K-profile step could not be completed: {e}",
  1221. reason=str(e)[:200],
  1222. )
  1223. results[RestoreCategory.KPROFILES.value] = tally
  1224. return results
  1225. # --- Per-category appliers --------------------------------------------
  1226. async def _restore_archives(
  1227. self,
  1228. db: AsyncSession,
  1229. payload,
  1230. overwrite: bool,
  1231. tally: _CategoryTally,
  1232. id_map: dict[int, int],
  1233. *,
  1234. runs_payload=None,
  1235. aliases_payload=None,
  1236. ) -> None:
  1237. archives = payload.get("archives") if isinstance(payload, dict) else None
  1238. if not isinstance(archives, list):
  1239. tally.note("noData", "No data of this kind in this backup")
  1240. return
  1241. # An archive merged into another is not restored as a row of its own:
  1242. # what pointed at it (its prints, spool usage) goes to the archive it
  1243. # went into, as the merge moved them.
  1244. aliases = _aliases_of(aliases_payload)
  1245. # Local archives an entry of this restore already went to.
  1246. claimed: set[int] = set()
  1247. # Its prints' starts recognise an archive printed again since the
  1248. # backup: the archive's own start is its latest print's.
  1249. run_starts: dict[int, set[datetime]] = {}
  1250. for run in _runs_of(runs_payload):
  1251. if isinstance(run, dict) and isinstance(run.get("archive_id"), int):
  1252. started = _parse_dt(run.get("started_at"))
  1253. if started is not None:
  1254. run_starts.setdefault(run["archive_id"], set()).add(started)
  1255. valid_printers = set((await db.execute(select(Printer.id))).scalars().all())
  1256. valid_projects = set((await db.execute(select(Project.id))).scalars().all())
  1257. # Ownership decides visibility, not just attribution: an archive with a
  1258. # NULL created_by_id is a 404 to every caller without archives:read_all
  1259. # (_ensure_archive_visible fails closed on it) and never appears in the
  1260. # ownership-scoped list queries. Hoisted like the two above.
  1261. #
  1262. # username is the natural key and wins, per the module's rule at the top
  1263. # of the file; created_by_id is the fallback for a pre-#2656 commit that
  1264. # carries no username. That ordering is what makes restoring onto a
  1265. # rebuilt instance safe: the users table renumbers there, so a live id
  1266. # can land on a different person, and the id path alone cannot tell that
  1267. # from a correct match. Resolving on the name instead means the one case
  1268. # it cannot resolve — a user renamed since the backup — falls through to
  1269. # ownerless-with-a-note below rather than misattributing in silence.
  1270. users = (await db.execute(select(User.id, User.username))).all()
  1271. valid_users = {user_id for user_id, _ in users}
  1272. users_by_name = {username: user_id for user_id, username in users}
  1273. # Only metadata is backed up, never the 3MF/thumbnail bytes, and
  1274. # print_archives.file_path is NOT NULL — so inserted rows get an empty
  1275. # path and are history-only. Say so once rather than per row.
  1276. warned_files = False
  1277. for entry in archives:
  1278. if not isinstance(entry, dict):
  1279. tally.failed += 1
  1280. continue
  1281. old_id = entry.get("id") if isinstance(entry.get("id"), int) else None
  1282. if old_id in aliases:
  1283. tally.skipped += 1
  1284. tally.note(
  1285. "archivesMergedFollowed",
  1286. "Archives merged into another are not restored as rows of their own - what pointed at them "
  1287. "goes to the archive they were merged into",
  1288. )
  1289. continue
  1290. started_at = _parse_dt(entry.get("started_at"))
  1291. existing = await self._find_archive(
  1292. db, entry, started_at, run_starts.get(old_id, set()) if old_id is not None else ()
  1293. )
  1294. # Merged here since the backup: its prints went into the archive
  1295. # it went into, and so does everything else of this entry. That
  1296. # archive is not overwritten with this one's fields, nor with a
  1297. # second entry's: it holds what the merge joined.
  1298. followed = False
  1299. if existing is not None:
  1300. into = await db.scalar(
  1301. select(ArchiveAlias.archive_id).where(ArchiveAlias.merged_archive_id == existing.id)
  1302. )
  1303. if into is not None:
  1304. existing = await db.get(PrintArchive, into)
  1305. followed = True
  1306. if existing is not None and (followed or existing.id in claimed):
  1307. if old_id is not None:
  1308. id_map[old_id] = existing.id
  1309. tally.skipped += 1
  1310. tally.note(
  1311. "archivesMergedFollowed",
  1312. "Archives merged into another are not restored as rows of their own - what pointed at them "
  1313. "goes to the archive they were merged into",
  1314. )
  1315. continue
  1316. numbers = {
  1317. "print_time_seconds": _restored_number(entry.get("print_time_seconds"), integer=True),
  1318. "filament_used_grams": _restored_number(entry.get("filament_used_grams"), maximum=_MAX_RESTORED_GRAMS),
  1319. "cost": _restored_number(entry.get("cost")),
  1320. "energy_kwh": _restored_number(entry.get("energy_kwh")),
  1321. "energy_cost": _restored_number(entry.get("energy_cost")),
  1322. "wear_cost": _restored_number(entry.get("wear_cost")),
  1323. }
  1324. if any(entry.get(key) is not None and value is None for key, value in numbers.items()):
  1325. tally.note(
  1326. "archivesNumbersDropped",
  1327. "Some archives had grams, costs, energy or print times that are not valid numbers - left empty",
  1328. )
  1329. # 0 is a real quantity (a ruined plate, #3051); the archive editor
  1330. # takes 0..10000. Anything else is the one print it stood for.
  1331. quantity = _restored_number(entry.get("quantity"), maximum=10_000, integer=True)
  1332. if quantity is None:
  1333. quantity = 1
  1334. # A print running when the backup was made is long over: closed
  1335. # here the way the stale-archive cleanup closes one. Restored as
  1336. # running, the next reconnect closed it as a print that had just
  1337. # failed, with a failure notification and a Print Log row (ST/I7).
  1338. status = entry.get("status") or "completed"
  1339. failure_reason = entry.get("failure_reason")
  1340. backed_up_in_flight = status in ("printing", "paused")
  1341. if backed_up_in_flight:
  1342. status, failure_reason = "cancelled", "noStatusUpdate"
  1343. fields = {
  1344. "print_name": entry.get("print_name"),
  1345. "print_time_seconds": numbers["print_time_seconds"],
  1346. "filament_used_grams": numbers["filament_used_grams"],
  1347. "filament_type": entry.get("filament_type"),
  1348. "filament_color": entry.get("filament_color"),
  1349. "layer_height": entry.get("layer_height"),
  1350. "total_layers": entry.get("total_layers"),
  1351. "nozzle_diameter": entry.get("nozzle_diameter"),
  1352. "bed_temperature": entry.get("bed_temperature"),
  1353. "nozzle_temperature": entry.get("nozzle_temperature"),
  1354. "sliced_for_model": entry.get("sliced_for_model"),
  1355. "status": status,
  1356. "started_at": started_at,
  1357. "completed_at": _parse_dt(entry.get("completed_at")),
  1358. "makerworld_url": entry.get("makerworld_url"),
  1359. "designer": entry.get("designer"),
  1360. "external_url": entry.get("external_url"),
  1361. "is_favorite": bool(entry.get("is_favorite")),
  1362. "tags": entry.get("tags"),
  1363. "notes": entry.get("notes"),
  1364. "cost": numbers["cost"],
  1365. "failure_reason": failure_reason,
  1366. "quantity": quantity,
  1367. "energy_kwh": numbers["energy_kwh"],
  1368. "energy_cost": numbers["energy_cost"],
  1369. "wear_cost": numbers["wear_cost"],
  1370. }
  1371. printer_id = entry.get("printer_id")
  1372. if printer_id is not None and printer_id not in valid_printers:
  1373. tally.note(
  1374. "archivesPrinterMissing", "Some archives referenced printers that no longer exist — link cleared"
  1375. )
  1376. printer_id = None
  1377. project_id = entry.get("project_id")
  1378. if project_id is not None and project_id not in valid_projects:
  1379. tally.note(
  1380. "archivesProjectMissing", "Some archives referenced projects that no longer exist — link cleared"
  1381. )
  1382. project_id = None
  1383. fields["printer_id"] = printer_id
  1384. fields["project_id"] = project_id
  1385. # The ownership pair and deleted_at are the late arrivals — a backup
  1386. # commit taken before the collector wrote them carries neither key.
  1387. # Absent is NOT the same as null here, because the overwrite branch
  1388. # below is a blanket setattr: treating a missing key as None would
  1389. # write NULL over a live owner (_ensure_archive_visible then 404s the
  1390. # archive for the very user who owns it — the failure carrying the
  1391. # column was added to fix) and silently un-delete a row the user
  1392. # deleted. So only carry a column the backup actually knows about;
  1393. # on insert, an absent key just takes the model default.
  1394. # An owner the backup names but this instance cannot resolve is the
  1395. # same epistemic state as an absent key — we do not know who owns
  1396. # this archive — so it takes the same action: the column is left out
  1397. # of ``fields`` entirely rather than set to None. Writing NULL there
  1398. # would take the owner away from a local archive that has a perfectly
  1399. # good one, which is the 404-for-its-own-owner failure this column is
  1400. # carried across to fix, and it would do it on the overwrite path
  1401. # where there is a local answer to keep. On insert there is nothing
  1402. # to keep, so the row takes the model default and lands ownerless,
  1403. # which is what the note says.
  1404. owner_cleared = False
  1405. backup_username = entry.get("created_by_username")
  1406. if isinstance(backup_username, str) and backup_username:
  1407. # The natural-key path. A miss here is a user renamed or deleted
  1408. # since the backup, and there is nothing else to resolve on: the
  1409. # id alongside it is from the source instance's numbering, so
  1410. # trusting it is exactly the misattribution the name is here to
  1411. # prevent. Not a reason to fail the row — the archive is still
  1412. # worth having, and an admin can reassign it — but said out loud
  1413. # on insert, because an ownerless archive is not silent-safe.
  1414. created_by_id = users_by_name.get(backup_username)
  1415. if created_by_id is None:
  1416. if existing is None:
  1417. tally.note(
  1418. "archivesOwnerUnmatched",
  1419. "Some archives name an owner this instance does not have — owner cleared rather than "
  1420. "guessed from the backup's user id, so they are visible only to users with the "
  1421. "archives:read_all permission until an admin reassigns them",
  1422. )
  1423. owner_cleared = True
  1424. else:
  1425. fields["created_by_id"] = created_by_id
  1426. elif "created_by_id" in entry:
  1427. # Fallback for a commit taken before the collector recorded the
  1428. # username. Validated rather than trusted, so a *stale* id is
  1429. # dropped instead of pointing somewhere wrong; a live id
  1430. # belonging to a different person on a rebuilt instance is the
  1431. # case this path cannot see, and is why the branch above exists.
  1432. # An explicit null is not a miss — the backup is saying the
  1433. # archive had no owner — so it is written, and overwrite keeps
  1434. # meaning "make the local row match the backup".
  1435. created_by_id = entry.get("created_by_id")
  1436. if created_by_id is not None and created_by_id not in valid_users:
  1437. if existing is None:
  1438. tally.note(
  1439. "archivesOwnerCleared",
  1440. "Some archives referenced users that no longer exist — owner cleared, so they are "
  1441. "visible only to users with the archives:read_all permission until an admin "
  1442. "reassigns them",
  1443. )
  1444. owner_cleared = True
  1445. else:
  1446. fields["created_by_id"] = created_by_id
  1447. if "deleted_at" in entry:
  1448. # A soft-deleted archive is still in the backup (its row is kept
  1449. # so stats keep counting it), so carry the flag across or the
  1450. # restore turns something the user deleted back into a visible
  1451. # archive.
  1452. fields["deleted_at"] = _parse_dt(entry.get("deleted_at"))
  1453. if existing is not None:
  1454. claimed.add(existing.id)
  1455. if old_id is not None:
  1456. id_map[old_id] = existing.id
  1457. if not overwrite:
  1458. tally.skipped += 1
  1459. continue
  1460. # Overwrite means "make the local row match the backup", which
  1461. # includes un-deleting one the user deleted after the backup was
  1462. # taken. Legitimate, but not obvious from a restored/skipped
  1463. # count, so say it.
  1464. if existing.deleted_at is not None and "deleted_at" in fields and fields["deleted_at"] is None:
  1465. tally.note(
  1466. "archivesUndeleted",
  1467. "Archive(s) deleted since the backup are visible again — overwrite was on",
  1468. )
  1469. if backed_up_in_flight:
  1470. # The local row is that same print, still running or since
  1471. # ended: it knows more about its outcome than the backup's
  1472. # snapshot of it mid-way. Closing it here would end a live
  1473. # print's archive, or overwrite a real result.
  1474. for key in ("status", "failure_reason", "completed_at"):
  1475. fields.pop(key, None)
  1476. for key, value in fields.items():
  1477. setattr(existing, key, value)
  1478. tally.restored += 1
  1479. continue
  1480. if not warned_files:
  1481. tally.note(
  1482. "archivesMetadataOnly",
  1483. "Restored archives carry metadata only — the 3MF and thumbnail files are not in a Git backup",
  1484. )
  1485. warned_files = True
  1486. # Insert-only, and the mirror of the rule above: an owner the backup
  1487. # cannot tell us is never written, so on overwrite the local one
  1488. # survives — but there is no local row here to fall back on, so the
  1489. # archive lands ownerless, a 404 for everyone without
  1490. # archives:read_all. Three ways to get here: a commit taken before
  1491. # the collector recorded the column (every pre-#2656 backup), an
  1492. # archive that genuinely had no owner on the source instance, or one
  1493. # whose owner this instance cannot resolve. All restore fine and all
  1494. # were silent, so the tally said "N archives restored" while the user
  1495. # who asked for them saw none. The unresolved cases above already
  1496. # said their piece; don't say it twice for the same row.
  1497. if fields.get("created_by_id") is None and not owner_cleared:
  1498. tally.note(
  1499. "archivesOwnerUnknown",
  1500. "Some archives were restored without an owner — this backup does not record one, so they "
  1501. "are visible only to users with the archives:read_all permission until an admin reassigns "
  1502. "them",
  1503. )
  1504. row = PrintArchive(
  1505. filename=entry.get("filename") or "restored-from-backup",
  1506. file_path="",
  1507. file_size=entry.get("file_size") or 0,
  1508. content_hash=entry.get("content_hash"),
  1509. # The file is not in the backup, so its G-code cannot be read
  1510. # here: the backup's fingerprint lets a print of the same slice
  1511. # sent from the slicer join this archive and give it the file
  1512. # (#1634). Older backups have none, and the fill marks it read.
  1513. gcode_hash=_gcode_hash_of(entry),
  1514. **fields,
  1515. )
  1516. created_at = _parse_dt(entry.get("created_at"))
  1517. if created_at is not None:
  1518. row.created_at = created_at
  1519. db.add(row)
  1520. await db.flush()
  1521. claimed.add(row.id)
  1522. if old_id is not None:
  1523. id_map[old_id] = row.id
  1524. tally.restored += 1
  1525. for merged, into in aliases.items():
  1526. if into in id_map:
  1527. id_map[merged] = id_map[into]
  1528. async def _find_archive(
  1529. self, db: AsyncSession, entry: dict, started_at: datetime | None, run_starts: Iterable[datetime] = ()
  1530. ) -> PrintArchive | None:
  1531. """Match a backed-up archive to a local row by natural key.
  1532. ``started_at`` is nullable and genuinely NULL for a whole class of rows —
  1533. the re-slice path in ``library.py`` constructs ``PrintArchive`` without
  1534. one — so it cannot be *required* by the key. It narrows the match instead:
  1535. a backed-up row with no ``started_at`` matches a local row that has none
  1536. either. Requiring it meant those archives never matched, so each restore
  1537. re-inserted them as duplicates and overwrite mode could never update them.
  1538. ``content_hash`` identifies the sliced file on its own, which is why it is
  1539. the branch allowed to run without a ``started_at``; ``filename`` is too
  1540. weak for that (re-slices share it) and still requires one. Two backed-up
  1541. rows sharing a hash *and* having no ``started_at`` are indistinguishable
  1542. in the backup, so they collapse onto one local row — better than
  1543. duplicating both on every restore.
  1544. Soft-deleted rows are matched deliberately: there is no ``deleted_at``
  1545. filter here because the row still exists, and matching it is what stops a
  1546. restore inserting a live duplicate of an archive the user has deleted.
  1547. """
  1548. created_at = _parse_dt(entry.get("created_at"))
  1549. # A print of it starting then, here or in the backup: printed again
  1550. # since, the archive's own start is a later print's (archive redesign
  1551. # P3).
  1552. starts = set(run_starts) | ({started_at} if started_at else set())
  1553. async def best(where, *, own_start: bool) -> PrintArchive | None:
  1554. rows = (await db.execute(select(PrintArchive).where(where).order_by(PrintArchive.id))).scalars().all()
  1555. if not rows:
  1556. return None
  1557. # Its own row first, by its own start: one merged since is then
  1558. # followed by the caller rather than taken for its survivor.
  1559. if own_start:
  1560. for row in rows:
  1561. if row.started_at == started_at:
  1562. return row
  1563. if starts:
  1564. # Core table, so a print running now counts too.
  1565. runs = PrintLogEntry.__table__.c
  1566. printed = set(
  1567. (
  1568. await db.execute(
  1569. select(runs.archive_id).where(
  1570. runs.archive_id.in_([row.id for row in rows]), runs.started_at.in_(starts)
  1571. )
  1572. )
  1573. )
  1574. .scalars()
  1575. .all()
  1576. )
  1577. for row in rows:
  1578. if row.id in printed:
  1579. return row
  1580. # When it was archived, which no print changes: an archive printed
  1581. # since, without a start or print in the backup to go by.
  1582. return next((row for row in rows if _created_at_matches(row, created_at)), None)
  1583. content_hash = entry.get("content_hash")
  1584. if content_hash:
  1585. row = await best(PrintArchive.content_hash == content_hash, own_start=True)
  1586. if row is not None:
  1587. return row
  1588. filename = entry.get("filename")
  1589. if filename:
  1590. return await best(PrintArchive.filename == filename, own_start=started_at is not None)
  1591. return None
  1592. async def _restore_runs(
  1593. self,
  1594. db: AsyncSession,
  1595. payload,
  1596. overwrite: bool,
  1597. tally: _CategoryTally,
  1598. id_map: dict[int, int],
  1599. ) -> None:
  1600. """Each print of the restored archives (archive redesign P3).
  1601. After the archives, through their ``old_id -> new_id`` map. A print
  1602. already here is the same archive's at the same start on the same
  1603. plate; it is left alone, or made to match the backup with overwrite.
  1604. Each archive that gained or changed a print then takes its latest
  1605. print's outcome, as everywhere else. A backup written before prints
  1606. were backed up has none, and its archives restore as before.
  1607. """
  1608. from backend.app.services.print_log import mirror_latest_run
  1609. runs = _runs_of(payload)
  1610. if not runs:
  1611. return
  1612. valid_printers = set((await db.execute(select(Printer.id))).scalars().all())
  1613. users_by_name = {name: user_id for user_id, name in (await db.execute(select(User.id, User.username))).all()}
  1614. touched: set[int] = set()
  1615. for entry in runs:
  1616. if not isinstance(entry, dict):
  1617. tally.failed += 1
  1618. continue
  1619. backup_archive = entry.get("archive_id")
  1620. archive_id = None
  1621. if backup_archive is not None:
  1622. archive_id = id_map.get(backup_archive) if isinstance(backup_archive, int) else None
  1623. if archive_id is None:
  1624. # Its archive is not in this restore (it failed, or the
  1625. # backup is damaged): there is nothing to attach it to.
  1626. tally.skipped += 1
  1627. tally.note("runsArchiveMissing", "Some prints were skipped - their archive is not in this restore")
  1628. continue
  1629. started_at = _parse_dt(entry.get("started_at"))
  1630. created_at = _parse_dt(entry.get("created_at"))
  1631. plate_id = entry.get("plate_id") if isinstance(entry.get("plate_id"), int) else None
  1632. existing = await self._find_run(db, archive_id, started_at, plate_id, created_at, entry)
  1633. numbers = {
  1634. "duration_seconds": _restored_number(entry.get("duration_seconds"), integer=True),
  1635. "filament_used_grams": _restored_number(entry.get("filament_used_grams"), maximum=_MAX_RESTORED_GRAMS),
  1636. "cost": _restored_number(entry.get("cost")),
  1637. "energy_kwh": _restored_number(entry.get("energy_kwh")),
  1638. "energy_cost": _restored_number(entry.get("energy_cost")),
  1639. "wear_cost": _restored_number(entry.get("wear_cost")),
  1640. "quantity": _restored_number(entry.get("quantity"), maximum=10_000, integer=True),
  1641. }
  1642. if any(entry.get(key) is not None and value is None for key, value in numbers.items()):
  1643. tally.note(
  1644. "runsNumbersDropped",
  1645. "Some prints had grams, costs, energy or durations that are not valid numbers - left empty",
  1646. )
  1647. # A print running when the backup was made is long over: closed
  1648. # as its archive is (see _restore_archives).
  1649. status = entry.get("status") if isinstance(entry.get("status"), str) else "completed"
  1650. failure_reason = entry.get("failure_reason")
  1651. in_flight = status in (RUN_OPEN, "paused")
  1652. if in_flight:
  1653. status, failure_reason = "cancelled", "noStatusUpdate"
  1654. verdict = entry.get("user_verdict") if entry.get("user_verdict") in _RUN_VERDICTS else None
  1655. printer_id = entry.get("printer_id")
  1656. if printer_id is not None and printer_id not in valid_printers:
  1657. tally.note(
  1658. "runsPrinterMissing",
  1659. "Some prints referenced printers that no longer exist - link cleared, their printer name kept",
  1660. )
  1661. printer_id = None
  1662. username = entry.get("created_by_username")
  1663. owner_name = entry.get("owner_username")
  1664. fields = {
  1665. "archive_id": archive_id,
  1666. "print_name": entry.get("print_name"),
  1667. "plate_id": plate_id,
  1668. "printer_id": printer_id,
  1669. "printer_name": entry.get("printer_name"),
  1670. "status": status,
  1671. "started_at": started_at,
  1672. "completed_at": _parse_dt(entry.get("completed_at")),
  1673. "duration_unmeasured": entry.get("duration_unmeasured"),
  1674. "ended_offline": entry.get("ended_offline"),
  1675. "filament_type": entry.get("filament_type"),
  1676. "filament_color": entry.get("filament_color"),
  1677. "failure_reason": failure_reason,
  1678. "user_verdict": verdict,
  1679. "user_verdict_source": entry.get("user_verdict_source") if verdict else None,
  1680. "user_verdict_at": _parse_dt(entry.get("user_verdict_at")) if verdict else None,
  1681. "notes": entry.get("notes"),
  1682. "created_by_username": username if isinstance(username, str) else None,
  1683. **numbers,
  1684. }
  1685. # By name, never the backup's id, and only the owner's: the
  1686. # recorded name stays when its user is deleted, and may be someone
  1687. # else's by now. A name this instance does not have is left out,
  1688. # as for archives: a local print keeps its owner, a new one has
  1689. # none and keeps the name.
  1690. owner = users_by_name.get(owner_name) if isinstance(owner_name, str) else None
  1691. if owner is not None or not isinstance(owner_name, str):
  1692. fields["created_by_id"] = owner
  1693. if existing is not None:
  1694. # The local print knows more than the backup's snapshot of it
  1695. # mid-way (its figures came at its end), and one printing now
  1696. # is still being written by its print.
  1697. if not overwrite or in_flight or existing.status == RUN_OPEN:
  1698. tally.skipped += 1
  1699. continue
  1700. for key, value in fields.items():
  1701. setattr(existing, key, value)
  1702. tally.restored += 1
  1703. else:
  1704. row = PrintLogEntry(**fields)
  1705. if created_at is not None:
  1706. # Statistics date a print by it.
  1707. row.created_at = created_at
  1708. db.add(row)
  1709. await db.flush()
  1710. tally.restored += 1
  1711. if archive_id is not None:
  1712. touched.add(archive_id)
  1713. for archive_id in touched:
  1714. await mirror_latest_run(db, archive_id)
  1715. async def _find_run(
  1716. self,
  1717. db: AsyncSession,
  1718. archive_id: int | None,
  1719. started_at: datetime | None,
  1720. plate_id: int | None,
  1721. created_at: datetime | None,
  1722. entry: dict,
  1723. ) -> PrintLogEntry | None:
  1724. """The same print already here: its archive's, started then, on that
  1725. plate. Without a start, by when it was logged (compared in Python, see
  1726. _created_at_matches); one without an archive by its printer and name
  1727. too. A print running now counts."""
  1728. query = (
  1729. select(PrintLogEntry)
  1730. .where(
  1731. PrintLogEntry.archive_id == archive_id
  1732. if archive_id is not None
  1733. else PrintLogEntry.archive_id.is_(None),
  1734. PrintLogEntry.plate_id == plate_id if plate_id is not None else PrintLogEntry.plate_id.is_(None),
  1735. PrintLogEntry.started_at == started_at if started_at else PrintLogEntry.started_at.is_(None),
  1736. )
  1737. .order_by(PrintLogEntry.id)
  1738. .execution_options(**{INCLUDE_OPEN_RUNS: True})
  1739. )
  1740. if archive_id is None:
  1741. query = query.where(
  1742. PrintLogEntry.printer_name == entry.get("printer_name"),
  1743. PrintLogEntry.print_name == entry.get("print_name"),
  1744. )
  1745. candidates = (await db.execute(query)).scalars().all()
  1746. if started_at:
  1747. return candidates[0] if candidates else None
  1748. return next((row for row in candidates if _created_at_matches(row, created_at)), None)
  1749. async def _restore_spools(
  1750. self,
  1751. db: AsyncSession,
  1752. inventory,
  1753. usage_payload,
  1754. overwrite: bool,
  1755. tally: _CategoryTally,
  1756. archive_id_map: dict[int, int],
  1757. ) -> None:
  1758. spools = inventory.get("spools") if isinstance(inventory, dict) else None
  1759. if not isinstance(spools, list):
  1760. usage = usage_payload.get("usage_history") if isinstance(usage_payload, dict) else None
  1761. if isinstance(usage, list) and usage:
  1762. # A Spoolman-mode backup: usage of Spoolman spools, no built-in
  1763. # spools to go with it (#3407).
  1764. await self._restore_spool_usage(db, usage_payload, tally, {}, archive_id_map)
  1765. return
  1766. tally.note("noData", "No data of this kind in this backup")
  1767. return
  1768. spool_id_map: dict[int, int] = {}
  1769. tags_kept = 0
  1770. for entry in spools:
  1771. if not isinstance(entry, dict):
  1772. tally.failed += 1
  1773. continue
  1774. old_id = entry.get("id") if isinstance(entry.get("id"), int) else None
  1775. existing, matched_on = await self._find_spool(db, entry)
  1776. fields = {
  1777. "material": entry.get("material") or "PLA",
  1778. "subtype": entry.get("subtype"),
  1779. "color_name": entry.get("color_name"),
  1780. "rgba": entry.get("rgba"),
  1781. "brand": entry.get("brand"),
  1782. "label_weight": entry.get("label_weight") or 1000,
  1783. "core_weight": entry.get("core_weight") or 250,
  1784. "weight_used": entry.get("weight_used") or 0,
  1785. "weight_locked": bool(entry.get("weight_locked")),
  1786. "slicer_filament": entry.get("slicer_filament"),
  1787. "slicer_filament_name": entry.get("slicer_filament_name"),
  1788. "nozzle_temp_min": entry.get("nozzle_temp_min"),
  1789. "nozzle_temp_max": entry.get("nozzle_temp_max"),
  1790. "note": entry.get("note"),
  1791. "cost_per_kg": entry.get("cost_per_kg"),
  1792. "tag_uid": entry.get("tag_uid"),
  1793. "tray_uuid": entry.get("tray_uuid"),
  1794. "data_origin": entry.get("data_origin"),
  1795. "tag_type": entry.get("tag_type"),
  1796. "archived_at": _parse_dt(entry.get("archived_at")),
  1797. }
  1798. # The user's own bookkeeping on the spool: purchasing number
  1799. # (#2870), category and low-stock override (#729), free-text
  1800. # storage. Added to the backup format after the fields above, so
  1801. # they are applied only when the file actually carries them —
  1802. # an older backup must not wipe what the live row holds.
  1803. # `location_id` is deliberately absent: the locations table is
  1804. # not backed up, so the ID would point at whatever owns it here.
  1805. for late_field in ("material_number", "category", "low_stock_threshold_pct", "storage_location"):
  1806. if late_field in entry:
  1807. fields[late_field] = entry[late_field]
  1808. # The "Reset Total Consumed" point (WU-7), same late-field rule; a
  1809. # value that is no amount of filament keeps what the row holds.
  1810. baseline = _restored_number(entry.get("weight_used_baseline"))
  1811. if baseline is not None:
  1812. fields["weight_used_baseline"] = baseline
  1813. # When it was last used (LU-4), same late-field rule.
  1814. if "last_used" in entry:
  1815. fields["last_used"] = _parse_dt(entry.get("last_used"))
  1816. # Last drying (#2863), same late-field rule.
  1817. if "last_dried_at" in entry:
  1818. fields["last_dried_at"] = _parse_dt(entry.get("last_dried_at"))
  1819. fields["last_dried_temp"] = entry.get("last_dried_temp")
  1820. fields["last_dried_hours"] = entry.get("last_dried_hours")
  1821. if existing is not None:
  1822. if old_id is not None:
  1823. spool_id_map[old_id] = existing.id
  1824. if not overwrite:
  1825. tally.skipped += 1
  1826. continue
  1827. tags_kept += await self._guard_tag_overwrite(db, existing, fields, matched_on)
  1828. for key, value in fields.items():
  1829. setattr(existing, key, value)
  1830. tally.restored += 1
  1831. continue
  1832. row = Spool(**fields)
  1833. # Carry the original created_at across. Without it the row would be
  1834. # stamped "now", and the composite fallback in _find_spool (which
  1835. # keys on created_at) would miss on a second restore and insert a
  1836. # duplicate instead of matching.
  1837. created_at = _parse_dt(entry.get("created_at"))
  1838. if created_at is not None:
  1839. row.created_at = created_at
  1840. db.add(row)
  1841. await db.flush()
  1842. if old_id is not None:
  1843. spool_id_map[old_id] = row.id
  1844. tally.restored += 1
  1845. if tags_kept:
  1846. tally.note(
  1847. "spoolTagKept",
  1848. f"{tags_kept} spool tag(s) left as they are — the backup would have cleared a tag that "
  1849. "has since been scanned, or moved one onto a second spool.",
  1850. count=tags_kept,
  1851. )
  1852. await self._restore_spool_usage(db, usage_payload, tally, spool_id_map, archive_id_map)
  1853. async def _find_spool(self, db: AsyncSession, entry: dict) -> tuple[Spool | None, str | None]:
  1854. """Match a backed-up spool to a local row, and say which key matched.
  1855. Physical identity first (an RFID/Bambu tag is the spool), then a
  1856. descriptive composite including ``created_at`` so two otherwise
  1857. identical spools added at different times stay distinct.
  1858. The second element names the column that matched — ``"tag_uid"``,
  1859. ``"tray_uuid"`` or ``None`` for the composite. ``_guard_tag_overwrite``
  1860. needs it: the matched column holds the incoming value by definition, so
  1861. it is the *other* one that overwrite can corrupt.
  1862. """
  1863. tag_uid = entry.get("tag_uid")
  1864. if tag_uid:
  1865. result = await db.execute(select(Spool).where(Spool.tag_uid == tag_uid))
  1866. row = result.scalars().first()
  1867. if row is not None:
  1868. return row, "tag_uid"
  1869. tray_uuid = entry.get("tray_uuid")
  1870. if tray_uuid:
  1871. result = await db.execute(select(Spool).where(Spool.tray_uuid == tray_uuid))
  1872. row = result.scalars().first()
  1873. if row is not None:
  1874. return row, "tray_uuid"
  1875. created_at = _parse_dt(entry.get("created_at"))
  1876. if created_at is None:
  1877. return None, None
  1878. # created_at is filtered in Python, not here — see _created_at_matches.
  1879. result = await db.execute(
  1880. select(Spool).where(
  1881. Spool.material == (entry.get("material") or "PLA"),
  1882. Spool.brand == entry.get("brand"),
  1883. Spool.subtype == entry.get("subtype"),
  1884. Spool.color_name == entry.get("color_name"),
  1885. )
  1886. )
  1887. for row in result.scalars():
  1888. if _created_at_matches(row, created_at):
  1889. return row, None
  1890. return None, None
  1891. @staticmethod
  1892. async def _guard_tag_overwrite(db: AsyncSession, existing: Spool, fields: dict, matched_on: str | None) -> int:
  1893. """Remove tag columns from ``fields`` that an overwrite would corrupt.
  1894. ``tag_uid`` and ``tray_uuid`` are both in ``fields`` and overwrite is a
  1895. blanket ``setattr`` loop, so a spool matched on one key gets the backup's
  1896. *other* key written onto it. Neither column has a unique constraint
  1897. (``models/spool.py``, and no unique index in the migrations), so nothing
  1898. errors — a duplicate tag simply appears, after which ``_find_spool``'s
  1899. ``.first()`` is non-deterministic and an AMS tag lookup resolves to an
  1900. arbitrary one of the two spools. The same loop can also *clear* a tag the
  1901. user has scanned since the backup was taken, when the backup entry holds
  1902. ``None``.
  1903. Two refusals, and the row is otherwise overwritten as normal:
  1904. * the incoming value is empty and the local row has one — the backup
  1905. predates the scan, so the local tag is the newer fact;
  1906. * the incoming value is already held by a different local spool — writing
  1907. it would create the duplicate described above.
  1908. Returns how many columns were left alone, so the caller can say so in the
  1909. tally rather than doing it silently.
  1910. """
  1911. kept = 0
  1912. for column in ("tag_uid", "tray_uuid"):
  1913. # The column we matched on already holds the incoming value.
  1914. if column == matched_on:
  1915. continue
  1916. incoming = fields.get(column)
  1917. current = getattr(existing, column)
  1918. if incoming == current:
  1919. continue
  1920. if not incoming:
  1921. if current:
  1922. fields.pop(column)
  1923. kept += 1
  1924. continue
  1925. clash = await db.execute(
  1926. select(Spool.id).where(getattr(Spool, column) == incoming, Spool.id != existing.id)
  1927. )
  1928. if clash.scalars().first() is not None:
  1929. fields.pop(column)
  1930. kept += 1
  1931. return kept
  1932. async def _restore_spool_usage(
  1933. self,
  1934. db: AsyncSession,
  1935. usage_payload,
  1936. tally: _CategoryTally,
  1937. spool_id_map: dict[int, int],
  1938. archive_id_map: dict[int, int],
  1939. ) -> None:
  1940. usage = usage_payload.get("usage_history") if isinstance(usage_payload, dict) else None
  1941. if not isinstance(usage, list) or not usage:
  1942. return
  1943. valid_printers = set((await db.execute(select(Printer.id))).scalars().all())
  1944. unresolved = 0
  1945. unlinked_archives = 0
  1946. for entry in usage:
  1947. if not isinstance(entry, dict):
  1948. tally.failed += 1
  1949. continue
  1950. old_spool_id = entry.get("spool_id")
  1951. spool_id = spool_id_map.get(old_spool_id) if isinstance(old_spool_id, int) else None
  1952. # Usage of a spool deleted before the backup has no spool, and is
  1953. # restored without one; only a spool id that resolves to nothing is skipped.
  1954. if spool_id is None and old_spool_id is not None:
  1955. # The parent spool never made it into the map: the backup's spool
  1956. # list didn't include it, or its entry carried no integer id. A
  1957. # spool that was merely *skipped* (matched locally, overwrite off)
  1958. # is mapped a few lines up in _restore_spools, so it never lands
  1959. # here — which is why the note below offers no remedy.
  1960. unresolved += 1
  1961. tally.skipped += 1
  1962. continue
  1963. created_at = _parse_dt(entry.get("created_at"))
  1964. # Usage history has no natural key of its own, so dedupe on the
  1965. # tuple that makes a consumption event unique in practice. As in
  1966. # _find_spool, created_at is compared in Python — see
  1967. # _created_at_matches. An entry carrying no created_at at all
  1968. # cannot be recognised and is re-inserted, which is what the
  1969. # IS NULL comparison this replaced did too: the column is
  1970. # non-nullable, so it never matched either.
  1971. # Spoolman's ids are its own, so they come back as they were.
  1972. spoolman_spool_id = entry.get("spoolman_spool_id")
  1973. if not isinstance(spoolman_spool_id, int) or isinstance(spoolman_spool_id, bool):
  1974. spoolman_spool_id = None
  1975. existing = await db.execute(
  1976. select(SpoolUsageHistory).where(
  1977. SpoolUsageHistory.spool_id == spool_id,
  1978. SpoolUsageHistory.spoolman_spool_id == spoolman_spool_id,
  1979. SpoolUsageHistory.weight_used == (entry.get("weight_used") or 0),
  1980. SpoolUsageHistory.print_name == entry.get("print_name"),
  1981. )
  1982. )
  1983. if any(_created_at_matches(row, created_at) for row in existing.scalars()):
  1984. tally.skipped += 1
  1985. continue
  1986. printer_id = entry.get("printer_id")
  1987. if printer_id is not None and printer_id not in valid_printers:
  1988. printer_id = None
  1989. old_archive_id = entry.get("archive_id")
  1990. archive_id = archive_id_map.get(old_archive_id) if isinstance(old_archive_id, int) else None
  1991. if archive_id is None and isinstance(old_archive_id, int):
  1992. # Restoring spools without archives leaves archive_id_map empty,
  1993. # so every "this print consumed that spool" link is dropped — the
  1994. # local archive may well exist, but its payload wasn't fetched,
  1995. # so there is no natural key here to match it on. Nor is it
  1996. # repairable by a later archives-only restore: the dedupe key
  1997. # above doesn't include archive_id, so these rows are recognised
  1998. # as already-present and skipped. Worth telling the user while
  1999. # they can still redo the run with both categories ticked.
  2000. unlinked_archives += 1
  2001. weight_used = _restored_number(entry.get("weight_used"))
  2002. percent_used = _restored_number(entry.get("percent_used"), integer=True)
  2003. cost = _restored_number(entry.get("cost"))
  2004. if any(
  2005. entry.get(key) is not None and value is None
  2006. for key, value in (("weight_used", weight_used), ("percent_used", percent_used), ("cost", cost))
  2007. ):
  2008. tally.note(
  2009. "usageNumbersDropped",
  2010. "Some usage entries had weights or costs that are not valid numbers - left empty",
  2011. )
  2012. row = SpoolUsageHistory(
  2013. spool_id=spool_id,
  2014. spoolman_spool_id=spoolman_spool_id,
  2015. printer_id=printer_id,
  2016. print_name=entry.get("print_name"),
  2017. archive_id=archive_id,
  2018. weight_used=weight_used or 0,
  2019. percent_used=percent_used or 0,
  2020. status=entry.get("status") or "completed",
  2021. cost=cost,
  2022. )
  2023. if created_at is not None:
  2024. row.created_at = created_at
  2025. db.add(row)
  2026. tally.restored += 1
  2027. if unresolved:
  2028. tally.note(
  2029. "spoolUsageUnresolved",
  2030. f"{unresolved} usage record(s) skipped — their spool is not in this backup's "
  2031. "spool list, so there is nothing to attach them to.",
  2032. count=unresolved,
  2033. )
  2034. if unlinked_archives:
  2035. tally.note(
  2036. "spoolUsageUnlinked",
  2037. f"{unlinked_archives} usage record(s) restored without their print-history link — "
  2038. "select Print archives alongside Spool inventory to keep it.",
  2039. count=unlinked_archives,
  2040. )
  2041. async def _restore_settings(
  2042. self,
  2043. db: AsyncSession,
  2044. payload,
  2045. overwrite: bool,
  2046. tally: _CategoryTally,
  2047. keys_written: set[str] | None = None,
  2048. ) -> None:
  2049. values = payload.get("settings") if isinstance(payload, dict) else None
  2050. if not isinstance(values, dict):
  2051. tally.note("noData", "No data of this kind in this backup")
  2052. return
  2053. # Planned before the first write, so the companion rule reads genuinely
  2054. # pre-restore local state, and so the preview and this run classify the
  2055. # payload identically.
  2056. plan = await self._plan_settings(db, values)
  2057. refused = plan.refused
  2058. for key, value in values.items():
  2059. if not isinstance(key, str) or not key:
  2060. tally.failed += 1
  2061. continue
  2062. if key in refused:
  2063. # Refusals are reported in the notes and nowhere else. They are
  2064. # already outside the preview's item count, and the preview is
  2065. # the number the user was shown, so counting them here would
  2066. # make restored + skipped + failed exceed it. The two skips
  2067. # below stay counted because they depend on this run's flags,
  2068. # which the preview cannot see.
  2069. continue
  2070. if value is None:
  2071. tally.skipped += 1
  2072. continue
  2073. result = await db.execute(select(Settings).where(Settings.key == key))
  2074. existing = result.scalar_one_or_none()
  2075. if existing is not None:
  2076. if not overwrite:
  2077. tally.skipped += 1
  2078. continue
  2079. existing.value = plan.stored[key]
  2080. tally.restored += 1
  2081. if keys_written is not None:
  2082. keys_written.add(key)
  2083. continue
  2084. db.add(Settings(key=key, value=plan.stored[key]))
  2085. tally.restored += 1
  2086. if keys_written is not None:
  2087. keys_written.add(key)
  2088. # A backup taken before a printer delete still lists it, and SQLite gives
  2089. # its id to the next printer created.
  2090. await drop_printers_from_obico(db)
  2091. if plan.blocked:
  2092. tally.note(
  2093. "settingsCredentialsSkipped",
  2094. f"{len(plan.blocked)} credential-like key(s) skipped — re-enter secrets manually",
  2095. count=len(plan.blocked),
  2096. )
  2097. if plan.protected:
  2098. tally.note(
  2099. "settingsAuthSkipped",
  2100. f"{len(plan.protected)} authentication setting(s) skipped — change those in Settings > "
  2101. "Authentication so the lockout checks still run",
  2102. count=len(plan.protected),
  2103. )
  2104. if plan.companion:
  2105. keys = ", ".join(sorted(plan.companion))
  2106. tally.note(
  2107. "settingsCompanionSkipped",
  2108. f"{keys} left switched off — the credential each one needs cannot be restored from a "
  2109. "backup and this instance has none stored, so switching them on would leave the "
  2110. "integration unauthenticated",
  2111. keys=keys,
  2112. count=len(plan.companion),
  2113. )
  2114. if plan.two_factor:
  2115. tally.note(
  2116. "settingsTwoFactorSkipped",
  2117. f"{len(plan.two_factor)} email sign-in code setting(s) skipped - each user switches email codes "
  2118. "on or off in their own account",
  2119. count=len(plan.two_factor),
  2120. )
  2121. if plan.address:
  2122. keys = ", ".join(sorted(plan.address))
  2123. tally.note(
  2124. "settingsAddressKept",
  2125. f"{keys} left as they are - they decide where Bambuddy sends data, emails or backups; change them "
  2126. "in Settings",
  2127. keys=keys,
  2128. count=len(plan.address),
  2129. )
  2130. if plan.invalid:
  2131. keys = ", ".join(sorted(plan.invalid))
  2132. tally.note(
  2133. "settingsInvalidSkipped",
  2134. f"{keys} skipped - the value in the backup is not one Settings would accept",
  2135. keys=keys,
  2136. count=len(plan.invalid),
  2137. )
  2138. if plan.unknown:
  2139. keys = ", ".join(sorted(plan.unknown))
  2140. tally.note(
  2141. "settingsUnknownSkipped",
  2142. f"{keys} skipped - not settings a backup restores (runtime state, account sessions or unknown keys)",
  2143. keys=keys,
  2144. count=len(plan.unknown),
  2145. )
  2146. if plan.destination:
  2147. keys = ", ".join(sorted(plan.destination))
  2148. tally.note(
  2149. "settingsDestinationKept",
  2150. f"{keys} left as they are - this instance has a stored credential that would otherwise be sent "
  2151. "to the address in the backup. Change them in Settings and enter the credential again.",
  2152. keys=keys,
  2153. count=len(plan.destination),
  2154. )
  2155. async def _reconfigure_mqtt_relay(self, db: AsyncSession, keys_written: set[str], tally: _CategoryTally) -> None:
  2156. """Push restored mqtt_* settings into the live relay.
  2157. The relay reads its broker config once, at configure() time — the
  2158. settings PUT handler reconfigures it for exactly this reason
  2159. (api/routes/settings.py). Writing the rows alone left the relay on the
  2160. pre-restore broker until the next backend restart while the UI showed
  2161. the restored values, which is the one way a restore could look applied
  2162. and not be.
  2163. Called after the commit, never before: configure() tears the connection
  2164. down and rebuilds it, so it must not run against values a later failure
  2165. could roll back. Only mqtt_password can't come back this way (the
  2166. credential blocklist skips it) — the row already in the database is
  2167. reused, so an unchanged broker keeps working.
  2168. """
  2169. if not _MQTT_SETTING_KEYS & keys_written:
  2170. return
  2171. try:
  2172. from backend.app.services.mqtt_relay import mqtt_relay
  2173. rows = await db.execute(select(Settings).where(Settings.key.in_(_MQTT_SETTING_KEYS)))
  2174. stored = {s.key: s.value for s in rows.scalars().all()}
  2175. # Same shape and defaults the settings PUT handler builds.
  2176. await mqtt_relay.configure(
  2177. {
  2178. "mqtt_enabled": (stored.get("mqtt_enabled") or "false") == "true",
  2179. "mqtt_broker": stored.get("mqtt_broker") or "",
  2180. "mqtt_port": int(stored.get("mqtt_port") or "1883"),
  2181. "mqtt_username": stored.get("mqtt_username") or "",
  2182. "mqtt_password": stored.get("mqtt_password") or "",
  2183. "mqtt_topic_prefix": stored.get("mqtt_topic_prefix") or "bambuddy",
  2184. "mqtt_use_tls": (stored.get("mqtt_use_tls") or "false") == "true",
  2185. "panda_breath_enabled": (stored.get("panda_breath_enabled") or "false") == "true",
  2186. }
  2187. )
  2188. except Exception:
  2189. # Same call is best-effort in the settings PUT handler: the rows are
  2190. # committed either way, and a broker that refuses the new config
  2191. # must not turn a successful restore into a failed one. Noted rather
  2192. # than swallowed silently, so the user knows to restart.
  2193. logger.warning("Could not reconfigure the MQTT relay after a settings restore", exc_info=True)
  2194. tally.note(
  2195. "settingsMqttRelayFailed",
  2196. "MQTT settings restored, but the relay could not be reconnected — restart Bambuddy",
  2197. )
  2198. async def _restore_kprofiles(self, db: AsyncSession, payload: dict, tally: _CategoryTally) -> None:
  2199. by_serial: dict[str, list[tuple[str, dict]]] = {}
  2200. for path, content in payload.items():
  2201. match = _KPROFILE_PATH_RE.match(path)
  2202. if not match or not isinstance(content, dict):
  2203. continue
  2204. by_serial.setdefault(match.group(1), []).append((match.group(2), content))
  2205. if not by_serial:
  2206. tally.note("noData", "No data of this kind in this backup")
  2207. return
  2208. result = await db.execute(select(Printer))
  2209. printers = {p.serial_number: p for p in result.scalars().all() if p.serial_number}
  2210. # Overwrite is not offered for K-profiles: extrusion_cali_set replaces
  2211. # the profile occupying a slot, so writing is always an overwrite on the
  2212. # printer side.
  2213. tally.note("kprofilesAlwaysOverwrite", "K-profiles always overwrite the matching slot on the printer")
  2214. # A refusal is now believed and counted failed (#2718 made the ack worth
  2215. # reading), but silence still counts restored, so the caveat stands —
  2216. # narrowed to what is actually left uncertain.
  2217. tally.note(
  2218. "kprofilesAckUnreliable",
  2219. "A printer that does not confirm the write still counts as restored — verify the profiles on the printer",
  2220. )
  2221. for serial, entries in sorted(by_serial.items()):
  2222. profile_total = self._kprofile_profile_count(c for _, c in entries)
  2223. printer = printers.get(serial)
  2224. if printer is None:
  2225. tally.skipped += profile_total
  2226. tally.note("kprofilesPrinterMissing", f"No printer with serial {serial} — skipped", serial=serial)
  2227. continue
  2228. client = printer_manager.get_client(printer.id)
  2229. if not client or not client.state.connected:
  2230. tally.skipped += profile_total
  2231. tally.note(
  2232. "kprofilesPrinterOffline",
  2233. f"{printer.name} ({serial}) is not connected — skipped",
  2234. printer=printer.name,
  2235. serial=serial,
  2236. )
  2237. continue
  2238. for nozzle, content in sorted(entries):
  2239. profiles = content.get("profiles")
  2240. if not isinstance(profiles, list) or not profiles:
  2241. continue
  2242. if nozzle not in _KNOWN_NOZZLES:
  2243. tally.note(
  2244. "kprofilesUnknownNozzle",
  2245. f"Unexpected nozzle diameter {nozzle} for {serial} — sent as-is",
  2246. nozzle=nozzle,
  2247. serial=serial,
  2248. )
  2249. # The backup's slot_id is a cali_idx, and cali_idx is as
  2250. # unstable as the autoincrement ids we already refuse to reuse
  2251. # for spools and archives: editing a profile in Bambuddy is a
  2252. # delete-then-add on a single-nozzle printer, which re-keys it.
  2253. # Addressing extrusion_cali_set at a slot that no longer exists
  2254. # is a silent no-op — the printer drops it and we would still
  2255. # report the profile restored. So resolve the live index first.
  2256. current = await self._current_kprofile_index(client, nozzle, serial)
  2257. if current is None:
  2258. # Unread is not empty: matching against nothing would send
  2259. # every profile as a new one beside the originals still on
  2260. # the printer (#3404). Nothing is sent.
  2261. tally.failed += len(profiles)
  2262. tally.note(
  2263. "kprofilesReadFailed",
  2264. f"{printer.name} ({serial}) did not answer the K-profile read for {nozzle} - not restored",
  2265. nozzle=nozzle,
  2266. printer=printer.name,
  2267. serial=serial,
  2268. )
  2269. continue
  2270. profile_dicts = []
  2271. unmatched = 0
  2272. # A live profile can only stand in for one backed-up entry. Two
  2273. # entries resolving to the same cali_idx both go into the batch,
  2274. # the second overwrites the first on the printer, and the tally
  2275. # counts two restored where one landed.
  2276. claimed: set[int] = set()
  2277. for p in profiles:
  2278. if not isinstance(p, dict):
  2279. # Counted, not dropped. _kprofile_profile_count includes
  2280. # it, so the offline and printer-missing paths already
  2281. # count the same entry skipped and the failure path
  2282. # counts it outstanding — leaving the tally here was the
  2283. # one place a profile could vanish from
  2284. # restored + skipped + failed entirely.
  2285. #
  2286. # failed here against skipped there is not a
  2287. # disagreement about the entry. The three counters say
  2288. # what happened to an item on this run, not whether it
  2289. # was ever usable: an offline printer skips everything it
  2290. # holds, well-formed or not, because nothing was
  2291. # attempted, while here the entry was reached and could
  2292. # not be used.
  2293. tally.failed += 1
  2294. continue
  2295. match = self._match_kprofile(p, current, claimed)
  2296. if match is None:
  2297. unmatched += 1
  2298. else:
  2299. claimed.add(match.slot_id)
  2300. entry = {
  2301. "filament_id": p.get("filament_id", ""),
  2302. "name": p.get("name", ""),
  2303. "k_value": p.get("k_value", "0.020000"),
  2304. "extruder_id": p.get("extruder_id", 0),
  2305. # Prefer the live setting_id when we matched: it is
  2306. # what the printer currently associates with the slot.
  2307. "setting_id": (match.setting_id if match else None) or p.get("setting_id"),
  2308. # cali_idx -1 tells the printer to add a new profile
  2309. # rather than address a slot that isn't there.
  2310. "cali_idx": match.slot_id if match else -1,
  2311. # Only consulted for the generated-setting_id
  2312. # fallback; cali_idx above takes precedence.
  2313. "slot_id": 0,
  2314. }
  2315. # Same precedence as setting_id, and set only when known.
  2316. # nozzle_id encodes the fitted nozzle's type and diameter
  2317. # ("HS00-0.4"), so the live value beats the backup's: the
  2318. # user may have swapped the nozzle since. When neither knows,
  2319. # the key has to be *absent* — set_kprofiles_batch supplies
  2320. # HS00-{diameter} via p.get(..., default), which a key
  2321. # present-and-None defeats, publishing a null nozzle_id.
  2322. # Printers that omit it are the reason the default is there
  2323. # (#1748), so it has to be reachable.
  2324. nozzle_id = (getattr(match, "nozzle_id", None) if match else None) or p.get("nozzle_id")
  2325. if nozzle_id:
  2326. entry["nozzle_id"] = nozzle_id
  2327. profile_dicts.append(entry)
  2328. if not profile_dicts:
  2329. continue
  2330. if unmatched:
  2331. tally.note(
  2332. "kprofilesUnmatched",
  2333. f"{unmatched} profile(s) for {nozzle} had no counterpart on {printer.name} "
  2334. "— added as new profiles",
  2335. count=unmatched,
  2336. nozzle=nozzle,
  2337. printer=printer.name,
  2338. )
  2339. try:
  2340. seq = client.set_kprofiles_batch(profile_dicts, nozzle)
  2341. except Exception as e:
  2342. logger.warning("K-profile restore failed for %s nozzle %s: %s", serial, nozzle, e)
  2343. seq = None
  2344. if not seq:
  2345. tally.failed += len(profile_dicts)
  2346. tally.note(
  2347. "kprofilesSendFailed",
  2348. f"Failed to send {nozzle} profiles to {printer.name} ({serial})",
  2349. nozzle=nozzle,
  2350. printer=printer.name,
  2351. serial=serial,
  2352. )
  2353. continue
  2354. # What came back is the sequence_id the command was published
  2355. # under, not a verdict (#2718) — a truthy string only means the
  2356. # command left the building. The printer answers separately, and
  2357. # every other caller of this API now reads that answer; without
  2358. # this the restore would be the one path left that reports a
  2359. # refused write as saved.
  2360. ok, detail = await self._kprofile_ack(client, seq, serial, nozzle)
  2361. if ok:
  2362. tally.restored += len(profile_dicts)
  2363. else:
  2364. tally.failed += len(profile_dicts)
  2365. tally.note(
  2366. "kprofilesRefused",
  2367. f"{printer.name} ({serial}) refused the {nozzle} profiles: {detail}",
  2368. nozzle=nozzle,
  2369. printer=printer.name,
  2370. serial=serial,
  2371. reason=detail,
  2372. )
  2373. @staticmethod
  2374. def _kprofile_profile_count(contents) -> int:
  2375. """Count the profiles across parsed K-profile files.
  2376. Defensive on purpose. A hand-edited or truncated backup can carry a
  2377. ``profiles`` value that is not a list, and this count runs *before* the
  2378. per-call guards in the loop below — after ``_apply`` has already
  2379. committed the database categories. A malformed file has to be a skipped
  2380. category, not an exception thrown over committed rows.
  2381. """
  2382. total = 0
  2383. for content in contents:
  2384. profiles = content.get("profiles") if isinstance(content, dict) else None
  2385. if isinstance(profiles, list):
  2386. total += len(profiles)
  2387. return total
  2388. @staticmethod
  2389. async def _kprofile_ack(client, seq: str, serial: str, nozzle: str) -> tuple[bool, str]:
  2390. """Read the printer's verdict on one batch write.
  2391. ``await_cali_ack`` already treats silence as success — no answer is not
  2392. evidence of refusal, and firmware that predates the ack never answers at
  2393. all. An exception reading it is the same situation one layer up, so it
  2394. degrades the same way rather than turning a write that most likely
  2395. landed into a reported failure.
  2396. """
  2397. try:
  2398. ok, detail = await client.await_cali_ack(seq)
  2399. return bool(ok), str(detail or "")
  2400. except Exception as e:
  2401. logger.warning("Could not read the K-profile ack for %s nozzle %s: %s", serial, nozzle, e)
  2402. return True, ""
  2403. @staticmethod
  2404. async def _current_kprofile_index(client, nozzle: str, serial: str) -> list | None:
  2405. """Read the printer's live profiles for one nozzle, or None when the
  2406. read failed.
  2407. A failed read used to be "nothing matched", which made every profile
  2408. an add next to the ones still on the printer (#3404). The caller skips
  2409. the nozzle instead; the rest of the restore goes on.
  2410. """
  2411. try:
  2412. return list(await client.get_kprofiles(nozzle_diameter=nozzle) or [])
  2413. except Exception as e:
  2414. logger.warning("Could not read live K-profiles for %s nozzle %s: %s", serial, nozzle, e)
  2415. return None
  2416. @staticmethod
  2417. def _match_kprofile(entry: dict, current: list, claimed: set[int]):
  2418. """Find the live profile a backed-up entry corresponds to.
  2419. ``setting_id`` is the filament preset the profile was calibrated for and
  2420. is the strongest signal; a delete-then-add edit regenerates it, so fall
  2421. back to the display name, which Bambuddy's own editor preserves.
  2422. Both are scoped by ``filament_id`` — the same preset on a different
  2423. filament is a different profile — and by ``extruder_id``, because on a
  2424. dual-nozzle printer the same preset on the other extruder is a different
  2425. profile too.
  2426. ``claimed`` holds the slot ids already taken by earlier entries in this
  2427. nozzle's loop, and no live profile may be claimed twice. Without it, two
  2428. backed-up entries sharing a ``filament_id`` and matching on neither
  2429. ``setting_id`` nor ``name`` both fell through to the single-candidate
  2430. arm and both took the same slot — reachable whenever the user has since
  2431. deleted one of a pair, because the delete-then-add re-key is what strips
  2432. the ``setting_id`` match. Returning None for the displaced entry means
  2433. ``cali_idx: -1``, i.e. add-as-new, which is the safe outcome.
  2434. """
  2435. filament_id = entry.get("filament_id")
  2436. if not filament_id:
  2437. return None
  2438. candidates = [c for c in current if c.filament_id == filament_id]
  2439. # The live index is read per nozzle *diameter*, so on an H2D both
  2440. # extruders' profiles come back together. With the same filament
  2441. # calibrated on both — the ordinary case on a dual-nozzle printer, not an
  2442. # exotic one — filament_id alone lets extruder 0's backed-up entry match
  2443. # extruder 1's live profile, and the batch then carries
  2444. # {extruder_id: 0, cali_idx: <extruder-1 slot>}: one extruder's
  2445. # calibration written over the other's, counted restored.
  2446. #
  2447. # Conditional on both sides saying which extruder they mean. A pre-#2656
  2448. # backup carries no extruder_id, and a live index that reports none must
  2449. # not turn every entry into an add.
  2450. extruder_id = entry.get("extruder_id")
  2451. if isinstance(extruder_id, int) and any(getattr(c, "extruder_id", None) is not None for c in candidates):
  2452. candidates = [c for c in candidates if getattr(c, "extruder_id", None) == extruder_id]
  2453. available = [c for c in candidates if c.slot_id not in claimed]
  2454. if not available:
  2455. return None
  2456. setting_id = entry.get("setting_id")
  2457. if setting_id:
  2458. for c in available:
  2459. if c.setting_id == setting_id:
  2460. return c
  2461. name = entry.get("name")
  2462. if name:
  2463. for c in available:
  2464. if c.name == name:
  2465. return c
  2466. # Exactly one profile for this filament and no better discriminator:
  2467. # treat it as the same profile rather than duplicating it. Judged
  2468. # against every candidate rather than the unclaimed ones, because two
  2469. # live profiles for one filament are ambiguous whether or not another
  2470. # entry has already taken one of them.
  2471. return available[0] if len(candidates) == 1 else None
  2472. # Singleton instance
  2473. github_restore_service = GitHubRestoreService()
  2474. _GCODE_HASH = re.compile(r"^[0-9a-f]{64}$")
  2475. def _gcode_hash_of(entry: dict) -> str | None:
  2476. """The backed-up G-code fingerprint (#1634), when it is one."""
  2477. value = entry.get("gcode_hash")
  2478. return value if isinstance(value, str) and _GCODE_HASH.match(value) else None