github_backup.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565
  1. """GitHub backup service for printer profiles.
  2. Handles scheduled and on-demand backups of K-profiles and cloud profiles to GitHub.
  3. """
  4. import asyncio
  5. import logging
  6. from datetime import datetime, timedelta, timezone
  7. import httpx
  8. from sqlalchemy import desc, select
  9. from sqlalchemy.ext.asyncio import AsyncSession
  10. from backend.app.core.database import async_session
  11. from backend.app.models.archive import PrintArchive
  12. from backend.app.models.github_backup import GitHubBackupConfig, GitHubBackupLog
  13. from backend.app.models.printer import Printer
  14. from backend.app.models.settings import Settings
  15. from backend.app.models.spool import Spool
  16. from backend.app.models.spool_usage_history import SpoolUsageHistory
  17. from backend.app.services.git_providers.factory import get_provider_backend
  18. from backend.app.services.printer_manager import printer_manager
  19. logger = logging.getLogger(__name__)
  20. # Schedule intervals in seconds
  21. SCHEDULE_INTERVALS = {
  22. "hourly": 3600,
  23. "daily": 86400,
  24. "weekly": 604800,
  25. }
  26. _PROVIDER_DISPLAY_NAMES = {
  27. "github": "GitHub",
  28. "gitlab": "GitLab",
  29. "gitea": "Gitea",
  30. "forgejo": "Forgejo",
  31. }
  32. class GitHubBackupService:
  33. """Service for backing up profiles to GitHub."""
  34. def __init__(self):
  35. self._scheduler_task: asyncio.Task | None = None
  36. self._check_interval = 60 # Check every minute for scheduled runs
  37. self._running_backup: bool = False
  38. self._backup_progress: str | None = None
  39. self._http_client: httpx.AsyncClient | None = None
  40. async def _get_client(self) -> httpx.AsyncClient:
  41. """Get or create HTTP client."""
  42. if self._http_client is None or self._http_client.is_closed:
  43. self._http_client = httpx.AsyncClient(timeout=60.0)
  44. return self._http_client
  45. async def start_scheduler(self):
  46. """Start the background scheduler loop."""
  47. if self._scheduler_task is not None:
  48. return
  49. logger.info("Starting GitHub backup scheduler")
  50. self._scheduler_task = asyncio.create_task(self._scheduler_loop())
  51. def stop_scheduler(self):
  52. """Stop the scheduler."""
  53. if self._scheduler_task:
  54. self._scheduler_task.cancel()
  55. self._scheduler_task = None
  56. logger.info("Stopped GitHub backup scheduler")
  57. async def _scheduler_loop(self):
  58. """Main scheduler loop - checks for due backups."""
  59. while True:
  60. try:
  61. await asyncio.sleep(self._check_interval)
  62. await self._check_scheduled_backups()
  63. except asyncio.CancelledError:
  64. break
  65. except Exception as e:
  66. logger.error("Error in GitHub backup scheduler: %s", e)
  67. await asyncio.sleep(60)
  68. async def _check_scheduled_backups(self):
  69. """Check if any scheduled backups are due."""
  70. async with async_session() as db:
  71. result = await db.execute(
  72. select(GitHubBackupConfig).where(
  73. GitHubBackupConfig.enabled == True, # noqa: E712
  74. GitHubBackupConfig.schedule_enabled == True, # noqa: E712
  75. )
  76. )
  77. configs = result.scalars().all()
  78. now = datetime.now(timezone.utc)
  79. for config in configs:
  80. # Handle both naive (from DB) and aware datetimes
  81. next_run = config.next_scheduled_run
  82. if next_run and next_run.tzinfo is None:
  83. next_run = next_run.replace(tzinfo=timezone.utc)
  84. if next_run and next_run <= now:
  85. logger.info("Running scheduled backup for config %s", config.id)
  86. await self.run_backup(config.id, trigger="scheduled")
  87. def calculate_next_run(self, schedule_type: str, from_time: datetime | None = None) -> datetime:
  88. """Calculate the next scheduled run time."""
  89. now = from_time or datetime.now(timezone.utc)
  90. interval = SCHEDULE_INTERVALS.get(schedule_type, SCHEDULE_INTERVALS["daily"])
  91. return now + timedelta(seconds=interval)
  92. async def test_connection(self, repo_url: str, token: str, provider: str = "github") -> dict:
  93. """Test connection and permissions for the given provider."""
  94. backend = get_provider_backend(provider)
  95. client = await self._get_client()
  96. return await backend.test_connection(repo_url, token, client)
  97. async def run_backup(self, config_id: int, trigger: str = "manual") -> dict:
  98. """Run a backup operation.
  99. Args:
  100. config_id: ID of the backup configuration
  101. trigger: "manual" or "scheduled"
  102. Returns:
  103. dict with success, message, log_id, commit_sha, files_changed
  104. """
  105. if self._running_backup:
  106. return {"success": False, "message": "A backup is already running", "log_id": None}
  107. self._running_backup = True
  108. log_id = None
  109. try:
  110. async with async_session() as db:
  111. # Get config
  112. result = await db.execute(select(GitHubBackupConfig).where(GitHubBackupConfig.id == config_id))
  113. config = result.scalar_one_or_none()
  114. if not config:
  115. return {"success": False, "message": "Configuration not found", "log_id": None}
  116. if not config.enabled:
  117. return {"success": False, "message": "Backup is disabled", "log_id": None}
  118. # Create log entry
  119. log = GitHubBackupLog(config_id=config_id, status="running", trigger=trigger)
  120. db.add(log)
  121. await db.commit()
  122. await db.refresh(log)
  123. log_id = log.id
  124. try:
  125. # Collect backup data
  126. self._backup_progress = "Collecting profiles..."
  127. backup_data = await self._collect_backup_data(db, config)
  128. if not backup_data:
  129. # No data to backup
  130. log.status = "skipped"
  131. log.completed_at = datetime.now(timezone.utc)
  132. log.error_message = "No data to backup"
  133. config.last_backup_at = datetime.now(timezone.utc)
  134. config.last_backup_status = "skipped"
  135. config.last_backup_message = "No data to backup"
  136. if config.schedule_enabled:
  137. config.next_scheduled_run = self.calculate_next_run(config.schedule_type)
  138. await db.commit()
  139. return {
  140. "success": True,
  141. "message": "No data to backup",
  142. "log_id": log_id,
  143. "commit_sha": None,
  144. "files_changed": 0,
  145. }
  146. provider_name = _PROVIDER_DISPLAY_NAMES.get(config.provider, config.provider)
  147. self._backup_progress = f"Pushing to {provider_name}..."
  148. push_result = await self._push_to_provider(config, backup_data)
  149. # Update log and config
  150. log.status = push_result["status"]
  151. log.completed_at = datetime.now(timezone.utc)
  152. log.commit_sha = push_result.get("commit_sha")
  153. log.files_changed = push_result.get("files_changed", 0)
  154. log.error_message = push_result.get("error")
  155. config.last_backup_at = datetime.now(timezone.utc)
  156. config.last_backup_status = push_result["status"]
  157. config.last_backup_message = push_result.get("message", "")
  158. config.last_backup_commit_sha = push_result.get("commit_sha")
  159. if config.schedule_enabled:
  160. config.next_scheduled_run = self.calculate_next_run(config.schedule_type)
  161. await db.commit()
  162. return {
  163. "success": push_result["status"] in ("success", "skipped"),
  164. "message": push_result.get("message", "Backup completed"),
  165. "log_id": log_id,
  166. "commit_sha": push_result.get("commit_sha"),
  167. "files_changed": push_result.get("files_changed", 0),
  168. }
  169. except Exception as e:
  170. logger.error("Backup failed: %s", e)
  171. log.status = "failed"
  172. log.completed_at = datetime.now(timezone.utc)
  173. log.error_message = str(e)
  174. config.last_backup_at = datetime.now(timezone.utc)
  175. config.last_backup_status = "failed"
  176. config.last_backup_message = str(e)
  177. if config.schedule_enabled:
  178. config.next_scheduled_run = self.calculate_next_run(config.schedule_type)
  179. await db.commit()
  180. return {
  181. "success": False,
  182. "message": str(e),
  183. "log_id": log_id,
  184. "commit_sha": None,
  185. "files_changed": 0,
  186. }
  187. finally:
  188. self._running_backup = False
  189. self._backup_progress = None
  190. async def _collect_backup_data(self, db: AsyncSession, config: GitHubBackupConfig) -> dict:
  191. """Collect data to backup based on config settings.
  192. Returns dict with structure:
  193. {
  194. "backup_metadata.json": {...},
  195. "kprofiles/{serial}/{nozzle}.json": {...},
  196. "cloud_profiles/filament.json": [...],
  197. "cloud_profiles/printer.json": [...],
  198. "cloud_profiles/process.json": [...],
  199. "settings/app_settings.json": {...},
  200. }
  201. """
  202. files: dict[str, dict | list] = {}
  203. # Metadata file (no timestamps - git tracks file history)
  204. metadata = {
  205. "version": "1.0",
  206. "backup_type": "bambuddy_profiles",
  207. "contents": {
  208. "kprofiles": config.backup_kprofiles,
  209. "cloud_profiles": config.backup_cloud_profiles,
  210. "settings": config.backup_settings,
  211. "spools": config.backup_spools,
  212. "archives": config.backup_archives,
  213. },
  214. }
  215. files["backup_metadata.json"] = metadata
  216. # Collect K-profiles from all connected printers
  217. if config.backup_kprofiles:
  218. self._backup_progress = "Collecting K-profiles from printers..."
  219. await self._collect_kprofiles(db, files)
  220. # Collect cloud profiles
  221. if config.backup_cloud_profiles:
  222. self._backup_progress = "Collecting cloud profiles from Bambu Cloud..."
  223. await self._collect_cloud_profiles(db, files)
  224. # Collect app settings
  225. if config.backup_settings:
  226. self._backup_progress = "Collecting app settings..."
  227. await self._collect_settings(db, files)
  228. # Collect spool inventory
  229. if config.backup_spools:
  230. self._backup_progress = "Collecting spool inventory..."
  231. await self._collect_spools(db, files)
  232. # Collect print archives
  233. if config.backup_archives:
  234. self._backup_progress = "Collecting print archives..."
  235. await self._collect_archives(db, files)
  236. return files
  237. async def _collect_kprofiles(self, db: AsyncSession, files: dict):
  238. """Collect K-profiles from all connected printers."""
  239. result = await db.execute(select(Printer).where(Printer.is_active == True)) # noqa: E712
  240. printers = result.scalars().all()
  241. nozzle_diameters = ["0.2", "0.4", "0.6", "0.8"]
  242. for printer in printers:
  243. client = printer_manager.get_client(printer.id)
  244. if not client or not client.state.connected:
  245. continue
  246. serial = printer.serial_number
  247. printer_profiles = {}
  248. for nozzle in nozzle_diameters:
  249. try:
  250. profiles = await client.get_kprofiles(nozzle_diameter=nozzle)
  251. if profiles:
  252. profile_data = {
  253. "version": "1.0",
  254. "printer_name": printer.name,
  255. "printer_serial": serial,
  256. "nozzle_diameter": nozzle,
  257. "profiles": [
  258. {
  259. "slot_id": p.slot_id,
  260. "name": p.name,
  261. "k_value": p.k_value,
  262. "filament_id": p.filament_id,
  263. "nozzle_id": p.nozzle_id,
  264. "extruder_id": p.extruder_id,
  265. "setting_id": p.setting_id,
  266. "n_coef": p.n_coef,
  267. }
  268. for p in profiles
  269. ],
  270. }
  271. files[f"kprofiles/{serial}/{nozzle}.json"] = profile_data
  272. printer_profiles[nozzle] = len(profiles)
  273. except Exception as e:
  274. logger.warning("Failed to get K-profiles for printer %s nozzle %s: %s", serial, nozzle, e)
  275. if printer_profiles:
  276. logger.info("Collected K-profiles for %s: %s", serial, printer_profiles)
  277. async def _collect_cloud_profiles(self, db: AsyncSession, files: dict):
  278. """Collect Bambu Cloud profiles if authenticated."""
  279. # Backup runs without a user context, so fall back to the auth-disabled
  280. # Settings storage. ``build_authenticated_cloud`` honours the stored
  281. # region so China-region tokens are validated against api.bambulab.cn.
  282. from backend.app.api.routes.cloud import build_authenticated_cloud
  283. cloud = await build_authenticated_cloud(db, user=None)
  284. if cloud is None or not cloud.is_authenticated:
  285. if cloud is not None:
  286. await cloud.close()
  287. logger.info("Cloud not authenticated, skipping cloud profiles")
  288. return
  289. try:
  290. settings = await cloud.get_slicer_settings()
  291. if not settings:
  292. return
  293. # Separate by type
  294. filament_settings = []
  295. printer_settings = []
  296. process_settings = []
  297. for setting in settings.get("setting", []) if isinstance(settings.get("setting"), list) else []:
  298. setting_type = setting.get("type", "")
  299. if setting_type == "filament":
  300. filament_settings.append(setting)
  301. elif setting_type == "printer":
  302. printer_settings.append(setting)
  303. elif setting_type == "process":
  304. process_settings.append(setting)
  305. if filament_settings:
  306. files["cloud_profiles/filament.json"] = {
  307. "version": "1.0",
  308. "profiles": filament_settings,
  309. }
  310. if printer_settings:
  311. files["cloud_profiles/printer.json"] = {
  312. "version": "1.0",
  313. "profiles": printer_settings,
  314. }
  315. if process_settings:
  316. files["cloud_profiles/process.json"] = {
  317. "version": "1.0",
  318. "profiles": process_settings,
  319. }
  320. logger.info(
  321. f"Collected cloud profiles: {len(filament_settings)} filament, "
  322. f"{len(printer_settings)} printer, {len(process_settings)} process"
  323. )
  324. except Exception as e:
  325. logger.warning("Failed to collect cloud profiles: %s", e)
  326. finally:
  327. await cloud.close()
  328. async def _collect_settings(self, db: AsyncSession, files: dict):
  329. """Collect app settings."""
  330. result = await db.execute(select(Settings))
  331. settings = result.scalars().all()
  332. # Filter out sensitive settings
  333. sensitive_keys = {"bambu_cloud_token", "auth_secret_key"}
  334. settings_data = {s.key: s.value for s in settings if s.key not in sensitive_keys}
  335. files["settings/app_settings.json"] = {
  336. "version": "1.0",
  337. "settings": settings_data,
  338. }
  339. async def _collect_spools(self, db: AsyncSession, files: dict):
  340. """Collect spool inventory data."""
  341. result = await db.execute(select(Spool))
  342. spools = result.scalars().all()
  343. if not spools:
  344. return
  345. spool_list = []
  346. for s in spools:
  347. spool_data = {
  348. "id": s.id,
  349. "material": s.material,
  350. "subtype": s.subtype,
  351. "color_name": s.color_name,
  352. "rgba": s.rgba,
  353. "brand": s.brand,
  354. "label_weight": s.label_weight,
  355. "core_weight": s.core_weight,
  356. "weight_used": s.weight_used,
  357. "weight_locked": s.weight_locked,
  358. "slicer_filament": s.slicer_filament,
  359. "slicer_filament_name": s.slicer_filament_name,
  360. "nozzle_temp_min": s.nozzle_temp_min,
  361. "nozzle_temp_max": s.nozzle_temp_max,
  362. "note": s.note,
  363. "cost_per_kg": s.cost_per_kg,
  364. "tag_uid": s.tag_uid,
  365. "tray_uuid": s.tray_uuid,
  366. "data_origin": s.data_origin,
  367. "tag_type": s.tag_type,
  368. "archived_at": str(s.archived_at) if s.archived_at else None,
  369. "created_at": str(s.created_at) if s.created_at else None,
  370. }
  371. spool_list.append(spool_data)
  372. files["spools/inventory.json"] = {
  373. "version": "1.0",
  374. "spools": spool_list,
  375. }
  376. # Collect usage history
  377. usage_result = await db.execute(select(SpoolUsageHistory))
  378. usages = usage_result.scalars().all()
  379. if usages:
  380. usage_list = []
  381. for u in usages:
  382. usage_list.append(
  383. {
  384. "id": u.id,
  385. "spool_id": u.spool_id,
  386. "printer_id": u.printer_id,
  387. "print_name": u.print_name,
  388. "archive_id": u.archive_id,
  389. "weight_used": u.weight_used,
  390. "percent_used": u.percent_used,
  391. "status": u.status,
  392. "cost": u.cost,
  393. "created_at": str(u.created_at) if u.created_at else None,
  394. }
  395. )
  396. files["spools/usage_history.json"] = {
  397. "version": "1.0",
  398. "usage_history": usage_list,
  399. }
  400. logger.info("Collected %d spools and %d usage records", len(spool_list), len(usages))
  401. async def _collect_archives(self, db: AsyncSession, files: dict):
  402. """Collect print archive metadata (no binary files)."""
  403. result = await db.execute(select(PrintArchive))
  404. archives = result.scalars().all()
  405. if not archives:
  406. return
  407. archive_list = []
  408. for a in archives:
  409. archive_data = {
  410. "id": a.id,
  411. "printer_id": a.printer_id,
  412. "project_id": a.project_id,
  413. "filename": a.filename,
  414. "file_size": a.file_size,
  415. "content_hash": a.content_hash,
  416. "print_name": a.print_name,
  417. "print_time_seconds": a.print_time_seconds,
  418. "filament_used_grams": a.filament_used_grams,
  419. "filament_type": a.filament_type,
  420. "filament_color": a.filament_color,
  421. "layer_height": a.layer_height,
  422. "total_layers": a.total_layers,
  423. "nozzle_diameter": a.nozzle_diameter,
  424. "bed_temperature": a.bed_temperature,
  425. "nozzle_temperature": a.nozzle_temperature,
  426. "sliced_for_model": a.sliced_for_model,
  427. "status": a.status,
  428. "started_at": str(a.started_at) if a.started_at else None,
  429. "completed_at": str(a.completed_at) if a.completed_at else None,
  430. "makerworld_url": a.makerworld_url,
  431. "designer": a.designer,
  432. "external_url": a.external_url,
  433. "is_favorite": a.is_favorite,
  434. "tags": a.tags,
  435. "notes": a.notes,
  436. "cost": a.cost,
  437. "failure_reason": a.failure_reason,
  438. "quantity": a.quantity,
  439. "energy_kwh": a.energy_kwh,
  440. "energy_cost": a.energy_cost,
  441. "created_at": str(a.created_at) if a.created_at else None,
  442. }
  443. archive_list.append(archive_data)
  444. files["archives/print_history.json"] = {
  445. "version": "1.0",
  446. "archives": archive_list,
  447. }
  448. logger.info("Collected %d print archives", len(archive_list))
  449. async def _push_to_provider(self, config: GitHubBackupConfig, files: dict) -> dict:
  450. """Push files to the configured Git provider."""
  451. backend = get_provider_backend(config.provider)
  452. client = await self._get_client()
  453. return await backend.push_files(
  454. repo_url=config.repository_url,
  455. token=config.access_token,
  456. branch=config.branch,
  457. files=files,
  458. client=client,
  459. )
  460. @property
  461. def is_running(self) -> bool:
  462. """Check if a backup is currently running."""
  463. return self._running_backup
  464. @property
  465. def progress(self) -> str | None:
  466. """Get current backup progress message."""
  467. return self._backup_progress
  468. async def get_logs(self, config_id: int, limit: int = 50, offset: int = 0) -> list[GitHubBackupLog]:
  469. """Get backup logs for a configuration."""
  470. async with async_session() as db:
  471. result = await db.execute(
  472. select(GitHubBackupLog)
  473. .where(GitHubBackupLog.config_id == config_id)
  474. .order_by(desc(GitHubBackupLog.started_at))
  475. .offset(offset)
  476. .limit(limit)
  477. )
  478. return list(result.scalars().all())
  479. # Singleton instance
  480. github_backup_service = GitHubBackupService()