github.py 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783
  1. """GitHub backend — implements GitProviderBackend using the GitHub Git Data API."""
  2. import base64
  3. import json
  4. import logging
  5. import re
  6. from datetime import datetime, timezone
  7. from urllib.parse import quote
  8. import httpx
  9. from backend.app.services.git_providers.base import GitProviderBackend
  10. logger = logging.getLogger(__name__)
  11. class GitHubBackend(GitProviderBackend):
  12. """Backend for github.com using the GitHub Git Data API."""
  13. def get_api_base(self, repo_url: str) -> str:
  14. m = re.match(r"https?://([\w.\-]+(:\d+)?)/", repo_url)
  15. if m:
  16. host = m.group(1)
  17. return "https://api.github.com" if host == "github.com" else f"https://{host}/api/v3"
  18. m = re.match(r"git@([\w.\-]+):", repo_url)
  19. if m:
  20. host = m.group(1)
  21. return "https://api.github.com" if host == "github.com" else f"https://{host}/api/v3"
  22. return "https://api.github.com"
  23. def parse_repo_url(self, url: str) -> tuple[str, str]:
  24. """Return (owner, repo) from a Git HTTPS or SSH URL."""
  25. if not url or len(url) > 500:
  26. raise ValueError("Invalid Git URL: URL too long or empty")
  27. # HTTPS: https://<host>[:<port>]/<owner>/<repo>[.git][/]
  28. match = re.match(
  29. r"https://[\w.\-]+(:\d+)?/([\w.\-]{1,100})/([\w.\-]{1,100})(?:\.git)?/?$",
  30. url,
  31. )
  32. if match:
  33. return match.group(2), match.group(3).removesuffix(".git")
  34. # SSH: git@<host>:<owner>/<repo>[.git]
  35. match = re.match(
  36. r"git@[\w.\-]+:([\w.\-]{1,100})/([\w.\-]{1,100})(?:\.git)?$",
  37. url,
  38. )
  39. if match:
  40. return match.group(1), match.group(2).removesuffix(".git")
  41. raise ValueError(f"Cannot parse repository URL: {url}")
  42. async def test_connection(self, repo_url: str, token: str, client: httpx.AsyncClient) -> dict:
  43. """Test API access and push permission for the repository."""
  44. try:
  45. owner, repo = self.parse_repo_url(repo_url)
  46. api_base = self.get_api_base(repo_url)
  47. headers = self.get_headers(token)
  48. response = await client.get(f"{api_base}/repos/{owner}/{repo}", headers=headers)
  49. if response.status_code == 401:
  50. return {"success": False, "message": "Invalid access token", "repo_name": None, "permissions": None}
  51. if response.status_code == 404:
  52. return {
  53. "success": False,
  54. "message": "Repository not found. Check URL and token permissions.",
  55. "repo_name": None,
  56. "permissions": None,
  57. }
  58. if response.status_code != 200:
  59. return {
  60. "success": False,
  61. "message": f"API error: {response.status_code}",
  62. "repo_name": None,
  63. "permissions": None,
  64. }
  65. data = response.json()
  66. permissions = data.get("permissions", {})
  67. is_private = bool(data.get("private", False))
  68. if not permissions.get("push", False):
  69. return {
  70. "success": False,
  71. "message": "Token does not have push permission to this repository",
  72. "repo_name": data.get("full_name"),
  73. "permissions": permissions,
  74. "is_private": is_private,
  75. }
  76. return {
  77. "success": True,
  78. "message": "Connection successful",
  79. "repo_name": data.get("full_name"),
  80. "permissions": permissions,
  81. "is_private": is_private,
  82. }
  83. except Exception as e:
  84. logger.exception("Git connection test failed")
  85. detail = str(e)[:200]
  86. message = (
  87. f"Connection failed: {type(e).__name__}: {detail}"
  88. if detail
  89. else f"Connection failed: {type(e).__name__}"
  90. )
  91. return {
  92. "success": False,
  93. "message": message,
  94. "repo_name": None,
  95. "permissions": None,
  96. "is_private": None,
  97. }
  98. async def list_commits(
  99. self,
  100. repo_url: str,
  101. token: str,
  102. branch: str,
  103. client: httpx.AsyncClient,
  104. limit: int = 20,
  105. ) -> dict:
  106. """List recent commits on ``branch`` via the repo commits API."""
  107. try:
  108. owner, repo = self.parse_repo_url(repo_url)
  109. api_base = self.get_api_base(repo_url)
  110. headers = self.get_headers(token)
  111. # GitHub pages with ``per_page`` and ignores ``limit``; Gitea/Forgejo
  112. # do the reverse. Sending both lets GiteaBackend inherit this method
  113. # unchanged instead of duplicating it for one query parameter.
  114. response = await client.get(
  115. f"{api_base}/repos/{owner}/{repo}/commits",
  116. headers=headers,
  117. params={"sha": branch, "per_page": limit, "limit": limit},
  118. )
  119. if response.status_code == 404:
  120. return {
  121. "success": False,
  122. "message": (
  123. f"Branch '{branch}' not found, or the repository has no commits yet. "
  124. "Run a backup before restoring."
  125. ),
  126. "commits": [],
  127. }
  128. if response.status_code != 200:
  129. msg = f"Failed to list commits (HTTP {response.status_code}): {self._truncated_response_text(response)}"
  130. logger.warning("list_commits %s/%s: %s", owner, repo, msg)
  131. return {"success": False, "message": msg, "commits": []}
  132. try:
  133. data = response.json()
  134. except ValueError:
  135. return {"success": False, "message": "Non-JSON response listing commits", "commits": []}
  136. if not isinstance(data, list):
  137. return {"success": False, "message": "Unexpected shape listing commits", "commits": []}
  138. return {"success": True, "message": "OK", "commits": self._parse_commit_entries(data, limit)}
  139. except Exception as e:
  140. logger.exception("list_commits failed for %s branch=%s", repo_url, branch)
  141. return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "commits": []}
  142. @staticmethod
  143. def _parse_commit_entries(data: list, limit: int) -> list[dict]:
  144. """Normalise GitHub/Gitea commit list entries to our flat shape."""
  145. commits = []
  146. for entry in data[:limit]:
  147. if not isinstance(entry, dict):
  148. continue
  149. sha = entry.get("sha")
  150. if not isinstance(sha, str) or not sha:
  151. continue
  152. commit = entry.get("commit") if isinstance(entry.get("commit"), dict) else {}
  153. author = commit.get("author") if isinstance(commit.get("author"), dict) else {}
  154. commits.append(
  155. {
  156. "sha": sha,
  157. "message": commit.get("message") or "",
  158. "author": author.get("name") or "",
  159. "date": author.get("date") or "",
  160. }
  161. )
  162. return commits
  163. async def _blob_shas_at(
  164. self,
  165. client: httpx.AsyncClient,
  166. headers: dict,
  167. api_base: str,
  168. owner: str,
  169. repo: str,
  170. ref: str,
  171. ) -> tuple[dict[str, str] | None, str]:
  172. """Return ``({path: blob_sha}, "")`` at ``ref``, or ``(None, error_message)``.
  173. A commit SHA is a valid tree-ish for the trees API, so this resolves the
  174. commit's tree in one request rather than commit -> tree -> list.
  175. """
  176. response = await client.get(
  177. f"{api_base}/repos/{owner}/{repo}/git/trees/{ref}?recursive=1",
  178. headers=headers,
  179. )
  180. if response.status_code == 404:
  181. return None, f"Commit or tree '{ref}' not found in the repository"
  182. if response.status_code != 200:
  183. return None, f"Failed to list tree (HTTP {response.status_code}): {self._truncated_response_text(response)}"
  184. try:
  185. data = response.json()
  186. except ValueError:
  187. return None, "Non-JSON response listing tree"
  188. # Same limit the push path guards against: a truncated listing would make
  189. # a restore silently skip categories that are actually in the backup.
  190. if data.get("truncated"):
  191. return None, (
  192. "Repository tree exceeds the API listing limit (truncated=true), so the backup "
  193. "contents cannot be enumerated reliably. Rotate the backup repository."
  194. )
  195. blobs: dict[str, str] = {}
  196. for item in data.get("tree", []):
  197. if not isinstance(item, dict) or item.get("type") != "blob":
  198. continue
  199. path, sha = item.get("path"), item.get("sha")
  200. if isinstance(path, str) and isinstance(sha, str) and path and sha:
  201. blobs[path] = sha
  202. return blobs, ""
  203. async def get_commit(self, repo_url: str, token: str, ref: str, client: httpx.AsyncClient) -> dict:
  204. """Read one commit's metadata directly, for refs outside the list window."""
  205. try:
  206. owner, repo = self.parse_repo_url(repo_url)
  207. api_base = self.get_api_base(repo_url)
  208. headers = self.get_headers(token)
  209. response = await client.get(f"{api_base}/repos/{owner}/{repo}/commits/{ref}", headers=headers)
  210. if response.status_code == 404:
  211. return {"success": False, "message": f"Commit '{ref}' not found in the repository", "commit": None}
  212. if response.status_code != 200:
  213. msg = f"Failed to read commit (HTTP {response.status_code}): {self._truncated_response_text(response)}"
  214. logger.warning("get_commit %s/%s ref=%s: %s", owner, repo, ref, msg)
  215. return {"success": False, "message": msg, "commit": None}
  216. try:
  217. data = response.json()
  218. except ValueError:
  219. return {"success": False, "message": "Non-JSON response reading commit", "commit": None}
  220. if not isinstance(data, dict):
  221. return {"success": False, "message": "Unexpected shape reading commit", "commit": None}
  222. # Same entry shape as list_commits, so callers can treat the two
  223. # interchangeably.
  224. parsed = self._parse_commit_entries([data], 1)
  225. if not parsed:
  226. return {"success": False, "message": "Commit response carried no SHA", "commit": None}
  227. return {"success": True, "message": "OK", "commit": parsed[0]}
  228. except Exception as e:
  229. logger.exception("get_commit failed for %s ref=%s", repo_url, ref)
  230. return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "commit": None}
  231. async def list_tree(
  232. self,
  233. repo_url: str,
  234. token: str,
  235. ref: str,
  236. client: httpx.AsyncClient,
  237. ) -> dict:
  238. """List blob paths present at ``ref`` via the Git Data trees API."""
  239. try:
  240. owner, repo = self.parse_repo_url(repo_url)
  241. api_base = self.get_api_base(repo_url)
  242. headers = self.get_headers(token)
  243. blobs, error = await self._blob_shas_at(client, headers, api_base, owner, repo, ref)
  244. if blobs is None:
  245. logger.warning("list_tree %s/%s ref=%s: %s", owner, repo, ref, error)
  246. return {"success": False, "message": error, "paths": [], "blob_shas": {}}
  247. # The map is handed back so fetch_files does not GET the same
  248. # recursive tree a second time for the same ref.
  249. return {"success": True, "message": "OK", "paths": sorted(blobs), "blob_shas": blobs}
  250. except Exception as e:
  251. logger.exception("list_tree failed for %s ref=%s", repo_url, ref)
  252. return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "paths": [], "blob_shas": {}}
  253. async def fetch_files(
  254. self,
  255. repo_url: str,
  256. token: str,
  257. ref: str,
  258. paths: list[str],
  259. client: httpx.AsyncClient,
  260. blob_shas: dict[str, str] | None = None,
  261. ) -> dict:
  262. """Read ``paths`` at ``ref`` via the Git Data blobs API.
  263. The blobs API is used rather than the contents API because contents
  264. inlines only files up to 1 MB — an archive-heavy ``print_history.json``
  265. can exceed that, and it would come back with an empty body instead of an
  266. error.
  267. """
  268. try:
  269. owner, repo = self.parse_repo_url(repo_url)
  270. api_base = self.get_api_base(repo_url)
  271. headers = self.get_headers(token)
  272. blobs = blob_shas
  273. if blobs is None:
  274. blobs, error = await self._blob_shas_at(client, headers, api_base, owner, repo, ref)
  275. if blobs is None:
  276. logger.warning("fetch_files %s/%s ref=%s: %s", owner, repo, ref, error)
  277. return {"success": False, "message": error, "files": {}}
  278. files: dict[str, str] = {}
  279. for path in paths:
  280. sha = blobs.get(path)
  281. if sha is None:
  282. continue
  283. response = await client.get(f"{api_base}/repos/{owner}/{repo}/git/blobs/{sha}", headers=headers)
  284. if response.status_code != 200:
  285. msg = f"Failed to read {path} (HTTP {response.status_code}): {self._truncated_response_text(response)}"
  286. logger.warning("fetch_files %s/%s: %s", owner, repo, msg)
  287. return {"success": False, "message": msg, "files": {}}
  288. text, error = self._decode_blob(response, path)
  289. if text is None:
  290. logger.warning("fetch_files %s/%s: %s", owner, repo, error)
  291. return {"success": False, "message": error, "files": {}}
  292. files[path] = text
  293. return {"success": True, "message": "OK", "files": files}
  294. except Exception as e:
  295. logger.exception("fetch_files failed for %s ref=%s", repo_url, ref)
  296. return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "files": {}}
  297. def _decode_blob(self, response: httpx.Response, path: str) -> tuple[str | None, str]:
  298. """Decode a blob API response body to text, or return an error message."""
  299. try:
  300. data = response.json()
  301. except ValueError:
  302. return None, f"Non-JSON response reading {path}"
  303. if not isinstance(data, dict):
  304. return None, f"Unexpected shape reading {path}"
  305. content = data.get("content")
  306. if not isinstance(content, str):
  307. return None, f"Missing content reading {path}"
  308. encoding = data.get("encoding", "base64")
  309. try:
  310. if encoding == "base64":
  311. # Both providers wrap base64 payloads at 60 chars; b64decode
  312. # tolerates the newlines, but be explicit about it.
  313. return base64.b64decode(content).decode("utf-8"), ""
  314. if encoding in ("utf-8", "text", "plain"):
  315. return content, ""
  316. except (ValueError, UnicodeDecodeError) as e:
  317. return None, f"Could not decode {path}: {type(e).__name__}"
  318. return None, f"Unsupported blob encoding {encoding!r} reading {path}"
  319. async def push_files(
  320. self,
  321. repo_url: str,
  322. token: str,
  323. branch: str,
  324. files: dict,
  325. client: httpx.AsyncClient,
  326. _allow_branch_create: bool = True,
  327. ) -> dict:
  328. """Push files to the repository using the Git Data API."""
  329. try:
  330. owner, repo = self.parse_repo_url(repo_url)
  331. api_base = self.get_api_base(repo_url)
  332. headers = self.get_headers(token)
  333. ref_response = await client.get(f"{api_base}/repos/{owner}/{repo}/git/refs/heads/{branch}", headers=headers)
  334. empty = self._is_empty_repository(ref_response)
  335. if ref_response.status_code == 404 or empty:
  336. if not _allow_branch_create:
  337. return {
  338. "status": "failed",
  339. "message": (
  340. f"Branch '{branch}' not found after creation — possible replication lag. "
  341. "The next scheduled backup will retry."
  342. ),
  343. }
  344. if empty:
  345. return await self._seed_empty_repository(
  346. client, headers, api_base, owner, repo, branch, files, repo_url, token
  347. )
  348. return await self._create_branch_and_push(
  349. client, headers, api_base, owner, repo, branch, files, repo_url, token
  350. )
  351. if ref_response.status_code != 200:
  352. msg = f"Failed to get branch ref (HTTP {ref_response.status_code}): {self._truncated_response_text(ref_response)}"
  353. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  354. return {"status": "failed", "message": msg, "error": self._truncated_response_text(ref_response)}
  355. current_commit_sha, err = self._read_sha(ref_response, "object", "sha")
  356. if err:
  357. msg = f"Malformed ref response ({err}): {self._truncated_response_text(ref_response)}"
  358. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  359. return {"status": "failed", "message": msg}
  360. commit_response = await client.get(
  361. f"{api_base}/repos/{owner}/{repo}/git/commits/{current_commit_sha}", headers=headers
  362. )
  363. if commit_response.status_code != 200:
  364. msg = f"Failed to get current commit (HTTP {commit_response.status_code}): {self._truncated_response_text(commit_response)}"
  365. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  366. return {"status": "failed", "message": msg}
  367. current_tree_sha, err = self._read_sha(commit_response, "tree", "sha")
  368. if err:
  369. msg = f"Malformed commit response ({err}): {self._truncated_response_text(commit_response)}"
  370. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  371. return {"status": "failed", "message": msg}
  372. tree_response = await client.get(
  373. f"{api_base}/repos/{owner}/{repo}/git/trees/{current_tree_sha}?recursive=1", headers=headers
  374. )
  375. if tree_response.status_code != 200:
  376. msg = f"Failed to list existing tree (HTTP {tree_response.status_code}): {self._truncated_response_text(tree_response)}"
  377. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  378. return {"status": "failed", "message": msg, "error": self._truncated_response_text(tree_response)}
  379. tree_data = tree_response.json()
  380. # GitHub's tree API truncates >7MB / >100k entries. A truncated tree
  381. # listing makes the SHA-equality dedup miss and every file gets
  382. # re-uploaded as a new blob each run — silent churn until someone
  383. # notices the bloated history. Fail loudly so the user rotates the
  384. # backup repo.
  385. if tree_data.get("truncated"):
  386. msg = (
  387. "Repository tree exceeds the GitHub API listing limit (truncated=true). "
  388. "Rotate the backup repository to avoid silent file-by-file churn on every backup."
  389. )
  390. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  391. return {"status": "failed", "message": msg}
  392. existing_files: dict[str, str] = {}
  393. for item in tree_data.get("tree", []):
  394. if item.get("type") != "blob":
  395. continue
  396. path, sha = item.get("path"), item.get("sha")
  397. if not path or not sha:
  398. logger.warning("push_files: skipping malformed tree entry: %s", item)
  399. continue
  400. existing_files[path] = sha
  401. tree_items = []
  402. files_changed = 0
  403. for path, content in files.items():
  404. content_str = json.dumps(content, indent=2, default=str)
  405. content_bytes = content_str.encode("utf-8")
  406. content_sha = self._blob_sha(content_bytes)
  407. if path in existing_files and existing_files[path] == content_sha:
  408. continue
  409. blob_response = await client.post(
  410. f"{api_base}/repos/{owner}/{repo}/git/blobs",
  411. headers=headers,
  412. json={"content": base64.b64encode(content_bytes).decode(), "encoding": "base64"},
  413. )
  414. if blob_response.status_code == 404:
  415. msg = "GitHub API returned 404 for POST /git/blobs — check repository visibility and token scope"
  416. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  417. return {"status": "failed", "message": msg}
  418. if blob_response.status_code != 201:
  419. msg = f"Failed to create blob for {path} (HTTP {blob_response.status_code}): {self._truncated_response_text(blob_response)}"
  420. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  421. return {"status": "failed", "message": msg}
  422. blob_sha, err = self._read_sha(blob_response, "sha")
  423. if err:
  424. msg = f"Malformed blob response for {path} ({err}): {self._truncated_response_text(blob_response)}"
  425. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  426. return {"status": "failed", "message": msg}
  427. tree_items.append({"path": path, "mode": "100644", "type": "blob", "sha": blob_sha})
  428. files_changed += 1
  429. if not tree_items:
  430. return {"status": "skipped", "message": "No changes to commit", "commit_sha": None, "files_changed": 0}
  431. tree_response = await client.post(
  432. f"{api_base}/repos/{owner}/{repo}/git/trees",
  433. headers=headers,
  434. json={"base_tree": current_tree_sha, "tree": tree_items},
  435. )
  436. if tree_response.status_code != 201:
  437. msg = f"Failed to create tree (HTTP {tree_response.status_code}): {self._truncated_response_text(tree_response)}"
  438. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  439. return {"status": "failed", "message": msg}
  440. new_tree_sha, err = self._read_sha(tree_response, "sha")
  441. if err:
  442. msg = f"Malformed tree-create response ({err}): {self._truncated_response_text(tree_response)}"
  443. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  444. return {"status": "failed", "message": msg}
  445. commit_message = f"Bambuddy backup - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}"
  446. commit_response = await client.post(
  447. f"{api_base}/repos/{owner}/{repo}/git/commits",
  448. headers=headers,
  449. json={"message": commit_message, "tree": new_tree_sha, "parents": [current_commit_sha]},
  450. )
  451. if commit_response.status_code != 201:
  452. msg = f"Failed to create commit (HTTP {commit_response.status_code}): {self._truncated_response_text(commit_response)}"
  453. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  454. return {"status": "failed", "message": msg}
  455. new_commit_sha, err = self._read_sha(commit_response, "sha")
  456. if err:
  457. msg = f"Malformed commit-create response ({err}): {self._truncated_response_text(commit_response)}"
  458. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  459. return {"status": "failed", "message": msg}
  460. ref_update = await client.patch(
  461. f"{api_base}/repos/{owner}/{repo}/git/refs/heads/{branch}",
  462. headers=headers,
  463. json={"sha": new_commit_sha},
  464. )
  465. if ref_update.status_code != 200:
  466. msg = f"Failed to update branch (HTTP {ref_update.status_code}): {self._truncated_response_text(ref_update)}"
  467. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  468. return {"status": "failed", "message": msg}
  469. return {
  470. "status": "success",
  471. "message": f"Backup successful - {files_changed} files updated",
  472. "commit_sha": new_commit_sha,
  473. "files_changed": files_changed,
  474. }
  475. except Exception as e:
  476. logger.exception("push_files failed for %s branch=%s", repo_url, branch)
  477. return {"status": "failed", "message": str(e), "error": str(e)}
  478. async def _create_branch_and_push(
  479. self,
  480. client: httpx.AsyncClient,
  481. headers: dict,
  482. api_base: str,
  483. owner: str,
  484. repo: str,
  485. branch: str,
  486. files: dict,
  487. repo_url: str,
  488. token: str,
  489. ) -> dict:
  490. """Create branch (from default branch or as initial commit) then push."""
  491. try:
  492. repo_response = await client.get(f"{api_base}/repos/{owner}/{repo}", headers=headers)
  493. if repo_response.status_code != 200:
  494. msg = f"Failed to get repo info (HTTP {repo_response.status_code}): {self._truncated_response_text(repo_response)}"
  495. logger.warning("_create_branch_and_push %s/%s: %s", owner, repo, msg)
  496. return {"status": "failed", "message": msg}
  497. try:
  498. default_branch = repo_response.json().get("default_branch", "main")
  499. except ValueError:
  500. msg = f"Malformed repo-info response (non-JSON body): {self._truncated_response_text(repo_response)}"
  501. logger.warning("_create_branch_and_push %s/%s: %s", owner, repo, msg)
  502. return {"status": "failed", "message": msg}
  503. ref_response = await client.get(
  504. f"{api_base}/repos/{owner}/{repo}/git/refs/heads/{default_branch}", headers=headers
  505. )
  506. if self._is_empty_repository(ref_response):
  507. return await self._seed_empty_repository(
  508. client, headers, api_base, owner, repo, branch, files, repo_url, token
  509. )
  510. if ref_response.status_code != 200:
  511. return await self._create_initial_commit(client, headers, api_base, owner, repo, branch, files)
  512. base_sha, err = self._read_sha(ref_response, "object", "sha")
  513. if err:
  514. msg = f"Malformed default-branch ref response ({err}): {self._truncated_response_text(ref_response)}"
  515. logger.warning("_create_branch_and_push %s/%s: %s", owner, repo, msg)
  516. return {"status": "failed", "message": msg}
  517. create_ref = await client.post(
  518. f"{api_base}/repos/{owner}/{repo}/git/refs",
  519. headers=headers,
  520. json={"ref": f"refs/heads/{branch}", "sha": base_sha},
  521. )
  522. if create_ref.status_code != 201:
  523. msg = f"Failed to create branch '{branch}' (HTTP {create_ref.status_code}): {self._truncated_response_text(create_ref)}"
  524. logger.warning("_create_branch_and_push %s/%s: %s", owner, repo, msg)
  525. return {"status": "failed", "message": msg}
  526. logger.info("Re-entering push_files after branch create %s/%s -> %s", owner, repo, branch)
  527. return await self.push_files(repo_url, token, branch, files, client, _allow_branch_create=False)
  528. except Exception as e:
  529. logger.exception("_create_branch_and_push failed for %s/%s branch=%s", owner, repo, branch)
  530. return {"status": "failed", "message": str(e), "error": str(e)}
  531. @staticmethod
  532. def _is_empty_repository(response: httpx.Response) -> bool:
  533. """Whether GitHub refused a Git Data call because the repository has no commits (#3565)."""
  534. if response.status_code != 409:
  535. return False
  536. try:
  537. body = response.json()
  538. except ValueError:
  539. return False
  540. message = body.get("message") if isinstance(body, dict) else None
  541. return isinstance(message, str) and "repository is empty" in message.lower()
  542. async def _seed_empty_repository(
  543. self,
  544. client: httpx.AsyncClient,
  545. headers: dict,
  546. api_base: str,
  547. owner: str,
  548. repo: str,
  549. branch: str,
  550. files: dict,
  551. repo_url: str,
  552. token: str,
  553. ) -> dict:
  554. """Give a repository with no commits its first one, then push the rest (#3565).
  555. GitHub answers every Git Data call -- refs, blobs, trees -- with 409 until a
  556. repository has a commit, so the blob/tree path cannot make the first one.
  557. The Contents API can: it writes one file and creates the commit and the
  558. branch. The other files then go through push_files like any later backup.
  559. """
  560. if not files:
  561. return {"status": "skipped", "message": "No changes to commit", "commit_sha": None, "files_changed": 0}
  562. try:
  563. path = next(iter(files))
  564. content_bytes = json.dumps(files[path], indent=2, default=str).encode("utf-8")
  565. payload = {
  566. "message": f"Initial Bambuddy backup - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}",
  567. "content": base64.b64encode(content_bytes).decode(),
  568. "branch": branch,
  569. }
  570. url = f"{api_base}/repos/{owner}/{repo}/contents/{quote(path)}"
  571. response = await client.put(url, headers=headers, json=payload)
  572. on_backup_branch = True
  573. if response.status_code in (404, 422):
  574. # Should GitHub not create a named branch in a repository that
  575. # has none yet, the first commit goes to the default branch and
  576. # push_files then branches the backup branch off it.
  577. logger.info(
  578. "_seed_empty_repository %s/%s: branch %s refused (HTTP %s), seeding the default branch",
  579. owner,
  580. repo,
  581. branch,
  582. response.status_code,
  583. )
  584. default_branch_payload = {k: v for k, v in payload.items() if k != "branch"}
  585. response = await client.put(url, headers=headers, json=default_branch_payload)
  586. on_backup_branch = False
  587. if response.status_code != 201:
  588. msg = f"Failed to create the first commit (HTTP {response.status_code}): {self._truncated_response_text(response)}"
  589. logger.warning("_seed_empty_repository %s/%s: %s", owner, repo, msg)
  590. return {"status": "failed", "message": msg}
  591. commit_sha, err = self._read_sha(response, "commit", "sha")
  592. if err:
  593. msg = f"Malformed contents response ({err}): {self._truncated_response_text(response)}"
  594. logger.warning("_seed_empty_repository %s/%s: %s", owner, repo, msg)
  595. return {"status": "failed", "message": msg}
  596. seeded = {
  597. "status": "success",
  598. "message": f"Initial backup created - {len(files)} files",
  599. "commit_sha": commit_sha,
  600. "files_changed": len(files),
  601. }
  602. if len(files) == 1 and on_backup_branch:
  603. return seeded
  604. logger.info("Re-entering push_files after seeding %s/%s", owner, repo)
  605. result = await self.push_files(
  606. repo_url, token, branch, files, client, _allow_branch_create=not on_backup_branch
  607. )
  608. if result.get("status") == "skipped":
  609. return seeded
  610. if result.get("status") == "success":
  611. return {**seeded, "commit_sha": result.get("commit_sha")}
  612. return result
  613. except Exception as e:
  614. logger.exception("_seed_empty_repository failed for %s/%s branch=%s", owner, repo, branch)
  615. return {"status": "failed", "message": str(e), "error": str(e)}
  616. async def _create_initial_commit(
  617. self,
  618. client: httpx.AsyncClient,
  619. headers: dict,
  620. api_base: str,
  621. owner: str,
  622. repo: str,
  623. branch: str,
  624. files: dict,
  625. ) -> dict:
  626. """Create the first commit in an empty repository."""
  627. try:
  628. tree_items = []
  629. for path, content in files.items():
  630. content_str = json.dumps(content, indent=2, default=str)
  631. blob_response = await client.post(
  632. f"{api_base}/repos/{owner}/{repo}/git/blobs",
  633. headers=headers,
  634. json={"content": base64.b64encode(content_str.encode()).decode(), "encoding": "base64"},
  635. )
  636. if blob_response.status_code == 404:
  637. msg = "GitHub API returned 404 for POST /git/blobs — check repository visibility and token scope"
  638. logger.warning("_create_initial_commit %s/%s: %s", owner, repo, msg)
  639. return {"status": "failed", "message": msg}
  640. if blob_response.status_code != 201:
  641. msg = f"Failed to create blob for {path} (HTTP {blob_response.status_code}): {self._truncated_response_text(blob_response)}"
  642. logger.warning("_create_initial_commit %s/%s: %s", owner, repo, msg)
  643. return {"status": "failed", "message": msg}
  644. blob_sha, err = self._read_sha(blob_response, "sha")
  645. if err:
  646. msg = f"Malformed blob response for {path} ({err}): {self._truncated_response_text(blob_response)}"
  647. logger.warning("_create_initial_commit %s/%s: %s", owner, repo, msg)
  648. return {"status": "failed", "message": msg}
  649. tree_items.append({"path": path, "mode": "100644", "type": "blob", "sha": blob_sha})
  650. tree_response = await client.post(
  651. f"{api_base}/repos/{owner}/{repo}/git/trees",
  652. headers=headers,
  653. json={"tree": tree_items},
  654. )
  655. if tree_response.status_code != 201:
  656. msg = f"Failed to create tree (HTTP {tree_response.status_code}): {self._truncated_response_text(tree_response)}"
  657. logger.warning("_create_initial_commit %s/%s: %s", owner, repo, msg)
  658. return {"status": "failed", "message": msg}
  659. tree_sha, err = self._read_sha(tree_response, "sha")
  660. if err:
  661. msg = f"Malformed tree-create response ({err}): {self._truncated_response_text(tree_response)}"
  662. logger.warning("_create_initial_commit %s/%s: %s", owner, repo, msg)
  663. return {"status": "failed", "message": msg}
  664. commit_response = await client.post(
  665. f"{api_base}/repos/{owner}/{repo}/git/commits",
  666. headers=headers,
  667. json={
  668. "message": f"Initial Bambuddy backup - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}",
  669. "tree": tree_sha,
  670. },
  671. )
  672. if commit_response.status_code != 201:
  673. msg = f"Failed to create commit (HTTP {commit_response.status_code}): {self._truncated_response_text(commit_response)}"
  674. logger.warning("_create_initial_commit %s/%s: %s", owner, repo, msg)
  675. return {"status": "failed", "message": msg}
  676. commit_sha, err = self._read_sha(commit_response, "sha")
  677. if err:
  678. msg = f"Malformed commit-create response ({err}): {self._truncated_response_text(commit_response)}"
  679. logger.warning("_create_initial_commit %s/%s: %s", owner, repo, msg)
  680. return {"status": "failed", "message": msg}
  681. ref_response = await client.post(
  682. f"{api_base}/repos/{owner}/{repo}/git/refs",
  683. headers=headers,
  684. json={"ref": f"refs/heads/{branch}", "sha": commit_sha},
  685. )
  686. if ref_response.status_code != 201:
  687. msg = f"Failed to create branch ref (HTTP {ref_response.status_code}): {self._truncated_response_text(ref_response)}"
  688. logger.warning("_create_initial_commit %s/%s: %s", owner, repo, msg)
  689. return {"status": "failed", "message": msg}
  690. return {
  691. "status": "success",
  692. "message": f"Initial backup created - {len(files)} files",
  693. "commit_sha": commit_sha,
  694. "files_changed": len(files),
  695. }
  696. except Exception as e:
  697. logger.exception("_create_initial_commit failed for %s/%s branch=%s", owner, repo, branch)
  698. return {"status": "failed", "message": str(e), "error": str(e)}