gitea.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372
  1. """Gitea backend — overrides GitHubBackend where Gitea's API diverges."""
  2. import base64
  3. import json
  4. import logging
  5. import re
  6. from datetime import datetime, timezone
  7. import httpx
  8. from backend.app.services.git_providers.github import GitHubBackend
  9. logger = logging.getLogger(__name__)
  10. class GiteaBackend(GitHubBackend):
  11. """Backend for Gitea instances.
  12. Gitea's Git Data API (/api/v1/repos/{owner}/{repo}/git/...) is *mostly*
  13. compatible with GitHub's, but diverges on three points that broke real-world
  14. backups (#1224, #1225, #1239):
  15. 1. ``GET /git/refs/heads/{branch}`` returns a *list* of matching refs even
  16. when only one matches; GitHub returns a single object. The push paths
  17. below extract the SHA via ``_ref_sha()`` instead of the GitHub-style
  18. ``["object"]["sha"]`` chain.
  19. 2. The Git Data API (blobs/trees/commits/refs) refuses writes against an
  20. empty repository — every blob POST returns 404 until the repo has at
  21. least one commit. ``_create_initial_commit()`` is overridden to use the
  22. Contents API, which seeds the branch + initial commit in a single call.
  23. 3. The Git Data API does not support atomic multi-file commits — each file
  24. requires a separate blob POST followed by a tree/commit/ref sequence.
  25. ``push_files()`` is overridden to use the Contents API
  26. (``POST /repos/.../contents`` with a ``files`` array), which commits all
  27. changed files in a single round-trip and avoids partial-commit failures.
  28. """
  29. @staticmethod
  30. def _ref_sha(ref_data) -> str:
  31. """Extract the commit SHA from Gitea's list-shaped ref response."""
  32. if isinstance(ref_data, list):
  33. if not ref_data:
  34. raise ValueError("Empty refs list returned by Gitea API")
  35. return ref_data[0]["object"]["sha"]
  36. return ref_data["object"]["sha"]
  37. @staticmethod
  38. def _commit_tree_sha(commit_data: dict) -> str | None:
  39. """Extract the tree SHA from a commit response.
  40. GitHub's ``GET /git/commits/{sha}`` returns the GitCommit schema with
  41. ``tree`` at the top level. Gitea's same-named endpoint may return the
  42. wrapped Commit schema where ``tree`` lives under ``commit``. Try the
  43. flat shape first (GitHub-compatible deployments and some Gitea/Forgejo
  44. versions) then fall back to the wrapped shape.
  45. """
  46. tree_node = commit_data.get("tree")
  47. if not isinstance(tree_node, dict):
  48. tree_node = (commit_data.get("commit") or {}).get("tree")
  49. if isinstance(tree_node, dict):
  50. return tree_node.get("sha")
  51. return None
  52. # Gitea/Forgejo can be hosted under a URL path prefix (ROOT_URL like
  53. # https://host/gitea), so the repo lives at /<prefix...>/<owner>/<repo>
  54. # rather than at the host root (#2642). Capture the scheme+host+prefix as
  55. # one group and the final two path segments as owner/repo; the lazy prefix
  56. # group is empty for a root-hosted instance. One shared pattern keeps
  57. # parse_repo_url() and get_api_base() from drifting.
  58. _HTTPS_REPO_RE = re.compile(
  59. r"(https?://[\w.\-]+(?::\d+)?(?:/[\w.\-]+)*?)/([\w.\-]{1,100})/([\w.\-]{1,100})(?:\.git)?/?$"
  60. )
  61. def parse_repo_url(self, url: str) -> tuple[str, str]:
  62. """Return (owner, repo) — accepts both https:// and http:// for self-hosted instances."""
  63. if not url or len(url) > 500:
  64. raise ValueError("Invalid Git URL: URL too long or empty")
  65. match = self._HTTPS_REPO_RE.match(url)
  66. if match:
  67. return match.group(2), match.group(3).removesuffix(".git")
  68. match = re.match(
  69. r"git@[\w.\-]+:([\w.\-]{1,100})/([\w.\-]{1,100})(?:\.git)?$",
  70. url,
  71. )
  72. if match:
  73. return match.group(1), match.group(2).removesuffix(".git")
  74. raise ValueError(f"Cannot parse repository URL: {url}")
  75. def get_api_base(self, repo_url: str) -> str:
  76. """Derive API base from the repository URL's scheme, host and any path prefix."""
  77. match = self._HTTPS_REPO_RE.match(repo_url)
  78. if match:
  79. return f"{match.group(1)}/api/v1"
  80. raise ValueError(f"Cannot derive API base from URL: {repo_url}")
  81. def get_headers(self, token: str) -> dict:
  82. headers = super().get_headers(token)
  83. headers["Accept"] = "application/json"
  84. return headers
  85. async def push_files(
  86. self,
  87. repo_url: str,
  88. token: str,
  89. branch: str,
  90. files: dict,
  91. client: httpx.AsyncClient,
  92. _allow_branch_create: bool = True,
  93. ) -> dict:
  94. """Push files via the Git Data API, normalising Gitea's list-shaped ref response."""
  95. try:
  96. owner, repo = self.parse_repo_url(repo_url)
  97. api_base = self.get_api_base(repo_url)
  98. headers = self.get_headers(token)
  99. ref_response = await client.get(f"{api_base}/repos/{owner}/{repo}/git/refs/heads/{branch}", headers=headers)
  100. if ref_response.status_code == 404:
  101. if not _allow_branch_create:
  102. return {
  103. "status": "failed",
  104. "message": (
  105. f"Branch '{branch}' not found after creation — possible replication lag. "
  106. "The next scheduled backup will retry."
  107. ),
  108. }
  109. return await self._create_branch_and_push(
  110. client, headers, api_base, owner, repo, branch, files, repo_url, token
  111. )
  112. if ref_response.status_code != 200:
  113. return {
  114. "status": "failed",
  115. "message": f"Failed to get branch ref: {ref_response.status_code}",
  116. "error": self._truncated_response_text(ref_response),
  117. }
  118. current_commit_sha = self._ref_sha(ref_response.json())
  119. commit_response = await client.get(
  120. f"{api_base}/repos/{owner}/{repo}/git/commits/{current_commit_sha}", headers=headers
  121. )
  122. if commit_response.status_code != 200:
  123. msg = f"Failed to get current commit (HTTP {commit_response.status_code}): {self._truncated_response_text(commit_response)}"
  124. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  125. return {"status": "failed", "message": msg}
  126. current_tree_sha = self._commit_tree_sha(commit_response.json())
  127. if not current_tree_sha:
  128. msg = (
  129. f"Failed to extract tree SHA from commit response: {self._truncated_response_text(commit_response)}"
  130. )
  131. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  132. return {"status": "failed", "message": msg}
  133. tree_response = await client.get(
  134. f"{api_base}/repos/{owner}/{repo}/git/trees/{current_tree_sha}?recursive=1", headers=headers
  135. )
  136. if tree_response.status_code != 200:
  137. msg = f"Failed to list existing tree (HTTP {tree_response.status_code}): {self._truncated_response_text(tree_response)}"
  138. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  139. return {"status": "failed", "message": msg, "error": self._truncated_response_text(tree_response)}
  140. tree_data = tree_response.json()
  141. # Gitea's tree API can report ``truncated: true`` for large
  142. # listings; if we honour the partial map, the dedup check misses
  143. # and every file gets re-uploaded each run.
  144. if tree_data.get("truncated"):
  145. msg = (
  146. "Repository tree exceeds the Gitea API listing limit (truncated=true). "
  147. "Rotate the backup repository to avoid silent file-by-file churn on every backup."
  148. )
  149. logger.warning("push_files %s/%s: %s", owner, repo, msg)
  150. return {"status": "failed", "message": msg}
  151. existing_files: dict[str, str] = {}
  152. for item in tree_data.get("tree", []):
  153. if item.get("type") != "blob":
  154. continue
  155. path, sha = item.get("path"), item.get("sha")
  156. if not path or not sha:
  157. logger.warning("push_files: skipping malformed tree entry: %s", item)
  158. continue
  159. existing_files[path] = sha
  160. api_files = []
  161. files_changed = 0
  162. for path, content in files.items():
  163. content_str = json.dumps(content, indent=2, default=str)
  164. content_bytes = content_str.encode("utf-8")
  165. content_b64 = base64.b64encode(content_bytes).decode()
  166. content_sha = self._blob_sha(content_bytes)
  167. if path in existing_files:
  168. if existing_files[path] == content_sha:
  169. continue
  170. api_files.append(
  171. {"operation": "update", "path": path, "content": content_b64, "sha": existing_files[path]}
  172. )
  173. else:
  174. api_files.append({"operation": "create", "path": path, "content": content_b64})
  175. files_changed += 1
  176. if not api_files:
  177. return {"status": "skipped", "message": "No changes to commit", "commit_sha": None, "files_changed": 0}
  178. commit_message = f"Bambuddy backup - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}"
  179. response = await client.post(
  180. f"{api_base}/repos/{owner}/{repo}/contents",
  181. headers=headers,
  182. json={"branch": branch, "message": commit_message, "files": api_files},
  183. )
  184. if response.status_code == 404:
  185. return {
  186. "status": "failed",
  187. "message": "Contents API endpoint not found — your Gitea instance may be older than v1.18 or the API may be disabled by an administrator (POST /contents returned 404)",
  188. }
  189. if response.status_code == 409:
  190. return {
  191. "status": "failed",
  192. "message": (
  193. "Conflict committing files — the branch likely advanced concurrently "
  194. "(web-UI edit, another backup run, or path-vs-tree collision). "
  195. "The next scheduled backup will re-read the current tree and resolve this."
  196. ),
  197. }
  198. if response.status_code not in (200, 201):
  199. return {
  200. "status": "failed",
  201. "message": f"Backup commit failed: {self._truncated_response_text(response)}",
  202. }
  203. commit_sha = (response.json().get("commit") or {}).get("sha")
  204. message = (
  205. f"Backup successful - {files_changed} files updated"
  206. if commit_sha
  207. else f"Backup successful - {files_changed} files updated (commit SHA not reported by server)"
  208. )
  209. return {
  210. "status": "success",
  211. "message": message,
  212. "commit_sha": commit_sha,
  213. "files_changed": files_changed,
  214. }
  215. except Exception as e:
  216. logger.exception("push_files failed for %s branch=%s", repo_url, branch)
  217. return {"status": "failed", "message": str(e), "error": str(e)}
  218. async def _create_branch_and_push(
  219. self,
  220. client: httpx.AsyncClient,
  221. headers: dict,
  222. api_base: str,
  223. owner: str,
  224. repo: str,
  225. branch: str,
  226. files: dict,
  227. repo_url: str,
  228. token: str,
  229. ) -> dict:
  230. """Create branch (from default branch or as initial commit) then push."""
  231. try:
  232. repo_response = await client.get(f"{api_base}/repos/{owner}/{repo}", headers=headers)
  233. if repo_response.status_code != 200:
  234. msg = f"Failed to get repo info (HTTP {repo_response.status_code}): {self._truncated_response_text(repo_response)}"
  235. logger.warning("_create_branch_and_push %s/%s: %s", owner, repo, msg)
  236. return {"status": "failed", "message": msg}
  237. default_branch = repo_response.json().get("default_branch", "main")
  238. # GET the default branch to confirm the repo is non-empty; SHA is intentionally unused —
  239. # POST /branches takes a branch name, not a SHA.
  240. ref_response = await client.get(
  241. f"{api_base}/repos/{owner}/{repo}/git/refs/heads/{default_branch}", headers=headers
  242. )
  243. if ref_response.status_code != 200:
  244. return await self._create_initial_commit(client, headers, api_base, owner, repo, branch, files)
  245. create_ref = await client.post(
  246. f"{api_base}/repos/{owner}/{repo}/branches",
  247. headers=headers,
  248. json={"new_branch_name": branch, "old_ref_name": default_branch},
  249. )
  250. if create_ref.status_code == 403:
  251. msg = f"Permission denied creating branch '{branch}' — token may lack write access to this repository"
  252. logger.warning("_create_branch_and_push %s/%s: 403 %s", owner, repo, msg)
  253. return {"status": "failed", "message": msg}
  254. if create_ref.status_code == 409:
  255. msg = f"Branch '{branch}' already exists (possible race condition)"
  256. logger.warning("_create_branch_and_push %s/%s: 409 %s", owner, repo, msg)
  257. return {"status": "failed", "message": msg}
  258. if create_ref.status_code != 201:
  259. msg = f"Failed to create branch '{branch}' (HTTP {create_ref.status_code}): {self._truncated_response_text(create_ref)}"
  260. logger.warning("_create_branch_and_push %s/%s: %s", owner, repo, msg)
  261. return {"status": "failed", "message": msg}
  262. logger.info("Re-entering push_files after branch create %s/%s -> %s", owner, repo, branch)
  263. return await self.push_files(repo_url, token, branch, files, client, _allow_branch_create=False)
  264. except Exception as e:
  265. logger.exception("_create_branch_and_push failed for %s/%s branch=%s", owner, repo, branch)
  266. return {"status": "failed", "message": str(e), "error": str(e)}
  267. async def _create_initial_commit(
  268. self,
  269. client: httpx.AsyncClient,
  270. headers: dict,
  271. api_base: str,
  272. owner: str,
  273. repo: str,
  274. branch: str,
  275. files: dict,
  276. ) -> dict:
  277. """Seed an empty Gitea repository via the Contents API.
  278. Gitea's Git Data API requires the repository to have at least one
  279. commit before it accepts blob/tree/commit writes; on an empty repo
  280. every ``POST /git/blobs`` returns 404. The Contents API is the
  281. documented bootstrap path: a single ``POST /repos/{owner}/{repo}/contents``
  282. with a ``files`` array creates the initial commit and the target
  283. branch in one round-trip (Gitea 1.18+, Forgejo all versions).
  284. """
  285. try:
  286. if not files:
  287. return {"status": "skipped", "message": "No files to commit", "commit_sha": None, "files_changed": 0}
  288. api_files = []
  289. for path, content in files.items():
  290. content_str = json.dumps(content, indent=2, default=str)
  291. content_b64 = base64.b64encode(content_str.encode("utf-8")).decode()
  292. api_files.append({"operation": "create", "path": path, "content": content_b64})
  293. commit_message = f"Initial Bambuddy backup - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}"
  294. body = {
  295. "branch": branch,
  296. "new_branch": branch,
  297. "message": commit_message,
  298. "files": api_files,
  299. }
  300. response = await client.post(
  301. f"{api_base}/repos/{owner}/{repo}/contents",
  302. headers=headers,
  303. json=body,
  304. )
  305. if response.status_code not in (200, 201):
  306. return {
  307. "status": "failed",
  308. "message": f"Failed to create initial commit: {self._truncated_response_text(response)}",
  309. }
  310. data = response.json()
  311. commit_sha = (data.get("commit") or {}).get("sha")
  312. message = (
  313. f"Initial backup created - {len(files)} files"
  314. if commit_sha
  315. else f"Initial backup created - {len(files)} files (commit SHA not reported by server)"
  316. )
  317. return {
  318. "status": "success",
  319. "message": message,
  320. "commit_sha": commit_sha,
  321. "files_changed": len(files),
  322. }
  323. except Exception as e:
  324. logger.exception("_create_initial_commit failed for %s/%s branch=%s", owner, repo, branch)
  325. return {"status": "failed", "message": str(e), "error": str(e)}