log_reader.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315
  1. """Shared primitives for reading, parsing, and sanitizing the Bambuddy app log.
  2. Extracted from ``routes/support.py`` so service-layer code (e.g. the log-health
  3. scanner in ``log_health.py``) can reuse log reading and redaction without
  4. importing from the API layer. ``support.py`` re-imports these helpers and keeps
  5. its own route handlers.
  6. """
  7. import logging
  8. import re
  9. from pydantic import BaseModel
  10. from sqlalchemy import select
  11. from sqlalchemy.ext.asyncio import AsyncSession
  12. from backend.app.core.config import settings
  13. from backend.app.core.logging_filters import (
  14. HOST_ONLY_LOGGERS,
  15. URL_CREDENTIALS_PATTERN,
  16. redact_device_urls,
  17. redact_query_tokens,
  18. redact_refused_headers,
  19. redact_url_credentials,
  20. )
  21. from backend.app.models.printer import Printer
  22. from backend.app.models.settings import Settings
  23. from backend.app.models.user import User
  24. from backend.app.models.virtual_printer import VirtualPrinter
  25. from backend.app.utils.printer_models import PRINTER_MODEL_ID_MAP, PRINTER_MODEL_MAP
  26. logger = logging.getLogger(__name__)
  27. # Log line format: "2024-01-15 10:30:45,123 INFO [module.name] [trace_id] Message"
  28. # The trace_id is left as part of the message group — callers that need it can
  29. # parse it out; the log-health scanner does not.
  30. LOG_LINE_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2},\d{3})\s+(\w+)\s+\[([^\]]+)\]\s+(.*)$")
  31. # LDAP Distinguished Names carry PII — the leaf ``CN=`` is the user's real name
  32. # (#2681). Match a run of at least two ``attr=value`` RDN components joined by
  33. # commas, where ``attr`` is a known LDAP attribute type. Requiring two components
  34. # keeps this from clobbering an incidental ``key=value`` in an unrelated log line,
  35. # while still catching DNs wherever they surface — the deliberate "auth successful"
  36. # line, ldap3 exception strings, and group DNs alike. Bias is intentionally toward
  37. # redaction: over-redacting a rare debug line to ``[DN]`` is a safe failure; leaking
  38. # a name is not.
  39. # The value char class excludes `<>;+` — RFC 4514 requires those escaped inside a
  40. # DN value, so an unescaped one marks the end of the DN, not part of it. That stops
  41. # the final (comma-unbounded) component from greedily swallowing trailing log text
  42. # such as ``… -> GroupName``.
  43. _LDAP_RDN = r"(?:CN|OU|DC|UID|O|L|ST|C|SN|GN|DN|E|MAIL|STREET|GIVENNAME|SURNAME)=[^,\n<>;+]+"
  44. _LDAP_DN_PATTERN = re.compile(rf"(?i)\b{_LDAP_RDN}(?:\s*,\s*{_LDAP_RDN})+")
  45. class LogEntry(BaseModel):
  46. """A single parsed log entry."""
  47. timestamp: str
  48. level: str
  49. logger_name: str
  50. message: str
  51. def parse_log_line(line: str) -> LogEntry | None:
  52. """Parse a single log line into a LogEntry, or None if it is not a line start."""
  53. match = LOG_LINE_PATTERN.match(line.strip())
  54. if match:
  55. return LogEntry(
  56. timestamp=match.group(1),
  57. level=match.group(2),
  58. logger_name=match.group(3),
  59. message=match.group(4),
  60. )
  61. return None
  62. def _logger_of(line: str) -> str | None:
  63. match = LOG_LINE_PATTERN.match(line.strip())
  64. return match.group(3) if match else None
  65. def _redacted(entry: LogEntry) -> LogEntry:
  66. # Lines on disk may predate a redaction: masked as they are read, before
  67. # any search, so a search cannot confirm a secret either.
  68. entry.message = redact_refused_headers(redact_url_credentials(redact_query_tokens(entry.message))) or ""
  69. if entry.logger_name in HOST_ONLY_LOGGERS:
  70. entry.message = redact_device_urls(entry.message) or ""
  71. return entry
  72. def read_log_entries(
  73. limit: int = 200,
  74. level_filter: str | None = None,
  75. search: str | None = None,
  76. ) -> tuple[list[LogEntry], int]:
  77. """Read and parse log entries from ``bambuddy.log``, newest first.
  78. Continuation lines (tracebacks etc.) are folded into the message of the
  79. entry they belong to. Returns ``(entries, total_lines_in_file)``.
  80. """
  81. log_file = settings.log_dir / "bambuddy.log"
  82. if not log_file.exists():
  83. return [], 0
  84. entries: list[LogEntry] = []
  85. total_lines = 0
  86. try:
  87. with open(log_file, encoding="utf-8", errors="replace") as f:
  88. lines = f.readlines()
  89. total_lines = len(lines)
  90. # Parse lines in reverse order (newest first)
  91. current_entry: LogEntry | None = None
  92. multi_line_buffer: list[str] = []
  93. for line in reversed(lines):
  94. parsed = parse_log_line(line)
  95. if parsed:
  96. # Found a new log entry start
  97. if current_entry:
  98. current_entry = _redacted(current_entry)
  99. # Apply filters and add previous entry (without multi_line_buffer - it belongs to new entry)
  100. should_include = True
  101. # Level filter
  102. if level_filter and current_entry.level.upper() != level_filter.upper():
  103. should_include = False
  104. # Search filter (case-insensitive)
  105. if search and should_include:
  106. search_lower = search.lower()
  107. if not (
  108. search_lower in current_entry.message.lower()
  109. or search_lower in current_entry.logger_name.lower()
  110. ):
  111. should_include = False
  112. if should_include:
  113. entries.append(current_entry)
  114. if len(entries) >= limit:
  115. break
  116. # Set new entry and attach any accumulated multi-line content to it
  117. # (in reverse order, continuation lines come before their parent entry)
  118. current_entry = parsed
  119. if multi_line_buffer:
  120. current_entry.message += "\n" + "\n".join(reversed(multi_line_buffer))
  121. multi_line_buffer = []
  122. elif line.strip():
  123. # Continuation of multi-line log entry (will be attached to next parsed entry)
  124. multi_line_buffer.append(line.rstrip())
  125. # Don't forget the last (oldest) entry
  126. # Note: any remaining multi_line_buffer would be orphaned lines before the first entry
  127. if current_entry and len(entries) < limit:
  128. current_entry = _redacted(current_entry)
  129. should_include = True
  130. if level_filter and current_entry.level.upper() != level_filter.upper():
  131. should_include = False
  132. if search and should_include:
  133. search_lower = search.lower()
  134. if not (
  135. search_lower in current_entry.message.lower()
  136. or search_lower in current_entry.logger_name.lower()
  137. ):
  138. should_include = False
  139. if should_include:
  140. entries.append(current_entry)
  141. except Exception as e:
  142. logger.error("Error reading log file: %s", e)
  143. return [], 0
  144. # Entries are already in newest-first order
  145. return entries, total_lines
  146. def sanitize_log_content(content: str, sensitive_strings: dict[str, str] | None = None) -> str:
  147. """Remove sensitive data from log content.
  148. ``sensitive_strings`` maps known exact values (printer names, serials, etc.)
  149. to replacement labels; pass the result of :func:`collect_sensitive_strings`.
  150. Regex passes additionally redact credentials in URLs, emails, serials, and
  151. IP addresses that were not captured by exact matching.
  152. """
  153. # Lines from loggers that name a device by its whole URL keep its host only.
  154. # First, while each line's logger can still be read: a known value below
  155. # may be part of a logger name.
  156. content = "\n".join(
  157. redact_device_urls(line) or "" if _logger_of(line) in HOST_ONLY_LOGGERS else line
  158. for line in content.split("\n")
  159. )
  160. # Then replace known sensitive values (database-aware exact matching)
  161. # This catches printer names, usernames, and other arbitrary user-chosen strings
  162. # that regex patterns cannot detect
  163. if sensitive_strings:
  164. # Sort by length descending to avoid partial matches (e.g. "My Printer 1" before "My Printer")
  165. for value, label in sorted(sensitive_strings.items(), key=lambda x: len(x[0]), reverse=True):
  166. if len(value) < 3:
  167. continue # Skip very short strings to prevent over-redaction
  168. content = re.sub(re.escape(value), label, content)
  169. # Replace credentials in URLs (e.g. http://user:pass@host, rtsps://bblp:code@host).
  170. # Shares its pattern with the log-pipeline redaction in ``core.logging_filters`` so
  171. # the two can't drift; the bundle drops the username too, where the live log keeps
  172. # it for diagnosis.
  173. content = URL_CREDENTIALS_PATTERN.sub(r"\g<scheme>[CREDENTIALS]@", content)
  174. # Query-string tokens (?token=...), for log lines written before the live
  175. # filter existed or by a logger it is not attached to.
  176. content = redact_query_tokens(content) or ""
  177. # A header an HTTP client refused, quoted with its value.
  178. content = redact_refused_headers(content) or ""
  179. # Replace email addresses
  180. content = re.sub(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", "[EMAIL]", content)
  181. # Replace LDAP Distinguished Names (#2681) — PII on par with email.
  182. content = _LDAP_DN_PATTERN.sub("[DN]", content)
  183. # Replace Bambu Lab printer serial numbers (format: 00M/01D/01S/01P/03W + alphanumeric, 12-16 chars total)
  184. content = re.sub(r"\b0[0-3][A-Z0-9][A-Z0-9]{9,13}\b", "[SERIAL]", content, flags=re.IGNORECASE)
  185. # Replace IPv4 addresses (skip firmware versions like 01.09.01.00 which have leading zeros,
  186. # and four-number runs inside a longer dotted number, such as the LDAP StartTLS OID
  187. # 1.3.6.1.4.1.1466.20037, whose masked "[IP].4.1.1466.20037" hid what failed, #3197)
  188. content = re.sub(
  189. r"(?<!\d\.)\b(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]\d|\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]\d|\d)\b(?!\.\d)",
  190. "[IP]",
  191. content,
  192. )
  193. # Replace paths with usernames
  194. content = re.sub(r"/home/[^/\s]+/", "/home/[user]/", content)
  195. content = re.sub(r"/Users/[^/\s]+/", "/Users/[user]/", content)
  196. content = re.sub(r"/opt/[^/\s]+/", "/opt/[user]/", content)
  197. return content
  198. def _model_key(value: str) -> str:
  199. return value.upper().replace(" ", "").replace("-", "")
  200. # Every model name and code Bambuddy knows. A printer named after its model
  201. # keeps nothing private in that name, and redacting it would wipe the model
  202. # from the whole support package (#3442).
  203. _MODEL_NAME_KEYS = frozenset(
  204. _model_key(v)
  205. for v in (*PRINTER_MODEL_MAP, *PRINTER_MODEL_MAP.values(), *PRINTER_MODEL_ID_MAP, *PRINTER_MODEL_ID_MAP.values())
  206. )
  207. def _is_model_name(name: str, model: str | None) -> bool:
  208. """True when the printer name is just a model name, so not worth redacting.
  209. The printer's own model counts too, which covers a model Bambuddy does not
  210. list yet.
  211. """
  212. key = _model_key(name.strip())
  213. return key in _MODEL_NAME_KEYS or (bool(model) and key == _model_key(model.strip()))
  214. async def collect_sensitive_strings(db: AsyncSession) -> dict[str, str]:
  215. """Collect known sensitive values from the database for log redaction.
  216. Covers printer names, serial numbers (virtual printers' too), IP
  217. addresses, access codes, auth usernames, and the Bambu Cloud email. Pass the result to
  218. :func:`sanitize_log_content`.
  219. """
  220. sensitive_strings: dict[str, str] = {}
  221. # Printer names, serial numbers, IP addresses, and access codes
  222. result = await db.execute(
  223. select(Printer.name, Printer.serial_number, Printer.ip_address, Printer.access_code, Printer.model)
  224. )
  225. for name, serial, ip_address, access_code, model in result.all():
  226. if name and not _is_model_name(name, model):
  227. sensitive_strings[name] = "[PRINTER]"
  228. if serial:
  229. sensitive_strings[serial] = "[SERIAL]"
  230. if ip_address:
  231. sensitive_strings[ip_address] = "[IP]"
  232. if access_code:
  233. sensitive_strings[access_code] = "[ACCESS_CODE]"
  234. # Virtual printer serials. Stored only as a suffix: the serial is the
  235. # model's prefix plus it, and many prefixes (A2L, P2S, H2D, X2D) are past
  236. # what the serial pattern below matches, so the certificate log lines
  237. # carried them whole (#3526). Every prefix, so a log written before the
  238. # VP's model was changed is covered as well.
  239. from backend.app.services.virtual_printer.manager import MODEL_SERIAL_PREFIXES, _get_serial_for_model
  240. result = await db.execute(select(VirtualPrinter.serial_suffix))
  241. for (suffix,) in result.all():
  242. if suffix:
  243. for model in (*MODEL_SERIAL_PREFIXES, ""):
  244. sensitive_strings[_get_serial_for_model(model, suffix)] = "[SERIAL]"
  245. # Auth usernames
  246. result = await db.execute(select(User.username))
  247. for (username,) in result.all():
  248. if username:
  249. sensitive_strings[username] = "[USER]"
  250. # Bambu Cloud email
  251. result = await db.execute(select(Settings.value).where(Settings.key == "bambu_cloud_email"))
  252. cloud_email = result.scalar_one_or_none()
  253. if cloud_email:
  254. sensitive_strings[cloud_email] = "[EMAIL]"
  255. return sensitive_strings