test_upload_failure_reason_2899.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440
  1. """Say what actually went wrong, not what usually does (#2899).
  2. Every failed dispatch upload used to carry the same sentence: "Failed to upload
  3. file to printer. Check if SD card is inserted and properly formatted
  4. (FAT32/exFAT)." The reporter got it after a TLS handshake failure and restarted
  5. the printer on the strength of it. That could not have helped -- the handshake
  6. never reached the printer's filesystem, and the cool-off that produced the
  7. repeat failure lives in Bambuddy's own memory, where power-cycling a printer
  8. does not reach.
  9. #2780 had already removed operator advice from this failure's *log* line, for
  10. exactly this reason. The advice survived in the string people actually read.
  11. The information was never missing. ``connect`` separates five failure classes
  12. and ``upload_file`` separates 553/552/550, each with its own log line -- and
  13. both then returned a bare ``False``. These tests pin the reason travelling out
  14. to the caller, and the card being named only where the printer itself raised
  15. storage.
  16. """
  17. import ftplib # nosec B402 -- tests construct real ftplib error types
  18. import ssl
  19. import time
  20. from contextlib import ExitStack
  21. from pathlib import Path
  22. from types import SimpleNamespace
  23. from unittest.mock import AsyncMock, MagicMock, patch
  24. import pytest
  25. from backend.app.services.bambu_ftp import (
  26. BambuFTPClient,
  27. FtpFailure,
  28. FtpFailureKind,
  29. FtpFailureReport,
  30. describe_upload_failure,
  31. upload_file_async,
  32. with_ftp_retry,
  33. )
  34. pytestmark = pytest.mark.unit
  35. IP = "192.168.50.142"
  36. @pytest.fixture(autouse=True)
  37. def _clean_state():
  38. BambuFTPClient._handshake_blocked_until.clear()
  39. BambuFTPClient._handshake_skip_logged.clear()
  40. BambuFTPClient._mode_cache.clear()
  41. yield
  42. BambuFTPClient._handshake_blocked_until.clear()
  43. BambuFTPClient._handshake_skip_logged.clear()
  44. BambuFTPClient._mode_cache.clear()
  45. # ---------------------------------------------------------------------------
  46. # The client records which of its own branches it took
  47. # ---------------------------------------------------------------------------
  48. @pytest.mark.parametrize(
  49. ("error", "kind", "code"),
  50. [
  51. (ssl.SSLError("[SSL: WRONG_VERSION_NUMBER] wrong version number"), FtpFailureKind.HANDSHAKE, None),
  52. (TimeoutError("handshake operation timed out"), FtpFailureKind.TIMEOUT, None),
  53. (ftplib.error_perm("530 Login incorrect."), FtpFailureKind.AUTH, "530"),
  54. (OSError("Connection reset by peer"), FtpFailureKind.NETWORK, None),
  55. ],
  56. ids=["handshake", "timeout", "auth", "network"],
  57. )
  58. def test_connect_records_which_failure_it_hit(error, kind, code):
  59. transport = MagicMock()
  60. transport.connect.side_effect = error
  61. with patch("backend.app.services.bambu_ftp.ImplicitFTP_TLS", return_value=transport):
  62. client = BambuFTPClient(IP, "12345678", printer_model="P2S")
  63. assert client.connect() is False
  64. assert client.last_failure is not None
  65. assert client.last_failure.kind is kind
  66. assert client.last_failure.code == code
  67. # The underlying text is kept too -- the sentence is for the operator, the
  68. # detail is for whoever reads the log next to it.
  69. assert str(error)[:20] in client.last_failure.detail
  70. def test_the_cooloff_skip_is_its_own_kind():
  71. """ "We did not try" is not the same failure as "we tried and it broke"."""
  72. BambuFTPClient._handshake_blocked_until[IP] = time.monotonic() + 300
  73. client = BambuFTPClient(IP, "12345678")
  74. assert client.connect() is False
  75. assert client.last_failure is not None
  76. assert client.last_failure.kind is FtpFailureKind.COOLOFF
  77. @pytest.mark.parametrize(
  78. ("reply", "kind"),
  79. [
  80. ("553 Could not create file.", FtpFailureKind.STORAGE),
  81. ("552 Storage quota exceeded.", FtpFailureKind.STORAGE),
  82. ("550 Permission denied.", FtpFailureKind.NOT_FOUND),
  83. ("500 Unknown command.", FtpFailureKind.UNKNOWN),
  84. ],
  85. ids=["553", "552", "550", "500"],
  86. )
  87. def test_upload_classifies_the_printers_reply_code(reply, kind, tmp_path):
  88. """553 and 552 are the printer talking about its own storage.
  89. That is the one case where naming the SD card is worth anything, and it is
  90. the case the blanket message was written for before it was applied to
  91. every failure alike.
  92. """
  93. local = tmp_path / "job.3mf"
  94. local.write_bytes(b"x" * 16)
  95. client = BambuFTPClient(IP, "12345678")
  96. client._ftp = MagicMock()
  97. client._ftp.transfercmd.side_effect = ftplib.error_perm(reply)
  98. assert client.upload_file(local, "/job.3mf") is False
  99. assert client.last_failure is not None
  100. assert client.last_failure.kind is kind
  101. assert client.last_failure.code == reply[:3]
  102. def test_a_successful_upload_leaves_no_failure_behind(tmp_path):
  103. """Otherwise a later failure inherits an earlier one's reason."""
  104. local = tmp_path / "job.3mf"
  105. local.write_bytes(b"x" * 16)
  106. client = BambuFTPClient(IP, "12345678")
  107. client._ftp = MagicMock()
  108. client.last_failure = FtpFailure(FtpFailureKind.STORAGE, "553 stale", "553")
  109. assert client.upload_file(local, "/job.3mf") is True
  110. assert client.last_failure is None
  111. # ---------------------------------------------------------------------------
  112. # The reason reaches the caller
  113. # ---------------------------------------------------------------------------
  114. class TestTheReportReachesTheCaller:
  115. @pytest.fixture()
  116. def refusing_printer(self):
  117. transport = MagicMock()
  118. transport.connect.side_effect = ssl.SSLError("[SSL: WRONG_VERSION_NUMBER] wrong version number")
  119. with patch("backend.app.services.bambu_ftp.ImplicitFTP_TLS", return_value=transport):
  120. yield transport
  121. async def test_upload_file_async_fills_the_slot(self, refusing_printer, tmp_path):
  122. local = tmp_path / "job.3mf"
  123. local.write_bytes(b"x" * 16)
  124. report = FtpFailureReport()
  125. assert await upload_file_async(IP, "12345678", local, "/job.3mf", timeout=5.0, failure=report) is False
  126. assert report.failure is not None
  127. assert report.failure.kind is FtpFailureKind.HANDSHAKE
  128. async def test_it_survives_the_retry_loop(self, refusing_printer, tmp_path):
  129. """with_ftp_retry forwards the slot untouched, so the last try wins.
  130. The last attempt is the one that decided the outcome, so its reason is
  131. the one the operator should be given.
  132. """
  133. local = tmp_path / "job.3mf"
  134. local.write_bytes(b"x" * 16)
  135. report = FtpFailureReport()
  136. result = await with_ftp_retry(
  137. upload_file_async,
  138. IP,
  139. "12345678",
  140. local,
  141. "/job.3mf",
  142. timeout=5.0,
  143. respect_handshake_cooloff=False,
  144. failure=report,
  145. max_retries=2,
  146. retry_delay=0.01,
  147. )
  148. assert result is None
  149. assert report.failure is not None
  150. assert report.failure.kind is FtpFailureKind.HANDSHAKE
  151. async def test_two_callers_do_not_cross(self, refusing_printer, tmp_path):
  152. """The slot belongs to the caller, not to the printer.
  153. A per-IP dict on the client would be the obvious way to do this, and
  154. it is the way that breaks: a background timelapse fetch running beside
  155. a dispatch would overwrite the dispatch's reason with its own, and
  156. report the wrong cause with total confidence.
  157. """
  158. local = tmp_path / "job.3mf"
  159. local.write_bytes(b"x" * 16)
  160. mine, theirs = FtpFailureReport(), FtpFailureReport()
  161. await upload_file_async(IP, "12345678", local, "/a.3mf", timeout=5.0, failure=mine)
  162. assert theirs.failure is None
  163. assert mine.failure is not None
  164. async def test_a_caller_that_does_not_ask_is_unaffected(self, refusing_printer, tmp_path):
  165. """Every other caller passes nothing and must keep working."""
  166. local = tmp_path / "job.3mf"
  167. local.write_bytes(b"x" * 16)
  168. assert await upload_file_async(IP, "12345678", local, "/job.3mf", timeout=5.0) is False
  169. # ---------------------------------------------------------------------------
  170. # The wording
  171. # ---------------------------------------------------------------------------
  172. class TestTheWording:
  173. def test_only_a_storage_reply_sends_anyone_to_the_card(self):
  174. """Advice about the card, not mention of it.
  175. The handshake message names the card too, to rule it out -- that is
  176. the opposite of what this is guarding against, so the marker is the
  177. instruction ("formatted FAT32 or exFAT"), not the noun.
  178. """
  179. advising = [k for k in FtpFailureKind if "FAT32" in describe_upload_failure(FtpFailure(k, "detail"))]
  180. assert advising == [FtpFailureKind.STORAGE]
  181. def test_no_other_failure_asks_anyone_to_touch_the_card(self):
  182. """Anything that is not a storage reply must not send them there.
  183. Checked as "do something to the card" rather than "say the words",
  184. since ruling the card out is exactly what the handshake message does.
  185. """
  186. for kind in FtpFailureKind:
  187. if kind is FtpFailureKind.STORAGE:
  188. continue
  189. message = describe_upload_failure(FtpFailure(kind, "detail"))
  190. assert "Check that its SD card" not in message, kind
  191. assert "inserted" not in message, kind
  192. @pytest.mark.parametrize(
  193. ("kind", "must_say"),
  194. [
  195. (FtpFailureKind.COOLOFF, "clears on its own"),
  196. (FtpFailureKind.HANDSHAKE, "not with TLS"),
  197. (FtpFailureKind.AUTH, "access code"),
  198. (FtpFailureKind.TIMEOUT, "did not respond in time"),
  199. (FtpFailureKind.STORAGE, "FAT32"),
  200. (FtpFailureKind.NOT_FOUND, "Bambuddy-side"),
  201. (FtpFailureKind.NETWORK, "server log"),
  202. (FtpFailureKind.UNKNOWN, "server log"),
  203. ],
  204. )
  205. def test_every_kind_says_something_of_its_own(self, kind, must_say):
  206. """One line per branch, so none can quietly collapse into the generic.
  207. Without this, deleting the access-code branch or the timeout branch
  208. leaves every other assertion here passing -- they only check that the
  209. card is not named, which the generic message also satisfies.
  210. """
  211. assert must_say in describe_upload_failure(FtpFailure(kind, "detail", "553"))
  212. def test_the_access_code_hint_names_a_screen_that_exists(self):
  213. """The Access Code field is on the printer form on the Printers page.
  214. Naming a screen that is not there would be its own version of this
  215. bug: confident, specific, and a waste of the reader's time.
  216. """
  217. message = describe_upload_failure(FtpFailure(FtpFailureKind.AUTH, "530 Login incorrect.", "530"))
  218. assert "Printers page" in message
  219. def test_a_handshake_failure_says_the_card_is_not_involved(self):
  220. message = describe_upload_failure(FtpFailure(FtpFailureKind.HANDSHAKE, "WRONG_VERSION_NUMBER"))
  221. assert "not with TLS" in message
  222. assert "SD card is not involved" in message
  223. def test_it_does_not_prescribe_a_power_cycle(self):
  224. """#2780 removed that advice from the log because it does not work.
  225. The reporter of this issue restarted a printer on the strength of the
  226. user-facing string, so the string has to carry the same restraint.
  227. """
  228. for kind in FtpFailureKind:
  229. message = describe_upload_failure(FtpFailure(kind, "detail"))
  230. assert "restart the printer" not in message.lower(), kind
  231. assert "reboot" not in message.lower(), kind
  232. def test_an_unclassified_failure_points_at_the_log_rather_than_guessing(self):
  233. for failure in (None, FtpFailure(FtpFailureKind.UNKNOWN, "500 what")):
  234. message = describe_upload_failure(failure)
  235. assert "server log" in message
  236. assert "SD card" not in message
  237. def test_the_storage_message_carries_the_reply_code(self):
  238. """So a support bundle and the queue entry can be lined up."""
  239. message = describe_upload_failure(FtpFailure(FtpFailureKind.STORAGE, "553 Could not create file.", "553"))
  240. assert "553" in message
  241. assert "FAT32" in message
  242. # ---------------------------------------------------------------------------
  243. # End to end: what the queue entry says
  244. # ---------------------------------------------------------------------------
  245. @pytest.fixture
  246. async def dispatch_case(tmp_path):
  247. """Minimal one-printer, one-queued-job database for ``_start_print``."""
  248. from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
  249. import backend.app.models # noqa: F401 - populate Base.metadata
  250. from backend.app.core.database import Base
  251. from backend.app.models.archive import PrintArchive
  252. from backend.app.models.print_queue import PrintQueueItem
  253. from backend.app.models.printer import Printer
  254. engine = create_async_engine("sqlite+aiosqlite:///:memory:", echo=False)
  255. async with engine.begin() as conn:
  256. await conn.run_sync(Base.metadata.create_all)
  257. session_maker = async_sessionmaker(engine, expire_on_commit=False)
  258. base_dir = tmp_path / "case"
  259. archive_rel = Path("archives") / "job.3mf"
  260. archive_abs = base_dir / archive_rel
  261. archive_abs.parent.mkdir(parents=True, exist_ok=True)
  262. archive_abs.write_bytes(b"archive payload")
  263. async with session_maker() as db:
  264. printer = Printer(
  265. name="Bambulab P2S-4",
  266. serial_number="SERIAL",
  267. ip_address=IP,
  268. access_code="12345678",
  269. model="P2S",
  270. )
  271. db.add(printer)
  272. await db.flush()
  273. archive = PrintArchive(
  274. printer_id=printer.id,
  275. filename="job.3mf",
  276. file_path=str(archive_rel),
  277. file_size=archive_abs.stat().st_size,
  278. status="completed",
  279. )
  280. db.add(archive)
  281. await db.flush()
  282. item = PrintQueueItem(printer_id=printer.id, archive_id=archive.id, status="pending")
  283. db.add(item)
  284. await db.commit()
  285. item_id = item.id
  286. try:
  287. yield SimpleNamespace(session_maker=session_maker, base_dir=base_dir, item_id=item_id)
  288. finally:
  289. await engine.dispose()
  290. async def _dispatch_failing_with(dispatch_case, failure: FtpFailure | None):
  291. """Run one dispatch whose upload fails with *failure*.
  292. Returns the queue item's message and the reason the notification carried,
  293. which have to agree -- a push saying something different from the screen is
  294. its own small bug.
  295. """
  296. import backend.app.services.print_scheduler as scheduler_module
  297. from backend.app.models.print_queue import PrintQueueItem
  298. from backend.app.services.print_scheduler import PrintScheduler
  299. from backend.tests._fixtures.background_tasks import discarding_spawn_patch
  300. async def _upload(*_args, **kwargs):
  301. # Stands in for the real wrapper: fills the caller's slot, then fails.
  302. # Reading kwargs["failure"] rather than accepting it as a parameter is
  303. # deliberate -- if the dispatch ever stops passing the slot, every
  304. # message below falls back to the generic one and these tests fail.
  305. if failure is not None and kwargs.get("failure") is not None:
  306. kwargs["failure"].failure = failure
  307. return False
  308. notify = AsyncMock()
  309. scheduler = PrintScheduler()
  310. async with dispatch_case.session_maker() as db:
  311. item = await db.get(PrintQueueItem, dispatch_case.item_id)
  312. patches = [
  313. patch.object(scheduler_module.settings, "base_dir", dispatch_case.base_dir),
  314. patch("backend.app.services.print_scheduler.printer_manager.is_connected", MagicMock(return_value=True)),
  315. patch("backend.app.services.print_scheduler.printer_manager.get_status", MagicMock(return_value=None)),
  316. patch(
  317. "backend.app.services.print_scheduler.get_ftp_retry_settings",
  318. AsyncMock(return_value=(False, 0, 0, 1.0)),
  319. ),
  320. patch("backend.app.services.print_scheduler.delete_file_async", AsyncMock(return_value=True)),
  321. patch("backend.app.services.print_scheduler.upload_file_async", _upload),
  322. patch("backend.app.services.print_scheduler.notification_service.on_queue_job_failed", notify),
  323. discarding_spawn_patch(),
  324. patch.object(scheduler, "_propagate_owner_to_printer_manager", AsyncMock()),
  325. patch.object(scheduler, "_power_off_if_needed", AsyncMock()),
  326. patch.object(scheduler, "_preheat_and_soak", AsyncMock()),
  327. ]
  328. with ExitStack() as stack:
  329. for p in patches:
  330. stack.enter_context(p)
  331. await scheduler._start_print(db, item)
  332. refreshed = await db.get(PrintQueueItem, dispatch_case.item_id)
  333. assert refreshed.status == "failed"
  334. return refreshed.error_message or "", notify.await_args.kwargs["reason"]
  335. class TestWhatTheQueueEntrySays:
  336. async def test_a_handshake_failure_does_not_send_anyone_to_the_sd_card(self, dispatch_case):
  337. """The report's own case: a TLS failure, answered with card advice.
  338. The reporter acted on it and restarted the printer. Nothing in that
  339. path reaches the printer's filesystem, and the cool-off that made the
  340. next dispatch fail identically lives in Bambuddy's memory, where
  341. power-cycling a printer does not reach.
  342. """
  343. message, reason = await _dispatch_failing_with(
  344. dispatch_case, FtpFailure(FtpFailureKind.HANDSHAKE, "WRONG_VERSION_NUMBER")
  345. )
  346. assert "inserted" not in message, message
  347. assert "FAT32" not in message, message
  348. assert "not with TLS" in message, message
  349. assert reason == message
  350. async def test_a_553_still_gets_the_card_advice(self, dispatch_case):
  351. """The advice was written for this case and belongs to it.
  352. Removing it everywhere would trade one wrong message for a vaguer one;
  353. the point is to attach it where the printer actually said storage.
  354. """
  355. message, reason = await _dispatch_failing_with(
  356. dispatch_case, FtpFailure(FtpFailureKind.STORAGE, "553 Could not create file.", "553")
  357. )
  358. assert "FAT32" in message, message
  359. assert "553" in message, message
  360. assert reason == message
  361. async def test_an_unclassified_failure_points_at_the_log(self, dispatch_case):
  362. """No reason recorded means no reason invented."""
  363. message, reason = await _dispatch_failing_with(dispatch_case, None)
  364. assert "server log" in message, message
  365. assert "SD card" not in message, message
  366. assert reason == message