test_cleartext_probe_2780.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383
  1. """Ask the printer what it actually said (#2780).
  2. ``[SSL: WRONG_VERSION_NUMBER]`` on port 990 means the printer's first bytes
  3. were not a TLS record. That is measured rather than assumed, and the two tests
  4. at the top of this file are the measurement: a cleartext banner reproduces the
  5. exact error the field reports, while a genuine TLS version mismatch produces a
  6. different one. Both matter, because the profile registry used to explain this
  7. failure as a TLS 1.3 problem and prescribe a version cap for it -- which cannot
  8. work, since the error was never about the negotiated version.
  9. What the error does not say is *which* cleartext message, and that is the part
  10. that would identify the fault. OpenSSL has consumed those bytes by the time the
  11. exception surfaces, so the client now opens one plain connection and reads them.
  12. The reporter with the affected farm offered a packet capture; this gets the same
  13. answer from every affected install instead of one.
  14. """
  15. import logging
  16. import socket
  17. import ssl
  18. import threading
  19. import time
  20. from unittest.mock import MagicMock, patch
  21. import pytest
  22. from backend.app.services import bambu_ftp
  23. from backend.app.services.bambu_ftp import BambuFTPClient
  24. pytestmark = pytest.mark.unit
  25. LOGGER = "backend.app.services.bambu_ftp"
  26. REFUSAL = b"421 Too many connections. Try again later.\r\n"
  27. @pytest.fixture(autouse=True)
  28. def _clean_state():
  29. BambuFTPClient._handshake_blocked_until.clear()
  30. BambuFTPClient._handshake_skip_logged.clear()
  31. BambuFTPClient._mode_cache.clear()
  32. yield
  33. BambuFTPClient._handshake_blocked_until.clear()
  34. BambuFTPClient._handshake_skip_logged.clear()
  35. BambuFTPClient._mode_cache.clear()
  36. class _Listener:
  37. """A socket on an ephemeral port that answers however the test says.
  38. ``mode="cleartext"`` sends an FTP refusal in the clear, the way a vsFTPd
  39. that is turning connections away does. ``mode="silent"`` accepts and says
  40. nothing, which is what a healthy implicit-FTPS service does while it waits
  41. for a ClientHello.
  42. """
  43. def __init__(self, mode: str):
  44. self.mode = mode
  45. self.accepts = 0
  46. self._sock = socket.socket()
  47. self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
  48. self._sock.bind(("127.0.0.1", 0))
  49. self._sock.listen(8)
  50. self.port = self._sock.getsockname()[1]
  51. self._stop = threading.Event()
  52. self._conns: list[socket.socket] = []
  53. self._thread = threading.Thread(target=self._serve, daemon=True)
  54. self._thread.start()
  55. def _serve(self):
  56. # A blocking accept() is not reliably woken by closing the socket from
  57. # another thread, which left every teardown here waiting out its join.
  58. self._sock.settimeout(0.1)
  59. while not self._stop.is_set():
  60. try:
  61. conn, _ = self._sock.accept()
  62. except TimeoutError:
  63. continue
  64. except OSError:
  65. return
  66. self.accepts += 1
  67. if self.mode == "cleartext":
  68. try:
  69. conn.sendall(REFUSAL)
  70. except OSError:
  71. pass
  72. conn.close()
  73. else:
  74. # Hold it open and stay quiet, so the probe has to time out.
  75. self._conns.append(conn)
  76. def stop(self):
  77. self._stop.set()
  78. self._sock.close()
  79. for c in self._conns:
  80. try:
  81. c.close()
  82. except OSError:
  83. pass
  84. self._thread.join(timeout=2)
  85. @pytest.fixture()
  86. def cleartext_printer():
  87. server = _Listener("cleartext")
  88. yield server
  89. server.stop()
  90. @pytest.fixture()
  91. def silent_printer():
  92. server = _Listener("silent")
  93. yield server
  94. server.stop()
  95. @pytest.fixture(autouse=True)
  96. def _fast_probe(monkeypatch):
  97. """A real timeout would make the silent case a two-second test."""
  98. monkeypatch.setattr(bambu_ftp, "_CLEARTEXT_PROBE_TIMEOUT", 0.25)
  99. # ---------------------------------------------------------------------------
  100. # The measurement the rest of this rests on
  101. # ---------------------------------------------------------------------------
  102. def test_a_cleartext_banner_is_what_produces_wrong_version_number(cleartext_printer):
  103. """The exact error the affected farm logs, from a non-TLS answer."""
  104. ctx = ssl.create_default_context()
  105. ctx.check_hostname = False
  106. ctx.verify_mode = ssl.CERT_NONE
  107. raw = socket.create_connection(("127.0.0.1", cleartext_printer.port), 5)
  108. with pytest.raises(ssl.SSLError) as caught:
  109. ctx.wrap_socket(raw, server_hostname="printer").do_handshake()
  110. assert caught.value.reason == "WRONG_VERSION_NUMBER"
  111. def test_a_version_mismatch_produces_a_different_error(tmp_path):
  112. """So "cap the TLS version" cannot be the fix for WRONG_VERSION_NUMBER.
  113. Two of the cap_tls_v1_2 profile entries were written on the belief that it
  114. was. A real mismatch reports itself as a protocol-version alert, and a
  115. server that only speaks 1.2 negotiates fine against our own context without
  116. any cap -- so neither half of that reasoning holds.
  117. """
  118. import subprocess # nosec B404 -- generating a throwaway cert for a local server
  119. subprocess.run( # nosec B603 B607
  120. [
  121. "openssl",
  122. "req",
  123. "-x509",
  124. "-newkey",
  125. "rsa:2048",
  126. "-keyout",
  127. str(tmp_path / "k.pem"),
  128. "-out",
  129. str(tmp_path / "c.pem"),
  130. "-days",
  131. "1",
  132. "-nodes",
  133. "-subj",
  134. "/CN=printer",
  135. ],
  136. check=True,
  137. capture_output=True,
  138. )
  139. server_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
  140. server_ctx.load_cert_chain(str(tmp_path / "c.pem"), str(tmp_path / "k.pem"))
  141. server_ctx.maximum_version = ssl.TLSVersion.TLSv1_2
  142. listener = socket.socket()
  143. listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
  144. listener.bind(("127.0.0.1", 0))
  145. listener.listen(4)
  146. port = listener.getsockname()[1]
  147. def serve():
  148. for _ in range(2):
  149. try:
  150. conn, _addr = listener.accept()
  151. except OSError:
  152. return
  153. try:
  154. server_ctx.wrap_socket(conn, server_side=True).close()
  155. except (ssl.SSLError, OSError):
  156. try:
  157. conn.close()
  158. except OSError:
  159. pass
  160. thread = threading.Thread(target=serve, daemon=True)
  161. thread.start()
  162. try:
  163. def attempt(*, force_tls13: bool):
  164. ctx = ssl.create_default_context()
  165. ctx.check_hostname = False
  166. ctx.verify_mode = ssl.CERT_NONE
  167. ctx.minimum_version = ssl.TLSVersion.TLSv1_3 if force_tls13 else ssl.TLSVersion.TLSv1_2
  168. if force_tls13:
  169. ctx.maximum_version = ssl.TLSVersion.TLSv1_3
  170. raw = socket.create_connection(("127.0.0.1", port), 5)
  171. try:
  172. ctx.wrap_socket(raw, server_hostname="printer").do_handshake()
  173. return None
  174. finally:
  175. try:
  176. raw.close()
  177. except OSError:
  178. pass
  179. with pytest.raises(ssl.SSLError) as caught:
  180. attempt(force_tls13=True)
  181. assert caught.value.reason != "WRONG_VERSION_NUMBER"
  182. assert "PROTOCOL_VERSION" in caught.value.reason
  183. # And the half that makes the caps no-ops: a 1.2-only peer needs no help.
  184. assert attempt(force_tls13=False) is None
  185. finally:
  186. listener.close()
  187. thread.join(timeout=2)
  188. # ---------------------------------------------------------------------------
  189. # The probe
  190. # ---------------------------------------------------------------------------
  191. class TestTheProbe:
  192. def _client(self, server):
  193. client = BambuFTPClient("127.0.0.1", "12345678", timeout=5.0, printer_model="P2S")
  194. client.FTP_PORT = server.port
  195. return client
  196. def test_it_puts_the_printers_own_words_in_the_log(self, cleartext_printer, caplog):
  197. with caplog.at_level(logging.WARNING, logger=LOGGER):
  198. assert self._client(cleartext_printer).connect() is False
  199. messages = [r.getMessage() for r in caplog.records]
  200. assert any("421 Too many connections" in m for m in messages), messages
  201. # And it has to be findable by someone filing a report.
  202. assert any("include this line if you report it" in m for m in messages), messages
  203. def test_the_reason_carries_it_too(self, cleartext_printer):
  204. """So the failure reaches the user's message, not only the log."""
  205. client = self._client(cleartext_printer)
  206. client.connect()
  207. assert client.last_failure is not None
  208. assert "421 Too many connections" in client.last_failure.detail
  209. def test_silence_is_reported_as_the_fault_having_passed(self, caplog):
  210. """The printer sent non-TLS bytes, then had recovered a moment later.
  211. Driven through a stubbed probe rather than a silent server, because a
  212. server that accepts and stays quiet never reaches this branch at all --
  213. it produces a handshake *timeout*, not WRONG_VERSION_NUMBER, and the
  214. TimeoutError branch handles that one. Reading nothing here means the
  215. refusal passed between the handshake and the question, which is worth
  216. saying rather than logging nothing at all.
  217. """
  218. transport = MagicMock()
  219. error = ssl.SSLError(1, "[SSL: WRONG_VERSION_NUMBER] wrong version number")
  220. error.reason = "WRONG_VERSION_NUMBER"
  221. transport.connect.side_effect = error
  222. with (
  223. patch("backend.app.services.bambu_ftp.ImplicitFTP_TLS", return_value=transport),
  224. patch("backend.app.services.bambu_ftp._read_cleartext_reply", return_value=None),
  225. caplog.at_level(logging.WARNING, logger=LOGGER),
  226. ):
  227. assert BambuFTPClient("192.0.2.10", "12345678").connect() is False
  228. assert any("nothing readable in cleartext" in r.getMessage() for r in caplog.records)
  229. # And a probe that finds nothing must not cost the cool-off: the
  230. # handshake still failed, whatever the printer said a moment later.
  231. assert BambuFTPClient.handshake_blocked("192.0.2.10") is True
  232. def test_an_accept_and_stay_quiet_printer_is_a_timeout_not_this(self, silent_printer):
  233. """The other half of #2780's theory, and it lands somewhere else.
  234. A vsFTPd answering its global connection limit by accepting and never
  235. speaking produces a handshake timeout. Probing that would read nothing
  236. by definition, so this branch is deliberately not reached.
  237. """
  238. client = self._client(silent_printer)
  239. client.timeout = 0.5
  240. with patch("backend.app.services.bambu_ftp._read_cleartext_reply") as probe:
  241. assert client.connect() is False
  242. probe.assert_not_called()
  243. assert client.last_failure is not None
  244. assert client.last_failure.kind.value == "timeout"
  245. def test_it_asks_once_per_cooloff_not_once_per_attempt(self, cleartext_printer):
  246. """A dispatch ignores the cool-off, so it reaches this branch four times.
  247. Probing each time would add a connection per attempt to a printer whose
  248. suspected fault is having too many -- the opposite of what #2780's
  249. socket-leak fix was for.
  250. """
  251. for _ in range(4):
  252. client = self._client(cleartext_printer)
  253. client.respect_handshake_cooloff = False
  254. client.connect()
  255. # Four handshakes, and exactly one probe on top of them.
  256. assert cleartext_printer.accepts == 5
  257. def test_a_fresh_cooloff_window_asks_again(self, cleartext_printer):
  258. """A printer that recovers and fails later is a new event to diagnose."""
  259. self._client(cleartext_printer).connect()
  260. before = cleartext_printer.accepts
  261. BambuFTPClient._handshake_blocked_until.clear()
  262. self._client(cleartext_printer).connect()
  263. assert cleartext_printer.accepts == before + 2 # handshake + probe
  264. def test_a_real_version_mismatch_is_not_probed(self):
  265. """Nothing to read: that peer spoke TLS, it just would not agree on one.
  266. Without this check the probe would connect and sit out its whole
  267. timeout on every such failure.
  268. """
  269. transport = MagicMock()
  270. error = ssl.SSLError(1, "[SSL: TLSV1_ALERT_PROTOCOL_VERSION] tlsv1 alert protocol version")
  271. error.reason = "TLSV1_ALERT_PROTOCOL_VERSION"
  272. transport.connect.side_effect = error
  273. with (
  274. patch("backend.app.services.bambu_ftp.ImplicitFTP_TLS", return_value=transport),
  275. patch("backend.app.services.bambu_ftp._read_cleartext_reply") as probe,
  276. ):
  277. assert BambuFTPClient("192.0.2.10", "12345678").connect() is False
  278. probe.assert_not_called()
  279. def test_a_refused_probe_reads_as_nothing_rather_than_raising(self):
  280. """Nothing is listening, so it must come back None, not blow up.
  281. Uses a port that was just released rather than patching
  282. ``socket.create_connection``, which is process-wide and would sit under
  283. anything else running in this worker.
  284. """
  285. released = socket.socket()
  286. released.bind(("127.0.0.1", 0))
  287. port = released.getsockname()[1]
  288. released.close()
  289. assert bambu_ftp._read_cleartext_reply("127.0.0.1", port) is None
  290. def test_the_dead_socket_is_closed_before_the_printer_is_asked_again(self):
  291. """Ordering, and it is the whole reason this is safe to do at all.
  292. The probe opens a second connection to a printer whose suspected fault
  293. is having no connection slots left. Holding the failed handshake open
  294. across that would be the leak #2780's cleanup was added to stop, with
  295. an extra connection layered on top.
  296. """
  297. transport = MagicMock()
  298. error = ssl.SSLError(1, "[SSL: WRONG_VERSION_NUMBER] wrong version number")
  299. error.reason = "WRONG_VERSION_NUMBER"
  300. transport.connect.side_effect = error
  301. client = BambuFTPClient("192.0.2.10", "12345678")
  302. observed = {}
  303. def _probe(*_args):
  304. observed["still_open"] = client._ftp is not None
  305. observed["closed"] = transport.close.called
  306. return "421 Too many connections."
  307. with (
  308. patch("backend.app.services.bambu_ftp.ImplicitFTP_TLS", return_value=transport),
  309. patch("backend.app.services.bambu_ftp._read_cleartext_reply", _probe),
  310. ):
  311. assert client.connect() is False
  312. assert observed == {"still_open": False, "closed": True}
  313. def test_the_probe_does_not_outlive_its_timeout(self, silent_printer):
  314. started = time.monotonic()
  315. assert bambu_ftp._read_cleartext_reply("127.0.0.1", silent_printer.port) is None
  316. assert time.monotonic() - started < 2.0