test_launcher_shutdown_config.py 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
  1. """Every launcher must be able to shut Bambuddy down gracefully.
  2. Two defects, found together, both invisible until you look for them:
  3. 1. **The Docker image never received SIGTERM at all.** ``CMD ["sh", "-c",
  4. "uvicorn ..."]`` leaves the shell as PID 1 with uvicorn as its child, and
  5. dash does not forward signals. Measured on the shipped image: ``docker stop``
  6. ran the full 10s grace period, exited 137 (SIGKILL), and the container log
  7. contained no "Shutting down" line. So *every* stop, restart and image update
  8. was a hard kill — no WAL checkpoint, no MQTT disconnect, no virtual-printer
  9. teardown. ``exec`` makes uvicorn PID 1 and the signal lands.
  10. 2. **Uvicorn waits forever for in-flight requests.**
  11. ``timeout_graceful_shutdown`` defaults to None, and an MJPEG camera stream is
  12. a response that never completes — ``httptools``'s connection ``shutdown()``
  13. only flips ``keep_alive = False`` on an in-flight cycle, it does not close the
  14. transport. One open camera tile pins the process indefinitely, and the app's
  15. own teardown never runs because uvicorn only fires the lifespan shutdown
  16. *after* connections drain. The flag caps the wait and cancels the tasks; the
  17. camera generators already unwind cleanly on CancelledError.
  18. Neither shows up in any functional test — the app is perfectly healthy right up
  19. until you ask it to stop. Hence this: pin the launchers themselves.
  20. """
  21. from __future__ import annotations
  22. import re
  23. from pathlib import Path
  24. import pytest
  25. REPO = Path(__file__).resolve().parents[3]
  26. FLAG = "--timeout-graceful-shutdown"
  27. # These pin repo-root launcher files (Dockerfile, compose, service units,
  28. # install scripts) that the Docker test image deliberately does not ship —
  29. # Dockerfile.test copies only backend/, pyproject.toml, gcode_viewer/ and
  30. # requirements. In a source checkout the files are always present and the
  31. # guard below is live (a moved/deleted launcher still fails loudly on every
  32. # `test_backend.sh` run); inside the stripped test image there is nothing to
  33. # check, so skip rather than fail. `frontend/package.json` is present in every
  34. # checkout but never in the test image, so it distinguishes the two.
  35. pytestmark = pytest.mark.skipif(
  36. not (REPO / "frontend" / "package.json").is_file(),
  37. reason="launcher config files aren't shipped in the Docker test image; verified in native runs",
  38. )
  39. def _read(rel: str) -> str:
  40. path = REPO / rel
  41. assert path.is_file(), f"launcher moved or was removed: {rel}"
  42. return path.read_text()
  43. def _uvicorn_lines(text: str) -> list[str]:
  44. """Lines that actually launch uvicorn, ignoring comments about it."""
  45. return [
  46. line for line in text.splitlines() if "uvicorn" in line and not line.lstrip().startswith(("#", "REM", "<!--"))
  47. ]
  48. class TestDockerImage:
  49. def test_cmd_execs_uvicorn_so_it_becomes_pid_1(self):
  50. """Without exec, `sh` is PID 1, dash eats the SIGTERM, and docker stop
  51. always ends in SIGKILL after the grace period.
  52. """
  53. cmd = next(line for line in _read("Dockerfile").splitlines() if line.startswith("CMD "))
  54. assert "exec uvicorn" in cmd, (
  55. "Dockerfile CMD must `exec` uvicorn. Without it the shell stays as PID 1, "
  56. "uvicorn never receives SIGTERM, and every docker stop is a SIGKILL:\n" + cmd
  57. )
  58. def test_cmd_bounds_the_graceful_shutdown(self):
  59. cmd = next(line for line in _read("Dockerfile").splitlines() if line.startswith("CMD "))
  60. assert FLAG in cmd, cmd
  61. def test_compose_allows_more_than_dockers_default_grace(self):
  62. compose = _read("docker-compose.yml")
  63. assert "stop_grace_period:" in compose, (
  64. "docker-compose.yml should raise stop_grace_period above Docker's 10s default, "
  65. "so a slow teardown on a Pi is not clipped by a SIGKILL."
  66. )
  67. class TestSystemdUnits:
  68. @pytest.mark.parametrize("unit", ["deploy/bambuddy.service"])
  69. def test_execstart_bounds_the_graceful_shutdown(self, unit):
  70. exec_start = next(line for line in _read(unit).splitlines() if line.startswith("ExecStart="))
  71. assert FLAG in exec_start, exec_start
  72. @pytest.mark.parametrize("unit", ["deploy/bambuddy.service"])
  73. def test_stop_timeout_leaves_room_for_the_teardown(self, unit):
  74. """systemd's timer is the backstop, not the mechanism — but it still has
  75. to outlast uvicorn's own 5s wait plus the app's ~1-2s of teardown.
  76. """
  77. match = re.search(r"^TimeoutStopSec=(\d+)", _read(unit), re.M)
  78. assert match, "unit should state a TimeoutStopSec rather than inherit the 90s default"
  79. assert int(match.group(1)) >= 15, (
  80. f"TimeoutStopSec={match.group(1)}s can clip the teardown: uvicorn waits up to 5s "
  81. "for in-flight requests, then the app checkpoints the WAL and stops the virtual "
  82. "printers."
  83. )
  84. class TestInstallScript:
  85. def test_generated_systemd_unit_bounds_the_shutdown(self):
  86. lines = _uvicorn_lines(_read("install/install.sh"))
  87. exec_start = [line for line in lines if line.startswith("ExecStart=")]
  88. assert exec_start, "install.sh no longer emits a systemd ExecStart line"
  89. for line in exec_start:
  90. assert FLAG in line, line
  91. def test_generated_launchd_plist_bounds_the_shutdown(self):
  92. """The macOS plist passes argv as a <string> array, so the flag and its
  93. value are two separate entries.
  94. """
  95. plist_region = _read("install/install.sh")
  96. assert f"<string>{FLAG}</string>" in plist_region, (
  97. "the launchd plist in install.sh does not pass --timeout-graceful-shutdown"
  98. )
  99. class TestWindowsService:
  100. def test_nssm_registration_bounds_the_shutdown(self):
  101. bat = _read("installers/windows/service/install-service.bat")
  102. install_line = next(line for line in _uvicorn_lines(bat) if "install Bambuddy" in line)
  103. assert FLAG in install_line, install_line
  104. def test_nssm_waits_long_enough_for_the_ctrl_c_stop(self):
  105. """NSSM's default AppStopMethodConsole is 1500ms. Uvicorn shuts down on
  106. the Ctrl-C but needs longer than that, so Windows was force-killing it
  107. mid-teardown.
  108. """
  109. bat = _read("installers/windows/service/install-service.bat")
  110. match = re.search(r"AppStopMethodConsole\s+(\d+)", bat)
  111. assert match, "install-service.bat must raise NSSM's 1500ms console-stop default"
  112. assert int(match.group(1)) >= 10000, match.group(1)