test_launcher_shutdown_config.py 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127
  1. """Every launcher must be able to shut Bambuddy down gracefully.
  2. Two defects, found together, both invisible until you look for them:
  3. 1. **The Docker image never received SIGTERM at all.** ``CMD ["sh", "-c",
  4. "uvicorn ..."]`` leaves the shell as PID 1 with uvicorn as its child, and
  5. dash does not forward signals. Measured on the shipped image: ``docker stop``
  6. ran the full 10s grace period, exited 137 (SIGKILL), and the container log
  7. contained no "Shutting down" line. So *every* stop, restart and image update
  8. was a hard kill — no WAL checkpoint, no MQTT disconnect, no virtual-printer
  9. teardown. ``exec`` makes uvicorn PID 1 and the signal lands.
  10. 2. **Uvicorn waits forever for in-flight requests.**
  11. ``timeout_graceful_shutdown`` defaults to None, and an MJPEG camera stream is
  12. a response that never completes — ``httptools``'s connection ``shutdown()``
  13. only flips ``keep_alive = False`` on an in-flight cycle, it does not close the
  14. transport. One open camera tile pins the process indefinitely, and the app's
  15. own teardown never runs because uvicorn only fires the lifespan shutdown
  16. *after* connections drain. The flag caps the wait and cancels the tasks; the
  17. camera generators already unwind cleanly on CancelledError.
  18. Neither shows up in any functional test — the app is perfectly healthy right up
  19. until you ask it to stop. Hence this: pin the launchers themselves.
  20. """
  21. from __future__ import annotations
  22. import re
  23. from pathlib import Path
  24. import pytest
  25. REPO = Path(__file__).resolve().parents[3]
  26. FLAG = "--timeout-graceful-shutdown"
  27. def _read(rel: str) -> str:
  28. path = REPO / rel
  29. assert path.is_file(), f"launcher moved or was removed: {rel}"
  30. return path.read_text()
  31. def _uvicorn_lines(text: str) -> list[str]:
  32. """Lines that actually launch uvicorn, ignoring comments about it."""
  33. return [
  34. line for line in text.splitlines() if "uvicorn" in line and not line.lstrip().startswith(("#", "REM", "<!--"))
  35. ]
  36. class TestDockerImage:
  37. def test_cmd_execs_uvicorn_so_it_becomes_pid_1(self):
  38. """Without exec, `sh` is PID 1, dash eats the SIGTERM, and docker stop
  39. always ends in SIGKILL after the grace period.
  40. """
  41. cmd = next(line for line in _read("Dockerfile").splitlines() if line.startswith("CMD "))
  42. assert "exec uvicorn" in cmd, (
  43. "Dockerfile CMD must `exec` uvicorn. Without it the shell stays as PID 1, "
  44. "uvicorn never receives SIGTERM, and every docker stop is a SIGKILL:\n" + cmd
  45. )
  46. def test_cmd_bounds_the_graceful_shutdown(self):
  47. cmd = next(line for line in _read("Dockerfile").splitlines() if line.startswith("CMD "))
  48. assert FLAG in cmd, cmd
  49. def test_compose_allows_more_than_dockers_default_grace(self):
  50. compose = _read("docker-compose.yml")
  51. assert "stop_grace_period:" in compose, (
  52. "docker-compose.yml should raise stop_grace_period above Docker's 10s default, "
  53. "so a slow teardown on a Pi is not clipped by a SIGKILL."
  54. )
  55. class TestSystemdUnits:
  56. @pytest.mark.parametrize("unit", ["deploy/bambuddy.service"])
  57. def test_execstart_bounds_the_graceful_shutdown(self, unit):
  58. exec_start = next(line for line in _read(unit).splitlines() if line.startswith("ExecStart="))
  59. assert FLAG in exec_start, exec_start
  60. @pytest.mark.parametrize("unit", ["deploy/bambuddy.service"])
  61. def test_stop_timeout_leaves_room_for_the_teardown(self, unit):
  62. """systemd's timer is the backstop, not the mechanism — but it still has
  63. to outlast uvicorn's own 5s wait plus the app's ~1-2s of teardown.
  64. """
  65. match = re.search(r"^TimeoutStopSec=(\d+)", _read(unit), re.M)
  66. assert match, "unit should state a TimeoutStopSec rather than inherit the 90s default"
  67. assert int(match.group(1)) >= 15, (
  68. f"TimeoutStopSec={match.group(1)}s can clip the teardown: uvicorn waits up to 5s "
  69. "for in-flight requests, then the app checkpoints the WAL and stops the virtual "
  70. "printers."
  71. )
  72. class TestInstallScript:
  73. def test_generated_systemd_unit_bounds_the_shutdown(self):
  74. lines = _uvicorn_lines(_read("install/install.sh"))
  75. exec_start = [line for line in lines if line.startswith("ExecStart=")]
  76. assert exec_start, "install.sh no longer emits a systemd ExecStart line"
  77. for line in exec_start:
  78. assert FLAG in line, line
  79. def test_generated_launchd_plist_bounds_the_shutdown(self):
  80. """The macOS plist passes argv as a <string> array, so the flag and its
  81. value are two separate entries.
  82. """
  83. plist_region = _read("install/install.sh")
  84. assert f"<string>{FLAG}</string>" in plist_region, (
  85. "the launchd plist in install.sh does not pass --timeout-graceful-shutdown"
  86. )
  87. class TestWindowsService:
  88. def test_nssm_registration_bounds_the_shutdown(self):
  89. bat = _read("installers/windows/service/install-service.bat")
  90. install_line = next(line for line in _uvicorn_lines(bat) if "install Bambuddy" in line)
  91. assert FLAG in install_line, install_line
  92. def test_nssm_waits_long_enough_for_the_ctrl_c_stop(self):
  93. """NSSM's default AppStopMethodConsole is 1500ms. Uvicorn shuts down on
  94. the Ctrl-C but needs longer than that, so Windows was force-killing it
  95. mid-teardown.
  96. """
  97. bat = _read("installers/windows/service/install-service.bat")
  98. match = re.search(r"AppStopMethodConsole\s+(\d+)", bat)
  99. assert match, "install-service.bat must raise NSSM's 1500ms console-stop default"
  100. assert int(match.group(1)) >= 10000, match.group(1)