test_fd_limit_2883.py 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165
  1. """The open-file limit is raised at startup, and bundles say what holds it (#2883).
  2. #2883 ran out of descriptors at Docker's default soft limit of 1024. From then
  3. on every new SQLite connection failed with "disk I/O error", and after 44 hours
  4. the database was corrupt. The soft limit is the process's own to raise up to the
  5. hard limit, so startup does that on every install. The support bundle now counts
  6. descriptors by kind, so the next report shows what held them.
  7. """
  8. import resource
  9. import sqlite3
  10. from unittest.mock import MagicMock, patch
  11. import pytest
  12. from backend.app.core import fd_limit
  13. INF = resource.RLIM_INFINITY
  14. @pytest.fixture(autouse=True)
  15. def _reset_status():
  16. fd_limit.startup_status = None
  17. yield
  18. fd_limit.startup_status = None
  19. def _run(soft, hard, setrlimit=None):
  20. calls = []
  21. def _set(which, limits):
  22. calls.append(limits)
  23. if setrlimit is not None:
  24. setrlimit(limits)
  25. with (
  26. patch.object(resource, "getrlimit", return_value=(soft, hard)),
  27. patch.object(resource, "setrlimit", side_effect=_set),
  28. ):
  29. status = fd_limit.raise_open_file_limit()
  30. return status, calls
  31. class TestRaiseOpenFileLimit:
  32. def test_the_soft_limit_is_raised_to_the_hard_one(self):
  33. status, calls = _run(1024, 524288)
  34. assert calls == [(524288, 524288)]
  35. assert status == {"soft_at_start": 1024, "hard": 524288, "soft": 524288, "raised": True}
  36. assert fd_limit.startup_status == status
  37. def test_a_limit_already_at_the_hard_one_is_left_alone(self):
  38. status, calls = _run(524288, 524288)
  39. assert calls == []
  40. assert status["raised"] is False
  41. def test_an_unlimited_soft_limit_is_left_alone(self):
  42. status, calls = _run(INF, INF)
  43. assert calls == []
  44. assert status["soft"] == "unlimited"
  45. def test_an_unlimited_hard_limit_gets_a_finite_soft_one(self):
  46. """macOS reports RLIM_INFINITY but refuses a soft limit above
  47. kern.maxfilesperproc, so finite values are tried in turn."""
  48. def refuse_the_first(limits):
  49. if limits[0] == 65536:
  50. raise ValueError("not allowed")
  51. status, calls = _run(256, INF, setrlimit=refuse_the_first)
  52. assert calls == [(65536, INF), (10240, INF)]
  53. assert status["soft"] == 10240
  54. assert status["hard"] == "unlimited"
  55. assert status["raised"] is True
  56. def test_a_refused_raise_is_logged_and_startup_carries_on(self, caplog):
  57. def refuse(limits):
  58. raise OSError("operation not permitted")
  59. status, _ = _run(1024, 4096, setrlimit=refuse)
  60. assert status["raised"] is False
  61. assert status["soft"] == 1024
  62. assert "operation not permitted" in status["error"]
  63. assert any("Could not raise the open-file limit" in r.getMessage() for r in caplog.records)
  64. def test_an_unreadable_limit_is_not_fatal(self):
  65. with patch.object(resource, "getrlimit", side_effect=OSError("nope")):
  66. assert fd_limit.raise_open_file_limit() is None
  67. def test_no_rlimit_module_is_not_fatal(self):
  68. """Windows has no ``resource`` module."""
  69. import builtins
  70. real_import = builtins.__import__
  71. def no_resource(name, *args, **kwargs):
  72. if name == "resource":
  73. raise ImportError(name)
  74. return real_import(name, *args, **kwargs)
  75. with patch("builtins.__import__", side_effect=no_resource):
  76. assert fd_limit.raise_open_file_limit() is None
  77. class TestDescriptorsInTheSupportBundle:
  78. @pytest.mark.parametrize(
  79. ("target", "kind"),
  80. [
  81. ("socket:[12345]", "socket"),
  82. ("pipe:[678]", "pipe"),
  83. ("anon_inode:[eventpoll]", "anon_inode"),
  84. ("/app/data/bambuddy.db", "database"),
  85. ("/app/data/bambuddy.db-wal", "database_wal"),
  86. ("/app/data/bambuddy.db-shm", "database_shm"),
  87. ("/dev/null", "device"),
  88. ("/app/logs/bambuddy.log", "file"),
  89. ],
  90. )
  91. def test_kinds(self, target, kind):
  92. from backend.app.api.routes.support import _fd_kind
  93. assert _fd_kind(target) == kind
  94. def test_counts_by_kind_against_the_limit_without_paths(self, tmp_path):
  95. import psutil
  96. from backend.app.api.routes.support import _collect_fd_info
  97. db = sqlite3.connect(tmp_path / "secret-name.db")
  98. db.execute("PRAGMA journal_mode = WAL")
  99. db.execute("CREATE TABLE t (x)")
  100. try:
  101. info = _collect_fd_info(psutil.Process())
  102. finally:
  103. db.close()
  104. assert info["num_fds"] > 0
  105. assert info["fds_by_type"]["database"] >= 1
  106. assert info["fds_by_type"]["database_wal"] >= 1
  107. assert set(info["fd_limit"]) == {"soft", "hard"}
  108. assert "secret-name" not in repr(info)
  109. def test_the_startup_result_is_carried(self):
  110. from backend.app.api.routes.support import _collect_fd_info
  111. fd_limit.startup_status = {"soft_at_start": 1024, "hard": 524288, "soft": 524288, "raised": True}
  112. info = _collect_fd_info(MagicMock())
  113. assert info["fd_limit_at_startup"]["soft_at_start"] == 1024
  114. def test_a_failing_probe_still_returns(self):
  115. from backend.app.api.routes.support import _collect_fd_info
  116. proc = MagicMock()
  117. proc.num_fds.side_effect = RuntimeError("restricted")
  118. with patch("os.listdir", side_effect=PermissionError("no /proc")):
  119. info = _collect_fd_info(proc)
  120. assert "num_fds" not in info
  121. assert "fds_by_type" not in info