test_run_with_retry.py 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198
  1. """Tests for database.run_with_retry — SQLite lock retry logic (#897)."""
  2. from __future__ import annotations
  3. from unittest.mock import AsyncMock, patch
  4. import pytest
  5. from sqlalchemy.exc import OperationalError
  6. @pytest.fixture(autouse=True)
  7. def _force_sqlite():
  8. """Make is_sqlite() return True for all tests in this module."""
  9. with patch("backend.app.core.database.is_sqlite", return_value=True):
  10. yield
  11. def _make_locked_error() -> OperationalError:
  12. """Create a realistic 'database is locked' OperationalError."""
  13. return OperationalError(
  14. statement="UPDATE print_queue SET status=?",
  15. params=("completed",),
  16. orig=Exception("database is locked"),
  17. )
  18. def _make_other_error() -> OperationalError:
  19. """Create a non-lock OperationalError."""
  20. return OperationalError(
  21. statement="SELECT 1",
  22. params=(),
  23. orig=Exception("no such table: foo"),
  24. )
  25. @pytest.mark.asyncio
  26. async def test_succeeds_on_first_attempt():
  27. """Happy path — fn succeeds immediately."""
  28. from backend.app.core.database import run_with_retry
  29. mock_fn = AsyncMock(return_value="ok")
  30. with patch("backend.app.core.database.async_session") as mock_session_factory:
  31. mock_db = AsyncMock()
  32. mock_session_factory.return_value.__aenter__ = AsyncMock(return_value=mock_db)
  33. mock_session_factory.return_value.__aexit__ = AsyncMock(return_value=False)
  34. result = await run_with_retry(mock_fn, label="test")
  35. assert result == "ok"
  36. mock_fn.assert_awaited_once_with(mock_db)
  37. @pytest.mark.asyncio
  38. async def test_retries_on_sqlite_locked():
  39. """fn fails with 'database is locked' then succeeds on retry."""
  40. from backend.app.core.database import run_with_retry
  41. call_count = 0
  42. async def flaky_fn(db):
  43. nonlocal call_count
  44. call_count += 1
  45. if call_count == 1:
  46. raise _make_locked_error()
  47. return "recovered"
  48. with (
  49. patch("backend.app.core.database.async_session") as mock_session_factory,
  50. patch("backend.app.core.database.asyncio.sleep", new_callable=AsyncMock) as mock_sleep,
  51. ):
  52. mock_db = AsyncMock()
  53. mock_session_factory.return_value.__aenter__ = AsyncMock(return_value=mock_db)
  54. mock_session_factory.return_value.__aexit__ = AsyncMock(return_value=False)
  55. result = await run_with_retry(flaky_fn, label="test")
  56. assert result == "recovered"
  57. assert call_count == 2
  58. mock_sleep.assert_awaited_once_with(0.5) # first retry: 0.5s delay
  59. @pytest.mark.asyncio
  60. async def test_raises_after_max_attempts():
  61. """fn fails with 'database is locked' on all attempts — raises."""
  62. from backend.app.core.database import run_with_retry
  63. async def always_locked(db):
  64. raise _make_locked_error()
  65. with (
  66. patch("backend.app.core.database.async_session") as mock_session_factory,
  67. patch("backend.app.core.database.asyncio.sleep", new_callable=AsyncMock),
  68. ):
  69. mock_db = AsyncMock()
  70. mock_session_factory.return_value.__aenter__ = AsyncMock(return_value=mock_db)
  71. mock_session_factory.return_value.__aexit__ = AsyncMock(return_value=False)
  72. with pytest.raises(OperationalError, match="database is locked"):
  73. await run_with_retry(always_locked, max_attempts=3, label="test")
  74. @pytest.mark.asyncio
  75. async def test_non_lock_error_not_retried():
  76. """Non-lock OperationalErrors are raised immediately, not retried."""
  77. from backend.app.core.database import run_with_retry
  78. call_count = 0
  79. async def bad_fn(db):
  80. nonlocal call_count
  81. call_count += 1
  82. raise _make_other_error()
  83. with (
  84. patch("backend.app.core.database.async_session") as mock_session_factory,
  85. patch("backend.app.core.database.asyncio.sleep", new_callable=AsyncMock) as mock_sleep,
  86. ):
  87. mock_db = AsyncMock()
  88. mock_session_factory.return_value.__aenter__ = AsyncMock(return_value=mock_db)
  89. mock_session_factory.return_value.__aexit__ = AsyncMock(return_value=False)
  90. with pytest.raises(OperationalError, match="no such table"):
  91. await run_with_retry(bad_fn, label="test")
  92. assert call_count == 1
  93. mock_sleep.assert_not_awaited()
  94. @pytest.mark.asyncio
  95. async def test_backoff_increases():
  96. """Retry delays increase: 0.5s, 1.0s, 1.5s."""
  97. from backend.app.core.database import run_with_retry
  98. call_count = 0
  99. async def recovers_on_third(db):
  100. nonlocal call_count
  101. call_count += 1
  102. if call_count < 3:
  103. raise _make_locked_error()
  104. return "ok"
  105. with (
  106. patch("backend.app.core.database.async_session") as mock_session_factory,
  107. patch("backend.app.core.database.asyncio.sleep", new_callable=AsyncMock) as mock_sleep,
  108. ):
  109. mock_db = AsyncMock()
  110. mock_session_factory.return_value.__aenter__ = AsyncMock(return_value=mock_db)
  111. mock_session_factory.return_value.__aexit__ = AsyncMock(return_value=False)
  112. result = await run_with_retry(recovers_on_third, max_attempts=3, label="test")
  113. assert result == "ok"
  114. assert call_count == 3
  115. assert mock_sleep.await_args_list[0].args == (0.5,)
  116. assert mock_sleep.await_args_list[1].args == (1.0,)
  117. @pytest.mark.asyncio
  118. async def test_postgres_no_retry():
  119. """On PostgreSQL, fn is called once with no retry logic."""
  120. from backend.app.core.database import run_with_retry
  121. mock_fn = AsyncMock(return_value="pg_ok")
  122. with (
  123. patch("backend.app.core.database.is_sqlite", return_value=False),
  124. patch("backend.app.core.database.async_session") as mock_session_factory,
  125. ):
  126. mock_db = AsyncMock()
  127. mock_session_factory.return_value.__aenter__ = AsyncMock(return_value=mock_db)
  128. mock_session_factory.return_value.__aexit__ = AsyncMock(return_value=False)
  129. result = await run_with_retry(mock_fn, label="test")
  130. assert result == "pg_ok"
  131. mock_fn.assert_awaited_once_with(mock_db)
  132. @pytest.mark.asyncio
  133. async def test_postgres_error_not_retried():
  134. """On PostgreSQL, OperationalErrors are raised immediately."""
  135. from backend.app.core.database import run_with_retry
  136. async def bad_fn(db):
  137. raise _make_locked_error()
  138. with (
  139. patch("backend.app.core.database.is_sqlite", return_value=False),
  140. patch("backend.app.core.database.async_session") as mock_session_factory,
  141. ):
  142. mock_db = AsyncMock()
  143. mock_session_factory.return_value.__aenter__ = AsyncMock(return_value=mock_db)
  144. mock_session_factory.return_value.__aexit__ = AsyncMock(return_value=False)
  145. with pytest.raises(OperationalError):
  146. await run_with_retry(bad_fn, label="test")