test_notification_write_lock.py 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102
  1. """A notification must never be sent while holding the SQLite write lock (#2770).
  2. The reporter's bundle has two "database is locked" failures, and both sit inside
  3. a Discord connect timeout::
  4. 17:36:55 Sending humidity alarm ... 15.0% > 14.0%
  5. 17:37:12 WARNING Printer sensor history recording failed: database is locked
  6. 17:37:25 ERROR httpx.ConnectTimeout <- exactly 30.000s later
  7. The mechanism is not contention from writing too much. The AMS sensor loop does
  8. ``db.add(history)`` and only commits *after* the alarms have been dispatched, so
  9. the first SELECT inside the notification path used to autoflush that pending
  10. INSERT — opening a write transaction — and the provider was then contacted over
  11. the network with that transaction still open. SQLite allows one writer, and the
  12. 30 s connect timeout comfortably outlived the 15 s ``busy_timeout``, so unrelated
  13. background tasks failed.
  14. These tests pin the two reads that run before the network call. They assert the
  15. caller's pending row is still unflushed afterwards, which is the same thing as
  16. "no write transaction was opened on its behalf" and holds on any dialect.
  17. """
  18. import pytest
  19. from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
  20. import backend.app.models # noqa: F401 - populate Base.metadata
  21. from backend.app.core.database import Base
  22. from backend.app.models.notification import NotificationProvider
  23. from backend.app.models.notification_template import NotificationTemplate
  24. from backend.app.services.notification_service import NotificationService
  25. @pytest.fixture
  26. async def session(tmp_path):
  27. engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'notify-lock.db'}")
  28. async with engine.begin() as conn:
  29. await conn.run_sync(Base.metadata.create_all)
  30. maker = async_sessionmaker(engine, expire_on_commit=False)
  31. async with maker() as s:
  32. yield s
  33. await engine.dispose()
  34. def _pending_row() -> NotificationProvider:
  35. """A row the caller has added but not committed — the sensor loop's position."""
  36. return NotificationProvider(name="pending", provider_type="discord", config="{}", enabled=False)
  37. @pytest.mark.asyncio
  38. async def test_provider_lookup_does_not_flush_the_callers_pending_writes(session):
  39. service = NotificationService()
  40. session.add(NotificationProvider(name="Discord", provider_type="discord", config="{}", enabled=True))
  41. await session.commit()
  42. pending = _pending_row()
  43. session.add(pending)
  44. providers = await service._get_providers_for_event(session, "on_ams_drying_suspended")
  45. assert [p.name for p in providers] == ["Discord"]
  46. assert pending in session.new, "the caller's pending INSERT was flushed, taking the SQLite write lock"
  47. @pytest.mark.asyncio
  48. async def test_template_lookup_does_not_flush_the_callers_pending_writes(session):
  49. service = NotificationService()
  50. session.add(
  51. NotificationTemplate(
  52. event_type="ams_drying_suspended",
  53. name="Auto-Drying Suspended",
  54. title_template="t",
  55. body_template="b",
  56. is_default=True,
  57. )
  58. )
  59. await session.commit()
  60. pending = _pending_row()
  61. session.add(pending)
  62. template = await service._get_template(session, "ams_drying_suspended")
  63. assert template is not None
  64. assert pending in session.new, "the caller's pending INSERT was flushed, taking the SQLite write lock"
  65. @pytest.mark.asyncio
  66. async def test_connect_timeout_stays_under_the_sqlite_busy_timeout():
  67. """15 s is the ``busy_timeout`` set in database.py. A connect timeout at or
  68. above it guarantees the "database is locked" failure whenever a site's
  69. internet is down, whatever else is fixed."""
  70. service = NotificationService()
  71. client = await service._get_client()
  72. try:
  73. assert client.timeout.connect is not None
  74. assert client.timeout.connect < 15.0
  75. # The body still gets the generous budget — image uploads on a slow
  76. # uplink must not start failing.
  77. assert client.timeout.read == 30.0
  78. assert client.timeout.write == 30.0
  79. finally:
  80. await service.close()