bambuddy/backend/tests/unit/test_camera_usb_stream_cleanup.py

"""External/USB camera ffmpeg-leak cleanup (#2675, reporter @bitbarista).

An external USB (V4L2) camera's ffmpeg used to be reachable only from its own
stream generator's ``finally`` — which an abrupt client disconnect can skip
(same cancellation-timing class as #776). Because external streams never
registered into ``_active_streams`` / ``_disconnect_events`` / the spawned-PID
map, both ``/camera/stop`` and ``cleanup_orphaned_streams`` were structurally
blind to the leak, leaving ``/dev/videoN`` locked. The fix registers the external
ffmpeg into the same registries the RTSP path uses.
"""

from __future__ import annotations

import asyncio
import os
import sys
import time
from contextlib import suppress
from unittest.mock import AsyncMock, mock_open, patch

import pytest

from backend.app.api.routes import camera
from backend.app.services import external_camera


async def _instant_sleep(*_args, **_kwargs) -> None:
    """Drop-in for asyncio.sleep that returns immediately (no self-recursion)."""
    return None


class _CleanProc:
    """ffmpeg that terminates cleanly when asked."""

    def __init__(self, pid: int) -> None:
        self.pid = pid
        self.returncode = None
        # Real Process objects always expose these (None when not piped), and
        # _terminate_ffmpeg drains them so a full pipe can't wedge the exit.
        self.stdout = None
        self.stderr = None

    def terminate(self) -> None:
        self.returncode = 0

    def kill(self) -> None:
        self.returncode = -9

    async def wait(self) -> int:
        return self.returncode if self.returncode is not None else 0


class _ImmediateEOFReader:
    async def read(self, _size: int = -1) -> bytes:
        return b""


class _UsbProc:
    """ffmpeg for a USB stream: yields no frames, exits at first read."""

    def __init__(self, pid: int = 52001) -> None:
        self.pid = pid
        self.returncode = None
        self.stdout = _ImmediateEOFReader()
        self.stderr = _ImmediateEOFReader()

    def terminate(self) -> None:
        self.returncode = 0

    def kill(self) -> None:
        self.returncode = -9

    async def wait(self) -> int:
        return 0


# ---------------------------------------------------------------------------
# 1. The stream generator hands its ffmpeg process to the on_process callback
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_stream_usb_registers_process_via_on_process(monkeypatch):
    """``_stream_usb`` must call ``on_process`` with the spawned ffmpeg so the
    route can register it — this is the linchpin of the whole fix."""

    class _FakePath:
        def __init__(self, _p: str) -> None:
            pass

        def exists(self) -> bool:
            return True

    proc = _UsbProc()

    async def fake_create_subprocess_exec(*_args, **_kwargs):
        return proc

    monkeypatch.setattr(external_camera, "get_ffmpeg_path", lambda: "/fake/ffmpeg")
    monkeypatch.setattr(external_camera, "Path", _FakePath)
    monkeypatch.setattr(external_camera.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
    monkeypatch.setattr(external_camera.asyncio, "sleep", _instant_sleep)

    captured: list[object] = []
    stream = external_camera._stream_usb("/dev/video0", 10, on_process=captured.append)
    try:
        async for _frame in stream:
            pass
    finally:
        with suppress(Exception):
            await stream.aclose()

    assert captured == [proc], "the spawned ffmpeg process must be handed to on_process"


# ---------------------------------------------------------------------------
# 2. /camera/stop now finds and kills a registered external USB process
#    (the reported {"stopped": 0} → {"stopped": 1})
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_stop_endpoint_terminates_registered_external_process(monkeypatch):
    monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.05)
    monkeypatch.setattr(camera, "get_subscriber_count", lambda _key: 0)

    async def fake_shutdown(_key):
        return False

    monkeypatch.setattr(camera, "shutdown_broadcaster", fake_shutdown)

    printer_id = 7
    sid = f"{printer_id}-ext-abc12345"
    proc = _CleanProc(pid=52010)
    event = asyncio.Event()
    camera._active_streams[sid] = proc
    camera._disconnect_events[sid] = event
    camera._spawned_ffmpeg_pids[proc.pid] = time.time()
    camera._stream_last_frame_times[sid] = time.time()

    try:
        result = await camera.stop_camera_stream(printer_id, _=None)
        assert result["stopped"] == 1
        assert proc.returncode is not None, "the external ffmpeg must be terminated"
        assert event.is_set(), "the stream's stop event must be signalled"
        # Registry fully cleaned so it can't be double-reaped.
        assert sid not in camera._active_streams
        assert sid not in camera._disconnect_events
        assert proc.pid not in camera._spawned_ffmpeg_pids
    finally:
        camera._active_streams.pop(sid, None)
        camera._disconnect_events.pop(sid, None)
        camera._spawned_ffmpeg_pids.pop(proc.pid, None)
        camera._stream_last_frame_times.pop(sid, None)


# ---------------------------------------------------------------------------
# 3. The orphan janitor reaps a stale registered external USB stream
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_cleanup_janitor_reaps_stale_external_usb_stream(monkeypatch):
    monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.05)
    monkeypatch.setattr(camera, "_scan_bambu_ffmpeg_pids", lambda: [])

    import os

    proc = _CleanProc(pid=os.getpid())  # real pid so layer-2 existence check keeps it
    sid = "7-ext-deadbeef"
    now = time.time()
    camera._active_streams[sid] = proc
    camera._spawned_ffmpeg_pids[proc.pid] = now - 120  # spawned long ago
    camera._stream_last_frame_times[sid] = now - 60  # stale: no frames >30s
    camera._disconnect_events[sid] = asyncio.Event()

    try:
        await asyncio.wait_for(camera.cleanup_orphaned_streams(), timeout=2.0)
        assert proc.returncode is not None, "stale external ffmpeg must be killed"
        assert sid not in camera._active_streams
    finally:
        camera._active_streams.pop(sid, None)
        camera._spawned_ffmpeg_pids.pop(proc.pid, None)
        camera._stream_last_frame_times.pop(sid, None)
        camera._disconnect_events.pop(sid, None)


# ---------------------------------------------------------------------------
# 4. The /proc "nuclear net" now matches USB (v4l2) ffmpeg from prior sessions
# ---------------------------------------------------------------------------


def test_scan_matches_v4l2_ffmpeg(monkeypatch):
    cmdline = b"ffmpeg\x00-f\x00v4l2\x00-i\x00/dev/video0\x00-f\x00mjpeg\x00-\x00"
    monkeypatch.setattr("os.listdir", lambda _p: ["52020"])
    monkeypatch.setattr(camera, "_read_ppid", lambda _pid: 1)
    with patch("builtins.open", mock_open(read_data=cmdline)):
        assert 52020 in camera._scan_bambu_ffmpeg_pids()


def test_scan_matches_an_orphaned_rtsp_ffmpeg(monkeypatch):
    cmdline = b"ffmpeg\x00-i\x00rtsp://bblp:code@127.0.0.1:4000/streaming/live/1\x00-\x00"
    monkeypatch.setattr("os.listdir", lambda _p: ["52022"])
    monkeypatch.setattr(camera, "_read_ppid", lambda _pid: 1)
    with patch("builtins.open", mock_open(read_data=cmdline)):
        assert camera._scan_bambu_ffmpeg_pids() == [52022]


@pytest.mark.skipif(sys.platform != "linux", reason="/proc")
def test_scan_leaves_another_processes_camera_ffmpeg_alone(tmp_path):
    """For real, through /proc: the same command run by another program (here
    a shell standing in for another Bambuddy) is not found; ours is."""
    import subprocess

    fake = tmp_path / "ffmpeg"
    fake.write_text("#!/bin/sh\nsleep 30\n")
    fake.chmod(0o755)
    url = "rtsp://bblp:code@127.0.0.1:4000/streaming/live/1"
    theirs = subprocess.Popen(["sh", "-c", f"'{fake}' -i '{url}' -; true"])
    ours = subprocess.Popen([str(fake), "-i", url, "-"])
    try:
        deadline = time.monotonic() + 5
        other = None
        while other is None and time.monotonic() < deadline:
            for pid in os.listdir("/proc"):
                if pid.isdigit() and camera._read_ppid(int(pid)) == theirs.pid:
                    other = int(pid)
            time.sleep(0.05)
        assert other is not None, "the other program's ffmpeg did not start"

        found = camera._scan_bambu_ffmpeg_pids()

        assert ours.pid in found
        assert other not in found
    finally:
        for proc in (theirs, ours):
            proc.kill()
            proc.wait()
        subprocess.run(["pkill", "-f", str(fake)], check=False)


def test_scan_ignores_unrelated_ffmpeg(monkeypatch):
    # A transcode of a local file is not ours — must not be reaped.
    cmdline = b"ffmpeg\x00-i\x00/home/user/movie.mp4\x00out.mkv\x00"
    monkeypatch.setattr("os.listdir", lambda _p: ["52021"])
    with patch("builtins.open", mock_open(read_data=cmdline)):
        assert camera._scan_bambu_ffmpeg_pids() == []


# ---------------------------------------------------------------------------
# 5. Only our own USB ffmpeg: another program's camera process is left alone
# ---------------------------------------------------------------------------

OWN_PID = 4242
ONE_SHOT = [b"/usr/bin/ffmpeg", b"-f", b"v4l2", b"-i", b"/dev/video0", b"-frames:v", b"1", b"-f", b"image2pipe", b"-"]
STREAM = [b"ffmpeg", b"-f", b"v4l2", b"-framerate", b"10", b"-i", b"/dev/video2", b"-f", b"mjpeg", b"-"]


def test_our_one_shot_and_stream_commands_are_ours():
    assert camera._is_bambuddy_ffmpeg(ONE_SHOT, OWN_PID, OWN_PID)
    assert camera._is_bambuddy_ffmpeg(STREAM, OWN_PID, OWN_PID)


def test_an_orphan_from_an_earlier_run_is_ours():
    """Reparented to init once the Bambuddy that spawned it died."""
    assert camera._is_bambuddy_ffmpeg(STREAM, 1, OWN_PID)


def test_another_programs_usb_ffmpeg_is_not_ours():
    """motion, an OctoPrint webcam or a recording on a bare-metal install."""
    assert not camera._is_bambuddy_ffmpeg(STREAM, 777, OWN_PID)


def test_a_different_v4l2_command_shape_is_not_ours():
    recording = [b"ffmpeg", b"-f", b"v4l2", b"-input_format", b"h264", b"-i", b"/dev/video0", b"/srv/cam.mkv"]
    assert not camera._is_bambuddy_ffmpeg(recording, OWN_PID, OWN_PID)


RTSP = [b"ffmpeg", b"-rtsp_transport", b"tcp", b"-i", b"rtsps://bblp:code@10.0.0.5:322/streaming/live/1", b"-"]


def test_our_bambu_rtsp_and_its_orphans_are_ours():
    assert camera._is_bambuddy_ffmpeg(RTSP, OWN_PID, OWN_PID)
    assert camera._is_bambuddy_ffmpeg(RTSP, 1, OWN_PID)


def test_another_bambuddys_rtsp_is_not_ours():
    """A second install or a test server on the same host runs the same
    command: its cleanup killed this one's live camera every minute."""
    assert not camera._is_bambuddy_ffmpeg(RTSP, 777, OWN_PID)
    assert not camera._is_bambuddy_ffmpeg(RTSP, None, OWN_PID)


def test_unknown_parent_is_not_ours_for_usb():
    assert not camera._is_bambuddy_ffmpeg(STREAM, None, OWN_PID)


# ---------------------------------------------------------------------------
# 6. A stuck ffmpeg can never hang the stream's own cleanup (#2580)
# ---------------------------------------------------------------------------


class _WedgedProc:
    """ffmpeg blocked in write() on a full pipe: SIGTERM is ignored, and after
    SIGKILL wait() never returns while its pipes are unread."""

    def __init__(self) -> None:
        self.pid = 52999
        self.returncode = None
        self.stdout = asyncio.StreamReader()
        self.stdout.feed_data(b"\xff\xd8" + b"x" * 200 + b"\xff\xd9")
        self.stderr = asyncio.StreamReader()
        self.killed = False

    def terminate(self) -> None:
        pass

    def kill(self) -> None:
        self.killed = True
        self.returncode = -9

    async def wait(self) -> int:
        await asyncio.Event().wait()
        return -9  # pragma: no cover


@pytest.mark.asyncio
async def test_a_wedged_usb_ffmpeg_does_not_hang_the_stream_cleanup(monkeypatch):
    """The shared upstream's cleanup runs in the fan-out broadcaster, which stop,
    a camera change, printer delete and shutdown all wait on."""
    from backend.app.services import external_camera

    monkeypatch.setattr(camera, "_FFMPEG_TERM_TIMEOUT", 0.05)
    monkeypatch.setattr(camera, "_FFMPEG_KILL_TIMEOUT", 0.05)
    proc = _WedgedProc()
    monkeypatch.setattr(external_camera, "get_ffmpeg_path", lambda: "/usr/bin/ffmpeg")
    monkeypatch.setattr(external_camera, "_safe_usb_device_path", lambda _d: "/dev/video0")
    monkeypatch.setattr("asyncio.create_subprocess_exec", AsyncMock(return_value=proc))

    stream = external_camera._stream_usb("/dev/video0", 5)
    assert await asyncio.wait_for(anext(stream), 2)

    await asyncio.wait_for(stream.aclose(), 2)

    assert proc.killed