tile-live.ps1, flagship-live.ps1, foodcam-live.ps1 — the whole thing

go2rtc (WSL2)  rtsp://127.0.0.1:8554/helen_food_face_sub ──► tile-live.ps1 -Cam helen_food_face      -Port 5107 ─┐
               rtsp://127.0.0.1:8554/cat_water_bowl_sub  ──► tile-live.ps1 -Cam cat_water_bowl       -Port 5103 ─┤  mpegts over
               rtsp://127.0.0.1:8554/cat_treat_dispenser_sub ► tile-live.ps1 -Cam cat_treat_dispenser -Port 5104 ─┤  udp://127.0.0.1
               rtsp://127.0.0.1:8554/bed_cam_sub         ──► tile-live.ps1 -Cam bed_cam              -Port 5105 ─┘
                                                                                                                  │
                                                         flagship-live.ps1: 4×scale=960:540 → hstack/vstack → h264_qsv 3500k @10fps → RTMP
                                                                                                                                (video HaRC7bsF-pM)
               rtsp://127.0.0.1:8554/helen_food_face ────► foodcam-live.ps1: d3d11va decode → fps=15 → h264_qsv 2500k → RTMP
                                                                                                                                (video JrPoY5QHLdw)

param(
    [Parameter(Mandatory=$true)][string]$Cam,
    [Parameter(Mandatory=$true)][int]$Port
)

# v6 (2026-08-29): restream go2rtc SUB as MPEG-TS copy. No decode, no x264.
# Compositor scales to 960x540. Food cam stays independently re-encoded.
$ffmpeg = "C:\Users\<YOUR_USER>\AppData\Local\Microsoft\WinGet\Links\ffmpeg.exe"
$logDir = "C:\ProgramData\HelenLive\logs"
New-Item -ItemType Directory -Force -Path $logDir | Out-Null

$memCapBytes = 250MB
$wrapperLog = Join-Path $logDir "flagship-tile-$Cam-wrapper.log"
$src = "rtsp://127.0.0.1:8554/${Cam}_sub"

while ($true) {
    $ts = Get-Date -Format "yyyyMMdd-HHmmss"
    $logFile = Join-Path $logDir "flagship-tile-$Cam-$ts.log"

    $ffmpegArgs = @(
        "-rtsp_transport","tcp","-i",$src,
        "-an","-c:v","copy",
        "-f","mpegts","udp://127.0.0.1:$Port`?pkt_size=1316"
    )

    $proc = Start-Process -FilePath $ffmpeg -ArgumentList $ffmpegArgs -NoNewWindow -PassThru `
        -RedirectStandardOutput $logFile -RedirectStandardError "$logFile.err"

    while (-not $proc.HasExited) {
        Start-Sleep -Seconds 5
        $proc.Refresh()
        if ($proc.WorkingSet64 -gt $memCapBytes) {
            Add-Content -Path $wrapperLog -Value "$(Get-Date -Format o) memory cap exceeded ($($proc.WorkingSet64) bytes), killing"
            Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
            break
        }
    }
    if (-not $proc.HasExited) { Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue }

    Add-Content -Path $wrapperLog -Value "$(Get-Date -Format o) ffmpeg exited (code $($proc.ExitCode)), restarting in 5s"
    Start-Sleep -Seconds 5
}

YT_FLAGSHIP_RTMP_URL=<YT_RTMP_URL>
YT_FLAGSHIP_STREAM_KEY=<YT_STREAM_KEY>
YT_FLAGSHIP_BROADCAST_ID=<YT_BROADCAST_ID>

$envFile = "C:\ProgramData\HelenLive\flagship-live.env"
Get-Content $envFile | ForEach-Object {
    if ($_ -match '^(.*?)=(.*)$') {
        [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2])
    }
}

$rtmpUrl = $env:YT_FLAGSHIP_RTMP_URL
$streamKey = $env:YT_FLAGSHIP_STREAM_KEY
$dest = "$rtmpUrl/$streamKey"

$ffmpeg = "C:\Users\<YOUR_USER>\AppData\Local\Microsoft\WinGet\Links\ffmpeg.exe"
$logDir = "C:\ProgramData\HelenLive\logs"
New-Item -ItemType Directory -Force -Path $logDir | Out-Null

$memCapBytes = 900MB
$wrapperLog = Join-Path $logDir "flagship-comp-wrapper.log"
$rtmpGraceSec = 45

# v11 (2026-08-30): pane 0 is Face (helen_food_face_sub) on UDP 5107.
# Privacy/framing decision: Court includes the left entrance where people
# may pass; Face avoids showing that entrance on the public mosaic.
# Layout: top row = face(5107), water(5103); bottom row = treat(5104),
# bed(5105). QSV compositor encode - do not regress to libx264.
#
# v11 adds an RTMP health check: the old watchdog only restarted ffmpeg on
# process EXIT, but YouTube can hang up the RTMP socket (state -> CloseWait)
# while ffmpeg itself stays alive and never exits, so the stream goes dark
# with `schtasks /Query` still reporting "Running". Diagnosed 2026-08-30
# 07:31 EDT after the mosaic stopped twice in under an hour; `schtasks` gave
# no signal either time, only `Get-NetTCPConnection -RemotePort 1935` showed
# the real state. After a $rtmpGraceSec startup grace period (handshake takes
# ~10-50s observed), if ffmpeg's PID has no RTMP connection to YouTube in the
# Established state, kill it so the outer while($true) loop restarts with a
# fresh socket.
while ($true) {
    $ts = Get-Date -Format "yyyyMMdd-HHmmss"
    $logFile = Join-Path $logDir "flagship-comp-$ts.log"

    $ffmpegArgs = @(
        "-fflags","nobuffer+genpts","-i","udp://127.0.0.1:5107?fifo_size=1000000&overrun_nonfatal=1",
        "-fflags","nobuffer+genpts","-i","udp://127.0.0.1:5103?fifo_size=1000000&overrun_nonfatal=1",
        "-fflags","nobuffer+genpts","-i","udp://127.0.0.1:5104?fifo_size=1000000&overrun_nonfatal=1",
        "-fflags","nobuffer+genpts","-i","udp://127.0.0.1:5105?fifo_size=1000000&overrun_nonfatal=1",
        "-stream_loop","-1","-i","C:\ProgramData\HelenLive\music\helen_serial.wav",
        "-filter_complex","[0:v]scale=960:540,setsar=1[v0];[1:v]scale=960:540,setsar=1[v1];[2:v]scale=960:540,setsar=1[v2];[3:v]scale=960:540,setsar=1[v3];[v0][v1]hstack=inputs=2[top];[v2][v3]hstack=inputs=2[bottom];[top][bottom]vstack=inputs=2[outv]",
        "-map","[outv]","-map","4:a",
        "-vsync","cfr","-r","10",
        "-c:v","h264_qsv","-preset","veryfast","-look_ahead","0","-bf","0","-b:v","3500k","-maxrate","3500k","-bufsize","7000k","-g","20",
        "-c:a","aac","-b:a","128k","-ar","44100",
        "-f","flv",$dest
    )

    $proc = Start-Process -FilePath $ffmpeg -ArgumentList $ffmpegArgs -NoNewWindow -PassThru `
        -RedirectStandardOutput $logFile -RedirectStandardError "$logFile.err"
    $startTime = Get-Date
    $killReason = $null

    while (-not $proc.HasExited) {
        Start-Sleep -Seconds 5
        $proc.Refresh()
        if ($proc.WorkingSet64 -gt $memCapBytes) {
            $killReason = "memory cap exceeded ($($proc.WorkingSet64) bytes)"
            Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
            break
        }
        $elapsed = (Get-Date) - $startTime
        if ($elapsed.TotalSeconds -gt $rtmpGraceSec) {
            $conn = Get-NetTCPConnection -OwningProcess $proc.Id -RemotePort 1935 -ErrorAction SilentlyContinue
            if (-not $conn) {
                $killReason = "no RTMP connection to YouTube after ${rtmpGraceSec}s"
                Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
                break
            } elseif (@($conn | Where-Object { $_.State -eq 'Established' }).Count -eq 0) {
                $badStates = ($conn | Select-Object -ExpandProperty State) -join ','
                $killReason = "RTMP connection unhealthy (state=$badStates)"
                Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
                break
            }
        }
    }
    if (-not $proc.HasExited) { Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue }

    if ($killReason) {
        Add-Content -Path $wrapperLog -Value "$(Get-Date -Format o) $killReason, killing and restarting in 5s"
    } else {
        Add-Content -Path $wrapperLog -Value "$(Get-Date -Format o) ffmpeg exited (code $($proc.ExitCode)), restarting in 5s"
    }
    Start-Sleep -Seconds 5
}

$envFile = "C:\ProgramData\HelenLive\foodcam-live.env"
Get-Content $envFile | ForEach-Object {
    if ($_ -match '^(.*?)=(.*)$') {
        [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2])
    }
}

$rtmpUrl = $env:YT_FOODCAM_RTMP_URL
$streamKey = $env:YT_FOODCAM_STREAM_KEY
$dest = "$rtmpUrl/$streamKey"

$ffmpeg = "C:\Users\<YOUR_USER>\AppData\Local\Microsoft\WinGet\Links\ffmpeg.exe"
$logDir = "C:\ProgramData\HelenLive\logs"
New-Item -ItemType Directory -Force -Path $logDir | Out-Null

$memCapBytes = 700MB
$wrapperLog = Join-Path $logDir "foodcam-wrapper.log"
$rtmpGraceSec = 45

# v5 (2026-08-30): adds the same RTMP health check built for flagship v11
# (see flagship-live.ps1 for the full incident note). The old watchdog only
# restarted ffmpeg on process EXIT or on a frozen `frame=` counter it never
# actually checked. If YouTube hangs up the RTMP socket (state -> CloseWait)
# ffmpeg can sit there alive and connected-to-nothing indefinitely. After a
# $rtmpGraceSec startup grace period, require an Established RTMP connection
# to YouTube or kill and restart.
while ($true) {
    $ts = Get-Date -Format "yyyyMMdd-HHmmss"
    $logFile = Join-Path $logDir "foodcam-$ts.log"

    # v4: d3d11va decode + QSV encode. Still re-time to 15fps CFR for YouTube
    # (copy to YT caused non-monotonic DTS / buffering). Never camera mic.
    $ffmpegArgs = @(
        "-hwaccel","d3d11va",
        "-rtsp_transport","tcp","-i","rtsp://127.0.0.1:8554/helen_food_face",
        "-stream_loop","-1","-i","C:\ProgramData\HelenLive\music\helen_face.wav",
        "-map","0:v","-map","1:a",
        "-vf","fps=15,format=nv12",
        "-c:v","h264_qsv","-preset","veryfast","-look_ahead","0","-bf","0",
        "-b:v","2500k","-maxrate","2500k","-bufsize","5000k",
        "-g","30","-r","15",
        "-c:a","aac","-b:a","128k","-ar","44100",
        "-f","flv",$dest
    )

    $proc = Start-Process -FilePath $ffmpeg -ArgumentList $ffmpegArgs -NoNewWindow -PassThru `
        -RedirectStandardOutput $logFile -RedirectStandardError "$logFile.err"
    $startTime = Get-Date
    $killReason = $null

    while (-not $proc.HasExited) {
        Start-Sleep -Seconds 5
        $proc.Refresh()
        if ($proc.WorkingSet64 -gt $memCapBytes) {
            $killReason = "memory cap exceeded ($($proc.WorkingSet64) bytes)"
            Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
            break
        }
        $elapsed = (Get-Date) - $startTime
        if ($elapsed.TotalSeconds -gt $rtmpGraceSec) {
            $conn = Get-NetTCPConnection -OwningProcess $proc.Id -RemotePort 1935 -ErrorAction SilentlyContinue
            if (-not $conn) {
                $killReason = "no RTMP connection to YouTube after ${rtmpGraceSec}s"
                Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
                break
            } elseif (@($conn | Where-Object { $_.State -eq 'Established' }).Count -eq 0) {
                $badStates = ($conn | Select-Object -ExpandProperty State) -join ','
                $killReason = "RTMP connection unhealthy (state=$badStates)"
                Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
                break
            }
        }
    }
    if (-not $proc.HasExited) { Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue }

    if ($killReason) {
        Add-Content -Path $wrapperLog -Value "$(Get-Date -Format o) $killReason, killing and restarting in 10s"
    } else {
        Add-Content -Path $wrapperLog -Value "$(Get-Date -Format o) ffmpeg exited (code $($proc.ExitCode)), restarting in 10s"
    }
    Start-Sleep -Seconds 10
}

# Daily maintenance: refresh the 4 flagship mosaic tile processes + the
# compositor that reads them. Added 2026-08-31.
#
# Why this exists: the 4 tile ffmpeg processes (Face/Water/Treat/Bed, each a
# plain `-c:v copy` from go2rtc's RTSP `_sub` stream to a loopback UDP port)
# have no health check of their own - only the flagship/foodcam RTMP
# encoders got the RTMP self-heal (see flagship-live.ps1 v11). Left running
# 10+ hours straight, their long-lived RTSP connections can degrade quietly
# (no crash, just bad data), which then feeds the compositor corrupt input
# it can't decode ("no frame!" / "non-existing PPS 0 referenced") - a
# failure the RTMP self-heal cannot detect or fix, because the local
# process/socket state looks fine throughout.
#
# Fix is mechanical: Stop+Start all 4 tiles for fresh RTSP connections, then
# Stop+Start the compositor so it picks up clean input. Same two steps that
# resolved the real outage on 2026-08-30 21:56 EDT, done here proactively
# once a day instead of waiting for someone to notice the stream is dark.

$logDir = "C:\ProgramData\HelenLive\logs"
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
$log = Join-Path $logDir "tile-refresh.log"

function Log($msg) {
    Add-Content -Path $log -Value "$(Get-Date -Format o) $msg"
}

Log "=== daily tile refresh starting ==="

$tiles = @('HelenFlagshipTileFace', 'HelenFlagshipTileWater', 'HelenFlagshipTileTreat', 'HelenFlagshipTileBed')

foreach ($t in $tiles) {
    $out = schtasks /End /TN $t 2>&1
    Log "End $t -> $out"
}

Start-Sleep -Seconds 5

foreach ($t in $tiles) {
    $out = schtasks /Run /TN $t 2>&1
    Log "Run $t -> $out"
}

Start-Sleep -Seconds 5

$out = schtasks /End /TN HelenFlagshipLive 2>&1
Log "End HelenFlagshipLive -> $out"

Start-Sleep -Seconds 3

$out = schtasks /Run /TN HelenFlagshipLive 2>&1
Log "Run HelenFlagshipLive -> $out"

Log "=== daily tile refresh done ==="

<?xml version="1.0" encoding="UTF-16"?>
<Task version="1.2" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task">
  <RegistrationInfo>
    <Date>2026-08-29T21:30:00</Date>
    <Author><YOUR_PC>\<YOUR_USER></Author>
    <URI>\HelenFlagshipTileFace</URI>
  </RegistrationInfo>
  <Principals>
    <Principal id="Author">
      <UserId>S-1-5-18</UserId>
      <RunLevel>HighestAvailable</RunLevel>
    </Principal>
  </Principals>
  <Settings>
    <DisallowStartIfOnBatteries>true</DisallowStartIfOnBatteries>
    <StopIfGoingOnBatteries>true</StopIfGoingOnBatteries>
    <MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy>
    <IdleSettings>
      <Duration>PT10M</Duration>
      <WaitTimeout>PT1H</WaitTimeout>
      <StopOnIdleEnd>true</StopOnIdleEnd>
      <RestartOnIdle>false</RestartOnIdle>
    </IdleSettings>
  </Settings>
  <Triggers>
    <BootTrigger>
      <StartBoundary>2026-08-29T21:30:00</StartBoundary>
    </BootTrigger>
  </Triggers>
  <Actions Context="Author">
    <Exec>
      <Command>powershell</Command>
      <Arguments>-ExecutionPolicy Bypass -File C:\ProgramData\HelenLive\tile-live.ps1 -Cam helen_food_face -Port 5107</Arguments>
    </Exec>
  </Actions>
</Task>

# mosaic and single cam: at boot, as SYSTEM, hidden window
schtasks /Create /TN HelenFlagshipLive /SC ONSTART /RU SYSTEM /RL HIGHEST /F `
  /TR "powershell.exe -ExecutionPolicy Bypass -WindowStyle Hidden -File C:\ProgramData\HelenLive\flagship-live.ps1"
schtasks /Create /TN HelenFoodCamLive /SC ONSTART /RU SYSTEM /RL HIGHEST /F `
  /TR "powershell.exe -ExecutionPolicy Bypass -WindowStyle Hidden -File C:\ProgramData\HelenLive\foodcam-live.ps1"

# daily tile + compositor reset at 04:00
schtasks /Create /TN HelenFlagshipTileRefresh /SC DAILY /ST 04:00 /RU SYSTEM /RL HIGHEST /F `
  /TR "powershell.exe -ExecutionPolicy Bypass -WindowStyle Hidden -File C:\ProgramData\HelenLive\refresh-tiles.ps1"

# start everything now without rebooting
'HelenFlagshipTileFace','HelenFlagshipTileWater','HelenFlagshipTileTreat','HelenFlagshipTileBed',
'HelenFlagshipLive','HelenFoodCamLive' | ForEach-Object { schtasks /Run /TN $_ }

# flagship-comp-20260902-164809.log.err
Input #0, mpegts, from 'udp://127.0.0.1:5107?fifo_size=1000000&overrun_nonfatal=1':
  Stream #0:0[0x100]: Video: h264 (High), yuv420p(progressive), 640x360, 20 fps, 20 tbr, 90k tbn, start 10.800000
  ... three more identical inputs on 5103, 5104, 5105 ...
Input #4, wav, from 'C:\ProgramData\HelenLive\music\helen_serial.wav':
  Stream #4:0: Audio: pcm_s16le, 44100 Hz, stereo, s16, 1411 kb/s
  vstack:default -> Stream #0:0 (h264_qsv)
      encoder         : Lavc62.28.101 h264_qsv
frame=47550 fps= 10 q=21.0 size= 2118844KiB time=01:19:14.90 bitrate=3650.5kbits/s dup=73 drop=55723 speed=0.999x

# foodcam-20260902-164809.log.err
Input #0, rtsp, from 'rtsp://127.0.0.1:8554/helen_food_face':
  Stream #0:0: Video: h264 (High), yuv420p(progressive), 2560x1440, 20 fps, 20 tbr, 90k tbn
  Stream #0:1: Audio: pcm_alaw, 8000 Hz, mono, s16, 64 kb/s          <- never mapped
Stream mapping:
  Stream #0:0 -> #0:0 (h264 (native) -> h264 (h264_qsv))
  Stream #1:0 -> #0:1 (pcm_s16le (native) -> aac (native))
Output #0, flv, to '<YT_RTMP_URL>/<YT_STREAM_KEY>':
  Stream #0:0: Video: h264, nv12(tv, progressive), 2560x1440, q=2-31, 2500 kb/s, 15 fps, 1k tbn
  Stream #0:1: Audio: aac (LC), 44100 Hz, stereo, fltp, 128 kb/s
frame=71684 fps= 15 q=29.0 size= 1540588KiB time=01:19:38.86 bitrate=2640.9kbits/s dup=21 drop=0 speed=   1x

2026-09-02T16:36:14-04:00 RTMP connection unhealthy (state=CloseWait), killing and restarting in 5s
2026-09-02T16:37:15-04:00 RTMP connection unhealthy (state=CloseWait), killing and restarting in 5s
2026-09-02T16:38:15-04:00 RTMP connection unhealthy (state=CloseWait), killing and restarting in 5s
   ... nine more, one a minute ...
2026-09-02T16:47:30-04:00 RTMP connection unhealthy (state=CloseWait), killing and restarting in 5s
(next session started 16:48:09 and was still Established at 18:07)

# foodcam-wrapper.log, previous 26 hours
2026-09-01T16:27:54-04:00 no RTMP connection to YouTube after 45s, killing and restarting in 10s
2026-09-01T16:34:04-04:00 RTMP connection unhealthy (state=CloseWait), killing and restarting in 10s
2026-09-02T03:02:38-04:00 RTMP connection unhealthy (state=CloseWait), killing and restarting in 10s
2026-09-02T03:05:22-04:00 RTMP connection unhealthy (state=CloseWait), killing and restarting in 10s
2026-09-02T03:09:38-04:00 RTMP connection unhealthy (state=CloseWait), killing and restarting in 10s
2026-09-02T03:09:53-04:00 ffmpeg exited (code ), restarting in 10s
2026-09-02T03:11:24-04:00 RTMP connection unhealthy (state=CloseWait), killing and restarting in 10s
2026-09-02T06:24:06-04:00 no RTMP connection to YouTube after 45s, killing and restarting in 10s