Wait for live streams to end before downloading

While a YouTube video is still a live stream (or upcoming), the static
formats bestaudio requests may not be available, so yt-dlp fails with
'Requested format is not available'. Once the stream ends, YouTube
converts it to a regular VOD that downloads normally.

Before downloading, probe the video's live_status via yt-dlp. If it is
still live, reschedule the job in 10-minute increments (live_wait_count)
without counting against the normal retry budget. After 24 consecutive
waits (~4h) the job falls back to a regular retry attempt with the
exponential backoff, so a video that never converts still fails
permanently instead of retrying forever. Live waits are reset on
success or any ordinary retry.

downloader/downloader.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -44,6 +45,22 @@ func NewPool(poolSize int, backoffBase time.Duration, sm *state.Manager) *Pool {
 	return &Pool{poolSize: poolSize, backoffBase: backoffBase, state: sm}
 }
 
+// liveWaitInterval is how often a still-live YouTube stream is re-checked.
+// While a stream is live, the static formats bestaudio requests may not be
+// available ("Requested format is not available"); once the stream ends,
+// YouTube converts it to a regular VOD whose formats download normally.
+const liveWaitInterval = 10 * time.Minute
+
+// maxLiveWaits bounds the live wait: after this many 10-minute waits
+// (~4 hours) the job falls back to a regular retry attempt with the
+// normal exponential backoff, so a video that never converts still ends
+// in a permanent failure rather than retrying forever.
+const maxLiveWaits = 24
+
+// errLiveWait is returned by downloadYT when the video is still live and
+// has been rescheduled; it is not a download failure.
+var errLiveWait = errors.New("video is live, waiting for stream to end")
+
 // Run starts the worker goroutines. It returns immediately after spawning the workers.
 func (p *Pool) Run(jobs <-chan DownloadJob) {
 	for i := 0; i < p.poolSize; i++ {
@@ -64,7 +81,11 @@ func (p *Pool) worker(job DownloadJob) {
 		err = p.download(job)
 	}
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "ERROR: download %s failed: %v\n", job.EnclosureURL, err)
+		if errors.Is(err, errLiveWait) {
+			fmt.Fprintf(os.Stderr, "rssd: %s still live, retrying in %s\n", job.EnclosureURL, liveWaitInterval)
+		} else {
+			fmt.Fprintf(os.Stderr, "ERROR: download %s failed: %v\n", job.EnclosureURL, err)
+		}
 	} else {
 		fmt.Fprintf(os.Stderr, "OK: downloaded %s -> %s\n", job.EnclosureURL, job.DestPath)
 	}
@@ -76,6 +97,15 @@ func (p *Pool) worker(job DownloadJob) {
 // WebM) with no re-encoding. The file lands at destPath + ".<actual-ext>";
 // state is updated to the real path.
 func (p *Pool) downloadYT(job DownloadJob) error {
+	// While the video is still a live stream, the static formats we request
+	// are not available yet. Instead of counting a failed attempt, reschedule
+	// the job in liveWaitInterval; once the stream ends and converts to a
+	// regular VOD the normal download succeeds.
+	if p.isStillLive(job) {
+		p.markLiveWait(job.EnclosureURL)
+		return errLiveWait
+	}
+
 	// Update state to downloading.
 	if !p.state.UpdateJob(job.EnclosureURL, func(j *state.Job) {
 		j.Status = state.StatusDownloading
@@ -123,6 +153,7 @@ func (p *Pool) downloadYT(job DownloadJob) error {
 	p.state.UpdateJob(job.EnclosureURL, func(j *state.Job) {
 		j.Status = state.StatusDownloaded
 		j.DestPath = actual
+		j.LiveWaitCount = 0
 	})
 	if err := p.state.Save(); err != nil {
 		return fmt.Errorf("save state (downloaded): %w", err)
@@ -132,6 +163,42 @@ func (p *Pool) downloadYT(job DownloadJob) error {
 	return nil
 }
 
+// isStillLive reports whether the video is a live stream that has not ended
+// yet (or is not even started). If the probe itself fails, false is returned
+// so the normal download path (with its own error handling) can proceed.
+func (p *Pool) isStillLive(job DownloadJob) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	info, err := ytdlp.Info(ctx, job.YTSettings, job.VideoID)
+	if err != nil {
+		return false
+	}
+	return info.LiveStatus == "is_live" || info.LiveStatus == "upcoming"
+}
+
+// markLiveWait reschedules a job whose video is still live. The wait does
+// not count against the job's retry budget; after maxLiveWaits consecutive
+// waits the next check falls back to a regular retry attempt.
+func (p *Pool) markLiveWait(enclosureURL string) {
+	p.state.UpdateJob(enclosureURL, func(j *state.Job) {
+		j.Status = state.StatusRetrying
+		j.LiveWaitCount++
+		t := time.Now()
+		if j.LiveWaitCount <= maxLiveWaits {
+			j.Error = fmt.Sprintf("video is live (wait %d/%d), retrying after stream ends", j.LiveWaitCount, maxLiveWaits)
+			t = t.Add(liveWaitInterval)
+		} else {
+			j.LiveWaitCount = 0
+			j.AttemptCount++
+			delay := p.backoffBase * time.Duration(1<<uint(j.AttemptCount-1))
+			j.Error = fmt.Sprintf("video still live after %d waits; falling back to regular retry", maxLiveWaits)
+			t = t.Add(delay)
+		}
+		j.NextAttemptAt = &t
+	})
+	p.state.Save()
+}
+
 // runYT executes yt-dlp for a single video, downloading audio only.
 func (p *Pool) runYT(ctx context.Context, job DownloadJob, args ...string) error {
 	cmd := ytdlp.CommandFor(ctx, job.YTSettings)
@@ -314,6 +381,7 @@ func ffprobeDuration(path string) (float64, error) {
 func (p *Pool) markRetry(enclosureURL string, err error) {
 	p.state.UpdateJob(enclosureURL, func(j *state.Job) {
 		j.Status = state.StatusRetrying
+		j.LiveWaitCount = 0
 		j.Error = err.Error()
 		// Exponential backoff: base * 2^attemptCount (attempt already incremented).
 		delay := p.backoffBase * time.Duration(1<<uint(j.AttemptCount-1))

state/state.go

@@ -33,6 +33,7 @@ type Job struct {
 	NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"` // nil means ready now
 	Error         string     `json:"error,omitempty"`
 	Duration      float64    `json:"duration,omitempty"` // expected media duration in seconds (YouTube)
+	LiveWaitCount int        `json:"live_wait_count,omitempty"` // consecutive 10-min waits while a video is still live (does not count against AttemptCount)
 }
 
 // JobStatus values.

ytdlp/ytdlp.go

@@ -81,6 +81,7 @@ type VideoInfo struct {
 	Title      string
 	Duration   float64 // seconds, 0 if unknown
 	UploadDate time.Time
+	LiveStatus string // "not_live", "is_live", "was_live", "upcoming", or "" if unknown
 }
 
 // Info resolves full metadata for a single video by ID.
@@ -124,6 +125,7 @@ func Info(ctx context.Context, s Settings, videoID string) (VideoInfo, error) {
 			info.UploadDate = t
 		}
 	}
+	info.LiveStatus, _ = data["live_status"].(string)
 	return info, nil
 }