rssd/ytdlp/ytdlp.go

// Package ytdlp provides a thin wrapper around the yt-dlp CLI for
// YouTube channel/playlist discovery and per-video metadata resolution.
package ytdlp

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"os/exec"
	"strings"
	"time"
)

// maxJSONLineBytes bounds one line of --print-json output. A full video's
// JSON (including the formats list) routinely exceeds 1 MB for long videos,
// so the limit must be generous.
const maxJSONLineBytes = 16 * 1024 * 1024

// Settings controls how yt-dlp is invoked.
type Settings struct {
	// BinPath is the path to the yt-dlp executable (or a name on $PATH).
	BinPath string
	// PlayerClient is passed via --extractor-args "youtube:player_client=..."
	// to work around YouTube client restrictions (see DefaultPlayerClient).
	PlayerClient string
}

// DefaultPlayerClient works around YouTube's SABR-only streaming experiment,
// which withholds plain HTTPS URLs from the default web/android clients.
const DefaultPlayerClient = "web_embedded"

// FlatVideo is one entry from a --flat-playlist enumeration.
type FlatVideo struct {
	ID       string   `json:"id"`
	Title    string   `json:"title"`
	Duration *float64 `json:"duration"` // seconds; may be nil (e.g. some shorts)
	Channel  string   `json:"channel"`
}

// Discovery enumerates all videos in a YouTube channel or playlist using
// `yt-dlp --flat-playlist` (fast; does not resolve each video).
// Returns videos in the order yt-dlp emits them:
//
//   - channel:   newest first
//   - playlist:  playlist order (not necessarily chronological)
func Discovery(ctx context.Context, s Settings, sourceURL string) ([]FlatVideo, error) {
	args := []string{"--flat-playlist", "--print-json", sourceURL}
	stdout, stderr, err := run(ctx, s, args...)
	if err != nil {
		return nil, fmt.Errorf("yt-dlp discovery %s: %v: %s", sourceURL, err, LastLines(string(stderr), 3))
	}

	var vids []FlatVideo
	sc := bufio.NewScanner(bytes.NewReader(stdout))
	sc.Buffer(make([]byte, 0, 64*1024), maxJSONLineBytes)
	for sc.Scan() {
		line := strings.TrimSpace(sc.Text())
		if line == "" {
			continue
		}
		var v FlatVideo
		if err := json.Unmarshal([]byte(line), &v); err != nil {
			continue // skip malformed lines
		}
		if v.ID == "" {
			continue
		}
		vids = append(vids, v)
	}
	if len(vids) == 0 {
		return nil, fmt.Errorf("yt-dlp discovery %s: no videos found", sourceURL)
	}
	return vids, nil
}

// VideoInfo carries resolved metadata for a single video.
type VideoInfo struct {
	ID         string
	Title      string
	Duration   float64 // seconds, 0 if unknown
	UploadDate time.Time
	LiveStatus string // "not_live", "is_live", "was_live", "is_upcoming", or "" if unknown
	// ReleaseTimestamp is the scheduled start time of a live stream in unix
	// seconds (from liveBroadcastDetails.startTimestamp); 0 if unknown.
	ReleaseTimestamp int64
}

// Info resolves full metadata for a single video by ID.
func Info(ctx context.Context, s Settings, videoID string) (VideoInfo, error) {
	url := "https://www.youtube.com/watch?v=" + videoID
	stdout, stderr, err := run(ctx, s,
		"--no-download", "--no-playlist", "--print-json", url)
	if err != nil {
		return VideoInfo{}, fmt.Errorf("yt-dlp info %s: %v: %s", videoID, err, LastLines(string(stderr), 3))
	}

	sc := bufio.NewScanner(bytes.NewReader(stdout))
	sc.Buffer(make([]byte, 0, 64*1024), maxJSONLineBytes)
	var data map[string]any
	for sc.Scan() {
		line := strings.TrimSpace(sc.Text())
		if line == "" {
			continue
		}
		if err := json.Unmarshal([]byte(line), &data); err == nil {
			break
		}
	}
	if data == nil {
		if err := sc.Err(); err != nil {
			return VideoInfo{}, fmt.Errorf("yt-dlp info %s: %v", videoID, err)
		}
		return VideoInfo{}, fmt.Errorf("yt-dlp info %s: empty or unparseable output", videoID)
	}

	info := VideoInfo{ID: videoID}
	if t, _ := data["title"].(string); t != "" {
		info.Title = t
	}
	if d, ok := num(data["duration"]); ok && d > 0 {
		info.Duration = d
	}
	// upload_date is "YYYYMMDD" or "NA" (or null).
	if ud, ok := data["upload_date"].(string); ok && ud != "" && ud != "NA" {
		if t, err := time.Parse("20060102", ud); err == nil {
			info.UploadDate = t
		}
	}
	info.LiveStatus, _ = data["live_status"].(string)
	if ts, ok := num(data["release_timestamp"]); ok {
		info.ReleaseTimestamp = int64(ts)
	}
	return info, nil
}

// CommandFor builds an *exec.Cmd running yt-dlp with the standard
// arguments for the given settings (binary + YouTube extractor args).
// Additional arguments may be appended to cmd.Args by the caller.
func CommandFor(ctx context.Context, s Settings) *exec.Cmd {
	bin := s.BinPath
	if bin == "" {
		bin = "yt-dlp"
	}
	pc := s.PlayerClient
	if pc == "" {
		pc = DefaultPlayerClient
	}
	return exec.CommandContext(ctx, bin,
		"--no-warnings", "--extractor-args", "youtube:player_client="+pc)
}

// run executes yt-dlp with the given arguments and returns stdout and stderr.
func run(ctx context.Context, s Settings, args ...string) ([]byte, []byte, error) {
	cmd := CommandFor(ctx, s)
	cmd.Args = append(cmd.Args, args...)
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return stdout.Bytes(), stderr.Bytes(), err
	}
	return stdout.Bytes(), stderr.Bytes(), nil
}

// LastLines returns the last n non-empty lines of s, joined with " | ".
// Useful for compact error messages from yt-dlp output.
func LastLines(s string, n int) string {
	var lines []string
	for _, l := range strings.Split(s, "\n") {
		if t := strings.TrimSpace(l); t != "" {
			lines = append(lines, t)
		}
	}
	if len(lines) > n {
		lines = lines[len(lines)-n:]
	}
	return strings.Join(lines, " | ")
}

func num(v any) (float64, bool) {
	switch x := v.(type) {
	case float64:
		return x, true
	case float32:
		return float64(x), true
	case int:
		return float64(x), true
	case int64:
		return float64(x), true
	case json.Number:
		f, err := x.Float64()
		return f, err == nil
	}
	return 0, false
}