Greg Pomerantz
bc26e73a90
- ytdlp: thin wrapper around yt-dlp CLI for channel/playlist discovery (--flat-playlist) and per-video metadata (--print-json); defaults to player_client=web_embedded to avoid SABR-only format restrictions. - downloader: YouTube path extracts audio via yt-dlp -x, probes duration with ffprobe and rejects clips far shorter than the expected length. - main: YouTube channel/playlist feeds, incremental discovery with pre-start-date boundary for newest-first channels, two-phase record-then-enqueue so discovered jobs survive crashes. - poller: propagate enclosure MIME type; computeDestPath appends the matching file extension. - state: jobs carry expected duration; AddJobWithStatus for skipped items. - cmd/probe: dry-run discovery/date-resolution diagnostics.
121 lines
2.7 KiB
Go
121 lines
2.7 KiB
Go
// Command probe performs a dry run of the YouTube discovery + date-resolution
|
|
// logic for one or more channel/playlist URLs. It resolves every video's
|
|
// upload date and reports how many fall before/after a start date, without
|
|
// downloading anything.
|
|
//
|
|
// Usage:
|
|
//
|
|
// probe <start-date YYYY-MM-DD> <url> [url ...]
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"time"
|
|
|
|
"rssd/ytdlp"
|
|
)
|
|
|
|
func main() {
|
|
if len(os.Args) < 3 {
|
|
fmt.Fprintln(os.Stderr, "usage: probe <start-date YYYY-MM-DD> <url> [url ...]")
|
|
os.Exit(2)
|
|
}
|
|
start, err := time.Parse("2006-01-02", os.Args[1])
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "bad start date:", err)
|
|
os.Exit(2)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Hour)
|
|
defer cancel()
|
|
s := ytdlp.Settings{}
|
|
|
|
for _, u := range os.Args[2:] {
|
|
probeOne(ctx, s, u, start)
|
|
}
|
|
}
|
|
|
|
func probeOne(ctx context.Context, s ytdlp.Settings, url string, start time.Time) {
|
|
fmt.Printf("\n=== %s ===\n", url)
|
|
vids, err := ytdlp.Discovery(ctx, s, url)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, " discovery error: %v\n", err)
|
|
return
|
|
}
|
|
fmt.Printf(" discovered %d videos\n", len(vids))
|
|
|
|
newestFirst := true
|
|
for _, tok := range []string{"list="} {
|
|
if contains(url, tok) {
|
|
newestFirst = false
|
|
}
|
|
}
|
|
fmt.Printf(" assumed newest-first: %v\n", newestFirst)
|
|
|
|
total := 0
|
|
inRange, beforeStart, noDate, failed := 0, 0, 0, 0
|
|
var oldest, newest time.Time
|
|
for _, v := range vids {
|
|
total++
|
|
info, err := ytdlp.Info(ctx, s, v.ID)
|
|
if err != nil {
|
|
failed++
|
|
if failed <= 5 {
|
|
fmt.Fprintf(os.Stderr, " resolve warn %s: %v\n", v.ID, err)
|
|
}
|
|
continue
|
|
}
|
|
if info.UploadDate.IsZero() {
|
|
noDate++
|
|
continue
|
|
}
|
|
if oldest.IsZero() || info.UploadDate.Before(oldest) {
|
|
oldest = info.UploadDate
|
|
}
|
|
if info.UploadDate.After(newest) {
|
|
newest = info.UploadDate
|
|
}
|
|
if info.UploadDate.Before(start) {
|
|
beforeStart++
|
|
if newestFirst {
|
|
// Channel is newest-first: everything after this is older, so stop.
|
|
fmt.Printf(" reached pre-start-date boundary, stopping\n")
|
|
break
|
|
}
|
|
continue
|
|
}
|
|
inRange++
|
|
if inRange <= 5 {
|
|
fmt.Printf(" IN %s %s (%s)\n", info.UploadDate.Format("2006-01-02"), v.ID, trunc(v.Title, 60))
|
|
}
|
|
}
|
|
fmt.Printf(" resolved=%d inRange(>=%s)=%d beforeStart=%d noDate=%d failed=%d\n",
|
|
total, start.Format("2006-01-02"), inRange, beforeStart, noDate, failed)
|
|
fmt.Printf(" oldest=%s newest=%s\n", fmtDate(oldest), fmtDate(newest))
|
|
}
|
|
|
|
func contains(s, sub string) bool {
|
|
for i := 0; i+len(sub) <= len(s); i++ {
|
|
if s[i:i+len(sub)] == sub {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func trunc(s string, n int) string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
return s[:n] + "…"
|
|
}
|
|
|
|
func fmtDate(t time.Time) string {
|
|
if t.IsZero() {
|
|
return "-"
|
|
}
|
|
return t.Format("2006-01-02")
|
|
}
|