Initial commit.
This commit is contained in:
commit
6f188d1939
64
cmd/debug/main.go
Normal file
64
cmd/debug/main.go
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
func main() {
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Get("https://feeds.transistor.fm/oxide-and-friends")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error fetching: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error reading body: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
parser := gofeed.NewParser()
|
||||
feed, err := parser.ParseString(string(body))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error parsing feed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("Feed title: %s\n", feed.Title)
|
||||
fmt.Printf("Total items: %d\n\n", len(feed.Items))
|
||||
|
||||
for i, item := range feed.Items {
|
||||
if i >= 5 {
|
||||
break
|
||||
}
|
||||
fmt.Printf("--- Item %d ---\n", i+1)
|
||||
fmt.Printf(" Title: %s\n", item.Title)
|
||||
fmt.Printf(" Published (raw): %s\n", item.Published)
|
||||
if item.PublishedParsed != nil {
|
||||
fmt.Printf(" PublishedParsed: %v\n", *item.PublishedParsed)
|
||||
} else {
|
||||
fmt.Printf(" PublishedParsed: <nil>\n")
|
||||
}
|
||||
|
||||
// Check for itunes:pubDate or other date fields
|
||||
for _, enc := range item.Enclosures {
|
||||
fmt.Printf(" Enclosure URL (truncated): %s...\n", enc.URL[:min(len(enc.URL), 80)])
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
147
config/config.go
Normal file
147
config/config.go
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// ExpandPath resolves a path that may start with ~ to an absolute path.
|
||||
func ExpandPath(p string) string {
|
||||
if strings.HasPrefix(p, "~") {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return p // fall back to literal on error
|
||||
}
|
||||
return filepath.Join(home, strings.TrimPrefix(p, "~"))
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// expandPath resolves a path that may start with ~ to an absolute path.
|
||||
func expandPath(p string) string {
|
||||
return ExpandPath(p)
|
||||
}
|
||||
|
||||
// Feed represents a single podcast feed to monitor.
|
||||
type Feed struct {
|
||||
URL string `yaml:"url"`
|
||||
OutputDir string `yaml:"output_dir"`
|
||||
PollInterval *time.Duration `yaml:"poll_interval,omitempty"` // nil means use global default
|
||||
StartDate string `yaml:"start_date,omitempty"` // optional, format "2006-01-02"
|
||||
}
|
||||
|
||||
// PublishedAt returns the parsed start date for this feed, or zero time if not set.
|
||||
func (f Feed) PublishedAt() time.Time {
|
||||
if f.StartDate == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
t, err := time.Parse("2006-01-02", f.StartDate)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// Settings holds global defaults for the daemon. Zero values are replaced by ApplyDefaults.
|
||||
type Settings struct {
|
||||
PollInterval time.Duration `yaml:"poll_interval"`
|
||||
MaxRetries int `yaml:"max_retries"`
|
||||
BackoffBase time.Duration `yaml:"backoff_base"`
|
||||
DownloadPoolSize int `yaml:"download_pool_size"`
|
||||
}
|
||||
|
||||
// Config is the top-level parsed configuration.
|
||||
type Config struct {
|
||||
Settings Settings `yaml:"settings,omitempty"`
|
||||
Feeds []Feed `yaml:"feeds"`
|
||||
}
|
||||
|
||||
// ApplyDefaults fills in zero-valued settings with sensible defaults and expands ~ paths.
|
||||
func (c *Config) ApplyDefaults() {
|
||||
if c.Settings.PollInterval == 0 {
|
||||
c.Settings.PollInterval = 6 * time.Hour
|
||||
}
|
||||
if c.Settings.MaxRetries == 0 {
|
||||
c.Settings.MaxRetries = 5
|
||||
}
|
||||
if c.Settings.BackoffBase == 0 {
|
||||
c.Settings.BackoffBase = 5 * time.Minute
|
||||
}
|
||||
if c.Settings.DownloadPoolSize == 0 {
|
||||
c.Settings.DownloadPoolSize = 4
|
||||
}
|
||||
|
||||
for i := range c.Feeds {
|
||||
c.Feeds[i].OutputDir = expandPath(c.Feeds[i].OutputDir)
|
||||
}
|
||||
}
|
||||
|
||||
// PollIntervalFor returns the effective poll interval for a feed, falling back to global default.
|
||||
func (c *Config) PollIntervalFor(f Feed) time.Duration {
|
||||
if f.PollInterval != nil && *f.PollInterval > 0 {
|
||||
return *f.PollInterval
|
||||
}
|
||||
return c.Settings.PollInterval
|
||||
}
|
||||
|
||||
// Validate checks that the config is usable.
|
||||
func (c *Config) Validate() error {
|
||||
if len(c.Feeds) == 0 {
|
||||
return fmt.Errorf("no feeds defined")
|
||||
}
|
||||
for i, f := range c.Feeds {
|
||||
if f.URL == "" {
|
||||
return fmt.Errorf("feed[%d]: url is required", i)
|
||||
}
|
||||
if f.OutputDir == "" {
|
||||
return fmt.Errorf("feed[%d] (%s): output_dir is required", i, f.URL)
|
||||
}
|
||||
if c.Settings.PollInterval <= 0 {
|
||||
return fmt.Errorf("settings.poll_interval must be > 0")
|
||||
}
|
||||
if c.Settings.MaxRetries == 0 {
|
||||
return fmt.Errorf("settings.max_retries must be > 0")
|
||||
}
|
||||
if c.Settings.BackoffBase <= 0 {
|
||||
return fmt.Errorf("settings.backoff_base must be > 0")
|
||||
}
|
||||
if c.Settings.DownloadPoolSize == 0 {
|
||||
return fmt.Errorf("settings.download_pool_size must be > 0")
|
||||
}
|
||||
// Validate start_date format if provided.
|
||||
if f.StartDate != "" {
|
||||
if _, err := time.Parse("2006-01-02", f.StartDate); err != nil {
|
||||
return fmt.Errorf("feed[%d] (%s): invalid start_date %q: %w", i, f.URL, f.StartDate, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Load reads and parses a YAML config from the given path.
|
||||
func Load(path string) (*Config, error) {
|
||||
path = expandPath(path)
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
|
||||
var cfg Config
|
||||
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("parse config: %w", err)
|
||||
}
|
||||
|
||||
cfg.ApplyDefaults()
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("validate config: %w", err)
|
||||
}
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
140
downloader/downloader.go
Normal file
140
downloader/downloader.go
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
package downloader
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"rssd/state"
|
||||
)
|
||||
|
||||
// DownloadJob carries all information needed for a single download.
|
||||
type DownloadJob struct {
|
||||
FeedURL string
|
||||
EnclosureURL string
|
||||
DestPath string
|
||||
}
|
||||
|
||||
// Pool manages a fixed-size worker pool for downloading files.
|
||||
type Pool struct {
|
||||
poolSize int
|
||||
backoffBase time.Duration
|
||||
state *state.Manager
|
||||
}
|
||||
|
||||
// NewPool creates a download pool with the given parameters.
|
||||
func NewPool(poolSize int, backoffBase time.Duration, sm *state.Manager) *Pool {
|
||||
return &Pool{poolSize: poolSize, backoffBase: backoffBase, state: sm}
|
||||
}
|
||||
|
||||
// Run starts the worker goroutines. It returns immediately after spawning the workers.
|
||||
func (p *Pool) Run(jobs <-chan DownloadJob) {
|
||||
for i := 0; i < p.poolSize; i++ {
|
||||
go func() {
|
||||
for job := range jobs {
|
||||
p.worker(job)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Pool) worker(job DownloadJob) {
|
||||
fmt.Fprintf(os.Stderr, "rssd: worker started download %s\n", job.EnclosureURL)
|
||||
if err := p.download(job); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "ERROR: download %s failed: %v\n", job.EnclosureURL, err)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "OK: downloaded %s -> %s\n", job.EnclosureURL, job.DestPath)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "rssd: worker finished download %s\n", job.EnclosureURL)
|
||||
}
|
||||
|
||||
func (p *Pool) download(job DownloadJob) error {
|
||||
// Update state to downloading.
|
||||
if !p.state.UpdateJob(job.EnclosureURL, func(j *state.Job) {
|
||||
j.Status = state.StatusDownloading
|
||||
j.AttemptCount++
|
||||
}) {
|
||||
return fmt.Errorf("job not found for %s", job.EnclosureURL)
|
||||
}
|
||||
if err := p.state.Save(); err != nil {
|
||||
return fmt.Errorf("save state (downloading): %w", err)
|
||||
}
|
||||
|
||||
// Ensure output directory exists.
|
||||
if err := os.MkdirAll(filepath.Dir(job.DestPath), 0755); err != nil {
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("create dest dir: %w", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// Create destination file.
|
||||
dstFile, err := os.Create(job.DestPath)
|
||||
if err != nil {
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("create dest file: %w", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// Download with timeout.
|
||||
client := &http.Client{Timeout: 30 * time.Minute}
|
||||
req, err := http.NewRequest("GET", job.EnclosureURL, nil)
|
||||
if err != nil {
|
||||
dstFile.Close()
|
||||
os.Remove(job.DestPath)
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("create request: %w", err))
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
dstFile.Close()
|
||||
os.Remove(job.DestPath)
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("fetch: %w", err))
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
dstFile.Close()
|
||||
os.Remove(job.DestPath)
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("HTTP %d", resp.StatusCode))
|
||||
return fmt.Errorf("HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Copy body to file.
|
||||
if _, err := io.Copy(dstFile, resp.Body); err != nil {
|
||||
dstFile.Close()
|
||||
os.Remove(job.DestPath)
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("write file: %w", err))
|
||||
return err
|
||||
}
|
||||
if err := dstFile.Close(); err != nil {
|
||||
os.Remove(job.DestPath)
|
||||
p.markRetry(job.EnclosureURL, fmt.Errorf("close file: %w", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// Mark as downloaded.
|
||||
p.state.UpdateJob(job.EnclosureURL, func(j *state.Job) {
|
||||
j.Status = state.StatusDownloaded
|
||||
})
|
||||
if err := p.state.Save(); err != nil {
|
||||
return fmt.Errorf("save state (downloaded): %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// markRetry updates a job to retrying status with the given error and computes backoff delay.
|
||||
func (p *Pool) markRetry(enclosureURL string, err error) {
|
||||
p.state.UpdateJob(enclosureURL, func(j *state.Job) {
|
||||
j.Status = state.StatusRetrying
|
||||
j.Error = err.Error()
|
||||
// Exponential backoff: base * 2^attemptCount (attempt already incremented).
|
||||
delay := p.backoffBase * time.Duration(1<<uint(j.AttemptCount-1))
|
||||
t := time.Now().Add(delay)
|
||||
j.NextAttemptAt = &t
|
||||
})
|
||||
p.state.Save()
|
||||
}
|
||||
16
go.mod
Normal file
16
go.mod
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
module rssd
|
||||
|
||||
go 1.24.2
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/mmcdole/gofeed v1.3.0 // indirect
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
golang.org/x/net v0.4.0 // indirect
|
||||
golang.org/x/text v0.5.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
34
go.sum
Normal file
34
go.sum
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
|
||||
github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
|
||||
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
|
||||
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
266
main.go
Normal file
266
main.go
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"rssd/config"
|
||||
"rssd/downloader"
|
||||
"rssd/poller"
|
||||
"rssd/state"
|
||||
)
|
||||
|
||||
const (
|
||||
stateFile = "state.json"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Fprintf(os.Stderr, "rssd: starting\n")
|
||||
|
||||
// Load configuration.
|
||||
cfgPath := os.Getenv("RSSD_CONFIG")
|
||||
if cfgPath == "" {
|
||||
cfgPath = "~/.config/rssd/config.yaml"
|
||||
}
|
||||
cfgPath = config.ExpandPath(cfgPath) // resolve ~ early for consistent path usage
|
||||
cfg, err := config.Load(cfgPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "rssd: fatal: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Load state — store alongside the config file.
|
||||
stateDir := filepath.Dir(cfgPath)
|
||||
sm, err := state.NewManager(filepath.Join(stateDir, stateFile))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "rssd: fatal: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := sm.Load(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "rssd: warning: could not load state (%v), starting fresh\n", err)
|
||||
}
|
||||
|
||||
// Start download worker pool.
|
||||
jobs := make(chan downloader.DownloadJob)
|
||||
pool := downloader.NewPool(cfg.Settings.DownloadPoolSize, cfg.Settings.BackoffBase, sm)
|
||||
go pool.Run(jobs)
|
||||
|
||||
// Re-enqueue any "pending" jobs from a previous run that were never picked up by workers.
|
||||
sm.ForEachJob(func(j *state.JobSnapshot) {
|
||||
if j.Status == state.StatusPending {
|
||||
fmt.Fprintf(os.Stderr, "rssd: re-enqueued pending job %s\n", j.EnclosureURL)
|
||||
jobs <- downloader.DownloadJob{
|
||||
FeedURL: j.FeedURL,
|
||||
EnclosureURL: j.EnclosureURL,
|
||||
DestPath: j.DestPath,
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
fmt.Fprintf(os.Stderr, "rssd: workers started, polling feeds\n")
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
|
||||
|
||||
fmt.Fprintf(os.Stderr, "rssd: %d feeds configured, polling started\n", len(cfg.Feeds))
|
||||
|
||||
// Launch a poller goroutine per feed.
|
||||
for _, feed := range cfg.Feeds {
|
||||
go pollFeed(feed, sm, jobs, cfg)
|
||||
}
|
||||
|
||||
// Periodically dispatch retrying jobs whose backoff has elapsed.
|
||||
retryTicker := time.NewTicker(1 * time.Minute)
|
||||
defer retryTicker.Stop()
|
||||
go func() {
|
||||
for range retryTicker.C {
|
||||
dispatchRetries(sm, jobs, cfg.Settings.MaxRetries)
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for shutdown signal.
|
||||
sig := <-sigCh
|
||||
fmt.Fprintf(os.Stderr, "rssd: received %v, shutting down...\n", sig)
|
||||
|
||||
// Close jobs channel to stop accepting new work. Workers will finish in-flight downloads
|
||||
// but we don't wait — just exit and let the next run handle any remaining state.
|
||||
close(jobs)
|
||||
|
||||
// Save final state before exiting so no progress is lost.
|
||||
if err := sm.Save(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "rssd: error saving final state: %v\n", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "rssd: stopped\n")
|
||||
}
|
||||
|
||||
func pollFeed(feed config.Feed, sm *state.Manager, jobs chan<- downloader.DownloadJob, cfg *config.Config) {
|
||||
fetcher := poller.NewFetcher()
|
||||
interval := cfg.PollIntervalFor(feed)
|
||||
|
||||
fmt.Fprintf(os.Stderr, "rssd: polling %s every %s\n", feed.URL, interval)
|
||||
|
||||
// Poll immediately on startup.
|
||||
pollAndEnqueue(feed, sm, jobs, fetcher, cfg)
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
pollAndEnqueue(feed, sm, jobs, fetcher, cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func pollAndEnqueue(feed config.Feed, sm *state.Manager, jobs chan<- downloader.DownloadJob, fetcher *poller.Fetcher, cfg *config.Config) {
|
||||
feedState := sm.FindFeedByURL(feed.URL)
|
||||
var lastModified, etag string
|
||||
if feedState != nil {
|
||||
lastModified = feedState.LastModified
|
||||
etag = feedState.ETag
|
||||
}
|
||||
|
||||
result, err := fetcher.Poll(feed.URL, lastModified, etag)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "rssd: error polling %s: %v\n", feed.URL, err)
|
||||
return
|
||||
}
|
||||
if result.NotModified {
|
||||
fmt.Fprintf(os.Stderr, "rssd: %s unchanged (skipping)\n", feed.URL)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "rssd: %s polled — %d enclosures found\n", feed.URL, len(result.NewEnclosures))
|
||||
|
||||
if len(result.NewEnclosures) == 0 {
|
||||
fmt.Fprintf(os.Stderr, "rssd: warning: %s has no enclosures in latest items\n", feed.URL)
|
||||
}
|
||||
|
||||
// Save updated feed metadata for next conditional request.
|
||||
sm.UpsertFeed(feed.URL, result.LastModified, result.ETag)
|
||||
|
||||
// Enqueue new enclosures (filtered by start_date).
|
||||
startDate := feed.PublishedAt()
|
||||
for _, encInfo := range result.NewEnclosures {
|
||||
// Skip items older than the configured start date.
|
||||
if !startDate.IsZero() && (!encInfo.PublishedAt.IsZero() && encInfo.PublishedAt.Before(startDate)) {
|
||||
fmt.Fprintf(os.Stderr, "rssd: skipping %s (published %s, before start_date %s)\n",
|
||||
encInfo.URL,
|
||||
encInfo.PublishedAt.Format("2006-01-02"),
|
||||
feed.StartDate)
|
||||
continue
|
||||
}
|
||||
|
||||
destPath := computeDestPath(feed.OutputDir, encInfo)
|
||||
if sm.AddJob(feed.URL, encInfo.URL, destPath, cfg.Settings.MaxRetries) {
|
||||
fmt.Fprintf(os.Stderr, "rssd: enqueued %s -> %s\n", encInfo.URL, destPath)
|
||||
jobs <- downloader.DownloadJob{
|
||||
FeedURL: feed.URL,
|
||||
EnclosureURL: encInfo.URL,
|
||||
DestPath: destPath,
|
||||
}
|
||||
} else {
|
||||
job := sm.FindJobByURL(encInfo.URL)
|
||||
if job != nil {
|
||||
fmt.Fprintf(os.Stderr, "rssd: skipping %s (already %s)\n", encInfo.URL, job.Status)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "rssd: skipping duplicate %s\n", encInfo.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := sm.Save(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "rssd: error saving state: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// dispatchRetries scans all jobs in state and re-enqueues those whose backoff has elapsed.
|
||||
func dispatchRetries(sm *state.Manager, jobs chan<- downloader.DownloadJob, maxRetries int) {
|
||||
// Collect snapshots (lock released before any callback invocation).
|
||||
var ready []downloader.DownloadJob
|
||||
var failed []string
|
||||
|
||||
sm.ForEachJob(func(snap *state.JobSnapshot) {
|
||||
if snap.Status != state.StatusRetrying {
|
||||
return
|
||||
}
|
||||
|
||||
if snap.AttemptCount >= maxRetries {
|
||||
// Permanently failed — use UpdateJob to mutate the live state.
|
||||
sm.UpdateJob(snap.EnclosureURL, func(j *state.Job) {
|
||||
j.Status = state.StatusFailed
|
||||
})
|
||||
failed = append(failed, fmt.Sprintf("%s (after %d attempts: %s)",
|
||||
snap.EnclosureURL, snap.AttemptCount, snap.Error))
|
||||
return
|
||||
}
|
||||
|
||||
if snap.NextAttemptAt == nil || time.Now().After(*snap.NextAttemptAt) {
|
||||
// Re-enqueue — use UpdateJob to mutate the live state.
|
||||
sm.UpdateJob(snap.EnclosureURL, func(j *state.Job) {
|
||||
j.Status = state.StatusPending
|
||||
j.NextAttemptAt = nil
|
||||
})
|
||||
ready = append(ready, downloader.DownloadJob{
|
||||
FeedURL: snap.FeedURL,
|
||||
EnclosureURL: snap.EnclosureURL,
|
||||
DestPath: snap.DestPath,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if len(failed) > 0 {
|
||||
for _, msg := range failed {
|
||||
fmt.Fprintf(os.Stderr, "rssd: %s permanently failed\n", msg)
|
||||
}
|
||||
}
|
||||
|
||||
// Send to channel and persist state (lock released).
|
||||
for _, job := range ready {
|
||||
jobs <- job
|
||||
}
|
||||
|
||||
if err := sm.Save(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "rssd: error saving state (retry dispatch): %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// sanitizeFilename replaces characters that are invalid or problematic in filenames.
|
||||
func sanitizeFilename(s string) string {
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "<", "_", ">", "_", "|", "_", "\"", "_", "?", "_", "*", "_",
|
||||
)
|
||||
s = replacer.Replace(s)
|
||||
for strings.Contains(s, "__") {
|
||||
s = strings.ReplaceAll(s, "__", "_")
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// computeDestPath constructs the local file path from an enclosure URL and output directory.
|
||||
// The filename is prefixed with YYYYMMDD - for chronological sorting when dates are available.
|
||||
func computeDestPath(outputDir string, encInfo poller.EnclosureInfo) string {
|
||||
baseName := sanitizeFilename(encInfo.Title)
|
||||
if baseName == "" {
|
||||
// Fallback to enclosure URL basename if title is empty or invalid.
|
||||
parsed, err := url.Parse(encInfo.URL)
|
||||
if err != nil || parsed.Path == "" || parsed.Path == "/" {
|
||||
return filepath.Join(outputDir, strings.ReplaceAll(encInfo.URL, "/", "_"))
|
||||
}
|
||||
baseName = sanitizeFilename(filepath.Base(parsed.Path))
|
||||
if baseName == "." || baseName == "" {
|
||||
baseName = "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// Prepend date prefix for chronological sorting if available.
|
||||
if !encInfo.PublishedAt.IsZero() {
|
||||
datePrefix := encInfo.PublishedAt.Format("2006-01-02")
|
||||
return filepath.Join(outputDir, fmt.Sprintf("%s - %s", datePrefix, baseName))
|
||||
}
|
||||
|
||||
return filepath.Join(outputDir, baseName)
|
||||
}
|
||||
118
poller/poller.go
Normal file
118
poller/poller.go
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
package poller
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
// EnclosureInfo carries an enclosure URL and its associated item's metadata.
|
||||
type EnclosureInfo struct {
|
||||
URL string
|
||||
Title string // episode title, if available
|
||||
PublishedAt time.Time // zero value means no date available
|
||||
}
|
||||
|
||||
// FeedResult holds the result of polling a single feed.
|
||||
type FeedResult struct {
|
||||
URL string
|
||||
NewEnclosures []EnclosureInfo
|
||||
LastModified string
|
||||
ETag string
|
||||
NotModified bool
|
||||
Error error
|
||||
}
|
||||
|
||||
// Fetcher abstracts the HTTP fetch + parse cycle for a single feed.
|
||||
type Fetcher struct {
|
||||
httpClient *http.Client
|
||||
parser *gofeed.Parser
|
||||
}
|
||||
|
||||
// NewFetcher creates a new Fetcher with sensible defaults.
|
||||
func NewFetcher() *Fetcher {
|
||||
return &Fetcher{
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
parser: gofeed.NewParser(),
|
||||
}
|
||||
}
|
||||
|
||||
// Poll fetches and parses an RSS/Atom feed, returning new enclosure info.
|
||||
// lastModified and etag are used for conditional requests (If-None-Match / If-Modified-Since).
|
||||
// Returns a FeedResult with NewEnclosures populated only if the feed changed.
|
||||
func (f *Fetcher) Poll(url, lastModified, etag string) (*FeedResult, error) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
|
||||
if etag != "" {
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
if lastModified != "" {
|
||||
req.Header.Set("If-Modified-Since", lastModified)
|
||||
}
|
||||
|
||||
resp, err := f.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch %s: %w", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Handle 304 Not Modified — feed hasn't changed.
|
||||
if resp.StatusCode == http.StatusNotModified {
|
||||
return &FeedResult{
|
||||
URL: url,
|
||||
NotModified: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("fetch %s: HTTP %d", url, resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body %s: %w", url, err)
|
||||
}
|
||||
|
||||
feed, err := f.parser.ParseString(string(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse feed %s: %w", url, err)
|
||||
}
|
||||
|
||||
// Extract enclosure info from feed items.
|
||||
var enclosures []EnclosureInfo
|
||||
for _, item := range feed.Items {
|
||||
for _, enc := range item.Enclosures {
|
||||
if enc.URL != "" {
|
||||
encInfo := EnclosureInfo{URL: enc.URL, Title: item.Title}
|
||||
if item.PublishedParsed != nil {
|
||||
encInfo.PublishedAt = *item.PublishedParsed
|
||||
}
|
||||
enclosures = append(enclosures, encInfo)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result := &FeedResult{
|
||||
URL: url,
|
||||
NewEnclosures: enclosures,
|
||||
NotModified: false,
|
||||
}
|
||||
|
||||
// Capture cache headers for the next conditional request.
|
||||
if v := resp.Header.Get("Last-Modified"); v != "" {
|
||||
result.LastModified = v
|
||||
}
|
||||
if v := resp.Header.Get("ETag"); v != "" {
|
||||
result.ETag = v
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
272
state/state.go
Normal file
272
state/state.go
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
package state
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// State is the full persisted state of the daemon.
|
||||
type State struct {
|
||||
Feeds []FeedState `json:"feeds"`
|
||||
Jobs []Job `json:"jobs"`
|
||||
}
|
||||
|
||||
// FeedState holds per-feed metadata from HTTP responses.
|
||||
type FeedState struct {
|
||||
URL string `json:"url"`
|
||||
LastModified string `json:"last_modified,omitempty"`
|
||||
ETag string `json:"etag,omitempty"`
|
||||
}
|
||||
|
||||
// Job represents a download job with retry state.
|
||||
type Job struct {
|
||||
FeedURL string `json:"feed_url"`
|
||||
EnclosureURL string `json:"enclosure_url"`
|
||||
DestPath string `json:"dest_path"`
|
||||
Status string `json:"status"`
|
||||
AttemptCount int `json:"attempt_count"`
|
||||
MaxRetries int `json:"max_retries"`
|
||||
NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"` // nil means ready now
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// JobStatus values.
|
||||
const (
|
||||
StatusPending = "pending"
|
||||
StatusDownloading = "downloading"
|
||||
StatusRetrying = "retrying"
|
||||
StatusDownloaded = "downloaded"
|
||||
StatusFailed = "failed"
|
||||
)
|
||||
|
||||
// Manager provides thread-safe access to persisted state.
|
||||
type Manager struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
state *State
|
||||
}
|
||||
|
||||
// NewManager creates a new state manager pointing at the given file path.
|
||||
func NewManager(path string) (*Manager, error) {
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("create state directory: %w", err)
|
||||
}
|
||||
return &Manager{path: path, state: &State{}}, nil
|
||||
}
|
||||
|
||||
// Load reads the state file from disk. If it doesn't exist or is empty, returns an empty state.
|
||||
func (m *Manager) Load() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
data, err := os.ReadFile(m.path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
m.state = &State{}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("read state: %w", err)
|
||||
}
|
||||
|
||||
if len(data) == 0 {
|
||||
m.state = &State{}
|
||||
return nil
|
||||
}
|
||||
|
||||
var s State
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
return fmt.Errorf("parse state: %w", err)
|
||||
}
|
||||
|
||||
m.state = &s
|
||||
return nil
|
||||
}
|
||||
|
||||
// Save atomically writes the current state to disk with pretty-printed JSON.
|
||||
func (m *Manager) Save() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
data, err := json.MarshalIndent(m.state, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal state: %w", err)
|
||||
}
|
||||
|
||||
dir := filepath.Dir(m.path)
|
||||
tmpFile, err := os.CreateTemp(dir, ".rssd-state-*.tmp")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create temp file: %w", err)
|
||||
}
|
||||
tmpName := tmpFile.Name()
|
||||
|
||||
if _, err := tmpFile.Write(data); err != nil {
|
||||
tmpFile.Close()
|
||||
os.Remove(tmpName)
|
||||
return fmt.Errorf("write temp state: %w", err)
|
||||
}
|
||||
if err := tmpFile.Close(); err != nil {
|
||||
os.Remove(tmpName)
|
||||
return fmt.Errorf("close temp state: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmpName, m.path); err != nil {
|
||||
os.Remove(tmpName)
|
||||
return fmt.Errorf("rename state file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// JobSnapshot is a copy of a Job suitable for read-only inspection.
|
||||
type JobSnapshot struct {
|
||||
FeedURL string
|
||||
EnclosureURL string
|
||||
DestPath string
|
||||
Status string
|
||||
AttemptCount int
|
||||
MaxRetries int
|
||||
NextAttemptAt *time.Time // nil means ready now
|
||||
Error string
|
||||
}
|
||||
|
||||
// ForEachJob iterates over all jobs and calls fn for each one.
|
||||
// The callback receives a pointer to a copy of the job (not the live state),
|
||||
// so it must not modify fields. Use UpdateJob or AddJob for mutations.
|
||||
// fn may perform I/O without risk of deadlock — the lock is released before
|
||||
// any callback invocation.
|
||||
func (m *Manager) ForEachJob(fn func(*JobSnapshot)) {
|
||||
m.mu.Lock()
|
||||
jobsCopy := make([]JobSnapshot, len(m.state.Jobs))
|
||||
for i := range m.state.Jobs {
|
||||
j := &m.state.Jobs[i]
|
||||
jobsCopy[i] = JobSnapshot{
|
||||
FeedURL: j.FeedURL,
|
||||
EnclosureURL: j.EnclosureURL,
|
||||
DestPath: j.DestPath,
|
||||
Status: j.Status,
|
||||
AttemptCount: j.AttemptCount,
|
||||
MaxRetries: j.MaxRetries,
|
||||
NextAttemptAt: j.NextAttemptAt,
|
||||
Error: j.Error,
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
for i := range jobsCopy {
|
||||
fn(&jobsCopy[i])
|
||||
}
|
||||
}
|
||||
|
||||
// State returns a copy of the current in-memory state for iteration.
|
||||
func (m *Manager) State() *State {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.state
|
||||
}
|
||||
|
||||
// AddJob adds a new download job if one with this enclosure URL doesn't already exist.
|
||||
// Returns true if the job was added, false if it already existed.
|
||||
func (m *Manager) AddJob(feedURL, enclosureURL, destPath string, maxRetries int) bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// Check if this enclosure URL already has a job.
|
||||
for _, j := range m.state.Jobs {
|
||||
if j.EnclosureURL == enclosureURL {
|
||||
return false // already exists
|
||||
}
|
||||
}
|
||||
|
||||
job := Job{
|
||||
FeedURL: feedURL,
|
||||
EnclosureURL: enclosureURL,
|
||||
DestPath: destPath,
|
||||
Status: StatusPending,
|
||||
AttemptCount: 0,
|
||||
MaxRetries: maxRetries,
|
||||
NextAttemptAt: nil,
|
||||
Error: "",
|
||||
}
|
||||
|
||||
m.state.Jobs = append(m.state.Jobs, job)
|
||||
return true
|
||||
}
|
||||
|
||||
// UpdateJob finds a job by enclosure URL and applies the update function.
|
||||
// Returns true if the job was found and updated.
|
||||
func (m *Manager) UpdateJob(enclosureURL string, fn func(*Job)) bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for i := range m.state.Jobs {
|
||||
if m.state.Jobs[i].EnclosureURL == enclosureURL {
|
||||
fn(&m.state.Jobs[i])
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FindJobByURL returns a pointer to the job for the given enclosure URL, or nil.
|
||||
func (m *Manager) FindJobByURL(enclosureURL string) *Job {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for i := range m.state.Jobs {
|
||||
if m.state.Jobs[i].EnclosureURL == enclosureURL {
|
||||
return &m.state.Jobs[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasJob returns true if a job with the given enclosure URL exists.
|
||||
func (m *Manager) HasJob(enclosureURL string) bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for _, j := range m.state.Jobs {
|
||||
if j.EnclosureURL == enclosureURL {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FindFeedByURL returns the FeedState for a given feed URL, or nil.
|
||||
func (m *Manager) FindFeedByURL(feedURL string) *FeedState {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for i := range m.state.Feeds {
|
||||
if m.state.Feeds[i].URL == feedURL {
|
||||
return &m.state.Feeds[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpsertFeed adds or updates a feed's metadata (ETag, Last-Modified).
|
||||
func (m *Manager) UpsertFeed(feedURL, lastModified, etag string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for i := range m.state.Feeds {
|
||||
if m.state.Feeds[i].URL == feedURL {
|
||||
m.state.Feeds[i].LastModified = lastModified
|
||||
m.state.Feeds[i].ETag = etag
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
m.state.Feeds = append(m.state.Feeds, FeedState{
|
||||
URL: feedURL,
|
||||
LastModified: lastModified,
|
||||
ETag: etag,
|
||||
})
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user