package downloader import ( "fmt" "io" "net/http" "os" "path/filepath" "time" "rssd/state" ) // DownloadJob carries all information needed for a single download. type DownloadJob struct { FeedURL string EnclosureURL string DestPath string } // Pool manages a fixed-size worker pool for downloading files. type Pool struct { poolSize int backoffBase time.Duration state *state.Manager } // NewPool creates a download pool with the given parameters. func NewPool(poolSize int, backoffBase time.Duration, sm *state.Manager) *Pool { return &Pool{poolSize: poolSize, backoffBase: backoffBase, state: sm} } // Run starts the worker goroutines. It returns immediately after spawning the workers. func (p *Pool) Run(jobs <-chan DownloadJob) { for i := 0; i < p.poolSize; i++ { go func() { for job := range jobs { p.worker(job) } }() } } func (p *Pool) worker(job DownloadJob) { fmt.Fprintf(os.Stderr, "rssd: worker started download %s\n", job.EnclosureURL) if err := p.download(job); err != nil { fmt.Fprintf(os.Stderr, "ERROR: download %s failed: %v\n", job.EnclosureURL, err) } else { fmt.Fprintf(os.Stderr, "OK: downloaded %s -> %s\n", job.EnclosureURL, job.DestPath) } fmt.Fprintf(os.Stderr, "rssd: worker finished download %s\n", job.EnclosureURL) } func (p *Pool) download(job DownloadJob) error { // Update state to downloading. if !p.state.UpdateJob(job.EnclosureURL, func(j *state.Job) { j.Status = state.StatusDownloading j.AttemptCount++ }) { return fmt.Errorf("job not found for %s", job.EnclosureURL) } if err := p.state.Save(); err != nil { return fmt.Errorf("save state (downloading): %w", err) } // Ensure output directory exists. if err := os.MkdirAll(filepath.Dir(job.DestPath), 0755); err != nil { p.markRetry(job.EnclosureURL, fmt.Errorf("create dest dir: %w", err)) return err } // Create destination file. dstFile, err := os.Create(job.DestPath) if err != nil { p.markRetry(job.EnclosureURL, fmt.Errorf("create dest file: %w", err)) return err } // Download with timeout. client := &http.Client{Timeout: 30 * time.Minute} req, err := http.NewRequest("GET", job.EnclosureURL, nil) if err != nil { dstFile.Close() os.Remove(job.DestPath) p.markRetry(job.EnclosureURL, fmt.Errorf("create request: %w", err)) return err } resp, err := client.Do(req) if err != nil { dstFile.Close() os.Remove(job.DestPath) p.markRetry(job.EnclosureURL, fmt.Errorf("fetch: %w", err)) return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { dstFile.Close() os.Remove(job.DestPath) p.markRetry(job.EnclosureURL, fmt.Errorf("HTTP %d", resp.StatusCode)) return fmt.Errorf("HTTP %d", resp.StatusCode) } // Copy body to file. if _, err := io.Copy(dstFile, resp.Body); err != nil { dstFile.Close() os.Remove(job.DestPath) p.markRetry(job.EnclosureURL, fmt.Errorf("write file: %w", err)) return err } if err := dstFile.Close(); err != nil { os.Remove(job.DestPath) p.markRetry(job.EnclosureURL, fmt.Errorf("close file: %w", err)) return err } // Mark as downloaded. p.state.UpdateJob(job.EnclosureURL, func(j *state.Job) { j.Status = state.StatusDownloaded }) if err := p.state.Save(); err != nil { return fmt.Errorf("save state (downloaded): %w", err) } return nil } // markRetry updates a job to retrying status with the given error and computes backoff delay. func (p *Pool) markRetry(enclosureURL string, err error) { p.state.UpdateJob(enclosureURL, func(j *state.Job) { j.Status = state.StatusRetrying j.Error = err.Error() // Exponential backoff: base * 2^attemptCount (attempt already incremented). delay := p.backoffBase * time.Duration(1<