diff --git a/scrape.go b/scrape.go index ed5cadf..985176a 100644 --- a/scrape.go +++ b/scrape.go @@ -126,21 +126,32 @@ func ScrapeAd(fetch *Fetcher, uri string) error { advertisement.CalculateExpire() - proceed := CheckAdVisited(fetch.Config, advertisement.Slug) + // prepare ad dir name + addir, err := AdDirName(fetch.Config, advertisement) + if err != nil { + return err + } + + proceed := CheckAdVisited(fetch.Config, addir) if !proceed { return nil } // write listing - addir, err := WriteAd(fetch.Config, advertisement) + err = WriteAd(fetch.Config, advertisement, addir) if err != nil { return err } + // tell the user slog.Debug("extracted ad listing", "ad", advertisement) + // stats fetch.Config.IncrAds() + // register for later checks + DirsVisited[addir] = 1 + return ScrapeImages(fetch, advertisement, addir) } diff --git a/store.go b/store.go index b47645c..f5878ff 100644 --- a/store.go +++ b/store.go @@ -72,19 +72,13 @@ func AdDirName(conf *Config, advertisement *Ad) (string, error) { return buf.String(), nil } -func WriteAd(conf *Config, advertisement *Ad) (string, error) { - // prepare ad dir name - addir, err := AdDirName(conf, advertisement) - if err != nil { - return "", err - } - +func WriteAd(conf *Config, advertisement *Ad, addir string) error { // prepare output dir dir := filepath.Join(conf.Outdir, addir) - err = Mkdir(dir) + err := Mkdir(dir) if err != nil { - return "", err + return err } // write ad file @@ -92,7 +86,7 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) { listingfd, err := os.Create(listingfile) if err != nil { - return "", fmt.Errorf("failed to create Adlisting.txt: %w", err) + return fmt.Errorf("failed to create Adlisting.txt: %w", err) } defer listingfd.Close() @@ -104,17 +98,17 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) { tmpl, err := tpl.New("adlisting").Parse(conf.Template) if err != nil { - return "", fmt.Errorf("failed to parse adlisting template: %w", err) + return fmt.Errorf("failed to parse adlisting template: %w", err) } err = tmpl.Execute(listingfd, advertisement) if err != nil { - return "", fmt.Errorf("failed to execute adlisting template: %w", err) + return fmt.Errorf("failed to execute adlisting template: %w", err) } slog.Info("wrote ad listing", "listingfile", listingfile) - return addir, nil + return nil } func WriteImage(filename string, reader *bytes.Reader) error { @@ -176,9 +170,6 @@ func CheckAdVisited(conf *Config, adname string) bool { return false } - // register - DirsVisited[adname] = 1 - // overwrite return true }