fix #77: use processed ad dir for duplicate checking, not slug

scrape.go

@@ -126,21 +126,32 @@ func ScrapeAd(fetch *Fetcher, uri string) error {
 
 	advertisement.CalculateExpire()
 
-	proceed := CheckAdVisited(fetch.Config, advertisement.Slug)
+	// prepare ad dir name
+	addir, err := AdDirName(fetch.Config, advertisement)
+	if err != nil {
+		return err
+	}
+
+	proceed := CheckAdVisited(fetch.Config, addir)
 	if !proceed {
 		return nil
 	}
 
 	// write listing
-	addir, err := WriteAd(fetch.Config, advertisement)
+	err = WriteAd(fetch.Config, advertisement, addir)
 	if err != nil {
 		return err
 	}
 
+	// tell the user
 	slog.Debug("extracted ad listing", "ad", advertisement)
 
+	// stats
 	fetch.Config.IncrAds()
 
+	// register for later checks
+	DirsVisited[addir] = 1
+
 	return ScrapeImages(fetch, advertisement, addir)
 }
 

store.go

@@ -72,27 +72,21 @@ func AdDirName(conf *Config, advertisement *Ad) (string, error) {
 	return buf.String(), nil
 }
 
-func WriteAd(conf *Config, advertisement *Ad) (string, error) {
-	// prepare ad dir name
-	addir, err := AdDirName(conf, advertisement)
-	if err != nil {
-		return "", err
-	}
-
+func WriteAd(conf *Config, advertisement *Ad, addir string) error {
 	// prepare output dir
 	dir := filepath.Join(conf.Outdir, addir)
 
-	err = Mkdir(dir)
+	err := Mkdir(dir)
 	if err != nil {
-		return "", err
+		return err
 	}
 
 	// write ad file
 	listingfile := filepath.Join(dir, "Adlisting.txt")
 
 	listingfd, err := os.Create(listingfile)
 	if err != nil {
-		return "", fmt.Errorf("failed to create Adlisting.txt: %w", err)
+		return fmt.Errorf("failed to create Adlisting.txt: %w", err)
 	}
 	defer listingfd.Close()
 
@@ -104,17 +98,17 @@ func WriteAd(conf *Config, advertisement *Ad) (string, error) {
 
 	tmpl, err := tpl.New("adlisting").Parse(conf.Template)
 	if err != nil {
-		return "", fmt.Errorf("failed to parse adlisting template: %w", err)
+		return fmt.Errorf("failed to parse adlisting template: %w", err)
 	}
 
 	err = tmpl.Execute(listingfd, advertisement)
 	if err != nil {
-		return "", fmt.Errorf("failed to execute adlisting template: %w", err)
+		return fmt.Errorf("failed to execute adlisting template: %w", err)
 	}
 
 	slog.Info("wrote ad listing", "listingfile", listingfile)
 
-	return addir, nil
+	return nil
 }
 
 func WriteImage(filename string, reader *bytes.Reader) error {
@@ -176,9 +170,6 @@ func CheckAdVisited(conf *Config, adname string) bool {
 		return false
 	}
 
-	// register
-	DirsVisited[adname] = 1
-
 	// overwrite
 	return true
 }