gogobee/cmd/fetch-esteemed/main.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/joho/godotenv"
)

type Entry struct {
	ID          string `json:"id"`
	Name        string `json:"name"`
	ImageURL    string `json:"image_url"`
	SearchQuery string `json:"search_query,omitempty"`
}

type WikiResponse struct {
	Query struct {
		Pages map[string]struct {
			Thumbnail struct {
				Source string `json:"source"`
			} `json:"thumbnail"`
		} `json:"pages"`
	} `json:"query"`
}

type SerpAPIResponse struct {
	ImagesResults []struct {
		Original  string `json:"original"`
		Thumbnail string `json:"thumbnail"`
	} `json:"images_results"`
}

func main() {
	const (
		esteemedPath = "esteemed.json"
		outputDir    = "data/esteemed"
		requestDelay = 500 * time.Millisecond
	)

	_ = godotenv.Load()
	serpAPIKey := os.Getenv("SERPAPI_KEY")

	client := &http.Client{
		Timeout: 30 * time.Second,
	}

	// Read esteemed.json
	data, err := os.ReadFile(esteemedPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to read %s: %v\n", esteemedPath, err)
		os.Exit(1)
	}

	var entries []Entry
	if err := json.Unmarshal(data, &entries); err != nil {
		fmt.Fprintf(os.Stderr, "Failed to parse JSON: %v\n", err)
		os.Exit(1)
	}

	// Create output directory
	if err := os.MkdirAll(outputDir, 0o755); err != nil {
		fmt.Fprintf(os.Stderr, "Failed to create output directory: %v\n", err)
		os.Exit(1)
	}

	if serpAPIKey != "" {
		fmt.Println("SerpAPI key found — will use as fallback for missing Wikipedia images")
	} else {
		fmt.Println("No SERPAPI_KEY set — Wikipedia only")
	}

	fmt.Printf("Processing %d entries...\n\n", len(entries))

	downloaded := 0
	skipped := 0
	failed := 0

	for i, entry := range entries {
		// Validate entry ID to prevent path traversal or command injection.
		if entry.ID == "" || strings.ContainsAny(entry.ID, "/\\;|&$`\"'") || strings.Contains(entry.ID, "..") {
			fmt.Printf("[%d/%d] SKIP (invalid ID): %q\n", i+1, len(entries), entry.ID)
			skipped++
			continue
		}
		outPath := filepath.Join(outputDir, entry.ID+".jpg")

		// Skip if already downloaded
		if _, err := os.Stat(outPath); err == nil {
			fmt.Printf("[%d/%d] SKIP (exists): %s\n", i+1, len(entries), entry.Name)
			skipped++
			continue
		}

		if entry.ImageURL == "" {
			fmt.Printf("[%d/%d] SKIP (no URL): %s\n", i+1, len(entries), entry.Name)
			skipped++
			continue
		}

		var imageURL string

		// Try Wikipedia first
		if isWikipediaURL(entry.ImageURL) {
			pageName := extractPageName(entry.ImageURL)
			if pageName != "" {
				var err error
				imageURL, err = fetchWikipediaImage(client, pageName)
				if err != nil {
					fmt.Printf("[%d/%d] WARN (wiki API): %s - %v\n", i+1, len(entries), entry.Name, err)
				}
			}
		} else {
			imageURL = entry.ImageURL
		}

		// Fallback to SerpAPI if no Wikipedia image
		if imageURL == "" && serpAPIKey != "" {
			// Use explicit search_query if provided, else derive from Wikipedia page name
			searchQuery := entry.Name
			if entry.SearchQuery != "" {
				searchQuery = entry.SearchQuery
			} else if isWikipediaURL(entry.ImageURL) {
				if pn := extractPageName(entry.ImageURL); pn != "" {
					q := strings.ReplaceAll(pn, "_", " ")
					q = strings.ReplaceAll(q, "(", "")
					q = strings.ReplaceAll(q, ")", "")
					q = strings.TrimSpace(q)
					if q != "" {
						searchQuery = q
					}
				}
			}
			var err error
			imageURL, err = fetchSerpAPIImage(client, serpAPIKey, searchQuery)
			if err != nil {
				fmt.Printf("[%d/%d] WARN (serpapi): %s - %v\n", i+1, len(entries), entry.Name, err)
			}
			if imageURL != "" {
				fmt.Printf("[%d/%d] SerpAPI fallback: %s\n", i+1, len(entries), entry.Name)
			}
		}

		if imageURL == "" {
			fmt.Printf("[%d/%d] FAIL (no image found): %s\n", i+1, len(entries), entry.Name)
			failed++
			time.Sleep(requestDelay)
			continue
		}

		if err := downloadImage(client, imageURL, outPath); err != nil {
			fmt.Printf("[%d/%d] FAIL (download): %s - %v\n", i+1, len(entries), entry.Name, err)
			failed++
		} else {
			// Convert non-JPEG files (WebP, PNG, etc.) to JPEG via ImageMagick
			convertToJPEG(outPath)
			fmt.Printf("[%d/%d] OK: %s\n", i+1, len(entries), entry.Name)
			downloaded++
		}

		time.Sleep(requestDelay)
	}

	fmt.Printf("\nDone. Downloaded: %d, Skipped: %d, Failed: %d\n", downloaded, skipped, failed)
}

func isWikipediaURL(rawURL string) bool {
	return strings.Contains(rawURL, "wikipedia.org/wiki/")
}

func extractPageName(wikiURL string) string {
	parsed, err := url.Parse(wikiURL)
	if err != nil {
		return ""
	}
	const prefix = "/wiki/"
	if !strings.HasPrefix(parsed.Path, prefix) {
		return ""
	}
	return parsed.Path[len(prefix):]
}

func fetchWikipediaImage(client *http.Client, pageName string) (string, error) {
	apiURL := fmt.Sprintf(
		"https://en.wikipedia.org/w/api.php?action=query&titles=%s&prop=pageimages&format=json&pithumbsize=400",
		url.PathEscape(pageName),
	)

	req, err := http.NewRequest("GET", apiURL, nil)
	if err != nil {
		return "", fmt.Errorf("creating request: %w", err)
	}
	req.Header.Set("User-Agent", "gogobee-fetch-esteemed/1.0 (bot; educational project)")

	resp, err := client.Do(req)
	if err != nil {
		return "", fmt.Errorf("API request: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("API returned status %d", resp.StatusCode)
	}

	var wikiResp WikiResponse
	if err := json.NewDecoder(resp.Body).Decode(&wikiResp); err != nil {
		return "", fmt.Errorf("decoding response: %w", err)
	}

	for _, page := range wikiResp.Query.Pages {
		if page.Thumbnail.Source != "" {
			return page.Thumbnail.Source, nil
		}
	}

	return "", nil
}

func fetchSerpAPIImage(client *http.Client, apiKey, query string) (string, error) {
	params := url.Values{
		"engine":  {"google_images"},
		"q":       {query},
		"api_key": {apiKey},
		"num":     {"1"},
	}
	apiURL := "https://serpapi.com/search.json?" + params.Encode()

	req, err := http.NewRequest("GET", apiURL, nil)
	if err != nil {
		return "", fmt.Errorf("creating request: %w", err)
	}

	resp, err := client.Do(req)
	if err != nil {
		return "", fmt.Errorf("API request: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("API returned status %d", resp.StatusCode)
	}

	var serpResp SerpAPIResponse
	if err := json.NewDecoder(resp.Body).Decode(&serpResp); err != nil {
		return "", fmt.Errorf("decoding response: %w", err)
	}

	if len(serpResp.ImagesResults) == 0 {
		return "", nil
	}

	// Prefer the original full-size image, fall back to thumbnail
	result := serpResp.ImagesResults[0]
	if result.Original != "" {
		return result.Original, nil
	}
	return result.Thumbnail, nil
}

func downloadImage(client *http.Client, imageURL, outPath string) error {
	req, err := http.NewRequest("GET", imageURL, nil)
	if err != nil {
		return fmt.Errorf("creating request: %w", err)
	}
	req.Header.Set("User-Agent", "gogobee-fetch-esteemed/1.0 (bot; educational project)")

	resp, err := client.Do(req)
	if err != nil {
		return fmt.Errorf("downloading: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("download returned status %d", resp.StatusCode)
	}

	// Write to a temp file first, then rename for atomicity
	tmpPath := outPath + ".tmp"
	f, err := os.Create(tmpPath)
	if err != nil {
		return fmt.Errorf("creating file: %w", err)
	}

	_, err = io.Copy(f, resp.Body)
	f.Close()
	if err != nil {
		os.Remove(tmpPath)
		return fmt.Errorf("writing file: %w", err)
	}

	if err := os.Rename(tmpPath, outPath); err != nil {
		os.Remove(tmpPath)
		return fmt.Errorf("renaming file: %w", err)
	}

	return nil
}

// convertToJPEG checks if the file is actually a JPEG; if not (e.g. WebP, PNG),
// it converts it to JPEG via ImageMagick, flattening any alpha channel.
func convertToJPEG(path string) {
	f, err := os.Open(path)
	if err != nil {
		return
	}

	// Read first 3 bytes to check for JPEG magic bytes (FF D8 FF)
	header := make([]byte, 3)
	n, _ := f.Read(header)
	f.Close()

	if n >= 3 && header[0] == 0xFF && header[1] == 0xD8 && header[2] == 0xFF {
		return // already JPEG
	}

	// Convert via ImageMagick
	tmpPath := path + ".conv.jpg"
	cmd := exec.Command("convert", path, "-background", "white", "-flatten", tmpPath)
	if err := cmd.Run(); err != nil {
		fmt.Fprintf(os.Stderr, "  convert failed for %s: %v\n", filepath.Base(path), err)
		os.Remove(tmpPath)
		return
	}

	if err := os.Rename(tmpPath, path); err != nil {
		fmt.Fprintf(os.Stderr, "  rename failed for %s: %v\n", filepath.Base(path), err)
		os.Remove(tmpPath)
		return
	}

	fmt.Printf("  converted %s to JPEG\n", filepath.Base(path))
}