veola/internal/apify/types.go

package apify

import (
	"encoding/json"
	"log/slog"
	"strconv"
	"strings"
	"time"
)

// jstLocation is the timezone ZenMarket reports Yahoo Auctions JP end times
// in when the value lacks an explicit offset. Falls back to UTC if the
// embedded tzdata lookup fails (main.go imports time/tzdata so in practice
// this always resolves).
var jstLocation = func() *time.Location {
	loc, err := time.LoadLocation("Asia/Tokyo")
	if err != nil {
		return time.UTC
	}
	return loc
}()

// yahooEndingDateLayouts is the ordered list of layouts attempted when
// parsing ZenMarket's `ending_date`. Some entries use Local-as-JST (no
// offset in the string) — see parseYahooEndingDate for the routing.
var yahooEndingDateLayouts = []struct {
	layout string
	jst    bool // true: parse as JST; false: parse with offset/zone from string
}{
	{time.RFC3339, false},
	{"2006-01-02T15:04:05", true},
	{"2006-01-02 15:04:05", true},
	{"2006/01/02 15:04:05", true},
	{"2006/01/02 15:04", true},
	{"2006-01-02 15:04", true},
}

// parseYahooEndingDate is a permissive parser for ZenMarket's ending_date
// field. The actor's output format has shifted over time and isn't documented;
// rather than wedging on one layout, try the known shapes in order and treat
// anything without an explicit zone as JST (Yahoo Auctions runs in Japan).
// Returns nil + logs a warning when nothing parses, so operators see the raw
// value and can extend this list.
func parseYahooEndingDate(s string) *time.Time {
	s = strings.TrimSpace(s)
	if s == "" {
		return nil
	}
	for _, l := range yahooEndingDateLayouts {
		var (
			t   time.Time
			err error
		)
		if l.jst {
			t, err = time.ParseInLocation(l.layout, s, jstLocation)
		} else {
			t, err = time.Parse(l.layout, s)
		}
		if err == nil {
			slog.Debug("yahoo ending_date parsed",
				"raw", s, "layout", l.layout, "jst_assumed", l.jst, "parsed", t.UTC().Format(time.RFC3339))
			return &t
		}
	}
	slog.Warn("yahoo ending_date: no layout matched", "raw", s)
	return nil
}

// ActiveListingInput is the input schema for `automation-lab/ebay-scraper`.
// The actor accepts keyword searches and standard filters; it targets
// ebay.com only (no per-marketplace routing in the actor itself), so
// non-US marketplaces won't return useful results with this actor.
type ActiveListingInput struct {
	SearchQueries        []string            `json:"searchQueries"`
	MaxProductsPerSearch int                 `json:"maxProductsPerSearch,omitempty"`
	MaxSearchPages       int                 `json:"maxSearchPages,omitempty"`
	Sort                 string              `json:"sort,omitempty"`
	ListingType          string              `json:"listingType,omitempty"`
	Condition            []string            `json:"condition,omitempty"`
	MinPrice             *int                `json:"minPrice,omitempty"`
	MaxPrice             *int                `json:"maxPrice,omitempty"`
	ProxyConfiguration   *ProxyConfiguration `json:"proxyConfiguration,omitempty"`
}

// ProxyConfiguration is the standard apify input block for proxy routing.
// eBay (and most retail sites) return 403 to datacenter IPs; passing
// {"useApifyProxy": true, "apifyProxyGroups": ["RESIDENTIAL"]} works.
type ProxyConfiguration struct {
	UseApifyProxy     bool     `json:"useApifyProxy"`
	ApifyProxyGroups  []string `json:"apifyProxyGroups,omitempty"`
	ApifyProxyCountry string   `json:"apifyProxyCountry,omitempty"`
}

// ActiveListingResult is decoded leniently to handle multiple eBay-scraper
// actors. delicious_zebu/ebay-product-listing-scraper returns productUrl /
// imageUrl / numeric price; harvestlab/ebay-scraper used url / price /
// currency. The decoder coalesces both shapes.
type ActiveListingResult struct {
	Title           string  `json:"title"`
	Price           any     `json:"price"`
	OriginalPrice   any     `json:"originalPrice"`
	Currency        string  `json:"currency"`
	URL             string   `json:"url"`
	ProductURL      string   `json:"productUrl"`
	Store           string   `json:"store"`
	ImageURL        string   `json:"imageUrl"`
	Image           string   `json:"image"`
	Thumbnail       string   `json:"thumbnail"`
	Images          []string `json:"images"`
	Condition       string  `json:"condition"`
	ListingType     string  `json:"listingType"`
	ShippingCost    any     `json:"shippingCost"`
	ShippingPrice   any     `json:"shippingPrice"`
	FreeShipping    bool    `json:"freeShipping"`
	Marketplace     string  `json:"marketplace"`
	MatchConfidence float64 `json:"matchConfidence"`
	Availability    string  `json:"availability"`
	WatchersCount  int      `json:"watchersCount"`
	QuantitySold   int      `json:"quantitySold"`
}

type SoldListingInput struct {
	Query              string              `json:"query"`
	Marketplace        string              `json:"marketplace,omitempty"`
	MaxResults         int                 `json:"maxResults,omitempty"`
	DaysBack           int                 `json:"daysBack,omitempty"`
	ProxyConfiguration *ProxyConfiguration `json:"proxyConfiguration,omitempty"`
}

type SoldListingResult struct {
	Title         string  `json:"title"`
	SoldPrice     float64 `json:"soldPrice"`
	Currency      string  `json:"soldCurrency"`
	SoldAt        string  `json:"endedAt"`
	Condition     string  `json:"condition"`
	ListingType   string  `json:"listingType"`
	ShippingPrice float64 `json:"shippingPrice"`
	URL           string  `json:"url"`
}

type PriceComparisonInput struct {
	Query              string              `json:"query,omitempty"`
	URL                string              `json:"url,omitempty"`
	MatchStrictness    string              `json:"matchStrictness,omitempty"`
	ProxyConfiguration *ProxyConfiguration `json:"proxyConfiguration,omitempty"`
}

type PriceComparisonResult struct {
	Title           string  `json:"title"`
	Price           float64 `json:"price"`
	Currency        string  `json:"currency"`
	URL             string  `json:"url"`
	Store           string  `json:"store"`
	ImageURL        string  `json:"imageUrl"`
	Availability    string  `json:"availability"`
	MatchConfidence float64 `json:"matchConfidence"`
}

// YahooAuctionsJPInput targets meron1122/zenmarket-scraper. ZenMarket is a
// buyer-proxy for Yahoo Auctions JP; its scraper returns ZenMarket-proxied
// listing URLs and USD-converted prices.
type YahooAuctionsJPInput struct {
	SearchTerm        string `json:"searchTerm"`
	CategoryID        string `json:"categoryID,omitempty"`
	MaxPages          int    `json:"maxPages,omitempty"`
	MaxRemainingHours int    `json:"maxRemainingHours,omitempty"`
}

// MercariJPInput targets cloud9_ai/mercari-scraper. The actor manages its
// own proxy (Japan datacenter with residential fallback), so we do not send
// a proxyConfiguration block.
type MercariJPInput struct {
	SearchKeywords []string `json:"searchKeywords,omitempty"`
	ProductUrls    []string `json:"productUrls,omitempty"`
	Status         string   `json:"status,omitempty"`
	SortBy         string   `json:"sortBy,omitempty"`
	PriceMin       *int     `json:"priceMin,omitempty"`
	PriceMax       *int     `json:"priceMax,omitempty"`
	ItemCondition  string   `json:"itemCondition,omitempty"`
	MaxResults     int      `json:"maxResults,omitempty"`
}

// YahooAuctionsJPResult matches meron1122/zenmarket-scraper output. Prices
// are USD-converted at the ZenMarket-published rate.
type YahooAuctionsJPResult struct {
	Name         string   `json:"name"`
	CurrentPrice any      `json:"current_price"`
	Photos       []string `json:"photos"`
	URL          string   `json:"url"`
	EndingDate   string   `json:"ending_date"`
}

// UnifiedResult is the common shape produced by ParseResults regardless of
// which actor type returned the data. The scheduler consumes this.
type UnifiedResult struct {
	Title           string
	Price           float64
	Currency        string
	URL             string
	Store           string
	ImageURL        string
	Source          string
	MatchConfidence float64
	OutOfStock      bool
	// MatchedQuery records which alias from the item's query list produced
	// this row. Empty for URL-only items or rows from non-search sources.
	MatchedQuery string
	// EndsAt is the auction end time, if known. Auction-format eBay listings
	// and Yahoo Auctions JP listings populate this; fixed-price listings
	// and most Apify scraper outputs leave it nil.
	EndsAt *time.Time
}

// Decode unmarshals a list of raw JSON items into UnifiedResult slices using
// the shape that matches the given source label.
func Decode(items []json.RawMessage, source string) ([]UnifiedResult, error) {
	out := make([]UnifiedResult, 0, len(items))
	switch source {
	case SourceActiveEbay, SourcePriceCompare:
		for _, raw := range items {
			var r ActiveListingResult
			if err := json.Unmarshal(raw, &r); err != nil {
				continue
			}
			url := r.URL
			if url == "" {
				url = r.ProductURL
			}
			img := r.ImageURL
			if img == "" {
				img = r.Image
			}
			if img == "" {
				img = r.Thumbnail
			}
			if img == "" && len(r.Images) > 0 {
				img = r.Images[0]
			}
			store := r.Store
			if store == "" {
				store = r.Marketplace
			}
			if store == "" && source == SourceActiveEbay {
				store = "ebay"
			}
			cur := r.Currency
			if cur == "" {
				cur = "USD"
			}
			out = append(out, UnifiedResult{
				Title:           r.Title,
				Price:           coercePrice(r.Price),
				Currency:        cur,
				URL:             url,
				Store:           store,
				ImageURL:        img,
				Source:          source,
				MatchConfidence: r.MatchConfidence,
				OutOfStock:      isOOS(r.Availability),
			})
		}
	case SourceYahooJP:
		for _, raw := range items {
			slog.Debug("yahoo raw item", "json", string(raw))
			var r YahooAuctionsJPResult
			if err := json.Unmarshal(raw, &r); err != nil {
				slog.Debug("yahoo decode failed", "err", err, "json", string(raw))
				continue
			}
			img := ""
			if len(r.Photos) > 0 {
				img = r.Photos[0]
			}
			out = append(out, UnifiedResult{
				Title:    r.Name,
				Price:    coercePrice(r.CurrentPrice),
				Currency: "USD",
				URL:      r.URL,
				Store:    "yahoo-auctions-jp (via zenmarket)",
				ImageURL: img,
				Source:   source,
				EndsAt:   parseYahooEndingDate(r.EndingDate),
			})
		}
	case SourceMercariJP:
		// Mercari actors vary in shape; accept either price/currentPrice and title/name.
		for _, raw := range items {
			var generic struct {
				Title        string  `json:"title"`
				Name         string  `json:"name"`
				Price        float64 `json:"price"`
				CurrentPrice float64 `json:"currentPrice"`
				Currency     string  `json:"currency"`
				URL          string  `json:"url"`
				ImageURL     string  `json:"imageUrl"`
				Image        string  `json:"image"`
				Status       string  `json:"status"`
			}
			if err := json.Unmarshal(raw, &generic); err != nil {
				continue
			}
			title := generic.Title
			if title == "" {
				title = generic.Name
			}
			price := generic.Price
			if price == 0 {
				price = generic.CurrentPrice
			}
			img := generic.ImageURL
			if img == "" {
				img = generic.Image
			}
			cur := generic.Currency
			if cur == "" {
				cur = "JPY"
			}
			out = append(out, UnifiedResult{
				Title:      title,
				Price:      price,
				Currency:   cur,
				URL:        generic.URL,
				Store:      "mercari-jp",
				ImageURL:   img,
				Source:     source,
				OutOfStock: isOOS(generic.Status),
			})
		}
	}
	return out, nil
}

const (
	SourceActiveEbay   = "ebay"
	SourcePriceCompare = "price-comparison"
	SourceYahooJP      = "yahoo-auctions-jp"
	SourceMercariJP    = "mercari-jp"
	SourceSoldEbay     = "ebay-sold"
	SourceSoldYahooJP  = "yahoo-auctions-jp-sold"
)

// coercePrice accepts a price field that might be a number or a string with
// currency symbols / commas (e.g. "$24.99", "1,299.00"). Returns 0 on failure
// so FilterResults can drop the row cleanly.
func coercePrice(v any) float64 {
	switch x := v.(type) {
	case nil:
		return 0
	case float64:
		return x
	case float32:
		return float64(x)
	case int:
		return float64(x)
	case int64:
		return float64(x)
	case string:
		s := strings.Map(func(r rune) rune {
			switch {
			case r >= '0' && r <= '9', r == '.', r == '-':
				return r
			}
			return -1
		}, x)
		f, err := strconv.ParseFloat(s, 64)
		if err != nil {
			return 0
		}
		return f
	}
	return 0
}

func isOOS(s string) bool {
	switch s {
	case "out_of_stock", "OUT_OF_STOCK", "sold", "SOLD", "ended":
		return true
	}
	return false
}