Add eBay Browse API integration with daily call quota

eBay marketplaces are now polled through eBay's official Buy > Browse API (client-credentials OAuth2) instead of an Apify scraper actor; Apify still handles Yahoo JP and Mercari. Browse API calls are tracked per day in a new ebay_api_usage table and capped (default 5000, configurable) on eBay's Pacific-time reset clock, so polling halts before the limit is hit. Credentials live in config.toml [ebay] and are overridable via /settings, which also surfaces the day's running call count. Also carries the server.secure_cookies config plumbing (field, accessor, example) consumed by the following commit. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-14 12:10:39 -07:00
parent cfa01bd4ef
commit 1ae2c50b9a
12 changed files with 1092 additions and 262 deletions
--- a/internal/scheduler/scheduler.go
+++ b/internal/scheduler/scheduler.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -13,14 +14,23 @@ import (
 	"veola/internal/apify"
 	"veola/internal/config"
 	"veola/internal/db"
+	"veola/internal/ebay"
 	"veola/internal/models"
 	"veola/internal/ntfy"
 )

+// Provider labels distinguish how a plan is executed: through an Apify actor
+// run, or through eBay's official Browse API.
+const (
+	providerApify = "apify"
+	providerEbay  = "ebay"
+)
+
 type Scheduler struct {
 	cfg      *config.Config
 	store    *db.Store
 	apify    *apify.Client
+	ebay     *ebay.Client
 	ntfy     *ntfy.Client
 	cron     *cron.Cron

@@ -37,6 +47,7 @@ func New(cfg *config.Config, store *db.Store, ap *apify.Client, nt *ntfy.Client)
 		cfg:     cfg,
 		store:   store,
 		apify:   ap,
+		ebay:    ebay.New(cfg.Ebay.ClientID, cfg.Ebay.ClientSecret, cfg.Ebay.Environment),
 		ntfy:    nt,
 		cron:    cron.New(),
 		entries: make(map[int64]cron.EntryID),
@@ -136,52 +147,16 @@ func (s *Scheduler) RunPoll(ctx context.Context, it models.Item) {
 	var errs []string
 	successes := 0
 	for _, p := range plans {
-		if p.actorID == "" {
-			errs = append(errs, fmt.Sprintf("%s: no actor configured", p.marketplace))
-			continue
-		}
-		raw, err := apifyClient.Run(ctx, p.actorID, p.input)
+		decoded, err := s.ExecutePlan(ctx, p)
 		if err != nil {
 			label := p.marketplace
 			if p.query != "" {
 				label = fmt.Sprintf("query %q on %s", p.query, p.marketplace)
 			}
 			errs = append(errs, fmt.Sprintf("%s: %s", label, err.Error()))
-			slog.Error("apify run failed", "item_id", it.ID, "marketplace", p.marketplace, "query", p.query, "err", err)
+			slog.Error("plan failed", "item_id", it.ID, "provider", p.provider, "marketplace", p.marketplace, "query", p.query, "err", err)
 			continue
 		}
-		decoded, _ := apify.Decode(raw, p.source)
-		usable := 0
-		for i := range decoded {
-			decoded[i].MatchedQuery = p.query
-			if decoded[i].URL != "" && decoded[i].Price > 0 {
-				usable++
-			}
-		}
-		slog.Info("apify run decoded",
-			"item_id", it.ID,
-			"marketplace", p.marketplace,
-			"query", p.query,
-			"actor", p.actorID,
-			"raw", len(raw),
-			"decoded", len(decoded),
-			"usable", usable,
-		)
-		if usable == 0 && len(raw) > 0 {
-			var sample map[string]any
-			if err := jsonUnmarshal(raw[0], &sample); err == nil {
-				keys := make([]string, 0, len(sample))
-				for k := range sample {
-					keys = append(keys, k)
-				}
-				slog.Warn("decoded zero usable rows; raw item keys",
-					"item_id", it.ID,
-					"marketplace", p.marketplace,
-					"actor", p.actorID,
-					"keys", keys,
-				)
-			}
-		}
 		results = append(results, decoded...)
 		successes++
 	}
@@ -322,6 +297,103 @@ func (s *Scheduler) apifyClient(ctx context.Context) *apify.Client {
 	return apify.New(key)
 }

+// ebayClient returns the shared eBay client with credentials refreshed from
+// settings (falling back to config.toml). The client caches its OAuth token
+// in memory, so the same instance is reused across polls; credentials are
+// only re-applied when they actually change.
+func (s *Scheduler) ebayClient(ctx context.Context) *ebay.Client {
+	id := s.cfg.Ebay.ClientID
+	secret := s.cfg.Ebay.ClientSecret
+	if v, _ := s.store.GetSetting(ctx, "ebay_client_id"); v != "" {
+		id = v
+	}
+	if v, _ := s.store.GetSetting(ctx, "ebay_client_secret"); v != "" {
+		secret = v
+	}
+	s.ebay.EnsureCredentials(id, secret)
+	return s.ebay
+}
+
+// EbayUsage returns the number of eBay Browse API calls made so far today and
+// the configured daily limit. A limit <= 0 means uncapped. Settings override
+// config.toml for the limit, mirroring how credentials are resolved.
+func (s *Scheduler) EbayUsage(ctx context.Context) (used, limit int) {
+	used, _ = s.store.EbayUsageToday(ctx)
+	limit = s.cfg.Ebay.DailyCallLimit
+	if v, _ := s.store.GetSetting(ctx, "ebay_daily_call_limit"); v != "" {
+		if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
+			limit = n
+		}
+	}
+	return used, limit
+}
+
+// ExecutePlan runs one plan and returns decoded, provider-agnostic results
+// with MatchedQuery already stamped. eBay plans go through the official
+// Browse API; all other plans run an Apify actor. Callers handle per-plan
+// errors without poisoning sibling plans.
+func (s *Scheduler) ExecutePlan(ctx context.Context, p actorPlan) ([]apify.UnifiedResult, error) {
+	var decoded []apify.UnifiedResult
+	switch p.provider {
+	case providerEbay:
+		sp, ok := p.input.(ebay.SearchParams)
+		if !ok {
+			return nil, fmt.Errorf("ebay plan has wrong input type %T", p.input)
+		}
+		used, limit := s.EbayUsage(ctx)
+		if limit > 0 && used >= limit {
+			return nil, fmt.Errorf("ebay daily API call limit reached (%d/%d); polling halted until the next reset (midnight US Pacific)", used, limit)
+		}
+		listings, err := s.ebayClient(ctx).Search(ctx, sp)
+		// The call hit eBay (or at least was attempted against it) whether
+		// or not it succeeded, so it counts against the daily allowance.
+		if n, incErr := s.store.IncrementEbayUsage(ctx); incErr != nil {
+			slog.Error("ebay usage increment failed", "err", incErr)
+		} else if limit > 0 && n >= limit {
+			slog.Warn("ebay daily API call limit reached", "used", n, "limit", limit)
+		}
+		if err != nil {
+			return nil, err
+		}
+		decoded = make([]apify.UnifiedResult, 0, len(listings))
+		for _, l := range listings {
+			decoded = append(decoded, apify.UnifiedResult{
+				Title:    l.Title,
+				Price:    l.Price,
+				Currency: l.Currency,
+				URL:      l.URL,
+				Store:    l.Store,
+				ImageURL: l.ImageURL,
+				Source:   apify.SourceActiveEbay,
+			})
+		}
+	default:
+		if p.actorID == "" {
+			return nil, fmt.Errorf("no actor configured for %s", p.marketplace)
+		}
+		raw, err := s.apifyClient(ctx).Run(ctx, p.actorID, p.input)
+		if err != nil {
+			return nil, err
+		}
+		decoded, _ = apify.Decode(raw, p.source)
+	}
+	usable := 0
+	for i := range decoded {
+		decoded[i].MatchedQuery = p.query
+		if decoded[i].URL != "" && decoded[i].Price > 0 {
+			usable++
+		}
+	}
+	slog.Info("plan executed",
+		"provider", p.provider,
+		"marketplace", p.marketplace,
+		"query", p.query,
+		"decoded", len(decoded),
+		"usable", usable,
+	)
+	return decoded, nil
+}
+
 func (s *Scheduler) sendNotification(ctx context.Context, it models.Item, r apify.UnifiedResult) error {
 	tags := []string{"mag"}
 	if it.TargetPrice != nil && r.Price <= *it.TargetPrice {
@@ -398,6 +470,7 @@ func (s *Scheduler) BuildPreviewInputs(it models.Item) []actorPlan {
 type actorPlan struct {
 	marketplace string
 	source      string
+	provider    string
 	actorID     string
 	query       string
 	input       any
@@ -418,6 +491,9 @@ func (p actorPlan) Query() string { return p.query }
 // Input returns the actor input payload as expected by apify.Client.Run.
 func (p actorPlan) Input() any { return p.input }

+// Provider returns "apify" or "ebay" — how this plan is executed.
+func (p actorPlan) Provider() string { return p.provider }
+
 // buildAllInputs returns one actor plan per (alias × marketplace) for the item.
 // For URL-only items (no aliases), produces one plan per marketplace with an
 // empty query string.
@@ -447,27 +523,51 @@ func (s *Scheduler) buildInputsForQuery(it models.Item, query string, markets []
 		switch {
 		case strings.Contains(mk, "yahoo") || strings.Contains(url, "yahoo.co.jp"):
 			actorID := firstNonEmpty(it.ActorActive, s.cfg.Apify.Actors.YahooAuctionsJP)
-			plans = append(plans, actorPlan{m, apify.SourceYahooJP, actorID, query, apify.YahooAuctionsJPInput{
-				SearchTerm: query,
-				MaxPages:   1,
-			}})
+			plans = append(plans, actorPlan{
+				marketplace: m, source: apify.SourceYahooJP, provider: providerApify,
+				actorID: actorID, query: query,
+				input: apify.YahooAuctionsJPInput{SearchTerm: query, MaxPages: 1},
+			})
 		case strings.Contains(mk, "mercari") || strings.Contains(url, "mercari"):
 			actorID := firstNonEmpty(it.ActorActive, s.cfg.Apify.Actors.MercariJP)
-			plans = append(plans, actorPlan{m, apify.SourceMercariJP, actorID, query, apify.MercariJPInput{
-				SearchKeywords: []string{query},
-				Status:         "on_sale",
-				MaxResults:     30,
-			}})
+			plans = append(plans, actorPlan{
+				marketplace: m, source: apify.SourceMercariJP, provider: providerApify,
+				actorID: actorID, query: query,
+				input: apify.MercariJPInput{
+					SearchKeywords: []string{query},
+					Status:         "on_sale",
+					MaxResults:     30,
+				},
+			})
+		case ebay.IsEbayMarketplace(mk):
+			// eBay marketplaces are polled through eBay's official Browse
+			// API, not an Apify scraper actor.
+			plans = append(plans, actorPlan{
+				marketplace: m, source: apify.SourceActiveEbay, provider: providerEbay,
+				query: query,
+				input: ebay.SearchParams{
+					MarketplaceID: ebay.MarketplaceID(mk),
+					Query:         query,
+					ListingType:   it.ListingType,
+					Limit:         30,
+				},
+			})
 		default:
+			// Non-eBay custom marketplaces still fall back to the Apify
+			// active-listings actor.
 			actorID := firstNonEmpty(it.ActorActive, s.cfg.Apify.Actors.ActiveListings)
-			plans = append(plans, actorPlan{m, apify.SourceActiveEbay, actorID, query, apify.ActiveListingInput{
-				SearchQueries:        []string{query},
-				MaxProductsPerSearch: 30,
-				MaxSearchPages:       1,
-				Sort:                 "best_match",
-				ListingType:          mapListingType(it.ListingType),
-				ProxyConfiguration:   s.proxyConfig(),
-			}})
+			plans = append(plans, actorPlan{
+				marketplace: m, source: apify.SourceActiveEbay, provider: providerApify,
+				actorID: actorID, query: query,
+				input: apify.ActiveListingInput{
+					SearchQueries:        []string{query},
+					MaxProductsPerSearch: 30,
+					MaxSearchPages:       1,
+					Sort:                 "best_match",
+					ListingType:          mapListingType(it.ListingType),
+					ProxyConfiguration:   s.proxyConfig(),
+				},
+			})
 		}
 	}
 	return plans