Add eBay Browse API integration with daily call quota

eBay marketplaces are now polled through eBay's official Buy > Browse API (client-credentials OAuth2) instead of an Apify scraper actor; Apify still handles Yahoo JP and Mercari. Browse API calls are tracked per day in a new ebay_api_usage table and capped (default 5000, configurable) on eBay's Pacific-time reset clock, so polling halts before the limit is hit. Credentials live in config.toml [ebay] and are overridable via /settings, which also surfaces the day's running call count.

Also carries the server.secure_cookies config plumbing (field, accessor, example) consumed by the following commit.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
prosolis
2026-05-14 12:10:39 -07:00
parent cfa01bd4ef
commit 1ae2c50b9a
12 changed files with 1092 additions and 262 deletions

View File

@@ -13,13 +13,43 @@ type Config struct {
Server ServerConfig `toml:"server"`
Security SecurityConfig `toml:"security"`
Apify ApifyConfig `toml:"apify"`
Ebay EbayConfig `toml:"ebay"`
Ntfy NtfyConfig `toml:"ntfy"`
Scheduler SchedulerConfig `toml:"scheduler"`
}
// EbayConfig holds credentials for eBay's official Buy > Browse API. When set,
// eBay marketplaces are polled through the Browse API instead of an Apify
// scraper actor. ClientID is the App ID and ClientSecret is the Cert ID from
// the eBay developer keyset. Environment is "production" (default) or
// "sandbox". Like the Apify key, both credentials can be overridden at
// runtime via the Settings page.
type EbayConfig struct {
ClientID string `toml:"client_id"`
ClientSecret string `toml:"client_secret"`
Environment string `toml:"environment"`
// DailyCallLimit caps Browse API calls per day, on eBay's own quota
// clock (midnight US Pacific). Once reached, eBay polling halts until
// the next reset. Defaults to 5000 (the standard Browse API allowance).
// Set to a negative value to disable the cap.
DailyCallLimit int `toml:"daily_call_limit"`
}
type ServerConfig struct {
Port int `toml:"port"`
DBPath string `toml:"db_path"`
// SecureCookies sets the Secure attribute on the session cookie. It must
// be true in any deployment reachable over HTTPS — including behind a
// TLS-terminating proxy like Traefik, where the browser-facing leg is
// HTTPS even though Veola itself speaks plain HTTP. Defaults to true;
// set false only for local non-TLS development.
SecureCookies *bool `toml:"secure_cookies"`
}
// UseSecureCookies resolves the SecureCookies setting, defaulting to true when
// the key is absent from config.
func (c ServerConfig) UseSecureCookies() bool {
return c.SecureCookies == nil || *c.SecureCookies
}
type SecurityConfig struct {
@@ -111,6 +141,9 @@ func (c *Config) validate() error {
if c.Ntfy.DefaultTopic == "" {
c.Ntfy.DefaultTopic = "veola"
}
if c.Ebay.DailyCallLimit == 0 {
c.Ebay.DailyCallLimit = 5000
}
return nil
}

View File

@@ -213,6 +213,60 @@ func (s *Store) SetSetting(ctx context.Context, key, value string) error {
return err
}
// ============ ebay api usage ============
// ebayResetLoc is the timezone eBay's API rate limits reset in: midnight
// Pacific (it observes US DST). If the zone database is somehow unavailable
// we fall back to UTC rather than failing — main.go embeds time/tzdata so in
// practice the lookup always succeeds.
var ebayResetLoc = func() *time.Location {
loc, err := time.LoadLocation("America/Los_Angeles")
if err != nil {
return time.UTC
}
return loc
}()
// ebayUsageDay returns the date key used to bucket eBay API calls, aligned to
// eBay's own quota reset (midnight US Pacific).
func ebayUsageDay() string {
return time.Now().In(ebayResetLoc).Format("2006-01-02")
}
// EbayUsageToday returns the number of eBay Browse API calls recorded for the
// current UTC day. A missing row counts as zero.
func (s *Store) EbayUsageToday(ctx context.Context) (int, error) {
var n int
err := s.DB.QueryRowContext(ctx,
`SELECT call_count FROM ebay_api_usage WHERE usage_date = ?`, ebayUsageDay()).Scan(&n)
if errors.Is(err, sql.ErrNoRows) {
return 0, nil
}
if err != nil {
return 0, err
}
return n, nil
}
// IncrementEbayUsage records one eBay Browse API call against the current UTC
// day and returns the new running total.
func (s *Store) IncrementEbayUsage(ctx context.Context) (int, error) {
day := ebayUsageDay()
_, err := s.DB.ExecContext(ctx, `
INSERT INTO ebay_api_usage (usage_date, call_count) VALUES (?, 1)
ON CONFLICT(usage_date) DO UPDATE SET call_count = call_count + 1
`, day)
if err != nil {
return 0, err
}
var n int
if err := s.DB.QueryRowContext(ctx,
`SELECT call_count FROM ebay_api_usage WHERE usage_date = ?`, day).Scan(&n); err != nil {
return 0, err
}
return n, nil
}
// ============ items ============
func (s *Store) CreateItem(ctx context.Context, it *models.Item) (int64, error) {

View File

@@ -90,6 +90,15 @@ INSERT OR IGNORE INTO settings (key, value) VALUES
('global_poll_interval_minutes', '60'),
('match_confidence_threshold', '0.6');
-- ebay_api_usage tracks Browse API calls per day so Veola can surface
-- consumption and halt polling before the developer keyset's daily call
-- limit is exceeded. usage_date is YYYY-MM-DD in US Pacific time, matching
-- eBay's own quota reset.
CREATE TABLE IF NOT EXISTS ebay_api_usage (
usage_date TEXT PRIMARY KEY,
call_count INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS sessions (
token TEXT PRIMARY KEY,
data BLOB NOT NULL,

227
internal/ebay/client.go Normal file
View File

@@ -0,0 +1,227 @@
// Package ebay is a thin client for eBay's official Buy > Browse API. It
// handles client-credentials OAuth2 token caching and active-listing search
// (item_summary/search). It deliberately covers only what Veola needs: a
// keyword search returning normalized listings. Sold/completed data (the
// Marketplace Insights API) is not implemented here.
package ebay
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
)
// endpoints bundles the production/sandbox base URLs for the two APIs used.
type endpoints struct {
oauth string
browse string
}
func endpointsFor(environment string) endpoints {
if strings.EqualFold(strings.TrimSpace(environment), "sandbox") {
return endpoints{
oauth: "https://api.sandbox.ebay.com/identity/v1/oauth2/token",
browse: "https://api.sandbox.ebay.com/buy/browse/v1",
}
}
return endpoints{
oauth: "https://api.ebay.com/identity/v1/oauth2/token",
browse: "https://api.ebay.com/buy/browse/v1",
}
}
// Client is safe for concurrent use. The application access token is cached
// in memory and refreshed shortly before it expires.
type Client struct {
HTTP *http.Client
mu sync.Mutex
clientID string
clientSecret string
ends endpoints
token string
tokenExpiry time.Time
}
// New builds a client for the given keyset. environment is "production"
// (default) or "sandbox". Credentials may be empty; calls then fail fast
// with a "not configured" error.
func New(clientID, clientSecret, environment string) *Client {
return &Client{
HTTP: &http.Client{Timeout: 30 * time.Second},
clientID: clientID,
clientSecret: clientSecret,
ends: endpointsFor(environment),
}
}
// EnsureCredentials updates the keyset if it changed, discarding any cached
// token so the next call re-authenticates. The environment is fixed at
// construction time and is not changed here. Safe to call on every poll.
func (c *Client) EnsureCredentials(clientID, clientSecret string) {
c.mu.Lock()
defer c.mu.Unlock()
if clientID == c.clientID && clientSecret == c.clientSecret {
return
}
c.clientID = clientID
c.clientSecret = clientSecret
c.token = ""
c.tokenExpiry = time.Time{}
}
func (c *Client) accessToken(ctx context.Context) (string, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.clientID == "" || c.clientSecret == "" {
return "", errors.New("ebay credentials not configured")
}
if c.token != "" && time.Now().Before(c.tokenExpiry) {
return c.token, nil
}
form := url.Values{}
form.Set("grant_type", "client_credentials")
form.Set("scope", "https://api.ebay.com/oauth/api_scope")
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.ends.oauth, strings.NewReader(form.Encode()))
if err != nil {
return "", err
}
basic := base64.StdEncoding.EncodeToString([]byte(c.clientID + ":" + c.clientSecret))
req.Header.Set("Authorization", "Basic "+basic)
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := c.HTTP.Do(req)
if err != nil {
return "", fmt.Errorf("ebay oauth: %w", err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
if resp.StatusCode >= 300 {
return "", fmt.Errorf("ebay oauth: http %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
var tr struct {
AccessToken string `json:"access_token"`
ExpiresIn int `json:"expires_in"`
TokenType string `json:"token_type"`
}
if err := json.Unmarshal(body, &tr); err != nil {
return "", fmt.Errorf("ebay oauth: decode: %w", err)
}
if tr.AccessToken == "" {
return "", errors.New("ebay oauth: empty access token")
}
c.token = tr.AccessToken
// Refresh a minute early to avoid racing the expiry.
ttl := time.Duration(tr.ExpiresIn) * time.Second
if ttl <= time.Minute {
ttl = time.Minute
}
c.tokenExpiry = time.Now().Add(ttl - time.Minute)
return c.token, nil
}
// browseItemSummary mirrors the subset of the item_summary/search response
// Veola consumes.
type browseItemSummary struct {
ItemID string `json:"itemId"`
Title string `json:"title"`
Price struct {
Value string `json:"value"`
Currency string `json:"currency"`
} `json:"price"`
ItemWebURL string `json:"itemWebUrl"`
Image struct {
ImageURL string `json:"imageUrl"`
} `json:"image"`
ThumbnailImages []struct {
ImageURL string `json:"imageUrl"`
} `json:"thumbnailImages"`
Seller struct {
Username string `json:"username"`
} `json:"seller"`
}
// Search runs one item_summary/search call and returns normalized listings.
// An empty query is rejected: the Browse API requires a non-empty q.
func (c *Client) Search(ctx context.Context, p SearchParams) ([]Listing, error) {
query := strings.TrimSpace(p.Query)
if query == "" {
return nil, errors.New("ebay search requires a non-empty query")
}
token, err := c.accessToken(ctx)
if err != nil {
return nil, err
}
marketplace := p.MarketplaceID
if marketplace == "" {
marketplace = "EBAY_US"
}
limit := p.Limit
if limit <= 0 || limit > 200 {
limit = 50
}
q := url.Values{}
q.Set("q", query)
q.Set("limit", strconv.Itoa(limit))
if f := buyingOptionsFilter(p.ListingType); f != "" {
q.Set("filter", f)
}
reqURL := c.ends.browse + "/item_summary/search?" + q.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("X-EBAY-C-MARKETPLACE-ID", marketplace)
req.Header.Set("Content-Type", "application/json")
resp, err := c.HTTP.Do(req)
if err != nil {
return nil, fmt.Errorf("ebay browse: %w", err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if resp.StatusCode >= 300 {
return nil, fmt.Errorf("ebay browse: http %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
var sr struct {
ItemSummaries []browseItemSummary `json:"itemSummaries"`
}
if err := json.Unmarshal(body, &sr); err != nil {
return nil, fmt.Errorf("ebay browse: decode: %w", err)
}
out := make([]Listing, 0, len(sr.ItemSummaries))
for _, s := range sr.ItemSummaries {
price, _ := strconv.ParseFloat(strings.TrimSpace(s.Price.Value), 64)
img := s.Image.ImageURL
if img == "" && len(s.ThumbnailImages) > 0 {
img = s.ThumbnailImages[0].ImageURL
}
store := "ebay"
if s.Seller.Username != "" {
store = "ebay (" + s.Seller.Username + ")"
}
out = append(out, Listing{
Title: s.Title,
Price: price,
Currency: s.Price.Currency,
URL: s.ItemWebURL,
Store: store,
ImageURL: img,
})
}
return out, nil
}

View File

@@ -0,0 +1,45 @@
package ebay
import "testing"
func TestMarketplaceID(t *testing.T) {
cases := map[string]string{
"ebay.com": "EBAY_US",
"ebay.co.uk": "EBAY_GB",
"ebay.de": "EBAY_DE",
"ebay.com.au": "EBAY_AU",
"EBAY.CA": "EBAY_CA",
"ebay": "EBAY_US",
"weird-market": "EBAY_US",
" ebay.it ": "EBAY_IT",
}
for in, want := range cases {
if got := MarketplaceID(in); got != want {
t.Errorf("MarketplaceID(%q) = %q, want %q", in, got, want)
}
}
}
func TestIsEbayMarketplace(t *testing.T) {
if !IsEbayMarketplace("ebay.co.uk") {
t.Error("ebay.co.uk should be an eBay marketplace")
}
if IsEbayMarketplace("yahoo-auctions-jp") {
t.Error("yahoo should not be an eBay marketplace")
}
}
func TestBuyingOptionsFilter(t *testing.T) {
cases := map[string]string{
"": "",
"all": "",
"bin": "buyingOptions:{FIXED_PRICE}",
"buy_it_now": "buyingOptions:{FIXED_PRICE}",
"auction": "buyingOptions:{AUCTION}",
}
for in, want := range cases {
if got := buyingOptionsFilter(in); got != want {
t.Errorf("buyingOptionsFilter(%q) = %q, want %q", in, got, want)
}
}
}

94
internal/ebay/types.go Normal file
View File

@@ -0,0 +1,94 @@
package ebay
import "strings"
// SearchParams is the input to a single Browse API item_summary/search call.
// It is provider-specific and is carried as the opaque input payload of a
// scheduler plan, mirroring how Apify actor inputs are carried.
type SearchParams struct {
// MarketplaceID is an eBay marketplace identifier such as EBAY_US.
MarketplaceID string
// Query is the keyword search string. Required; the Browse API rejects
// an empty q.
Query string
// ListingType is Veola's vocabulary ("all", "bin"/"buy_it_now",
// "auction"); it is mapped to a buyingOptions filter.
ListingType string
// Limit caps the number of results requested (Browse API max is 200).
Limit int
}
// Listing is one normalized active eBay listing. The scheduler converts these
// into the shared apify.UnifiedResult shape so the rest of the pipeline
// (dedup, filter, alert) is provider-agnostic.
type Listing struct {
Title string
Price float64
Currency string
URL string
Store string
ImageURL string
}
// MarketplaceID maps a Veola marketplace string (e.g. "ebay.com",
// "ebay.co.uk") to an eBay Browse API marketplace identifier. Unknown or
// bare "ebay" values fall back to EBAY_US.
func MarketplaceID(marketplace string) string {
m := strings.ToLower(strings.TrimSpace(marketplace))
switch {
case strings.Contains(m, "ebay.co.uk"):
return "EBAY_GB"
case strings.Contains(m, "ebay.de"):
return "EBAY_DE"
case strings.Contains(m, "ebay.com.au"):
return "EBAY_AU"
case strings.Contains(m, "ebay.ca"):
return "EBAY_CA"
case strings.Contains(m, "ebay.fr"):
return "EBAY_FR"
case strings.Contains(m, "ebay.it"):
return "EBAY_IT"
case strings.Contains(m, "ebay.es"):
return "EBAY_ES"
case strings.Contains(m, "ebay.at"):
return "EBAY_AT"
case strings.Contains(m, "ebay.ch"):
return "EBAY_CH"
case strings.Contains(m, "ebay.ie"):
return "EBAY_IE"
case strings.Contains(m, "ebay.nl"):
return "EBAY_NL"
case strings.Contains(m, "ebay.com.hk"):
return "EBAY_HK"
case strings.Contains(m, "ebay.com.sg"):
return "EBAY_SG"
case strings.Contains(m, "ebay.com.my"):
return "EBAY_MY"
case strings.Contains(m, "ebay.ph"):
return "EBAY_PH"
case strings.Contains(m, "ebay.pl"):
return "EBAY_PL"
default:
// "ebay.com" and any bare/unknown eBay marketplace.
return "EBAY_US"
}
}
// IsEbayMarketplace reports whether a Veola marketplace string should be
// polled through the official eBay Browse API.
func IsEbayMarketplace(marketplace string) bool {
return strings.Contains(strings.ToLower(marketplace), "ebay")
}
// buyingOptionsFilter maps Veola's listing-type vocabulary to the Browse API
// "filter" query parameter. An empty string means no filter ("all").
func buyingOptionsFilter(listingType string) string {
switch strings.ToLower(strings.TrimSpace(listingType)) {
case "bin", "buy_it_now", "fixed_price":
return "buyingOptions:{FIXED_PRICE}"
case "auction":
return "buyingOptions:{AUCTION}"
default:
return ""
}
}

View File

@@ -3,10 +3,12 @@ package handlers
import (
"fmt"
"net/http"
"strconv"
"strings"
"veola/internal/apify"
"veola/internal/auth"
"veola/internal/ebay"
"veola/internal/models"
"veola/internal/ntfy"
"veola/templates"
@@ -14,6 +16,9 @@ import (
var settingsKeys = []string{
"apify_api_key",
"ebay_client_id",
"ebay_client_secret",
"ebay_daily_call_limit",
"ntfy_base_url",
"ntfy_default_topic",
"ntfy_token",
@@ -31,11 +36,14 @@ func (a *App) settingsData(r *http.Request) (templates.SettingsData, error) {
}
users, _ := a.Store.ListUsers(r.Context())
cur := auth.CurrentUserFromRequest(r)
ebayUsed, ebayLimit := a.Scheduler.EbayUsage(r.Context())
return templates.SettingsData{
Page: a.page(r, "Settings", "settings"),
Values: values,
IsAdmin: cur != nil && cur.Role == models.RoleAdmin,
Users: users,
Page: a.page(r, "Settings", "settings"),
Values: values,
IsAdmin: cur != nil && cur.Role == models.RoleAdmin,
Users: users,
EbayUsedToday: ebayUsed,
EbayDailyLimit: ebayLimit,
}, nil
}
@@ -193,3 +201,90 @@ func (a *App) PostTestApify(w http.ResponseWriter, r *http.Request) {
}
render(w, r, templates.Settings(d))
}
func (a *App) PostTestEbay(w http.ResponseWriter, r *http.Request) {
cur := auth.CurrentUserFromRequest(r)
if cur == nil || cur.Role != models.RoleAdmin {
http.Error(w, "forbidden", http.StatusForbidden)
return
}
d, err := a.settingsData(r)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Settings-table values win over config.toml. Both paths are trimmed:
// a stray newline in the TOML would otherwise reach eBay verbatim.
clientID := strings.TrimSpace(d.Values["ebay_client_id"])
if clientID == "" {
clientID = strings.TrimSpace(a.Cfg.Ebay.ClientID)
}
clientSecret := strings.TrimSpace(d.Values["ebay_client_secret"])
if clientSecret == "" {
clientSecret = strings.TrimSpace(a.Cfg.Ebay.ClientSecret)
}
if clientID == "" || clientSecret == "" {
d.TestEbayOK = "Set the eBay App ID and Cert ID first."
render(w, r, templates.Settings(d))
return
}
if d.EbayDailyLimit > 0 && d.EbayUsedToday >= d.EbayDailyLimit {
d.TestEbayOK = fmt.Sprintf("Daily eBay API call limit reached (%d/%d). Test skipped.", d.EbayUsedToday, d.EbayDailyLimit)
render(w, r, templates.Settings(d))
return
}
env := "production"
if strings.EqualFold(strings.TrimSpace(a.Cfg.Ebay.Environment), "sandbox") {
env = "sandbox"
}
// Echo back exactly what was sent (App ID masked, Cert ID length only) so
// a failure points at the inputs, not just "it failed".
inputs := fmt.Sprintf("%s, App ID %s, Cert ID %d chars", env, maskID(clientID), len(clientSecret))
client := ebay.New(clientID, clientSecret, a.Cfg.Ebay.Environment)
listings, err := client.Search(r.Context(), ebay.SearchParams{
MarketplaceID: "EBAY_US",
Query: "test",
Limit: 1,
})
// A real call was made; count it against the daily allowance.
if n, incErr := a.Store.IncrementEbayUsage(r.Context()); incErr == nil {
d.EbayUsedToday = n
}
if err != nil {
msg := fmt.Sprintf("eBay test failed (%s): %s", inputs, err.Error())
if ks := ebayKeysetEnv(clientID); ks != "" && ks != env {
msg += fmt.Sprintf(" — the App ID looks like a %s keyset, but environment is %q. Set environment = %q in the [ebay] config block (or use your %s keyset).", ks, env, ks, env)
}
d.TestEbayOK = msg
} else {
d.TestEbayOK = fmt.Sprintf("eBay Browse API reachable (%s). Returned %d item(s).", inputs, len(listings))
}
render(w, r, templates.Settings(d))
}
// maskID returns a fingerprint of a credential for display: enough of the head
// to recognize it, the rest elided. Used only for the App ID (the non-secret
// half of the OAuth pair) — never for the Cert ID.
func maskID(s string) string {
if len(s) <= 12 {
return strings.Repeat("•", len(s))
}
return s[:12] + "…(" + strconv.Itoa(len(s)) + " chars)"
}
// ebayKeysetEnv guesses which environment an eBay App ID belongs to from the
// SBX/PRD marker eBay embeds in it (e.g. "Name-app-PRD-1a2b..."). Returns ""
// when no marker is present.
func ebayKeysetEnv(clientID string) string {
up := strings.ToUpper(clientID)
switch {
case strings.Contains(up, "-SBX-"):
return "sandbox"
case strings.Contains(up, "-PRD-"):
return "production"
default:
return ""
}
}

View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log/slog"
"strconv"
"strings"
"sync"
"time"
@@ -13,14 +14,23 @@ import (
"veola/internal/apify"
"veola/internal/config"
"veola/internal/db"
"veola/internal/ebay"
"veola/internal/models"
"veola/internal/ntfy"
)
// Provider labels distinguish how a plan is executed: through an Apify actor
// run, or through eBay's official Browse API.
const (
providerApify = "apify"
providerEbay = "ebay"
)
type Scheduler struct {
cfg *config.Config
store *db.Store
apify *apify.Client
ebay *ebay.Client
ntfy *ntfy.Client
cron *cron.Cron
@@ -37,6 +47,7 @@ func New(cfg *config.Config, store *db.Store, ap *apify.Client, nt *ntfy.Client)
cfg: cfg,
store: store,
apify: ap,
ebay: ebay.New(cfg.Ebay.ClientID, cfg.Ebay.ClientSecret, cfg.Ebay.Environment),
ntfy: nt,
cron: cron.New(),
entries: make(map[int64]cron.EntryID),
@@ -136,52 +147,16 @@ func (s *Scheduler) RunPoll(ctx context.Context, it models.Item) {
var errs []string
successes := 0
for _, p := range plans {
if p.actorID == "" {
errs = append(errs, fmt.Sprintf("%s: no actor configured", p.marketplace))
continue
}
raw, err := apifyClient.Run(ctx, p.actorID, p.input)
decoded, err := s.ExecutePlan(ctx, p)
if err != nil {
label := p.marketplace
if p.query != "" {
label = fmt.Sprintf("query %q on %s", p.query, p.marketplace)
}
errs = append(errs, fmt.Sprintf("%s: %s", label, err.Error()))
slog.Error("apify run failed", "item_id", it.ID, "marketplace", p.marketplace, "query", p.query, "err", err)
slog.Error("plan failed", "item_id", it.ID, "provider", p.provider, "marketplace", p.marketplace, "query", p.query, "err", err)
continue
}
decoded, _ := apify.Decode(raw, p.source)
usable := 0
for i := range decoded {
decoded[i].MatchedQuery = p.query
if decoded[i].URL != "" && decoded[i].Price > 0 {
usable++
}
}
slog.Info("apify run decoded",
"item_id", it.ID,
"marketplace", p.marketplace,
"query", p.query,
"actor", p.actorID,
"raw", len(raw),
"decoded", len(decoded),
"usable", usable,
)
if usable == 0 && len(raw) > 0 {
var sample map[string]any
if err := jsonUnmarshal(raw[0], &sample); err == nil {
keys := make([]string, 0, len(sample))
for k := range sample {
keys = append(keys, k)
}
slog.Warn("decoded zero usable rows; raw item keys",
"item_id", it.ID,
"marketplace", p.marketplace,
"actor", p.actorID,
"keys", keys,
)
}
}
results = append(results, decoded...)
successes++
}
@@ -322,6 +297,103 @@ func (s *Scheduler) apifyClient(ctx context.Context) *apify.Client {
return apify.New(key)
}
// ebayClient returns the shared eBay client with credentials refreshed from
// settings (falling back to config.toml). The client caches its OAuth token
// in memory, so the same instance is reused across polls; credentials are
// only re-applied when they actually change.
func (s *Scheduler) ebayClient(ctx context.Context) *ebay.Client {
id := s.cfg.Ebay.ClientID
secret := s.cfg.Ebay.ClientSecret
if v, _ := s.store.GetSetting(ctx, "ebay_client_id"); v != "" {
id = v
}
if v, _ := s.store.GetSetting(ctx, "ebay_client_secret"); v != "" {
secret = v
}
s.ebay.EnsureCredentials(id, secret)
return s.ebay
}
// EbayUsage returns the number of eBay Browse API calls made so far today and
// the configured daily limit. A limit <= 0 means uncapped. Settings override
// config.toml for the limit, mirroring how credentials are resolved.
func (s *Scheduler) EbayUsage(ctx context.Context) (used, limit int) {
used, _ = s.store.EbayUsageToday(ctx)
limit = s.cfg.Ebay.DailyCallLimit
if v, _ := s.store.GetSetting(ctx, "ebay_daily_call_limit"); v != "" {
if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
limit = n
}
}
return used, limit
}
// ExecutePlan runs one plan and returns decoded, provider-agnostic results
// with MatchedQuery already stamped. eBay plans go through the official
// Browse API; all other plans run an Apify actor. Callers handle per-plan
// errors without poisoning sibling plans.
func (s *Scheduler) ExecutePlan(ctx context.Context, p actorPlan) ([]apify.UnifiedResult, error) {
var decoded []apify.UnifiedResult
switch p.provider {
case providerEbay:
sp, ok := p.input.(ebay.SearchParams)
if !ok {
return nil, fmt.Errorf("ebay plan has wrong input type %T", p.input)
}
used, limit := s.EbayUsage(ctx)
if limit > 0 && used >= limit {
return nil, fmt.Errorf("ebay daily API call limit reached (%d/%d); polling halted until the next reset (midnight US Pacific)", used, limit)
}
listings, err := s.ebayClient(ctx).Search(ctx, sp)
// The call hit eBay (or at least was attempted against it) whether
// or not it succeeded, so it counts against the daily allowance.
if n, incErr := s.store.IncrementEbayUsage(ctx); incErr != nil {
slog.Error("ebay usage increment failed", "err", incErr)
} else if limit > 0 && n >= limit {
slog.Warn("ebay daily API call limit reached", "used", n, "limit", limit)
}
if err != nil {
return nil, err
}
decoded = make([]apify.UnifiedResult, 0, len(listings))
for _, l := range listings {
decoded = append(decoded, apify.UnifiedResult{
Title: l.Title,
Price: l.Price,
Currency: l.Currency,
URL: l.URL,
Store: l.Store,
ImageURL: l.ImageURL,
Source: apify.SourceActiveEbay,
})
}
default:
if p.actorID == "" {
return nil, fmt.Errorf("no actor configured for %s", p.marketplace)
}
raw, err := s.apifyClient(ctx).Run(ctx, p.actorID, p.input)
if err != nil {
return nil, err
}
decoded, _ = apify.Decode(raw, p.source)
}
usable := 0
for i := range decoded {
decoded[i].MatchedQuery = p.query
if decoded[i].URL != "" && decoded[i].Price > 0 {
usable++
}
}
slog.Info("plan executed",
"provider", p.provider,
"marketplace", p.marketplace,
"query", p.query,
"decoded", len(decoded),
"usable", usable,
)
return decoded, nil
}
func (s *Scheduler) sendNotification(ctx context.Context, it models.Item, r apify.UnifiedResult) error {
tags := []string{"mag"}
if it.TargetPrice != nil && r.Price <= *it.TargetPrice {
@@ -398,6 +470,7 @@ func (s *Scheduler) BuildPreviewInputs(it models.Item) []actorPlan {
type actorPlan struct {
marketplace string
source string
provider string
actorID string
query string
input any
@@ -418,6 +491,9 @@ func (p actorPlan) Query() string { return p.query }
// Input returns the actor input payload as expected by apify.Client.Run.
func (p actorPlan) Input() any { return p.input }
// Provider returns "apify" or "ebay" — how this plan is executed.
func (p actorPlan) Provider() string { return p.provider }
// buildAllInputs returns one actor plan per (alias × marketplace) for the item.
// For URL-only items (no aliases), produces one plan per marketplace with an
// empty query string.
@@ -447,27 +523,51 @@ func (s *Scheduler) buildInputsForQuery(it models.Item, query string, markets []
switch {
case strings.Contains(mk, "yahoo") || strings.Contains(url, "yahoo.co.jp"):
actorID := firstNonEmpty(it.ActorActive, s.cfg.Apify.Actors.YahooAuctionsJP)
plans = append(plans, actorPlan{m, apify.SourceYahooJP, actorID, query, apify.YahooAuctionsJPInput{
SearchTerm: query,
MaxPages: 1,
}})
plans = append(plans, actorPlan{
marketplace: m, source: apify.SourceYahooJP, provider: providerApify,
actorID: actorID, query: query,
input: apify.YahooAuctionsJPInput{SearchTerm: query, MaxPages: 1},
})
case strings.Contains(mk, "mercari") || strings.Contains(url, "mercari"):
actorID := firstNonEmpty(it.ActorActive, s.cfg.Apify.Actors.MercariJP)
plans = append(plans, actorPlan{m, apify.SourceMercariJP, actorID, query, apify.MercariJPInput{
SearchKeywords: []string{query},
Status: "on_sale",
MaxResults: 30,
}})
plans = append(plans, actorPlan{
marketplace: m, source: apify.SourceMercariJP, provider: providerApify,
actorID: actorID, query: query,
input: apify.MercariJPInput{
SearchKeywords: []string{query},
Status: "on_sale",
MaxResults: 30,
},
})
case ebay.IsEbayMarketplace(mk):
// eBay marketplaces are polled through eBay's official Browse
// API, not an Apify scraper actor.
plans = append(plans, actorPlan{
marketplace: m, source: apify.SourceActiveEbay, provider: providerEbay,
query: query,
input: ebay.SearchParams{
MarketplaceID: ebay.MarketplaceID(mk),
Query: query,
ListingType: it.ListingType,
Limit: 30,
},
})
default:
// Non-eBay custom marketplaces still fall back to the Apify
// active-listings actor.
actorID := firstNonEmpty(it.ActorActive, s.cfg.Apify.Actors.ActiveListings)
plans = append(plans, actorPlan{m, apify.SourceActiveEbay, actorID, query, apify.ActiveListingInput{
SearchQueries: []string{query},
MaxProductsPerSearch: 30,
MaxSearchPages: 1,
Sort: "best_match",
ListingType: mapListingType(it.ListingType),
ProxyConfiguration: s.proxyConfig(),
}})
plans = append(plans, actorPlan{
marketplace: m, source: apify.SourceActiveEbay, provider: providerApify,
actorID: actorID, query: query,
input: apify.ActiveListingInput{
SearchQueries: []string{query},
MaxProductsPerSearch: 30,
MaxSearchPages: 1,
Sort: "best_match",
ListingType: mapListingType(it.ListingType),
ProxyConfiguration: s.proxyConfig(),
},
})
}
}
return plans