Add eBay Browse API integration with daily call quota
eBay marketplaces are now polled through eBay's official Buy > Browse API (client-credentials OAuth2) instead of an Apify scraper actor; Apify still handles Yahoo JP and Mercari. Browse API calls are tracked per day in a new ebay_api_usage table and capped (default 5000, configurable) on eBay's Pacific-time reset clock, so polling halts before the limit is hit. Credentials live in config.toml [ebay] and are overridable via /settings, which also surfaces the day's running call count. Also carries the server.secure_cookies config plumbing (field, accessor, example) consumed by the following commit. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
prosolis
227
internal/ebay/client.go
Normal file
227
internal/ebay/client.go
Normal file
@@ -0,0 +1,227 @@
|
||||
// Package ebay is a thin client for eBay's official Buy > Browse API. It
|
||||
// handles client-credentials OAuth2 token caching and active-listing search
|
||||
// (item_summary/search). It deliberately covers only what Veola needs: a
|
||||
// keyword search returning normalized listings. Sold/completed data (the
|
||||
// Marketplace Insights API) is not implemented here.
|
||||
package ebay
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// endpoints bundles the production/sandbox base URLs for the two APIs used.
|
||||
type endpoints struct {
|
||||
oauth string
|
||||
browse string
|
||||
}
|
||||
|
||||
func endpointsFor(environment string) endpoints {
|
||||
if strings.EqualFold(strings.TrimSpace(environment), "sandbox") {
|
||||
return endpoints{
|
||||
oauth: "https://api.sandbox.ebay.com/identity/v1/oauth2/token",
|
||||
browse: "https://api.sandbox.ebay.com/buy/browse/v1",
|
||||
}
|
||||
}
|
||||
return endpoints{
|
||||
oauth: "https://api.ebay.com/identity/v1/oauth2/token",
|
||||
browse: "https://api.ebay.com/buy/browse/v1",
|
||||
}
|
||||
}
|
||||
|
||||
// Client is safe for concurrent use. The application access token is cached
|
||||
// in memory and refreshed shortly before it expires.
|
||||
type Client struct {
|
||||
HTTP *http.Client
|
||||
|
||||
mu sync.Mutex
|
||||
clientID string
|
||||
clientSecret string
|
||||
ends endpoints
|
||||
token string
|
||||
tokenExpiry time.Time
|
||||
}
|
||||
|
||||
// New builds a client for the given keyset. environment is "production"
|
||||
// (default) or "sandbox". Credentials may be empty; calls then fail fast
|
||||
// with a "not configured" error.
|
||||
func New(clientID, clientSecret, environment string) *Client {
|
||||
return &Client{
|
||||
HTTP: &http.Client{Timeout: 30 * time.Second},
|
||||
clientID: clientID,
|
||||
clientSecret: clientSecret,
|
||||
ends: endpointsFor(environment),
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureCredentials updates the keyset if it changed, discarding any cached
|
||||
// token so the next call re-authenticates. The environment is fixed at
|
||||
// construction time and is not changed here. Safe to call on every poll.
|
||||
func (c *Client) EnsureCredentials(clientID, clientSecret string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if clientID == c.clientID && clientSecret == c.clientSecret {
|
||||
return
|
||||
}
|
||||
c.clientID = clientID
|
||||
c.clientSecret = clientSecret
|
||||
c.token = ""
|
||||
c.tokenExpiry = time.Time{}
|
||||
}
|
||||
|
||||
func (c *Client) accessToken(ctx context.Context) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.clientID == "" || c.clientSecret == "" {
|
||||
return "", errors.New("ebay credentials not configured")
|
||||
}
|
||||
if c.token != "" && time.Now().Before(c.tokenExpiry) {
|
||||
return c.token, nil
|
||||
}
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("grant_type", "client_credentials")
|
||||
form.Set("scope", "https://api.ebay.com/oauth/api_scope")
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.ends.oauth, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
basic := base64.StdEncoding.EncodeToString([]byte(c.clientID + ":" + c.clientSecret))
|
||||
req.Header.Set("Authorization", "Basic "+basic)
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
resp, err := c.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ebay oauth: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
if resp.StatusCode >= 300 {
|
||||
return "", fmt.Errorf("ebay oauth: http %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
var tr struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
ExpiresIn int `json:"expires_in"`
|
||||
TokenType string `json:"token_type"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &tr); err != nil {
|
||||
return "", fmt.Errorf("ebay oauth: decode: %w", err)
|
||||
}
|
||||
if tr.AccessToken == "" {
|
||||
return "", errors.New("ebay oauth: empty access token")
|
||||
}
|
||||
c.token = tr.AccessToken
|
||||
// Refresh a minute early to avoid racing the expiry.
|
||||
ttl := time.Duration(tr.ExpiresIn) * time.Second
|
||||
if ttl <= time.Minute {
|
||||
ttl = time.Minute
|
||||
}
|
||||
c.tokenExpiry = time.Now().Add(ttl - time.Minute)
|
||||
return c.token, nil
|
||||
}
|
||||
|
||||
// browseItemSummary mirrors the subset of the item_summary/search response
|
||||
// Veola consumes.
|
||||
type browseItemSummary struct {
|
||||
ItemID string `json:"itemId"`
|
||||
Title string `json:"title"`
|
||||
Price struct {
|
||||
Value string `json:"value"`
|
||||
Currency string `json:"currency"`
|
||||
} `json:"price"`
|
||||
ItemWebURL string `json:"itemWebUrl"`
|
||||
Image struct {
|
||||
ImageURL string `json:"imageUrl"`
|
||||
} `json:"image"`
|
||||
ThumbnailImages []struct {
|
||||
ImageURL string `json:"imageUrl"`
|
||||
} `json:"thumbnailImages"`
|
||||
Seller struct {
|
||||
Username string `json:"username"`
|
||||
} `json:"seller"`
|
||||
}
|
||||
|
||||
// Search runs one item_summary/search call and returns normalized listings.
|
||||
// An empty query is rejected: the Browse API requires a non-empty q.
|
||||
func (c *Client) Search(ctx context.Context, p SearchParams) ([]Listing, error) {
|
||||
query := strings.TrimSpace(p.Query)
|
||||
if query == "" {
|
||||
return nil, errors.New("ebay search requires a non-empty query")
|
||||
}
|
||||
token, err := c.accessToken(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
marketplace := p.MarketplaceID
|
||||
if marketplace == "" {
|
||||
marketplace = "EBAY_US"
|
||||
}
|
||||
limit := p.Limit
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 50
|
||||
}
|
||||
|
||||
q := url.Values{}
|
||||
q.Set("q", query)
|
||||
q.Set("limit", strconv.Itoa(limit))
|
||||
if f := buyingOptionsFilter(p.ListingType); f != "" {
|
||||
q.Set("filter", f)
|
||||
}
|
||||
reqURL := c.ends.browse + "/item_summary/search?" + q.Encode()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("X-EBAY-C-MARKETPLACE-ID", marketplace)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ebay browse: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("ebay browse: http %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
var sr struct {
|
||||
ItemSummaries []browseItemSummary `json:"itemSummaries"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &sr); err != nil {
|
||||
return nil, fmt.Errorf("ebay browse: decode: %w", err)
|
||||
}
|
||||
|
||||
out := make([]Listing, 0, len(sr.ItemSummaries))
|
||||
for _, s := range sr.ItemSummaries {
|
||||
price, _ := strconv.ParseFloat(strings.TrimSpace(s.Price.Value), 64)
|
||||
img := s.Image.ImageURL
|
||||
if img == "" && len(s.ThumbnailImages) > 0 {
|
||||
img = s.ThumbnailImages[0].ImageURL
|
||||
}
|
||||
store := "ebay"
|
||||
if s.Seller.Username != "" {
|
||||
store = "ebay (" + s.Seller.Username + ")"
|
||||
}
|
||||
out = append(out, Listing{
|
||||
Title: s.Title,
|
||||
Price: price,
|
||||
Currency: s.Price.Currency,
|
||||
URL: s.ItemWebURL,
|
||||
Store: store,
|
||||
ImageURL: img,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
45
internal/ebay/ebay_test.go
Normal file
45
internal/ebay/ebay_test.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package ebay
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMarketplaceID(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"ebay.com": "EBAY_US",
|
||||
"ebay.co.uk": "EBAY_GB",
|
||||
"ebay.de": "EBAY_DE",
|
||||
"ebay.com.au": "EBAY_AU",
|
||||
"EBAY.CA": "EBAY_CA",
|
||||
"ebay": "EBAY_US",
|
||||
"weird-market": "EBAY_US",
|
||||
" ebay.it ": "EBAY_IT",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := MarketplaceID(in); got != want {
|
||||
t.Errorf("MarketplaceID(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsEbayMarketplace(t *testing.T) {
|
||||
if !IsEbayMarketplace("ebay.co.uk") {
|
||||
t.Error("ebay.co.uk should be an eBay marketplace")
|
||||
}
|
||||
if IsEbayMarketplace("yahoo-auctions-jp") {
|
||||
t.Error("yahoo should not be an eBay marketplace")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuyingOptionsFilter(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"": "",
|
||||
"all": "",
|
||||
"bin": "buyingOptions:{FIXED_PRICE}",
|
||||
"buy_it_now": "buyingOptions:{FIXED_PRICE}",
|
||||
"auction": "buyingOptions:{AUCTION}",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := buyingOptionsFilter(in); got != want {
|
||||
t.Errorf("buyingOptionsFilter(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
94
internal/ebay/types.go
Normal file
94
internal/ebay/types.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package ebay
|
||||
|
||||
import "strings"
|
||||
|
||||
// SearchParams is the input to a single Browse API item_summary/search call.
|
||||
// It is provider-specific and is carried as the opaque input payload of a
|
||||
// scheduler plan, mirroring how Apify actor inputs are carried.
|
||||
type SearchParams struct {
|
||||
// MarketplaceID is an eBay marketplace identifier such as EBAY_US.
|
||||
MarketplaceID string
|
||||
// Query is the keyword search string. Required; the Browse API rejects
|
||||
// an empty q.
|
||||
Query string
|
||||
// ListingType is Veola's vocabulary ("all", "bin"/"buy_it_now",
|
||||
// "auction"); it is mapped to a buyingOptions filter.
|
||||
ListingType string
|
||||
// Limit caps the number of results requested (Browse API max is 200).
|
||||
Limit int
|
||||
}
|
||||
|
||||
// Listing is one normalized active eBay listing. The scheduler converts these
|
||||
// into the shared apify.UnifiedResult shape so the rest of the pipeline
|
||||
// (dedup, filter, alert) is provider-agnostic.
|
||||
type Listing struct {
|
||||
Title string
|
||||
Price float64
|
||||
Currency string
|
||||
URL string
|
||||
Store string
|
||||
ImageURL string
|
||||
}
|
||||
|
||||
// MarketplaceID maps a Veola marketplace string (e.g. "ebay.com",
|
||||
// "ebay.co.uk") to an eBay Browse API marketplace identifier. Unknown or
|
||||
// bare "ebay" values fall back to EBAY_US.
|
||||
func MarketplaceID(marketplace string) string {
|
||||
m := strings.ToLower(strings.TrimSpace(marketplace))
|
||||
switch {
|
||||
case strings.Contains(m, "ebay.co.uk"):
|
||||
return "EBAY_GB"
|
||||
case strings.Contains(m, "ebay.de"):
|
||||
return "EBAY_DE"
|
||||
case strings.Contains(m, "ebay.com.au"):
|
||||
return "EBAY_AU"
|
||||
case strings.Contains(m, "ebay.ca"):
|
||||
return "EBAY_CA"
|
||||
case strings.Contains(m, "ebay.fr"):
|
||||
return "EBAY_FR"
|
||||
case strings.Contains(m, "ebay.it"):
|
||||
return "EBAY_IT"
|
||||
case strings.Contains(m, "ebay.es"):
|
||||
return "EBAY_ES"
|
||||
case strings.Contains(m, "ebay.at"):
|
||||
return "EBAY_AT"
|
||||
case strings.Contains(m, "ebay.ch"):
|
||||
return "EBAY_CH"
|
||||
case strings.Contains(m, "ebay.ie"):
|
||||
return "EBAY_IE"
|
||||
case strings.Contains(m, "ebay.nl"):
|
||||
return "EBAY_NL"
|
||||
case strings.Contains(m, "ebay.com.hk"):
|
||||
return "EBAY_HK"
|
||||
case strings.Contains(m, "ebay.com.sg"):
|
||||
return "EBAY_SG"
|
||||
case strings.Contains(m, "ebay.com.my"):
|
||||
return "EBAY_MY"
|
||||
case strings.Contains(m, "ebay.ph"):
|
||||
return "EBAY_PH"
|
||||
case strings.Contains(m, "ebay.pl"):
|
||||
return "EBAY_PL"
|
||||
default:
|
||||
// "ebay.com" and any bare/unknown eBay marketplace.
|
||||
return "EBAY_US"
|
||||
}
|
||||
}
|
||||
|
||||
// IsEbayMarketplace reports whether a Veola marketplace string should be
|
||||
// polled through the official eBay Browse API.
|
||||
func IsEbayMarketplace(marketplace string) bool {
|
||||
return strings.Contains(strings.ToLower(marketplace), "ebay")
|
||||
}
|
||||
|
||||
// buyingOptionsFilter maps Veola's listing-type vocabulary to the Browse API
|
||||
// "filter" query parameter. An empty string means no filter ("all").
|
||||
func buyingOptionsFilter(listingType string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(listingType)) {
|
||||
case "bin", "buy_it_now", "fixed_price":
|
||||
return "buyingOptions:{FIXED_PRICE}"
|
||||
case "auction":
|
||||
return "buyingOptions:{AUCTION}"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user