Initial commit
This commit is contained in:
152
internal/apify/client.go
Normal file
152
internal/apify/client.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package apify
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
apiBase = "https://api.apify.com/v2"
|
||||
pollEvery = 3 * time.Second
|
||||
pollTimeout = 5 * time.Minute
|
||||
)
|
||||
|
||||
// Client is a thin wrapper around the Apify run-and-fetch lifecycle.
|
||||
type Client struct {
|
||||
APIKey string
|
||||
HTTP *http.Client
|
||||
}
|
||||
|
||||
func New(apiKey string) *Client {
|
||||
return &Client{
|
||||
APIKey: apiKey,
|
||||
HTTP: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
type runResponse struct {
|
||||
Data struct {
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"`
|
||||
DefaultDatasetID string `json:"defaultDatasetId"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// Run starts an actor run, waits for SUCCEEDED, and returns dataset items as raw JSON.
|
||||
func (c *Client) Run(ctx context.Context, actorID string, input any) ([]json.RawMessage, error) {
|
||||
if c.APIKey == "" {
|
||||
return nil, errors.New("apify api_key not configured")
|
||||
}
|
||||
if actorID == "" {
|
||||
return nil, errors.New("apify actor id is empty")
|
||||
}
|
||||
|
||||
body, err := json.Marshal(input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Apify URLs use "~" to separate username and actor name, never "/".
|
||||
// Accept either form in config and normalize before path-escaping.
|
||||
urlActorID := strings.ReplaceAll(actorID, "/", "~")
|
||||
startURL := fmt.Sprintf("%s/acts/%s/runs?token=%s", apiBase, url.PathEscape(urlActorID), url.QueryEscape(c.APIKey))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, startURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := c.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("start run: %w", err)
|
||||
}
|
||||
var runResp runResponse
|
||||
if err := decodeJSON(resp, &runResp); err != nil {
|
||||
return nil, fmt.Errorf("start run: %w", err)
|
||||
}
|
||||
if runResp.Data.ID == "" {
|
||||
return nil, errors.New("start run: missing run id")
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(pollTimeout)
|
||||
pollCtx, cancel := context.WithDeadline(ctx, deadline)
|
||||
defer cancel()
|
||||
|
||||
status, datasetID, err := c.waitForRun(pollCtx, runResp.Data.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status != "SUCCEEDED" {
|
||||
return nil, fmt.Errorf("apify run terminated with status %s", status)
|
||||
}
|
||||
|
||||
return c.fetchDataset(ctx, datasetID)
|
||||
}
|
||||
|
||||
func (c *Client) waitForRun(ctx context.Context, runID string) (string, string, error) {
|
||||
pollURL := fmt.Sprintf("%s/actor-runs/%s?token=%s", apiBase, url.PathEscape(runID), url.QueryEscape(c.APIKey))
|
||||
for {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pollURL, nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
resp, err := c.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("poll run: %w", err)
|
||||
}
|
||||
var r runResponse
|
||||
if err := decodeJSON(resp, &r); err != nil {
|
||||
return "", "", fmt.Errorf("poll run: %w", err)
|
||||
}
|
||||
switch r.Data.Status {
|
||||
case "SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT":
|
||||
return r.Data.Status, r.Data.DefaultDatasetID, nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", "", ctx.Err()
|
||||
case <-time.After(pollEvery):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) fetchDataset(ctx context.Context, datasetID string) ([]json.RawMessage, error) {
|
||||
if datasetID == "" {
|
||||
return nil, errors.New("missing dataset id")
|
||||
}
|
||||
dsURL := fmt.Sprintf("%s/datasets/%s/items?clean=true&format=json&token=%s", apiBase, url.PathEscape(datasetID), url.QueryEscape(c.APIKey))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, dsURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch dataset: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
return nil, fmt.Errorf("dataset returned %d: %s", resp.StatusCode, string(b))
|
||||
}
|
||||
var items []json.RawMessage
|
||||
if err := json.NewDecoder(resp.Body).Decode(&items); err != nil {
|
||||
return nil, fmt.Errorf("decode dataset: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func decodeJSON(resp *http.Response, dst any) error {
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
return fmt.Errorf("http %d: %s", resp.StatusCode, string(b))
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(dst)
|
||||
}
|
||||
313
internal/apify/types.go
Normal file
313
internal/apify/types.go
Normal file
@@ -0,0 +1,313 @@
|
||||
package apify
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ActiveListingInput is the input schema for `automation-lab/ebay-scraper`.
|
||||
// The actor accepts keyword searches and standard filters; it targets
|
||||
// ebay.com only (no per-marketplace routing in the actor itself), so
|
||||
// non-US marketplaces won't return useful results with this actor.
|
||||
type ActiveListingInput struct {
|
||||
SearchQueries []string `json:"searchQueries"`
|
||||
MaxProductsPerSearch int `json:"maxProductsPerSearch,omitempty"`
|
||||
MaxSearchPages int `json:"maxSearchPages,omitempty"`
|
||||
Sort string `json:"sort,omitempty"`
|
||||
ListingType string `json:"listingType,omitempty"`
|
||||
Condition []string `json:"condition,omitempty"`
|
||||
MinPrice *int `json:"minPrice,omitempty"`
|
||||
MaxPrice *int `json:"maxPrice,omitempty"`
|
||||
ProxyConfiguration *ProxyConfiguration `json:"proxyConfiguration,omitempty"`
|
||||
}
|
||||
|
||||
// ProxyConfiguration is the standard apify input block for proxy routing.
|
||||
// eBay (and most retail sites) return 403 to datacenter IPs; passing
|
||||
// {"useApifyProxy": true, "apifyProxyGroups": ["RESIDENTIAL"]} works.
|
||||
type ProxyConfiguration struct {
|
||||
UseApifyProxy bool `json:"useApifyProxy"`
|
||||
ApifyProxyGroups []string `json:"apifyProxyGroups,omitempty"`
|
||||
ApifyProxyCountry string `json:"apifyProxyCountry,omitempty"`
|
||||
}
|
||||
|
||||
// ActiveListingResult is decoded leniently to handle multiple eBay-scraper
|
||||
// actors. delicious_zebu/ebay-product-listing-scraper returns productUrl /
|
||||
// imageUrl / numeric price; harvestlab/ebay-scraper used url / price /
|
||||
// currency. The decoder coalesces both shapes.
|
||||
type ActiveListingResult struct {
|
||||
Title string `json:"title"`
|
||||
Price any `json:"price"`
|
||||
OriginalPrice any `json:"originalPrice"`
|
||||
Currency string `json:"currency"`
|
||||
URL string `json:"url"`
|
||||
ProductURL string `json:"productUrl"`
|
||||
Store string `json:"store"`
|
||||
ImageURL string `json:"imageUrl"`
|
||||
Image string `json:"image"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
Images []string `json:"images"`
|
||||
Condition string `json:"condition"`
|
||||
ListingType string `json:"listingType"`
|
||||
ShippingCost any `json:"shippingCost"`
|
||||
ShippingPrice any `json:"shippingPrice"`
|
||||
FreeShipping bool `json:"freeShipping"`
|
||||
Marketplace string `json:"marketplace"`
|
||||
MatchConfidence float64 `json:"matchConfidence"`
|
||||
Availability string `json:"availability"`
|
||||
WatchersCount int `json:"watchersCount"`
|
||||
QuantitySold int `json:"quantitySold"`
|
||||
}
|
||||
|
||||
type SoldListingInput struct {
|
||||
Query string `json:"query"`
|
||||
Marketplace string `json:"marketplace,omitempty"`
|
||||
MaxResults int `json:"maxResults,omitempty"`
|
||||
DaysBack int `json:"daysBack,omitempty"`
|
||||
ProxyConfiguration *ProxyConfiguration `json:"proxyConfiguration,omitempty"`
|
||||
}
|
||||
|
||||
type SoldListingResult struct {
|
||||
Title string `json:"title"`
|
||||
SoldPrice float64 `json:"soldPrice"`
|
||||
Currency string `json:"soldCurrency"`
|
||||
SoldAt string `json:"endedAt"`
|
||||
Condition string `json:"condition"`
|
||||
ListingType string `json:"listingType"`
|
||||
ShippingPrice float64 `json:"shippingPrice"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type PriceComparisonInput struct {
|
||||
Query string `json:"query,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
MatchStrictness string `json:"matchStrictness,omitempty"`
|
||||
ProxyConfiguration *ProxyConfiguration `json:"proxyConfiguration,omitempty"`
|
||||
}
|
||||
|
||||
type PriceComparisonResult struct {
|
||||
Title string `json:"title"`
|
||||
Price float64 `json:"price"`
|
||||
Currency string `json:"currency"`
|
||||
URL string `json:"url"`
|
||||
Store string `json:"store"`
|
||||
ImageURL string `json:"imageUrl"`
|
||||
Availability string `json:"availability"`
|
||||
MatchConfidence float64 `json:"matchConfidence"`
|
||||
}
|
||||
|
||||
// YahooAuctionsJPInput targets meron1122/zenmarket-scraper. ZenMarket is a
|
||||
// buyer-proxy for Yahoo Auctions JP; its scraper returns ZenMarket-proxied
|
||||
// listing URLs and USD-converted prices.
|
||||
type YahooAuctionsJPInput struct {
|
||||
SearchTerm string `json:"searchTerm"`
|
||||
CategoryID string `json:"categoryID,omitempty"`
|
||||
MaxPages int `json:"maxPages,omitempty"`
|
||||
MaxRemainingHours int `json:"maxRemainingHours,omitempty"`
|
||||
}
|
||||
|
||||
// MercariJPInput targets cloud9_ai/mercari-scraper. The actor manages its
|
||||
// own proxy (Japan datacenter with residential fallback), so we do not send
|
||||
// a proxyConfiguration block.
|
||||
type MercariJPInput struct {
|
||||
SearchKeywords []string `json:"searchKeywords,omitempty"`
|
||||
ProductUrls []string `json:"productUrls,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
SortBy string `json:"sortBy,omitempty"`
|
||||
PriceMin *int `json:"priceMin,omitempty"`
|
||||
PriceMax *int `json:"priceMax,omitempty"`
|
||||
ItemCondition string `json:"itemCondition,omitempty"`
|
||||
MaxResults int `json:"maxResults,omitempty"`
|
||||
}
|
||||
|
||||
// YahooAuctionsJPResult matches meron1122/zenmarket-scraper output. Prices
|
||||
// are USD-converted at the ZenMarket-published rate.
|
||||
type YahooAuctionsJPResult struct {
|
||||
Name string `json:"name"`
|
||||
CurrentPrice any `json:"current_price"`
|
||||
Photos []string `json:"photos"`
|
||||
URL string `json:"url"`
|
||||
EndingDate string `json:"ending_date"`
|
||||
}
|
||||
|
||||
// UnifiedResult is the common shape produced by ParseResults regardless of
|
||||
// which actor type returned the data. The scheduler consumes this.
|
||||
type UnifiedResult struct {
|
||||
Title string
|
||||
Price float64
|
||||
Currency string
|
||||
URL string
|
||||
Store string
|
||||
ImageURL string
|
||||
Source string
|
||||
MatchConfidence float64
|
||||
OutOfStock bool
|
||||
// MatchedQuery records which alias from the item's query list produced
|
||||
// this row. Empty for URL-only items or rows from non-search sources.
|
||||
MatchedQuery string
|
||||
}
|
||||
|
||||
// Decode unmarshals a list of raw JSON items into UnifiedResult slices using
|
||||
// the shape that matches the given source label.
|
||||
func Decode(items []json.RawMessage, source string) ([]UnifiedResult, error) {
|
||||
out := make([]UnifiedResult, 0, len(items))
|
||||
switch source {
|
||||
case SourceActiveEbay, SourcePriceCompare:
|
||||
for _, raw := range items {
|
||||
var r ActiveListingResult
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
continue
|
||||
}
|
||||
url := r.URL
|
||||
if url == "" {
|
||||
url = r.ProductURL
|
||||
}
|
||||
img := r.ImageURL
|
||||
if img == "" {
|
||||
img = r.Image
|
||||
}
|
||||
if img == "" {
|
||||
img = r.Thumbnail
|
||||
}
|
||||
if img == "" && len(r.Images) > 0 {
|
||||
img = r.Images[0]
|
||||
}
|
||||
store := r.Store
|
||||
if store == "" {
|
||||
store = r.Marketplace
|
||||
}
|
||||
if store == "" && source == SourceActiveEbay {
|
||||
store = "ebay"
|
||||
}
|
||||
cur := r.Currency
|
||||
if cur == "" {
|
||||
cur = "USD"
|
||||
}
|
||||
out = append(out, UnifiedResult{
|
||||
Title: r.Title,
|
||||
Price: coercePrice(r.Price),
|
||||
Currency: cur,
|
||||
URL: url,
|
||||
Store: store,
|
||||
ImageURL: img,
|
||||
Source: source,
|
||||
MatchConfidence: r.MatchConfidence,
|
||||
OutOfStock: isOOS(r.Availability),
|
||||
})
|
||||
}
|
||||
case SourceYahooJP:
|
||||
for _, raw := range items {
|
||||
var r YahooAuctionsJPResult
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
continue
|
||||
}
|
||||
img := ""
|
||||
if len(r.Photos) > 0 {
|
||||
img = r.Photos[0]
|
||||
}
|
||||
out = append(out, UnifiedResult{
|
||||
Title: r.Name,
|
||||
Price: coercePrice(r.CurrentPrice),
|
||||
Currency: "USD",
|
||||
URL: r.URL,
|
||||
Store: "yahoo-auctions-jp (via zenmarket)",
|
||||
ImageURL: img,
|
||||
Source: source,
|
||||
})
|
||||
}
|
||||
case SourceMercariJP:
|
||||
// Mercari actors vary in shape; accept either price/currentPrice and title/name.
|
||||
for _, raw := range items {
|
||||
var generic struct {
|
||||
Title string `json:"title"`
|
||||
Name string `json:"name"`
|
||||
Price float64 `json:"price"`
|
||||
CurrentPrice float64 `json:"currentPrice"`
|
||||
Currency string `json:"currency"`
|
||||
URL string `json:"url"`
|
||||
ImageURL string `json:"imageUrl"`
|
||||
Image string `json:"image"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &generic); err != nil {
|
||||
continue
|
||||
}
|
||||
title := generic.Title
|
||||
if title == "" {
|
||||
title = generic.Name
|
||||
}
|
||||
price := generic.Price
|
||||
if price == 0 {
|
||||
price = generic.CurrentPrice
|
||||
}
|
||||
img := generic.ImageURL
|
||||
if img == "" {
|
||||
img = generic.Image
|
||||
}
|
||||
cur := generic.Currency
|
||||
if cur == "" {
|
||||
cur = "JPY"
|
||||
}
|
||||
out = append(out, UnifiedResult{
|
||||
Title: title,
|
||||
Price: price,
|
||||
Currency: cur,
|
||||
URL: generic.URL,
|
||||
Store: "mercari-jp",
|
||||
ImageURL: img,
|
||||
Source: source,
|
||||
OutOfStock: isOOS(generic.Status),
|
||||
})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
const (
|
||||
SourceActiveEbay = "ebay"
|
||||
SourcePriceCompare = "price-comparison"
|
||||
SourceYahooJP = "yahoo-auctions-jp"
|
||||
SourceMercariJP = "mercari-jp"
|
||||
SourceSoldEbay = "ebay-sold"
|
||||
SourceSoldYahooJP = "yahoo-auctions-jp-sold"
|
||||
)
|
||||
|
||||
// coercePrice accepts a price field that might be a number or a string with
|
||||
// currency symbols / commas (e.g. "$24.99", "1,299.00"). Returns 0 on failure
|
||||
// so FilterResults can drop the row cleanly.
|
||||
func coercePrice(v any) float64 {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
return 0
|
||||
case float64:
|
||||
return x
|
||||
case float32:
|
||||
return float64(x)
|
||||
case int:
|
||||
return float64(x)
|
||||
case int64:
|
||||
return float64(x)
|
||||
case string:
|
||||
s := strings.Map(func(r rune) rune {
|
||||
switch {
|
||||
case r >= '0' && r <= '9', r == '.', r == '-':
|
||||
return r
|
||||
}
|
||||
return -1
|
||||
}, x)
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return f
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func isOOS(s string) bool {
|
||||
switch s {
|
||||
case "out_of_stock", "OUT_OF_STOCK", "sold", "SOLD", "ended":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
214
internal/auth/auth.go
Normal file
214
internal/auth/auth.go
Normal file
@@ -0,0 +1,214 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/alexedwards/scs/sqlite3store"
|
||||
"github.com/alexedwards/scs/v2"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"veola/internal/db"
|
||||
"veola/internal/models"
|
||||
)
|
||||
|
||||
const (
|
||||
BcryptCost = 12
|
||||
MinPasswordLen = 12
|
||||
|
||||
sessionUserIDKey = "user_id"
|
||||
sessionCSRFKey = "csrf_token"
|
||||
|
||||
csrfFormField = "csrf_token"
|
||||
csrfHeaderName = "X-CSRF-Token"
|
||||
)
|
||||
|
||||
// Manager bundles session manager + DB store and serves as the auth surface.
|
||||
type Manager struct {
|
||||
Sessions *scs.SessionManager
|
||||
Store *db.Store
|
||||
hmacKey []byte
|
||||
}
|
||||
|
||||
func NewManager(sqlDB *sql.DB, store *db.Store, sessionSecret string) (*Manager, error) {
|
||||
if len(sessionSecret) < 32 {
|
||||
return nil, errors.New("session secret too short")
|
||||
}
|
||||
sm := scs.New()
|
||||
sm.Store = sqlite3store.New(sqlDB)
|
||||
sm.Lifetime = 7 * 24 * time.Hour
|
||||
sm.IdleTimeout = 7 * 24 * time.Hour
|
||||
sm.Cookie.Name = "veola_session"
|
||||
sm.Cookie.HttpOnly = true
|
||||
sm.Cookie.Path = "/"
|
||||
sm.Cookie.SameSite = http.SameSiteLaxMode
|
||||
sm.Cookie.Persist = true
|
||||
// Cookie.Secure left false for self-hosted HTTP deployments; flip via env in deploy.
|
||||
|
||||
mac := sha256.New()
|
||||
mac.Write([]byte(sessionSecret))
|
||||
return &Manager{Sessions: sm, Store: store, hmacKey: mac.Sum(nil)}, nil
|
||||
}
|
||||
|
||||
func HashPassword(plain string) (string, error) {
|
||||
b, err := bcrypt.GenerateFromPassword([]byte(plain), BcryptCost)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
func CheckPassword(hash, plain string) bool {
|
||||
return bcrypt.CompareHashAndPassword([]byte(hash), []byte(plain)) == nil
|
||||
}
|
||||
|
||||
// LogIn writes the user id into the session and rotates the token.
|
||||
func (m *Manager) LogIn(ctx context.Context, userID int64) error {
|
||||
if err := m.Sessions.RenewToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
m.Sessions.Put(ctx, sessionUserIDKey, userID)
|
||||
m.Sessions.Put(ctx, sessionCSRFKey, newCSRFToken())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) LogOut(ctx context.Context) error {
|
||||
return m.Sessions.Destroy(ctx)
|
||||
}
|
||||
|
||||
func (m *Manager) UserID(ctx context.Context) int64 {
|
||||
return m.Sessions.GetInt64(ctx, sessionUserIDKey)
|
||||
}
|
||||
|
||||
func (m *Manager) CurrentUser(ctx context.Context) (*models.User, error) {
|
||||
id := m.UserID(ctx)
|
||||
if id == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return m.Store.GetUserByID(ctx, id)
|
||||
}
|
||||
|
||||
// ============ CSRF ============
|
||||
|
||||
func newCSRFToken() string {
|
||||
b := make([]byte, 32)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
// extremely unlikely; fall back to a time-based token rather than crashing
|
||||
return hex.EncodeToString([]byte(time.Now().String()))
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
func (m *Manager) CSRFToken(ctx context.Context) string {
|
||||
tok := m.Sessions.GetString(ctx, sessionCSRFKey)
|
||||
if tok == "" {
|
||||
tok = newCSRFToken()
|
||||
m.Sessions.Put(ctx, sessionCSRFKey, tok)
|
||||
}
|
||||
return tok
|
||||
}
|
||||
|
||||
// CSRFFieldName is the HTML form field expected by the middleware.
|
||||
func CSRFFieldName() string { return csrfFormField }
|
||||
|
||||
// ============ Middleware ============
|
||||
|
||||
type ctxKey int
|
||||
|
||||
const (
|
||||
ctxKeyUser ctxKey = iota
|
||||
)
|
||||
|
||||
func userFromContext(ctx context.Context) *models.User {
|
||||
u, _ := ctx.Value(ctxKeyUser).(*models.User)
|
||||
return u
|
||||
}
|
||||
|
||||
// CurrentUserFromRequest is the public accessor for handlers and templates.
|
||||
func CurrentUserFromRequest(r *http.Request) *models.User {
|
||||
return userFromContext(r.Context())
|
||||
}
|
||||
|
||||
// LoadUser populates the user into the context for any logged-in session.
|
||||
// Routes still need RequireAuth/RequireAdmin to gate access.
|
||||
func (m *Manager) LoadUser(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
u, err := m.CurrentUser(r.Context())
|
||||
if err == nil && u != nil {
|
||||
ctx := context.WithValue(r.Context(), ctxKeyUser, u)
|
||||
r = r.WithContext(ctx)
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// RequireAuth redirects to /login if no user is present. Skips /login, /setup,
|
||||
// /static. The setup-gate (redirect to /setup if no users exist) is applied
|
||||
// at the router level via SetupGate so it can short-circuit before auth runs.
|
||||
func (m *Manager) RequireAuth(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if userFromContext(r.Context()) == nil {
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Manager) RequireAdmin(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
u := userFromContext(r.Context())
|
||||
if u == nil {
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
if u.Role != models.RoleAdmin {
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// CSRFProtect validates the CSRF token on non-idempotent requests.
|
||||
func (m *Manager) CSRFProtect(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet, http.MethodHead, http.MethodOptions:
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
expected := m.Sessions.GetString(r.Context(), sessionCSRFKey)
|
||||
if expected == "" {
|
||||
http.Error(w, "csrf token missing from session", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
got := r.Header.Get(csrfHeaderName)
|
||||
if got == "" {
|
||||
if err := r.ParseForm(); err == nil {
|
||||
got = r.PostFormValue(csrfFormField)
|
||||
}
|
||||
}
|
||||
if subtle.ConstantTimeCompare([]byte(got), []byte(expected)) != 1 {
|
||||
http.Error(w, "invalid csrf token", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// HMAC is exposed for non-session use cases (e.g. signed setup links). Not
|
||||
// currently called by handlers but kept available since the secret is loaded.
|
||||
func (m *Manager) HMAC(payload []byte) string {
|
||||
h := hmac.New(sha256.New, m.hmacKey)
|
||||
h.Write(payload)
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
125
internal/config/config.go
Normal file
125
internal/config/config.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Server ServerConfig `toml:"server"`
|
||||
Security SecurityConfig `toml:"security"`
|
||||
Apify ApifyConfig `toml:"apify"`
|
||||
Ntfy NtfyConfig `toml:"ntfy"`
|
||||
Scheduler SchedulerConfig `toml:"scheduler"`
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
Port int `toml:"port"`
|
||||
DBPath string `toml:"db_path"`
|
||||
}
|
||||
|
||||
type SecurityConfig struct {
|
||||
SessionSecret string `toml:"session_secret"`
|
||||
EncryptionKey string `toml:"encryption_key"`
|
||||
}
|
||||
|
||||
type ApifyConfig struct {
|
||||
APIKey string `toml:"api_key"`
|
||||
Actors ActorConfig `toml:"actors"`
|
||||
Proxy ProxyConfig `toml:"proxy"`
|
||||
}
|
||||
|
||||
// ProxyConfig controls the proxyConfiguration block passed to apify actors
|
||||
// that scrape sites which block datacenter IPs (e.g. eBay returns 403 without
|
||||
// a residential proxy).
|
||||
type ProxyConfig struct {
|
||||
UseApifyProxy bool `toml:"use_apify_proxy"`
|
||||
Groups []string `toml:"groups"`
|
||||
Country string `toml:"country"`
|
||||
}
|
||||
|
||||
type ActorConfig struct {
|
||||
ActiveListings string `toml:"active_listings"`
|
||||
SoldListings string `toml:"sold_listings"`
|
||||
PriceComparison string `toml:"price_comparison"`
|
||||
YahooAuctionsJP string `toml:"yahoo_auctions_jp"`
|
||||
YahooAuctionsJPSold string `toml:"yahoo_auctions_jp_sold"`
|
||||
MercariJP string `toml:"mercari_jp"`
|
||||
}
|
||||
|
||||
type NtfyConfig struct {
|
||||
BaseURL string `toml:"base_url"`
|
||||
DefaultTopic string `toml:"default_topic"`
|
||||
}
|
||||
|
||||
type SchedulerConfig struct {
|
||||
GlobalPollIntervalMinutes int `toml:"global_poll_interval_minutes"`
|
||||
MatchConfidenceThreshold float64 `toml:"match_confidence_threshold"`
|
||||
}
|
||||
|
||||
func Load(path string) (*Config, error) {
|
||||
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("config file not found at %s. Copy config.toml.example to that path and fill it in", path)
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("stat config: %w", err)
|
||||
}
|
||||
|
||||
var c Config
|
||||
if _, err := toml.DecodeFile(path, &c); err != nil {
|
||||
return nil, fmt.Errorf("parse config: %w", err)
|
||||
}
|
||||
|
||||
if err := c.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
func (c *Config) validate() error {
|
||||
if len(c.Security.SessionSecret) < 32 {
|
||||
return errors.New("security.session_secret must be at least 32 bytes")
|
||||
}
|
||||
if len(c.Security.EncryptionKey) < 32 {
|
||||
return errors.New("security.encryption_key must be at least 32 bytes")
|
||||
}
|
||||
if c.Security.SessionSecret == c.Security.EncryptionKey {
|
||||
return errors.New("security.session_secret and security.encryption_key must not be equal")
|
||||
}
|
||||
if c.Server.DBPath == "" {
|
||||
return errors.New("server.db_path must be set")
|
||||
}
|
||||
dir := filepath.Dir(c.Server.DBPath)
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
if err := checkWritable(dir); err != nil {
|
||||
return fmt.Errorf("server.db_path directory %s not writable: %w", dir, err)
|
||||
}
|
||||
if c.Server.Port == 0 {
|
||||
c.Server.Port = 8080
|
||||
}
|
||||
if c.Scheduler.GlobalPollIntervalMinutes == 0 {
|
||||
c.Scheduler.GlobalPollIntervalMinutes = 60
|
||||
}
|
||||
if c.Scheduler.MatchConfidenceThreshold == 0 {
|
||||
c.Scheduler.MatchConfidenceThreshold = 0.6
|
||||
}
|
||||
if c.Ntfy.DefaultTopic == "" {
|
||||
c.Ntfy.DefaultTopic = "veola"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkWritable(dir string) error {
|
||||
f, err := os.CreateTemp(dir, ".veola-write-test-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := f.Name()
|
||||
f.Close()
|
||||
return os.Remove(name)
|
||||
}
|
||||
94
internal/crypto/crypto.go
Normal file
94
internal/crypto/crypto.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/crypto/hkdf"
|
||||
)
|
||||
|
||||
const (
|
||||
prefix = "enc:"
|
||||
infoLabel = "veola-v1"
|
||||
keyLen = 32
|
||||
nonceLen = 12
|
||||
)
|
||||
|
||||
// DeriveKey derives a 32-byte AES key from raw key material via HKDF-SHA256.
|
||||
func DeriveKey(rawKey []byte) ([]byte, error) {
|
||||
if len(rawKey) < 32 {
|
||||
return nil, errors.New("raw encryption key must be at least 32 bytes")
|
||||
}
|
||||
r := hkdf.New(sha256.New, rawKey, nil, []byte(infoLabel))
|
||||
out := make([]byte, keyLen)
|
||||
if _, err := io.ReadFull(r, out); err != nil {
|
||||
return nil, fmt.Errorf("derive key: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func Encrypt(key []byte, plaintext string) (string, error) {
|
||||
if plaintext == "" {
|
||||
return "", nil
|
||||
}
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
nonce := make([]byte, nonceLen)
|
||||
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
|
||||
return "", err
|
||||
}
|
||||
ct := gcm.Seal(nil, nonce, []byte(plaintext), nil)
|
||||
buf := make([]byte, 0, len(nonce)+len(ct))
|
||||
buf = append(buf, nonce...)
|
||||
buf = append(buf, ct...)
|
||||
return prefix + base64.StdEncoding.EncodeToString(buf), nil
|
||||
}
|
||||
|
||||
func Decrypt(key []byte, value string) (string, error) {
|
||||
if value == "" {
|
||||
return "", nil
|
||||
}
|
||||
if !IsEncrypted(value) {
|
||||
// Plaintext passthrough lets us decrypt rows that pre-date encryption
|
||||
// without a separate migration.
|
||||
return value, nil
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(value, prefix))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("decode ciphertext: %w", err)
|
||||
}
|
||||
if len(raw) < nonceLen {
|
||||
return "", errors.New("ciphertext too short")
|
||||
}
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
nonce, ct := raw[:nonceLen], raw[nonceLen:]
|
||||
pt, err := gcm.Open(nil, nonce, ct, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("decrypt: %w", err)
|
||||
}
|
||||
return string(pt), nil
|
||||
}
|
||||
|
||||
func IsEncrypted(value string) bool {
|
||||
return strings.HasPrefix(value, prefix)
|
||||
}
|
||||
86
internal/crypto/crypto_test.go
Normal file
86
internal/crypto/crypto_test.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func testKey(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
k, err := DeriveKey([]byte("0123456789abcdef0123456789abcdef-aaa"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
func TestRoundTrip(t *testing.T) {
|
||||
k := testKey(t)
|
||||
cases := []string{
|
||||
"hello",
|
||||
"",
|
||||
"ツインビー グラディウス パロディウス",
|
||||
strings.Repeat("a", 4096),
|
||||
"line1\nline2\ttab",
|
||||
}
|
||||
for _, pt := range cases {
|
||||
ct, err := Encrypt(k, pt)
|
||||
if err != nil {
|
||||
t.Fatalf("encrypt %q: %v", pt, err)
|
||||
}
|
||||
if pt != "" && !IsEncrypted(ct) {
|
||||
t.Errorf("expected enc: prefix on %q", ct)
|
||||
}
|
||||
got, err := Decrypt(k, ct)
|
||||
if err != nil {
|
||||
t.Fatalf("decrypt: %v", err)
|
||||
}
|
||||
if got != pt {
|
||||
t.Errorf("round-trip mismatch: got %q want %q", got, pt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonceUnique(t *testing.T) {
|
||||
k := testKey(t)
|
||||
a, _ := Encrypt(k, "same plaintext")
|
||||
b, _ := Encrypt(k, "same plaintext")
|
||||
if a == b {
|
||||
t.Error("two encryptions of the same plaintext produced identical ciphertext (nonce not random)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTamperRejected(t *testing.T) {
|
||||
k := testKey(t)
|
||||
ct, _ := Encrypt(k, "secret")
|
||||
tampered := ct[:len(ct)-2] + "AA"
|
||||
if _, err := Decrypt(k, tampered); err == nil {
|
||||
t.Error("expected tampered ciphertext to fail decryption")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrongKeyRejected(t *testing.T) {
|
||||
k1 := testKey(t)
|
||||
k2, _ := DeriveKey([]byte("a-different-32-byte-key-aaaaaaaaaaaa"))
|
||||
ct, _ := Encrypt(k1, "secret")
|
||||
if _, err := Decrypt(k2, ct); err == nil {
|
||||
t.Error("expected decryption with wrong key to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlaintextPassthrough(t *testing.T) {
|
||||
k := testKey(t)
|
||||
got, err := Decrypt(k, "not-encrypted")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "not-encrypted" {
|
||||
t.Errorf("plaintext passthrough failed: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveKeyRequiresMinLength(t *testing.T) {
|
||||
if _, err := DeriveKey([]byte("too short")); err == nil {
|
||||
t.Error("expected error on short key material")
|
||||
}
|
||||
}
|
||||
65
internal/db/db.go
Normal file
65
internal/db/db.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
//go:embed schema.sql
|
||||
var schemaSQL string
|
||||
|
||||
func Open(path string) (*sql.DB, error) {
|
||||
dsn := fmt.Sprintf("file:%s?_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)", path)
|
||||
conn, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open sqlite: %w", err)
|
||||
}
|
||||
if err := conn.Ping(); err != nil {
|
||||
conn.Close()
|
||||
return nil, fmt.Errorf("ping sqlite: %w", err)
|
||||
}
|
||||
if _, err := conn.Exec(schemaSQL); err != nil {
|
||||
conn.Close()
|
||||
return nil, fmt.Errorf("apply schema: %w", err)
|
||||
}
|
||||
if err := addColumnIfMissing(conn, "items", "min_price", "REAL"); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := addColumnIfMissing(conn, "items", "exclude_keywords", "TEXT"); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
if err := addColumnIfMissing(conn, "results", "matched_query", "TEXT"); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func addColumnIfMissing(conn *sql.DB, table, column, typ string) error {
|
||||
rows, err := conn.Query(fmt.Sprintf(`PRAGMA table_info(%s)`, table))
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect %s: %w", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, ctype string
|
||||
var notnull, pk int
|
||||
var dflt sql.NullString
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
||||
return err
|
||||
}
|
||||
if name == column {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if _, err := conn.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN %s %s`, table, column, typ)); err != nil {
|
||||
return fmt.Errorf("add column %s.%s: %w", table, column, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
97
internal/db/dedup_test.go
Normal file
97
internal/db/dedup_test.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"veola/internal/crypto"
|
||||
"veola/internal/models"
|
||||
)
|
||||
|
||||
func newTestStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
conn, err := Open(filepath.Join(dir, "test.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { conn.Close() })
|
||||
key, _ := crypto.DeriveKey([]byte("0123456789abcdef0123456789abcdef-aaa"))
|
||||
return NewStore(conn, key)
|
||||
}
|
||||
|
||||
func TestDedupByItemAndURL(t *testing.T) {
|
||||
if os.Getenv("CI_SKIP_SQLITE") != "" {
|
||||
t.Skip()
|
||||
}
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
id, err := s.CreateItem(ctx, &models.Item{
|
||||
Name: "TwinBee", NtfyTopic: "veola", Active: true,
|
||||
PollIntervalMinutes: 60, NtfyPriority: "default",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r := &models.Result{
|
||||
ItemID: id,
|
||||
Title: "TwinBee Famicom",
|
||||
Currency: "USD",
|
||||
URL: "https://example.com/listing/1",
|
||||
}
|
||||
if _, err := s.InsertResult(ctx, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
exists, err := s.ResultExists(ctx, id, "https://example.com/listing/1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Error("expected result to be detected as duplicate")
|
||||
}
|
||||
|
||||
missing, _ := s.ResultExists(ctx, id, "https://example.com/listing/2")
|
||||
if missing {
|
||||
t.Error("expected unknown URL to not be flagged as duplicate")
|
||||
}
|
||||
|
||||
// Different item, same URL should not collide.
|
||||
id2, _ := s.CreateItem(ctx, &models.Item{Name: "Other", NtfyTopic: "veola", PollIntervalMinutes: 60, NtfyPriority: "default"})
|
||||
other, _ := s.ResultExists(ctx, id2, "https://example.com/listing/1")
|
||||
if other {
|
||||
t.Error("dedup should be scoped to item_id")
|
||||
}
|
||||
|
||||
// Empty URL should not collide.
|
||||
emptyExists, _ := s.ResultExists(ctx, id, "")
|
||||
if emptyExists {
|
||||
t.Error("empty URL should never be flagged as duplicate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCJKRoundTrip(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
id, err := s.CreateItem(ctx, &models.Item{
|
||||
Name: "ツインビー",
|
||||
SearchQuery: "ツインビー グラディウス パロディウス",
|
||||
NtfyTopic: "veola",
|
||||
Active: true,
|
||||
PollIntervalMinutes: 60, NtfyPriority: "default",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := s.GetItem(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Name != "ツインビー" || got.SearchQuery != "ツインビー グラディウス パロディウス" {
|
||||
t.Errorf("CJK round-trip failed: name=%q query=%q", got.Name, got.SearchQuery)
|
||||
}
|
||||
}
|
||||
747
internal/db/queries.go
Normal file
747
internal/db/queries.go
Normal file
@@ -0,0 +1,747 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"veola/internal/crypto"
|
||||
"veola/internal/models"
|
||||
)
|
||||
|
||||
// Store wraps a *sql.DB with the encryption key used for column-level crypto.
|
||||
type Store struct {
|
||||
DB *sql.DB
|
||||
Key []byte
|
||||
}
|
||||
|
||||
func NewStore(db *sql.DB, key []byte) *Store {
|
||||
return &Store{DB: db, Key: key}
|
||||
}
|
||||
|
||||
// enc encrypts plaintext, logging and returning empty string on failure.
|
||||
func (s *Store) enc(plain string) string {
|
||||
if plain == "" {
|
||||
return ""
|
||||
}
|
||||
v, err := crypto.Encrypt(s.Key, plain)
|
||||
if err != nil {
|
||||
slog.Error("encrypt failed", "err", err)
|
||||
return ""
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// dec decrypts a value; on failure returns "" per spec line 333.
|
||||
func (s *Store) dec(v string) string {
|
||||
if v == "" {
|
||||
return ""
|
||||
}
|
||||
out, err := crypto.Decrypt(s.Key, v)
|
||||
if err != nil {
|
||||
slog.Error("decrypt failed", "err", err)
|
||||
return ""
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func nullStr(s string) sql.NullString {
|
||||
if s == "" {
|
||||
return sql.NullString{}
|
||||
}
|
||||
return sql.NullString{String: s, Valid: true}
|
||||
}
|
||||
|
||||
func nullFloat(f *float64) sql.NullFloat64 {
|
||||
if f == nil {
|
||||
return sql.NullFloat64{}
|
||||
}
|
||||
return sql.NullFloat64{Float64: *f, Valid: true}
|
||||
}
|
||||
|
||||
func nullTime(t *time.Time) sql.NullTime {
|
||||
if t == nil {
|
||||
return sql.NullTime{}
|
||||
}
|
||||
return sql.NullTime{Time: *t, Valid: true}
|
||||
}
|
||||
|
||||
func ptrFloat(f sql.NullFloat64) *float64 {
|
||||
if !f.Valid {
|
||||
return nil
|
||||
}
|
||||
v := f.Float64
|
||||
return &v
|
||||
}
|
||||
|
||||
func ptrTime(t sql.NullTime) *time.Time {
|
||||
if !t.Valid {
|
||||
return nil
|
||||
}
|
||||
v := t.Time
|
||||
return &v
|
||||
}
|
||||
|
||||
// ============ users ============
|
||||
|
||||
func (s *Store) UserCount(ctx context.Context) (int, error) {
|
||||
var n int
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT COUNT(*) FROM users`).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (s *Store) CreateUser(ctx context.Context, username, hash string, role models.Role) (int64, error) {
|
||||
res, err := s.DB.ExecContext(ctx,
|
||||
`INSERT INTO users (username, password_hash, role) VALUES (?, ?, ?)`,
|
||||
username, hash, string(role))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.LastInsertId()
|
||||
}
|
||||
|
||||
func (s *Store) GetUserByUsername(ctx context.Context, username string) (*models.User, error) {
|
||||
row := s.DB.QueryRowContext(ctx,
|
||||
`SELECT id, username, password_hash, role, created_at FROM users WHERE username = ?`,
|
||||
username)
|
||||
var u models.User
|
||||
var role string
|
||||
if err := row.Scan(&u.ID, &u.Username, &u.PasswordHash, &role, &u.CreatedAt); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
u.Role = models.Role(role)
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetUserByID(ctx context.Context, id int64) (*models.User, error) {
|
||||
row := s.DB.QueryRowContext(ctx,
|
||||
`SELECT id, username, password_hash, role, created_at FROM users WHERE id = ?`, id)
|
||||
var u models.User
|
||||
var role string
|
||||
if err := row.Scan(&u.ID, &u.Username, &u.PasswordHash, &role, &u.CreatedAt); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
u.Role = models.Role(role)
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListUsers(ctx context.Context) ([]models.User, error) {
|
||||
rows, err := s.DB.QueryContext(ctx,
|
||||
`SELECT id, username, password_hash, role, created_at FROM users ORDER BY id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []models.User
|
||||
for rows.Next() {
|
||||
var u models.User
|
||||
var role string
|
||||
if err := rows.Scan(&u.ID, &u.Username, &u.PasswordHash, &role, &u.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u.Role = models.Role(role)
|
||||
out = append(out, u)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) UpdateUserPassword(ctx context.Context, id int64, hash string) error {
|
||||
_, err := s.DB.ExecContext(ctx, `UPDATE users SET password_hash = ? WHERE id = ?`, hash, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) DeleteUser(ctx context.Context, id int64) error {
|
||||
_, err := s.DB.ExecContext(ctx, `DELETE FROM users WHERE id = ?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// ============ settings ============
|
||||
|
||||
func (s *Store) GetSetting(ctx context.Context, key string) (string, error) {
|
||||
var v sql.NullString
|
||||
err := s.DB.QueryRowContext(ctx, `SELECT value FROM settings WHERE key = ?`, key).Scan(&v)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !v.Valid {
|
||||
return "", nil
|
||||
}
|
||||
return s.dec(v.String), nil
|
||||
}
|
||||
|
||||
func (s *Store) GetAllSettings(ctx context.Context) (map[string]string, error) {
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT key, value FROM settings`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]string)
|
||||
for rows.Next() {
|
||||
var k string
|
||||
var v sql.NullString
|
||||
if err := rows.Scan(&k, &v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v.Valid {
|
||||
out[k] = s.dec(v.String)
|
||||
} else {
|
||||
out[k] = ""
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) SetSetting(ctx context.Context, key, value string) error {
|
||||
enc := s.enc(value)
|
||||
_, err := s.DB.ExecContext(ctx, `
|
||||
INSERT INTO settings (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = CURRENT_TIMESTAMP
|
||||
`, key, enc)
|
||||
return err
|
||||
}
|
||||
|
||||
// ============ items ============
|
||||
|
||||
func (s *Store) CreateItem(ctx context.Context, it *models.Item) (int64, error) {
|
||||
res, err := s.DB.ExecContext(ctx, `
|
||||
INSERT INTO items (
|
||||
name, search_query, url, category, target_price, ntfy_topic, ntfy_priority,
|
||||
poll_interval_minutes, include_out_of_stock, min_price, exclude_keywords,
|
||||
listing_type,
|
||||
actor_active, actor_sold, actor_price_compare, use_price_comparison,
|
||||
active, best_price, best_price_store, best_price_url, best_price_image_url,
|
||||
best_price_title, last_polled_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`,
|
||||
it.Name, s.enc(it.SearchQuery), nullStr(it.URL), nullStr(it.Category),
|
||||
nullFloat(it.TargetPrice), s.enc(it.NtfyTopic), it.NtfyPriority,
|
||||
it.PollIntervalMinutes, boolToInt(it.IncludeOutOfStock),
|
||||
nullFloat(it.MinPrice), nullStr(s.enc(it.ExcludeKeywords)),
|
||||
nullStr(it.ListingType),
|
||||
nullStr(it.ActorActive), nullStr(it.ActorSold), nullStr(it.ActorPriceCompare),
|
||||
boolToInt(it.UsePriceComparison), boolToInt(it.Active),
|
||||
nullFloat(it.BestPrice), nullStr(it.BestPriceStore),
|
||||
nullStr(s.enc(it.BestPriceURL)), nullStr(s.enc(it.BestPriceImageURL)),
|
||||
nullStr(s.enc(it.BestPriceTitle)), nullTime(it.LastPolledAt),
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
id, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := s.SetItemMarketplaces(ctx, id, it.Marketplaces); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (s *Store) UpdateItem(ctx context.Context, it *models.Item) error {
|
||||
if _, err := s.DB.ExecContext(ctx, `
|
||||
UPDATE items SET
|
||||
name = ?, search_query = ?, url = ?, category = ?, target_price = ?,
|
||||
ntfy_topic = ?, ntfy_priority = ?, poll_interval_minutes = ?,
|
||||
include_out_of_stock = ?, min_price = ?, exclude_keywords = ?,
|
||||
listing_type = ?,
|
||||
actor_active = ?, actor_sold = ?, actor_price_compare = ?,
|
||||
use_price_comparison = ?, active = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
`,
|
||||
it.Name, s.enc(it.SearchQuery), nullStr(it.URL), nullStr(it.Category),
|
||||
nullFloat(it.TargetPrice), s.enc(it.NtfyTopic), it.NtfyPriority,
|
||||
it.PollIntervalMinutes, boolToInt(it.IncludeOutOfStock),
|
||||
nullFloat(it.MinPrice), nullStr(s.enc(it.ExcludeKeywords)),
|
||||
nullStr(it.ListingType),
|
||||
nullStr(it.ActorActive), nullStr(it.ActorSold), nullStr(it.ActorPriceCompare),
|
||||
boolToInt(it.UsePriceComparison), boolToInt(it.Active),
|
||||
it.ID,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.SetItemMarketplaces(ctx, it.ID, it.Marketplaces)
|
||||
}
|
||||
|
||||
// SetItemMarketplaces replaces the marketplace list for an item. Order is
|
||||
// preserved via the `position` column.
|
||||
func (s *Store) SetItemMarketplaces(ctx context.Context, itemID int64, markets []string) error {
|
||||
tx, err := s.DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM item_marketplaces WHERE item_id = ?`, itemID); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, m := range markets {
|
||||
if m == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx,
|
||||
`INSERT INTO item_marketplaces (item_id, position, marketplace) VALUES (?, ?, ?)`,
|
||||
itemID, i, m); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// getItemMarketplaces returns the ordered marketplace list for one item.
|
||||
func (s *Store) getItemMarketplaces(ctx context.Context, itemID int64) ([]string, error) {
|
||||
rows, err := s.DB.QueryContext(ctx,
|
||||
`SELECT marketplace FROM item_marketplaces WHERE item_id = ? ORDER BY position`, itemID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var m string
|
||||
if err := rows.Scan(&m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// loadMarketplacesForItems bulk-loads marketplaces for a list of items in one
|
||||
// query. Returns a map keyed by item ID.
|
||||
func (s *Store) loadMarketplacesForItems(ctx context.Context, ids []int64) (map[int64][]string, error) {
|
||||
out := make(map[int64][]string, len(ids))
|
||||
if len(ids) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
placeholders := make([]string, len(ids))
|
||||
args := make([]any, len(ids))
|
||||
for i, id := range ids {
|
||||
placeholders[i] = "?"
|
||||
args[i] = id
|
||||
}
|
||||
q := `SELECT item_id, marketplace FROM item_marketplaces WHERE item_id IN (` +
|
||||
strings.Join(placeholders, ",") + `) ORDER BY item_id, position`
|
||||
rows, err := s.DB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var m string
|
||||
if err := rows.Scan(&id, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[id] = append(out[id], m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) SetItemActive(ctx context.Context, id int64, active bool) error {
|
||||
_, err := s.DB.ExecContext(ctx,
|
||||
`UPDATE items SET active = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?`,
|
||||
boolToInt(active), id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) DeleteItem(ctx context.Context, id int64) error {
|
||||
_, err := s.DB.ExecContext(ctx, `DELETE FROM items WHERE id = ?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) GetItem(ctx context.Context, id int64) (*models.Item, error) {
|
||||
row := s.DB.QueryRowContext(ctx, itemSelect+` WHERE id = ?`, id)
|
||||
it, err := scanItem(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
s.decryptItem(it)
|
||||
markets, err := s.getItemMarketplaces(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
it.Marketplaces = markets
|
||||
return it, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListItems(ctx context.Context) ([]models.Item, error) {
|
||||
return s.listItemsWhere(ctx, itemSelect+` ORDER BY name COLLATE NOCASE`)
|
||||
}
|
||||
|
||||
func (s *Store) ListActiveItems(ctx context.Context) ([]models.Item, error) {
|
||||
return s.listItemsWhere(ctx, itemSelect+` WHERE active = 1 ORDER BY id`)
|
||||
}
|
||||
|
||||
func (s *Store) listItemsWhere(ctx context.Context, q string, args ...any) ([]models.Item, error) {
|
||||
rows, err := s.DB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []models.Item
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
it, err := scanItem(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.decryptItem(it)
|
||||
out = append(out, *it)
|
||||
ids = append(ids, it.ID)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
markets, err := s.loadMarketplacesForItems(ctx, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range out {
|
||||
out[i].Marketplaces = markets[out[i].ID]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListCategories(ctx context.Context) ([]string, error) {
|
||||
rows, err := s.DB.QueryContext(ctx, `SELECT DISTINCT category FROM items WHERE category IS NOT NULL AND category != '' ORDER BY category COLLATE NOCASE`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var c string
|
||||
if err := rows.Scan(&c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateItemPollResult writes best-price fields, last_polled_at, last_poll_error.
|
||||
func (s *Store) UpdateItemPollResult(ctx context.Context, id int64, best *models.Item, errMsg string) error {
|
||||
var (
|
||||
bestPrice sql.NullFloat64
|
||||
bestStore, bestURL, bestImage, bestTitle, errField sql.NullString
|
||||
)
|
||||
if best != nil {
|
||||
bestPrice = nullFloat(best.BestPrice)
|
||||
bestStore = nullStr(best.BestPriceStore)
|
||||
bestURL = nullStr(s.enc(best.BestPriceURL))
|
||||
bestImage = nullStr(s.enc(best.BestPriceImageURL))
|
||||
bestTitle = nullStr(s.enc(best.BestPriceTitle))
|
||||
}
|
||||
if errMsg != "" {
|
||||
errField = nullStr(s.enc(errMsg))
|
||||
}
|
||||
_, err := s.DB.ExecContext(ctx, `
|
||||
UPDATE items SET
|
||||
best_price = ?, best_price_store = ?, best_price_url = ?,
|
||||
best_price_image_url = ?, best_price_title = ?,
|
||||
last_polled_at = CURRENT_TIMESTAMP, last_poll_error = ?
|
||||
WHERE id = ?
|
||||
`, bestPrice, bestStore, bestURL, bestImage, bestTitle, errField, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const itemSelect = `
|
||||
SELECT id, name, search_query, url, category, target_price, ntfy_topic, ntfy_priority,
|
||||
poll_interval_minutes, include_out_of_stock, min_price, exclude_keywords,
|
||||
listing_type,
|
||||
actor_active, actor_sold, actor_price_compare, use_price_comparison,
|
||||
active, last_polled_at, last_poll_error, best_price, best_price_store,
|
||||
best_price_url, best_price_image_url, best_price_title, created_at, updated_at
|
||||
FROM items
|
||||
`
|
||||
|
||||
type rowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
func scanItem(r rowScanner) (*models.Item, error) {
|
||||
var (
|
||||
it models.Item
|
||||
searchQuery, urlS, category, listingType sql.NullString
|
||||
excludeKw sql.NullString
|
||||
actorA, actorS, actorP sql.NullString
|
||||
ntfyTopic, lastPollErr sql.NullString
|
||||
bestStore, bestURL, bestImage, bestTitle sql.NullString
|
||||
targetPrice, minPrice, bestPrice sql.NullFloat64
|
||||
includeOOS, usePC, active int
|
||||
lastPolledAt sql.NullTime
|
||||
)
|
||||
if err := r.Scan(
|
||||
&it.ID, &it.Name, &searchQuery, &urlS, &category, &targetPrice, &ntfyTopic, &it.NtfyPriority,
|
||||
&it.PollIntervalMinutes, &includeOOS, &minPrice, &excludeKw,
|
||||
&listingType,
|
||||
&actorA, &actorS, &actorP, &usePC,
|
||||
&active, &lastPolledAt, &lastPollErr, &bestPrice, &bestStore,
|
||||
&bestURL, &bestImage, &bestTitle, &it.CreatedAt, &it.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
it.ExcludeKeywords = excludeKw.String
|
||||
it.MinPrice = ptrFloat(minPrice)
|
||||
it.SearchQuery = searchQuery.String
|
||||
it.URL = urlS.String
|
||||
it.Category = category.String
|
||||
it.ListingType = listingType.String
|
||||
it.ActorActive = actorA.String
|
||||
it.ActorSold = actorS.String
|
||||
it.ActorPriceCompare = actorP.String
|
||||
it.NtfyTopic = ntfyTopic.String
|
||||
it.LastPollError = lastPollErr.String
|
||||
it.BestPriceStore = bestStore.String
|
||||
it.BestPriceURL = bestURL.String
|
||||
it.BestPriceImageURL = bestImage.String
|
||||
it.BestPriceTitle = bestTitle.String
|
||||
it.TargetPrice = ptrFloat(targetPrice)
|
||||
it.BestPrice = ptrFloat(bestPrice)
|
||||
it.IncludeOutOfStock = includeOOS != 0
|
||||
it.UsePriceComparison = usePC != 0
|
||||
it.Active = active != 0
|
||||
it.LastPolledAt = ptrTime(lastPolledAt)
|
||||
return &it, nil
|
||||
}
|
||||
|
||||
func (s *Store) decryptItem(it *models.Item) *models.Item {
|
||||
it.SearchQuery = s.dec(it.SearchQuery)
|
||||
it.ExcludeKeywords = s.dec(it.ExcludeKeywords)
|
||||
it.NtfyTopic = s.dec(it.NtfyTopic)
|
||||
it.LastPollError = s.dec(it.LastPollError)
|
||||
it.BestPriceURL = s.dec(it.BestPriceURL)
|
||||
it.BestPriceImageURL = s.dec(it.BestPriceImageURL)
|
||||
it.BestPriceTitle = s.dec(it.BestPriceTitle)
|
||||
return it
|
||||
}
|
||||
|
||||
// ============ results ============
|
||||
|
||||
func (s *Store) InsertResult(ctx context.Context, r *models.Result) (int64, error) {
|
||||
res, err := s.DB.ExecContext(ctx, `
|
||||
INSERT INTO results (item_id, title, price, currency, url, source, image_url, matched_query, alerted, found_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
`,
|
||||
r.ItemID, s.enc(r.Title), nullFloat(r.Price), r.Currency,
|
||||
nullStr(r.URL), nullStr(r.Source), s.enc(r.ImageURL),
|
||||
nullStr(s.enc(r.MatchedQuery)),
|
||||
boolToInt(r.Alerted),
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.LastInsertId()
|
||||
}
|
||||
|
||||
// ResultExists returns true if a row with this item_id and url already exists.
|
||||
// URL is stored as plaintext per agreed deviation #1, so equality works.
|
||||
func (s *Store) ResultExists(ctx context.Context, itemID int64, url string) (bool, error) {
|
||||
if url == "" {
|
||||
return false, nil
|
||||
}
|
||||
var n int
|
||||
err := s.DB.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM results WHERE item_id = ? AND url = ?`, itemID, url,
|
||||
).Scan(&n)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
func (s *Store) MarkResultAlerted(ctx context.Context, id int64) error {
|
||||
_, err := s.DB.ExecContext(ctx, `UPDATE results SET alerted = 1 WHERE id = ?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
type ResultsQuery struct {
|
||||
ItemID int64 // 0 = all items
|
||||
Limit int
|
||||
Offset int
|
||||
Order string // "price_asc", "price_desc", "found_desc" (default), "found_asc"
|
||||
}
|
||||
|
||||
func (s *Store) ListResults(ctx context.Context, q ResultsQuery) ([]models.Result, error) {
|
||||
order := `found_at DESC`
|
||||
switch q.Order {
|
||||
case "price_asc":
|
||||
order = `price ASC NULLS LAST`
|
||||
case "price_desc":
|
||||
order = `price DESC NULLS LAST`
|
||||
case "found_asc":
|
||||
order = `found_at ASC`
|
||||
}
|
||||
limit := q.Limit
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
args := []any{}
|
||||
where := ""
|
||||
if q.ItemID != 0 {
|
||||
where = `WHERE item_id = ?`
|
||||
args = append(args, q.ItemID)
|
||||
}
|
||||
args = append(args, limit, q.Offset)
|
||||
rows, err := s.DB.QueryContext(ctx, fmt.Sprintf(`
|
||||
SELECT id, item_id, title, price, currency, url, source, image_url, matched_query, alerted, found_at
|
||||
FROM results %s ORDER BY %s LIMIT ? OFFSET ?
|
||||
`, where, order), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []models.Result
|
||||
for rows.Next() {
|
||||
var (
|
||||
r models.Result
|
||||
title, urlS, source, imageS, matchQ sql.NullString
|
||||
price sql.NullFloat64
|
||||
alerted int
|
||||
)
|
||||
if err := rows.Scan(&r.ID, &r.ItemID, &title, &price, &r.Currency, &urlS, &source, &imageS, &matchQ, &alerted, &r.FoundAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.Title = s.dec(title.String)
|
||||
r.URL = urlS.String
|
||||
r.Source = source.String
|
||||
r.ImageURL = s.dec(imageS.String)
|
||||
r.MatchedQuery = s.dec(matchQ.String)
|
||||
r.Price = ptrFloat(price)
|
||||
r.Alerted = alerted != 0
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) CountResults(ctx context.Context, itemID int64) (int, error) {
|
||||
var n int
|
||||
q := `SELECT COUNT(*) FROM results`
|
||||
args := []any{}
|
||||
if itemID != 0 {
|
||||
q += ` WHERE item_id = ?`
|
||||
args = append(args, itemID)
|
||||
}
|
||||
err := s.DB.QueryRowContext(ctx, q, args...).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ============ price_history ============
|
||||
|
||||
func (s *Store) InsertPricePoint(ctx context.Context, p *models.PricePoint) error {
|
||||
if p.PolledAt.IsZero() {
|
||||
_, err := s.DB.ExecContext(ctx,
|
||||
`INSERT INTO price_history (item_id, price, store) VALUES (?, ?, ?)`,
|
||||
p.ItemID, p.Price, s.enc(p.Store))
|
||||
return err
|
||||
}
|
||||
_, err := s.DB.ExecContext(ctx,
|
||||
`INSERT INTO price_history (item_id, price, store, polled_at) VALUES (?, ?, ?, ?)`,
|
||||
p.ItemID, p.Price, s.enc(p.Store), p.PolledAt)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) ListPriceHistory(ctx context.Context, itemID int64) ([]models.PricePoint, error) {
|
||||
rows, err := s.DB.QueryContext(ctx,
|
||||
`SELECT id, item_id, price, store, polled_at FROM price_history WHERE item_id = ? ORDER BY polled_at ASC`,
|
||||
itemID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []models.PricePoint
|
||||
for rows.Next() {
|
||||
var p models.PricePoint
|
||||
var store sql.NullString
|
||||
if err := rows.Scan(&p.ID, &p.ItemID, &p.Price, &store, &p.PolledAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Store = s.dec(store.String)
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ============ stats ============
|
||||
|
||||
type DashboardStats struct {
|
||||
TotalItems int
|
||||
ActiveItems int
|
||||
ResultsToday int
|
||||
AlertsToday int
|
||||
PotentialSpend float64
|
||||
PricedItemCount int
|
||||
UnpricedCount int
|
||||
MoneySaved float64
|
||||
SavedItemCount int
|
||||
}
|
||||
|
||||
func (s *Store) GetDashboardStats(ctx context.Context) (*DashboardStats, error) {
|
||||
d := &DashboardStats{}
|
||||
queries := map[string]any{
|
||||
`SELECT COUNT(*) FROM items`: &d.TotalItems,
|
||||
`SELECT COUNT(*) FROM items WHERE active = 1`: &d.ActiveItems,
|
||||
`SELECT COUNT(*) FROM results WHERE found_at >= datetime('now', '-1 day')`: &d.ResultsToday,
|
||||
`SELECT COUNT(*) FROM results WHERE alerted = 1 AND found_at >= datetime('now', '-1 day')`: &d.AlertsToday,
|
||||
}
|
||||
for q, dst := range queries {
|
||||
if err := s.DB.QueryRowContext(ctx, q).Scan(dst); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := s.DB.QueryRowContext(ctx, `
|
||||
SELECT COALESCE(SUM(best_price), 0), COUNT(*)
|
||||
FROM items WHERE active = 1 AND best_price IS NOT NULL
|
||||
`).Scan(&d.PotentialSpend, &d.PricedItemCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.DB.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM items WHERE active = 1 AND best_price IS NULL
|
||||
`).Scan(&d.UnpricedCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.DB.QueryContext(ctx, `
|
||||
SELECT i.best_price, AVG(p.price) AS avg_price
|
||||
FROM items i
|
||||
JOIN price_history p ON p.item_id = i.id
|
||||
WHERE i.active = 1 AND i.best_price IS NOT NULL
|
||||
GROUP BY i.id
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var bp, avg float64
|
||||
if err := rows.Scan(&bp, &avg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if bp < avg {
|
||||
d.MoneySaved += avg - bp
|
||||
d.SavedItemCount++
|
||||
}
|
||||
}
|
||||
return d, rows.Err()
|
||||
}
|
||||
|
||||
func boolToInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
99
internal/db/schema.sql
Normal file
99
internal/db/schema.sql
Normal file
@@ -0,0 +1,99 @@
|
||||
PRAGMA journal_mode=WAL;
|
||||
PRAGMA foreign_keys=ON;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL,
|
||||
role TEXT NOT NULL DEFAULT 'user',
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS items (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
search_query TEXT,
|
||||
url TEXT,
|
||||
category TEXT,
|
||||
target_price REAL,
|
||||
ntfy_topic TEXT NOT NULL,
|
||||
ntfy_priority TEXT DEFAULT 'default',
|
||||
poll_interval_minutes INTEGER DEFAULT 60,
|
||||
include_out_of_stock INTEGER DEFAULT 0,
|
||||
min_price REAL,
|
||||
exclude_keywords TEXT,
|
||||
listing_type TEXT,
|
||||
actor_active TEXT,
|
||||
actor_sold TEXT,
|
||||
actor_price_compare TEXT,
|
||||
use_price_comparison INTEGER DEFAULT 0,
|
||||
active INTEGER DEFAULT 1,
|
||||
last_polled_at DATETIME,
|
||||
last_poll_error TEXT,
|
||||
best_price REAL,
|
||||
best_price_store TEXT,
|
||||
best_price_url TEXT,
|
||||
best_price_image_url TEXT,
|
||||
best_price_title TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_items_active ON items(active);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS item_marketplaces (
|
||||
item_id INTEGER NOT NULL REFERENCES items(id) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL,
|
||||
marketplace TEXT NOT NULL,
|
||||
PRIMARY KEY (item_id, position)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_item_marketplaces_item ON item_marketplaces(item_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS results (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
item_id INTEGER NOT NULL REFERENCES items(id) ON DELETE CASCADE,
|
||||
title TEXT,
|
||||
price REAL,
|
||||
currency TEXT NOT NULL,
|
||||
url TEXT,
|
||||
source TEXT,
|
||||
image_url TEXT,
|
||||
matched_query TEXT,
|
||||
alerted INTEGER DEFAULT 0,
|
||||
found_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_results_item ON results(item_id, found_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_results_dedup ON results(item_id, url);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS price_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
item_id INTEGER NOT NULL REFERENCES items(id) ON DELETE CASCADE,
|
||||
price REAL NOT NULL,
|
||||
store TEXT,
|
||||
polled_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_price_history_item ON price_history(item_id, polled_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
INSERT OR IGNORE INTO settings (key, value) VALUES
|
||||
('apify_api_key', ''),
|
||||
('ntfy_base_url', ''),
|
||||
('ntfy_default_topic', 'veola'),
|
||||
('global_poll_interval_minutes', '60'),
|
||||
('match_confidence_threshold', '0.6');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token TEXT PRIMARY KEY,
|
||||
data BLOB NOT NULL,
|
||||
expiry REAL NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_expiry ON sessions(expiry);
|
||||
113
internal/handlers/auth.go
Normal file
113
internal/handlers/auth.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"veola/internal/auth"
|
||||
"veola/internal/models"
|
||||
"veola/templates"
|
||||
)
|
||||
|
||||
func (a *App) GetLogin(w http.ResponseWriter, r *http.Request) {
|
||||
if auth.CurrentUserFromRequest(r) != nil {
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
render(w, r, templates.Login(templates.LoginData{
|
||||
Page: a.page(r, "Sign in", ""),
|
||||
}))
|
||||
}
|
||||
|
||||
func (a *App) PostLogin(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "bad form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
username := strings.TrimSpace(r.PostFormValue("username"))
|
||||
password := r.PostFormValue("password")
|
||||
u, err := a.Store.GetUserByUsername(r.Context(), username)
|
||||
if err != nil || u == nil || !auth.CheckPassword(u.PasswordHash, password) {
|
||||
render(w, r, templates.Login(templates.LoginData{
|
||||
Page: a.page(r, "Sign in", ""),
|
||||
Error: "Invalid username or password",
|
||||
Username: username,
|
||||
}))
|
||||
return
|
||||
}
|
||||
if err := a.Auth.LogIn(r.Context(), u.ID); err != nil {
|
||||
http.Error(w, "session error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) PostLogout(w http.ResponseWriter, r *http.Request) {
|
||||
_ = a.Auth.LogOut(r.Context())
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) GetSetup(w http.ResponseWriter, r *http.Request) {
|
||||
n, err := a.Store.UserCount(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, "db error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
render(w, r, templates.Setup(templates.SetupData{
|
||||
Page: a.page(r, "Setup", ""),
|
||||
}))
|
||||
}
|
||||
|
||||
func (a *App) PostSetup(w http.ResponseWriter, r *http.Request) {
|
||||
n, err := a.Store.UserCount(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, "db error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "bad form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
username := strings.TrimSpace(r.PostFormValue("username"))
|
||||
password := r.PostFormValue("password")
|
||||
confirm := r.PostFormValue("password_confirm")
|
||||
errMsg := ""
|
||||
switch {
|
||||
case username == "":
|
||||
errMsg = "Username is required"
|
||||
case len(password) < auth.MinPasswordLen:
|
||||
errMsg = "Password must be at least 12 characters"
|
||||
case password != confirm:
|
||||
errMsg = "Passwords do not match"
|
||||
}
|
||||
if errMsg != "" {
|
||||
render(w, r, templates.Setup(templates.SetupData{
|
||||
Page: a.page(r, "Setup", ""),
|
||||
Error: errMsg,
|
||||
Username: username,
|
||||
}))
|
||||
return
|
||||
}
|
||||
hash, err := auth.HashPassword(password)
|
||||
if err != nil {
|
||||
http.Error(w, "hash error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if _, err := a.Store.CreateUser(r.Context(), username, hash, models.RoleAdmin); err != nil {
|
||||
render(w, r, templates.Setup(templates.SetupData{
|
||||
Page: a.page(r, "Setup", ""),
|
||||
Error: "Could not create user: " + err.Error(),
|
||||
Username: username,
|
||||
}))
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
}
|
||||
106
internal/handlers/dashboard.go
Normal file
106
internal/handlers/dashboard.go
Normal file
@@ -0,0 +1,106 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"veola/internal/db"
|
||||
"veola/templates"
|
||||
)
|
||||
|
||||
func (a *App) GetDashboard(w http.ResponseWriter, r *http.Request) {
|
||||
d, err := a.dashboardData(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
render(w, r, templates.Dashboard(d))
|
||||
}
|
||||
|
||||
func (a *App) GetDashboardRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
d, err := a.dashboardData(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Render only the inner block by reusing the full body component; the
|
||||
// outer hx-swap="outerHTML" replaces the same wrapper. The full Dashboard
|
||||
// template is overkill but keeps a single source of truth.
|
||||
render(w, r, templates.Dashboard(d))
|
||||
}
|
||||
|
||||
func (a *App) dashboardData(r *http.Request) (templates.DashboardData, error) {
|
||||
stats, err := a.Store.GetDashboardStats(r.Context())
|
||||
if err != nil {
|
||||
return templates.DashboardData{}, err
|
||||
}
|
||||
results, err := a.Store.ListResults(r.Context(), db.ResultsQuery{Limit: 20})
|
||||
if err != nil {
|
||||
return templates.DashboardData{}, err
|
||||
}
|
||||
itemNames := map[int64]string{}
|
||||
all, _ := a.Store.ListItems(r.Context())
|
||||
for _, it := range all {
|
||||
itemNames[it.ID] = it.Name
|
||||
}
|
||||
rrs := make([]templates.ResultRow, 0, len(results))
|
||||
for _, r := range results {
|
||||
rrs = append(rrs, templates.ResultRow{
|
||||
ItemID: r.ItemID,
|
||||
ItemName: itemNames[r.ItemID],
|
||||
Title: r.Title,
|
||||
Price: r.Price,
|
||||
Currency: r.Currency,
|
||||
Source: r.Source,
|
||||
URL: r.URL,
|
||||
FoundAt: r.FoundAt,
|
||||
Alerted: r.Alerted,
|
||||
})
|
||||
}
|
||||
alerts, err := alertsRecent(a, r, itemNames)
|
||||
if err != nil {
|
||||
return templates.DashboardData{}, err
|
||||
}
|
||||
return templates.DashboardData{
|
||||
Page: a.page(r, "Dashboard", "dashboard"),
|
||||
Stats: stats,
|
||||
RecentResults: rrs,
|
||||
RecentAlerts: alerts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func alertsRecent(a *App, r *http.Request, itemNames map[int64]string) ([]templates.AlertRow, error) {
|
||||
rows, err := a.Store.DB.QueryContext(r.Context(), `
|
||||
SELECT item_id, price, currency, found_at FROM results
|
||||
WHERE alerted = 1 ORDER BY found_at DESC LIMIT 5
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []templates.AlertRow
|
||||
for rows.Next() {
|
||||
var (
|
||||
itemID int64
|
||||
price sql.NullFloat64
|
||||
currency string
|
||||
foundAt time.Time
|
||||
)
|
||||
if err := rows.Scan(&itemID, &price, ¤cy, &foundAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var p *float64
|
||||
if price.Valid {
|
||||
v := price.Float64
|
||||
p = &v
|
||||
}
|
||||
out = append(out, templates.AlertRow{
|
||||
ItemName: itemNames[itemID],
|
||||
Price: p,
|
||||
Currency: currency,
|
||||
FoundAt: foundAt,
|
||||
})
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
146
internal/handlers/handlers.go
Normal file
146
internal/handlers/handlers.go
Normal file
@@ -0,0 +1,146 @@
|
||||
// Package handlers wires HTTP routes for Veola. Each file in the package owns
|
||||
// a related cluster of routes; this file holds the shared App container and
|
||||
// helper functions used across all handlers.
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/a-h/templ"
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"veola/internal/apify"
|
||||
"veola/internal/auth"
|
||||
"veola/internal/config"
|
||||
"veola/internal/db"
|
||||
"veola/internal/ntfy"
|
||||
"veola/internal/scheduler"
|
||||
"veola/templates"
|
||||
)
|
||||
|
||||
type App struct {
|
||||
Cfg *config.Config
|
||||
Store *db.Store
|
||||
Auth *auth.Manager
|
||||
Apify *apify.Client
|
||||
Ntfy *ntfy.Client
|
||||
Scheduler *scheduler.Scheduler
|
||||
Preview *PreviewCache
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, store *db.Store, am *auth.Manager, ap *apify.Client, nt *ntfy.Client, sc *scheduler.Scheduler) *App {
|
||||
return &App{
|
||||
Cfg: cfg, Store: store, Auth: am,
|
||||
Apify: ap, Ntfy: nt, Scheduler: sc,
|
||||
Preview: NewPreviewCache(10 * time.Minute),
|
||||
}
|
||||
}
|
||||
|
||||
// Routes returns the chi router with everything wired up.
|
||||
func (a *App) Routes() http.Handler {
|
||||
r := chi.NewRouter()
|
||||
fs := http.FileServer(http.Dir("./static"))
|
||||
r.Handle("/static/*", http.StripPrefix("/static/", fs))
|
||||
|
||||
// All other routes pass through session loading + setup gate.
|
||||
r.Group(func(r chi.Router) {
|
||||
r.Use(a.Auth.Sessions.LoadAndSave)
|
||||
r.Use(a.Auth.LoadUser)
|
||||
r.Use(a.setupGate)
|
||||
|
||||
// Public auth pages.
|
||||
r.Get("/login", a.GetLogin)
|
||||
r.With(a.Auth.CSRFProtect).Post("/login", a.PostLogin)
|
||||
r.Get("/setup", a.GetSetup)
|
||||
r.With(a.Auth.CSRFProtect).Post("/setup", a.PostSetup)
|
||||
|
||||
// Authenticated section.
|
||||
r.Group(func(r chi.Router) {
|
||||
r.Use(a.Auth.RequireAuth)
|
||||
r.With(a.Auth.CSRFProtect).Post("/logout", a.PostLogout)
|
||||
|
||||
r.Get("/", a.GetDashboard)
|
||||
r.Get("/dashboard/refresh", a.GetDashboardRefresh)
|
||||
|
||||
r.Get("/items", a.GetItems)
|
||||
r.Get("/items/new", a.GetNewItem)
|
||||
r.With(a.Auth.CSRFProtect).Post("/items/preview", a.PostPreview)
|
||||
r.With(a.Auth.CSRFProtect).Post("/items", a.PostCreateItem)
|
||||
r.Get("/items/{id}/edit", a.GetEditItem)
|
||||
r.With(a.Auth.CSRFProtect).Post("/items/{id}", a.PostUpdateItem)
|
||||
r.With(a.Auth.CSRFProtect).Post("/items/{id}/toggle", a.PostToggleItem)
|
||||
r.With(a.Auth.CSRFProtect).Post("/items/{id}/delete", a.PostDeleteItem)
|
||||
r.With(a.Auth.CSRFProtect).Post("/items/{id}/run", a.PostRunItem)
|
||||
r.Get("/items/{id}/error", a.GetItemError)
|
||||
r.Get("/items/{id}/results", a.GetItemResults)
|
||||
|
||||
r.Get("/results", a.GetGlobalResults)
|
||||
|
||||
r.Get("/settings", a.GetSettings)
|
||||
r.With(a.Auth.CSRFProtect).Post("/settings", a.PostSettings)
|
||||
r.With(a.Auth.CSRFProtect).Post("/settings/password", a.PostPasswordChange)
|
||||
r.With(a.Auth.CSRFProtect).Post("/settings/test-ntfy", a.PostTestNtfy)
|
||||
r.With(a.Auth.CSRFProtect).Post("/settings/test-apify", a.PostTestApify)
|
||||
r.With(a.Auth.CSRFProtect, a.Auth.RequireAdmin).Post("/users", a.PostCreateUser)
|
||||
r.With(a.Auth.CSRFProtect, a.Auth.RequireAdmin).Post("/users/{id}/delete", a.PostDeleteUser)
|
||||
r.With(a.Auth.CSRFProtect, a.Auth.RequireAdmin).Post("/users/{id}/reset-password", a.PostResetPassword)
|
||||
})
|
||||
})
|
||||
return r
|
||||
}
|
||||
|
||||
// setupGate redirects every request to /setup if no users exist.
|
||||
func (a *App) setupGate(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/setup" || isStaticPath(r.URL.Path) {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
n, err := a.Store.UserCount(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, "db error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if n == 0 {
|
||||
http.Redirect(w, r, "/setup", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
// Once at least one user exists, /setup is a 404.
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func isStaticPath(p string) bool {
|
||||
return len(p) >= 8 && p[:8] == "/static/"
|
||||
}
|
||||
|
||||
func (a *App) page(r *http.Request, title, active string) templates.Page {
|
||||
return templates.Page{
|
||||
Title: title,
|
||||
Active: active,
|
||||
CSRFToken: a.Auth.CSRFToken(r.Context()),
|
||||
CurrentUser: auth.CurrentUserFromRequest(r),
|
||||
}
|
||||
}
|
||||
|
||||
func render(w http.ResponseWriter, r *http.Request, c templ.Component) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := c.Render(r.Context(), w); err != nil {
|
||||
slog.Error("render failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func parseInt64(s string) int64 {
|
||||
n, _ := strconv.ParseInt(s, 10, 64)
|
||||
return n
|
||||
}
|
||||
|
||||
func intParam(r *http.Request, key string) int64 {
|
||||
return parseInt64(chi.URLParam(r, key))
|
||||
}
|
||||
|
||||
func ctxBg() context.Context { return context.Background() }
|
||||
453
internal/handlers/items.go
Normal file
453
internal/handlers/items.go
Normal file
@@ -0,0 +1,453 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"veola/internal/apify"
|
||||
"veola/internal/models"
|
||||
"veola/internal/scheduler"
|
||||
"veola/templates"
|
||||
)
|
||||
|
||||
func (a *App) GetItems(w http.ResponseWriter, r *http.Request) {
|
||||
cat := r.URL.Query().Get("category")
|
||||
all, err := a.Store.ListItems(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, "db error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
var items []models.Item
|
||||
for _, it := range all {
|
||||
if cat == "" || it.Category == cat {
|
||||
items = append(items, it)
|
||||
}
|
||||
}
|
||||
cats, _ := a.Store.ListCategories(r.Context())
|
||||
render(w, r, templates.Items(templates.ItemsData{
|
||||
Page: a.page(r, "Items", "items"),
|
||||
Items: items,
|
||||
Categories: cats,
|
||||
SelectedCategory: cat,
|
||||
}))
|
||||
}
|
||||
|
||||
func (a *App) GetNewItem(w http.ResponseWriter, r *http.Request) {
|
||||
cats, _ := a.Store.ListCategories(r.Context())
|
||||
render(w, r, templates.ItemForm(templates.ItemFormData{
|
||||
Page: a.page(r, "Add Item", "items"),
|
||||
IsEdit: false,
|
||||
Categories: cats,
|
||||
Item: models.Item{
|
||||
NtfyPriority: "default",
|
||||
PollIntervalMinutes: a.Cfg.Scheduler.GlobalPollIntervalMinutes,
|
||||
Marketplaces: []string{"ebay.com"},
|
||||
ListingType: "all",
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
func (a *App) GetEditItem(w http.ResponseWriter, r *http.Request) {
|
||||
id := intParam(r, "id")
|
||||
it, err := a.Store.GetItem(r.Context(), id)
|
||||
if err != nil || it == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
cats, _ := a.Store.ListCategories(r.Context())
|
||||
render(w, r, templates.ItemForm(templates.ItemFormData{
|
||||
Page: a.page(r, "Edit "+it.Name, "items"),
|
||||
IsEdit: true,
|
||||
Item: *it,
|
||||
Categories: cats,
|
||||
}))
|
||||
}
|
||||
|
||||
// parseItemForm pulls form fields into a models.Item plus a list of validation
|
||||
// errors. Used by preview, create, and update.
|
||||
func parseItemForm(r *http.Request) (models.Item, []string) {
|
||||
var it models.Item
|
||||
var errs []string
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return it, []string{"could not parse form"}
|
||||
}
|
||||
it.Name = strings.TrimSpace(r.PostFormValue("name"))
|
||||
it.SearchQuery = strings.Join(models.SplitList(r.PostFormValue("search_query"), 10), "\n")
|
||||
it.ExcludeKeywords = strings.Join(models.SplitList(r.PostFormValue("exclude_keywords"), 20), "\n")
|
||||
it.URL = strings.TrimSpace(r.PostFormValue("url"))
|
||||
if newCat := strings.TrimSpace(r.PostFormValue("category_new")); newCat != "" {
|
||||
it.Category = newCat
|
||||
} else {
|
||||
it.Category = strings.TrimSpace(r.PostFormValue("category"))
|
||||
}
|
||||
it.NtfyTopic = strings.TrimSpace(r.PostFormValue("ntfy_topic"))
|
||||
it.NtfyPriority = strings.TrimSpace(r.PostFormValue("ntfy_priority"))
|
||||
if it.NtfyPriority == "" {
|
||||
it.NtfyPriority = "default"
|
||||
}
|
||||
it.Marketplaces = collectMarketplaces(r.PostForm["marketplace"], r.PostFormValue("marketplace_custom"))
|
||||
it.ListingType = strings.TrimSpace(r.PostFormValue("listing_type"))
|
||||
it.ActorActive = strings.TrimSpace(r.PostFormValue("actor_active"))
|
||||
it.ActorSold = strings.TrimSpace(r.PostFormValue("actor_sold"))
|
||||
it.ActorPriceCompare = strings.TrimSpace(r.PostFormValue("actor_price_compare"))
|
||||
it.IncludeOutOfStock = r.PostFormValue("include_out_of_stock") == "1"
|
||||
it.UsePriceComparison = r.PostFormValue("use_price_comparison") == "1"
|
||||
it.Active = true
|
||||
|
||||
if tp := strings.TrimSpace(r.PostFormValue("target_price")); tp != "" {
|
||||
if v, err := strconv.ParseFloat(tp, 64); err == nil && v >= 0 {
|
||||
it.TargetPrice = &v
|
||||
}
|
||||
}
|
||||
if mp := strings.TrimSpace(r.PostFormValue("min_price")); mp != "" {
|
||||
if v, err := strconv.ParseFloat(mp, 64); err == nil && v >= 0 {
|
||||
it.MinPrice = &v
|
||||
}
|
||||
}
|
||||
if pi := strings.TrimSpace(r.PostFormValue("poll_interval_minutes")); pi != "" {
|
||||
if v, err := strconv.Atoi(pi); err == nil && v > 0 {
|
||||
it.PollIntervalMinutes = v
|
||||
}
|
||||
}
|
||||
if it.PollIntervalMinutes == 0 {
|
||||
it.PollIntervalMinutes = 60
|
||||
}
|
||||
|
||||
if it.Name == "" {
|
||||
errs = append(errs, "name is required")
|
||||
}
|
||||
if it.SearchQuery == "" && it.URL == "" {
|
||||
errs = append(errs, "either search query or product URL is required")
|
||||
}
|
||||
if it.NtfyTopic == "" {
|
||||
// Default to a slug of the name.
|
||||
it.NtfyTopic = slugify(it.Name)
|
||||
}
|
||||
return it, errs
|
||||
}
|
||||
|
||||
// collectMarketplaces dedupes the checkbox values + custom CSV input into an
|
||||
// ordered slice. Checkbox order first, then custom entries in the order the
|
||||
// user typed them.
|
||||
func collectMarketplaces(checked []string, custom string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
add := func(v string) {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" || seen[v] {
|
||||
return
|
||||
}
|
||||
seen[v] = true
|
||||
out = append(out, v)
|
||||
}
|
||||
for _, v := range checked {
|
||||
add(v)
|
||||
}
|
||||
for _, v := range strings.Split(custom, ",") {
|
||||
add(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func slugify(s string) string {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == ' ', r == '_', r == '-':
|
||||
b.WriteRune('-')
|
||||
}
|
||||
}
|
||||
out := b.String()
|
||||
if out == "" {
|
||||
return "veola-item"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *App) PostPreview(w http.ResponseWriter, r *http.Request) {
|
||||
it, errs := parseItemForm(r)
|
||||
if len(errs) > 0 {
|
||||
render(w, r, templates.ItemPreview(templates.PreviewData{
|
||||
CSRFToken: a.Auth.CSRFToken(r.Context()),
|
||||
Form: formValuesFromItem(it, r),
|
||||
Error: strings.Join(errs, "; "),
|
||||
}))
|
||||
return
|
||||
}
|
||||
results, source, cached, err := a.runPreview(r.Context(), it)
|
||||
if err != nil {
|
||||
render(w, r, templates.ItemPreview(templates.PreviewData{
|
||||
CSRFToken: a.Auth.CSRFToken(r.Context()),
|
||||
Form: formValuesFromItem(it, r),
|
||||
Error: err.Error(),
|
||||
}))
|
||||
return
|
||||
}
|
||||
results = scheduler.FilterResults(results, a.Cfg.Scheduler.MatchConfidenceThreshold, it.IncludeOutOfStock)
|
||||
results = scheduler.ApplyItemFilters(results, it.MinPrice, it.ExcludeKeywordsList())
|
||||
if len(results) == 0 {
|
||||
render(w, r, templates.ItemPreview(templates.PreviewData{
|
||||
CSRFToken: a.Auth.CSRFToken(r.Context()),
|
||||
Form: formValuesFromItem(it, r),
|
||||
Empty: true,
|
||||
}))
|
||||
return
|
||||
}
|
||||
bestIdx := scheduler.PickBest(results)
|
||||
minP, maxP := results[0].Price, results[0].Price
|
||||
stores := map[string]struct{}{}
|
||||
cur := results[0].Currency
|
||||
for _, r := range results {
|
||||
if r.Price < minP {
|
||||
minP = r.Price
|
||||
}
|
||||
if r.Price > maxP {
|
||||
maxP = r.Price
|
||||
}
|
||||
stores[r.Store] = struct{}{}
|
||||
}
|
||||
render(w, r, templates.ItemPreview(templates.PreviewData{
|
||||
CSRFToken: a.Auth.CSRFToken(r.Context()),
|
||||
Form: formValuesFromItem(it, r),
|
||||
Results: results,
|
||||
BestIndex: bestIdx,
|
||||
MinPrice: minP,
|
||||
MaxPrice: maxP,
|
||||
StoreCount: len(stores),
|
||||
Cached: cached,
|
||||
Currency: cur,
|
||||
}))
|
||||
_ = source
|
||||
}
|
||||
|
||||
func (a *App) runPreview(ctx context.Context, it models.Item) ([]apify.UnifiedResult, string, bool, error) {
|
||||
plans := a.Scheduler.BuildPreviewInputs(it)
|
||||
if len(plans) == 0 {
|
||||
return nil, "", false, fmt.Errorf("no actor configured for this item")
|
||||
}
|
||||
previewMarket := ""
|
||||
if len(it.Marketplaces) > 0 {
|
||||
previewMarket = it.Marketplaces[0]
|
||||
}
|
||||
queries := it.SearchQueries()
|
||||
sortedQ := make([]string, len(queries))
|
||||
copy(sortedQ, queries)
|
||||
sort.Strings(sortedQ)
|
||||
actorIDs := make([]string, 0, len(plans))
|
||||
for _, p := range plans {
|
||||
actorIDs = append(actorIDs, p.ActorID())
|
||||
}
|
||||
sort.Strings(actorIDs)
|
||||
key := previewKey{
|
||||
Queries: strings.Join(sortedQ, "\n"),
|
||||
URL: it.URL,
|
||||
Marketplace: previewMarket,
|
||||
ListingType: it.ListingType,
|
||||
ActorIDs: strings.Join(actorIDs, ","),
|
||||
MaxResults: 30,
|
||||
}
|
||||
if cached, src, ok := a.Preview.Get(key); ok {
|
||||
return cached, src, true, nil
|
||||
}
|
||||
var merged []apify.UnifiedResult
|
||||
primarySource := ""
|
||||
for _, p := range plans {
|
||||
actorID := p.ActorID()
|
||||
if actorID == "" {
|
||||
continue
|
||||
}
|
||||
raw, err := a.Apify.Run(ctx, actorID, p.Input())
|
||||
if err != nil {
|
||||
slog.Warn("preview run failed", "actor", actorID, "query", p.Query(), "err", err)
|
||||
continue
|
||||
}
|
||||
decoded, _ := apify.Decode(raw, p.Source())
|
||||
for i := range decoded {
|
||||
decoded[i].MatchedQuery = p.Query()
|
||||
}
|
||||
usable := 0
|
||||
for _, r := range decoded {
|
||||
if r.URL != "" && r.Price > 0 {
|
||||
usable++
|
||||
}
|
||||
}
|
||||
slog.Info("preview decoded",
|
||||
"marketplace", previewMarket,
|
||||
"actor", actorID,
|
||||
"query", p.Query(),
|
||||
"raw", len(raw),
|
||||
"decoded", len(decoded),
|
||||
"usable", usable,
|
||||
)
|
||||
if usable == 0 && len(raw) > 0 {
|
||||
var sample map[string]any
|
||||
if err := json.Unmarshal(raw[0], &sample); err == nil {
|
||||
ks := make([]string, 0, len(sample))
|
||||
for k := range sample {
|
||||
ks = append(ks, k)
|
||||
}
|
||||
slog.Warn("preview decoded zero usable rows; raw item keys",
|
||||
"actor", actorID,
|
||||
"keys", ks,
|
||||
)
|
||||
}
|
||||
}
|
||||
merged = append(merged, decoded...)
|
||||
if primarySource == "" {
|
||||
primarySource = p.Source()
|
||||
}
|
||||
}
|
||||
merged = scheduler.DedupByURL(merged)
|
||||
a.Preview.Put(key, merged, primarySource)
|
||||
return merged, primarySource, false, nil
|
||||
}
|
||||
|
||||
func formValuesFromItem(it models.Item, r *http.Request) templates.FormValues {
|
||||
tp := ""
|
||||
if it.TargetPrice != nil {
|
||||
tp = fmt.Sprintf("%.2f", *it.TargetPrice)
|
||||
}
|
||||
mp := ""
|
||||
if it.MinPrice != nil {
|
||||
mp = fmt.Sprintf("%.2f", *it.MinPrice)
|
||||
}
|
||||
return templates.FormValues{
|
||||
Name: it.Name,
|
||||
SearchQuery: it.SearchQuery,
|
||||
URL: it.URL,
|
||||
Category: it.Category,
|
||||
TargetPrice: tp,
|
||||
MinPrice: mp,
|
||||
ExcludeKeywords: it.ExcludeKeywords,
|
||||
NtfyTopic: it.NtfyTopic,
|
||||
NtfyPriority: it.NtfyPriority,
|
||||
PollIntervalMinutes: fmt.Sprintf("%d", it.PollIntervalMinutes),
|
||||
IncludeOutOfStock: it.IncludeOutOfStock,
|
||||
Marketplaces: it.Marketplaces,
|
||||
ListingType: it.ListingType,
|
||||
ActorActive: it.ActorActive,
|
||||
ActorSold: it.ActorSold,
|
||||
ActorPriceCompare: it.ActorPriceCompare,
|
||||
UsePriceComparison: it.UsePriceComparison,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) PostCreateItem(w http.ResponseWriter, r *http.Request) {
|
||||
it, errs := parseItemForm(r)
|
||||
if len(errs) > 0 {
|
||||
http.Error(w, strings.Join(errs, "; "), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
id, err := a.Store.CreateItem(r.Context(), &it)
|
||||
if err != nil {
|
||||
http.Error(w, "could not save item: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
it.ID = id
|
||||
a.Scheduler.SyncItem(it)
|
||||
|
||||
go func() {
|
||||
bg := context.Background()
|
||||
fresh, err := a.Store.GetItem(bg, id)
|
||||
if err != nil || fresh == nil {
|
||||
return
|
||||
}
|
||||
a.Scheduler.SeedSoldHistory(bg, *fresh)
|
||||
a.Scheduler.RunPoll(bg, *fresh)
|
||||
}()
|
||||
|
||||
http.Redirect(w, r, fmt.Sprintf("/items/%d/results", id), http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) PostUpdateItem(w http.ResponseWriter, r *http.Request) {
|
||||
id := intParam(r, "id")
|
||||
existing, err := a.Store.GetItem(r.Context(), id)
|
||||
if err != nil || existing == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
updated, errs := parseItemForm(r)
|
||||
if len(errs) > 0 {
|
||||
cats, _ := a.Store.ListCategories(r.Context())
|
||||
updated.ID = id
|
||||
render(w, r, templates.ItemForm(templates.ItemFormData{
|
||||
Page: a.page(r, "Edit "+updated.Name, "items"),
|
||||
IsEdit: true,
|
||||
Item: updated,
|
||||
Errors: errs,
|
||||
Categories: cats,
|
||||
}))
|
||||
return
|
||||
}
|
||||
updated.ID = id
|
||||
updated.Active = existing.Active
|
||||
if err := a.Store.UpdateItem(r.Context(), &updated); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
a.Scheduler.SyncItem(updated)
|
||||
http.Redirect(w, r, "/items", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) PostToggleItem(w http.ResponseWriter, r *http.Request) {
|
||||
id := intParam(r, "id")
|
||||
it, err := a.Store.GetItem(r.Context(), id)
|
||||
if err != nil || it == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
it.Active = !it.Active
|
||||
if err := a.Store.SetItemActive(r.Context(), id, it.Active); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
a.Scheduler.SyncItem(*it)
|
||||
render(w, r, templates.ItemRow(*it, a.Auth.CSRFToken(r.Context())))
|
||||
}
|
||||
|
||||
func (a *App) PostDeleteItem(w http.ResponseWriter, r *http.Request) {
|
||||
id := intParam(r, "id")
|
||||
if err := a.Store.DeleteItem(r.Context(), id); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
a.Scheduler.RemoveItem(id)
|
||||
render(w, r, templates.EmptyRow())
|
||||
}
|
||||
|
||||
func (a *App) PostRunItem(w http.ResponseWriter, r *http.Request) {
|
||||
id := intParam(r, "id")
|
||||
it, err := a.Store.GetItem(r.Context(), id)
|
||||
if err != nil || it == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
go a.Scheduler.RunPoll(context.Background(), *it)
|
||||
// Re-render the row immediately so HTMX has something to swap in.
|
||||
render(w, r, templates.ItemRow(*it, a.Auth.CSRFToken(r.Context())))
|
||||
}
|
||||
|
||||
func (a *App) GetItemError(w http.ResponseWriter, r *http.Request) {
|
||||
id := intParam(r, "id")
|
||||
it, err := a.Store.GetItem(r.Context(), id)
|
||||
if err != nil || it == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
fmt.Fprintf(w, "<span>%s</span>", htmlEscape(it.LastPollError))
|
||||
}
|
||||
|
||||
func htmlEscape(s string) string {
|
||||
r := strings.NewReplacer("&", "&", "<", "<", ">", ">", "\"", """)
|
||||
return r.Replace(s)
|
||||
}
|
||||
66
internal/handlers/preview_cache.go
Normal file
66
internal/handlers/preview_cache.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"veola/internal/apify"
|
||||
)
|
||||
|
||||
// previewKey caches the *raw* apify result set (post-decode, post-merge,
|
||||
// pre-filter). Filters like min_price and exclude_keywords are applied after
|
||||
// the cache lookup so the operator can iterate on them without burning credits.
|
||||
type previewKey struct {
|
||||
Queries, URL, Marketplace, ListingType, ActorIDs string
|
||||
MaxResults int
|
||||
}
|
||||
|
||||
type previewEntry struct {
|
||||
results []apify.UnifiedResult
|
||||
source string
|
||||
stored time.Time
|
||||
}
|
||||
|
||||
type PreviewCache struct {
|
||||
ttl time.Duration
|
||||
mu sync.Mutex
|
||||
entries map[previewKey]previewEntry
|
||||
}
|
||||
|
||||
func NewPreviewCache(ttl time.Duration) *PreviewCache {
|
||||
return &PreviewCache{
|
||||
ttl: ttl,
|
||||
entries: make(map[previewKey]previewEntry),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *PreviewCache) Get(k previewKey) ([]apify.UnifiedResult, string, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
e, ok := c.entries[k]
|
||||
if !ok {
|
||||
return nil, "", false
|
||||
}
|
||||
if time.Since(e.stored) > c.ttl {
|
||||
delete(c.entries, k)
|
||||
return nil, "", false
|
||||
}
|
||||
return e.results, e.source, true
|
||||
}
|
||||
|
||||
func (c *PreviewCache) Put(k previewKey, results []apify.UnifiedResult, source string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.entries[k] = previewEntry{results: results, source: source, stored: time.Now()}
|
||||
if len(c.entries) > 64 {
|
||||
c.evictExpired()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *PreviewCache) evictExpired() {
|
||||
for k, e := range c.entries {
|
||||
if time.Since(e.stored) > c.ttl {
|
||||
delete(c.entries, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
145
internal/handlers/results.go
Normal file
145
internal/handlers/results.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"veola/internal/db"
|
||||
"veola/internal/models"
|
||||
"veola/internal/scheduler"
|
||||
"veola/templates"
|
||||
)
|
||||
|
||||
const resultsPerPage = 20
|
||||
|
||||
func (a *App) GetItemResults(w http.ResponseWriter, r *http.Request) {
|
||||
id := intParam(r, "id")
|
||||
it, err := a.Store.GetItem(r.Context(), id)
|
||||
if err != nil || it == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
order := r.URL.Query().Get("order")
|
||||
if order == "" {
|
||||
order = "found_desc"
|
||||
}
|
||||
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
|
||||
total, err := a.Store.CountResults(r.Context(), id)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
totalPages := (total + resultsPerPage - 1) / resultsPerPage
|
||||
if totalPages < 1 {
|
||||
totalPages = 1
|
||||
}
|
||||
if page > totalPages {
|
||||
page = totalPages
|
||||
}
|
||||
|
||||
results, err := a.Store.ListResults(r.Context(), db.ResultsQuery{
|
||||
ItemID: id,
|
||||
Limit: resultsPerPage,
|
||||
Offset: (page - 1) * resultsPerPage,
|
||||
Order: order,
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
history, err := a.Store.ListPriceHistory(r.Context(), id)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
badge := scheduler.PickBadge(*it, history, time.Now())
|
||||
chart := buildChartJSON(history)
|
||||
|
||||
render(w, r, templates.ItemResults(templates.ItemResultsData{
|
||||
Page: a.page(r, it.Name, "items"),
|
||||
Item: *it,
|
||||
Badge: badge,
|
||||
History: history,
|
||||
Results: results,
|
||||
Page_: page,
|
||||
TotalPages: totalPages,
|
||||
Order: order,
|
||||
HistoryChartJSON: chart,
|
||||
}))
|
||||
}
|
||||
|
||||
func buildChartJSON(history []models.PricePoint) string {
|
||||
c := templates.ChartJSON{
|
||||
Labels: make([]string, 0, len(history)),
|
||||
Points: make([]float64, 0, len(history)),
|
||||
}
|
||||
for _, p := range history {
|
||||
c.Labels = append(c.Labels, p.PolledAt.Format("2006-01-02"))
|
||||
c.Points = append(c.Points, p.Price)
|
||||
}
|
||||
return templates.MustChartJSON(c)
|
||||
}
|
||||
|
||||
func (a *App) GetGlobalResults(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
itemID, _ := strconv.ParseInt(q.Get("item_id"), 10, 64)
|
||||
from := strings.TrimSpace(q.Get("from"))
|
||||
to := strings.TrimSpace(q.Get("to"))
|
||||
|
||||
items, err := a.Store.ListItems(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
names := make(map[int64]string, len(items))
|
||||
for _, it := range items {
|
||||
names[it.ID] = it.Name
|
||||
}
|
||||
|
||||
results, err := a.Store.ListResults(r.Context(), db.ResultsQuery{
|
||||
ItemID: itemID,
|
||||
Limit: 200,
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
fromT, _ := time.Parse("2006-01-02", from)
|
||||
toT, _ := time.Parse("2006-01-02", to)
|
||||
if !toT.IsZero() {
|
||||
toT = toT.Add(24 * time.Hour)
|
||||
}
|
||||
|
||||
rows := make([]templates.ItemResultRow, 0, len(results))
|
||||
for _, res := range results {
|
||||
if !fromT.IsZero() && res.FoundAt.Before(fromT) {
|
||||
continue
|
||||
}
|
||||
if !toT.IsZero() && !res.FoundAt.Before(toT) {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, templates.ItemResultRow{
|
||||
Result: res,
|
||||
ItemName: names[res.ItemID],
|
||||
})
|
||||
}
|
||||
|
||||
render(w, r, templates.GlobalResults(templates.GlobalResultsData{
|
||||
Page: a.page(r, "Results", "results"),
|
||||
Items: items,
|
||||
Results: rows,
|
||||
ItemID: itemID,
|
||||
From: from,
|
||||
To: to,
|
||||
}))
|
||||
}
|
||||
195
internal/handlers/settings.go
Normal file
195
internal/handlers/settings.go
Normal file
@@ -0,0 +1,195 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"veola/internal/apify"
|
||||
"veola/internal/auth"
|
||||
"veola/internal/models"
|
||||
"veola/internal/ntfy"
|
||||
"veola/templates"
|
||||
)
|
||||
|
||||
var settingsKeys = []string{
|
||||
"apify_api_key",
|
||||
"ntfy_base_url",
|
||||
"ntfy_default_topic",
|
||||
"ntfy_token",
|
||||
"global_poll_interval_minutes",
|
||||
"match_confidence_threshold",
|
||||
}
|
||||
|
||||
func (a *App) settingsData(r *http.Request) (templates.SettingsData, error) {
|
||||
values, err := a.Store.GetAllSettings(r.Context())
|
||||
if err != nil {
|
||||
return templates.SettingsData{}, err
|
||||
}
|
||||
if values == nil {
|
||||
values = map[string]string{}
|
||||
}
|
||||
users, _ := a.Store.ListUsers(r.Context())
|
||||
cur := auth.CurrentUserFromRequest(r)
|
||||
return templates.SettingsData{
|
||||
Page: a.page(r, "Settings", "settings"),
|
||||
Values: values,
|
||||
IsAdmin: cur != nil && cur.Role == models.RoleAdmin,
|
||||
Users: users,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *App) GetSettings(w http.ResponseWriter, r *http.Request) {
|
||||
d, err := a.settingsData(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
render(w, r, templates.Settings(d))
|
||||
}
|
||||
|
||||
func (a *App) PostSettings(w http.ResponseWriter, r *http.Request) {
|
||||
cur := auth.CurrentUserFromRequest(r)
|
||||
if cur == nil || cur.Role != models.RoleAdmin {
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "bad form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, k := range settingsKeys {
|
||||
v := strings.TrimSpace(r.PostFormValue(k))
|
||||
if err := a.Store.SetSetting(r.Context(), k, v); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
http.Redirect(w, r, "/settings", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (a *App) PostPasswordChange(w http.ResponseWriter, r *http.Request) {
|
||||
cur := auth.CurrentUserFromRequest(r)
|
||||
if cur == nil {
|
||||
http.Redirect(w, r, "/login", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "bad form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
current := r.PostFormValue("current_password")
|
||||
next := r.PostFormValue("new_password")
|
||||
confirm := r.PostFormValue("new_password_confirm")
|
||||
|
||||
d, err := a.settingsData(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
switch {
|
||||
case !auth.CheckPassword(cur.PasswordHash, current):
|
||||
d.PasswordError = "Current password is incorrect"
|
||||
case len(next) < auth.MinPasswordLen:
|
||||
d.PasswordError = fmt.Sprintf("New password must be at least %d characters", auth.MinPasswordLen)
|
||||
case next != confirm:
|
||||
d.PasswordError = "New passwords do not match"
|
||||
}
|
||||
if d.PasswordError != "" {
|
||||
render(w, r, templates.Settings(d))
|
||||
return
|
||||
}
|
||||
|
||||
hash, err := auth.HashPassword(next)
|
||||
if err != nil {
|
||||
http.Error(w, "hash error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if err := a.Store.UpdateUserPassword(r.Context(), cur.ID, hash); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
d.PasswordMsg = "Password updated"
|
||||
render(w, r, templates.Settings(d))
|
||||
}
|
||||
|
||||
func (a *App) PostTestNtfy(w http.ResponseWriter, r *http.Request) {
|
||||
cur := auth.CurrentUserFromRequest(r)
|
||||
if cur == nil || cur.Role != models.RoleAdmin {
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
d, err := a.settingsData(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
baseURL := strings.TrimSpace(d.Values["ntfy_base_url"])
|
||||
topic := strings.TrimSpace(d.Values["ntfy_default_topic"])
|
||||
token := strings.TrimSpace(d.Values["ntfy_token"])
|
||||
if baseURL == "" || topic == "" {
|
||||
d.TestNtfyOK = "Set ntfy base URL and default topic first."
|
||||
render(w, r, templates.Settings(d))
|
||||
return
|
||||
}
|
||||
client := ntfy.NewWithToken(baseURL, token)
|
||||
if err := client.Send(r.Context(), ntfy.Notification{
|
||||
Topic: topic,
|
||||
Title: "Veola test",
|
||||
Message: "Test notification from Veola settings.",
|
||||
Priority: "default",
|
||||
Tags: []string{"white_check_mark"},
|
||||
}); err != nil {
|
||||
d.TestNtfyOK = "Ntfy test failed: " + err.Error()
|
||||
} else {
|
||||
d.TestNtfyOK = fmt.Sprintf("Sent test notification to %s/%s", baseURL, topic)
|
||||
}
|
||||
render(w, r, templates.Settings(d))
|
||||
}
|
||||
|
||||
func (a *App) PostTestApify(w http.ResponseWriter, r *http.Request) {
|
||||
cur := auth.CurrentUserFromRequest(r)
|
||||
if cur == nil || cur.Role != models.RoleAdmin {
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
d, err := a.settingsData(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
apiKey := strings.TrimSpace(d.Values["apify_api_key"])
|
||||
actorID := a.Cfg.Apify.Actors.ActiveListings
|
||||
if apiKey == "" {
|
||||
apiKey = a.Cfg.Apify.APIKey
|
||||
}
|
||||
if apiKey == "" || actorID == "" {
|
||||
d.TestApifyOK = "Apify API key or active_listings actor is not configured."
|
||||
render(w, r, templates.Settings(d))
|
||||
return
|
||||
}
|
||||
client := apify.New(apiKey)
|
||||
var proxy *apify.ProxyConfiguration
|
||||
p := a.Cfg.Apify.Proxy
|
||||
if p.UseApifyProxy {
|
||||
proxy = &apify.ProxyConfiguration{
|
||||
UseApifyProxy: true,
|
||||
ApifyProxyGroups: p.Groups,
|
||||
ApifyProxyCountry: p.Country,
|
||||
}
|
||||
}
|
||||
raw, err := client.Run(r.Context(), actorID, apify.ActiveListingInput{
|
||||
SearchQueries: []string{"test"},
|
||||
MaxProductsPerSearch: 1,
|
||||
MaxSearchPages: 1,
|
||||
ListingType: "all",
|
||||
ProxyConfiguration: proxy,
|
||||
})
|
||||
if err != nil {
|
||||
d.TestApifyOK = "Apify test failed: " + err.Error()
|
||||
} else {
|
||||
d.TestApifyOK = fmt.Sprintf("Apify returned %d item(s).", len(raw))
|
||||
}
|
||||
render(w, r, templates.Settings(d))
|
||||
}
|
||||
101
internal/handlers/users.go
Normal file
101
internal/handlers/users.go
Normal file
@@ -0,0 +1,101 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"veola/internal/auth"
|
||||
"veola/internal/models"
|
||||
"veola/templates"
|
||||
)
|
||||
|
||||
func (a *App) renderSettingsWithUserMsg(w http.ResponseWriter, r *http.Request, msg, errMsg string) {
|
||||
d, err := a.settingsData(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
d.UserMsg = msg
|
||||
d.UserError = errMsg
|
||||
render(w, r, templates.Settings(d))
|
||||
}
|
||||
|
||||
func (a *App) PostCreateUser(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "bad form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
username := strings.TrimSpace(r.PostFormValue("username"))
|
||||
password := r.PostFormValue("password")
|
||||
role := strings.TrimSpace(r.PostFormValue("role"))
|
||||
if role != string(models.RoleAdmin) {
|
||||
role = string(models.RoleUser)
|
||||
}
|
||||
|
||||
switch {
|
||||
case username == "":
|
||||
a.renderSettingsWithUserMsg(w, r, "", "Username is required")
|
||||
return
|
||||
case len(password) < auth.MinPasswordLen:
|
||||
a.renderSettingsWithUserMsg(w, r, "", fmt.Sprintf("Password must be at least %d characters", auth.MinPasswordLen))
|
||||
return
|
||||
}
|
||||
existing, _ := a.Store.GetUserByUsername(r.Context(), username)
|
||||
if existing != nil {
|
||||
a.renderSettingsWithUserMsg(w, r, "", "User already exists")
|
||||
return
|
||||
}
|
||||
hash, err := auth.HashPassword(password)
|
||||
if err != nil {
|
||||
a.renderSettingsWithUserMsg(w, r, "", "hash error")
|
||||
return
|
||||
}
|
||||
if _, err := a.Store.CreateUser(r.Context(), username, hash, models.Role(role)); err != nil {
|
||||
a.renderSettingsWithUserMsg(w, r, "", err.Error())
|
||||
return
|
||||
}
|
||||
a.renderSettingsWithUserMsg(w, r, "Created user "+username, "")
|
||||
}
|
||||
|
||||
func (a *App) PostDeleteUser(w http.ResponseWriter, r *http.Request) {
|
||||
id := intParam(r, "id")
|
||||
cur := auth.CurrentUserFromRequest(r)
|
||||
if cur != nil && cur.ID == id {
|
||||
a.renderSettingsWithUserMsg(w, r, "", "You cannot delete your own account")
|
||||
return
|
||||
}
|
||||
if err := a.Store.DeleteUser(r.Context(), id); err != nil {
|
||||
a.renderSettingsWithUserMsg(w, r, "", err.Error())
|
||||
return
|
||||
}
|
||||
a.renderSettingsWithUserMsg(w, r, "User removed", "")
|
||||
}
|
||||
|
||||
func (a *App) PostResetPassword(w http.ResponseWriter, r *http.Request) {
|
||||
id := intParam(r, "id")
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, "bad form", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
next := r.PostFormValue("new_password")
|
||||
if len(next) < auth.MinPasswordLen {
|
||||
a.renderSettingsWithUserMsg(w, r, "", fmt.Sprintf("Password must be at least %d characters", auth.MinPasswordLen))
|
||||
return
|
||||
}
|
||||
u, err := a.Store.GetUserByID(r.Context(), id)
|
||||
if err != nil || u == nil {
|
||||
a.renderSettingsWithUserMsg(w, r, "", "User not found")
|
||||
return
|
||||
}
|
||||
hash, err := auth.HashPassword(next)
|
||||
if err != nil {
|
||||
a.renderSettingsWithUserMsg(w, r, "", "hash error")
|
||||
return
|
||||
}
|
||||
if err := a.Store.UpdateUserPassword(r.Context(), id, hash); err != nil {
|
||||
a.renderSettingsWithUserMsg(w, r, "", err.Error())
|
||||
return
|
||||
}
|
||||
a.renderSettingsWithUserMsg(w, r, "Password reset for "+u.Username, "")
|
||||
}
|
||||
122
internal/models/models.go
Normal file
122
internal/models/models.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Role string
|
||||
|
||||
const (
|
||||
RoleAdmin Role = "admin"
|
||||
RoleUser Role = "user"
|
||||
)
|
||||
|
||||
type User struct {
|
||||
ID int64
|
||||
Username string
|
||||
PasswordHash string
|
||||
Role Role
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type Item struct {
|
||||
ID int64
|
||||
Name string
|
||||
SearchQuery string
|
||||
URL string
|
||||
Category string
|
||||
TargetPrice *float64
|
||||
NtfyTopic string
|
||||
NtfyPriority string
|
||||
PollIntervalMinutes int
|
||||
IncludeOutOfStock bool
|
||||
MinPrice *float64
|
||||
ExcludeKeywords string
|
||||
Marketplaces []string
|
||||
ListingType string
|
||||
ActorActive string
|
||||
ActorSold string
|
||||
ActorPriceCompare string
|
||||
UsePriceComparison bool
|
||||
Active bool
|
||||
LastPolledAt *time.Time
|
||||
LastPollError string
|
||||
BestPrice *float64
|
||||
BestPriceStore string
|
||||
BestPriceURL string
|
||||
BestPriceImageURL string
|
||||
BestPriceTitle string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
ID int64
|
||||
ItemID int64
|
||||
Title string
|
||||
Price *float64
|
||||
Currency string
|
||||
URL string
|
||||
Source string
|
||||
ImageURL string
|
||||
MatchedQuery string
|
||||
Alerted bool
|
||||
FoundAt time.Time
|
||||
}
|
||||
|
||||
// SearchQueries returns the item's alias list. Splits on newline, comma, and
|
||||
// semicolon; trims; drops blanks; dedupes case-insensitively. Result order is
|
||||
// the user's input order (first occurrence wins).
|
||||
func (it *Item) SearchQueries() []string {
|
||||
return SplitList(it.SearchQuery, 10)
|
||||
}
|
||||
|
||||
// ExcludeKeywordsList returns the item's exclude-keyword list, normalized the
|
||||
// same way as SearchQueries.
|
||||
func (it *Item) ExcludeKeywordsList() []string {
|
||||
return SplitList(it.ExcludeKeywords, 20)
|
||||
}
|
||||
|
||||
// SplitList splits a user-entered list on newline, comma, or semicolon,
|
||||
// trims whitespace, drops empty entries, dedupes case-insensitively, and caps
|
||||
// the result at max entries (0 = no cap).
|
||||
func SplitList(s string, max int) []string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, part := range strings.FieldsFunc(s, func(r rune) bool {
|
||||
return r == '\n' || r == '\r' || r == ',' || r == ';'
|
||||
}) {
|
||||
t := strings.TrimSpace(part)
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
k := strings.ToLower(t)
|
||||
if seen[k] {
|
||||
continue
|
||||
}
|
||||
seen[k] = true
|
||||
out = append(out, t)
|
||||
if max > 0 && len(out) >= max {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type PricePoint struct {
|
||||
ID int64
|
||||
ItemID int64
|
||||
Price float64
|
||||
Store string
|
||||
PolledAt time.Time
|
||||
}
|
||||
|
||||
type Setting struct {
|
||||
Key string
|
||||
Value string
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
98
internal/ntfy/client.go
Normal file
98
internal/ntfy/client.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package ntfy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
BaseURL string
|
||||
Token string
|
||||
HTTP *http.Client
|
||||
}
|
||||
|
||||
func New(baseURL string) *Client {
|
||||
return &Client{
|
||||
BaseURL: strings.TrimRight(baseURL, "/"),
|
||||
HTTP: &http.Client{Timeout: 15 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// NewWithToken returns a ntfy Client with bearer-token auth set. Use this
|
||||
// when the ntfy server requires authentication.
|
||||
func NewWithToken(baseURL, token string) *Client {
|
||||
c := New(baseURL)
|
||||
c.Token = strings.TrimSpace(token)
|
||||
return c
|
||||
}
|
||||
|
||||
type Notification struct {
|
||||
Topic string
|
||||
Title string
|
||||
Message string
|
||||
Priority string
|
||||
Tags []string
|
||||
Click string
|
||||
}
|
||||
|
||||
// Send publishes to ntfy using the topic-path + header style
|
||||
// (POST {base}/{topic} with metadata in HTTP headers and the message as the
|
||||
// raw body). This is the most broadly compatible ntfy publish method —
|
||||
// works on every ntfy version including self-hosted, and on any path layout
|
||||
// the server is mounted under.
|
||||
func (c *Client) Send(ctx context.Context, n Notification) error {
|
||||
if c.BaseURL == "" {
|
||||
return fmt.Errorf("ntfy base_url not configured")
|
||||
}
|
||||
if n.Topic == "" {
|
||||
return fmt.Errorf("ntfy topic required")
|
||||
}
|
||||
url := c.BaseURL + "/" + strings.TrimLeft(n.Topic, "/")
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(n.Message))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "text/plain; charset=utf-8")
|
||||
if c.Token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.Token)
|
||||
}
|
||||
if n.Title != "" {
|
||||
req.Header.Set("Title", n.Title)
|
||||
}
|
||||
if n.Priority != "" {
|
||||
req.Header.Set("Priority", n.Priority)
|
||||
}
|
||||
if len(n.Tags) > 0 {
|
||||
req.Header.Set("Tags", strings.Join(n.Tags, ","))
|
||||
}
|
||||
if n.Click != "" {
|
||||
req.Header.Set("Click", n.Click)
|
||||
}
|
||||
tokenLen := len(c.Token)
|
||||
tokenPrefix := ""
|
||||
if tokenLen >= 4 {
|
||||
tokenPrefix = c.Token[:4]
|
||||
}
|
||||
slog.Info("ntfy publish",
|
||||
"url", url,
|
||||
"topic", n.Topic,
|
||||
"auth_header_set", c.Token != "",
|
||||
"token_prefix", tokenPrefix,
|
||||
"token_len", tokenLen,
|
||||
)
|
||||
resp, err := c.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ntfy POST: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
return fmt.Errorf("ntfy returned %d: %s", resp.StatusCode, string(b))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
84
internal/scheduler/alert.go
Normal file
84
internal/scheduler/alert.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"veola/internal/apify"
|
||||
)
|
||||
|
||||
// ShouldAlert returns true if the result should trigger an alert.
|
||||
//
|
||||
// - Result must not already be in DB (caller checks that first).
|
||||
// - If targetPrice is nil, alert on every new result.
|
||||
// - If targetPrice is non-nil, alert only when price <= targetPrice.
|
||||
//
|
||||
// price=0 is treated as "unknown" and never alerts under a target.
|
||||
func ShouldAlert(targetPrice *float64, price float64) bool {
|
||||
if targetPrice == nil {
|
||||
return true
|
||||
}
|
||||
if price <= 0 {
|
||||
return false
|
||||
}
|
||||
return price <= *targetPrice
|
||||
}
|
||||
|
||||
// FilterResults applies match-confidence and out-of-stock filtering. Returns
|
||||
// a fresh slice; the input is not mutated.
|
||||
func FilterResults(in []apify.UnifiedResult, minConfidence float64, includeOOS bool) []apify.UnifiedResult {
|
||||
out := make([]apify.UnifiedResult, 0, len(in))
|
||||
for _, r := range in {
|
||||
if !includeOOS && r.OutOfStock {
|
||||
continue
|
||||
}
|
||||
if r.MatchConfidence != 0 && r.MatchConfidence < minConfidence {
|
||||
continue
|
||||
}
|
||||
if r.URL == "" || r.Price <= 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ApplyItemFilters drops results below minPrice (when set) and any whose title
|
||||
// contains one of excludeKeywords (case-insensitive substring match). Pass nil
|
||||
// or empty for either to skip that filter. Returns a fresh slice.
|
||||
func ApplyItemFilters(in []apify.UnifiedResult, minPrice *float64, excludeKeywords []string) []apify.UnifiedResult {
|
||||
lowered := make([]string, 0, len(excludeKeywords))
|
||||
for _, k := range excludeKeywords {
|
||||
k = strings.ToLower(strings.TrimSpace(k))
|
||||
if k != "" {
|
||||
lowered = append(lowered, k)
|
||||
}
|
||||
}
|
||||
out := make([]apify.UnifiedResult, 0, len(in))
|
||||
outer:
|
||||
for _, r := range in {
|
||||
if minPrice != nil && r.Price < *minPrice {
|
||||
continue
|
||||
}
|
||||
if len(lowered) > 0 {
|
||||
title := strings.ToLower(r.Title)
|
||||
for _, k := range lowered {
|
||||
if strings.Contains(title, k) {
|
||||
continue outer
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PickBest returns the index of the lowest-priced result, or -1 if none.
|
||||
func PickBest(rs []apify.UnifiedResult) int {
|
||||
best := -1
|
||||
for i, r := range rs {
|
||||
if best == -1 || r.Price < rs[best].Price {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
107
internal/scheduler/alert_test.go
Normal file
107
internal/scheduler/alert_test.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"veola/internal/apify"
|
||||
)
|
||||
|
||||
func ptr(f float64) *float64 { return &f }
|
||||
|
||||
func TestShouldAlert(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
target *float64
|
||||
price float64
|
||||
want bool
|
||||
}{
|
||||
{"no target alerts on any positive price", nil, 12.34, true},
|
||||
{"no target alerts even on zero price", nil, 0, true},
|
||||
{"price below target", ptr(60), 42, true},
|
||||
{"price equal to target", ptr(60), 60, true},
|
||||
{"price above target", ptr(60), 70, false},
|
||||
{"target set but price unknown", ptr(60), 0, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := ShouldAlert(c.target, c.price)
|
||||
if got != c.want {
|
||||
t.Errorf("got %v want %v", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterResults(t *testing.T) {
|
||||
in := []apify.UnifiedResult{
|
||||
{URL: "a", Price: 10, MatchConfidence: 0.9},
|
||||
{URL: "b", Price: 10, MatchConfidence: 0.4},
|
||||
{URL: "c", Price: 10, OutOfStock: true},
|
||||
{URL: "", Price: 10},
|
||||
{URL: "e", Price: 0},
|
||||
{URL: "f", Price: 12},
|
||||
}
|
||||
got := FilterResults(in, 0.6, false)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("expected 2 results, got %d", len(got))
|
||||
}
|
||||
if got[0].URL != "a" || got[1].URL != "f" {
|
||||
t.Errorf("unexpected filter output: %+v", got)
|
||||
}
|
||||
got2 := FilterResults(in, 0.6, true)
|
||||
if len(got2) != 3 {
|
||||
t.Errorf("expected 3 with OOS, got %d", len(got2))
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyItemFilters(t *testing.T) {
|
||||
in := []apify.UnifiedResult{
|
||||
{URL: "a", Title: "Sony A7 III body", Price: 1200},
|
||||
{URL: "b", Title: "Sony A7 III battery grip", Price: 45},
|
||||
{URL: "c", Title: "Sony A7 III lens cap", Price: 12},
|
||||
{URL: "d", Title: "Sony A7 III with strap", Price: 1100},
|
||||
{URL: "e", Title: "for parts not working", Price: 800},
|
||||
}
|
||||
got := ApplyItemFilters(in, ptr(100), []string{"grip", "lens cap", "for parts"})
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("expected 2 results after filter, got %d: %+v", len(got), got)
|
||||
}
|
||||
if got[0].URL != "a" || got[1].URL != "d" {
|
||||
t.Errorf("unexpected filter output: %+v", got)
|
||||
}
|
||||
|
||||
// Nil/empty filters are no-ops.
|
||||
got = ApplyItemFilters(in, nil, nil)
|
||||
if len(got) != len(in) {
|
||||
t.Errorf("nil filters dropped rows: got %d want %d", len(got), len(in))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupByURL(t *testing.T) {
|
||||
in := []apify.UnifiedResult{
|
||||
{Source: "ebay", URL: "https://a", MatchedQuery: "alpha"},
|
||||
{Source: "ebay", URL: "https://b", MatchedQuery: "alpha"},
|
||||
{Source: "ebay", URL: "https://a", MatchedQuery: "beta"}, // dup of #0
|
||||
{Source: "yahoo-auctions-jp", URL: "https://a"}, // different source, same url -> kept
|
||||
}
|
||||
got := DedupByURL(in)
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("expected 3 deduped, got %d: %+v", len(got), got)
|
||||
}
|
||||
if got[0].MatchedQuery != "alpha" {
|
||||
t.Errorf("first-occurrence MatchedQuery lost: %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickBest(t *testing.T) {
|
||||
rs := []apify.UnifiedResult{
|
||||
{Price: 50}, {Price: 30}, {Price: 90}, {Price: 30},
|
||||
}
|
||||
got := PickBest(rs)
|
||||
if got != 1 {
|
||||
t.Errorf("expected index 1, got %d", got)
|
||||
}
|
||||
if PickBest(nil) != -1 {
|
||||
t.Error("expected -1 for empty")
|
||||
}
|
||||
}
|
||||
76
internal/scheduler/badge.go
Normal file
76
internal/scheduler/badge.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"veola/internal/models"
|
||||
"veola/templates"
|
||||
)
|
||||
|
||||
// PickBadge returns the highest-priority deal-quality badge that applies to
|
||||
// an item, or an empty BadgeData if none match. Order:
|
||||
// 1. All-time low
|
||||
// 2. X% below 30-day avg (only when at least 10% below)
|
||||
// 3. X% below target
|
||||
func PickBadge(it models.Item, history []models.PricePoint, now time.Time) templates.BadgeData {
|
||||
if it.BestPrice == nil {
|
||||
return templates.BadgeData{}
|
||||
}
|
||||
best := *it.BestPrice
|
||||
|
||||
// 1. All-time low
|
||||
if isAllTimeLow(best, history) {
|
||||
return templates.BadgeData{Label: "All-time low", Class: "v-badge-low"}
|
||||
}
|
||||
|
||||
// 2. X% below 30-day average
|
||||
if avg, ok := windowedMean(history, now, 30*24*time.Hour); ok && best > 0 && avg > 0 {
|
||||
pct := (avg - best) / avg * 100
|
||||
if pct >= 10 {
|
||||
return templates.BadgeData{
|
||||
Label: fmt.Sprintf("%d%% below 30-day avg", int(pct+0.5)),
|
||||
Class: "v-badge-avg",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. X% below target
|
||||
if it.TargetPrice != nil && *it.TargetPrice > 0 && best < *it.TargetPrice {
|
||||
pct := (*it.TargetPrice - best) / *it.TargetPrice * 100
|
||||
return templates.BadgeData{
|
||||
Label: fmt.Sprintf("%d%% below target", int(pct+0.5)),
|
||||
Class: "v-badge-target",
|
||||
}
|
||||
}
|
||||
|
||||
return templates.BadgeData{}
|
||||
}
|
||||
|
||||
func isAllTimeLow(best float64, history []models.PricePoint) bool {
|
||||
if len(history) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, p := range history {
|
||||
if p.Price > 0 && p.Price < best {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func windowedMean(history []models.PricePoint, now time.Time, window time.Duration) (float64, bool) {
|
||||
cutoff := now.Add(-window)
|
||||
sum, n := 0.0, 0
|
||||
for _, p := range history {
|
||||
if p.PolledAt.Before(cutoff) {
|
||||
continue
|
||||
}
|
||||
sum += p.Price
|
||||
n++
|
||||
}
|
||||
if n == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return sum / float64(n), true
|
||||
}
|
||||
95
internal/scheduler/badge_test.go
Normal file
95
internal/scheduler/badge_test.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"veola/internal/models"
|
||||
)
|
||||
|
||||
func bestItem(best, target float64) models.Item {
|
||||
bp := best
|
||||
it := models.Item{BestPrice: &bp}
|
||||
if target > 0 {
|
||||
t := target
|
||||
it.TargetPrice = &t
|
||||
}
|
||||
return it
|
||||
}
|
||||
|
||||
func TestPickBadgeAllTimeLow(t *testing.T) {
|
||||
now := time.Now()
|
||||
hist := []models.PricePoint{
|
||||
{Price: 100, PolledAt: now.Add(-40 * 24 * time.Hour)},
|
||||
{Price: 80, PolledAt: now.Add(-10 * 24 * time.Hour)},
|
||||
{Price: 60, PolledAt: now.Add(-1 * 24 * time.Hour)},
|
||||
}
|
||||
it := bestItem(50, 0)
|
||||
got := PickBadge(it, hist, now)
|
||||
if got.Label != "All-time low" {
|
||||
t.Errorf("expected all-time low, got %q", got.Label)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickBadgeBelowAverage(t *testing.T) {
|
||||
now := time.Now()
|
||||
hist := []models.PricePoint{
|
||||
{Price: 100, PolledAt: now.Add(-25 * 24 * time.Hour)},
|
||||
{Price: 100, PolledAt: now.Add(-10 * 24 * time.Hour)},
|
||||
{Price: 100, PolledAt: now.Add(-5 * 24 * time.Hour)},
|
||||
}
|
||||
it := bestItem(80, 0) // 20% below 100 avg, not lowest because there's no lower in history but best is below points
|
||||
// add an older lower point so all-time-low is NOT triggered
|
||||
hist = append(hist, models.PricePoint{Price: 70, PolledAt: now.Add(-90 * 24 * time.Hour)})
|
||||
got := PickBadge(it, hist, now)
|
||||
if got.Label != "20% below 30-day avg" {
|
||||
t.Errorf("expected 20%% below 30-day avg, got %q", got.Label)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickBadgeBelowTarget(t *testing.T) {
|
||||
now := time.Now()
|
||||
// 30-day window mean equals best (50) so avg badge does not fire.
|
||||
// An older lower point disables the all-time-low badge.
|
||||
hist := []models.PricePoint{
|
||||
{Price: 50, PolledAt: now.Add(-2 * 24 * time.Hour)},
|
||||
{Price: 50, PolledAt: now.Add(-1 * 24 * time.Hour)},
|
||||
{Price: 40, PolledAt: now.Add(-90 * 24 * time.Hour)},
|
||||
}
|
||||
it := bestItem(50, 100) // 50% below target
|
||||
got := PickBadge(it, hist, now)
|
||||
if got.Label != "50% below target" {
|
||||
t.Errorf("expected 50%% below target, got %q", got.Label)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickBadgeNone(t *testing.T) {
|
||||
now := time.Now()
|
||||
// best matches recent avg, no target, and an older lower point exists -
|
||||
// no badge should fire.
|
||||
hist := []models.PricePoint{
|
||||
{Price: 50, PolledAt: now.Add(-1 * 24 * time.Hour)},
|
||||
{Price: 40, PolledAt: now.Add(-90 * 24 * time.Hour)},
|
||||
}
|
||||
it := bestItem(50, 0)
|
||||
got := PickBadge(it, hist, now)
|
||||
if got.Label != "" {
|
||||
t.Errorf("expected no badge, got %q", got.Label)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickBadgeIgnoresShortAvgGap(t *testing.T) {
|
||||
now := time.Now()
|
||||
hist := []models.PricePoint{
|
||||
{Price: 100, PolledAt: now.Add(-1 * 24 * time.Hour)},
|
||||
{Price: 95, PolledAt: now.Add(-2 * 24 * time.Hour)},
|
||||
}
|
||||
// best 92 is only ~5.6% below avg 97.5 — under the 10% floor
|
||||
older := models.PricePoint{Price: 80, PolledAt: now.Add(-90 * 24 * time.Hour)}
|
||||
hist = append(hist, older)
|
||||
it := bestItem(92, 0)
|
||||
got := PickBadge(it, hist, now)
|
||||
if got.Label != "" {
|
||||
t.Errorf("expected no badge for <10%% gap, got %q", got.Label)
|
||||
}
|
||||
}
|
||||
7
internal/scheduler/json.go
Normal file
7
internal/scheduler/json.go
Normal file
@@ -0,0 +1,7 @@
|
||||
package scheduler
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
func jsonUnmarshal(b []byte, dst any) error {
|
||||
return json.Unmarshal(b, dst)
|
||||
}
|
||||
599
internal/scheduler/scheduler.go
Normal file
599
internal/scheduler/scheduler.go
Normal file
@@ -0,0 +1,599 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/robfig/cron/v3"
|
||||
|
||||
"veola/internal/apify"
|
||||
"veola/internal/config"
|
||||
"veola/internal/db"
|
||||
"veola/internal/models"
|
||||
"veola/internal/ntfy"
|
||||
)
|
||||
|
||||
type Scheduler struct {
|
||||
cfg *config.Config
|
||||
store *db.Store
|
||||
apify *apify.Client
|
||||
ntfy *ntfy.Client
|
||||
cron *cron.Cron
|
||||
|
||||
mu sync.Mutex
|
||||
entries map[int64]cron.EntryID
|
||||
|
||||
rootCtx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, store *db.Store, ap *apify.Client, nt *ntfy.Client) *Scheduler {
|
||||
rootCtx, cancel := context.WithCancel(context.Background())
|
||||
return &Scheduler{
|
||||
cfg: cfg,
|
||||
store: store,
|
||||
apify: ap,
|
||||
ntfy: nt,
|
||||
cron: cron.New(),
|
||||
entries: make(map[int64]cron.EntryID),
|
||||
rootCtx: rootCtx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) Start(ctx context.Context) error {
|
||||
items, err := s.store.ListActiveItems(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, it := range items {
|
||||
s.register(it)
|
||||
}
|
||||
s.cron.Start()
|
||||
slog.Info("scheduler started", "items", len(items))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop blocks until running jobs complete.
|
||||
func (s *Scheduler) Stop() {
|
||||
s.cancel()
|
||||
stopCtx := s.cron.Stop()
|
||||
<-stopCtx.Done()
|
||||
slog.Info("scheduler stopped")
|
||||
}
|
||||
|
||||
// SyncItem registers, re-registers, or removes the cron job for an item based
|
||||
// on its current Active flag. Call after create/update/toggle/delete.
|
||||
func (s *Scheduler) SyncItem(it models.Item) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if existing, ok := s.entries[it.ID]; ok {
|
||||
s.cron.Remove(existing)
|
||||
delete(s.entries, it.ID)
|
||||
}
|
||||
if !it.Active {
|
||||
return
|
||||
}
|
||||
s.registerLocked(it)
|
||||
}
|
||||
|
||||
func (s *Scheduler) RemoveItem(id int64) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if existing, ok := s.entries[id]; ok {
|
||||
s.cron.Remove(existing)
|
||||
delete(s.entries, id)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) register(it models.Item) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.registerLocked(it)
|
||||
}
|
||||
|
||||
func (s *Scheduler) registerLocked(it models.Item) {
|
||||
mins := it.PollIntervalMinutes
|
||||
if mins <= 0 {
|
||||
mins = s.cfg.Scheduler.GlobalPollIntervalMinutes
|
||||
}
|
||||
if mins <= 0 {
|
||||
mins = 60
|
||||
}
|
||||
spec := fmt.Sprintf("@every %dm", mins)
|
||||
id := it.ID
|
||||
entryID, err := s.cron.AddFunc(spec, func() {
|
||||
ctx, cancel := context.WithTimeout(s.rootCtx, 10*time.Minute)
|
||||
defer cancel()
|
||||
fresh, err := s.store.GetItem(ctx, id)
|
||||
if err != nil || fresh == nil || !fresh.Active {
|
||||
return
|
||||
}
|
||||
s.RunPoll(ctx, *fresh)
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("schedule failed", "item_id", it.ID, "err", err)
|
||||
return
|
||||
}
|
||||
s.entries[it.ID] = entryID
|
||||
}
|
||||
|
||||
// RunPoll executes one poll cycle for an item. Public so handlers can trigger
|
||||
// "Run Now" without going through cron. Iterates over each (alias × marketplace)
|
||||
// pair; a single failing combo does not poison the others.
|
||||
func (s *Scheduler) RunPoll(ctx context.Context, it models.Item) {
|
||||
plans := s.buildAllInputs(it)
|
||||
if len(plans) == 0 {
|
||||
s.recordError(ctx, it.ID, "no marketplaces configured for this item")
|
||||
return
|
||||
}
|
||||
apifyClient := s.apifyClient(ctx)
|
||||
var results []apify.UnifiedResult
|
||||
var errs []string
|
||||
successes := 0
|
||||
for _, p := range plans {
|
||||
if p.actorID == "" {
|
||||
errs = append(errs, fmt.Sprintf("%s: no actor configured", p.marketplace))
|
||||
continue
|
||||
}
|
||||
raw, err := apifyClient.Run(ctx, p.actorID, p.input)
|
||||
if err != nil {
|
||||
label := p.marketplace
|
||||
if p.query != "" {
|
||||
label = fmt.Sprintf("query %q on %s", p.query, p.marketplace)
|
||||
}
|
||||
errs = append(errs, fmt.Sprintf("%s: %s", label, err.Error()))
|
||||
slog.Error("apify run failed", "item_id", it.ID, "marketplace", p.marketplace, "query", p.query, "err", err)
|
||||
continue
|
||||
}
|
||||
decoded, _ := apify.Decode(raw, p.source)
|
||||
usable := 0
|
||||
for i := range decoded {
|
||||
decoded[i].MatchedQuery = p.query
|
||||
if decoded[i].URL != "" && decoded[i].Price > 0 {
|
||||
usable++
|
||||
}
|
||||
}
|
||||
slog.Info("apify run decoded",
|
||||
"item_id", it.ID,
|
||||
"marketplace", p.marketplace,
|
||||
"query", p.query,
|
||||
"actor", p.actorID,
|
||||
"raw", len(raw),
|
||||
"decoded", len(decoded),
|
||||
"usable", usable,
|
||||
)
|
||||
if usable == 0 && len(raw) > 0 {
|
||||
var sample map[string]any
|
||||
if err := jsonUnmarshal(raw[0], &sample); err == nil {
|
||||
keys := make([]string, 0, len(sample))
|
||||
for k := range sample {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
slog.Warn("decoded zero usable rows; raw item keys",
|
||||
"item_id", it.ID,
|
||||
"marketplace", p.marketplace,
|
||||
"actor", p.actorID,
|
||||
"keys", keys,
|
||||
)
|
||||
}
|
||||
}
|
||||
results = append(results, decoded...)
|
||||
successes++
|
||||
}
|
||||
if successes == 0 {
|
||||
s.recordError(ctx, it.ID, strings.Join(errs, "; "))
|
||||
return
|
||||
}
|
||||
|
||||
if it.UsePriceComparison {
|
||||
pcID := it.ActorPriceCompare
|
||||
if pcID == "" {
|
||||
pcID = s.cfg.Apify.Actors.PriceComparison
|
||||
}
|
||||
if pcID != "" {
|
||||
pcQueries := it.SearchQueries()
|
||||
if len(pcQueries) == 0 && it.URL != "" {
|
||||
pcQueries = []string{""}
|
||||
}
|
||||
for _, q := range pcQueries {
|
||||
pcRaw, err := apifyClient.Run(ctx, pcID, apify.PriceComparisonInput{
|
||||
Query: q, URL: it.URL,
|
||||
ProxyConfiguration: s.proxyConfig(),
|
||||
})
|
||||
if err == nil {
|
||||
pc, _ := apify.Decode(pcRaw, apify.SourcePriceCompare)
|
||||
for i := range pc {
|
||||
pc[i].MatchedQuery = q
|
||||
}
|
||||
results = append(results, pc...)
|
||||
} else {
|
||||
slog.Warn("price comparison failed", "item_id", it.ID, "query", q, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
beforeDedup := len(results)
|
||||
results = DedupByURL(results)
|
||||
|
||||
threshold := s.cfg.Scheduler.MatchConfidenceThreshold
|
||||
beforeFilter := len(results)
|
||||
results = FilterResults(results, threshold, it.IncludeOutOfStock)
|
||||
results = ApplyItemFilters(results, it.MinPrice, it.ExcludeKeywordsList())
|
||||
slog.Info("filter applied",
|
||||
"item_id", it.ID,
|
||||
"before_dedup", beforeDedup,
|
||||
"before_filter", beforeFilter,
|
||||
"after", len(results),
|
||||
"min_confidence", threshold,
|
||||
"min_price", it.MinPrice,
|
||||
"exclude_count", len(it.ExcludeKeywordsList()),
|
||||
"include_out_of_stock", it.IncludeOutOfStock,
|
||||
)
|
||||
|
||||
bestIdx := PickBest(results)
|
||||
alertsSent := 0
|
||||
for _, r := range results {
|
||||
exists, err := s.store.ResultExists(ctx, it.ID, r.URL)
|
||||
if err != nil {
|
||||
slog.Error("dedup check failed", "err", err)
|
||||
continue
|
||||
}
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
alerted := false
|
||||
if ShouldAlert(it.TargetPrice, r.Price) {
|
||||
if err := s.sendNotification(ctx, it, r); err != nil {
|
||||
slog.Error("ntfy send failed", "err", err)
|
||||
} else {
|
||||
alerted = true
|
||||
alertsSent++
|
||||
}
|
||||
}
|
||||
price := r.Price
|
||||
_, err = s.store.InsertResult(ctx, &models.Result{
|
||||
ItemID: it.ID,
|
||||
Title: r.Title,
|
||||
Price: &price,
|
||||
Currency: r.Currency,
|
||||
URL: r.URL,
|
||||
Source: r.Source,
|
||||
ImageURL: r.ImageURL,
|
||||
MatchedQuery: r.MatchedQuery,
|
||||
Alerted: alerted,
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("insert result failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
errMsg := ""
|
||||
if len(errs) > 0 {
|
||||
errMsg = strings.Join(errs, "; ")
|
||||
}
|
||||
if bestIdx >= 0 {
|
||||
best := results[bestIdx]
|
||||
bp := best.Price
|
||||
_ = s.store.UpdateItemPollResult(ctx, it.ID, &models.Item{
|
||||
BestPrice: &bp,
|
||||
BestPriceStore: best.Store,
|
||||
BestPriceURL: best.URL,
|
||||
BestPriceImageURL: best.ImageURL,
|
||||
BestPriceTitle: best.Title,
|
||||
}, errMsg)
|
||||
_ = s.store.InsertPricePoint(ctx, &models.PricePoint{
|
||||
ItemID: it.ID,
|
||||
Price: bp,
|
||||
Store: best.Store,
|
||||
})
|
||||
} else {
|
||||
_ = s.store.UpdateItemPollResult(ctx, it.ID, nil, errMsg)
|
||||
}
|
||||
|
||||
slog.Info("poll completed",
|
||||
"item_id", it.ID,
|
||||
"item_name", it.Name,
|
||||
"marketplaces", len(plans),
|
||||
"successes", successes,
|
||||
"results", len(results),
|
||||
"alerts_sent", alertsSent,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *Scheduler) recordError(ctx context.Context, id int64, msg string) {
|
||||
if err := s.store.UpdateItemPollResult(ctx, id, nil, msg); err != nil {
|
||||
slog.Error("record error failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// apifyClient returns an apify.Client whose API key reflects the latest
|
||||
// value from settings, falling back to config.toml.
|
||||
func (s *Scheduler) apifyClient(ctx context.Context) *apify.Client {
|
||||
key := s.cfg.Apify.APIKey
|
||||
if v, _ := s.store.GetSetting(ctx, "apify_api_key"); v != "" {
|
||||
key = v
|
||||
}
|
||||
return apify.New(key)
|
||||
}
|
||||
|
||||
func (s *Scheduler) sendNotification(ctx context.Context, it models.Item, r apify.UnifiedResult) error {
|
||||
tags := []string{"mag"}
|
||||
if it.TargetPrice != nil && r.Price <= *it.TargetPrice {
|
||||
tags = []string{"shopping_cart", "tada"}
|
||||
}
|
||||
priority := it.NtfyPriority
|
||||
if priority == "" {
|
||||
priority = "default"
|
||||
}
|
||||
topic := it.NtfyTopic
|
||||
if topic == "" {
|
||||
if v, _ := s.store.GetSetting(ctx, "ntfy_default_topic"); v != "" {
|
||||
topic = v
|
||||
} else {
|
||||
topic = s.cfg.Ntfy.DefaultTopic
|
||||
}
|
||||
}
|
||||
msg := fmt.Sprintf("%s %s%.2f", r.Store, currencyPrefix(r.Currency), r.Price)
|
||||
if it.TargetPrice != nil {
|
||||
msg += fmt.Sprintf(" (target: %s%.2f)", currencyPrefix(r.Currency), *it.TargetPrice)
|
||||
}
|
||||
if r.Title != "" {
|
||||
msg += "\n" + r.Title
|
||||
}
|
||||
baseURL := s.cfg.Ntfy.BaseURL
|
||||
if v, _ := s.store.GetSetting(ctx, "ntfy_base_url"); v != "" {
|
||||
baseURL = v
|
||||
}
|
||||
token, _ := s.store.GetSetting(ctx, "ntfy_token")
|
||||
client := ntfy.NewWithToken(baseURL, token)
|
||||
return client.Send(ctx, ntfy.Notification{
|
||||
Topic: topic,
|
||||
Title: fmt.Sprintf("Veola Alert: %s", it.Name),
|
||||
Message: msg,
|
||||
Priority: priority,
|
||||
Tags: tags,
|
||||
Click: r.URL,
|
||||
})
|
||||
}
|
||||
|
||||
func currencyPrefix(c string) string {
|
||||
switch c {
|
||||
case "USD", "":
|
||||
return "$"
|
||||
case "GBP":
|
||||
return "£"
|
||||
case "EUR":
|
||||
return "€"
|
||||
case "JPY":
|
||||
return "¥"
|
||||
}
|
||||
return c + " "
|
||||
}
|
||||
|
||||
// BuildPreviewInputs returns one actor plan per alias for the first marketplace
|
||||
// on the item. Preview deliberately uses only one marketplace to limit actor
|
||||
// runs, but exercises every alias so the operator sees the full result set.
|
||||
func (s *Scheduler) BuildPreviewInputs(it models.Item) []actorPlan {
|
||||
queries := it.SearchQueries()
|
||||
if len(queries) == 0 {
|
||||
queries = []string{""}
|
||||
}
|
||||
markets := it.Marketplaces
|
||||
if len(markets) > 1 {
|
||||
markets = markets[:1]
|
||||
}
|
||||
var out []actorPlan
|
||||
for _, q := range queries {
|
||||
out = append(out, s.buildInputsForQuery(it, q, markets)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type actorPlan struct {
|
||||
marketplace string
|
||||
source string
|
||||
actorID string
|
||||
query string
|
||||
input any
|
||||
}
|
||||
|
||||
// Marketplace returns the marketplace for this plan.
|
||||
func (p actorPlan) Marketplace() string { return p.marketplace }
|
||||
|
||||
// Source returns the result-source label (used to pick a decoder).
|
||||
func (p actorPlan) Source() string { return p.source }
|
||||
|
||||
// ActorID returns the Apify actor ID this plan will invoke.
|
||||
func (p actorPlan) ActorID() string { return p.actorID }
|
||||
|
||||
// Query returns the alias string this plan searches for. Empty for URL-only items.
|
||||
func (p actorPlan) Query() string { return p.query }
|
||||
|
||||
// Input returns the actor input payload as expected by apify.Client.Run.
|
||||
func (p actorPlan) Input() any { return p.input }
|
||||
|
||||
// buildAllInputs returns one actor plan per (alias × marketplace) for the item.
|
||||
// For URL-only items (no aliases), produces one plan per marketplace with an
|
||||
// empty query string.
|
||||
func (s *Scheduler) buildAllInputs(it models.Item) []actorPlan {
|
||||
queries := it.SearchQueries()
|
||||
if len(queries) == 0 {
|
||||
queries = []string{""}
|
||||
}
|
||||
markets := it.Marketplaces
|
||||
if len(markets) == 0 {
|
||||
markets = []string{"ebay.com"}
|
||||
}
|
||||
var out []actorPlan
|
||||
for _, q := range queries {
|
||||
out = append(out, s.buildInputsForQuery(it, q, markets)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildInputsForQuery returns one actor plan per marketplace, all using the
|
||||
// same query string. Used by both the scheduler and the preview path.
|
||||
func (s *Scheduler) buildInputsForQuery(it models.Item, query string, markets []string) []actorPlan {
|
||||
url := strings.ToLower(it.URL)
|
||||
plans := make([]actorPlan, 0, len(markets))
|
||||
for _, m := range markets {
|
||||
mk := strings.ToLower(m)
|
||||
switch {
|
||||
case strings.Contains(mk, "yahoo") || strings.Contains(url, "yahoo.co.jp"):
|
||||
actorID := firstNonEmpty(it.ActorActive, s.cfg.Apify.Actors.YahooAuctionsJP)
|
||||
plans = append(plans, actorPlan{m, apify.SourceYahooJP, actorID, query, apify.YahooAuctionsJPInput{
|
||||
SearchTerm: query,
|
||||
MaxPages: 1,
|
||||
}})
|
||||
case strings.Contains(mk, "mercari") || strings.Contains(url, "mercari"):
|
||||
actorID := firstNonEmpty(it.ActorActive, s.cfg.Apify.Actors.MercariJP)
|
||||
plans = append(plans, actorPlan{m, apify.SourceMercariJP, actorID, query, apify.MercariJPInput{
|
||||
SearchKeywords: []string{query},
|
||||
Status: "on_sale",
|
||||
MaxResults: 30,
|
||||
}})
|
||||
default:
|
||||
actorID := firstNonEmpty(it.ActorActive, s.cfg.Apify.Actors.ActiveListings)
|
||||
plans = append(plans, actorPlan{m, apify.SourceActiveEbay, actorID, query, apify.ActiveListingInput{
|
||||
SearchQueries: []string{query},
|
||||
MaxProductsPerSearch: 30,
|
||||
MaxSearchPages: 1,
|
||||
Sort: "best_match",
|
||||
ListingType: mapListingType(it.ListingType),
|
||||
ProxyConfiguration: s.proxyConfig(),
|
||||
}})
|
||||
}
|
||||
}
|
||||
return plans
|
||||
}
|
||||
|
||||
// DedupByURL collapses duplicates within a single result set. When the same
|
||||
// listing matches multiple aliases the first occurrence wins, including its
|
||||
// MatchedQuery tag.
|
||||
func DedupByURL(in []apify.UnifiedResult) []apify.UnifiedResult {
|
||||
seen := map[string]bool{}
|
||||
out := make([]apify.UnifiedResult, 0, len(in))
|
||||
for _, r := range in {
|
||||
if r.URL == "" {
|
||||
out = append(out, r)
|
||||
continue
|
||||
}
|
||||
key := r.Source + "|" + r.URL
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// proxyConfig returns the apify proxyConfiguration block built from
|
||||
// config.toml. Returns nil — meaning omit the field from actor input
|
||||
// entirely — if use_apify_proxy is false. Group and country are ignored when
|
||||
// use_apify_proxy is false to prevent contradictory input.
|
||||
func (s *Scheduler) proxyConfig() *apify.ProxyConfiguration {
|
||||
p := s.cfg.Apify.Proxy
|
||||
if !p.UseApifyProxy {
|
||||
return nil
|
||||
}
|
||||
return &apify.ProxyConfiguration{
|
||||
UseApifyProxy: true,
|
||||
ApifyProxyGroups: p.Groups,
|
||||
ApifyProxyCountry: p.Country,
|
||||
}
|
||||
}
|
||||
|
||||
// mapListingType translates Veola's listing-type vocabulary ("all", "BIN",
|
||||
// "auction") into the automation-lab/ebay-scraper input vocabulary
|
||||
// ("all", "buy_it_now", "auction"). Unrecognized values fall through as-is
|
||||
// in case the user pasted a value the actor accepts but we don't.
|
||||
func mapListingType(s string) string {
|
||||
switch strings.ToLower(s) {
|
||||
case "", "all":
|
||||
return "all"
|
||||
case "bin", "buy_it_now":
|
||||
return "buy_it_now"
|
||||
case "auction":
|
||||
return "auction"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func firstNonEmpty(vs ...string) string {
|
||||
for _, v := range vs {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SeedSoldHistory runs the sold-listings actor and writes price_history rows
|
||||
// for an item just added. Errors are logged and swallowed: a missing baseline
|
||||
// is not fatal.
|
||||
func (s *Scheduler) SeedSoldHistory(ctx context.Context, it models.Item) {
|
||||
queries := it.SearchQueries()
|
||||
if len(queries) == 0 {
|
||||
return
|
||||
}
|
||||
markets := it.Marketplaces
|
||||
if len(markets) == 0 {
|
||||
markets = []string{"ebay.com"}
|
||||
}
|
||||
for _, q := range queries {
|
||||
for _, m := range markets {
|
||||
s.seedSoldHistoryFor(ctx, it, q, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scheduler) seedSoldHistoryFor(ctx context.Context, it models.Item, query, marketplace string) {
|
||||
actorID := firstNonEmpty(it.ActorSold, s.cfg.Apify.Actors.SoldListings)
|
||||
source := apify.SourceSoldEbay
|
||||
if strings.Contains(strings.ToLower(marketplace), "yahoo") {
|
||||
actorID = firstNonEmpty(it.ActorSold, s.cfg.Apify.Actors.YahooAuctionsJPSold)
|
||||
source = apify.SourceSoldYahooJP
|
||||
}
|
||||
if actorID == "" {
|
||||
return
|
||||
}
|
||||
raw, err := s.apifyClient(ctx).Run(ctx, actorID, apify.SoldListingInput{
|
||||
Query: query, Marketplace: marketplace, MaxResults: 50, DaysBack: 30,
|
||||
ProxyConfiguration: s.proxyConfig(),
|
||||
})
|
||||
if err != nil {
|
||||
slog.Warn("sold history seed failed", "item_id", it.ID, "marketplace", marketplace, "query", query, "err", err)
|
||||
return
|
||||
}
|
||||
for _, r := range raw {
|
||||
var sold apify.SoldListingResult
|
||||
if err := jsonUnmarshal(r, &sold); err != nil || sold.SoldPrice <= 0 {
|
||||
continue
|
||||
}
|
||||
t, _ := time.Parse(time.RFC3339, sold.SoldAt)
|
||||
if t.IsZero() {
|
||||
t = time.Now()
|
||||
}
|
||||
_ = s.store.InsertPricePoint(ctx, &models.PricePoint{
|
||||
ItemID: it.ID,
|
||||
Price: sold.SoldPrice,
|
||||
Store: sourceLabelToStore(source),
|
||||
PolledAt: t,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func sourceLabelToStore(src string) string {
|
||||
switch src {
|
||||
case apify.SourceSoldYahooJP:
|
||||
return "yahoo-auctions-jp-sold"
|
||||
}
|
||||
return "ebay-sold"
|
||||
}
|
||||
Reference in New Issue
Block a user