Files
Pete/internal/config/config.go
prosolis 4bdf9a7615 Add per-source user_agent override for WAF-gated feeds
The Portugal News moved its RSS behind AWS WAF, which 405s Pete's honest
bot UA (confirmed: bot UA 0/6, browser UA 6/6 from the server IP). Add an
optional per-source user_agent that falls back to the default bot UA, and
set The Portugal News to a browser UA. Not load-related — 30-min polls and
the IP isn't banned.
2026-06-21 16:40:05 -07:00

231 lines
7.7 KiB
Go

package config
import (
"fmt"
"log/slog"
"os"
"regexp"
"github.com/BurntSushi/toml"
)
// envBracketRe matches only ${VAR} style env references, not bare $VAR.
var envBracketRe = regexp.MustCompile(`\$\{([^}]+)\}`)
type Config struct {
Matrix MatrixConfig `toml:"matrix"`
Posting PostingConfig `toml:"posting"`
Storage StorageConfig `toml:"storage"`
Web WebConfig `toml:"web"`
Sources []SourceConfig `toml:"sources"`
}
// WebConfig controls the read-only HTTP interface (news.parodia.dev style).
type WebConfig struct {
Enabled bool `toml:"enabled"`
ListenAddr string `toml:"listen_addr"` // e.g. ":8080" or "127.0.0.1:8080"
SiteTitle string `toml:"site_title"` // display name in the header
BaseURL string `toml:"base_url"` // public URL (used in metadata only)
Auth AuthConfig `toml:"auth"` // optional OIDC sign-in (Authentik)
}
// AuthConfig wires Pete's web UI to an OIDC provider (our Authentik instance).
// When enabled, signed-in users get their preferences stored server-side keyed
// by the OIDC subject; anonymous visitors keep using browser localStorage.
type AuthConfig struct {
Enabled bool `toml:"enabled"`
Issuer string `toml:"issuer"` // e.g. https://authentik.parodia.dev/application/o/pete/
ClientID string `toml:"client_id"`
ClientSecret string `toml:"client_secret"`
RedirectURL string `toml:"redirect_url"` // e.g. https://news.parodia.dev/auth/callback
SessionSecret string `toml:"session_secret"` // HMAC key for signing the session cookie
}
type MatrixConfig struct {
Homeserver string `toml:"homeserver"`
UserID string `toml:"user_id"`
Password string `toml:"password"`
PickleKey string `toml:"pickle_key"`
DisplayName string `toml:"display_name"`
DataDir string `toml:"data_dir"`
AdminRoom string `toml:"admin_room"`
Channels map[string]string `toml:"channels"`
// Admins is the allowlist of Matrix user IDs permitted to run in-room
// commands like !post. Empty means commands are disabled — the channel
// rooms are public, so an empty list must NOT mean "anyone".
Admins []string `toml:"admins"`
}
type PostingConfig struct {
MinIntervalSeconds int `toml:"min_interval_seconds"`
BurstCapCount int `toml:"burst_cap_count"`
BurstCapWindowSeconds int `toml:"burst_cap_window_seconds"`
DedupCooldownHours int `toml:"dedup_cooldown_hours"`
// DailyCapTotal is the hard global cap on posts across ALL channels in a
// rolling 24-hour window. 0 disables the cap.
DailyCapTotal int `toml:"daily_cap_total"`
RoundRobin RoundRobinConfig `toml:"round_robin"`
}
// RoundRobinConfig switches Pete from immediate-on-classify posting to a
// paced rotation: one story per IntervalHours, picking the next channel in
// rotation order that has a postable story (skip-and-advance, newest-first).
type RoundRobinConfig struct {
Enabled bool `toml:"enabled"`
IntervalHours int `toml:"interval_hours"`
}
type StorageConfig struct {
DBPath string `toml:"db_path"`
RecentWindowHours int `toml:"recent_window_hours"`
}
type SourceConfig struct {
Name string `toml:"name"`
FeedURL string `toml:"feed_url"`
Tier int `toml:"tier"`
PollIntervalMinutes int `toml:"poll_interval_minutes"`
DirectRoute string `toml:"direct_route"`
Enabled bool `toml:"enabled"`
// Language, when set, drops feed items whose per-item language tag is
// present and does not match (prefix). Useful for multilingual feeds
// like Politico Europe that publish English + French side-by-side.
Language string `toml:"language"`
// UserAgent overrides the User-Agent sent when fetching this feed. Empty
// uses Pete's honest default bot UA. Set a browser-like string only for
// sources whose WAF (e.g. AWS WAF on The Portugal News) blocks bot UAs.
UserAgent string `toml:"user_agent"`
}
func Load(path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read config: %w", err)
}
// Expand only ${VAR} style env references, not bare $VAR
// (bare $ in passwords like "pa$$word" must not be mangled)
expanded := envBracketRe.ReplaceAllStringFunc(string(data), func(match string) string {
varName := match[2 : len(match)-1] // strip ${ and }
val := os.Getenv(varName)
if val == "" {
slog.Warn("config: env var referenced but not set", "var", varName)
}
return val
})
var cfg Config
if _, err := toml.Decode(expanded, &cfg); err != nil {
return nil, fmt.Errorf("parse config: %w", err)
}
cfg.applyDefaults()
if err := cfg.validate(); err != nil {
return nil, fmt.Errorf("validate config: %w", err)
}
return &cfg, nil
}
func (c *Config) validate() error {
if c.Matrix.Homeserver == "" {
return fmt.Errorf("matrix.homeserver is required")
}
if c.Matrix.UserID == "" {
return fmt.Errorf("matrix.user_id is required")
}
if c.Matrix.Password == "" {
return fmt.Errorf("matrix.password is required")
}
// pickle_key encrypts the E2EE crypto store (olm/megolm + cross-signing
// keys) at rest. Never fall back to a hardcoded default: a key baked into
// the source is no protection if crypto.db ever leaks.
if len(c.Matrix.PickleKey) < 16 {
return fmt.Errorf("matrix.pickle_key must be set to a strong secret (>=16 chars); it encrypts the E2EE crypto store at rest")
}
if len(c.Matrix.Channels) == 0 {
return fmt.Errorf("matrix.channels must have at least one entry")
}
if c.Storage.DBPath == "" {
return fmt.Errorf("storage.db_path is required")
}
if c.Web.Auth.Enabled {
a := c.Web.Auth
if a.Issuer == "" || a.ClientID == "" || a.ClientSecret == "" || a.RedirectURL == "" {
return fmt.Errorf("web.auth requires issuer, client_id, client_secret, and redirect_url when enabled")
}
if len(a.SessionSecret) < 16 {
return fmt.Errorf("web.auth.session_secret must be at least 16 characters")
}
}
for i, s := range c.Sources {
if s.Name == "" {
return fmt.Errorf("sources[%d].name is required", i)
}
if s.FeedURL == "" {
return fmt.Errorf("sources[%d].feed_url is required", i)
}
if s.Tier < 1 || s.Tier > 3 {
return fmt.Errorf("sources[%d].tier must be 1-3", i)
}
if s.PollIntervalMinutes <= 0 {
return fmt.Errorf("sources[%d].poll_interval_minutes must be > 0", i)
}
if !s.Enabled {
continue
}
if s.DirectRoute == "" {
return fmt.Errorf("sources[%d] (%s): direct_route is required for enabled sources", i, s.Name)
}
if _, ok := c.Matrix.Channels[s.DirectRoute]; !ok {
// Not a Matrix channel — treated as web-only (stories visible in the
// UI but never posted). Warn so typos still surface.
slog.Warn("source routes to non-Matrix channel (web-only)",
"source", s.Name, "direct_route", s.DirectRoute)
}
}
return nil
}
func (c *Config) applyDefaults() {
if c.Matrix.DataDir == "" {
c.Matrix.DataDir = "./data"
}
if c.Matrix.DisplayName == "" {
c.Matrix.DisplayName = "Pete"
}
if c.Posting.MinIntervalSeconds == 0 {
c.Posting.MinIntervalSeconds = 300
}
if c.Posting.BurstCapCount == 0 {
c.Posting.BurstCapCount = 3
}
if c.Posting.BurstCapWindowSeconds == 0 {
c.Posting.BurstCapWindowSeconds = 1800
}
if c.Posting.DedupCooldownHours == 0 {
c.Posting.DedupCooldownHours = 48
}
if c.Posting.RoundRobin.IntervalHours == 0 {
c.Posting.RoundRobin.IntervalHours = 4
}
if c.Storage.RecentWindowHours == 0 {
c.Storage.RecentWindowHours = 24
}
if c.Web.ListenAddr == "" {
c.Web.ListenAddr = ":8080"
}
if c.Web.SiteTitle == "" {
c.Web.SiteTitle = "Pete"
}
for i := range c.Sources {
if c.Sources[i].PollIntervalMinutes == 0 {
c.Sources[i].PollIntervalMinutes = 20
}
}
}