Compare commits

...

2 Commits

Author SHA1 Message Date
prosolis
ec1f130ed5 Add full-text search, EU channel theme, and web-only sources
Search: new FTS5-backed SearchStories query, /search JSON endpoint,
client-side overlay (search.js) wired into the layout header. EU
channel gets its own theme color (#003399) across bg/text/border/glow
classes. Sources can now route to non-Matrix channels without
validation error (web-only mode); a warning still flags typos.
2026-05-25 11:23:52 -07:00
prosolis
509a0fc7a7 Harden paywall detection + render paywalled stamp on cards
Bypass-UA retry (Googlebot + Google referer) for soft paywalls, JSON-LD
gating scoped to Article-typed nodes, HTTP 402 treated as explicit
paywall, Wayback freshness filter (30d cap), archive.today as secondary
archive fallback, and transport failures no longer trigger snapshot
swaps. When gating is detected and no archive workaround succeeds, the
story is stored with paywalled=1 and the web card renders a diagonal
red rubber-stamp overlay so readers know the link is gated.
2026-05-25 11:23:22 -07:00
16 changed files with 679 additions and 89 deletions

View File

@@ -140,7 +140,10 @@ func (c *Config) validate() error {
return fmt.Errorf("sources[%d] (%s): direct_route is required for enabled sources", i, s.Name)
}
if _, ok := c.Matrix.Channels[s.DirectRoute]; !ok {
return fmt.Errorf("sources[%d] (%s): direct_route %q is not a configured matrix.channels key", i, s.Name, s.DirectRoute)
// Not a Matrix channel — treated as web-only (stories visible in the
// UI but never posted). Warn so typos still surface.
slog.Warn("source routes to non-Matrix channel (web-only)",
"source", s.Name, "direct_route", s.DirectRoute)
}
}

View File

@@ -202,23 +202,6 @@ feed_url = "https://e.com/rss"
tier = 1
poll_interval_minutes = 20
enabled = true
`},
{"direct_route not in channels", `
[matrix]
homeserver = "https://h"
user_id = "@p:h"
password = "pw"
[matrix.channels]
tech = "!t:e"
[storage]
db_path = "/tmp/t.db"
[[sources]]
name = "S"
feed_url = "https://e.com/rss"
tier = 1
poll_interval_minutes = 20
direct_route = "gaming"
enabled = true
`},
}

View File

@@ -48,54 +48,128 @@ func resolveURL(base, ref string) string {
// as paywalled / gated, and the caller should fall back to an archive snapshot.
const PaywallBodyThreshold = 500
// googlebotUA is what many metered publishers grant first-click access to.
// Re-tried automatically when the default-UA fetch looks gated.
const googlebotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
// ArticleMeta is what we can learn from fetching an article page directly.
type ArticleMeta struct {
ImageURL string // og:image or twitter:image, absolute URL
BodyChars int // length of extracted visible body text
Fetched bool // true if we got an HTTP 200 with HTML
Paywalled bool // true if the page explicitly declares gated access
ImageURL string // og:image or twitter:image, absolute URL
BodyChars int // length of extracted visible body text
Fetched bool // true if we got an HTTP 200 with HTML
FetchError bool // true if the fetch failed at the network/HTTP layer
Paywalled bool // true if the page explicitly declares gated access
Status int // last HTTP status seen (0 on transport error)
}
// Gated reports whether the response carries a strong gating signal: an
// explicit paywall meta/JSON-LD declaration, an HTTP 402 Payment Required,
// or a 403 Forbidden after the bypass retry. A short body alone is not
// considered gating — that's a heuristic used by callers separately.
func (m ArticleMeta) Gated() bool {
if m.Paywalled {
return true
}
return m.Status == http.StatusPaymentRequired || m.Status == http.StatusForbidden
}
var articleClient = &http.Client{Timeout: 12 * time.Second}
// FetchArticleMeta fetches an article URL and returns its og:image plus the
// length of the visible body text (concatenation of <p> tags under <article>
// or <main>, falling back to all <p> tags). Returns Fetched=false on any
// network/HTTP failure so callers can branch on accessibility.
// FetchArticleMeta fetches an article URL with the default UA. If the result
// looks gated (explicit paywall signal, HTTP 402/403, or body too short) it
// retries once with a Googlebot UA + Google referer — the combination most
// metered publishers grant first-click access to. Returns the best of the
// two attempts.
func FetchArticleMeta(articleURL string) ArticleMeta {
if articleURL == "" {
return ArticleMeta{}
}
first := fetchArticleMetaOnce(articleURL, userAgent, "")
if !shouldRetryAsBot(first) {
return first
}
second := fetchArticleMetaOnce(articleURL, googlebotUA, "https://www.google.com/")
return pickBetter(first, second)
}
// shouldRetryAsBot returns true when the first attempt looks gated or too
// thin to be the real article body.
func shouldRetryAsBot(m ArticleMeta) bool {
if m.FetchError {
return false // transport failure won't be fixed by a different UA
}
if m.Gated() {
return true
}
if m.Fetched && m.BodyChars < PaywallBodyThreshold {
return true
}
return false
}
// pickBetter chooses the more useful of two fetch attempts: prefer the one
// that isn't gated, then the one with more body, then the one that fetched
// at all.
func pickBetter(a, b ArticleMeta) ArticleMeta {
aGated, bGated := a.Gated(), b.Gated()
if aGated != bGated {
if bGated {
return a
}
return b
}
if a.Fetched != b.Fetched {
if b.Fetched {
return b
}
return a
}
if b.BodyChars > a.BodyChars {
return b
}
return a
}
func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
if err != nil {
return ArticleMeta{}
return ArticleMeta{FetchError: true}
}
req.Header.Set("User-Agent", userAgent)
req.Header.Set("User-Agent", ua)
req.Header.Set("Accept", "text/html,application/xhtml+xml")
if referer != "" {
req.Header.Set("Referer", referer)
}
resp, err := articleClient.Do(req)
if err != nil {
return ArticleMeta{}
return ArticleMeta{FetchError: true}
}
defer resp.Body.Close()
meta := ArticleMeta{Status: resp.StatusCode}
// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
// We still try to parse the body for whatever signals it contains.
if resp.StatusCode != http.StatusOK {
return ArticleMeta{}
if resp.StatusCode == http.StatusPaymentRequired {
meta.Paywalled = true
}
return meta
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return ArticleMeta{}
return meta
}
return ArticleMeta{
ImageURL: extractOGImage(doc, articleURL),
BodyChars: extractBodyChars(doc),
Fetched: true,
Paywalled: detectPaywall(doc),
}
meta.Fetched = true
meta.ImageURL = extractOGImage(doc, articleURL)
meta.BodyChars = extractBodyChars(doc)
meta.Paywalled = detectPaywall(doc)
return meta
}
// detectPaywall checks the page for explicit gating signals that publishers
@@ -121,9 +195,11 @@ func detectPaywall(doc *goquery.Document) bool {
return gated
}
// jsonLDDeclaresGated returns true if the JSON-LD payload contains
// "isAccessibleForFree": false anywhere in its (possibly nested or arrayed)
// structure. Publishers vary wildly in shape, so we walk generically.
// jsonLDDeclaresGated returns true if the JSON-LD payload contains an
// Article-typed object (Article, NewsArticle, Report, BlogPosting, etc.)
// with "isAccessibleForFree" set falsy. We restrict to Article types so
// that embedded related-content or breadcrumb markup doesn't flip a free
// article to gated.
func jsonLDDeclaresGated(raw string) bool {
raw = strings.TrimSpace(raw)
if raw == "" {
@@ -136,21 +212,27 @@ func jsonLDDeclaresGated(raw string) bool {
return walkJSONLDForGated(v)
}
// articleSchemaTypes are the schema.org @type values we treat as "the main
// article" for paywall purposes.
var articleSchemaTypes = map[string]bool{
"article": true,
"newsarticle": true,
"report": true,
"reportagenewsarticle": true,
"blogposting": true,
"scholarlyarticle": true,
"techarticle": true,
"opinionnewsarticle": true,
"analysisnewsarticle": true,
"backgroundnewsarticle": true,
"reviewnewsarticle": true,
}
func walkJSONLDForGated(v any) bool {
switch x := v.(type) {
case map[string]any:
if raw, ok := x["isAccessibleForFree"]; ok {
switch r := raw.(type) {
case bool:
if !r {
return true
}
case string:
s := strings.ToLower(strings.TrimSpace(r))
if s == "false" || s == "no" {
return true
}
}
if isArticleType(x["@type"]) && declaresGated(x["isAccessibleForFree"]) {
return true
}
for _, vv := range x {
if walkJSONLDForGated(vv) {
@@ -167,6 +249,31 @@ func walkJSONLDForGated(v any) bool {
return false
}
func isArticleType(v any) bool {
switch t := v.(type) {
case string:
return articleSchemaTypes[strings.ToLower(strings.TrimSpace(t))]
case []any:
for _, e := range t {
if isArticleType(e) {
return true
}
}
}
return false
}
func declaresGated(v any) bool {
switch r := v.(type) {
case bool:
return !r
case string:
s := strings.ToLower(strings.TrimSpace(r))
return s == "false" || s == "no"
}
return false
}
// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
// that only care about the image. Returns "" when not found.
func FetchOGImage(articleURL string) string {

View File

@@ -126,32 +126,47 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
continue
}
// Genuinely new story — fetch the article page once. We use it for:
// Genuinely new story — fetch the article page once (with an internal
// bypass-UA retry on gating signals). We use it for:
// 1. og:image fallback when the feed didn't give us one.
// 2. Paywall detection: visible body text below the threshold means
// the article is gated, so swap in a Wayback snapshot for both
// Pete (image) and the reader (posted link).
// 2. Paywall detection: explicit gating signals or a body too short
// to be a real article means we swap in an archive snapshot for
// both Pete (image) and the reader (posted link).
//
// Pure transport failures (FetchError) are NOT treated as gating —
// swapping in a stale snapshot for a transient blip is worse than
// posting the live link.
meta := FetchArticleMeta(originalURL)
paywalled := !meta.Fetched || meta.Paywalled || meta.BodyChars < PaywallBodyThreshold
if paywalled {
if snap := ResolveWayback(originalURL); snap != "" {
gated := meta.Gated() || (meta.Fetched && meta.BodyChars < PaywallBodyThreshold)
// paywalled tracks whether the link the user will click is still
// gated — i.e. gating was detected AND no archive workaround worked.
// When a snapshot swap succeeds, the reader gets a readable page, so
// we don't stamp it.
paywalled := false
if gated {
workedAround := false
if snap := resolveArchive(originalURL); snap != "" {
snapMeta := FetchArticleMeta(snap)
if snapMeta.Fetched {
items[i].ArticleURL = snap
if items[i].ImageURL == "" && snapMeta.ImageURL != "" {
items[i].ImageURL = snapMeta.ImageURL
}
slog.Info("paywall detected, using wayback snapshot",
workedAround = true
slog.Info("paywall detected, using archive snapshot",
"guid", items[i].GUID, "original", originalURL,
"snapshot", snap, "body_chars", meta.BodyChars)
"snapshot", snap, "body_chars", meta.BodyChars,
"status", meta.Status)
} else {
slog.Debug("paywall detected but snapshot fetch failed",
"guid", items[i].GUID, "original", originalURL, "snapshot", snap)
}
} else {
slog.Debug("paywall detected but no wayback snapshot available",
"guid", items[i].GUID, "url", originalURL, "body_chars", meta.BodyChars)
slog.Debug("paywall detected but no archive snapshot available",
"guid", items[i].GUID, "url", originalURL,
"body_chars", meta.BodyChars, "status", meta.Status)
}
paywalled = !workedAround
} else if items[i].ImageURL == "" && meta.ImageURL != "" {
items[i].ImageURL = meta.ImageURL
slog.Debug("og:image fallback used",
@@ -172,6 +187,7 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
URLCanonical: canonical,
HeadlineNorm: headlineNorm,
Source: items[i].Source,
Paywalled: paywalled,
SeenAt: time.Now().Unix(),
}); err != nil {
slog.Error("failed to insert story", "guid", items[i].GUID, "err", err)
@@ -188,3 +204,13 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
return nil
}
// resolveArchive tries Wayback first (well-behaved availability API, fresh
// snapshots filtered) and falls back to archive.today, which often has
// captures Wayback doesn't — especially for hard-walled publishers.
func resolveArchive(articleURL string) string {
if snap := ResolveWayback(articleURL); snap != "" {
return snap
}
return ResolveArchiveToday(articleURL)
}

View File

@@ -5,29 +5,40 @@ import (
"encoding/json"
"net/http"
"net/url"
"strings"
"time"
)
var waybackClient = &http.Client{Timeout: 10 * time.Second}
// maxSnapshotAge bounds how stale a Wayback snapshot can be before we treat
// it as useless for a freshly-published news article. The "closest" snapshot
// returned by the availability API can otherwise be years old.
const maxSnapshotAge = 30 * 24 * time.Hour
type waybackResp struct {
ArchivedSnapshots struct {
Closest struct {
Available bool `json:"available"`
URL string `json:"url"`
Status string `json:"status"`
Timestamp string `json:"timestamp"` // YYYYMMDDhhmmss
} `json:"closest"`
} `json:"archived_snapshots"`
}
// ResolveWayback asks the Internet Archive's Wayback availability API for the
// closest snapshot of the given URL. Returns the snapshot URL (https) or ""
// if no snapshot exists or the request fails.
// ResolveWayback asks the Internet Archive's Wayback availability API for
// the closest snapshot of the given URL. Returns the snapshot URL (https)
// or "" if no snapshot exists, the request fails, or the closest snapshot
// is older than maxSnapshotAge.
func ResolveWayback(articleURL string) string {
if articleURL == "" {
return ""
}
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL)
// Anchor the lookup to "now" so we get the freshest snapshot rather
// than whichever happens to be Wayback's default closest match.
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL) +
"&timestamp=" + time.Now().UTC().Format("20060102150405")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -54,9 +65,76 @@ func ResolveWayback(articleURL string) string {
if !snap.Available || snap.Status != "200" || snap.URL == "" {
return ""
}
if !snapshotFreshEnough(snap.Timestamp) {
return ""
}
// Wayback sometimes returns http:// even when https is available.
if len(snap.URL) > 7 && snap.URL[:7] == "http://" {
if strings.HasPrefix(snap.URL, "http://") {
return "https://" + snap.URL[7:]
}
return snap.URL
}
func snapshotFreshEnough(ts string) bool {
if ts == "" {
return false
}
t, err := time.Parse("20060102150405", ts)
if err != nil {
return false
}
return time.Since(t) <= maxSnapshotAge
}
var archiveTodayClient = &http.Client{
Timeout: 10 * time.Second,
// Don't follow the redirect — we just want the Location header pointing
// at the most recent capture.
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
}
// ResolveArchiveToday looks up the newest archive.today / archive.ph snapshot
// for the given URL. archive.ph has no public availability API, but its
// `/newest/<url>` endpoint redirects (HTTP 302) to the most recent capture
// when one exists, or returns a non-redirect response otherwise. Returns
// "" on any failure or when no snapshot exists.
func ResolveArchiveToday(articleURL string) string {
if articleURL == "" {
return ""
}
api := "https://archive.ph/newest/" + articleURL
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
if err != nil {
return ""
}
req.Header.Set("User-Agent", userAgent)
resp, err := archiveTodayClient.Do(req)
if err != nil {
return ""
}
defer resp.Body.Close()
if resp.StatusCode < 300 || resp.StatusCode >= 400 {
return ""
}
loc := strings.TrimSpace(resp.Header.Get("Location"))
if loc == "" {
return ""
}
// archive.ph occasionally returns a relative Location; absolutize it.
if strings.HasPrefix(loc, "/") {
loc = "https://archive.ph" + loc
}
// Bounce back the input as Location means "no snapshot, here's the form"
// — distinguish a real capture URL (contains /YYYY/ or a short hash path).
if strings.Contains(loc, "://archive.ph/") && !strings.Contains(loc, "://archive.ph/newest/") {
return loc
}
return ""
}

View File

@@ -80,6 +80,7 @@ func runMigrations(d *sql.DB) error {
// we swallow that specifically.
addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"log/slog"
"strings"
"time"
)
@@ -28,10 +29,14 @@ func InsertStory(s *Story) error {
if s.Classified {
classified = 1
}
paywalled := 0
if s.Paywalled {
paywalled = 1
}
_, err := Get().Exec(
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, seen_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, s.SeenAt,
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt,
)
return err
}
@@ -220,7 +225,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
// channels (_discarded, _duplicate) are excluded.
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at,
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
@@ -234,7 +239,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt, &s.Posted); err != nil {
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
@@ -246,7 +251,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
// channels, newest first. Sentinel channels are excluded.
func ListAllClassified(limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at,
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
@@ -261,7 +266,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt, &s.Posted); err != nil {
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
@@ -273,7 +278,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
// Matrix, ordered by post time (newest first). Posted is always true.
func ListRecentlyPosted(limit int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.seen_at
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
FROM stories s
JOIN post_log p ON p.guid = s.guid
GROUP BY s.guid
@@ -286,7 +291,7 @@ func ListRecentlyPosted(limit int) ([]Story, error) {
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
return nil, err
}
s.Posted = true
@@ -343,6 +348,67 @@ func SetRoundRobinState(lastChannel string, tickAt int64) {
lastChannel, tickAt)
}
// SearchStories runs an FTS5 query against headline + lede and returns the
// best-ranked classified stories (real channels only), newest-ish first via
// bm25. The user query is tokenized and each token becomes a prefix match;
// FTS5 special characters are stripped so user input cannot break syntax.
func SearchStories(query string, limit int) ([]Story, error) {
fts := buildFTSQuery(query)
if fts == "" {
return nil, nil
}
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories_fts f
JOIN stories s ON s.id = f.rowid
WHERE f.stories_fts MATCH ?
AND s.classified = 1
AND s.channel IS NOT NULL
AND s.channel NOT IN ('_discarded', '_duplicate')
ORDER BY bm25(stories_fts) ASC, s.seen_at DESC
LIMIT ?`, fts, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
}
return out, rows.Err()
}
func buildFTSQuery(raw string) string {
var tokens []string
var cur strings.Builder
flush := func() {
if cur.Len() > 0 {
tokens = append(tokens, cur.String())
cur.Reset()
}
}
for _, r := range raw {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
cur.WriteRune(r)
case r > 127:
cur.WriteRune(r)
default:
flush()
}
}
flush()
for i, t := range tokens {
tokens[i] = "\"" + t + "\"*"
}
return strings.Join(tokens, " ")
}
// MarshalPlatforms converts a string slice to a JSON array string for storage.
func MarshalPlatforms(platforms []string) string {
if len(platforms) == 0 {

View File

@@ -14,6 +14,7 @@ CREATE TABLE IF NOT EXISTS stories (
platforms TEXT,
channel TEXT,
classified INTEGER NOT NULL DEFAULT 0,
paywalled INTEGER NOT NULL DEFAULT 0,
seen_at INTEGER NOT NULL
);

View File

@@ -14,6 +14,7 @@ type Story struct {
Platforms string // JSON array e.g. '["nintendo-switch","multi"]'
Channel string
Classified bool
Paywalled bool // source article is gated; ArticleURL may be an archive snapshot
SeenAt int64
Posted bool // true if this story has been posted to Matrix (joined from post_log)
}

View File

@@ -1,11 +1,13 @@
package web
import (
"encoding/json"
"fmt"
"html/template"
"log/slog"
"net/http"
"strconv"
"strings"
"time"
"pete/internal/storage"
@@ -22,6 +24,7 @@ type StoryView struct {
Source string
SeenAt time.Time
Posted bool
Paywalled bool // source is gated and no archive workaround succeeded
Channel string // channel slug; also the theme key
}
@@ -34,6 +37,7 @@ func toView(s storage.Story) StoryView {
Source: s.Source,
SeenAt: time.Unix(s.SeenAt, 0),
Posted: s.Posted,
Paywalled: s.Paywalled,
Channel: s.Channel,
}
}
@@ -235,6 +239,78 @@ func (s *Server) handleWeatherDemo(w http.ResponseWriter, r *http.Request) {
})
}
type searchResult struct {
Headline string `json:"headline"`
Lede string `json:"lede,omitempty"`
ArticleURL string `json:"article_url"`
ThumbURL string `json:"thumb_url,omitempty"`
Source string `json:"source"`
Channel string `json:"channel"`
ChannelTitle string `json:"channel_title"`
ChannelTheme string `json:"channel_theme"`
ChannelEmoji string `json:"channel_emoji"`
TimeAgo string `json:"time_ago"`
Posted bool `json:"posted"`
}
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
q := strings.TrimSpace(r.URL.Query().Get("q"))
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
if q == "" {
_, _ = w.Write([]byte(`{"results":[]}`))
return
}
const searchLimit = 12
rows, err := storage.SearchStories(q, searchLimit)
if err != nil {
slog.Error("web: search failed", "q", q, "err", err)
http.Error(w, "internal error", http.StatusInternalServerError)
return
}
channelByName := make(map[string]Channel, len(channels))
for _, ch := range channels {
channelByName[ch.Slug] = ch
}
results := make([]searchResult, 0, len(rows))
for _, row := range rows {
ch, ok := channelByName[row.Channel]
if !ok {
ch = Channel{Slug: row.Channel, Title: row.Channel, Theme: row.Channel}
}
results = append(results, searchResult{
Headline: row.Headline,
Lede: row.Lede,
ArticleURL: row.ArticleURL,
ThumbURL: thumbURL(row.ImageURL),
Source: row.Source,
Channel: ch.Slug,
ChannelTitle: ch.Title,
ChannelTheme: ch.Theme,
ChannelEmoji: ch.Emoji,
TimeAgo: shortTimeAgo(time.Unix(row.SeenAt, 0)),
Posted: row.Posted,
})
}
_ = json.NewEncoder(w).Encode(map[string]any{"results": results})
}
func shortTimeAgo(t time.Time) string {
d := time.Since(t)
switch {
case d < time.Minute:
return "just now"
case d < time.Hour:
return fmt.Sprintf("%dm ago", int(d.Minutes()))
case d < 24*time.Hour:
return fmt.Sprintf("%dh ago", int(d.Hours()))
case d < 7*24*time.Hour:
return fmt.Sprintf("%dd ago", int(d.Hours())/24)
default:
return t.Format("Jan 2")
}
}
func (s *Server) render(w http.ResponseWriter, page string, data any) {
tpl, ok := s.tpls[page]
if !ok {

View File

@@ -34,6 +34,7 @@ var channels = []Channel{
{Slug: "gaming", Title: "Gaming", Theme: "gaming", Emoji: "🎮", Blurb: "Releases, platforms, and the people making the games."},
{Slug: "tech", Title: "Tech", Theme: "tech", Emoji: "💻", Blurb: "Industry, products, and the wires that hold it all together."},
{Slug: "politics", Title: "Politics", Theme: "politics", Emoji: "🏛️", Blurb: "Policy, power, and the news of the day."},
{Slug: "eu", Title: "EU", Theme: "eu", Emoji: "🇪🇺", Blurb: "Portugal and the wider European beat. Read-only — these stories don't post to Matrix."},
{Slug: "music", Title: "Music", Theme: "music", Emoji: "🎵", Blurb: "Records, scenes, and the artists shaping the sound."},
}
@@ -73,6 +74,7 @@ func New(cfg config.WebConfig) (*Server, error) {
mux.HandleFunc("GET /{$}", s.handleIndex)
mux.HandleFunc("GET /weather", s.handleWeatherDemo)
mux.HandleFunc("GET /api/search", s.handleSearch)
for _, ch := range channels {
ch := ch
mux.HandleFunc("GET /"+ch.Slug, func(w http.ResponseWriter, r *http.Request) {

View File

@@ -58,27 +58,32 @@ html[data-phase="night"] {
.bg-theme-gaming { background-color: #4caf7d; }
.bg-theme-tech { background-color: #5aa9e6; }
.bg-theme-politics { background-color: #e07a5f; }
.bg-theme-eu { background-color: #003399; }
.bg-theme-music { background-color: #b079d6; }
.text-theme-gaming { color: #2d8a5a; }
.text-theme-tech { color: #2f7fb8; }
.text-theme-politics { color: #b8523a; }
.text-theme-eu { color: #003399; }
.text-theme-music { color: #8a4fb8; }
.decoration-theme-gaming { text-decoration-color: #4caf7d; }
.decoration-theme-tech { text-decoration-color: #5aa9e6; }
.decoration-theme-politics { text-decoration-color: #e07a5f; }
.decoration-theme-eu { text-decoration-color: #003399; }
.decoration-theme-music { text-decoration-color: #b079d6; }
.border-theme-gaming { border-color: #4caf7d; }
.border-theme-tech { border-color: #5aa9e6; }
.border-theme-politics { border-color: #e07a5f; }
.border-theme-eu { border-color: #003399; }
.border-theme-music { border-color: #b079d6; }
/* "Posted to Matrix" glow — soft pulsing aura in the channel's theme color. */
.glow-theme-gaming { box-shadow: 0 0 0 2px rgba(76,175,125,0.35), 0 0 24px 4px rgba(76,175,125,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
.glow-theme-tech { box-shadow: 0 0 0 2px rgba(90,169,230,0.35), 0 0 24px 4px rgba(90,169,230,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
.glow-theme-politics { box-shadow: 0 0 0 2px rgba(224,122,95,0.35), 0 0 24px 4px rgba(224,122,95,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
.glow-theme-eu { box-shadow: 0 0 0 2px rgba(0,51,153,0.35), 0 0 24px 4px rgba(0,51,153,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
.glow-theme-music { box-shadow: 0 0 0 2px rgba(176,121,214,0.35), 0 0 24px 4px rgba(176,121,214,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
@keyframes pete-glow-pulse {
@@ -92,4 +97,36 @@ html[data-phase="night"] {
-webkit-box-orient: vertical;
overflow: hidden;
}
/* "PAYWALLED" diagonal rubber stamp overlay — slapped across the whole
card when the source is gated and no archive snapshot worked. The card
stays clickable (link still goes to the original); the stamp is just
visual warning. pointer-events:none keeps it from eating clicks. */
.paywall-stamp {
position: absolute;
inset: 0;
pointer-events: none;
display: flex;
align-items: center;
justify-content: center;
z-index: 5;
background: rgba(255, 250, 240, 0.18);
}
.paywall-stamp::before {
content: "PAYWALLED";
font-family: var(--font-display, ui-sans-serif), system-ui, sans-serif;
font-weight: 900;
font-size: clamp(1rem, 3.2vw, 1.75rem);
letter-spacing: 0.12em;
color: rgba(180, 30, 30, 0.85);
border: 0.28rem double rgba(180, 30, 30, 0.85);
padding: 0.3rem 0.9rem;
transform: rotate(-15deg);
text-shadow: 0 1px 0 rgba(255,255,255,0.4);
box-shadow: inset 0 0 0 2px rgba(255,255,255,0.15);
background: rgba(255, 240, 230, 0.55);
border-radius: 0.25rem;
/* slight roughness to mimic a real ink stamp */
filter: contrast(1.05);
}
}

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,177 @@
// Cmd+K / Ctrl+K command palette. Hits /api/search and renders editorial-card
// results. Vanilla JS, no deps; styles come from output.css (Tailwind).
(function () {
const overlay = document.getElementById("pete-search");
if (!overlay) return;
const input = overlay.querySelector("[data-search-input]");
const list = overlay.querySelector("[data-search-list]");
const meta = overlay.querySelector("[data-search-meta]");
let activeIndex = -1;
let items = [];
let inflight = null;
let debounceTimer = 0;
let lastQuery = "";
function open() {
if (!overlay.classList.contains("hidden")) return;
overlay.classList.remove("hidden");
document.body.classList.add("overflow-hidden");
requestAnimationFrame(() => input.focus());
}
function close() {
overlay.classList.add("hidden");
document.body.classList.remove("overflow-hidden");
input.value = "";
list.innerHTML = "";
meta.textContent = "";
items = [];
activeIndex = -1;
lastQuery = "";
}
function escapeHTML(s) {
return String(s == null ? "" : s)
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;")
.replace(/"/g, "&quot;")
.replace(/'/g, "&#39;");
}
function render(results) {
items = results || [];
activeIndex = items.length > 0 ? 0 : -1;
if (items.length === 0) {
list.innerHTML = "";
return;
}
const html = items.map((r, i) => {
const thumb = r.thumb_url
? `<div class="hidden sm:block w-20 h-20 shrink-0 overflow-hidden rounded-2xl bg-[color:var(--ink)]/5">
<img src="${escapeHTML(r.thumb_url)}" alt="" loading="lazy" decoding="async"
class="h-full w-full object-cover">
</div>`
: `<div class="hidden sm:flex w-20 h-20 shrink-0 items-center justify-center rounded-2xl bg-[color:var(--ink)]/5 text-2xl">${escapeHTML(r.channel_emoji || "·")}</div>`;
const postedRing = r.posted ? ` border-theme-${escapeHTML(r.channel_theme)}` : "";
return `
<a href="${escapeHTML(r.article_url)}" target="_blank" rel="noopener noreferrer"
data-idx="${i}"
class="search-result group flex items-start gap-4 rounded-2xl p-3 transition border-2 border-transparent hover:bg-[color:var(--ink)]/5${postedRing}">
${thumb}
<div class="min-w-0 flex-1 space-y-1">
<div class="flex items-center gap-2 text-xs flex-wrap">
<span class="inline-flex items-center rounded-full bg-theme-${escapeHTML(r.channel_theme)} text-white px-2.5 py-0.5 font-semibold uppercase tracking-wider">
<span aria-hidden="true" class="mr-1">${escapeHTML(r.channel_emoji)}</span>${escapeHTML(r.channel_title)}
</span>
<span class="text-[color:var(--ink)]/60 font-semibold">${escapeHTML(r.source)}</span>
<span class="text-[color:var(--ink)]/50">${escapeHTML(r.time_ago)}</span>
</div>
<h3 class="font-display text-base sm:text-lg font-semibold leading-snug">${escapeHTML(r.headline)}</h3>
${r.lede ? `<p class="text-sm text-[color:var(--ink)]/70 line-clamp-2">${escapeHTML(r.lede)}</p>` : ""}
</div>
</a>`;
}).join("");
list.innerHTML = html;
paintActive();
}
function paintActive() {
const nodes = list.querySelectorAll(".search-result");
nodes.forEach((n, i) => {
if (i === activeIndex) {
n.classList.add("bg-[color:var(--ink)]/5", "border-[color:var(--ink)]/15");
n.scrollIntoView({ block: "nearest" });
} else {
n.classList.remove("bg-[color:var(--ink)]/5", "border-[color:var(--ink)]/15");
}
});
}
async function runQuery(q) {
if (q === lastQuery) return;
lastQuery = q;
if (!q) {
render([]);
meta.textContent = "type to search · esc to close";
return;
}
if (inflight) inflight.abort();
const ctrl = new AbortController();
inflight = ctrl;
meta.textContent = "searching…";
try {
const res = await fetch("/api/search?q=" + encodeURIComponent(q), { signal: ctrl.signal });
if (!res.ok) throw new Error("search failed: " + res.status);
const data = await res.json();
if (ctrl !== inflight) return;
render(data.results || []);
const n = (data.results || []).length;
meta.textContent = n === 0 ? "no matches" : `${n} result${n === 1 ? "" : "s"}`;
} catch (err) {
if (err.name === "AbortError") return;
console.error(err);
meta.textContent = "search error";
} finally {
if (ctrl === inflight) inflight = null;
}
}
input.addEventListener("input", () => {
clearTimeout(debounceTimer);
const q = input.value.trim();
debounceTimer = window.setTimeout(() => runQuery(q), 120);
});
input.addEventListener("keydown", (e) => {
if (e.key === "ArrowDown") {
e.preventDefault();
if (items.length === 0) return;
activeIndex = (activeIndex + 1) % items.length;
paintActive();
} else if (e.key === "ArrowUp") {
e.preventDefault();
if (items.length === 0) return;
activeIndex = (activeIndex - 1 + items.length) % items.length;
paintActive();
} else if (e.key === "Enter") {
if (activeIndex >= 0 && items[activeIndex]) {
e.preventDefault();
window.open(items[activeIndex].article_url, "_blank", "noopener");
}
}
});
overlay.addEventListener("click", (e) => {
if (e.target === overlay) close();
});
document.addEventListener("keydown", (e) => {
const isK = e.key === "k" || e.key === "K";
if ((e.metaKey || e.ctrlKey) && isK) {
e.preventDefault();
if (overlay.classList.contains("hidden")) open();
else close();
return;
}
if (e.key === "Escape" && !overlay.classList.contains("hidden")) {
close();
}
if (e.key === "/" && overlay.classList.contains("hidden")) {
const tag = (document.activeElement && document.activeElement.tagName) || "";
if (tag === "INPUT" || tag === "TEXTAREA") return;
e.preventDefault();
open();
}
});
document.querySelectorAll("[data-search-trigger]").forEach((btn) => {
btn.addEventListener("click", (e) => {
e.preventDefault();
open();
});
});
meta.textContent = "type to search · esc to close";
})();

View File

@@ -1,6 +1,6 @@
{{define "card"}}
<a href="{{.Story.ArticleURL}}" target="_blank" rel="noopener noreferrer"
class="group block rounded-3xl bg-[color:var(--card)] border-2 {{if .Story.Posted}}border-theme-{{.Theme}} glow-theme-{{.Theme}}{{else}}border-[color:var(--ink)]/10{{end}} shadow-pete overflow-hidden hover:-translate-y-1 hover:shadow-pete-lg transition">
class="group relative block rounded-3xl bg-[color:var(--card)] border-2 {{if .Story.Posted}}border-theme-{{.Theme}} glow-theme-{{.Theme}}{{else}}border-[color:var(--ink)]/10{{end}} shadow-pete overflow-hidden hover:-translate-y-1 hover:shadow-pete-lg transition">
{{if .Story.ImageURL}}
<div class="aspect-[16/10] w-full overflow-hidden bg-[color:var(--ink)]/5">
<img src="{{thumb .Story.ImageURL}}" alt="" loading="lazy" decoding="async"
@@ -28,5 +28,8 @@
<p class="text-sm text-[color:var(--ink)]/70 line-clamp-3">{{.Story.Lede}}</p>
{{end}}
</div>
{{if .Story.Paywalled}}
<div class="paywall-stamp" aria-label="Paywalled article"></div>
{{end}}
</a>
{{end}}

View File

@@ -39,19 +39,47 @@
<span class="font-display text-3xl font-bold tracking-tight">{{.SiteTitle}}</span>
<span class="hidden sm:inline rounded-full bg-[color:var(--accent)]/20 px-3 py-1 text-xs font-semibold uppercase tracking-wider text-[color:var(--ink)]/70" data-phase-label>day</span>
</a>
<nav class="flex items-center gap-1 sm:gap-2 rounded-full bg-[color:var(--card)] p-1 shadow-pete border-2 border-[color:var(--ink)]/10">
{{$active := .Active}}
{{range .Channels}}
<a href="/{{.Slug}}"
class="rounded-full px-3 py-2 text-sm font-semibold transition
{{if eq $active .Slug}}bg-theme-{{.Theme}} text-white shadow-sm{{else}}hover:bg-[color:var(--ink)]/5{{end}}">
<span aria-hidden="true" class="mr-1">{{.Emoji}}</span><span class="hidden sm:inline">{{.Title}}</span>
</a>
{{end}}
</nav>
<div class="flex items-center gap-2">
<button type="button" data-search-trigger title="Search (⌘K)"
class="flex items-center gap-2 rounded-full bg-[color:var(--card)] px-3 py-2 text-sm shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition">
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
stroke-linecap="round" stroke-linejoin="round" class="h-4 w-4 text-[color:var(--ink)]/60">
<circle cx="11" cy="11" r="7"></circle>
<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
</svg>
<span class="hidden sm:inline text-[color:var(--ink)]/60 font-semibold">Search</span>
<kbd class="hidden sm:inline-flex items-center rounded-md bg-[color:var(--ink)]/5 px-1.5 py-0.5 text-[10px] font-mono font-semibold text-[color:var(--ink)]/60">⌘K</kbd>
</button>
<nav class="flex items-center gap-1 sm:gap-2 rounded-full bg-[color:var(--card)] p-1 shadow-pete border-2 border-[color:var(--ink)]/10">
{{$active := .Active}}
{{range .Channels}}
<a href="/{{.Slug}}"
class="rounded-full px-3 py-2 text-sm font-semibold transition
{{if eq $active .Slug}}bg-theme-{{.Theme}} text-white shadow-sm{{else}}hover:bg-[color:var(--ink)]/5{{end}}">
<span aria-hidden="true" class="mr-1">{{.Emoji}}</span><span class="hidden sm:inline">{{.Title}}</span>
</a>
{{end}}
</nav>
</div>
</div>
</header>
<div id="pete-search" class="hidden fixed inset-0 z-50 bg-[color:var(--ink)]/40 backdrop-blur-sm flex items-start justify-center pt-[10vh] px-4" aria-modal="true" role="dialog">
<div class="w-full max-w-2xl rounded-3xl bg-[color:var(--card)] shadow-pete-lg border-2 border-[color:var(--ink)]/10 overflow-hidden">
<div class="flex items-center gap-3 px-5 py-4 border-b border-[color:var(--ink)]/10">
<kbd class="hidden sm:inline-flex items-center rounded-md bg-[color:var(--ink)]/5 px-1.5 py-0.5 text-[10px] font-mono font-semibold text-[color:var(--ink)]/60">⌘K</kbd>
<input type="search" data-search-input autocomplete="off" spellcheck="false"
placeholder="Search headlines…"
class="flex-1 bg-transparent text-lg outline-none placeholder:text-[color:var(--ink)]/40">
</div>
<div data-search-list class="max-h-[60vh] overflow-y-auto p-3 space-y-1"></div>
<div class="px-5 py-3 border-t border-[color:var(--ink)]/10 flex items-center justify-between text-xs text-[color:var(--ink)]/50">
<span><kbd class="font-mono">↑↓</kbd> navigate · <kbd class="font-mono"></kbd> open · <kbd class="font-mono">esc</kbd> close</span>
<span data-search-meta></span>
</div>
</div>
</div>
<main class="mx-auto max-w-6xl px-4 pb-24">
{{block "main" .}}{{end}}
</main>
@@ -80,5 +108,6 @@
</script>
<script src="/static/js/weather.js" defer></script>
<script src="/static/js/search.js" defer></script>
</body>
</html>{{end}}