Compare commits
45 Commits
ec1f130ed5
...
weather-fx
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0a723418ff | ||
|
|
9b20040b49 | ||
|
|
e91b423b1a | ||
|
|
dbcb459908 | ||
|
|
fceeb12ad5 | ||
|
|
74aa578a2d | ||
|
|
8f9fcc45f3 | ||
|
|
616055a704 | ||
|
|
2ea5f7a6f7 | ||
|
|
ff3a0e87be | ||
|
|
77581ac152 | ||
|
|
35850eaf73 | ||
|
|
8863b75916 | ||
|
|
71f7050f41 | ||
|
|
55aa167151 | ||
|
|
410f8dda0a | ||
|
|
e65ffb1373 | ||
|
|
aaf6e551a0 | ||
|
|
4bdf9a7615 | ||
|
|
d5d0656dba | ||
|
|
cbbedd9894 | ||
|
|
1a43616248 | ||
|
|
95f6e71933 | ||
|
|
7b76f9ed23 | ||
|
|
0b94ce7fe5 | ||
|
|
8aceb259ca | ||
|
|
fc4cab3ad6 | ||
|
|
3e29acaa23 | ||
|
|
78fc3ef811 | ||
|
|
7d469cf8c5 | ||
|
|
59658e7ebb | ||
|
|
b617d403b7 | ||
|
|
a15025089d | ||
|
|
92d700f2bb | ||
|
|
f52f26ff8d | ||
|
|
58493006e1 | ||
|
|
04a3c41bed | ||
|
|
b3002c8e12 | ||
|
|
70e1dfc2b8 | ||
|
|
83b1216541 | ||
|
|
dd99f2c7ab | ||
|
|
9e5ba5aafc | ||
|
|
e428f37c9e | ||
|
|
b8dcaa2aa1 | ||
|
|
71dc6d77ab |
22
README.md
22
README.md
@@ -14,21 +14,29 @@ A Matrix news bot that ingests RSS feeds from curated sources and routes each st
|
||||
- **Paywall detection** — if an article's visible body text is below threshold, Pete swaps in a Wayback Machine snapshot URL for both the lead image and the posted link
|
||||
- **FTS5 search** — full-text search across headlines and ledes
|
||||
- **Image validation** — HEAD-based checks filter tracking pixels, uploads valid images via MXC
|
||||
- **Web UI** — read-only browser interface (Tailwind, Animal-Crossing-ish vibe, day/night palette that follows the visitor's clock) at `/`, `/gaming`, `/tech`, `/politics`. Designed for a `news.parodia.dev`-style deployment
|
||||
- **Video thumbnails** — extracts a frame via `ffmpeg` for video sources; also falls back to `ffmpeg` when Go's stdlib JPEG decoder rejects a source image
|
||||
- **Seasonal weather** — canvas-based ambient overlay (snow, leaves, rain, etc.) driven by a Portugal/Lisbon calendar; visitors can toggle it with the ⭐ star button in the nav
|
||||
- **Web UI** — read-only browser interface (Tailwind, Animal-Crossing-ish vibe, day/night palette that follows the visitor's clock) at `/` plus a section per channel. Designed for a `news.parodia.dev`-style deployment
|
||||
|
||||
## Channels
|
||||
|
||||
| Channel | Purpose |
|
||||
|---|---|
|
||||
| `gaming` | Gaming news, releases, platform announcements |
|
||||
| `tech` | Technology news, product/industry stories |
|
||||
| `politics` | Political, policy, current events |
|
||||
| `gaming` | Gaming news, releases, platform announcements |
|
||||
| `eu` | Portugal and the wider European beat (web-only — does not post to Matrix) |
|
||||
| `music` | Records, scenes, artists |
|
||||
| `anime` | Series, studios, manga |
|
||||
| `foss` | Kernel, distros, free/open source |
|
||||
| `kids` | World news written for younger readers (web-only) |
|
||||
|
||||
## Requirements
|
||||
|
||||
- Go 1.25+
|
||||
- A Matrix account for Pete with access to target rooms
|
||||
- SQLite (bundled via pure Go driver, no CGo)
|
||||
- `ffmpeg` on `PATH` — used for video-frame thumbnails and as a fallback decoder for JPEGs the Go stdlib rejects
|
||||
- No CGo dependencies — E2EE uses pure Go crypto (goolm) via mautrix v0.28
|
||||
|
||||
## Setup
|
||||
@@ -39,7 +47,7 @@ git clone <repo-url> && cd pete
|
||||
go build -tags goolm .
|
||||
|
||||
# Create config from example
|
||||
cp config.example.yaml config.toml
|
||||
cp config.example.toml config.toml
|
||||
# Edit config.toml — fill in Matrix credentials, room IDs, and a direct_route for every source
|
||||
|
||||
# First run: seed current feed items as seen (prevents flood)
|
||||
@@ -54,19 +62,19 @@ cp config.example.yaml config.toml
|
||||
|
||||
## Configuration
|
||||
|
||||
See `config.example.yaml` for the full structure. Key sections:
|
||||
See `config.example.toml` for the full structure. Key sections:
|
||||
|
||||
- `matrix` — homeserver, credentials, channel room IDs, optional admin room
|
||||
- `posting` — rate limiting (min interval, burst cap, daily cap), optional `round_robin` block
|
||||
- `storage` — database path, retention windows
|
||||
- `sources` — RSS feeds with tier, polling interval, and **required** `direct_route` (must match a key in `matrix.channels`)
|
||||
- `sources` — RSS feeds with tier, polling interval, and **required** `direct_route` (must match a key in `matrix.channels`). Optional `language = "en"` drops items whose per-item `<language>` tag doesn't prefix-match — handy for multilingual feeds like Politico Europe
|
||||
- `web` — read-only HTTP UI (enabled toggle, listen address, site title, public base URL)
|
||||
|
||||
Environment variables can be referenced with `${VAR}` syntax in the TOML.
|
||||
|
||||
### Web UI
|
||||
|
||||
Set `web.enabled: true` (default port `:8080`) to expose Pete's classified-story archive over HTTP. Three sections (`/gaming`, `/tech`, `/politics`) plus a landing page at `/`, all pulling from the `stories` table. Adding a section is two lines in `internal/web/server.go` (the `channels` slice) plus a matching theme color in `internal/web/static/css/input.css`.
|
||||
Set `web.enabled: true` (default port `:8080`) to expose Pete's classified-story archive over HTTP. A landing page at `/` plus one section per channel (`/gaming`, `/tech`, `/politics`, `/eu`, `/music`, `/anime`, `/foss`, `/kids`), all pulling from the `stories` table. A `/weather` page demos the seasonal canvas overlay in isolation. Adding a section is two lines in `internal/web/server.go` (the `channels` slice) plus a matching theme color in `internal/web/static/css/input.css`.
|
||||
|
||||
The frontend uses a small Tailwind build:
|
||||
|
||||
@@ -116,7 +124,7 @@ RSS Feed → Poller → GUID/Canonical/Headline Dedup → Article Fetch → Stor
|
||||
|
||||
| Package | Purpose |
|
||||
|---|---|
|
||||
| `internal/config` | YAML config loading with `${ENV}` expansion |
|
||||
| `internal/config` | TOML config loading with `${ENV}` expansion |
|
||||
| `internal/storage` | SQLite with WAL, FTS5, all queries |
|
||||
| `internal/ingestion` | Per-source RSS polling, feed parsing, image validation, paywall detection |
|
||||
| `internal/dedup` | Canonical URL + headline normalization helpers |
|
||||
|
||||
@@ -6,13 +6,18 @@ pickle_key = "${PETE_PICKLE_KEY}"
|
||||
display_name = "Pete"
|
||||
data_dir = "./data"
|
||||
admin_room = "!adminroomid:matrix.example.org"
|
||||
# Matrix user IDs allowed to run in-room commands like !post. Empty disables
|
||||
# commands entirely (channel rooms may be public, so empty != "anyone").
|
||||
admins = ["@you:matrix.example.org"]
|
||||
|
||||
[matrix.channels]
|
||||
tech = "!techroomid:matrix.example.org"
|
||||
politics = "!politicsroomid:matrix.example.org"
|
||||
gaming = "!gamingroomid:matrix.example.org"
|
||||
lego = "!legoroomid:matrix.example.org"
|
||||
|
||||
[posting]
|
||||
enabled = true # master switch for auto-posting news to Matrix; false = web-only (commands still work)
|
||||
min_interval_seconds = 300
|
||||
burst_cap_count = 3
|
||||
burst_cap_window_seconds = 1800
|
||||
@@ -32,8 +37,71 @@ listen_addr = ":8080"
|
||||
site_title = "Pete"
|
||||
base_url = "https://news.parodia.dev"
|
||||
|
||||
# OIDC subjects allowed to view the owner-facing source-health dashboard at
|
||||
# /status (per-feed poll status, failures, content stats). Requires web.auth
|
||||
# below. Empty = /status returns 404 for everyone. Find a user's subject in the
|
||||
# server logs ("auth: user signed in" sub=...) after they sign in once.
|
||||
admin_subs = []
|
||||
|
||||
# Optional OIDC sign-in (Authentik). When enabled, signed-in users get their
|
||||
# preferences (hidden feeds, weather location, toggles) stored server-side keyed
|
||||
# by their OIDC subject and synced across devices. Anonymous visitors keep using
|
||||
# browser localStorage — the site stays public. If the provider is unreachable
|
||||
# at startup, Pete logs a warning and serves anonymously rather than refusing to
|
||||
# boot. Create an OAuth2/OIDC provider + application in Authentik, set the
|
||||
# redirect URI to <base_url>/auth/callback, then fill these in.
|
||||
[web.auth]
|
||||
enabled = false
|
||||
issuer = "https://authentik.parodia.dev/application/o/pete/"
|
||||
client_id = "${PETE_OIDC_CLIENT_ID}"
|
||||
client_secret = "${PETE_OIDC_CLIENT_SECRET}"
|
||||
redirect_url = "https://news.parodia.dev/auth/callback"
|
||||
# HMAC key that signs the session cookie. Generate with: openssl rand -hex 32
|
||||
session_secret = "${PETE_SESSION_SECRET}"
|
||||
|
||||
# Optional Web Push digests. When enabled, signed-in users can opt in (from the
|
||||
# feed-settings panel) to a periodic "N new stories" notification, delivered via
|
||||
# a service worker so it also works when the site is installed as a PWA. Push is
|
||||
# signed-in only, so it needs web.auth above; it does nothing otherwise.
|
||||
# Generate the VAPID keypair once with: pete -genvapid
|
||||
# then paste the two keys here (the private key is a secret — treat it like a
|
||||
# password). subject identifies you to the push service (a mailto: or https: URL).
|
||||
[web.push]
|
||||
enabled = false
|
||||
vapid_public_key = "${PETE_VAPID_PUBLIC_KEY}"
|
||||
vapid_private_key = "${PETE_VAPID_PRIVATE_KEY}"
|
||||
subject = "mailto:admin@parodia.dev"
|
||||
# How often the sender wakes to look for new stories per subscriber (minutes).
|
||||
interval_minutes = 360
|
||||
# Smallest number of new (non-hidden) stories that triggers a digest, so a lone
|
||||
# item doesn't ping everyone.
|
||||
min_stories = 3
|
||||
|
||||
# Server-side neural read-aloud (Piper, https://github.com/rhasspy/piper).
|
||||
# When enabled, the reader's "Listen" button streams real Piper voices instead
|
||||
# of the browser's robotic Web Speech voice. Signed-in only, so it needs
|
||||
# [web.auth] on too. Install the piper binary and one or more voice models
|
||||
# (<id>.onnx + <id>.onnx.json) into voices_dir first.
|
||||
[web.tts]
|
||||
enabled = false
|
||||
piper_bin = "/opt/piper/piper" # path to the piper executable
|
||||
voices_dir = "/opt/piper/voices" # dir holding <id>.onnx (+ .onnx.json) models
|
||||
default = "en_US-amy-medium" # voice id selected until the reader picks another
|
||||
# List the voices to offer, in menu order. Omit the whole [[web.tts.voices]]
|
||||
# list to auto-discover every *.onnx in voices_dir (labelled by filename).
|
||||
[[web.tts.voices]]
|
||||
id = "en_US-amy-medium"
|
||||
label = "Amy (US, female)"
|
||||
[[web.tts.voices]]
|
||||
id = "en_US-ryan-high"
|
||||
label = "Ryan (US, male, HQ)"
|
||||
|
||||
# Every enabled source MUST set direct_route to a key from [matrix.channels] above.
|
||||
# There is no automatic classification — Pete posts each story to its configured channel.
|
||||
# Optional: language = "en" drops feed items whose per-item <language> tag is
|
||||
# set and doesn't match (prefix). Useful for multilingual feeds like Politico
|
||||
# Europe that publish the same story in en / fr / de side-by-side. Items with
|
||||
# no language tag pass through unchanged.
|
||||
[[sources]]
|
||||
name = "The Guardian — World"
|
||||
feed_url = "https://www.theguardian.com/world/rss"
|
||||
@@ -41,6 +109,9 @@ tier = 1
|
||||
poll_interval_minutes = 20
|
||||
direct_route = "politics"
|
||||
enabled = true
|
||||
# Optional: override the User-Agent for this feed only. Leave unset to use
|
||||
# Pete's honest bot UA; set a browser string for feeds whose WAF blocks bots.
|
||||
# user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"
|
||||
|
||||
[[sources]]
|
||||
name = "The Guardian — Politics"
|
||||
@@ -73,3 +144,131 @@ tier = 1
|
||||
poll_interval_minutes = 20
|
||||
direct_route = "gaming"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "Pitchfork"
|
||||
feed_url = "https://pitchfork.com/feed/feed-news/rss"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "music"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "The Quietus"
|
||||
feed_url = "https://thequietus.com/feed"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "music"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "Consequence"
|
||||
feed_url = "https://consequence.net/feed/"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "music"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "Brooklyn Vegan"
|
||||
feed_url = "https://www.brooklynvegan.com/feed/"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "music"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "NME — Music"
|
||||
feed_url = "https://www.nme.com/news/music/feed"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "music"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "BBC Newsround"
|
||||
feed_url = "http://feeds.bbci.co.uk/newsround/rss.xml"
|
||||
tier = 1
|
||||
poll_interval_minutes = 30
|
||||
direct_route = "kids"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "DOGO News"
|
||||
feed_url = "https://www.dogonews.com/articles.rss"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "kids"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "Science News Explores"
|
||||
feed_url = "https://www.snexplores.org/feed"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "kids"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "NASA for Students"
|
||||
feed_url = "https://www.nasa.gov/rss/dyn/educationnews.rss"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "kids"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "Post-Punk.com"
|
||||
feed_url = "https://post-punk.com/feed/"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "music"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "Naked Capitalism"
|
||||
feed_url = "https://www.nakedcapitalism.com/feed"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "finance"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "Wolf Street"
|
||||
feed_url = "https://wolfstreet.com/feed/"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "finance"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "MarketBeat"
|
||||
feed_url = "https://www.marketbeat.com/feed/"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "finance"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "Seeking Alpha"
|
||||
feed_url = "https://seekingalpha.com/feed.xml"
|
||||
tier = 2
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "finance"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "Brickset"
|
||||
feed_url = "https://brickset.com/feed"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "lego"
|
||||
enabled = true
|
||||
|
||||
[[sources]]
|
||||
name = "The Brick Fan"
|
||||
feed_url = "https://www.thebrickfan.com/feed/"
|
||||
tier = 1
|
||||
poll_interval_minutes = 60
|
||||
direct_route = "lego"
|
||||
enabled = true
|
||||
|
||||
9
go.mod
9
go.mod
@@ -5,7 +5,12 @@ go 1.25.0
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.6.0
|
||||
github.com/PuerkitoBio/goquery v1.12.0
|
||||
github.com/SherClockHolmes/webpush-go v1.4.0
|
||||
github.com/coreos/go-oidc/v3 v3.19.0
|
||||
github.com/mmcdole/gofeed v1.3.0
|
||||
go.mau.fi/util v0.9.9
|
||||
golang.org/x/image v0.41.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
maunium.net/go/mautrix v0.28.0
|
||||
modernc.org/sqlite v1.50.1
|
||||
)
|
||||
@@ -14,6 +19,8 @@ require (
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
@@ -30,10 +37,8 @@ require (
|
||||
github.com/tidwall/match v1.2.0 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/sjson v1.2.5 // indirect
|
||||
go.mau.fi/util v0.9.9 // indirect
|
||||
golang.org/x/crypto v0.52.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a // indirect
|
||||
golang.org/x/image v0.41.0 // indirect
|
||||
golang.org/x/net v0.55.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
|
||||
10
go.sum
10
go.sum
@@ -6,13 +6,21 @@ github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7Oputl
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
|
||||
github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
|
||||
github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
|
||||
github.com/SherClockHolmes/webpush-go v1.4.0 h1:ocnzNKWN23T9nvHi6IfyrQjkIc0oJWv1B1pULsf9i3s=
|
||||
github.com/SherClockHolmes/webpush-go v1.4.0/go.mod h1:XSq8pKX11vNV8MJEMwjrlTkxhAj1zKfxmyhdV7Pd6UA=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/coreos/go-oidc/v3 v3.19.0 h1:F/xyOi3x1UnG1U27YVnM1N6bHiL1K2upi6U/0qr8r+I=
|
||||
github.com/coreos/go-oidc/v3 v3.19.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
@@ -96,6 +104,8 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
||||
@@ -22,10 +22,71 @@ type Config struct {
|
||||
|
||||
// WebConfig controls the read-only HTTP interface (news.parodia.dev style).
|
||||
type WebConfig struct {
|
||||
Enabled bool `toml:"enabled"`
|
||||
ListenAddr string `toml:"listen_addr"` // e.g. ":8080" or "127.0.0.1:8080"
|
||||
SiteTitle string `toml:"site_title"` // display name in the header
|
||||
BaseURL string `toml:"base_url"` // public URL (used in metadata only)
|
||||
Enabled bool `toml:"enabled"`
|
||||
ListenAddr string `toml:"listen_addr"` // e.g. ":8080" or "127.0.0.1:8080"
|
||||
SiteTitle string `toml:"site_title"` // display name in the header
|
||||
BaseURL string `toml:"base_url"` // public URL (used in metadata only)
|
||||
Auth AuthConfig `toml:"auth"` // optional OIDC sign-in (Authentik)
|
||||
Push PushConfig `toml:"push"` // optional Web Push digests (VAPID)
|
||||
TTS TTSConfig `toml:"tts"` // optional server-side neural read-aloud (Piper)
|
||||
// AdminSubs is the allowlist of OIDC subjects allowed to view the
|
||||
// owner-facing source-health dashboard at /status. Empty means the page is
|
||||
// inaccessible to everyone (returns 404). Requires auth to be enabled.
|
||||
AdminSubs []string `toml:"admin_subs"`
|
||||
}
|
||||
|
||||
// PushConfig wires the Web Push digest sender. Push is signed-in only (it keys
|
||||
// off the OIDC subject) so it does nothing unless auth is also enabled. Generate
|
||||
// a VAPID keypair once with `pete -genvapid` and paste the two keys here.
|
||||
type PushConfig struct {
|
||||
Enabled bool `toml:"enabled"`
|
||||
VAPIDPublicKey string `toml:"vapid_public_key"`
|
||||
VAPIDPrivateKey string `toml:"vapid_private_key"`
|
||||
// Subject identifies the sender to the push service; a mailto: or https: URL
|
||||
// per RFC 8292. Defaults to mailto:admin@<base_url host> is not attempted —
|
||||
// set it explicitly.
|
||||
Subject string `toml:"subject"`
|
||||
// IntervalMinutes is how often the digest sender wakes to look for new
|
||||
// stories per subscriber. Defaults to 360 (6h).
|
||||
IntervalMinutes int `toml:"interval_minutes"`
|
||||
// MinStories is the smallest number of new stories that triggers a digest
|
||||
// for a subscriber, so they aren't pinged for a single item. Defaults to 3.
|
||||
MinStories int `toml:"min_stories"`
|
||||
}
|
||||
|
||||
// TTSConfig wires server-side neural read-aloud (Piper). When enabled,
|
||||
// signed-in users get the reader's "Listen" button backed by real Piper voices
|
||||
// instead of the browser's robotic Web Speech voice. Read-aloud is a signed-in
|
||||
// perk, so this does nothing unless auth is also enabled.
|
||||
type TTSConfig struct {
|
||||
Enabled bool `toml:"enabled"`
|
||||
PiperBin string `toml:"piper_bin"` // path to the piper executable
|
||||
VoicesDir string `toml:"voices_dir"` // directory holding <id>.onnx (+ .onnx.json) models
|
||||
// Voices lists the voices to offer, in menu order. Each id is a model
|
||||
// filename stem, so id "en_US-ryan-high" maps to <voices_dir>/en_US-ryan-high.onnx.
|
||||
// Leave empty to auto-discover every *.onnx in voices_dir.
|
||||
Voices []VoiceConfig `toml:"voices"`
|
||||
// Default is the voice id selected until the reader picks another. Empty
|
||||
// falls back to the first available voice.
|
||||
Default string `toml:"default"`
|
||||
}
|
||||
|
||||
// VoiceConfig is one selectable Piper voice.
|
||||
type VoiceConfig struct {
|
||||
ID string `toml:"id"` // model filename stem, e.g. "en_US-ryan-high"
|
||||
Label string `toml:"label"` // menu label, e.g. "Ryan — US, male"
|
||||
}
|
||||
|
||||
// AuthConfig wires Pete's web UI to an OIDC provider (our Authentik instance).
|
||||
// When enabled, signed-in users get their preferences stored server-side keyed
|
||||
// by the OIDC subject; anonymous visitors keep using browser localStorage.
|
||||
type AuthConfig struct {
|
||||
Enabled bool `toml:"enabled"`
|
||||
Issuer string `toml:"issuer"` // e.g. https://authentik.parodia.dev/application/o/pete/
|
||||
ClientID string `toml:"client_id"`
|
||||
ClientSecret string `toml:"client_secret"`
|
||||
RedirectURL string `toml:"redirect_url"` // e.g. https://news.parodia.dev/auth/callback
|
||||
SessionSecret string `toml:"session_secret"` // HMAC key for signing the session cookie
|
||||
}
|
||||
|
||||
type MatrixConfig struct {
|
||||
@@ -37,13 +98,23 @@ type MatrixConfig struct {
|
||||
DataDir string `toml:"data_dir"`
|
||||
AdminRoom string `toml:"admin_room"`
|
||||
Channels map[string]string `toml:"channels"`
|
||||
// Admins is the allowlist of Matrix user IDs permitted to run in-room
|
||||
// commands like !post. Empty means commands are disabled — the channel
|
||||
// rooms are public, so an empty list must NOT mean "anyone".
|
||||
Admins []string `toml:"admins"`
|
||||
}
|
||||
|
||||
type PostingConfig struct {
|
||||
MinIntervalSeconds int `toml:"min_interval_seconds"`
|
||||
BurstCapCount int `toml:"burst_cap_count"`
|
||||
BurstCapWindowSeconds int `toml:"burst_cap_window_seconds"`
|
||||
DedupCooldownHours int `toml:"dedup_cooldown_hours"`
|
||||
// Enabled is the master switch for automatic news posting to Matrix.
|
||||
// When false, stories are still ingested, classified, and served to the
|
||||
// web UI, but Pete never auto-posts them to rooms. Command replies
|
||||
// (!post, !petestats) still work. Pointer so an absent key defaults to
|
||||
// true (posting on) rather than Go's zero-value false.
|
||||
Enabled *bool `toml:"enabled"`
|
||||
MinIntervalSeconds int `toml:"min_interval_seconds"`
|
||||
BurstCapCount int `toml:"burst_cap_count"`
|
||||
BurstCapWindowSeconds int `toml:"burst_cap_window_seconds"`
|
||||
DedupCooldownHours int `toml:"dedup_cooldown_hours"`
|
||||
// DailyCapTotal is the hard global cap on posts across ALL channels in a
|
||||
// rolling 24-hour window. 0 disables the cap.
|
||||
DailyCapTotal int `toml:"daily_cap_total"`
|
||||
@@ -70,6 +141,14 @@ type SourceConfig struct {
|
||||
PollIntervalMinutes int `toml:"poll_interval_minutes"`
|
||||
DirectRoute string `toml:"direct_route"`
|
||||
Enabled bool `toml:"enabled"`
|
||||
// Language, when set, drops feed items whose per-item language tag is
|
||||
// present and does not match (prefix). Useful for multilingual feeds
|
||||
// like Politico Europe that publish English + French side-by-side.
|
||||
Language string `toml:"language"`
|
||||
// UserAgent overrides the User-Agent sent when fetching this feed. Empty
|
||||
// uses Pete's honest default bot UA. Set a browser-like string only for
|
||||
// sources whose WAF (e.g. AWS WAF on The Portugal News) blocks bot UAs.
|
||||
UserAgent string `toml:"user_agent"`
|
||||
}
|
||||
|
||||
func Load(path string) (*Config, error) {
|
||||
@@ -113,6 +192,12 @@ func (c *Config) validate() error {
|
||||
if c.Matrix.Password == "" {
|
||||
return fmt.Errorf("matrix.password is required")
|
||||
}
|
||||
// pickle_key encrypts the E2EE crypto store (olm/megolm + cross-signing
|
||||
// keys) at rest. Never fall back to a hardcoded default: a key baked into
|
||||
// the source is no protection if crypto.db ever leaks.
|
||||
if len(c.Matrix.PickleKey) < 16 {
|
||||
return fmt.Errorf("matrix.pickle_key must be set to a strong secret (>=16 chars); it encrypts the E2EE crypto store at rest")
|
||||
}
|
||||
if len(c.Matrix.Channels) == 0 {
|
||||
return fmt.Errorf("matrix.channels must have at least one entry")
|
||||
}
|
||||
@@ -120,6 +205,29 @@ func (c *Config) validate() error {
|
||||
return fmt.Errorf("storage.db_path is required")
|
||||
}
|
||||
|
||||
if c.Web.Auth.Enabled {
|
||||
a := c.Web.Auth
|
||||
if a.Issuer == "" || a.ClientID == "" || a.ClientSecret == "" || a.RedirectURL == "" {
|
||||
return fmt.Errorf("web.auth requires issuer, client_id, client_secret, and redirect_url when enabled")
|
||||
}
|
||||
if len(a.SessionSecret) < 16 {
|
||||
return fmt.Errorf("web.auth.session_secret must be at least 16 characters")
|
||||
}
|
||||
}
|
||||
|
||||
if c.Web.Push.Enabled {
|
||||
if !c.Web.Auth.Enabled {
|
||||
return fmt.Errorf("web.push requires web.auth to be enabled (push is signed-in only)")
|
||||
}
|
||||
p := c.Web.Push
|
||||
if p.VAPIDPublicKey == "" || p.VAPIDPrivateKey == "" {
|
||||
return fmt.Errorf("web.push requires vapid_public_key and vapid_private_key (generate with: pete -genvapid)")
|
||||
}
|
||||
if p.Subject == "" {
|
||||
return fmt.Errorf("web.push.subject is required (a mailto: or https: URL identifying the sender)")
|
||||
}
|
||||
}
|
||||
|
||||
for i, s := range c.Sources {
|
||||
if s.Name == "" {
|
||||
return fmt.Errorf("sources[%d].name is required", i)
|
||||
@@ -157,8 +265,9 @@ func (c *Config) applyDefaults() {
|
||||
if c.Matrix.DisplayName == "" {
|
||||
c.Matrix.DisplayName = "Pete"
|
||||
}
|
||||
if c.Matrix.PickleKey == "" {
|
||||
c.Matrix.PickleKey = "pete_pickle_key"
|
||||
if c.Posting.Enabled == nil {
|
||||
on := true
|
||||
c.Posting.Enabled = &on
|
||||
}
|
||||
if c.Posting.MinIntervalSeconds == 0 {
|
||||
c.Posting.MinIntervalSeconds = 300
|
||||
@@ -184,6 +293,12 @@ func (c *Config) applyDefaults() {
|
||||
if c.Web.SiteTitle == "" {
|
||||
c.Web.SiteTitle = "Pete"
|
||||
}
|
||||
if c.Web.Push.IntervalMinutes == 0 {
|
||||
c.Web.Push.IntervalMinutes = 360
|
||||
}
|
||||
if c.Web.Push.MinStories == 0 {
|
||||
c.Web.Push.MinStories = 3
|
||||
}
|
||||
for i := range c.Sources {
|
||||
if c.Sources[i].PollIntervalMinutes == 0 {
|
||||
c.Sources[i].PollIntervalMinutes = 20
|
||||
|
||||
@@ -12,6 +12,7 @@ const validMatrix = `
|
||||
homeserver = "https://matrix.example.org"
|
||||
user_id = "@pete:example.org"
|
||||
password = "testpass"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!tech:example.org"
|
||||
politics = "!politics:example.org"
|
||||
@@ -57,6 +58,7 @@ func TestEnvVarExpansion(t *testing.T) {
|
||||
homeserver = "https://matrix.example.org"
|
||||
user_id = "@pete:example.org"
|
||||
password = "${TEST_PETE_PASS}"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!tech:example.org"
|
||||
|
||||
@@ -114,6 +116,7 @@ func TestValidationErrors(t *testing.T) {
|
||||
[matrix]
|
||||
user_id = "@p:h"
|
||||
password = "pw"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!t:e"
|
||||
[storage]
|
||||
@@ -123,6 +126,7 @@ db_path = "/tmp/t.db"
|
||||
[matrix]
|
||||
homeserver = "https://h"
|
||||
password = "pw"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!t:e"
|
||||
[storage]
|
||||
@@ -142,6 +146,7 @@ db_path = "/tmp/t.db"
|
||||
homeserver = "https://h"
|
||||
user_id = "@p:h"
|
||||
password = "pw"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[storage]
|
||||
db_path = "/tmp/t.db"
|
||||
`},
|
||||
@@ -150,6 +155,7 @@ db_path = "/tmp/t.db"
|
||||
homeserver = "https://h"
|
||||
user_id = "@p:h"
|
||||
password = "pw"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!t:e"
|
||||
[storage]
|
||||
@@ -159,6 +165,7 @@ tech = "!t:e"
|
||||
homeserver = "https://h"
|
||||
user_id = "@p:h"
|
||||
password = "pw"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!t:e"
|
||||
[storage]
|
||||
@@ -176,6 +183,7 @@ enabled = true
|
||||
homeserver = "https://h"
|
||||
user_id = "@p:h"
|
||||
password = "pw"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!t:e"
|
||||
[storage]
|
||||
@@ -192,6 +200,7 @@ enabled = true
|
||||
homeserver = "https://h"
|
||||
user_id = "@p:h"
|
||||
password = "pw"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!t:e"
|
||||
[storage]
|
||||
@@ -202,6 +211,43 @@ feed_url = "https://e.com/rss"
|
||||
tier = 1
|
||||
poll_interval_minutes = 20
|
||||
enabled = true
|
||||
`},
|
||||
{"push enabled without auth", `
|
||||
[matrix]
|
||||
homeserver = "https://h"
|
||||
user_id = "@p:h"
|
||||
password = "pw"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!t:e"
|
||||
[storage]
|
||||
db_path = "/tmp/t.db"
|
||||
[web.push]
|
||||
enabled = true
|
||||
vapid_public_key = "pub"
|
||||
vapid_private_key = "priv"
|
||||
subject = "mailto:a@b.c"
|
||||
`},
|
||||
{"push enabled missing keys", `
|
||||
[matrix]
|
||||
homeserver = "https://h"
|
||||
user_id = "@p:h"
|
||||
password = "pw"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!t:e"
|
||||
[storage]
|
||||
db_path = "/tmp/t.db"
|
||||
[web.auth]
|
||||
enabled = true
|
||||
issuer = "https://i"
|
||||
client_id = "id"
|
||||
client_secret = "secret"
|
||||
redirect_url = "https://r/cb"
|
||||
session_secret = "session_secret_16chars_long"
|
||||
[web.push]
|
||||
enabled = true
|
||||
subject = "mailto:a@b.c"
|
||||
`},
|
||||
}
|
||||
|
||||
@@ -225,6 +271,7 @@ func TestDisabledSourceSkipsDirectRouteCheck(t *testing.T) {
|
||||
homeserver = "https://h"
|
||||
user_id = "@p:h"
|
||||
password = "pw"
|
||||
pickle_key = "test_pickle_key_1234"
|
||||
[matrix.channels]
|
||||
tech = "!t:e"
|
||||
[storage]
|
||||
|
||||
@@ -21,7 +21,7 @@ var trackingParams = map[string]struct{}{
|
||||
"share": {},
|
||||
"shared": {},
|
||||
"cmpid": {},
|
||||
"CMP": {},
|
||||
"cmp": {},
|
||||
"icid": {},
|
||||
"ito": {},
|
||||
"yclid": {},
|
||||
|
||||
@@ -4,13 +4,23 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
)
|
||||
|
||||
// articleBodyMax bounds how much of an article response goquery will
|
||||
// ever buffer. ~5 MiB comfortably fits any real news article while
|
||||
// preventing a hostile origin from OOMing the process by streaming an
|
||||
// endless body within the request timeout window.
|
||||
const articleBodyMax = 5 << 20
|
||||
|
||||
// resolveURL turns a possibly-relative URL into an absolute one using
|
||||
// the base URL. Returns the raw input on parse failure.
|
||||
func resolveURL(base, ref string) string {
|
||||
@@ -55,11 +65,17 @@ const googlebotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.
|
||||
// ArticleMeta is what we can learn from fetching an article page directly.
|
||||
type ArticleMeta struct {
|
||||
ImageURL string // og:image or twitter:image, absolute URL
|
||||
BodyText string // extracted visible article body text (capped at MaxBodyChars)
|
||||
BodyChars int // length of extracted visible body text
|
||||
Fetched bool // true if we got an HTTP 200 with HTML
|
||||
FetchError bool // true if the fetch failed at the network/HTTP layer
|
||||
Paywalled bool // true if the page explicitly declares gated access
|
||||
Status int // last HTTP status seen (0 on transport error)
|
||||
// SubscriberOnly is true when the source publishes the article in a
|
||||
// form that genuinely has no public version — no archive snapshot, no
|
||||
// bypass UA, nothing. Callers should drop these from view entirely
|
||||
// rather than stamp them as paywalled.
|
||||
SubscriberOnly bool
|
||||
Status int // last HTTP status seen (0 on transport error)
|
||||
}
|
||||
|
||||
// Gated reports whether the response carries a strong gating signal: an
|
||||
@@ -73,7 +89,7 @@ func (m ArticleMeta) Gated() bool {
|
||||
return m.Status == http.StatusPaymentRequired || m.Status == http.StatusForbidden
|
||||
}
|
||||
|
||||
var articleClient = &http.Client{Timeout: 12 * time.Second}
|
||||
var articleClient = safehttp.NewClient(12 * time.Second)
|
||||
|
||||
// FetchArticleMeta fetches an article URL with the default UA. If the result
|
||||
// looks gated (explicit paywall signal, HTTP 402/403, or body too short) it
|
||||
@@ -98,6 +114,9 @@ func shouldRetryAsBot(m ArticleMeta) bool {
|
||||
if m.FetchError {
|
||||
return false // transport failure won't be fixed by a different UA
|
||||
}
|
||||
if m.SubscriberOnly {
|
||||
return false // genuinely subscriber-only; no UA changes that
|
||||
}
|
||||
if m.Gated() {
|
||||
return true
|
||||
}
|
||||
@@ -152,26 +171,94 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
|
||||
|
||||
meta := ArticleMeta{Status: resp.StatusCode}
|
||||
// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
|
||||
// We still try to parse the body for whatever signals it contains.
|
||||
// But many sites 403 scrapers via Cloudflare while serving humans fine —
|
||||
// peek at the body and treat a Cloudflare challenge as a transport failure
|
||||
// rather than a paywall, so we don't stamp readable articles.
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusServiceUnavailable {
|
||||
if isCloudflareChallenge(resp) {
|
||||
return ArticleMeta{FetchError: true}
|
||||
}
|
||||
}
|
||||
if resp.StatusCode == http.StatusPaymentRequired {
|
||||
meta.Paywalled = true
|
||||
}
|
||||
return meta
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
doc, err := goquery.NewDocumentFromReader(safehttp.LimitedBody(resp.Body, articleBodyMax))
|
||||
if err != nil {
|
||||
return meta
|
||||
}
|
||||
|
||||
meta.Fetched = true
|
||||
meta.ImageURL = extractOGImage(doc, articleURL)
|
||||
meta.BodyChars = extractBodyChars(doc)
|
||||
// Paywall detection reads <script type="application/ld+json"> and the LWN
|
||||
// marker, so run it before we strip non-content nodes below.
|
||||
meta.Paywalled = detectPaywall(doc)
|
||||
meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
|
||||
// Strip <script>/<style>/etc. before pulling body text — goquery's .Text()
|
||||
// includes the source of inline scripts (e.g. Datawrapper embed resizers),
|
||||
// which would otherwise leak verbatim into reader mode.
|
||||
stripNonContent(doc)
|
||||
// One body extraction serves both purposes: the text feeds reader mode and
|
||||
// its length is the paywall/thin-body signal.
|
||||
meta.BodyText = extractBodyText(doc)
|
||||
meta.BodyChars = len(meta.BodyText)
|
||||
return meta
|
||||
}
|
||||
|
||||
// detectSubscriberOnly returns true for sources known to publish articles
|
||||
// with no public version reachable by any workaround we have (archive
|
||||
// snapshots, bypass UAs, etc). Currently only LWN's subscriber-only
|
||||
// articles, which display a fixed marker in the article body.
|
||||
func detectSubscriberOnly(articleURL string, doc *goquery.Document) bool {
|
||||
u, err := url.Parse(articleURL)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
if host != "lwn.net" && !strings.HasSuffix(host, ".lwn.net") {
|
||||
return false
|
||||
}
|
||||
body := strings.ToLower(doc.Find("body").Text())
|
||||
return strings.Contains(body, "available only to lwn subscribers")
|
||||
}
|
||||
|
||||
// isCloudflareChallenge reports whether a non-200 response is Cloudflare's
|
||||
// bot-block or JS-challenge page rather than a real publisher gate. The
|
||||
// signatures are stable: cf-* response headers, the cf-error-details body
|
||||
// markup, and a handful of recognizable titles. We read at most 8 KiB so a
|
||||
// huge body can't stall the poller.
|
||||
func isCloudflareChallenge(resp *http.Response) bool {
|
||||
if resp == nil {
|
||||
return false
|
||||
}
|
||||
if resp.Header.Get("Server") == "cloudflare" || resp.Header.Get("CF-RAY") != "" || resp.Header.Get("cf-mitigated") != "" {
|
||||
// Header alone isn't enough — plenty of real publishers sit behind
|
||||
// Cloudflare. Confirm with a body marker below.
|
||||
}
|
||||
buf, err := io.ReadAll(io.LimitReader(resp.Body, 8192))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s := strings.ToLower(string(buf))
|
||||
markers := []string{
|
||||
"/cdn-cgi/styles/cf",
|
||||
"cf-error-details",
|
||||
"attention required! | cloudflare",
|
||||
"<title>just a moment",
|
||||
"sorry, you have been blocked",
|
||||
"cloudflare ray id",
|
||||
}
|
||||
for _, m := range markers {
|
||||
if strings.Contains(s, m) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// detectPaywall checks the page for explicit gating signals that publishers
|
||||
// expose for Google News and crawlers. We treat the article as paywalled if:
|
||||
// - <meta name="article:content_tier" content="metered|locked"> (Conde Nast,
|
||||
@@ -310,18 +397,71 @@ func FetchArticleBody(articleURL string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
doc, err := goquery.NewDocumentFromReader(safehttp.LimitedBody(resp.Body, articleBodyMax))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
stripNonContent(doc)
|
||||
return extractBodyText(doc)
|
||||
}
|
||||
|
||||
// stripNonContent removes nodes whose text content is never article prose but
|
||||
// which goquery's .Text() would otherwise concatenate into the extracted body:
|
||||
// inline scripts (Datawrapper/embed resizers, analytics), CSS, and the fallback
|
||||
// markup inside <noscript>/<template>. Must run after any logic that inspects
|
||||
// these nodes (e.g. JSON-LD paywall detection).
|
||||
func stripNonContent(doc *goquery.Document) {
|
||||
doc.Find("script, style, noscript, template").Remove()
|
||||
}
|
||||
|
||||
func extractBodyText(doc *goquery.Document) string {
|
||||
sel := doc.Find("article p, main p")
|
||||
if sel.Length() == 0 {
|
||||
sel = doc.Find("p")
|
||||
out := joinParagraphText(doc.Find("article p, main p"))
|
||||
if len(out) < PaywallBodyThreshold {
|
||||
if alt := joinParagraphText(doc.Find("p")); len(alt) > len(out) {
|
||||
out = alt
|
||||
}
|
||||
}
|
||||
// Sites like Phoronix use <br>-separated text inside <article> rather
|
||||
// than <p> tags, and sites like Anime News Network skip semantic
|
||||
// article/main tags entirely and put the body in a custom div. Fall
|
||||
// back to the raw text of the longest known container.
|
||||
if len(out) < PaywallBodyThreshold {
|
||||
if alt := bestContainerText(doc); len(alt) > len(out) {
|
||||
out = alt
|
||||
}
|
||||
}
|
||||
return truncateUTF8(out, MaxBodyChars)
|
||||
}
|
||||
|
||||
// bodyContainerSelectors is the list of selectors we try when <article>/<main>
|
||||
// aren't present or are too thin. Ordered loosely from semantic to generic;
|
||||
// we take the longest match across all of them.
|
||||
var bodyContainerSelectors = []string{
|
||||
"article",
|
||||
"main",
|
||||
`[itemprop="articleBody"]`,
|
||||
`[role="main"]`,
|
||||
".article-body",
|
||||
".article-content",
|
||||
".entry-content",
|
||||
".post-content",
|
||||
".story-body",
|
||||
".KonaBody", // Anime News Network
|
||||
"#maincontent",
|
||||
"#content",
|
||||
}
|
||||
|
||||
func bestContainerText(doc *goquery.Document) string {
|
||||
var best string
|
||||
for _, sel := range bodyContainerSelectors {
|
||||
if t := containerText(doc.Find(sel).First()); len(t) > len(best) {
|
||||
best = t
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func joinParagraphText(sel *goquery.Selection) string {
|
||||
var b strings.Builder
|
||||
sel.Each(func(_ int, s *goquery.Selection) {
|
||||
t := strings.TrimSpace(s.Text())
|
||||
@@ -332,15 +472,17 @@ func extractBodyText(doc *goquery.Document) string {
|
||||
b.WriteString("\n\n")
|
||||
}
|
||||
b.WriteString(t)
|
||||
if b.Len() >= MaxBodyChars {
|
||||
return
|
||||
}
|
||||
})
|
||||
out := strings.TrimSpace(b.String())
|
||||
if len(out) > MaxBodyChars {
|
||||
out = out[:MaxBodyChars]
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
func containerText(sel *goquery.Selection) string {
|
||||
if sel.Length() == 0 {
|
||||
return ""
|
||||
}
|
||||
return out
|
||||
// Collapse whitespace runs so <br>-laden markup doesn't produce a wall
|
||||
// of blank lines, but preserve enough structure to look like prose.
|
||||
return strings.Join(strings.Fields(sel.Text()), " ")
|
||||
}
|
||||
|
||||
func extractOGImage(doc *goquery.Document, base string) string {
|
||||
@@ -356,26 +498,99 @@ func extractOGImage(doc *goquery.Document, base string) string {
|
||||
return resolveURL(base, strings.TrimSpace(v))
|
||||
}
|
||||
}
|
||||
// Sites like Hardcore Gaming 101 publish no og:image — fall back to the
|
||||
// WordPress featured image (.wp-post-image), then the first reasonably-
|
||||
// sized <img> inside <article>/<main>.
|
||||
if v, ok := doc.Find(`img.wp-post-image`).First().Attr("src"); ok {
|
||||
if url := pickImageSrc(doc.Find(`img.wp-post-image`).First(), v); url != "" {
|
||||
return resolveURL(base, url)
|
||||
}
|
||||
}
|
||||
var found string
|
||||
doc.Find(`article img, main img`).EachWithBreak(func(_ int, s *goquery.Selection) bool {
|
||||
raw, _ := s.Attr("src")
|
||||
url := pickImageSrc(s, raw)
|
||||
if url == "" || !looksLikeContentImage(s, url) {
|
||||
return true
|
||||
}
|
||||
found = url
|
||||
return false
|
||||
})
|
||||
if found != "" {
|
||||
return resolveURL(base, found)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractBodyChars concatenates the text of <p> tags inside <article> or
|
||||
// <main>, falling back to all <p> tags, and returns the trimmed length.
|
||||
func extractBodyChars(doc *goquery.Document) int {
|
||||
sel := doc.Find("article p, main p")
|
||||
if sel.Length() == 0 {
|
||||
sel = doc.Find("p")
|
||||
// pickImageSrc returns the most useful image URL from an <img> selection,
|
||||
// preferring data-src/data-lazy-src (lazy-load) over src when src is a
|
||||
// placeholder (empty or data: URI).
|
||||
func pickImageSrc(s *goquery.Selection, src string) string {
|
||||
src = strings.TrimSpace(src)
|
||||
if !strings.HasPrefix(src, "data:") && src != "" {
|
||||
return src
|
||||
}
|
||||
var b strings.Builder
|
||||
sel.Each(func(_ int, s *goquery.Selection) {
|
||||
t := strings.TrimSpace(s.Text())
|
||||
if t == "" {
|
||||
return
|
||||
for _, attr := range []string{"data-src", "data-lazy-src", "data-original"} {
|
||||
if v, ok := s.Attr(attr); ok {
|
||||
v = strings.TrimSpace(v)
|
||||
if v != "" && !strings.HasPrefix(v, "data:") {
|
||||
return v
|
||||
}
|
||||
}
|
||||
if b.Len() > 0 {
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
b.WriteString(t)
|
||||
})
|
||||
return len(strings.TrimSpace(b.String()))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// looksLikeContentImage filters out obvious non-article images: gravatars,
|
||||
// social-share icons, tracking pixels. Anything with declared dimensions
|
||||
// under 100px on either side is treated as chrome.
|
||||
func looksLikeContentImage(s *goquery.Selection, url string) bool {
|
||||
low := strings.ToLower(url)
|
||||
for _, bad := range []string{"gravatar.com", "/avatar/", "/icons/", "share-", "pixel.gif", "sprite"} {
|
||||
if strings.Contains(low, bad) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if w := atoiAttr(s, "width"); w > 0 && w < 100 {
|
||||
return false
|
||||
}
|
||||
if h := atoiAttr(s, "height"); h > 0 && h < 100 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func atoiAttr(s *goquery.Selection, name string) int {
|
||||
v, ok := s.Attr(name)
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, c := range strings.TrimSpace(v) {
|
||||
if c < '0' || c > '9' {
|
||||
break
|
||||
}
|
||||
n = n*10 + int(c-'0')
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// extractBodyChars returns the visible body length used to gauge whether a
|
||||
// page looks like a real article. It tries <p> tags inside <article>/<main>,
|
||||
// then all <p> tags, then the raw text inside <article>/<main> — that last
|
||||
// fallback catches sites like Phoronix that separate paragraphs with <br>
|
||||
// instead of <p>.
|
||||
func extractBodyChars(doc *goquery.Document) int {
|
||||
best := len(joinParagraphText(doc.Find("article p, main p")))
|
||||
if best < PaywallBodyThreshold {
|
||||
if n := len(joinParagraphText(doc.Find("p"))); n > best {
|
||||
best = n
|
||||
}
|
||||
}
|
||||
if best < PaywallBodyThreshold {
|
||||
if n := len(bestContainerText(doc)); n > best {
|
||||
best = n
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
)
|
||||
|
||||
// NormalizeImageURL rewrites known CDN thumbnail URLs to a higher-resolution
|
||||
@@ -36,15 +38,10 @@ func NormalizeImageURL(raw string) string {
|
||||
return u.String()
|
||||
}
|
||||
|
||||
var imageClient = &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 3 {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
// imageClient validates feed-supplied image URLs. It routes through safehttp
|
||||
// so a hostile feed can't steer the HEAD probe at loopback / RFC1918 / cloud
|
||||
// metadata IPs — the dial-time guard also re-checks every redirect target.
|
||||
var imageClient = safehttp.NewClient(10 * time.Second)
|
||||
|
||||
// ValidateImageURL checks that an image URL returns a valid image response.
|
||||
// Returns false (with no error) on any failure — story posts without image.
|
||||
|
||||
@@ -5,8 +5,18 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// httptest servers bind to 127.0.0.1; allow the safe-dial check to
|
||||
// accept them in tests only.
|
||||
safehttp.AllowPrivate = true
|
||||
}
|
||||
|
||||
func TestFetchOGImage(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -70,3 +80,91 @@ func TestFirstImgSrc(t *testing.T) {
|
||||
t.Errorf("empty input should yield empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractBodyCharsANNLayout regresses a false-positive paywall flag on
|
||||
// Anime News Network: their layout has no <article>/<main> tags and most of
|
||||
// the prose lives outside <p> tags, inside <div class="KonaBody"> containers.
|
||||
// Before the fix, body-length came out below PaywallBodyThreshold and the
|
||||
// poller flagged the story as gated.
|
||||
func TestExtractBodyCharsANNLayout(t *testing.T) {
|
||||
body := `<html><body>
|
||||
<div id="maincontent" class="maincontent news article">
|
||||
<h1>Yen Press Licenses Wistoria</h1>
|
||||
<div id="content-zone"><div class="news"><div class="text-zone"><div class="KonaBody">
|
||||
<div class="meat"><p>Yen Press announced on Friday during its panel that it has licensed seven manga and two novel series for release in December 2026 and winter 2027.</p>
|
||||
Snowmelt and Agapanthus is a manga series being released in the coming season; the publisher described it as a romance story with supernatural undertones, drawn in a delicate watercolor style that should appeal to fans of slice-of-life fantasy. Witch and Cat follows a young magic apprentice and her talking familiar through a series of low-stakes adventures across a sleepy countryside town, leaning into cozy atmosphere over plot. Sinful Is the Angel Who Loves is positioned for an older audience and runs in the publisher's mature imprint. Each release will be available in print and digital across English-language markets, with the first volumes scheduled for the December 2026 wave.
|
||||
</div>
|
||||
</div></div></div></div>
|
||||
</div></body></html>`
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if n := extractBodyChars(doc); n < PaywallBodyThreshold {
|
||||
t.Fatalf("extractBodyChars = %d, want >= %d (ANN-style body was incorrectly treated as paywalled)", n, PaywallBodyThreshold)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractBodyTextStripsInlineScript guards against inline embed scripts
|
||||
// (Datawrapper iframe resizers, analytics) leaking into reader mode. goquery's
|
||||
// .Text() concatenates the source of <script> nodes, so extraction must strip
|
||||
// them first. The script here sits between two prose paragraphs, mirroring the
|
||||
// Politico/Datawrapper layout that surfaced the bug.
|
||||
func TestExtractBodyTextStripsInlineScript(t *testing.T) {
|
||||
body := `<html><body><article>
|
||||
<p>While he has an overall net trust rating of +11 percent nationally, this masks a large regional gap between the north and south of the country.</p>
|
||||
<figure><script type="text/javascript">!function(){"use strict";window.addEventListener("message",(function(a){if(void 0!==a.data["datawrapper-height"]){var e=document.querySelectorAll("iframe");for(var t in a.data["datawrapper-height"])for(var r=0;r<e.length;r++);}}))}();</script></figure>
|
||||
<p>Burnham is on the cusp of becoming the U.K.'s seventh prime minister in 10 years and will enter Number 10 later this month after being elected unopposed.</p>
|
||||
</article></body></html>`
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
stripNonContent(doc)
|
||||
out := extractBodyText(doc)
|
||||
if strings.Contains(out, "datawrapper-height") || strings.Contains(out, "addEventListener") {
|
||||
t.Fatalf("inline script leaked into body text:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "net trust rating") || !strings.Contains(out, "seventh prime minister") {
|
||||
t.Fatalf("prose paragraphs missing from body text:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectSubscriberOnly(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
url string
|
||||
body string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "lwn subscriber-only marker",
|
||||
url: "https://lwn.net/Articles/999999/",
|
||||
body: `<html><body><p>The page you have requested is available only to LWN subscribers. Please log in.</p></body></html>`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "lwn free article",
|
||||
url: "https://lwn.net/Articles/123456/",
|
||||
body: `<html><body><article><p>Full free article body here.</p></article></body></html>`,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "non-lwn host with same marker text is ignored",
|
||||
url: "https://example.com/x",
|
||||
body: `<html><body>available only to lwn subscribers</body></html>`,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(tc.body))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if got := detectSubscriberOnly(tc.url, doc); got != tc.want {
|
||||
t.Fatalf("detectSubscriberOnly = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package ingestion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"html"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -9,42 +10,84 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
ext "github.com/mmcdole/gofeed/extensions"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
)
|
||||
|
||||
const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"
|
||||
|
||||
var feedClient = &http.Client{Timeout: 30 * time.Second}
|
||||
// maxFeedBytes caps how much of a feed body we'll buffer into the parser, so a
|
||||
// hostile feed streaming an endless body within the request timeout can't OOM
|
||||
// the process. Generous headroom — real RSS/Atom feeds are well under this.
|
||||
const maxFeedBytes = 16 << 20
|
||||
|
||||
var feedClient = safehttp.NewClient(30 * time.Second)
|
||||
|
||||
// FeedItem represents a parsed RSS item ready for routing.
|
||||
type FeedItem struct {
|
||||
GUID string
|
||||
Headline string
|
||||
Lede string
|
||||
Content string // full article text from content:encoded, block structure preserved; "" when the feed ships only a snippet
|
||||
ImageURL string
|
||||
ArticleURL string
|
||||
Source string
|
||||
DirectRoute string
|
||||
Language string // per-item <language> (or dc:language) when present; "" otherwise
|
||||
PublishedAt int64 // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable
|
||||
}
|
||||
|
||||
var (
|
||||
htmlTagRe = regexp.MustCompile(`<[^>]*>`)
|
||||
wsRe = regexp.MustCompile(`\s+`)
|
||||
imgSrcRe = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
|
||||
|
||||
// nonContentElemRe strips whole <script>/<style>/<noscript> elements —
|
||||
// tags AND their text — before the generic tag strip runs. htmlTagRe alone
|
||||
// removes only the tags, which would leave inline JavaScript (e.g. a
|
||||
// Datawrapper embed resizer) as literal text in reader mode.
|
||||
nonContentElemRe = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script\s*>|<style\b[^>]*>.*?</style\s*>|<noscript\b[^>]*>.*?</noscript\s*>`)
|
||||
|
||||
// Used by extractContentText to preserve paragraph structure when turning
|
||||
// content:encoded HTML into plain text.
|
||||
blockCloseRe = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|blockquote|article|section|figure|figcaption|ul|ol|table|tr|pre)>`)
|
||||
brRe = regexp.MustCompile(`(?i)<br\s*/?>`)
|
||||
hspaceRe = regexp.MustCompile(`[ \t\f\r]+`)
|
||||
manyNewlineRe = regexp.MustCompile(`\n{3,}`)
|
||||
)
|
||||
|
||||
// maxContentChars bounds the stored article text. Generous enough for any real
|
||||
// article while keeping row size and the reader payload sane.
|
||||
const maxContentChars = 20000
|
||||
|
||||
// FetchFeed fetches and parses an RSS/Atom feed, returning items.
|
||||
func FetchFeed(feedURL string) ([]FeedItem, error) {
|
||||
// FetchFeed fetches and parses an RSS/Atom feed, returning items. ua overrides
|
||||
// the User-Agent header; pass "" to use Pete's honest default bot UA.
|
||||
func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
fp := gofeed.NewParser()
|
||||
fp.Client = feedClient
|
||||
fp.UserAgent = userAgent
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ua == "" {
|
||||
ua = userAgent
|
||||
}
|
||||
req.Header.Set("User-Agent", ua)
|
||||
resp, err := feedClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("feed fetch %s: status %d", feedURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
feed, err := fp.ParseURLWithContext(feedURL, ctx)
|
||||
feed, err := gofeed.NewParser().Parse(safehttp.LimitedBody(resp.Body, maxFeedBytes))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -52,11 +95,14 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
|
||||
items := make([]FeedItem, 0, len(feed.Items))
|
||||
for _, item := range feed.Items {
|
||||
fi := FeedItem{
|
||||
GUID: itemGUID(item),
|
||||
Headline: strings.TrimSpace(item.Title),
|
||||
Lede: extractLede(item.Description),
|
||||
ImageURL: NormalizeImageURL(extractImageURL(item)),
|
||||
ArticleURL: strings.TrimSpace(item.Link),
|
||||
GUID: itemGUID(item),
|
||||
Headline: strings.TrimSpace(item.Title),
|
||||
Lede: extractLede(item.Description),
|
||||
Content: extractContentText(item.Content),
|
||||
ImageURL: NormalizeImageURL(extractImageURL(item)),
|
||||
ArticleURL: strings.TrimSpace(item.Link),
|
||||
Language: itemLanguage(item),
|
||||
PublishedAt: itemPublished(item),
|
||||
}
|
||||
if fi.GUID == "" || fi.ArticleURL == "" {
|
||||
continue
|
||||
@@ -68,6 +114,22 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// itemLanguage returns the per-item language tag if the feed provides one.
|
||||
// Politico Europe puts a raw <language> inside each <item>; gofeed parks
|
||||
// unknown item-level elements in item.Custom. We also check the standard
|
||||
// dc:language extension for feeds that use Dublin Core.
|
||||
func itemLanguage(item *gofeed.Item) string {
|
||||
if item.Custom != nil {
|
||||
if v, ok := item.Custom["language"]; ok {
|
||||
return strings.ToLower(strings.TrimSpace(v))
|
||||
}
|
||||
}
|
||||
if item.DublinCoreExt != nil && len(item.DublinCoreExt.Language) > 0 {
|
||||
return strings.ToLower(strings.TrimSpace(item.DublinCoreExt.Language[0]))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func itemGUID(item *gofeed.Item) string {
|
||||
if item.GUID != "" {
|
||||
return item.GUID
|
||||
@@ -75,16 +137,79 @@ func itemGUID(item *gofeed.Item) string {
|
||||
return item.Link
|
||||
}
|
||||
|
||||
// itemPublished returns the item's publish time as unix seconds, falling back
|
||||
// to the updated time if pubDate was absent. Returns 0 when neither parses;
|
||||
// the poller clamps future timestamps to ingest time.
|
||||
func itemPublished(item *gofeed.Item) int64 {
|
||||
if item.PublishedParsed != nil {
|
||||
return item.PublishedParsed.Unix()
|
||||
}
|
||||
if item.UpdatedParsed != nil {
|
||||
return item.UpdatedParsed.Unix()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// extractLede returns the feed description verbatim, with HTML stripped.
|
||||
func extractLede(desc string) string {
|
||||
if desc == "" {
|
||||
return ""
|
||||
}
|
||||
text := htmlTagRe.ReplaceAllString(desc, "")
|
||||
// Drop <script>/<style>/<noscript> (tags and their inner text) before the
|
||||
// generic tag strip, which would otherwise leave the inner text behind.
|
||||
desc = nonContentElemRe.ReplaceAllString(desc, " ")
|
||||
// Replace tags with a space so adjacent blocks like </p><p> don't fuse words.
|
||||
text := htmlTagRe.ReplaceAllString(desc, " ")
|
||||
text = html.UnescapeString(text)
|
||||
text = wsRe.ReplaceAllString(text, " ")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
// extractContentText converts a feed's content:encoded HTML into plain text
|
||||
// with paragraph breaks preserved, for reader mode. Block-level boundaries
|
||||
// (</p>, </div>, <br>, list items, headings…) become newlines before the
|
||||
// remaining tags are stripped, so the reader can re-wrap the text into
|
||||
// paragraphs. Returns "" when the feed carried no content:encoded.
|
||||
func extractContentText(raw string) string {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return ""
|
||||
}
|
||||
// Remove <script>/<style>/<noscript> elements whole (tags + inner text)
|
||||
// first — otherwise htmlTagRe below strips only the tags and leaves inline
|
||||
// JavaScript (e.g. a Datawrapper embed resizer) as literal reader text.
|
||||
s := nonContentElemRe.ReplaceAllString(raw, "\n")
|
||||
s = brRe.ReplaceAllString(s, "\n")
|
||||
s = blockCloseRe.ReplaceAllString(s, "\n\n")
|
||||
s = htmlTagRe.ReplaceAllString(s, "")
|
||||
s = html.UnescapeString(s)
|
||||
lines := strings.Split(s, "\n")
|
||||
for i, ln := range lines {
|
||||
lines[i] = strings.TrimSpace(hspaceRe.ReplaceAllString(ln, " "))
|
||||
}
|
||||
s = manyNewlineRe.ReplaceAllString(strings.Join(lines, "\n"), "\n\n")
|
||||
s = strings.TrimSpace(s)
|
||||
return truncateUTF8(s, maxContentChars)
|
||||
}
|
||||
|
||||
// truncateUTF8 caps s at max bytes without splitting a multi-byte rune. A plain
|
||||
// byte slice can leave a partial trailing rune, which is invalid UTF-8; that
|
||||
// corrupt byte then breaks the RSS content:encoded XML (Go's xml encoder writes
|
||||
// it raw) and shows as a replacement char in JSON. Trim back to a rune boundary.
|
||||
func truncateUTF8(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
s = s[:max]
|
||||
for len(s) > 0 {
|
||||
if r, size := utf8.DecodeLastRuneInString(s); r == utf8.RuneError && size <= 1 {
|
||||
s = s[:len(s)-1] // strip one byte of an incomplete trailing rune
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// extractImageURL tries to find an image URL from feed item metadata.
|
||||
// Order: media:content/thumbnail, enclosures, item.Image, then <img> tags
|
||||
// scraped from content:encoded / description (catches feeds like Ars that
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package ingestion
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractLede(t *testing.T) {
|
||||
tests := []struct {
|
||||
@@ -48,6 +51,11 @@ func TestExtractLede(t *testing.T) {
|
||||
"It's a new era—one of change.",
|
||||
"It's a new era\u2014one of change.",
|
||||
},
|
||||
{
|
||||
"adjacent block tags inject a space",
|
||||
"<p>...end of moments</p><p>Clarence B Jones, a former...</p>",
|
||||
"...end of moments Clarence B Jones, a former...",
|
||||
},
|
||||
{
|
||||
"tags and entities combined",
|
||||
"<p>Oil prices rose & gas fell by >5%.</p>",
|
||||
@@ -64,3 +72,48 @@ func TestExtractLede(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractContentText(t *testing.T) {
|
||||
if got := extractContentText(""); got != "" {
|
||||
t.Errorf("empty input = %q, want empty", got)
|
||||
}
|
||||
if got := extractContentText(" \n "); got != "" {
|
||||
t.Errorf("whitespace-only input = %q, want empty", got)
|
||||
}
|
||||
|
||||
in := `<p>First paragraph with <a href="/x">a link</a> & entity.</p>` +
|
||||
`<p>Second line one.<br>line two.</p><ul><li>item</li></ul>`
|
||||
got := extractContentText(in)
|
||||
want := "First paragraph with a link & entity.\n\nSecond line one.\nline two.\n\nitem"
|
||||
if got != want {
|
||||
t.Errorf("extractContentText:\n got %q\n want %q", got, want)
|
||||
}
|
||||
|
||||
// Output is capped so a runaway body can't bloat the row.
|
||||
long := "<p>" + strings.Repeat("x", maxContentChars+500) + "</p>"
|
||||
if n := len(extractContentText(long)); n > maxContentChars {
|
||||
t.Errorf("capped length = %d, want <= %d", n, maxContentChars)
|
||||
}
|
||||
|
||||
// Inline embed scripts (Datawrapper resizers, analytics) must be dropped
|
||||
// whole — tags and body — not left as literal reader text. htmlTagRe alone
|
||||
// strips only the tags, so extractContentText has to remove the element first.
|
||||
embed := `<p>Net trust rating of +11 percent nationally.</p>` +
|
||||
`<figure><script type="text/javascript">!function(){"use strict";window.addEventListener("message",(function(a){var e=document.querySelectorAll("iframe");}))}();</script></figure>` +
|
||||
`<p>Burnham is on the cusp of becoming prime minister.</p>`
|
||||
gotEmbed := extractContentText(embed)
|
||||
if strings.Contains(gotEmbed, "datawrapper") || strings.Contains(gotEmbed, "addEventListener") || strings.Contains(gotEmbed, "querySelectorAll") {
|
||||
t.Errorf("inline script leaked into content text:\n%s", gotEmbed)
|
||||
}
|
||||
for _, want := range []string{"Net trust rating", "prime minister"} {
|
||||
if !strings.Contains(gotEmbed, want) {
|
||||
t.Errorf("prose %q missing from content text:\n%s", want, gotEmbed)
|
||||
}
|
||||
}
|
||||
|
||||
// The same guard applies to descriptions/ledes.
|
||||
ledeIn := `Trust gap widens.<script>track("x");</script> North vs south.`
|
||||
if gotLede := extractLede(ledeIn); strings.Contains(gotLede, "track") {
|
||||
t.Errorf("inline script leaked into lede: %q", gotLede)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package ingestion
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -94,10 +95,18 @@ func (p *Poller) pollOnce(ctx context.Context, src config.SourceConfig) {
|
||||
}
|
||||
|
||||
func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) error {
|
||||
items, err := FetchFeed(src.FeedURL)
|
||||
items, err := FetchFeed(src.FeedURL, src.UserAgent)
|
||||
if err != nil {
|
||||
// Persist the failure for the owner-facing source-health dashboard. The
|
||||
// in-memory consecutiveFailures in pollSource drives log warnings; this
|
||||
// row is the authoritative record the dashboard reads.
|
||||
storage.RecordPollResult(src.Name, false, 0, err)
|
||||
return err
|
||||
}
|
||||
// A successful fetch: record health now (before per-item processing) with
|
||||
// the number of items the feed returned, so the dashboard reflects feed
|
||||
// reachability even if individual items are later dropped as duplicates.
|
||||
storage.RecordPollResult(src.Name, true, len(items), nil)
|
||||
|
||||
newCount := 0
|
||||
for i := range items {
|
||||
@@ -109,6 +118,17 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
continue
|
||||
}
|
||||
|
||||
// Drop items in languages this source isn't configured to surface.
|
||||
// Items without a language tag pass through — only filter when the
|
||||
// feed explicitly marks an item with a non-matching language.
|
||||
if src.Language != "" && items[i].Language != "" &&
|
||||
!strings.HasPrefix(items[i].Language, strings.ToLower(src.Language)) {
|
||||
slog.Info("dropping story (language filter)",
|
||||
"guid", items[i].GUID, "source", src.Name,
|
||||
"item_lang", items[i].Language, "want", src.Language)
|
||||
continue
|
||||
}
|
||||
|
||||
// Dedup checks first — canonical and headline use the ORIGINAL URL so
|
||||
// they stay stable whether we end up swapping to a Wayback snapshot.
|
||||
originalURL := items[i].ArticleURL
|
||||
@@ -137,6 +157,21 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
// swapping in a stale snapshot for a transient blip is worse than
|
||||
// posting the live link.
|
||||
meta := FetchArticleMeta(originalURL)
|
||||
// Some sources (e.g. LWN's subscriber-only articles) have no public
|
||||
// version reachable by any workaround. Drop these from view entirely
|
||||
// rather than stamping them as paywalled.
|
||||
if meta.SubscriberOnly {
|
||||
// No DB row written, so the next poll will re-fetch and re-detect
|
||||
// this URL — fine in practice since LWN's feed is tiny and the
|
||||
// alternative (a tombstone row) leaks into every story query.
|
||||
slog.Info("dropping subscriber-only story",
|
||||
"guid", items[i].GUID, "url", originalURL, "source", src.Name)
|
||||
continue
|
||||
}
|
||||
// bodyText is the article text we may store for reader mode. It starts
|
||||
// as the body scraped during the fetch above and is upgraded to the
|
||||
// snapshot's body if we end up swapping in an archive snapshot.
|
||||
bodyText := meta.BodyText
|
||||
gated := meta.Gated() || (meta.Fetched && meta.BodyChars < PaywallBodyThreshold)
|
||||
// paywalled tracks whether the link the user will click is still
|
||||
// gated — i.e. gating was detected AND no archive workaround worked.
|
||||
@@ -152,6 +187,9 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
if items[i].ImageURL == "" && snapMeta.ImageURL != "" {
|
||||
items[i].ImageURL = snapMeta.ImageURL
|
||||
}
|
||||
if snapMeta.BodyText != "" {
|
||||
bodyText = snapMeta.BodyText
|
||||
}
|
||||
workedAround = true
|
||||
slog.Info("paywall detected, using archive snapshot",
|
||||
"guid", items[i].GUID, "original", originalURL,
|
||||
@@ -177,18 +215,35 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
items[i].Source = src.Name
|
||||
items[i].DirectRoute = src.DirectRoute
|
||||
|
||||
// Store immediately (before routing) to prevent re-ingestion
|
||||
// Store immediately (before routing) to prevent re-ingestion.
|
||||
// Future-dated pubDates (publishers occasionally post-date drafts or
|
||||
// have a misconfigured clock) get clamped to ingest time so they don't
|
||||
// stick at the top of the queue indefinitely.
|
||||
seenAt := time.Now().Unix()
|
||||
publishedAt := items[i].PublishedAt
|
||||
if publishedAt > seenAt {
|
||||
publishedAt = seenAt
|
||||
}
|
||||
// Reader-mode body: prefer whichever source gave us more complete text.
|
||||
// Feeds that ship full content:encoded usually win; feeds that only
|
||||
// syndicate a snippet fall back to the scraped article body.
|
||||
content := items[i].Content
|
||||
if len(bodyText) > len(content) {
|
||||
content = bodyText
|
||||
}
|
||||
if err := storage.InsertStory(&storage.Story{
|
||||
GUID: items[i].GUID,
|
||||
Headline: items[i].Headline,
|
||||
Lede: items[i].Lede,
|
||||
Content: content,
|
||||
ImageURL: items[i].ImageURL,
|
||||
ArticleURL: items[i].ArticleURL,
|
||||
URLCanonical: canonical,
|
||||
HeadlineNorm: headlineNorm,
|
||||
Source: items[i].Source,
|
||||
Paywalled: paywalled,
|
||||
SeenAt: time.Now().Unix(),
|
||||
SeenAt: seenAt,
|
||||
PublishedAt: publishedAt,
|
||||
}); err != nil {
|
||||
slog.Error("failed to insert story", "guid", items[i].GUID, "err", err)
|
||||
continue
|
||||
|
||||
@@ -7,9 +7,11 @@ import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
)
|
||||
|
||||
var waybackClient = &http.Client{Timeout: 10 * time.Second}
|
||||
var waybackClient = safehttp.NewClient(10 * time.Second)
|
||||
|
||||
// maxSnapshotAge bounds how stale a Wayback snapshot can be before we treat
|
||||
// it as useless for a freshly-published news article. The "closest" snapshot
|
||||
@@ -86,14 +88,16 @@ func snapshotFreshEnough(ts string) bool {
|
||||
return time.Since(t) <= maxSnapshotAge
|
||||
}
|
||||
|
||||
var archiveTodayClient = &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
// Don't follow the redirect — we just want the Location header pointing
|
||||
// at the most recent capture.
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
// archiveTodayClient routes through safehttp for the dial-time SSRF guard, but
|
||||
// overrides CheckRedirect so we read the 302 Location header (the most recent
|
||||
// capture) instead of following it.
|
||||
var archiveTodayClient = func() *http.Client {
|
||||
c := safehttp.NewClient(10 * time.Second)
|
||||
c.CheckRedirect = func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
}
|
||||
}
|
||||
return c
|
||||
}()
|
||||
|
||||
// ResolveArchiveToday looks up the newest archive.today / archive.ph snapshot
|
||||
// for the given URL. archive.ph has no public availability API, but its
|
||||
|
||||
@@ -13,12 +13,13 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
"pete/internal/safehttp"
|
||||
|
||||
_ "go.mau.fi/util/dbutil/litestream" // registers "sqlite3-fk-wal" driver used by cryptohelper
|
||||
|
||||
@@ -142,30 +143,45 @@ func New(cfg config.MatrixConfig) (*Client, error) {
|
||||
"room", evt.RoomID, "event_id", evt.ID, "sender", evt.Sender, "err", err)
|
||||
}
|
||||
|
||||
// LoginAs enables the cryptohelper to re-login if the token expires
|
||||
ch.LoginAs = &mautrix.ReqLogin{
|
||||
Type: mautrix.AuthTypePassword,
|
||||
Identifier: mautrix.UserIdentifier{
|
||||
Type: mautrix.IdentifierTypeUser,
|
||||
User: cfg.UserID,
|
||||
},
|
||||
Password: cfg.Password,
|
||||
InitialDeviceDisplayName: cfg.DisplayName,
|
||||
}
|
||||
|
||||
// IMPORTANT: do NOT set ch.LoginAs here. We already performed an explicit
|
||||
// password login above (or restored a saved token), so mx is fully
|
||||
// authenticated against device.json's device. When LoginAs is set AND the
|
||||
// crypto store is fresh, cryptohelper.Init() performs a SECOND login
|
||||
// (StoreCredentials=true), minting a separate device and pinning the olm
|
||||
// account to it — while device.json still records the first device. That
|
||||
// split-brain is exactly the "device never verifies" bug: the device the
|
||||
// token authenticates as is not the device the crypto account belongs to,
|
||||
// and each cold start leaks another orphan device. Token-expiry recovery is
|
||||
// already handled by the isTokenValid() check + re-login path above.
|
||||
if err := ch.Init(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("crypto helper init: %w", err)
|
||||
}
|
||||
|
||||
mx.Crypto = ch
|
||||
|
||||
// Bootstrap cross-signing only if not already set up.
|
||||
// Ensure our CURRENT device is cross-signed. We bootstrap (generate + upload
|
||||
// fresh cross-signing keys, then self-sign) whenever either no keys exist yet,
|
||||
// or keys exist on the server but our device is not signed by them. The latter
|
||||
// is exactly the post-wipe state: when crypto.db is deleted the private
|
||||
// self-signing key is lost (it is never persisted locally without an SSSS
|
||||
// passphrase, and we discard the random recovery key), so the lingering server
|
||||
// master key is useless to us and must be replaced.
|
||||
//
|
||||
// CRITICAL: do NOT gate this on IsDeviceTrusted(mach.OwnIdentity()).
|
||||
// OwnIdentity() hard-codes Trust=id.TrustStateVerified, and ResolveTrustContext
|
||||
// short-circuits on that, so the call is a tautology that ALWAYS returns true —
|
||||
// it makes this branch unreachable and the device never gets signed.
|
||||
// GetOwnVerificationStatus does a real check: is our own device key signed by
|
||||
// our self-signing key?
|
||||
mach := ch.Machine()
|
||||
existingKeys, err := mach.GetOwnCrossSigningPublicKeys(context.Background())
|
||||
hasKeys, deviceCrossSigned, err := mach.GetOwnVerificationStatus(context.Background())
|
||||
if err != nil {
|
||||
slog.Warn("cross-signing: failed to fetch existing keys", "err", err)
|
||||
slog.Warn("cross-signing: failed to check verification status", "err", err)
|
||||
}
|
||||
if existingKeys == nil || existingKeys.MasterKey == "" {
|
||||
if !hasKeys || !deviceCrossSigned {
|
||||
if hasKeys {
|
||||
slog.Warn("cross-signing: keys exist but current device is not cross-signed, resetting cross-signing")
|
||||
}
|
||||
_, _, err = mach.GenerateAndUploadCrossSigningKeys(context.Background(), func(ui *mautrix.RespUserInteractive) interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": mautrix.AuthTypePassword,
|
||||
@@ -195,7 +211,7 @@ func New(cfg config.MatrixConfig) (*Client, error) {
|
||||
slog.Info("cross-signing: master key signed")
|
||||
}
|
||||
} else {
|
||||
slog.Info("cross-signing: already configured, skipping bootstrap")
|
||||
slog.Info("cross-signing: already configured and current device cross-signed, skipping bootstrap")
|
||||
}
|
||||
|
||||
slog.Info("E2EE initialized",
|
||||
@@ -415,14 +431,18 @@ func (c *Client) PostStory(channel string, story *PostableStory) (eventID id.Eve
|
||||
return resp.EventID, imageSent, nil
|
||||
}
|
||||
|
||||
// imageDownloadClient fetches feed-supplied image URLs for re-upload to Matrix.
|
||||
// It routes through safehttp so a hostile feed can't point ImageURL at internal
|
||||
// services (loopback / RFC1918 / cloud metadata) — including via redirects.
|
||||
var imageDownloadClient = safehttp.NewClient(15 * time.Second)
|
||||
|
||||
// sendImage downloads an image, uploads it to Matrix, and sends it as m.image.
|
||||
func (c *Client) sendImage(ctx context.Context, roomID id.RoomID, imageURL string) error {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", imageURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create image request: %w", err)
|
||||
}
|
||||
httpClient := &http.Client{Timeout: 15 * time.Second}
|
||||
resp, err := httpClient.Do(req)
|
||||
resp, err := imageDownloadClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download image: %w", err)
|
||||
}
|
||||
|
||||
@@ -3,9 +3,21 @@ package matrix
|
||||
import (
|
||||
"fmt"
|
||||
"html"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// safeHref returns raw if it is an http(s) URL, else "". Keeps feed-supplied
|
||||
// links from injecting javascript:/data: schemes into the formatted_body href
|
||||
// (html.EscapeString blocks attribute breakout but not the scheme itself).
|
||||
func safeHref(raw string) string {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return ""
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// PostableStory contains the data needed to format and send a story.
|
||||
type PostableStory struct {
|
||||
ImageURL string
|
||||
@@ -30,11 +42,16 @@ func formatPost(s *PostableStory) (plain, htmlBody string) {
|
||||
|
||||
// HTML body
|
||||
var htmlParts []string
|
||||
htmlParts = append(htmlParts, fmt.Sprintf(
|
||||
`<strong><a href="%s">%s</a></strong>`,
|
||||
html.EscapeString(s.ArticleURL),
|
||||
html.EscapeString(s.Headline),
|
||||
))
|
||||
if href := safeHref(s.ArticleURL); href != "" {
|
||||
htmlParts = append(htmlParts, fmt.Sprintf(
|
||||
`<strong><a href="%s">%s</a></strong>`,
|
||||
html.EscapeString(href),
|
||||
html.EscapeString(s.Headline),
|
||||
))
|
||||
} else {
|
||||
// Non-http(s) link: render the headline without an anchor.
|
||||
htmlParts = append(htmlParts, fmt.Sprintf(`<strong>%s</strong>`, html.EscapeString(s.Headline)))
|
||||
}
|
||||
if s.Lede != "" {
|
||||
htmlParts = append(htmlParts, html.EscapeString(s.Lede))
|
||||
}
|
||||
|
||||
@@ -92,7 +92,10 @@ func (q *Queue) Wait() {
|
||||
|
||||
// ForcePost pops the next queued item for the given channel and posts it
|
||||
// immediately, bypassing min-interval, burst cap, and daily cap. Dedup
|
||||
// (canonical URL cooldown) still applies. Returns true if an item was sent.
|
||||
// (canonical URL cooldown) still applies. Returns true only if a Matrix
|
||||
// message was actually sent — a dedup-skip returns false so the caller
|
||||
// can fall back (e.g., to a DB lookup) instead of silently consuming the
|
||||
// queued item with no user-visible result.
|
||||
func (q *Queue) ForcePost(channel string) bool {
|
||||
q.mu.Lock()
|
||||
items := q.queues[channel]
|
||||
@@ -106,8 +109,7 @@ func (q *Queue) ForcePost(channel string) bool {
|
||||
|
||||
slog.Info("force-posting story on demand",
|
||||
"guid", item.GUID, "channel", channel)
|
||||
q.postItem(item)
|
||||
return true
|
||||
return q.postItem(item, true)
|
||||
}
|
||||
|
||||
func (q *Queue) drain() {
|
||||
@@ -192,17 +194,23 @@ func (q *Queue) drainChannel(channel string) {
|
||||
q.queues[channel] = items[1:]
|
||||
q.mu.Unlock()
|
||||
|
||||
q.postItem(item)
|
||||
q.postItem(item, false)
|
||||
}
|
||||
|
||||
// PostNow sends a story immediately, bypassing the in-memory queue and all
|
||||
// pacing limits. Last-mile canonical-URL dedup still applies. Used by !post
|
||||
// to satisfy on-demand requests with stories pulled directly from storage.
|
||||
func (q *Queue) PostNow(item QueueItem) {
|
||||
q.postItem(item)
|
||||
q.postItem(item, true)
|
||||
}
|
||||
|
||||
func (q *Queue) postItem(item QueueItem) {
|
||||
// postItem returns true when a Matrix event was actually sent, false on
|
||||
// any non-success path (dedup-skip, transport failure with or without
|
||||
// retry, dead-letter). ForcePost uses the return value to decide whether
|
||||
// to acknowledge the user-initiated !post or fall back to a DB lookup.
|
||||
// forced=true marks the resulting post_log row so it's excluded from the
|
||||
// global daily cap (manual overrides shouldn't steal the rotation budget).
|
||||
func (q *Queue) postItem(item QueueItem, forced bool) bool {
|
||||
// Last-mile dedup: if this canonical URL was already posted to this channel
|
||||
// within the cooldown window, drop silently. Catches "same article, different
|
||||
// GUID across feeds" and any race where two items slipped past ingest dedup.
|
||||
@@ -215,7 +223,7 @@ func (q *Queue) postItem(item QueueItem) {
|
||||
"url_canonical", canonical,
|
||||
"cooldown_hours", q.config.DedupCooldownHours,
|
||||
)
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
story := &matrix.PostableStory{
|
||||
@@ -237,7 +245,7 @@ func (q *Queue) postItem(item QueueItem) {
|
||||
"channel", item.Channel,
|
||||
"err", err,
|
||||
)
|
||||
return
|
||||
return false
|
||||
}
|
||||
slog.Error("failed to post story, will retry",
|
||||
"guid", item.GUID,
|
||||
@@ -254,15 +262,16 @@ func (q *Queue) postItem(item QueueItem) {
|
||||
q.mu.Lock()
|
||||
q.queues[item.Channel] = append([]QueueItem{item}, q.queues[item.Channel]...)
|
||||
q.mu.Unlock()
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
// Record in post log (INSERT OR IGNORE via unique index)
|
||||
storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical)
|
||||
storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical, forced)
|
||||
|
||||
slog.Info("story posted",
|
||||
"guid", item.GUID,
|
||||
"channel", item.Channel,
|
||||
"event_id", eventID,
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -18,6 +18,15 @@ type ReactionCallback func(roomID id.RoomID, targetEventID id.EventID, guid, cha
|
||||
var (
|
||||
reactionCallbackMu sync.RWMutex
|
||||
reactionCallback ReactionCallback
|
||||
|
||||
// reactionSem bounds concurrent in-flight reaction callbacks. Each callback
|
||||
// may do DB / Matrix / LLM I/O, so an unbounded fan-out (`go cb(...)` per
|
||||
// reaction) lets a reaction flood — legitimate burst or an abusive room
|
||||
// member spamming emoji — exhaust goroutines, connections, and memory.
|
||||
// Tries to acquire non-blocking: if the cap is saturated we drop the
|
||||
// callback with a warn rather than queueing, on the theory that the room
|
||||
// is already busy enough.
|
||||
reactionSem = make(chan struct{}, 8)
|
||||
)
|
||||
|
||||
// SetReactionCallback installs a hook called after every recorded reaction.
|
||||
@@ -48,7 +57,19 @@ func HandleReaction(roomID id.RoomID, eventID id.EventID, targetEventID id.Event
|
||||
reactionCallbackMu.RLock()
|
||||
cb := reactionCallback
|
||||
reactionCallbackMu.RUnlock()
|
||||
if cb != nil {
|
||||
go cb(roomID, targetEventID, guid, channel, emoji)
|
||||
if cb == nil {
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case reactionSem <- struct{}{}:
|
||||
default:
|
||||
slog.Warn("reaction callback dropped: worker pool saturated",
|
||||
"guid", guid, "channel", channel, "emoji", emoji)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer func() { <-reactionSem }()
|
||||
cb(roomID, targetEventID, guid, channel, emoji)
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestHandleReaction_KnownPost(t *testing.T) {
|
||||
setupTrackerTestDB(t)
|
||||
|
||||
// Insert a post log entry
|
||||
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "")
|
||||
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "", false)
|
||||
|
||||
// Handle a reaction to that post
|
||||
HandleReaction(
|
||||
@@ -63,7 +63,7 @@ func TestHandleReaction_UnknownPost(t *testing.T) {
|
||||
func TestHandleReaction_DuplicateIgnored(t *testing.T) {
|
||||
setupTrackerTestDB(t)
|
||||
|
||||
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "")
|
||||
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "", false)
|
||||
|
||||
// Send same reaction twice
|
||||
HandleReaction(
|
||||
|
||||
147
internal/safehttp/safehttp.go
Normal file
147
internal/safehttp/safehttp.go
Normal file
@@ -0,0 +1,147 @@
|
||||
// Package safehttp provides an http.Client hardened against SSRF and
|
||||
// memory-DoS via hostile upstreams. Every outbound fetch the bot makes
|
||||
// against feed-supplied URLs (RSS feeds, article pages, image hosts)
|
||||
// should go through one of these clients so a malicious feed can't
|
||||
// steer Pete at loopback, link-local, RFC1918, or cloud metadata IPs,
|
||||
// and can't OOM the process by streaming an unbounded body.
|
||||
package safehttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrBlockedHost is returned when a URL resolves to a non-public IP.
|
||||
var ErrBlockedHost = errors.New("safehttp: blocked non-public host")
|
||||
|
||||
// AllowPrivate, when true, disables the loopback/RFC1918 dial guard. It
|
||||
// exists for tests that spin up httptest.NewServer on 127.0.0.1 — never
|
||||
// set this in production.
|
||||
var AllowPrivate bool
|
||||
|
||||
// safeDialContext refuses connections to non-public IPs. It runs after
|
||||
// DNS resolution, so a hostile DNS rebinding that returns 127.0.0.1
|
||||
// still gets blocked at dial time.
|
||||
func safeDialContext(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ips, err := (&net.Resolver{}).LookupIP(ctx, "ip", host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var allowed net.IP
|
||||
for _, ip := range ips {
|
||||
if AllowPrivate || isPublicIP(ip) {
|
||||
allowed = ip
|
||||
break
|
||||
}
|
||||
}
|
||||
if allowed == nil {
|
||||
return nil, fmt.Errorf("%w: %s", ErrBlockedHost, host)
|
||||
}
|
||||
d := &net.Dialer{Timeout: 5 * time.Second, KeepAlive: 30 * time.Second}
|
||||
return d.DialContext(ctx, network, net.JoinHostPort(allowed.String(), port))
|
||||
}
|
||||
|
||||
// isPublicIP reports whether ip is a globally routable unicast address.
|
||||
// Rejects loopback, link-local, multicast, RFC1918, CGNAT, and the
|
||||
// AWS/GCP/Azure metadata IPs 169.254.169.254 / fd00:ec2::254 (these
|
||||
// already fall under link-local but spell it out for clarity).
|
||||
func isPublicIP(ip net.IP) bool {
|
||||
if ip == nil || ip.IsUnspecified() || ip.IsLoopback() ||
|
||||
ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
||||
ip.IsMulticast() || ip.IsPrivate() {
|
||||
return false
|
||||
}
|
||||
// 100.64.0.0/10 (CGNAT) is not covered by IsPrivate on older Go.
|
||||
if v4 := ip.To4(); v4 != nil {
|
||||
if v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127 {
|
||||
return false
|
||||
}
|
||||
// 0.0.0.0/8, 192.0.0.0/24, 192.0.2.0/24, 198.18.0.0/15, 198.51.100.0/24, 203.0.113.0/24
|
||||
if v4[0] == 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ValidateURL returns nil if the URL is http(s) and parseable. It does
|
||||
// not resolve DNS — the dial step does that — but it does reject bare
|
||||
// schemes (file://, gopher://, etc.) before we even open a connection.
|
||||
func ValidateURL(raw string) error {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return fmt.Errorf("safehttp: unsupported scheme %q", u.Scheme)
|
||||
}
|
||||
if u.Host == "" {
|
||||
return errors.New("safehttp: empty host")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewClient returns an http.Client whose transport blocks non-public
|
||||
// destinations at dial time, caps redirects at 5, and re-validates each
|
||||
// redirect target's scheme. timeout is the per-request overall budget.
|
||||
func NewClient(timeout time.Duration) *http.Client {
|
||||
tr := &http.Transport{
|
||||
DialContext: safeDialContext,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 32,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 5 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
ResponseHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
return &http.Client{
|
||||
Transport: tr,
|
||||
Timeout: timeout,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 5 {
|
||||
return errors.New("safehttp: stopped after 5 redirects")
|
||||
}
|
||||
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
|
||||
return fmt.Errorf("safehttp: unsupported redirect scheme %q", req.URL.Scheme)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// LimitedBody wraps r in an io.LimitReader that returns io.ErrUnexpectedEOF
|
||||
// once more than max bytes have been read. Use to cap how much of a
|
||||
// response body downstream parsers (goquery, gofeed, image.Decode) will
|
||||
// ever see — a hostile origin streaming an endless body otherwise OOMs
|
||||
// the process.
|
||||
func LimitedBody(r io.Reader, max int64) io.Reader {
|
||||
return &limitedReader{R: r, N: max}
|
||||
}
|
||||
|
||||
type limitedReader struct {
|
||||
R io.Reader
|
||||
N int64
|
||||
}
|
||||
|
||||
func (l *limitedReader) Read(p []byte) (int, error) {
|
||||
if l.N <= 0 {
|
||||
return 0, fmt.Errorf("safehttp: response body exceeded cap")
|
||||
}
|
||||
if int64(len(p)) > l.N {
|
||||
p = p[:l.N]
|
||||
}
|
||||
n, err := l.R.Read(p)
|
||||
l.N -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
@@ -156,7 +156,7 @@ func TestTick_AlreadyPostedExcluded(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
insertPostable(t, "t-posted", "GuardianTech", "tech", 200)
|
||||
insertPostable(t, "t-fresh", "GuardianTech", "tech", 100)
|
||||
storage.InsertPostLog("t-posted", "tech", "$evt1", "https://example.com/t-posted")
|
||||
storage.InsertPostLog("t-posted", "tech", "$evt1", "https://example.com/t-posted", false)
|
||||
|
||||
fq := &fakeQueue{}
|
||||
rr := New([]string{"tech"}, 4, fq)
|
||||
|
||||
@@ -81,13 +81,26 @@ func runMigrations(d *sql.DB) error {
|
||||
addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
|
||||
addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
|
||||
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
|
||||
// content holds the full article text (feed content:encoded when present,
|
||||
// else the body scraped during paywall detection) for reader mode. Stories
|
||||
// ingested before this column existed simply have NULL and fall back to lede.
|
||||
addColumnIfMissing(d, "stories", "content", "TEXT")
|
||||
// content_chars caches the character count of content so the "N min read"
|
||||
// chip never has to LENGTH() the full body on the hot listing path. Filled at
|
||||
// insert time; the backfill below populates rows that predate the column.
|
||||
addColumnIfMissing(d, "stories", "content_chars", "INTEGER NOT NULL DEFAULT 0")
|
||||
backfillContentChars(d)
|
||||
addColumnIfMissing(d, "stories", "published_at", "INTEGER")
|
||||
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
|
||||
addColumnIfMissing(d, "post_log", "forced", "INTEGER NOT NULL DEFAULT 0")
|
||||
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
|
||||
|
||||
// FTS5 virtual tables don't support IF NOT EXISTS reliably.
|
||||
// Check sqlite_master before creating.
|
||||
var ftsExists int
|
||||
d.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='stories_fts'`).Scan(&ftsExists)
|
||||
if err := d.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='stories_fts'`).Scan(&ftsExists); err != nil {
|
||||
return fmt.Errorf("probe FTS5 table: %w", err)
|
||||
}
|
||||
if ftsExists == 0 {
|
||||
if _, err := d.Exec(ftsSchema); err != nil {
|
||||
return fmt.Errorf("create FTS5 table: %w", err)
|
||||
@@ -112,17 +125,54 @@ func RunMaintenance() {
|
||||
exec("prune old reactions",
|
||||
`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)
|
||||
|
||||
// Drop per-user read/bookmark rows whose story has been pruned above, so the
|
||||
// table can't accumulate dangling references as stories age out.
|
||||
exec("prune orphan user_story_state",
|
||||
`DELETE FROM user_story_state WHERE story_id NOT IN (SELECT id FROM stories)`)
|
||||
|
||||
// Same for per-story view counts once their story has aged out.
|
||||
exec("prune orphan story_views",
|
||||
`DELETE FROM story_views WHERE story_id NOT IN (SELECT id FROM stories)`)
|
||||
|
||||
// Daily unique tokens are only useful for the recent window; their salts are
|
||||
// long gone. page_views is kept forever (tiny aggregate, all-time totals).
|
||||
exec("prune old daily_visitors",
|
||||
`DELETE FROM daily_visitors WHERE day < ?`, unixDay()-30)
|
||||
|
||||
exec("wal checkpoint", "PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
exec("optimize", "PRAGMA optimize")
|
||||
}
|
||||
|
||||
// exec is a fire-and-forget helper that logs errors.
|
||||
// exec is a fire-and-forget helper that logs errors. Several callers run it from
|
||||
// background goroutines (metrics, view counts), which can outlive a Close() — so
|
||||
// unlike Get() it must not panic on a nil handle: it simply skips the write.
|
||||
func exec(label, query string, args ...any) {
|
||||
if _, err := Get().Exec(query, args...); err != nil {
|
||||
mu.RLock()
|
||||
db := globalDB
|
||||
mu.RUnlock()
|
||||
if db == nil {
|
||||
slog.Warn("db exec skipped: no database", "op", label)
|
||||
return
|
||||
}
|
||||
if _, err := db.Exec(query, args...); err != nil {
|
||||
slog.Error("db exec failed", "op", label, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// backfillContentChars populates content_chars for rows carrying a body but a
|
||||
// zero count — i.e. stories ingested before the column existed. LENGTH() counts
|
||||
// characters (code points) for TEXT, matching the utf8.RuneCountInString done at
|
||||
// insert. After the first run this matches no rows (bodied stories are set,
|
||||
// bodyless ones stay 0 and are filtered by content IS NOT NULL), so it's a cheap
|
||||
// startup no-op thereafter.
|
||||
func backfillContentChars(d *sql.DB) {
|
||||
if _, err := d.Exec(
|
||||
`UPDATE stories SET content_chars = LENGTH(content)
|
||||
WHERE content_chars = 0 AND content IS NOT NULL AND content <> ''`); err != nil {
|
||||
slog.Error("backfill content_chars failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func addColumnIfMissing(d *sql.DB, table, column, columnType string) {
|
||||
q := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, columnType)
|
||||
if _, err := d.Exec(q); err != nil {
|
||||
|
||||
171
internal/storage/metrics.go
Normal file
171
internal/storage/metrics.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"sort"
|
||||
)
|
||||
|
||||
const secondsPerDay = 86400
|
||||
|
||||
// unixDay returns the current UTC day number (floor(unix / 86400)).
|
||||
func unixDay() int64 { return nowUnix() / secondsPerDay }
|
||||
|
||||
// UnixDay is the exported current UTC day number, for callers building
|
||||
// day-windowed queries (e.g. the trending rail's 7-day cutoff).
|
||||
func UnixDay() int64 { return unixDay() }
|
||||
|
||||
// RecordPageView increments the all-time and per-day view counter for a coarse
|
||||
// path label ("home", a channel slug, …). Fire-and-forget: a failure here must
|
||||
// never affect serving a page, so errors are logged and swallowed.
|
||||
func RecordPageView(path string) {
|
||||
exec("record page view",
|
||||
`INSERT INTO page_views (path, day, views) VALUES (?, ?, 1)
|
||||
ON CONFLICT(path, day) DO UPDATE SET views = views + 1`,
|
||||
path, unixDay())
|
||||
}
|
||||
|
||||
// RecordVisitor records a (already-hashed, salted) visitor token for today's
|
||||
// unique estimate. Duplicate tokens within the same day are ignored.
|
||||
func RecordVisitor(token string) {
|
||||
exec("record visitor",
|
||||
`INSERT OR IGNORE INTO daily_visitors (day, visitor) VALUES (?, ?)`,
|
||||
unixDay(), token)
|
||||
}
|
||||
|
||||
// RecordStoryView bumps the per-day read counter for a single story. Called
|
||||
// when a visitor opens the story in reader mode. Fire-and-forget like the other
|
||||
// metrics writes: a failure here must never affect serving the article.
|
||||
func RecordStoryView(id int64) {
|
||||
exec("record story view",
|
||||
`INSERT INTO story_views (story_id, day, views) VALUES (?, ?, 1)
|
||||
ON CONFLICT(story_id, day) DO UPDATE SET views = views + 1`,
|
||||
id, unixDay())
|
||||
}
|
||||
|
||||
// StoryViewTotals returns all-time read counts for the given story ids, as a
|
||||
// map keyed by id. Ids with no recorded views are simply absent from the map
|
||||
// (callers treat missing as zero). Best-effort: on error it returns whatever it
|
||||
// managed to read, so a metrics hiccup never blanks a page.
|
||||
func StoryViewTotals(ids []int64) map[int64]int {
|
||||
out := make(map[int64]int, len(ids))
|
||||
if len(ids) == 0 {
|
||||
return out
|
||||
}
|
||||
ph, args := intInClause(ids)
|
||||
rows, err := Get().Query(
|
||||
`SELECT story_id, SUM(views) FROM story_views
|
||||
WHERE story_id IN (`+ph+`) GROUP BY story_id`, args...)
|
||||
if err != nil {
|
||||
slog.Error("metrics: story view totals query failed", "err", err)
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var n int
|
||||
if err := rows.Scan(&id, &n); err != nil {
|
||||
slog.Error("metrics: scan story view total failed", "err", err)
|
||||
continue
|
||||
}
|
||||
out[id] = n
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
slog.Error("metrics: story view totals iteration failed", "err", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// intInClause builds a "?, ?, …" placeholder string and matching args slice for
|
||||
// a SQL IN (…) over int64 ids.
|
||||
func intInClause(ids []int64) (string, []any) {
|
||||
args := make([]any, len(ids))
|
||||
for i, id := range ids {
|
||||
args[i] = id
|
||||
}
|
||||
return placeholders(len(ids)), args
|
||||
}
|
||||
|
||||
// PathStat is one row of the per-page usage breakdown.
|
||||
type PathStat struct {
|
||||
Path string
|
||||
Total int // all-time views
|
||||
Today int // views since the start of the current UTC day
|
||||
}
|
||||
|
||||
// DayStat is one day's unique-visitor estimate.
|
||||
type DayStat struct {
|
||||
Day int64 // unix day number
|
||||
Uniques int
|
||||
}
|
||||
|
||||
// MetricsSummary is the aggregate usage readout surfaced by !petestats.
|
||||
type MetricsSummary struct {
|
||||
Pages []PathStat // per-path, busiest first
|
||||
TotalViews int // all-time, all paths
|
||||
ViewsToday int // all paths, current UTC day
|
||||
UniquesToday int // distinct visitor tokens, current UTC day
|
||||
Last7Days []DayStat // daily uniques, oldest → newest (note: salt rotates
|
||||
// daily, so these CANNOT be summed into a cross-day unique count)
|
||||
}
|
||||
|
||||
// GetMetricsSummary assembles the usage readout. Best-effort: any sub-query
|
||||
// failure leaves that field at its zero value rather than failing the whole
|
||||
// command.
|
||||
func GetMetricsSummary() MetricsSummary {
|
||||
today := unixDay()
|
||||
var m MetricsSummary
|
||||
|
||||
rows, err := Get().Query(
|
||||
`SELECT path,
|
||||
SUM(views) AS total,
|
||||
COALESCE(SUM(CASE WHEN day = ? THEN views END), 0) AS today
|
||||
FROM page_views
|
||||
GROUP BY path`, today)
|
||||
if err != nil {
|
||||
slog.Error("metrics: page_views query failed", "err", err)
|
||||
} else {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var p PathStat
|
||||
if err := rows.Scan(&p.Path, &p.Total, &p.Today); err != nil {
|
||||
slog.Error("metrics: scan page stat failed", "err", err)
|
||||
continue
|
||||
}
|
||||
m.Pages = append(m.Pages, p)
|
||||
m.TotalViews += p.Total
|
||||
m.ViewsToday += p.Today
|
||||
}
|
||||
}
|
||||
sort.Slice(m.Pages, func(i, j int) bool {
|
||||
if m.Pages[i].Total != m.Pages[j].Total {
|
||||
return m.Pages[i].Total > m.Pages[j].Total
|
||||
}
|
||||
return m.Pages[i].Path < m.Pages[j].Path
|
||||
})
|
||||
|
||||
if err := Get().QueryRow(
|
||||
`SELECT COUNT(*) FROM daily_visitors WHERE day = ?`, today,
|
||||
).Scan(&m.UniquesToday); err != nil {
|
||||
slog.Error("metrics: uniques-today query failed", "err", err)
|
||||
}
|
||||
|
||||
since := today - 6 // inclusive 7-day window
|
||||
vrows, err := Get().Query(
|
||||
`SELECT day, COUNT(*) FROM daily_visitors
|
||||
WHERE day >= ? GROUP BY day ORDER BY day`, since)
|
||||
if err != nil {
|
||||
slog.Error("metrics: 7-day uniques query failed", "err", err)
|
||||
} else {
|
||||
defer vrows.Close()
|
||||
for vrows.Next() {
|
||||
var d DayStat
|
||||
if err := vrows.Scan(&d.Day, &d.Uniques); err != nil {
|
||||
slog.Error("metrics: scan day stat failed", "err", err)
|
||||
continue
|
||||
}
|
||||
m.Last7Days = append(m.Last7Days, d)
|
||||
}
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
42
internal/storage/metrics_test.go
Normal file
42
internal/storage/metrics_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package storage
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMetrics_ViewsAndUniques(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
// Three views on home, two on gaming.
|
||||
RecordPageView("home")
|
||||
RecordPageView("home")
|
||||
RecordPageView("home")
|
||||
RecordPageView("gaming")
|
||||
RecordPageView("gaming")
|
||||
|
||||
// Two distinct visitors today; the repeat token must not double-count.
|
||||
RecordVisitor("tok-a")
|
||||
RecordVisitor("tok-a")
|
||||
RecordVisitor("tok-b")
|
||||
|
||||
m := GetMetricsSummary()
|
||||
|
||||
if m.TotalViews != 5 {
|
||||
t.Errorf("TotalViews = %d, want 5", m.TotalViews)
|
||||
}
|
||||
if m.ViewsToday != 5 {
|
||||
t.Errorf("ViewsToday = %d, want 5", m.ViewsToday)
|
||||
}
|
||||
if m.UniquesToday != 2 {
|
||||
t.Errorf("UniquesToday = %d, want 2 (deduped)", m.UniquesToday)
|
||||
}
|
||||
if len(m.Pages) == 0 || m.Pages[0].Path != "home" || m.Pages[0].Total != 3 {
|
||||
t.Errorf("Pages[0] = %+v, want home with 3 views first", m.Pages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetrics_EmptyIsZero(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
m := GetMetricsSummary()
|
||||
if m.TotalViews != 0 || m.ViewsToday != 0 || m.UniquesToday != 0 || len(m.Pages) != 0 {
|
||||
t.Errorf("expected zero-value summary, got %+v", m)
|
||||
}
|
||||
}
|
||||
50
internal/storage/prefs.go
Normal file
50
internal/storage/prefs.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// UserPrefs is one signed-in user's stored state. Prefs is an opaque JSON blob
|
||||
// owned by the web frontend (disabled feeds, weather location, toggles, …) —
|
||||
// the backend just persists and returns it, keyed by the OIDC subject.
|
||||
type UserPrefs struct {
|
||||
Sub string
|
||||
Prefs string
|
||||
Username string
|
||||
Email string
|
||||
UpdatedAt int64
|
||||
}
|
||||
|
||||
// GetUserPrefs returns the stored prefs blob for an OIDC subject. It returns
|
||||
// ("", nil) when the user has no record yet (first sign-in).
|
||||
func GetUserPrefs(sub string) (string, error) {
|
||||
var prefs string
|
||||
err := Get().QueryRow(`SELECT prefs FROM user_preferences WHERE user_sub = ?`, sub).Scan(&prefs)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get user prefs: %w", err)
|
||||
}
|
||||
return prefs, nil
|
||||
}
|
||||
|
||||
// PutUserPrefs upserts a user's prefs blob along with identity hints (username,
|
||||
// email) for later features. The prefs string is stored verbatim.
|
||||
func PutUserPrefs(sub, prefs, username, email string) error {
|
||||
_, err := Get().Exec(`
|
||||
INSERT INTO user_preferences (user_sub, prefs, username, email, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(user_sub) DO UPDATE SET
|
||||
prefs = excluded.prefs,
|
||||
username = excluded.username,
|
||||
email = excluded.email,
|
||||
updated_at = excluded.updated_at`,
|
||||
sub, prefs, username, email, nowUnix())
|
||||
if err != nil {
|
||||
return fmt.Errorf("put user prefs: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
44
internal/storage/prefs_test.go
Normal file
44
internal/storage/prefs_test.go
Normal file
@@ -0,0 +1,44 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUserPrefsRoundTrip(t *testing.T) {
|
||||
if err := Init(filepath.Join(t.TempDir(), "prefs.db")); err != nil {
|
||||
t.Fatalf("init db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = Close() })
|
||||
|
||||
// Missing user → empty, no error.
|
||||
got, err := GetUserPrefs("nobody")
|
||||
if err != nil {
|
||||
t.Fatalf("get missing: %v", err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Fatalf("expected empty for missing user, got %q", got)
|
||||
}
|
||||
|
||||
blob := `{"pete.weather.loc.v1":"{\"postal\":\"1000\"}"}`
|
||||
if err := PutUserPrefs("ak-sub-1", blob, "misaki", "m@example.com"); err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
got, err = GetUserPrefs("ak-sub-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got != blob {
|
||||
t.Fatalf("blob mismatch:\n got %q\nwant %q", got, blob)
|
||||
}
|
||||
|
||||
// Upsert: second put for same sub replaces the blob.
|
||||
blob2 := `{"pete-weather-off":"1"}`
|
||||
if err := PutUserPrefs("ak-sub-1", blob2, "misaki", "m@example.com"); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
got, _ = GetUserPrefs("ak-sub-1")
|
||||
if got != blob2 {
|
||||
t.Fatalf("upsert mismatch: got %q want %q", got, blob2)
|
||||
}
|
||||
}
|
||||
115
internal/storage/push.go
Normal file
115
internal/storage/push.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package storage
|
||||
|
||||
import "fmt"
|
||||
|
||||
// PushSubscription is one browser/device endpoint a signed-in user has opted in
|
||||
// for Web Push digests. See the push_subscriptions schema for the field roles.
|
||||
type PushSubscription struct {
|
||||
Endpoint string
|
||||
UserSub string
|
||||
P256dh string
|
||||
Auth string
|
||||
CreatedAt int64
|
||||
LastNotifiedAt int64
|
||||
}
|
||||
|
||||
// AddPushSubscription records (or refreshes) a push endpoint for a user. The
|
||||
// endpoint is the primary key, so a re-subscribe from the same browser updates
|
||||
// the keys and resets the digest watermark to now — the user shouldn't be
|
||||
// paged for everything published before they opted in.
|
||||
func AddPushSubscription(sub, endpoint, p256dh, auth string) error {
|
||||
now := nowUnix()
|
||||
_, err := Get().Exec(`
|
||||
INSERT INTO push_subscriptions (endpoint, user_sub, p256dh, auth, created_at, last_notified_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(endpoint) DO UPDATE SET
|
||||
user_sub = excluded.user_sub,
|
||||
p256dh = excluded.p256dh,
|
||||
auth = excluded.auth,
|
||||
last_notified_at = excluded.last_notified_at`,
|
||||
endpoint, sub, p256dh, auth, now, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("add push subscription: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemovePushSubscription drops one endpoint regardless of owner. Reserved for
|
||||
// the digest sender's prune path, where a push service has reported the endpoint
|
||||
// gone (404/410) and there's no caller identity to scope by. User-initiated
|
||||
// opt-outs must use RemovePushSubscriptionForUser.
|
||||
func RemovePushSubscription(endpoint string) error {
|
||||
_, err := Get().Exec(`DELETE FROM push_subscriptions WHERE endpoint = ?`, endpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("remove push subscription: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemovePushSubscriptionForUser drops an endpoint only if it belongs to sub, so
|
||||
// a signed-in user can't unsubscribe another account's device by presenting its
|
||||
// endpoint string. A no-op (no matching row) is not an error.
|
||||
func RemovePushSubscriptionForUser(sub, endpoint string) error {
|
||||
_, err := Get().Exec(
|
||||
`DELETE FROM push_subscriptions WHERE endpoint = ? AND user_sub = ?`, endpoint, sub)
|
||||
if err != nil {
|
||||
return fmt.Errorf("remove push subscription: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListPushSubscriptions returns every stored subscription, for the digest sender.
|
||||
func ListPushSubscriptions() ([]PushSubscription, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT endpoint, user_sub, p256dh, auth, created_at, last_notified_at
|
||||
FROM push_subscriptions`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []PushSubscription
|
||||
for rows.Next() {
|
||||
var p PushSubscription
|
||||
if err := rows.Scan(&p.Endpoint, &p.UserSub, &p.P256dh, &p.Auth, &p.CreatedAt, &p.LastNotifiedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// TouchPushSubscription advances an endpoint's digest watermark so its next
|
||||
// digest only considers stories seen after ts.
|
||||
func TouchPushSubscription(endpoint string, ts int64) error {
|
||||
_, err := Get().Exec(
|
||||
`UPDATE push_subscriptions SET last_notified_at = ? WHERE endpoint = ?`, ts, endpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("touch push subscription: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewClassifiedSince returns classified stories first seen after sinceUnix,
|
||||
// newest first, capped at limit. The digest sender uses it to count and preview
|
||||
// what's new for a subscriber; it carries just the fields a digest needs.
|
||||
func NewClassifiedSince(sinceUnix int64, limit int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT id, headline, source, channel, seen_at
|
||||
FROM stories
|
||||
WHERE classified = 1 AND channel NOT IN ('_discarded', '_duplicate') AND seen_at > ?
|
||||
ORDER BY seen_at DESC
|
||||
LIMIT ?`, sinceUnix, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.ID, &s.Headline, &s.Source, &s.Channel, &s.SeenAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
105
internal/storage/push_test.go
Normal file
105
internal/storage/push_test.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package storage
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestPushSubscriptionLifecycle(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
if err := AddPushSubscription("sub-1", "https://push.example/ep-a", "p256-a", "auth-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A second endpoint for the same user (e.g. a second device).
|
||||
if err := AddPushSubscription("sub-1", "https://push.example/ep-b", "p256-b", "auth-b"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A different user.
|
||||
if err := AddPushSubscription("sub-2", "https://push.example/ep-c", "p256-c", "auth-c"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
subs, err := ListPushSubscriptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(subs) != 3 {
|
||||
t.Fatalf("got %d subscriptions, want 3", len(subs))
|
||||
}
|
||||
|
||||
// Re-subscribing the same endpoint updates keys in place, not a new row.
|
||||
if err := AddPushSubscription("sub-1", "https://push.example/ep-a", "p256-a2", "auth-a2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subs, _ = ListPushSubscriptions()
|
||||
if len(subs) != 3 {
|
||||
t.Fatalf("after re-subscribe got %d rows, want 3 (upsert, not insert)", len(subs))
|
||||
}
|
||||
var epA PushSubscription
|
||||
for _, s := range subs {
|
||||
if s.Endpoint == "https://push.example/ep-a" {
|
||||
epA = s
|
||||
}
|
||||
}
|
||||
if epA.P256dh != "p256-a2" || epA.Auth != "auth-a2" {
|
||||
t.Errorf("re-subscribe did not refresh keys: %+v", epA)
|
||||
}
|
||||
|
||||
if err := RemovePushSubscription("https://push.example/ep-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subs, _ = ListPushSubscriptions()
|
||||
if len(subs) != 2 {
|
||||
t.Fatalf("after remove got %d rows, want 2", len(subs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTouchPushSubscriptionAdvancesWatermark(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
if err := AddPushSubscription("sub-1", "https://push.example/ep", "p", "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subs, _ := ListPushSubscriptions()
|
||||
orig := subs[0].LastNotifiedAt
|
||||
|
||||
want := orig + 5000
|
||||
if err := TouchPushSubscription("https://push.example/ep", want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subs, _ = ListPushSubscriptions()
|
||||
if subs[0].LastNotifiedAt != want {
|
||||
t.Errorf("watermark = %d, want %d", subs[0].LastNotifiedAt, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClassifiedSince(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
now := nowUnix()
|
||||
|
||||
insert := func(guid, headline, source, channel string, classified bool, seenAt int64) {
|
||||
if err := InsertStory(&Story{
|
||||
GUID: guid, Headline: headline, ArticleURL: "https://x/" + guid,
|
||||
Source: source, Channel: channel, Classified: classified, SeenAt: seenAt,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
insert("old", "Old news", "Feed A", "tech", true, now-1000)
|
||||
insert("fresh1", "Fresh one", "Feed A", "tech", true, now-100)
|
||||
insert("fresh2", "Fresh two", "Feed B", "gaming", true, now-50)
|
||||
insert("unclassified", "Pending", "Feed A", "", false, now-10)
|
||||
insert("discarded", "Junk", "Feed A", "_discarded", true, now-10)
|
||||
|
||||
got, err := NewClassifiedSince(now-500, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d stories, want 2 (fresh classified, non-discarded, after watermark)", len(got))
|
||||
}
|
||||
// Newest first.
|
||||
if got[0].GUID != "" && got[0].Headline != "Fresh two" {
|
||||
t.Errorf("first result = %q, want newest 'Fresh two'", got[0].Headline)
|
||||
}
|
||||
if got[0].Source != "Feed B" || got[1].Source != "Feed A" {
|
||||
t.Errorf("unexpected order/sources: %q then %q", got[0].Source, got[1].Source)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func nowUnix() int64 {
|
||||
@@ -33,14 +34,38 @@ func InsertStory(s *Story) error {
|
||||
if s.Paywalled {
|
||||
paywalled = 1
|
||||
}
|
||||
var publishedAt any
|
||||
if s.PublishedAt > 0 {
|
||||
publishedAt = s.PublishedAt
|
||||
}
|
||||
_, err := Get().Exec(
|
||||
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt,
|
||||
`INSERT INTO stories (guid, headline, lede, content, content_chars, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), utf8.RuneCountInString(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetStoryReaderText returns the stored full article text and lede for a single
|
||||
// story, for reader mode. found is false when no story has that id. The query
|
||||
// mirrors the visible-story filter (classified, non-sentinel channel) so the
|
||||
// public /api/article endpoint can't be enumerated to pull the captured bodies
|
||||
// of discarded or not-yet-classified stories that never surface in the UI.
|
||||
func GetStoryReaderText(id int64) (content, lede string, found bool, err error) {
|
||||
var c sql.NullString
|
||||
var l sql.NullString
|
||||
row := Get().QueryRow(
|
||||
`SELECT content, lede FROM stories
|
||||
WHERE id = ? AND classified = 1 AND channel NOT IN ('_discarded', '_duplicate')`, id)
|
||||
switch err = row.Scan(&c, &l); {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
return "", "", false, nil
|
||||
case err != nil:
|
||||
return "", "", false, err
|
||||
}
|
||||
return c.String, l.String, true, nil
|
||||
}
|
||||
|
||||
// nullIfEmpty returns a SQL NULL for empty strings so partial unique indexes
|
||||
// (WHERE col IS NOT NULL) can hold multiple "unknown" rows without conflict.
|
||||
func nullIfEmpty(s string) any {
|
||||
@@ -118,10 +143,16 @@ func GetStoryByGUID(guid string) (*Story, error) {
|
||||
|
||||
// InsertPostLog records that a story was posted to a channel.
|
||||
// Uses OR IGNORE to prevent duplicate posts for the same story+channel.
|
||||
func InsertPostLog(guid, channel, eventID, urlCanonical string) {
|
||||
// forced=true marks the row as a manual !post override so it doesn't count
|
||||
// against the global daily cap.
|
||||
func InsertPostLog(guid, channel, eventID, urlCanonical string, forced bool) {
|
||||
f := 0
|
||||
if forced {
|
||||
f = 1
|
||||
}
|
||||
exec("insert post_log",
|
||||
`INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at) VALUES (?, ?, ?, ?, ?)`,
|
||||
guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix())
|
||||
`INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at, forced) VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix(), f)
|
||||
}
|
||||
|
||||
// GetLastPostTime returns the unix timestamp of the most recent post to a channel.
|
||||
@@ -135,10 +166,11 @@ func GetLastPostTime(channel string) int64 {
|
||||
}
|
||||
|
||||
// CountAllPostsInWindow counts posts across ALL channels within a time window.
|
||||
// Used for the global daily cap.
|
||||
// Used for the global daily cap. Excludes forced (!post) entries so manual
|
||||
// overrides don't eat into the auto-rotation budget.
|
||||
func CountAllPostsInWindow(windowStart int64) int {
|
||||
var count int
|
||||
if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE posted_at >= ?`, windowStart).Scan(&count); err != nil {
|
||||
if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE posted_at >= ? AND forced = 0`, windowStart).Scan(&count); err != nil {
|
||||
slog.Error("CountAllPostsInWindow query failed", "err", err)
|
||||
return 1<<31 - 1 // fail closed: huge number prevents posting
|
||||
}
|
||||
@@ -185,7 +217,7 @@ func GetNewestPostableStory(source string) (*Story, error) {
|
||||
AND channel IS NOT NULL
|
||||
AND channel NOT IN ('_discarded', '_duplicate')
|
||||
AND guid NOT IN (SELECT guid FROM post_log)
|
||||
ORDER BY seen_at DESC
|
||||
ORDER BY COALESCE(published_at, seen_at) DESC
|
||||
LIMIT 1`, source)
|
||||
var s Story
|
||||
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
@@ -208,7 +240,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
|
||||
WHERE classified = 1
|
||||
AND channel = ?
|
||||
AND guid NOT IN (SELECT guid FROM post_log)
|
||||
ORDER BY seen_at DESC
|
||||
ORDER BY COALESCE(published_at, seen_at) DESC
|
||||
LIMIT 1`, channel)
|
||||
var s Story
|
||||
if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil {
|
||||
@@ -225,12 +257,12 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
|
||||
// channels (_discarded, _duplicate) are excluded.
|
||||
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
AND s.channel = ?
|
||||
ORDER BY s.seen_at DESC
|
||||
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
|
||||
LIMIT ? OFFSET ?`, channel, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -239,7 +271,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
@@ -251,13 +283,13 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
|
||||
// channels, newest first. Sentinel channels are excluded.
|
||||
func ListAllClassified(limit, offset int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
ORDER BY s.seen_at DESC
|
||||
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
|
||||
LIMIT ? OFFSET ?`, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -266,7 +298,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
@@ -274,11 +306,60 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListForFeed returns up to `limit` classified stories for the outbound RSS and
|
||||
// JSON feeds, newest first. Unlike the web list queries it also selects the full
|
||||
// `content` and `published_at`, which the feeds need for content:encoded bodies
|
||||
// and real pubDates. channel == "" spans all real channels; otherwise it scopes
|
||||
// to that one channel. Sentinel channels are always excluded.
|
||||
func ListForFeed(channel string, limit int) ([]Story, error) {
|
||||
const cols = `s.id, s.guid, s.headline, s.lede, s.content, s.image_url, s.article_url, s.url_canonical, s.source, s.channel, s.seen_at, s.published_at`
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if channel == "" {
|
||||
rows, err = Get().Query(
|
||||
`SELECT `+cols+`
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
|
||||
LIMIT ?`, limit)
|
||||
} else {
|
||||
rows, err = Get().Query(
|
||||
`SELECT `+cols+`
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
AND s.channel = ?
|
||||
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
|
||||
LIMIT ?`, channel, limit)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
var content, canonical sql.NullString
|
||||
var published sql.NullInt64
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &content, &s.ImageURL, &s.ArticleURL, &canonical, &s.Source, &s.Channel, &s.SeenAt, &published); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Content = content.String
|
||||
s.URLCanonical = canonical.String
|
||||
s.PublishedAt = published.Int64
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListRecentlyPosted returns up to `limit` stories that have been posted to
|
||||
// Matrix, ordered by post time (newest first). Posted is always true.
|
||||
func ListRecentlyPosted(limit int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
|
||||
FROM stories s
|
||||
JOIN post_log p ON p.guid = s.guid
|
||||
GROUP BY s.guid
|
||||
@@ -291,7 +372,7 @@ func ListRecentlyPosted(limit int) ([]Story, error) {
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Posted = true
|
||||
@@ -300,6 +381,72 @@ func ListRecentlyPosted(limit int) ([]Story, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// TrendingStories returns the most-read classified stories (real channels only)
|
||||
// counting views recorded on or after sinceDay (a unix day number), newest-ish
|
||||
// as a tiebreak. Used for the home page's "popular this week" rail. Stories with
|
||||
// no views in the window don't appear, so the rail is empty until reads exist.
|
||||
func TrendingStories(limit int, sinceDay int64) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories s
|
||||
JOIN story_views v ON v.story_id = s.id
|
||||
WHERE s.classified = 1
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
AND v.day >= ?
|
||||
GROUP BY s.id
|
||||
ORDER BY SUM(v.views) DESC, COALESCE(s.published_at, s.seen_at) DESC
|
||||
LIMIT ?`, sinceDay, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// StoryContentLengths returns the captured article text length (in characters)
|
||||
// for the given story ids, as a map keyed by id. Used to estimate a "N min read"
|
||||
// chip. It reads the precomputed content_chars column rather than LENGTH()-ing
|
||||
// the full body, so the hot listing path never scans article text. Ids with no
|
||||
// captured content have content_chars = 0 and are absent from the map (treated
|
||||
// as zero by callers).
|
||||
func StoryContentLengths(ids []int64) map[int64]int {
|
||||
out := make(map[int64]int, len(ids))
|
||||
if len(ids) == 0 {
|
||||
return out
|
||||
}
|
||||
ph, args := intInClause(ids)
|
||||
rows, err := Get().Query(
|
||||
`SELECT id, content_chars FROM stories
|
||||
WHERE id IN (`+ph+`) AND content_chars > 0`, args...)
|
||||
if err != nil {
|
||||
slog.Error("story content lengths query failed", "err", err)
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id, n int64
|
||||
if err := rows.Scan(&id, &n); err != nil {
|
||||
slog.Error("scan content length failed", "err", err)
|
||||
continue
|
||||
}
|
||||
out[id] = int(n)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
slog.Error("story content lengths iteration failed", "err", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CountClassifiedByChannel returns how many classified stories exist for a channel.
|
||||
func CountClassifiedByChannel(channel string) (int, error) {
|
||||
var n int
|
||||
|
||||
275
internal/storage/rank.go
Normal file
275
internal/storage/rank.go
Normal file
@@ -0,0 +1,275 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Ranking weights for the "For you" feed. Bookmarks are a stronger signal of
|
||||
// interest than a plain read, and channel affinity outweighs source affinity
|
||||
// (a reader follows topics more than outlets). Recency keeps the feed fresh so
|
||||
// a heavily-read channel doesn't surface week-old stories over today's news.
|
||||
const (
|
||||
affinityReadWeight = 1.0
|
||||
affinityBookmarkWeight = 3.0
|
||||
|
||||
forYouChannelWeight = 1.0
|
||||
forYouSourceWeight = 0.7
|
||||
forYouRecencyWeight = 0.9
|
||||
|
||||
// forYouCandidatePool bounds how many recent unread stories we score in Go.
|
||||
forYouCandidatePool = 400
|
||||
// forYouRecencyHalfLifeHours sets how fast the recency term decays.
|
||||
forYouRecencyHalfLifeHours = 48.0
|
||||
)
|
||||
|
||||
// userAffinity builds normalized channel and source affinity scores (each 0..1)
|
||||
// for a signed-in user from their read + bookmark history. Bookmarks count for
|
||||
// more than plain reads. total is the number of signal-bearing rows; when it is
|
||||
// zero the user has no history yet and both maps are empty.
|
||||
func userAffinity(sub string) (channel, source map[string]float64, total int, err error) {
|
||||
channel = make(map[string]float64)
|
||||
source = make(map[string]float64)
|
||||
if sub == "" {
|
||||
return channel, source, 0, nil
|
||||
}
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.channel, s.source, u.read_at, u.bookmarked_at
|
||||
FROM user_story_state u
|
||||
JOIN stories s ON s.id = u.story_id
|
||||
WHERE u.user_sub = ?
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')`, sub)
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var ch, src string
|
||||
var readAt, markAt sql.NullInt64
|
||||
if err := rows.Scan(&ch, &src, &readAt, &markAt); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
w := 0.0
|
||||
if readAt.Valid {
|
||||
w += affinityReadWeight
|
||||
}
|
||||
if markAt.Valid {
|
||||
w += affinityBookmarkWeight
|
||||
}
|
||||
if w == 0 {
|
||||
continue
|
||||
}
|
||||
channel[ch] += w
|
||||
source[src] += w
|
||||
total++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
normalizeMax(channel)
|
||||
normalizeMax(source)
|
||||
return channel, source, total, nil
|
||||
}
|
||||
|
||||
// normalizeMax scales a map's values so the largest becomes 1.0, leaving an
|
||||
// all-zero (or empty) map untouched.
|
||||
func normalizeMax(m map[string]float64) {
|
||||
var max float64
|
||||
for _, v := range m {
|
||||
if v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
if max == 0 {
|
||||
return
|
||||
}
|
||||
for k := range m {
|
||||
m[k] /= max
|
||||
}
|
||||
}
|
||||
|
||||
// ForYou returns a personalized ranking of recent, unread stories for a
|
||||
// signed-in user, blending channel affinity, source affinity, and recency. It
|
||||
// returns (nil, nil) when the user has no read/bookmark history yet, so callers
|
||||
// can fall back to the plain latest feed.
|
||||
func ForYou(sub string, limit int) ([]Story, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
chAff, srcAff, total, err := userAffinity(sub)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if total == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at, COALESCE(s.published_at, s.seen_at),
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
AND s.id NOT IN (
|
||||
SELECT story_id FROM user_story_state
|
||||
WHERE user_sub = ? AND read_at IS NOT NULL)
|
||||
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
|
||||
LIMIT ?`, sub, forYouCandidatePool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
now := float64(nowUnix())
|
||||
type scored struct {
|
||||
s Story
|
||||
score float64
|
||||
}
|
||||
var cands []scored
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
var effectiveAt int64
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &effectiveAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ageHours := (now - float64(effectiveAt)) / 3600.0
|
||||
if ageHours < 0 {
|
||||
ageHours = 0
|
||||
}
|
||||
recency := math.Exp(-ageHours / forYouRecencyHalfLifeHours)
|
||||
score := forYouChannelWeight*chAff[s.Channel] +
|
||||
forYouSourceWeight*srcAff[s.Source] +
|
||||
forYouRecencyWeight*recency
|
||||
cands = append(cands, scored{s, score})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Candidates arrive newest-first, and a stable sort keeps that order for
|
||||
// equal scores, so ties break toward the more recent story.
|
||||
sort.SliceStable(cands, func(i, j int) bool { return cands[i].score > cands[j].score })
|
||||
if len(cands) > limit {
|
||||
cands = cands[:limit]
|
||||
}
|
||||
out := make([]Story, 0, len(cands))
|
||||
for _, c := range cands {
|
||||
out = append(out, c.s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// relatedRecencyWindowSeconds bounds "related" results to roughly the same
|
||||
// window the story feed lives in (stories are pruned at 30 days anyway).
|
||||
const relatedRecencyWindowSeconds = 30 * 86400
|
||||
|
||||
// relatedStopwords are high-frequency, low-signal tokens dropped when building
|
||||
// the "related" FTS query so matches key off the story's actual subject matter.
|
||||
var relatedStopwords = map[string]bool{
|
||||
"the": true, "and": true, "for": true, "with": true, "that": true,
|
||||
"this": true, "from": true, "has": true, "have": true, "are": true,
|
||||
"was": true, "were": true, "will": true, "its": true, "into": true,
|
||||
"out": true, "how": true, "why": true, "what": true, "who": true,
|
||||
"you": true, "your": true, "not": true, "but": true, "all": true,
|
||||
"new": true, "now": true, "more": true, "after": true, "over": true,
|
||||
"about": true, "says": true, "said": true, "of": true, "to": true,
|
||||
"in": true, "is": true, "on": true, "at": true, "by": true, "as": true,
|
||||
"an": true, "be": true, "it": true, "or": true, "if": true, "we": true,
|
||||
"he": true, "do": true, "up": true, "so": true, "no": true, "my": true,
|
||||
}
|
||||
|
||||
// RelatedStories returns classified stories textually similar to the given
|
||||
// story, ranked by FTS5 relevance and excluding the story itself. The match
|
||||
// query is an OR of significant tokens from the seed story's headline and lede,
|
||||
// so it surfaces stories that share subject matter rather than requiring every
|
||||
// term (which is what SearchStories' implicit-AND query would demand).
|
||||
func RelatedStories(storyID int64, limit int) ([]Story, error) {
|
||||
if storyID <= 0 || limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var headline, lede sql.NullString
|
||||
err := Get().QueryRow(`SELECT headline, lede FROM stories WHERE id = ?`, storyID).Scan(&headline, &lede)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
fts := buildRelatedFTSQuery(headline.String + " " + lede.String)
|
||||
if fts == "" {
|
||||
return nil, nil
|
||||
}
|
||||
cutoff := nowUnix() - relatedRecencyWindowSeconds
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories_fts f
|
||||
JOIN stories s ON s.id = f.rowid
|
||||
WHERE f.stories_fts MATCH ?
|
||||
AND s.id != ?
|
||||
AND s.classified = 1
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
AND s.seen_at >= ?
|
||||
ORDER BY bm25(stories_fts) ASC, s.seen_at DESC
|
||||
LIMIT ?`, fts, storyID, cutoff, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// buildRelatedFTSQuery tokenizes a story's headline+lede into a de-duplicated,
|
||||
// stopword-filtered OR query of quoted terms, capped so a long lede can't build
|
||||
// a pathological query. Returns "" when nothing usable remains.
|
||||
func buildRelatedFTSQuery(raw string) string {
|
||||
var tokens []string
|
||||
var cur strings.Builder
|
||||
seen := make(map[string]bool)
|
||||
flush := func() {
|
||||
if cur.Len() == 0 {
|
||||
return
|
||||
}
|
||||
t := strings.ToLower(cur.String())
|
||||
cur.Reset()
|
||||
if len([]rune(t)) < 2 || relatedStopwords[t] || seen[t] {
|
||||
return
|
||||
}
|
||||
seen[t] = true
|
||||
tokens = append(tokens, t)
|
||||
}
|
||||
for _, r := range raw {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
|
||||
cur.WriteRune(r)
|
||||
case r > 127:
|
||||
cur.WriteRune(r)
|
||||
default:
|
||||
flush()
|
||||
}
|
||||
}
|
||||
flush()
|
||||
if len(tokens) == 0 {
|
||||
return ""
|
||||
}
|
||||
const maxTokens = 16
|
||||
if len(tokens) > maxTokens {
|
||||
tokens = tokens[:maxTokens]
|
||||
}
|
||||
for i, t := range tokens {
|
||||
tokens[i] = `"` + t + `"`
|
||||
}
|
||||
return strings.Join(tokens, " OR ")
|
||||
}
|
||||
141
internal/storage/rank_test.go
Normal file
141
internal/storage/rank_test.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// seedStoryFull inserts a story with an explicit channel + source so ranking
|
||||
// tests can build a skewed affinity profile.
|
||||
func seedStoryFull(t *testing.T, guid, channel, source, headline, lede string) int64 {
|
||||
t.Helper()
|
||||
s := &Story{
|
||||
GUID: guid,
|
||||
Headline: headline,
|
||||
Lede: lede,
|
||||
ArticleURL: "https://example.com/" + guid,
|
||||
Source: source,
|
||||
Channel: channel,
|
||||
Classified: true,
|
||||
SeenAt: nowUnix(),
|
||||
}
|
||||
if err := InsertStory(s); err != nil {
|
||||
t.Fatalf("insert story %s: %v", guid, err)
|
||||
}
|
||||
var id int64
|
||||
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, guid).Scan(&id); err != nil {
|
||||
t.Fatalf("lookup id %s: %v", guid, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestForYou_LeansToAffinityAndExcludesRead(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
const sub = "sub-1"
|
||||
|
||||
// A skewed history: the user reads/bookmarks gaming stories.
|
||||
g1 := seedStoryFull(t, "g1", "gaming", "GameSite", "Game one", "")
|
||||
g2 := seedStoryFull(t, "g2", "gaming", "GameSite", "Game two", "")
|
||||
// Candidates to rank (all unread until we mark some).
|
||||
gc := seedStoryFull(t, "gc", "gaming", "GameSite", "Fresh gaming story", "")
|
||||
tc := seedStoryFull(t, "tc", "tech", "TechSite", "Fresh tech story", "")
|
||||
|
||||
if err := SetRead(sub, g1, true); err != nil {
|
||||
t.Fatalf("read g1: %v", err)
|
||||
}
|
||||
if err := SetBookmark(sub, g2, true); err != nil {
|
||||
t.Fatalf("bookmark g2: %v", err)
|
||||
}
|
||||
|
||||
got, err := ForYou(sub, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ForYou: %v", err)
|
||||
}
|
||||
if len(got) == 0 {
|
||||
t.Fatal("expected results")
|
||||
}
|
||||
// Affinity leans gaming, so a gaming story tops the list and the tech story
|
||||
// sinks to the bottom. (gc and the unread-but-bookmarked g2 both qualify and
|
||||
// tie on score, so we assert on channel rather than a specific id.)
|
||||
if got[0].Channel != "gaming" {
|
||||
t.Fatalf("expected a gaming story first, got channel %q (id %d)", got[0].Channel, got[0].ID)
|
||||
}
|
||||
if got[len(got)-1].ID != tc {
|
||||
t.Fatalf("expected tech candidate ranked last, got id %d", got[len(got)-1].ID)
|
||||
}
|
||||
// Read stories must never appear.
|
||||
for _, s := range got {
|
||||
if s.ID == g1 {
|
||||
t.Fatalf("read story g1 leaked into ForYou")
|
||||
}
|
||||
}
|
||||
// Sanity: the gaming candidate that was never touched should be present.
|
||||
var sawGC bool
|
||||
for _, s := range got {
|
||||
if s.ID == gc {
|
||||
sawGC = true
|
||||
}
|
||||
}
|
||||
if !sawGC {
|
||||
t.Fatalf("expected unread gaming candidate gc in results")
|
||||
}
|
||||
}
|
||||
|
||||
func TestForYou_NoHistoryReturnsNil(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
seedStoryFull(t, "a", "tech", "TechSite", "Something", "")
|
||||
got, err := ForYou("nobody", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ForYou: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Fatalf("expected nil for a user with no history, got %d rows", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRelatedStories_OnTopicExcludesSelf(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
seed := seedStoryFull(t, "seed", "tech", "TechSite",
|
||||
"Apple unveils new iPhone camera", "The new camera sensor is larger.")
|
||||
rel := seedStoryFull(t, "rel", "tech", "OtherSite",
|
||||
"iPhone camera teardown reveals sensor", "A look at the new iPhone camera.")
|
||||
seedStoryFull(t, "off", "gaming", "GameSite",
|
||||
"Nintendo announces handheld", "A brand new portable console.")
|
||||
|
||||
got, err := RelatedStories(seed, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("RelatedStories: %v", err)
|
||||
}
|
||||
if len(got) == 0 {
|
||||
t.Fatal("expected at least one related story")
|
||||
}
|
||||
for _, s := range got {
|
||||
if s.ID == seed {
|
||||
t.Fatal("seed story returned as its own related")
|
||||
}
|
||||
}
|
||||
if got[0].ID != rel {
|
||||
t.Fatalf("expected the on-topic iPhone story first, got id %d", got[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRelatedFTSQuery(t *testing.T) {
|
||||
// Stopwords and 1-char tokens drop out; the rest become an OR of quoted terms.
|
||||
q := buildRelatedFTSQuery("The new iPhone camera is a big deal")
|
||||
if q == "" {
|
||||
t.Fatal("expected a non-empty query")
|
||||
}
|
||||
for _, bad := range []string{`"the"`, `"is"`, `"new"`} {
|
||||
if strings.Contains(q, bad) {
|
||||
t.Fatalf("stopword leaked into query: %s in %q", bad, q)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{`"iphone"`, `"camera"`} {
|
||||
if !strings.Contains(q, want) {
|
||||
t.Fatalf("expected %s in query %q", want, q)
|
||||
}
|
||||
}
|
||||
if buildRelatedFTSQuery("the a of to") != "" {
|
||||
t.Fatal("expected empty query when only stopwords remain")
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,8 @@ CREATE TABLE IF NOT EXISTS stories (
|
||||
guid TEXT UNIQUE NOT NULL,
|
||||
headline TEXT NOT NULL,
|
||||
lede TEXT,
|
||||
content TEXT,
|
||||
content_chars INTEGER NOT NULL DEFAULT 0,
|
||||
image_url TEXT,
|
||||
article_url TEXT NOT NULL,
|
||||
url_canonical TEXT,
|
||||
@@ -15,7 +17,8 @@ CREATE TABLE IF NOT EXISTS stories (
|
||||
channel TEXT,
|
||||
classified INTEGER NOT NULL DEFAULT 0,
|
||||
paywalled INTEGER NOT NULL DEFAULT 0,
|
||||
seen_at INTEGER NOT NULL
|
||||
seen_at INTEGER NOT NULL,
|
||||
published_at INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS post_log (
|
||||
@@ -24,7 +27,8 @@ CREATE TABLE IF NOT EXISTS post_log (
|
||||
channel TEXT NOT NULL,
|
||||
event_id TEXT,
|
||||
url_canonical TEXT,
|
||||
posted_at INTEGER NOT NULL
|
||||
posted_at INTEGER NOT NULL,
|
||||
forced INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS round_robin_state (
|
||||
@@ -43,15 +47,105 @@ CREATE TABLE IF NOT EXISTS reactions (
|
||||
reacted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS user_preferences (
|
||||
user_sub TEXT PRIMARY KEY,
|
||||
prefs TEXT NOT NULL,
|
||||
username TEXT,
|
||||
email TEXT,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Per-user read + bookmark state for signed-in visitors, keyed by OIDC subject.
|
||||
-- One row carries both signals; a NULL timestamp means "not set". A row with
|
||||
-- both timestamps NULL is meaningless and is pruned, so presence of a row means
|
||||
-- the story is read, bookmarked, or both.
|
||||
CREATE TABLE IF NOT EXISTS user_story_state (
|
||||
user_sub TEXT NOT NULL,
|
||||
story_id INTEGER NOT NULL,
|
||||
read_at INTEGER,
|
||||
bookmarked_at INTEGER,
|
||||
PRIMARY KEY (user_sub, story_id)
|
||||
);
|
||||
|
||||
-- Aggregate web usage. page_views holds running view counts keyed by a coarse
|
||||
-- path label ("home", channel slug, …) and the UTC day, so we can report both
|
||||
-- all-time totals and per-day breakdowns without storing any per-request rows.
|
||||
CREATE TABLE IF NOT EXISTS page_views (
|
||||
path TEXT NOT NULL,
|
||||
day INTEGER NOT NULL, -- unix day (floor(unix / 86400)), UTC
|
||||
views INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (path, day)
|
||||
);
|
||||
|
||||
-- Per-source poll health, one row per configured feed (keyed by source name).
|
||||
-- Written on every poll (success and failure) so the owner-facing dashboard can
|
||||
-- show which feeds are healthy without keeping the poller's in-memory state.
|
||||
-- last_success_at / last_item_count survive failures so a broken feed still
|
||||
-- shows when it last worked and how much it last returned.
|
||||
CREATE TABLE IF NOT EXISTS source_health (
|
||||
source TEXT PRIMARY KEY,
|
||||
last_poll_at INTEGER, -- unix, most recent poll attempt
|
||||
last_success_at INTEGER, -- unix, most recent successful fetch
|
||||
last_error TEXT, -- last failure message ('' when healthy)
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||
last_item_count INTEGER NOT NULL DEFAULT 0, -- items in the last successful fetch
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Web Push subscriptions for signed-in users, one row per browser/device
|
||||
-- endpoint. p256dh + auth are the client's encryption keys (RFC 8291); the
|
||||
-- server needs them to encrypt each push. last_notified_at is the per-endpoint
|
||||
-- digest watermark: the sender only counts stories seen after it. A user can
|
||||
-- have several endpoints (phone, desktop) — each is notified independently.
|
||||
CREATE TABLE IF NOT EXISTS push_subscriptions (
|
||||
endpoint TEXT PRIMARY KEY,
|
||||
user_sub TEXT NOT NULL,
|
||||
p256dh TEXT NOT NULL,
|
||||
auth TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
last_notified_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Privacy-preserving daily unique estimate. visitor is a salted hash of
|
||||
-- IP+User-Agent; the salt rotates every UTC day and is never persisted, so the
|
||||
-- hashes are irreversible and cannot be linked across days. We keep only enough
|
||||
-- to dedup within a single day, then prune.
|
||||
CREATE TABLE IF NOT EXISTS daily_visitors (
|
||||
day INTEGER NOT NULL,
|
||||
visitor TEXT NOT NULL,
|
||||
PRIMARY KEY (day, visitor)
|
||||
);
|
||||
|
||||
-- Per-story read counts, keyed by story id and UTC day. Incremented whenever a
|
||||
-- visitor opens a story in reader mode (/api/article). The day dimension lets
|
||||
-- us surface "popular this week" without a separate rollup; summing across all
|
||||
-- days gives the all-time count shown on cards. Rows age out with their story
|
||||
-- via the foreign-key-less prune in RunMaintenance.
|
||||
CREATE TABLE IF NOT EXISTS story_views (
|
||||
story_id INTEGER NOT NULL,
|
||||
day INTEGER NOT NULL, -- unix day (floor(unix / 86400)), UTC
|
||||
views INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (story_id, day)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_post_log_guid_channel ON post_log(guid, channel);
|
||||
CREATE INDEX IF NOT EXISTS idx_post_log_event_id ON post_log(event_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_post_log_channel_posted ON post_log(channel, posted_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_post_log_canonical_channel ON post_log(url_canonical, channel, posted_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_stories_classified_source ON stories(classified, source);
|
||||
CREATE INDEX IF NOT EXISTS idx_stories_channel_classified_seen ON stories(channel, classified, seen_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_stories_classified_seen ON stories(classified, seen_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_stories_image_url ON stories(image_url) WHERE image_url IS NOT NULL AND image_url <> '';
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_url_canonical ON stories(url_canonical) WHERE url_canonical IS NOT NULL AND url_canonical <> '';
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_source_headline_norm ON stories(source, headline_norm) WHERE headline_norm IS NOT NULL AND headline_norm <> '';
|
||||
CREATE INDEX IF NOT EXISTS idx_reactions_post_guid ON reactions(post_guid);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_reactions_event_id ON reactions(event_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_page_views_day ON page_views(day);
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_visitors_day ON daily_visitors(day);
|
||||
CREATE INDEX IF NOT EXISTS idx_story_views_day ON story_views(day);
|
||||
CREATE INDEX IF NOT EXISTS idx_user_state_bookmarks ON user_story_state(user_sub, bookmarked_at) WHERE bookmarked_at IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_user_state_reads ON user_story_state(user_sub, read_at) WHERE read_at IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_push_sub_user ON push_subscriptions(user_sub);
|
||||
`
|
||||
|
||||
const ftsSchema = `
|
||||
|
||||
149
internal/storage/sourcehealth.go
Normal file
149
internal/storage/sourcehealth.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// SourceHealth is one source's persisted poll health, as written by the poller.
|
||||
type SourceHealth struct {
|
||||
Source string
|
||||
LastPollAt int64 // 0 = never polled
|
||||
LastSuccessAt int64 // 0 = never succeeded
|
||||
LastError string
|
||||
ConsecutiveFailures int
|
||||
LastItemCount int
|
||||
UpdatedAt int64
|
||||
}
|
||||
|
||||
// SourceContentStat is per-source content derived from the stories table (plus
|
||||
// post_log for last-posted), independent of poll health.
|
||||
type SourceContentStat struct {
|
||||
Source string
|
||||
Total int // stories currently retained for this source
|
||||
Classified int // of those, how many are classified
|
||||
Paywalled int // of those, how many are gated
|
||||
LastSeenAt int64 // MAX(seen_at); 0 = none
|
||||
LastPostedAt int64 // MAX(post_log.posted_at) joined by guid; 0 = never posted
|
||||
}
|
||||
|
||||
// RecordPollResult upserts a source's health row after a poll attempt. On
|
||||
// success it clears the error and failure counter and records the item count;
|
||||
// on failure it bumps consecutive_failures and stores the message while
|
||||
// preserving the last successful timestamp and item count. Fire-and-forget:
|
||||
// a failure here must never disrupt polling, so errors are logged and swallowed.
|
||||
func RecordPollResult(source string, ok bool, itemCount int, pollErr error) {
|
||||
now := nowUnix()
|
||||
if ok {
|
||||
exec("record poll success",
|
||||
`INSERT INTO source_health
|
||||
(source, last_poll_at, last_success_at, last_error, consecutive_failures, last_item_count, updated_at)
|
||||
VALUES (?, ?, ?, '', 0, ?, ?)
|
||||
ON CONFLICT(source) DO UPDATE SET
|
||||
last_poll_at = excluded.last_poll_at,
|
||||
last_success_at = excluded.last_success_at,
|
||||
last_error = '',
|
||||
consecutive_failures = 0,
|
||||
last_item_count = excluded.last_item_count,
|
||||
updated_at = excluded.updated_at`,
|
||||
source, now, now, itemCount, now)
|
||||
return
|
||||
}
|
||||
msg := ""
|
||||
if pollErr != nil {
|
||||
msg = pollErr.Error()
|
||||
}
|
||||
exec("record poll failure",
|
||||
`INSERT INTO source_health
|
||||
(source, last_poll_at, last_success_at, last_error, consecutive_failures, last_item_count, updated_at)
|
||||
VALUES (?, ?, NULL, ?, 1, 0, ?)
|
||||
ON CONFLICT(source) DO UPDATE SET
|
||||
last_poll_at = excluded.last_poll_at,
|
||||
last_error = excluded.last_error,
|
||||
consecutive_failures = source_health.consecutive_failures + 1,
|
||||
updated_at = excluded.updated_at`,
|
||||
source, now, msg, now)
|
||||
}
|
||||
|
||||
// ListSourceHealth returns the poll-health row for every source that has been
|
||||
// polled at least once, keyed by source name.
|
||||
func ListSourceHealth() (map[string]SourceHealth, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT source, last_poll_at, last_success_at, last_error,
|
||||
consecutive_failures, last_item_count, updated_at
|
||||
FROM source_health`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list source health: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]SourceHealth)
|
||||
for rows.Next() {
|
||||
var h SourceHealth
|
||||
var lastPoll, lastSuccess sql.NullInt64
|
||||
var lastErr sql.NullString
|
||||
if err := rows.Scan(&h.Source, &lastPoll, &lastSuccess, &lastErr,
|
||||
&h.ConsecutiveFailures, &h.LastItemCount, &h.UpdatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.LastPollAt = lastPoll.Int64
|
||||
h.LastSuccessAt = lastSuccess.Int64
|
||||
h.LastError = lastErr.String
|
||||
out[h.Source] = h
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SourceContentStats derives per-source content counts from the stories table,
|
||||
// with last-posted joined from post_log by guid. Keyed by source name. Only
|
||||
// sources with at least one retained story appear; the caller pads out the rest
|
||||
// from its configured source list.
|
||||
func SourceContentStats() (map[string]SourceContentStat, error) {
|
||||
out := make(map[string]SourceContentStat)
|
||||
|
||||
rows, err := Get().Query(
|
||||
`SELECT source,
|
||||
COUNT(*),
|
||||
COALESCE(SUM(classified), 0),
|
||||
COALESCE(SUM(paywalled), 0),
|
||||
COALESCE(MAX(seen_at), 0)
|
||||
FROM stories
|
||||
GROUP BY source`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("source content stats: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var st SourceContentStat
|
||||
if err := rows.Scan(&st.Source, &st.Total, &st.Classified, &st.Paywalled, &st.LastSeenAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[st.Source] = st
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Last-posted per source, joined by guid. Kept separate so sources with
|
||||
// stories but no posts still appear above with a zero last-posted.
|
||||
prows, err := Get().Query(
|
||||
`SELECT s.source, MAX(p.posted_at)
|
||||
FROM post_log p
|
||||
JOIN stories s ON s.guid = p.guid
|
||||
GROUP BY s.source`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("source last-posted: %w", err)
|
||||
}
|
||||
defer prows.Close()
|
||||
for prows.Next() {
|
||||
var source string
|
||||
var lastPosted int64
|
||||
if err := prows.Scan(&source, &lastPosted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
st := out[source]
|
||||
st.Source = source
|
||||
st.LastPostedAt = lastPosted
|
||||
out[source] = st
|
||||
}
|
||||
return out, prows.Err()
|
||||
}
|
||||
104
internal/storage/sourcehealth_test.go
Normal file
104
internal/storage/sourcehealth_test.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRecordPollResult_SuccessThenFailure(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
// First a successful poll returning 7 items.
|
||||
RecordPollResult("Feed A", true, 7, nil)
|
||||
h, err := ListSourceHealth()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, ok := h["Feed A"]
|
||||
if !ok {
|
||||
t.Fatal("expected a health row for Feed A")
|
||||
}
|
||||
if a.LastItemCount != 7 || a.ConsecutiveFailures != 0 || a.LastError != "" {
|
||||
t.Errorf("after success: items=%d fails=%d err=%q", a.LastItemCount, a.ConsecutiveFailures, a.LastError)
|
||||
}
|
||||
if a.LastPollAt == 0 || a.LastSuccessAt == 0 {
|
||||
t.Errorf("after success: last_poll=%d last_success=%d, want both set", a.LastPollAt, a.LastSuccessAt)
|
||||
}
|
||||
successTS := a.LastSuccessAt
|
||||
|
||||
// Two failures in a row: failures accumulate, but the last-success timestamp
|
||||
// and item count are preserved so the dashboard still shows when it last worked.
|
||||
RecordPollResult("Feed A", false, 0, errors.New("dial tcp: timeout"))
|
||||
RecordPollResult("Feed A", false, 0, errors.New("502 bad gateway"))
|
||||
h, _ = ListSourceHealth()
|
||||
a = h["Feed A"]
|
||||
if a.ConsecutiveFailures != 2 {
|
||||
t.Errorf("consecutive failures = %d, want 2", a.ConsecutiveFailures)
|
||||
}
|
||||
if a.LastError != "502 bad gateway" {
|
||||
t.Errorf("last error = %q, want %q", a.LastError, "502 bad gateway")
|
||||
}
|
||||
if a.LastSuccessAt != successTS {
|
||||
t.Errorf("last success = %d, want preserved %d", a.LastSuccessAt, successTS)
|
||||
}
|
||||
if a.LastItemCount != 7 {
|
||||
t.Errorf("last item count = %d, want preserved 7", a.LastItemCount)
|
||||
}
|
||||
|
||||
// Recovery clears the error and resets the counter.
|
||||
RecordPollResult("Feed A", true, 3, nil)
|
||||
h, _ = ListSourceHealth()
|
||||
a = h["Feed A"]
|
||||
if a.ConsecutiveFailures != 0 || a.LastError != "" || a.LastItemCount != 3 {
|
||||
t.Errorf("after recovery: fails=%d err=%q items=%d", a.ConsecutiveFailures, a.LastError, a.LastItemCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceContentStats(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
// Source X: two classified stories, one paywalled; one posted.
|
||||
InsertStory(&Story{GUID: "x1", Headline: "X one", ArticleURL: "https://x.com/1", Source: "X", Channel: "tech", Classified: true, SeenAt: 100})
|
||||
InsertStory(&Story{GUID: "x2", Headline: "X two", ArticleURL: "https://x.com/2", Source: "X", Channel: "tech", Classified: true, Paywalled: true, SeenAt: 200})
|
||||
// Source Y: one unclassified story, never posted.
|
||||
InsertStory(&Story{GUID: "y1", Headline: "Y one", ArticleURL: "https://y.com/1", Source: "Y", SeenAt: 150})
|
||||
|
||||
InsertPostLog("x1", "tech", "$e1", "", false)
|
||||
// Backdate/forward the post so we can assert MAX(posted_at).
|
||||
Get().Exec(`UPDATE post_log SET posted_at = ? WHERE guid = ?`, int64(500), "x1")
|
||||
|
||||
stats, err := SourceContentStats()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
x := stats["X"]
|
||||
if x.Total != 2 || x.Classified != 2 || x.Paywalled != 1 {
|
||||
t.Errorf("X: total=%d classified=%d paywalled=%d, want 2/2/1", x.Total, x.Classified, x.Paywalled)
|
||||
}
|
||||
if x.LastSeenAt != 200 {
|
||||
t.Errorf("X last seen = %d, want 200", x.LastSeenAt)
|
||||
}
|
||||
if x.LastPostedAt != 500 {
|
||||
t.Errorf("X last posted = %d, want 500", x.LastPostedAt)
|
||||
}
|
||||
|
||||
y := stats["Y"]
|
||||
if y.Total != 1 || y.Classified != 0 || y.Paywalled != 0 {
|
||||
t.Errorf("Y: total=%d classified=%d paywalled=%d, want 1/0/0", y.Total, y.Classified, y.Paywalled)
|
||||
}
|
||||
if y.LastPostedAt != 0 {
|
||||
t.Errorf("Y last posted = %d, want 0 (never posted)", y.LastPostedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSourceHealth_Empty(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
h, err := ListSourceHealth()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(h) != 0 {
|
||||
t.Errorf("expected no health rows, got %d", len(h))
|
||||
}
|
||||
}
|
||||
@@ -61,6 +61,85 @@ func TestInsertStoryAndGUIDSeen(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetStoryReaderText(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
// Reader text is only served for classified, non-sentinel stories — the same
|
||||
// filter the public /api/article endpoint applies so discarded/unclassified
|
||||
// bodies can't be pulled by id enumeration.
|
||||
s := &Story{
|
||||
GUID: "reader-guid",
|
||||
Headline: "Reader Headline",
|
||||
Lede: "Short lede.",
|
||||
Content: "First paragraph.\n\nSecond paragraph.",
|
||||
ArticleURL: "https://example.com/reader",
|
||||
Source: "Src",
|
||||
Channel: "tech",
|
||||
Classified: true,
|
||||
SeenAt: 1700000000,
|
||||
}
|
||||
if err := InsertStory(s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var id int64
|
||||
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, s.GUID).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
content, lede, found, err := GetStoryReaderText(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("expected story to be found")
|
||||
}
|
||||
if content != s.Content {
|
||||
t.Errorf("content = %q, want %q", content, s.Content)
|
||||
}
|
||||
if lede != s.Lede {
|
||||
t.Errorf("lede = %q, want %q", lede, s.Lede)
|
||||
}
|
||||
|
||||
// A story ingested before content capture (no content) is still found, with
|
||||
// an empty content string so the reader falls back to the lede.
|
||||
bare := &Story{GUID: "bare-guid", Headline: "H", Lede: "Only a lede.", ArticleURL: "https://a.com", Source: "S", Channel: "tech", Classified: true, SeenAt: 1}
|
||||
if err := InsertStory(bare); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var bareID int64
|
||||
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, bare.GUID).Scan(&bareID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, lede, found, err = GetStoryReaderText(bareID)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("bare story: found=%v err=%v", found, err)
|
||||
}
|
||||
if content != "" {
|
||||
t.Errorf("bare content = %q, want empty", content)
|
||||
}
|
||||
if lede != "Only a lede." {
|
||||
t.Errorf("bare lede = %q", lede)
|
||||
}
|
||||
|
||||
// Unknown id reports not-found rather than erroring.
|
||||
if _, _, found, err = GetStoryReaderText(999999); err != nil || found {
|
||||
t.Errorf("unknown id: found=%v err=%v, want found=false err=nil", found, err)
|
||||
}
|
||||
|
||||
// A discarded story must not be readable via /api/article, even by direct id.
|
||||
discarded := &Story{GUID: "disc-guid", Headline: "H", Lede: "hidden", Content: "secret body", ArticleURL: "https://d.com", Source: "S", Channel: "_discarded", Classified: true, SeenAt: 1}
|
||||
if err := InsertStory(discarded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var discID int64
|
||||
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, discarded.GUID).Scan(&discID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, found, err = GetStoryReaderText(discID); err != nil || found {
|
||||
t.Errorf("discarded story: found=%v err=%v, want found=false", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInsertStoryDuplicateGUID(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
@@ -98,8 +177,8 @@ func TestMarkClassified(t *testing.T) {
|
||||
func TestPostLogAndLookup(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
InsertPostLog("story-1", "tech", "$event1:example.org", "")
|
||||
InsertPostLog("story-2", "politics", "$event2:example.org", "")
|
||||
InsertPostLog("story-1", "tech", "$event1:example.org", "", false)
|
||||
InsertPostLog("story-2", "politics", "$event2:example.org", "", false)
|
||||
|
||||
guid, channel, found := LookupPostGUID("$event1:example.org")
|
||||
if !found {
|
||||
@@ -193,9 +272,9 @@ func TestMarshalUnmarshalPlatforms(t *testing.T) {
|
||||
func TestInsertPostLog_DuplicateIgnored(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
InsertPostLog("story-1", "tech", "$event1:example.org", "")
|
||||
InsertPostLog("story-1", "tech", "$event1:example.org", "", false)
|
||||
// Same guid+channel should be silently ignored
|
||||
InsertPostLog("story-1", "tech", "$event1-retry:example.org", "")
|
||||
InsertPostLog("story-1", "tech", "$event1-retry:example.org", "", false)
|
||||
|
||||
var count int
|
||||
Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ? AND channel = ?`, "story-1", "tech").Scan(&count)
|
||||
@@ -204,7 +283,7 @@ func TestInsertPostLog_DuplicateIgnored(t *testing.T) {
|
||||
}
|
||||
|
||||
// Different channel should be allowed
|
||||
InsertPostLog("story-1", "politics", "$event2:example.org", "")
|
||||
InsertPostLog("story-1", "politics", "$event2:example.org", "", false)
|
||||
Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ?`, "story-1").Scan(&count)
|
||||
if count != 2 {
|
||||
t.Errorf("expected 2 post_log entries across channels, got %d", count)
|
||||
|
||||
91
internal/storage/story_views_test.go
Normal file
91
internal/storage/story_views_test.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package storage
|
||||
|
||||
import "testing"
|
||||
|
||||
// insertClassified is a tiny helper to seed a visible story and return its id.
|
||||
func insertClassified(t *testing.T, guid, headline, content string) int64 {
|
||||
t.Helper()
|
||||
s := &Story{
|
||||
GUID: guid,
|
||||
Headline: headline,
|
||||
Content: content,
|
||||
ArticleURL: "https://example.com/" + guid,
|
||||
Source: "Example Wire",
|
||||
Channel: "tech",
|
||||
Classified: true,
|
||||
SeenAt: nowUnix(),
|
||||
}
|
||||
if err := InsertStory(s); err != nil {
|
||||
t.Fatalf("insert %s: %v", guid, err)
|
||||
}
|
||||
var id int64
|
||||
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, guid).Scan(&id); err != nil {
|
||||
t.Fatalf("lookup %s: %v", guid, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestStoryViews_TotalsAndTrending(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
a := insertClassified(t, "s-a", "Story A", "some body text here")
|
||||
b := insertClassified(t, "s-b", "Story B", "")
|
||||
c := insertClassified(t, "s-c", "Story C", "another body")
|
||||
|
||||
// A read three times, C twice, B never.
|
||||
RecordStoryView(a)
|
||||
RecordStoryView(a)
|
||||
RecordStoryView(a)
|
||||
RecordStoryView(c)
|
||||
RecordStoryView(c)
|
||||
|
||||
totals := StoryViewTotals([]int64{a, b, c})
|
||||
if totals[a] != 3 {
|
||||
t.Errorf("totals[a] = %d, want 3", totals[a])
|
||||
}
|
||||
if _, ok := totals[b]; ok {
|
||||
t.Errorf("totals[b] present = %v, want absent (zero views)", totals[b])
|
||||
}
|
||||
if totals[c] != 2 {
|
||||
t.Errorf("totals[c] = %d, want 2", totals[c])
|
||||
}
|
||||
|
||||
// Trending over the last week: A (3) before C (2); B is absent (no views).
|
||||
trend, err := TrendingStories(10, unixDay()-6)
|
||||
if err != nil {
|
||||
t.Fatalf("trending: %v", err)
|
||||
}
|
||||
if len(trend) != 2 {
|
||||
t.Fatalf("trending len = %d, want 2 (B has no views)", len(trend))
|
||||
}
|
||||
if trend[0].ID != a || trend[1].ID != c {
|
||||
t.Errorf("trending order = [%d, %d], want [%d, %d]", trend[0].ID, trend[1].ID, a, c)
|
||||
}
|
||||
|
||||
// A window that starts after today excludes everything.
|
||||
future, err := TrendingStories(10, unixDay()+1)
|
||||
if err != nil {
|
||||
t.Fatalf("trending future: %v", err)
|
||||
}
|
||||
if len(future) != 0 {
|
||||
t.Errorf("trending (future window) len = %d, want 0", len(future))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoryContentLengths(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
a := insertClassified(t, "c-a", "Has body", "hello world body")
|
||||
b := insertClassified(t, "c-b", "No body", "")
|
||||
|
||||
lengths := StoryContentLengths([]int64{a, b})
|
||||
if lengths[a] != len("hello world body") {
|
||||
t.Errorf("lengths[a] = %d, want %d", lengths[a], len("hello world body"))
|
||||
}
|
||||
if _, ok := lengths[b]; ok {
|
||||
t.Errorf("lengths[b] present, want absent (empty content)")
|
||||
}
|
||||
if got := StoryContentLengths(nil); len(got) != 0 {
|
||||
t.Errorf("StoryContentLengths(nil) = %v, want empty", got)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ type Story struct {
|
||||
GUID string
|
||||
Headline string
|
||||
Lede string
|
||||
Content string // full article text for reader mode (feed content:encoded or scraped body); "" when unavailable
|
||||
ImageURL string
|
||||
ArticleURL string
|
||||
URLCanonical string
|
||||
@@ -16,5 +17,6 @@ type Story struct {
|
||||
Classified bool
|
||||
Paywalled bool // source article is gated; ArticleURL may be an archive snapshot
|
||||
SeenAt int64
|
||||
Posted bool // true if this story has been posted to Matrix (joined from post_log)
|
||||
PublishedAt int64 // from RSS pubDate/Atom published; 0 if missing. Future dates are clamped to SeenAt at ingest.
|
||||
Posted bool // true if this story has been posted to Matrix (joined from post_log)
|
||||
}
|
||||
|
||||
145
internal/storage/userstate.go
Normal file
145
internal/storage/userstate.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SetRead marks (read=true) or clears (read=false) the read flag for one story
|
||||
// and one signed-in user (keyed by OIDC subject). Read and bookmark state share
|
||||
// a row; clearing the last remaining flag removes the row.
|
||||
func SetRead(sub string, storyID int64, read bool) error {
|
||||
var ts any
|
||||
if read {
|
||||
ts = nowUnix()
|
||||
}
|
||||
_, err := Get().Exec(`
|
||||
INSERT INTO user_story_state (user_sub, story_id, read_at, bookmarked_at)
|
||||
VALUES (?, ?, ?, NULL)
|
||||
ON CONFLICT(user_sub, story_id) DO UPDATE SET read_at = excluded.read_at`,
|
||||
sub, storyID, ts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("set read: %w", err)
|
||||
}
|
||||
return pruneEmptyState(sub, storyID)
|
||||
}
|
||||
|
||||
// SetBookmark adds (on=true) or removes (on=false) a bookmark for one story and
|
||||
// one signed-in user. See SetRead for the shared-row semantics.
|
||||
func SetBookmark(sub string, storyID int64, on bool) error {
|
||||
var ts any
|
||||
if on {
|
||||
ts = nowUnix()
|
||||
}
|
||||
_, err := Get().Exec(`
|
||||
INSERT INTO user_story_state (user_sub, story_id, read_at, bookmarked_at)
|
||||
VALUES (?, ?, NULL, ?)
|
||||
ON CONFLICT(user_sub, story_id) DO UPDATE SET bookmarked_at = excluded.bookmarked_at`,
|
||||
sub, storyID, ts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("set bookmark: %w", err)
|
||||
}
|
||||
return pruneEmptyState(sub, storyID)
|
||||
}
|
||||
|
||||
// pruneEmptyState drops a row once neither flag is set, keeping the table to
|
||||
// only meaningful state.
|
||||
func pruneEmptyState(sub string, storyID int64) error {
|
||||
_, err := Get().Exec(
|
||||
`DELETE FROM user_story_state
|
||||
WHERE user_sub = ? AND story_id = ? AND read_at IS NULL AND bookmarked_at IS NULL`,
|
||||
sub, storyID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prune user state: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UserStoryState reports, for the given story ids, which are read and which are
|
||||
// bookmarked by this user. Both maps contain only ids whose flag is set, so a
|
||||
// missing key means false. An empty sub or id list returns empty maps.
|
||||
func UserStoryState(sub string, ids []int64) (read, bookmarked map[int64]bool, err error) {
|
||||
read = make(map[int64]bool)
|
||||
bookmarked = make(map[int64]bool)
|
||||
if sub == "" || len(ids) == 0 {
|
||||
return read, bookmarked, nil
|
||||
}
|
||||
q := `SELECT story_id, read_at, bookmarked_at FROM user_story_state
|
||||
WHERE user_sub = ? AND story_id IN (` + placeholders(len(ids)) + `)`
|
||||
args := make([]any, 0, len(ids)+1)
|
||||
args = append(args, sub)
|
||||
for _, id := range ids {
|
||||
args = append(args, id)
|
||||
}
|
||||
rows, err := Get().Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("user story state: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var r, b sql.NullInt64
|
||||
if err := rows.Scan(&id, &r, &b); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if r.Valid {
|
||||
read[id] = true
|
||||
}
|
||||
if b.Valid {
|
||||
bookmarked[id] = true
|
||||
}
|
||||
}
|
||||
return read, bookmarked, rows.Err()
|
||||
}
|
||||
|
||||
// ListBookmarks returns the user's bookmarked stories, most recently bookmarked
|
||||
// first, restricted to still-classified stories.
|
||||
func ListBookmarks(sub string, limit, offset int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM user_story_state u
|
||||
JOIN stories s ON s.id = u.story_id
|
||||
WHERE u.user_sub = ?
|
||||
AND u.bookmarked_at IS NOT NULL
|
||||
AND s.classified = 1
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
ORDER BY u.bookmarked_at DESC, u.story_id DESC
|
||||
LIMIT ? OFFSET ?`, sub, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// CountBookmarks returns how many bookmarked stories the user would see on the
|
||||
// bookmarks page. It applies the same classified filter as ListBookmarks so the
|
||||
// "N saved" header can't exceed the number of rendered cards.
|
||||
func CountBookmarks(sub string) (int, error) {
|
||||
var n int
|
||||
err := Get().QueryRow(
|
||||
`SELECT COUNT(*) FROM user_story_state u
|
||||
JOIN stories s ON s.id = u.story_id
|
||||
WHERE u.user_sub = ? AND u.bookmarked_at IS NOT NULL AND s.classified = 1
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')`,
|
||||
sub).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// placeholders returns "?, ?, …" with n slots for an IN clause.
|
||||
func placeholders(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Repeat("?, ", n-1) + "?"
|
||||
}
|
||||
126
internal/storage/userstate_test.go
Normal file
126
internal/storage/userstate_test.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package storage
|
||||
|
||||
import "testing"
|
||||
|
||||
func seedStory(t *testing.T, guid string) int64 {
|
||||
t.Helper()
|
||||
s := &Story{
|
||||
GUID: guid,
|
||||
Headline: "Headline " + guid,
|
||||
Lede: "Lede.",
|
||||
ArticleURL: "https://example.com/" + guid,
|
||||
Source: "Test Source",
|
||||
Channel: "tech",
|
||||
Classified: true,
|
||||
SeenAt: 1700000000,
|
||||
}
|
||||
if err := InsertStory(s); err != nil {
|
||||
t.Fatalf("insert story %s: %v", guid, err)
|
||||
}
|
||||
var id int64
|
||||
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, guid).Scan(&id); err != nil {
|
||||
t.Fatalf("lookup id %s: %v", guid, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestUserStoryState_ReadAndBookmark(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
id1 := seedStory(t, "s1")
|
||||
id2 := seedStory(t, "s2")
|
||||
const sub = "ak-sub-1"
|
||||
|
||||
// Nothing set yet.
|
||||
read, marks, err := UserStoryState(sub, []int64{id1, id2})
|
||||
if err != nil {
|
||||
t.Fatalf("initial state: %v", err)
|
||||
}
|
||||
if len(read) != 0 || len(marks) != 0 {
|
||||
t.Fatalf("expected empty state, got read=%v bookmarked=%v", read, marks)
|
||||
}
|
||||
|
||||
// Mark id1 read, bookmark id1 and id2.
|
||||
if err := SetRead(sub, id1, true); err != nil {
|
||||
t.Fatalf("set read: %v", err)
|
||||
}
|
||||
if err := SetBookmark(sub, id1, true); err != nil {
|
||||
t.Fatalf("set bookmark id1: %v", err)
|
||||
}
|
||||
if err := SetBookmark(sub, id2, true); err != nil {
|
||||
t.Fatalf("set bookmark id2: %v", err)
|
||||
}
|
||||
read, marks, _ = UserStoryState(sub, []int64{id1, id2})
|
||||
if !read[id1] || read[id2] {
|
||||
t.Fatalf("read map wrong: %v", read)
|
||||
}
|
||||
if !marks[id1] || !marks[id2] {
|
||||
t.Fatalf("bookmark map wrong: %v", marks)
|
||||
}
|
||||
// id1 carries both flags in a single row.
|
||||
if !read[id1] || !marks[id1] {
|
||||
t.Fatalf("expected id1 read+bookmarked, read=%v marks=%v", read, marks)
|
||||
}
|
||||
|
||||
// ListBookmarks orders newest bookmark first, tiebreaking on story id so the
|
||||
// result is deterministic within a single second.
|
||||
bm, err := ListBookmarks(sub, 10, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list bookmarks: %v", err)
|
||||
}
|
||||
if len(bm) != 2 {
|
||||
t.Fatalf("expected 2 bookmarks, got %d", len(bm))
|
||||
}
|
||||
if bm[0].ID != id2 || bm[1].ID != id1 {
|
||||
t.Fatalf("bookmark order wrong: %d then %d", bm[0].ID, bm[1].ID)
|
||||
}
|
||||
if n, _ := CountBookmarks(sub); n != 2 {
|
||||
t.Fatalf("count bookmarks = %d, want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserStoryState_ClearRemovesRow(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
id := seedStory(t, "s1")
|
||||
const sub = "ak-sub-1"
|
||||
|
||||
if err := SetRead(sub, id, true); err != nil {
|
||||
t.Fatalf("set read: %v", err)
|
||||
}
|
||||
if err := SetRead(sub, id, false); err != nil {
|
||||
t.Fatalf("unset read: %v", err)
|
||||
}
|
||||
// Row should be gone since neither flag is set.
|
||||
var n int
|
||||
if err := Get().QueryRow(`SELECT COUNT(*) FROM user_story_state WHERE user_sub = ?`, sub).Scan(&n); err != nil {
|
||||
t.Fatalf("count rows: %v", err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatalf("expected row pruned, found %d", n)
|
||||
}
|
||||
|
||||
// Setting read then bookmark then clearing only read must keep the row.
|
||||
_ = SetRead(sub, id, true)
|
||||
_ = SetBookmark(sub, id, true)
|
||||
_ = SetRead(sub, id, false)
|
||||
read, marks, _ := UserStoryState(sub, []int64{id})
|
||||
if read[id] {
|
||||
t.Fatalf("expected not read")
|
||||
}
|
||||
if !marks[id] {
|
||||
t.Fatalf("expected still bookmarked after clearing read")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserStoryState_ScopedBySub(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
id := seedStory(t, "s1")
|
||||
_ = SetBookmark("user-a", id, true)
|
||||
|
||||
read, marks, _ := UserStoryState("user-b", []int64{id})
|
||||
if len(read) != 0 || len(marks) != 0 {
|
||||
t.Fatalf("user-b should see no state, got read=%v marks=%v", read, marks)
|
||||
}
|
||||
if n, _ := CountBookmarks("user-b"); n != 0 {
|
||||
t.Fatalf("user-b count = %d, want 0", n)
|
||||
}
|
||||
}
|
||||
273
internal/web/auth.go
Normal file
273
internal/web/auth.go
Normal file
@@ -0,0 +1,273 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
"golang.org/x/oauth2"
|
||||
|
||||
"pete/internal/config"
|
||||
)
|
||||
|
||||
const (
|
||||
sessionCookie = "pete_session"
|
||||
oauthCookie = "pete_oauth"
|
||||
sessionTTL = 30 * 24 * time.Hour // stay signed in for a month
|
||||
oauthTTL = 10 * time.Minute // login round-trip window
|
||||
)
|
||||
|
||||
// Authenticator holds the OIDC plumbing for optional sign-in via Authentik.
|
||||
type Authenticator struct {
|
||||
oauth *oauth2.Config
|
||||
verifier *oidc.IDTokenVerifier
|
||||
secret []byte
|
||||
}
|
||||
|
||||
// SessionUser is the identity carried in the signed session cookie.
|
||||
type SessionUser struct {
|
||||
Sub string `json:"sub"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Email string `json:"email,omitempty"`
|
||||
Exp int64 `json:"exp"`
|
||||
}
|
||||
|
||||
// Display is the friendly name shown in the header (name, else email, else sub).
|
||||
func (u *SessionUser) Display() string {
|
||||
if u.Name != "" {
|
||||
return u.Name
|
||||
}
|
||||
if u.Email != "" {
|
||||
return u.Email
|
||||
}
|
||||
return u.Sub
|
||||
}
|
||||
|
||||
// Initial is the single uppercase glyph for the avatar bubble.
|
||||
func (u *SessionUser) Initial() string {
|
||||
d := strings.TrimSpace(u.Display())
|
||||
if d == "" {
|
||||
return "?"
|
||||
}
|
||||
return strings.ToUpper(d[:1])
|
||||
}
|
||||
|
||||
// oauthState is the short-lived CSRF/nonce envelope for the login round-trip.
|
||||
type oauthState struct {
|
||||
State string `json:"s"`
|
||||
Nonce string `json:"n"`
|
||||
Next string `json:"r"`
|
||||
Exp int64 `json:"e"`
|
||||
}
|
||||
|
||||
// newAuthenticator performs OIDC discovery against the configured issuer. It is
|
||||
// non-fatal at the call site: if discovery fails (provider down at boot), the
|
||||
// site still serves anonymously.
|
||||
func newAuthenticator(ctx context.Context, cfg config.AuthConfig) (*Authenticator, error) {
|
||||
provider, err := oidc.NewProvider(ctx, strings.TrimRight(cfg.Issuer, "/")+"/")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("oidc discovery: %w", err)
|
||||
}
|
||||
return &Authenticator{
|
||||
oauth: &oauth2.Config{
|
||||
ClientID: cfg.ClientID,
|
||||
ClientSecret: cfg.ClientSecret,
|
||||
RedirectURL: cfg.RedirectURL,
|
||||
Endpoint: provider.Endpoint(),
|
||||
Scopes: []string{oidc.ScopeOpenID, "profile", "email"},
|
||||
},
|
||||
verifier: provider.Verifier(&oidc.Config{ClientID: cfg.ClientID}),
|
||||
secret: []byte(cfg.SessionSecret),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ---- signed-cookie helpers ------------------------------------------------
|
||||
|
||||
// sign returns base64url(payload).base64url(hmac) — a compact stateless token.
|
||||
func (a *Authenticator) sign(payload []byte) string {
|
||||
mac := hmac.New(sha256.New, a.secret)
|
||||
mac.Write(payload)
|
||||
b64 := base64.RawURLEncoding
|
||||
return b64.EncodeToString(payload) + "." + b64.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// verify checks the HMAC and returns the raw payload bytes.
|
||||
func (a *Authenticator) verify(token string) ([]byte, bool) {
|
||||
parts := strings.SplitN(token, ".", 2)
|
||||
if len(parts) != 2 {
|
||||
return nil, false
|
||||
}
|
||||
b64 := base64.RawURLEncoding
|
||||
payload, err := b64.DecodeString(parts[0])
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
want, err := b64.DecodeString(parts[1])
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
mac := hmac.New(sha256.New, a.secret)
|
||||
mac.Write(payload)
|
||||
if subtle.ConstantTimeCompare(want, mac.Sum(nil)) != 1 {
|
||||
return nil, false
|
||||
}
|
||||
return payload, true
|
||||
}
|
||||
|
||||
func randToken() string {
|
||||
b := make([]byte, 32)
|
||||
_, _ = rand.Read(b)
|
||||
return base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
|
||||
// userFromRequest returns the signed-in user, or nil for anonymous visitors.
|
||||
func (a *Authenticator) userFromRequest(r *http.Request) *SessionUser {
|
||||
c, err := r.Cookie(sessionCookie)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
payload, ok := a.verify(c.Value)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var u SessionUser
|
||||
if json.Unmarshal(payload, &u) != nil {
|
||||
return nil
|
||||
}
|
||||
if u.Sub == "" || time.Now().Unix() > u.Exp {
|
||||
return nil
|
||||
}
|
||||
return &u
|
||||
}
|
||||
|
||||
func (a *Authenticator) setCookie(w http.ResponseWriter, name, value string, ttl time.Duration) {
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: name,
|
||||
Value: value,
|
||||
Path: "/",
|
||||
Expires: time.Now().Add(ttl),
|
||||
MaxAge: int(ttl.Seconds()),
|
||||
HttpOnly: true,
|
||||
Secure: true,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
})
|
||||
}
|
||||
|
||||
func (a *Authenticator) clearCookie(w http.ResponseWriter, name string) {
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: name, Value: "", Path: "/", MaxAge: -1,
|
||||
HttpOnly: true, Secure: true, SameSite: http.SameSiteLaxMode,
|
||||
})
|
||||
}
|
||||
|
||||
// ---- handlers -------------------------------------------------------------
|
||||
|
||||
// handleLogin starts the OIDC authorization-code flow.
|
||||
func (a *Authenticator) handleLogin(w http.ResponseWriter, r *http.Request) {
|
||||
st := oauthState{
|
||||
State: randToken(),
|
||||
Nonce: randToken(),
|
||||
Next: safeNext(r.URL.Query().Get("next")),
|
||||
Exp: time.Now().Add(oauthTTL).Unix(),
|
||||
}
|
||||
payload, _ := json.Marshal(st)
|
||||
a.setCookie(w, oauthCookie, a.sign(payload), oauthTTL)
|
||||
http.Redirect(w, r, a.oauth.AuthCodeURL(st.State, oidc.Nonce(st.Nonce)), http.StatusFound)
|
||||
}
|
||||
|
||||
// handleCallback completes the flow: validates state, exchanges the code,
|
||||
// verifies the ID token, and issues a session cookie.
|
||||
func (a *Authenticator) handleCallback(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := r.Cookie(oauthCookie)
|
||||
if err != nil {
|
||||
http.Error(w, "login expired, please try again", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
a.clearCookie(w, oauthCookie)
|
||||
payload, ok := a.verify(c.Value)
|
||||
if !ok {
|
||||
http.Error(w, "invalid login state", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var st oauthState
|
||||
if json.Unmarshal(payload, &st) != nil || time.Now().Unix() > st.Exp {
|
||||
http.Error(w, "login expired, please try again", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if subtle.ConstantTimeCompare([]byte(st.State), []byte(r.URL.Query().Get("state"))) != 1 {
|
||||
http.Error(w, "state mismatch", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
tok, err := a.oauth.Exchange(ctx, r.URL.Query().Get("code"))
|
||||
if err != nil {
|
||||
slog.Error("auth: code exchange failed", "err", err)
|
||||
http.Error(w, "sign-in failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
rawID, ok := tok.Extra("id_token").(string)
|
||||
if !ok {
|
||||
http.Error(w, "no id_token in response", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
idToken, err := a.verifier.Verify(ctx, rawID)
|
||||
if err != nil {
|
||||
slog.Error("auth: id_token verify failed", "err", err)
|
||||
http.Error(w, "sign-in failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if idToken.Nonce != st.Nonce {
|
||||
http.Error(w, "nonce mismatch", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var claims struct {
|
||||
Sub string `json:"sub"`
|
||||
Name string `json:"name"`
|
||||
PreferredUsername string `json:"preferred_username"`
|
||||
Email string `json:"email"`
|
||||
}
|
||||
if err := idToken.Claims(&claims); err != nil {
|
||||
http.Error(w, "sign-in failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
name := claims.Name
|
||||
if name == "" {
|
||||
name = claims.PreferredUsername
|
||||
}
|
||||
u := SessionUser{Sub: claims.Sub, Name: name, Email: claims.Email, Exp: time.Now().Add(sessionTTL).Unix()}
|
||||
sess, _ := json.Marshal(u)
|
||||
a.setCookie(w, sessionCookie, a.sign(sess), sessionTTL)
|
||||
slog.Info("auth: user signed in", "sub", claims.Sub, "name", name)
|
||||
http.Redirect(w, r, st.Next, http.StatusFound)
|
||||
}
|
||||
|
||||
// handleLogout clears the local session. It does not end the upstream Authentik
|
||||
// SSO session (that stays the user's choice).
|
||||
func (a *Authenticator) handleLogout(w http.ResponseWriter, r *http.Request) {
|
||||
a.clearCookie(w, sessionCookie)
|
||||
http.Redirect(w, r, "/", http.StatusFound)
|
||||
}
|
||||
|
||||
// safeNext keeps post-login redirects on-site (no open redirects). It rejects
|
||||
// "//host" and "/\host" — browsers normalize the backslash to a slash, so the
|
||||
// latter would otherwise resolve to a protocol-relative off-site URL.
|
||||
func safeNext(next string) string {
|
||||
if next == "" || !strings.HasPrefix(next, "/") ||
|
||||
strings.HasPrefix(next, "//") || strings.HasPrefix(next, "/\\") {
|
||||
return "/"
|
||||
}
|
||||
return next
|
||||
}
|
||||
55
internal/web/auth_test.go
Normal file
55
internal/web/auth_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package web
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSignVerifyRoundTrip(t *testing.T) {
|
||||
a := &Authenticator{secret: []byte("test-secret-key-at-least-16")}
|
||||
payload := []byte(`{"sub":"abc","exp":123}`)
|
||||
tok := a.sign(payload)
|
||||
|
||||
got, ok := a.verify(tok)
|
||||
if !ok {
|
||||
t.Fatal("verify rejected a freshly signed token")
|
||||
}
|
||||
if string(got) != string(payload) {
|
||||
t.Fatalf("payload mismatch: got %q want %q", got, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyRejectsTamperAndForeignKey(t *testing.T) {
|
||||
a := &Authenticator{secret: []byte("test-secret-key-at-least-16")}
|
||||
tok := a.sign([]byte(`{"sub":"abc"}`))
|
||||
|
||||
// Flip a byte in the payload segment.
|
||||
bad := []byte(tok)
|
||||
bad[0] ^= 0x01
|
||||
if _, ok := a.verify(string(bad)); ok {
|
||||
t.Error("verify accepted a tampered token")
|
||||
}
|
||||
|
||||
// A different key must not validate the same token.
|
||||
other := &Authenticator{secret: []byte("a-totally-different-key-16")}
|
||||
if _, ok := other.verify(tok); ok {
|
||||
t.Error("verify accepted a token signed with a different key")
|
||||
}
|
||||
|
||||
if _, ok := a.verify("not-a-valid-token"); ok {
|
||||
t.Error("verify accepted a malformed token")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeNext(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"": "/",
|
||||
"/tech": "/tech",
|
||||
"/tech?page=2": "/tech?page=2",
|
||||
"//evil.com": "/", // protocol-relative open redirect
|
||||
"https://evil.com": "/", // absolute URL
|
||||
"javascript:alert": "/", // not a path
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := safeNext(in); got != want {
|
||||
t.Errorf("safeNext(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
292
internal/web/feed.go
Normal file
292
internal/web/feed.go
Normal file
@@ -0,0 +1,292 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"html"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// feedItemLimit caps how many stories each outbound feed carries.
|
||||
const feedItemLimit = 50
|
||||
|
||||
// feedCacheControl is a short cache window; feeds refresh a few times an hour.
|
||||
const feedCacheControl = "public, max-age=300"
|
||||
|
||||
const (
|
||||
rssContentNS = "http://purl.org/rss/1.0/modules/content/"
|
||||
rssAtomNS = "http://www.w3.org/2005/Atom"
|
||||
jsonFeedVer = "https://jsonfeed.org/version/1.1"
|
||||
)
|
||||
|
||||
// --- RSS 2.0 ---
|
||||
|
||||
type rssFeedXML struct {
|
||||
XMLName xml.Name `xml:"rss"`
|
||||
Version string `xml:"version,attr"`
|
||||
ContentNS string `xml:"xmlns:content,attr"`
|
||||
AtomNS string `xml:"xmlns:atom,attr"`
|
||||
Channel rssChannel `xml:"channel"`
|
||||
}
|
||||
|
||||
type rssChannel struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Description string `xml:"description"`
|
||||
Language string `xml:"language,omitempty"`
|
||||
LastBuildDate string `xml:"lastBuildDate,omitempty"`
|
||||
Generator string `xml:"generator,omitempty"`
|
||||
AtomLink rssAtomLink `xml:"atom:link"`
|
||||
Items []rssItem `xml:"item"`
|
||||
}
|
||||
|
||||
type rssAtomLink struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
Type string `xml:"type,attr"`
|
||||
}
|
||||
|
||||
type rssItem struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
GUID rssGUID `xml:"guid"`
|
||||
PubDate string `xml:"pubDate,omitempty"`
|
||||
Category string `xml:"category,omitempty"`
|
||||
Description string `xml:"description"`
|
||||
Content *rssContent `xml:"content:encoded,omitempty"`
|
||||
}
|
||||
|
||||
type rssGUID struct {
|
||||
IsPermaLink string `xml:"isPermaLink,attr"`
|
||||
Value string `xml:",chardata"`
|
||||
}
|
||||
|
||||
type rssContent struct {
|
||||
Value string `xml:",cdata"`
|
||||
}
|
||||
|
||||
// --- JSON Feed 1.1 ---
|
||||
|
||||
type jsonFeed struct {
|
||||
Version string `json:"version"`
|
||||
Title string `json:"title"`
|
||||
HomePageURL string `json:"home_page_url"`
|
||||
FeedURL string `json:"feed_url"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Items []jsonFeedItem `json:"items"`
|
||||
}
|
||||
|
||||
type jsonFeedItem struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
ContentText string `json:"content_text,omitempty"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
Image string `json:"image,omitempty"`
|
||||
DatePublished string `json:"date_published,omitempty"`
|
||||
Authors []jsonFeedAuthor `json:"authors,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
}
|
||||
|
||||
type jsonFeedAuthor struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// handleFeedXML serves an RSS 2.0 feed for the given channel ("" = all channels).
|
||||
func (s *Server) handleFeedXML(w http.ResponseWriter, r *http.Request, channel string) {
|
||||
stories, err := storage.ListForFeed(channel, feedItemLimit)
|
||||
if err != nil {
|
||||
slog.Error("web: feed query failed", "channel", channel, "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
base := s.publicBase(r)
|
||||
title, homeURL, desc := s.feedMeta(channel, base)
|
||||
selfURL := base + feedPath(channel, "xml")
|
||||
|
||||
feed := rssFeedXML{
|
||||
Version: "2.0",
|
||||
ContentNS: rssContentNS,
|
||||
AtomNS: rssAtomNS,
|
||||
Channel: rssChannel{
|
||||
Title: title,
|
||||
Link: homeURL,
|
||||
Description: desc,
|
||||
Language: "en",
|
||||
Generator: "Pete",
|
||||
AtomLink: rssAtomLink{Href: selfURL, Rel: "self", Type: "application/rss+xml"},
|
||||
Items: make([]rssItem, 0, len(stories)),
|
||||
},
|
||||
}
|
||||
if len(stories) > 0 {
|
||||
feed.Channel.LastBuildDate = feedTime(stories[0]).UTC().Format(time.RFC1123Z)
|
||||
}
|
||||
for _, st := range stories {
|
||||
link := storyLink(st)
|
||||
item := rssItem{
|
||||
Title: st.Headline,
|
||||
Link: link,
|
||||
GUID: rssGUID{IsPermaLink: "false", Value: st.GUID},
|
||||
PubDate: feedTime(st).UTC().Format(time.RFC1123Z),
|
||||
Category: st.Channel,
|
||||
Description: st.Lede,
|
||||
}
|
||||
if html := contentToHTML(st.Content); html != "" {
|
||||
item.Content = &rssContent{Value: html}
|
||||
}
|
||||
feed.Channel.Items = append(feed.Channel.Items, item)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/rss+xml; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", feedCacheControl)
|
||||
if _, err := w.Write([]byte(xml.Header)); err != nil {
|
||||
return
|
||||
}
|
||||
enc := xml.NewEncoder(w)
|
||||
enc.Indent("", " ")
|
||||
if err := enc.Encode(feed); err != nil {
|
||||
slog.Error("web: rss encode failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handleFeedJSON serves a JSON Feed 1.1 for the given channel ("" = all).
|
||||
func (s *Server) handleFeedJSON(w http.ResponseWriter, r *http.Request, channel string) {
|
||||
stories, err := storage.ListForFeed(channel, feedItemLimit)
|
||||
if err != nil {
|
||||
slog.Error("web: feed query failed", "channel", channel, "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
base := s.publicBase(r)
|
||||
title, homeURL, desc := s.feedMeta(channel, base)
|
||||
|
||||
feed := jsonFeed{
|
||||
Version: jsonFeedVer,
|
||||
Title: title,
|
||||
HomePageURL: homeURL,
|
||||
FeedURL: base + feedPath(channel, "json"),
|
||||
Description: desc,
|
||||
Items: make([]jsonFeedItem, 0, len(stories)),
|
||||
}
|
||||
for _, st := range stories {
|
||||
item := jsonFeedItem{
|
||||
ID: st.GUID,
|
||||
URL: storyLink(st),
|
||||
Title: st.Headline,
|
||||
ContentText: strings.TrimSpace(st.Content),
|
||||
Summary: st.Lede,
|
||||
Image: st.ImageURL,
|
||||
DatePublished: feedTime(st).UTC().Format(time.RFC3339),
|
||||
Tags: []string{st.Channel},
|
||||
}
|
||||
if item.ContentText == "" {
|
||||
item.ContentText = st.Lede
|
||||
}
|
||||
if st.Source != "" {
|
||||
item.Authors = []jsonFeedAuthor{{Name: st.Source}}
|
||||
}
|
||||
feed.Items = append(feed.Items, item)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/feed+json; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", feedCacheControl)
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(feed); err != nil {
|
||||
slog.Error("web: json feed encode failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// feedMeta returns the title, home-page URL, and description for a feed. An
|
||||
// empty channel is the site-wide feed; a slug scopes to that channel.
|
||||
func (s *Server) feedMeta(channel, base string) (title, homeURL, desc string) {
|
||||
site := s.cfg.SiteTitle
|
||||
if site == "" {
|
||||
site = "Pete"
|
||||
}
|
||||
if channel == "" {
|
||||
return site, base + "/", "The latest across every channel."
|
||||
}
|
||||
ch := channelBySlug(channel)
|
||||
return site + " — " + ch.Title, base + "/" + channel, ch.Blurb
|
||||
}
|
||||
|
||||
// feedPath is the site-relative path of a feed, e.g. "/feed.xml" or
|
||||
// "/gaming/feed.json".
|
||||
func feedPath(channel, ext string) string {
|
||||
if channel == "" {
|
||||
return "/feed." + ext
|
||||
}
|
||||
return "/" + channel + "/feed." + ext
|
||||
}
|
||||
|
||||
// storyLink is the canonical outbound link for a story: its canonical URL when
|
||||
// known, otherwise the raw article URL.
|
||||
func storyLink(s storage.Story) string {
|
||||
if s.URLCanonical != "" {
|
||||
return s.URLCanonical
|
||||
}
|
||||
return s.ArticleURL
|
||||
}
|
||||
|
||||
// feedTime picks the best timestamp for a story: its published date when
|
||||
// present, otherwise when Pete first saw it.
|
||||
func feedTime(s storage.Story) time.Time {
|
||||
if s.PublishedAt > 0 {
|
||||
return time.Unix(s.PublishedAt, 0)
|
||||
}
|
||||
return time.Unix(s.SeenAt, 0)
|
||||
}
|
||||
|
||||
// channelBySlug looks up a channel by slug, falling back to a bare title-cased
|
||||
// entry so an unknown slug never panics.
|
||||
func channelBySlug(slug string) Channel {
|
||||
for _, ch := range channels {
|
||||
if ch.Slug == slug {
|
||||
return ch
|
||||
}
|
||||
}
|
||||
return Channel{Slug: slug, Title: slug, Theme: slug}
|
||||
}
|
||||
|
||||
// publicBase returns the site's public origin (scheme://host, no trailing
|
||||
// slash). It prefers the configured base_url and falls back to the request's
|
||||
// host so feeds still carry absolute URLs when base_url is unset.
|
||||
func (s *Server) publicBase(r *http.Request) string {
|
||||
if b := strings.TrimRight(s.cfg.BaseURL, "/"); b != "" {
|
||||
return b
|
||||
}
|
||||
scheme := "http"
|
||||
if r.TLS != nil || strings.EqualFold(r.Header.Get("X-Forwarded-Proto"), "https") {
|
||||
scheme = "https"
|
||||
}
|
||||
return scheme + "://" + r.Host
|
||||
}
|
||||
|
||||
// contentToHTML turns Pete's stored plain-text article body (paragraphs split by
|
||||
// blank lines, soft line breaks inside) into simple, escaped HTML for RSS
|
||||
// content:encoded. Returns "" when there is no body.
|
||||
func contentToHTML(text string) string {
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, para := range strings.Split(text, "\n\n") {
|
||||
para = strings.TrimSpace(para)
|
||||
if para == "" {
|
||||
continue
|
||||
}
|
||||
b.WriteString("<p>")
|
||||
b.WriteString(strings.ReplaceAll(html.EscapeString(para), "\n", "<br>"))
|
||||
b.WriteString("</p>")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
210
internal/web/feed_test.go
Normal file
210
internal/web/feed_test.go
Normal file
@@ -0,0 +1,210 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// TestFeeds exercises the outbound RSS and JSON feeds end to end: a classified
|
||||
// story appears in both formats, carrying its canonical link, full content, and
|
||||
// a real pubDate; a channel-scoped feed excludes stories from other channels.
|
||||
func TestFeeds(t *testing.T) {
|
||||
storage.Close()
|
||||
if err := storage.Init(filepath.Join(t.TempDir(), "feed.db")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { storage.Close() })
|
||||
|
||||
pub := time.Date(2026, 3, 1, 12, 0, 0, 0, time.UTC).Unix()
|
||||
tech := &storage.Story{
|
||||
GUID: "feed-tech-1",
|
||||
Headline: "Chips & Dips: A Tech Tale",
|
||||
Lede: "The lede for the tech story.",
|
||||
Content: "First paragraph.\n\nSecond paragraph with <html> & symbols.",
|
||||
ArticleURL: "https://example.com/tech/story?utm=x",
|
||||
URLCanonical: "https://example.com/tech/story",
|
||||
Source: "Example Wire",
|
||||
Channel: "tech",
|
||||
Classified: true,
|
||||
SeenAt: time.Now().Unix(),
|
||||
PublishedAt: pub,
|
||||
}
|
||||
gaming := &storage.Story{
|
||||
GUID: "feed-gaming-1",
|
||||
Headline: "A Gaming Headline",
|
||||
Lede: "Gaming lede.",
|
||||
ArticleURL: "https://example.com/gaming/story",
|
||||
Source: "Play Wire",
|
||||
Channel: "gaming",
|
||||
Classified: true,
|
||||
SeenAt: time.Now().Unix(),
|
||||
}
|
||||
for _, st := range []*storage.Story{tech, gaming} {
|
||||
if err := storage.InsertStory(st); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
s, err := New(config.WebConfig{SiteTitle: "Pete", ListenAddr: ":0", BaseURL: "https://news.example/"}, nil, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// --- Site-wide RSS ---
|
||||
rw := httptest.NewRecorder()
|
||||
s.handleFeedXML(rw, httptest.NewRequest("GET", "/feed.xml", nil), "")
|
||||
if rw.Code != 200 {
|
||||
t.Fatalf("rss status = %d", rw.Code)
|
||||
}
|
||||
if ct := rw.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/rss+xml") {
|
||||
t.Errorf("rss content-type = %q", ct)
|
||||
}
|
||||
var rss struct {
|
||||
Channel struct {
|
||||
Title string `xml:"title"`
|
||||
AtomLink struct {
|
||||
Href string `xml:"href,attr"`
|
||||
} `xml:"link"`
|
||||
Items []struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
GUID string `xml:"guid"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
Content string `xml:"encoded"` // content:encoded
|
||||
} `xml:"item"`
|
||||
} `xml:"channel"`
|
||||
}
|
||||
if err := xml.Unmarshal(rw.Body.Bytes(), &rss); err != nil {
|
||||
t.Fatalf("rss did not parse as XML: %v\n%s", err, rw.Body.String())
|
||||
}
|
||||
if len(rss.Channel.Items) != 2 {
|
||||
t.Fatalf("rss items = %d, want 2", len(rss.Channel.Items))
|
||||
}
|
||||
// Newest-first: PublishedAt March 2026 vs gaming's seen_at (now) — gaming is
|
||||
// newer, so it sorts first. Find the tech item to assert on.
|
||||
var techItem *struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
GUID string `xml:"guid"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
Content string `xml:"encoded"`
|
||||
}
|
||||
for i := range rss.Channel.Items {
|
||||
if rss.Channel.Items[i].GUID == "feed-tech-1" {
|
||||
techItem = &rss.Channel.Items[i]
|
||||
}
|
||||
}
|
||||
if techItem == nil {
|
||||
t.Fatal("tech item missing from rss")
|
||||
}
|
||||
if techItem.Link != "https://example.com/tech/story" {
|
||||
t.Errorf("rss link = %q, want canonical", techItem.Link)
|
||||
}
|
||||
if techItem.Title != "Chips & Dips: A Tech Tale" {
|
||||
t.Errorf("rss title = %q", techItem.Title)
|
||||
}
|
||||
if !strings.Contains(techItem.Content, "<p>First paragraph.</p>") {
|
||||
t.Errorf("rss content missing paragraph markup: %q", techItem.Content)
|
||||
}
|
||||
if !strings.Contains(techItem.Content, "<html>") {
|
||||
t.Errorf("rss content did not escape embedded markup: %q", techItem.Content)
|
||||
}
|
||||
if !strings.Contains(techItem.PubDate, "2026") {
|
||||
t.Errorf("rss pubDate = %q, want the published date", techItem.PubDate)
|
||||
}
|
||||
if !strings.Contains(rw.Body.String(), `href="https://news.example/feed.xml"`) {
|
||||
t.Errorf("rss missing atom self link with base_url")
|
||||
}
|
||||
|
||||
// --- Site-wide JSON Feed ---
|
||||
rw = httptest.NewRecorder()
|
||||
s.handleFeedJSON(rw, httptest.NewRequest("GET", "/feed.json", nil), "")
|
||||
if rw.Code != 200 {
|
||||
t.Fatalf("json status = %d", rw.Code)
|
||||
}
|
||||
if ct := rw.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/feed+json") {
|
||||
t.Errorf("json content-type = %q", ct)
|
||||
}
|
||||
var jf struct {
|
||||
Version string `json:"version"`
|
||||
FeedURL string `json:"feed_url"`
|
||||
HomePageURL string `json:"home_page_url"`
|
||||
Items []struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
ContentText string `json:"content_text"`
|
||||
Tags []string `json:"tags"`
|
||||
Authors []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"authors"`
|
||||
} `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(rw.Body.Bytes(), &jf); err != nil {
|
||||
t.Fatalf("json feed did not parse: %v", err)
|
||||
}
|
||||
if !strings.Contains(jf.Version, "jsonfeed.org") {
|
||||
t.Errorf("json version = %q", jf.Version)
|
||||
}
|
||||
if jf.FeedURL != "https://news.example/feed.json" {
|
||||
t.Errorf("json feed_url = %q", jf.FeedURL)
|
||||
}
|
||||
if len(jf.Items) != 2 {
|
||||
t.Fatalf("json items = %d, want 2", len(jf.Items))
|
||||
}
|
||||
var jTech *struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
ContentText string `json:"content_text"`
|
||||
Tags []string `json:"tags"`
|
||||
Authors []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"authors"`
|
||||
}
|
||||
for i := range jf.Items {
|
||||
if jf.Items[i].ID == "feed-tech-1" {
|
||||
jTech = &jf.Items[i]
|
||||
}
|
||||
}
|
||||
if jTech == nil {
|
||||
t.Fatal("tech item missing from json feed")
|
||||
}
|
||||
if jTech.URL != "https://example.com/tech/story" {
|
||||
t.Errorf("json url = %q, want canonical", jTech.URL)
|
||||
}
|
||||
if !strings.Contains(jTech.ContentText, "Second paragraph") {
|
||||
t.Errorf("json content_text = %q", jTech.ContentText)
|
||||
}
|
||||
if len(jTech.Tags) != 1 || jTech.Tags[0] != "tech" {
|
||||
t.Errorf("json tags = %v", jTech.Tags)
|
||||
}
|
||||
if len(jTech.Authors) != 1 || jTech.Authors[0].Name != "Example Wire" {
|
||||
t.Errorf("json authors = %v", jTech.Authors)
|
||||
}
|
||||
|
||||
// --- Channel-scoped RSS excludes other channels ---
|
||||
rw = httptest.NewRecorder()
|
||||
s.handleFeedXML(rw, httptest.NewRequest("GET", "/tech/feed.xml", nil), "tech")
|
||||
rss.Channel.Items = nil // xml.Unmarshal appends; clear the site-wide items
|
||||
if err := xml.Unmarshal(rw.Body.Bytes(), &rss); err != nil {
|
||||
t.Fatalf("channel rss parse: %v", err)
|
||||
}
|
||||
if len(rss.Channel.Items) != 1 {
|
||||
t.Fatalf("channel rss items = %d, want 1", len(rss.Channel.Items))
|
||||
}
|
||||
if rss.Channel.Items[0].GUID != "feed-tech-1" {
|
||||
t.Errorf("channel rss wrong item: %q", rss.Channel.Items[0].GUID)
|
||||
}
|
||||
if !strings.Contains(rss.Channel.Title, "Tech") {
|
||||
t.Errorf("channel rss title = %q, want channel name", rss.Channel.Title)
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ const pageSize = 24
|
||||
|
||||
// StoryView is the trimmed-down record used in templates.
|
||||
type StoryView struct {
|
||||
ID int64
|
||||
Headline string
|
||||
Lede string
|
||||
ImageURL string
|
||||
@@ -26,10 +27,13 @@ type StoryView struct {
|
||||
Posted bool
|
||||
Paywalled bool // source is gated and no archive workaround succeeded
|
||||
Channel string // channel slug; also the theme key
|
||||
ReadMins int // estimated reading time in minutes; 0 = unknown (no chip)
|
||||
Views int // all-time reader-mode opens; 0 = none yet (no badge)
|
||||
}
|
||||
|
||||
func toView(s storage.Story) StoryView {
|
||||
return StoryView{
|
||||
ID: s.ID,
|
||||
Headline: s.Headline,
|
||||
Lede: s.Lede,
|
||||
ImageURL: s.ImageURL,
|
||||
@@ -42,24 +46,77 @@ func toView(s storage.Story) StoryView {
|
||||
}
|
||||
}
|
||||
|
||||
// decorate fills the ReadMins and Views fields on one or more groups of story
|
||||
// cards using two cheap batch queries over the whole id set. Called just before
|
||||
// render so every listing (home rails, channel pages, bookmarks, for-you) shows
|
||||
// a reading-time chip and a read count without each list query having to carry
|
||||
// those columns. Best-effort: a metrics hiccup just leaves the badges off.
|
||||
func decorate(groups ...[]StoryView) {
|
||||
var ids []int64
|
||||
seen := make(map[int64]bool)
|
||||
for _, g := range groups {
|
||||
for _, v := range g {
|
||||
if v.ID > 0 && !seen[v.ID] {
|
||||
seen[v.ID] = true
|
||||
ids = append(ids, v.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return
|
||||
}
|
||||
lengths := storage.StoryContentLengths(ids)
|
||||
views := storage.StoryViewTotals(ids)
|
||||
for _, g := range groups {
|
||||
for i := range g {
|
||||
g[i].ReadMins = readMinutes(lengths[g[i].ID])
|
||||
g[i].Views = views[g[i].ID]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readMinutes turns a character count into a rounded minutes-to-read estimate,
|
||||
// assuming ~200 wpm and ~6 characters per word (5-letter words plus a space).
|
||||
// Any non-empty body reads as at least "1 min".
|
||||
func readMinutes(chars int) int {
|
||||
if chars <= 0 {
|
||||
return 0
|
||||
}
|
||||
mins := (chars + 600) / 1200 // round to nearest minute (200 wpm * 6 chars)
|
||||
if mins < 1 {
|
||||
mins = 1
|
||||
}
|
||||
return mins
|
||||
}
|
||||
|
||||
type pageData struct {
|
||||
SiteTitle string
|
||||
Channels []Channel
|
||||
Active string // slug of active channel, "" for landing
|
||||
Weather Weather
|
||||
Phase string // when set, forces day/dawn/dusk/night and disables the clock-driven phase JS
|
||||
SiteTitle string
|
||||
Channels []Channel
|
||||
Active string // slug of active channel, "" for landing
|
||||
Weather Weather
|
||||
Phase string // when set, forces day/dawn/dusk/night and disables the clock-driven phase JS
|
||||
AllSources template.JS // JSON array of {name, channel} for the settings panel
|
||||
AuthEnabled bool // sign-in is available
|
||||
User *SessionUser // nil for anonymous visitors
|
||||
UserPrefs template.JS // signed-in user's stored prefs blob (JSON), or "null"
|
||||
Path string // current request path, for post-login return
|
||||
PostingEnabled bool // false = web-only mode; hide Matrix-posting UI
|
||||
IsAdmin bool // signed-in user is on the admin allowlist (shows /status link)
|
||||
PushEnabled bool // Web Push is configured (shows the notifications toggle to signed-in users)
|
||||
PushPublicKey string // VAPID public key handed to the client to subscribe
|
||||
TTS template.JS // JSON {enabled, default, voices:[{id,label}]} for read-aloud, or "null"
|
||||
}
|
||||
|
||||
type channelPage struct {
|
||||
pageData
|
||||
Channel Channel
|
||||
Stories []StoryView
|
||||
Page int
|
||||
HasPrev bool
|
||||
HasNext bool
|
||||
PrevURL string
|
||||
NextURL string
|
||||
Total int
|
||||
Channel Channel
|
||||
Stories []StoryView
|
||||
Page int
|
||||
HasPrev bool
|
||||
HasNext bool
|
||||
PrevURL string
|
||||
NextURL string
|
||||
Total int
|
||||
}
|
||||
|
||||
type indexPage struct {
|
||||
@@ -67,6 +124,8 @@ type indexPage struct {
|
||||
JustPosted []StoryView
|
||||
Stats []channelStat
|
||||
Latest []StoryView
|
||||
ForYou []StoryView // personalized rail; empty for anon / no-history users
|
||||
Trending []StoryView // most-read this week; empty until reads accumulate
|
||||
}
|
||||
|
||||
type channelStat struct {
|
||||
@@ -77,18 +136,60 @@ type channelStat struct {
|
||||
Total int
|
||||
}
|
||||
|
||||
func (s *Server) base() pageData {
|
||||
return pageData{
|
||||
SiteTitle: s.cfg.SiteTitle,
|
||||
Channels: channels,
|
||||
Weather: currentWeather(time.Now()),
|
||||
}
|
||||
// jsForScript marks already-valid JSON as safe to embed in an inline <script>,
|
||||
// after neutralizing the byte sequences that could break out of that context
|
||||
// (</script>, HTML comments, and the JS line separators U+2028/U+2029). Without
|
||||
// this, a stored prefs blob containing "</script>" could inject markup.
|
||||
func jsForScript(b []byte) template.JS {
|
||||
r := strings.NewReplacer(
|
||||
"<", "\\u003c",
|
||||
">", "\\u003e",
|
||||
"&", "\\u0026",
|
||||
"
", "\\u2028",
|
||||
"
", "\\u2029",
|
||||
)
|
||||
return template.JS(r.Replace(string(b)))
|
||||
}
|
||||
|
||||
func (s *Server) handleIndex(w http.ResponseWriter, _ *http.Request) {
|
||||
func (s *Server) base(r *http.Request) pageData {
|
||||
srcJSON, err := json.Marshal(s.sources)
|
||||
if err != nil {
|
||||
srcJSON = []byte("[]")
|
||||
}
|
||||
d := pageData{
|
||||
SiteTitle: s.cfg.SiteTitle,
|
||||
Channels: channels,
|
||||
Weather: currentWeather(time.Now()),
|
||||
AllSources: jsForScript(srcJSON),
|
||||
AuthEnabled: s.auth != nil,
|
||||
UserPrefs: template.JS("null"),
|
||||
Path: r.URL.Path,
|
||||
PostingEnabled: s.postingEnabled,
|
||||
PushEnabled: s.auth != nil && s.cfg.Push.Enabled,
|
||||
PushPublicKey: s.cfg.Push.VAPIDPublicKey,
|
||||
TTS: template.JS("null"),
|
||||
}
|
||||
if s.tts != nil {
|
||||
d.TTS = s.tts.clientConfig()
|
||||
}
|
||||
if s.auth != nil {
|
||||
if u := s.auth.userFromRequest(r); u != nil {
|
||||
d.User = u
|
||||
d.IsAdmin = s.adminSubs[u.Sub]
|
||||
if blob, err := storage.GetUserPrefs(u.Sub); err == nil && blob != "" {
|
||||
d.UserPrefs = jsForScript([]byte(blob))
|
||||
}
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
|
||||
s.track(r, "home")
|
||||
const (
|
||||
justPostedLimit = 6
|
||||
latestLimit = 16
|
||||
trendingLimit = 8
|
||||
)
|
||||
|
||||
postedRows, err := storage.ListRecentlyPosted(justPostedLimit)
|
||||
@@ -130,16 +231,76 @@ func (s *Server) handleIndex(w http.ResponseWriter, _ *http.Request) {
|
||||
stats = append(stats, stat)
|
||||
}
|
||||
|
||||
// Popular this week: most-read stories over the trailing 7 UTC days. Empty
|
||||
// until reads accumulate, so the section simply doesn't render on a fresh DB.
|
||||
var trending []StoryView
|
||||
if trendRows, err := storage.TrendingStories(trendingLimit, storage.UnixDay()-6); err != nil {
|
||||
slog.Error("web: trending query failed", "err", err)
|
||||
} else {
|
||||
for _, row := range trendRows {
|
||||
trending = append(trending, toView(row))
|
||||
}
|
||||
}
|
||||
|
||||
data := indexPage{
|
||||
pageData: s.base(),
|
||||
pageData: s.base(r),
|
||||
JustPosted: justPosted,
|
||||
Stats: stats,
|
||||
Latest: latest,
|
||||
Trending: trending,
|
||||
}
|
||||
// For signed-in users with some read/bookmark history, lead with a small
|
||||
// personalized rail. ForYou returns nothing for anon / no-history users, so
|
||||
// the section simply doesn't render for them.
|
||||
if s.auth != nil {
|
||||
if u := s.auth.userFromRequest(r); u != nil {
|
||||
const forYouLimit = 8
|
||||
if fy, err := storage.ForYou(u.Sub, forYouLimit); err != nil {
|
||||
slog.Error("web: for-you rail failed", "sub", u.Sub, "err", err)
|
||||
} else {
|
||||
for _, row := range fy {
|
||||
data.ForYou = append(data.ForYou, toView(row))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
decorate(data.Trending, data.ForYou, data.JustPosted, data.Latest)
|
||||
s.render(w, "index", data)
|
||||
}
|
||||
|
||||
type forYouPage struct {
|
||||
pageData
|
||||
Stories []StoryView
|
||||
}
|
||||
|
||||
// handleForYou renders the dedicated personalized feed. Anonymous visitors are
|
||||
// sent to sign-in (the route is only registered when auth is on).
|
||||
func (s *Server) handleForYou(w http.ResponseWriter, r *http.Request) {
|
||||
u := s.auth.userFromRequest(r)
|
||||
if u == nil {
|
||||
http.Redirect(w, r, "/auth/login?next=/for-you", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
s.track(r, "for-you")
|
||||
const limit = 30
|
||||
rows, err := storage.ForYou(u.Sub, limit)
|
||||
if err != nil {
|
||||
slog.Error("web: for-you page failed", "sub", u.Sub, "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
views := make([]StoryView, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
views = append(views, toView(row))
|
||||
}
|
||||
decorate(views)
|
||||
base := s.base(r)
|
||||
base.Active = "for-you"
|
||||
s.render(w, "for-you", forYouPage{pageData: base, Stories: views})
|
||||
}
|
||||
|
||||
func (s *Server) handleChannel(w http.ResponseWriter, r *http.Request, ch Channel) {
|
||||
s.track(r, ch.Slug)
|
||||
page := 1
|
||||
if p := r.URL.Query().Get("page"); p != "" {
|
||||
if n, err := strconv.Atoi(p); err == nil && n > 0 {
|
||||
@@ -162,8 +323,9 @@ func (s *Server) handleChannel(w http.ResponseWriter, r *http.Request, ch Channe
|
||||
views = append(views, toView(r))
|
||||
}
|
||||
total, _ := storage.CountClassifiedByChannel(ch.Slug)
|
||||
decorate(views)
|
||||
|
||||
base := s.base()
|
||||
base := s.base(r)
|
||||
base.Active = ch.Slug
|
||||
data := channelPage{
|
||||
pageData: base,
|
||||
@@ -179,8 +341,67 @@ func (s *Server) handleChannel(w http.ResponseWriter, r *http.Request, ch Channe
|
||||
s.render(w, "channel", data)
|
||||
}
|
||||
|
||||
type bookmarksPage struct {
|
||||
pageData
|
||||
Stories []StoryView
|
||||
Page int
|
||||
HasPrev bool
|
||||
HasNext bool
|
||||
PrevURL string
|
||||
NextURL string
|
||||
Total int
|
||||
}
|
||||
|
||||
// handleBookmarks lists the signed-in user's bookmarked stories. Anonymous
|
||||
// visitors are sent to sign-in (the route is only registered when auth is on).
|
||||
func (s *Server) handleBookmarks(w http.ResponseWriter, r *http.Request) {
|
||||
u := s.auth.userFromRequest(r)
|
||||
if u == nil {
|
||||
http.Redirect(w, r, "/auth/login?next=/bookmarks", http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
s.track(r, "bookmarks")
|
||||
page := 1
|
||||
if p := r.URL.Query().Get("page"); p != "" {
|
||||
if n, err := strconv.Atoi(p); err == nil && n > 0 {
|
||||
page = n
|
||||
}
|
||||
}
|
||||
offset := (page - 1) * pageSize
|
||||
rows, err := storage.ListBookmarks(u.Sub, pageSize+1, offset)
|
||||
if err != nil {
|
||||
slog.Error("web: list bookmarks failed", "sub", u.Sub, "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
hasNext := len(rows) > pageSize
|
||||
if hasNext {
|
||||
rows = rows[:pageSize]
|
||||
}
|
||||
views := make([]StoryView, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
views = append(views, toView(row))
|
||||
}
|
||||
decorate(views)
|
||||
total, _ := storage.CountBookmarks(u.Sub)
|
||||
|
||||
base := s.base(r)
|
||||
base.Active = "bookmarks"
|
||||
data := bookmarksPage{
|
||||
pageData: base,
|
||||
Stories: views,
|
||||
Page: page,
|
||||
HasPrev: page > 1,
|
||||
HasNext: hasNext,
|
||||
PrevURL: fmt.Sprintf("/bookmarks?page=%d", page-1),
|
||||
NextURL: fmt.Sprintf("/bookmarks?page=%d", page+1),
|
||||
Total: total,
|
||||
}
|
||||
s.render(w, "bookmarks", data)
|
||||
}
|
||||
|
||||
var (
|
||||
demoVariants = []string{"rain", "petals", "jacaranda", "motes", "leaves"}
|
||||
demoVariants = []string{"clear", "clouds", "rain", "storm", "hail", "snow", "fog", "haze", "wind", "aurora", "petals", "jacaranda", "motes", "leaves"}
|
||||
demoIntensities = []string{"light", "medium", "heavy"}
|
||||
demoPhases = []string{"day", "dawn", "dusk", "night"}
|
||||
)
|
||||
@@ -210,10 +431,12 @@ func seasonForVariant(v string) string {
|
||||
return "winter"
|
||||
case "petals", "jacaranda":
|
||||
return "spring"
|
||||
case "motes":
|
||||
case "motes", "haze":
|
||||
return "summer"
|
||||
case "leaves":
|
||||
case "leaves", "wind":
|
||||
return "autumn"
|
||||
case "hail":
|
||||
return "winter"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -224,7 +447,7 @@ func (s *Server) handleWeatherDemo(w http.ResponseWriter, r *http.Request) {
|
||||
intensity := pickOr(q.Get("intensity"), demoIntensities, "heavy")
|
||||
phase := pickOr(q.Get("phase"), demoPhases, "day")
|
||||
|
||||
base := s.base()
|
||||
base := s.base(r)
|
||||
base.Weather = Weather{Season: seasonForVariant(variant), Variant: variant, Intensity: intensity}
|
||||
base.Phase = phase
|
||||
|
||||
@@ -268,6 +491,12 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"results": toSearchResults(rows)})
|
||||
}
|
||||
|
||||
// toSearchResults maps stories to the JSON shape shared by /api/search and
|
||||
// /api/related, resolving each story's channel to its display metadata.
|
||||
func toSearchResults(rows []storage.Story) []searchResult {
|
||||
channelByName := make(map[string]Channel, len(channels))
|
||||
for _, ch := range channels {
|
||||
channelByName[ch.Slug] = ch
|
||||
@@ -292,7 +521,80 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
Posted: row.Posted,
|
||||
})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"results": results})
|
||||
return results
|
||||
}
|
||||
|
||||
// handleRelated returns stories textually similar to a given story, for the
|
||||
// "You might also like" rail in reader mode. It is public (reader mode works
|
||||
// for anonymous visitors too); for signed-in users it drops already-read
|
||||
// stories so recommendations stay fresh.
|
||||
func (s *Server) handleRelated(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
id, err := strconv.ParseInt(strings.TrimSpace(r.URL.Query().Get("id")), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
http.Error(w, `{"error":"bad id"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
const relatedLimit = 6
|
||||
// Over-fetch so dropping already-read stories doesn't starve the rail.
|
||||
rows, err := storage.RelatedStories(id, relatedLimit*2)
|
||||
if err != nil {
|
||||
slog.Error("web: related failed", "id", id, "err", err)
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if s.auth != nil && len(rows) > 0 {
|
||||
if u := s.auth.userFromRequest(r); u != nil {
|
||||
ids := make([]int64, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
ids = append(ids, row.ID)
|
||||
}
|
||||
if read, _, err := storage.UserStoryState(u.Sub, ids); err == nil {
|
||||
kept := rows[:0]
|
||||
for _, row := range rows {
|
||||
if !read[row.ID] {
|
||||
kept = append(kept, row)
|
||||
}
|
||||
}
|
||||
rows = kept
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(rows) > relatedLimit {
|
||||
rows = rows[:relatedLimit]
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"results": toSearchResults(rows)})
|
||||
}
|
||||
|
||||
// handleArticle serves the stored full text of a single story for reader mode.
|
||||
// The client already has headline/image/source/time from the card's data
|
||||
// attributes, so this returns just the body text (and the lede as a fallback
|
||||
// for stories ingested before content was captured).
|
||||
func (s *Server) handleArticle(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
id, err := strconv.ParseInt(strings.TrimSpace(r.URL.Query().Get("id")), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
http.Error(w, `{"error":"bad id"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
content, lede, found, err := storage.GetStoryReaderText(id)
|
||||
if err != nil {
|
||||
slog.Error("web: article read failed", "id", id, "err", err)
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
http.Error(w, `{"error":"not found"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
// Opening a story in reader mode is our per-story read signal. found==true
|
||||
// means the id passed the same visibility filter the listings use, so this
|
||||
// can't be driven to inflate counts for hidden/unclassified stories. Fired
|
||||
// in the background so serving the body never waits on the write.
|
||||
go storage.RecordStoryView(id)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"content": content, "lede": lede})
|
||||
}
|
||||
|
||||
func shortTimeAgo(t time.Time) string {
|
||||
|
||||
93
internal/web/metrics.go
Normal file
93
internal/web/metrics.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// track records one page view plus a privacy-preserving unique-visitor token
|
||||
// for the given coarse label ("home", a channel slug, …). It is deliberately
|
||||
// cheap and best-effort: the visitor token is derived synchronously (so it sees
|
||||
// the live request) but the DB writes are fired in the background so page
|
||||
// rendering never waits on them.
|
||||
func (s *Server) track(r *http.Request, label string) {
|
||||
token := s.visitorToken(r)
|
||||
go func() {
|
||||
storage.RecordPageView(label)
|
||||
if token != "" {
|
||||
storage.RecordVisitor(token)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// visitorToken returns an irreversible per-day token for the requester: a
|
||||
// SHA-256 of (daily salt || client IP || User-Agent), truncated. Because the
|
||||
// salt rotates each UTC day and is never stored, tokens cannot be reversed back
|
||||
// to an IP or linked across days. Returns "" if no client IP can be determined.
|
||||
func (s *Server) visitorToken(r *http.Request) string {
|
||||
ip := clientIP(r)
|
||||
if ip == "" {
|
||||
return ""
|
||||
}
|
||||
salt := s.dailySalt()
|
||||
h := sha256.New()
|
||||
h.Write(salt[:])
|
||||
h.Write([]byte(ip))
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte(r.UserAgent()))
|
||||
return hex.EncodeToString(h.Sum(nil)[:16])
|
||||
}
|
||||
|
||||
// dailySalt returns the current day's salt, generating a fresh random one
|
||||
// whenever the UTC day rolls over. The previous salt is discarded, which is
|
||||
// what makes cross-day correlation impossible.
|
||||
func (s *Server) dailySalt() [16]byte {
|
||||
day := time.Now().UTC().Unix() / 86400
|
||||
s.metricsMu.Lock()
|
||||
defer s.metricsMu.Unlock()
|
||||
if day != s.saltDay || isZero(s.salt) {
|
||||
if _, err := rand.Read(s.salt[:]); err != nil {
|
||||
// crypto/rand should never fail; if it does, fall back to a
|
||||
// day-derived constant so we still dedup within the day.
|
||||
slog.Error("web: salt generation failed; using weak fallback", "err", err)
|
||||
for i := range s.salt {
|
||||
s.salt[i] = byte(day >> (uint(i%8) * 8))
|
||||
}
|
||||
}
|
||||
s.saltDay = day
|
||||
}
|
||||
return s.salt
|
||||
}
|
||||
|
||||
func isZero(b [16]byte) bool {
|
||||
for _, x := range b {
|
||||
if x != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// clientIP extracts the requester's IP, honoring the leftmost X-Forwarded-For
|
||||
// entry when present (Pete runs behind a reverse proxy) and otherwise falling
|
||||
// back to RemoteAddr. Only used to derive a one-way hash, never stored.
|
||||
func clientIP(r *http.Request) string {
|
||||
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
|
||||
if first := strings.TrimSpace(strings.Split(xff, ",")[0]); first != "" {
|
||||
return first
|
||||
}
|
||||
}
|
||||
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
return r.RemoteAddr
|
||||
}
|
||||
return host
|
||||
}
|
||||
74
internal/web/prefs_api.go
Normal file
74
internal/web/prefs_api.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// maxPrefsBytes caps the stored blob. Preferences are small (disabled feeds,
|
||||
// a postal code, a couple toggles); this is generous headroom, not a target.
|
||||
const maxPrefsBytes = 64 * 1024
|
||||
|
||||
// handlePrefs serves GET (load) and PUT (save) of the signed-in user's
|
||||
// preferences blob. The blob is opaque JSON owned by the frontend. Both verbs
|
||||
// require a session; anonymous callers get 401 and fall back to localStorage.
|
||||
func (s *Server) handlePrefs(w http.ResponseWriter, r *http.Request) {
|
||||
if s.auth == nil {
|
||||
http.Error(w, "auth disabled", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
u := s.auth.userFromRequest(r)
|
||||
if u == nil {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
http.Error(w, `{"error":"not signed in"}`, http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
blob, err := storage.GetUserPrefs(u.Sub)
|
||||
if err != nil {
|
||||
slog.Error("prefs: load failed", "sub", u.Sub, "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
if blob == "" {
|
||||
_, _ = w.Write([]byte(`{"prefs":null}`))
|
||||
return
|
||||
}
|
||||
// blob is already valid JSON; embed it verbatim.
|
||||
_, _ = w.Write([]byte(`{"prefs":`))
|
||||
_, _ = io.WriteString(w, blob)
|
||||
_, _ = w.Write([]byte(`}`))
|
||||
|
||||
case http.MethodPut:
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, maxPrefsBytes+1))
|
||||
if err != nil {
|
||||
http.Error(w, "read error", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if len(body) > maxPrefsBytes {
|
||||
http.Error(w, "preferences too large", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
if !json.Valid(body) {
|
||||
http.Error(w, "invalid JSON", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := storage.PutUserPrefs(u.Sub, string(body), u.Name, u.Email); err != nil {
|
||||
slog.Error("prefs: save failed", "sub", u.Sub, "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
default:
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
236
internal/web/push_sender.go
Normal file
236
internal/web/push_sender.go
Normal file
@@ -0,0 +1,236 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
"pete/internal/storage"
|
||||
|
||||
webpush "github.com/SherClockHolmes/webpush-go"
|
||||
)
|
||||
|
||||
// digestScan caps how many new stories the sender inspects per subscriber in one
|
||||
// pass. Well past MinStories; the digest only needs a count and one headline.
|
||||
const digestScan = 60
|
||||
|
||||
// pushSendTimeout bounds one push delivery. The endpoint is user-supplied, so a
|
||||
// hostile or dead push service must not be able to wedge the (serial) digest
|
||||
// loop and starve every other subscriber.
|
||||
const pushSendTimeout = 15 * time.Second
|
||||
|
||||
// runPushSender periodically builds and delivers a "N new stories" digest to
|
||||
// each subscriber, respecting their disabled-sources preference. It's started
|
||||
// only when push is configured. Best-effort throughout: a failed send never
|
||||
// stops the loop, and a permanently-gone endpoint is pruned.
|
||||
func (s *Server) runPushSender(ctx context.Context) {
|
||||
interval := time.Duration(s.cfg.Push.IntervalMinutes) * time.Minute
|
||||
if interval <= 0 {
|
||||
interval = 6 * time.Hour
|
||||
}
|
||||
slog.Info("web: push digest sender started", "interval", interval, "min_stories", s.cfg.Push.MinStories)
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.sendDigests()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sendDigests walks every subscription once, notifying those with enough new
|
||||
// stories since their last digest.
|
||||
func (s *Server) sendDigests() {
|
||||
subs, err := storage.ListPushSubscriptions()
|
||||
if err != nil {
|
||||
slog.Error("push: list subscriptions failed", "err", err)
|
||||
return
|
||||
}
|
||||
if len(subs) == 0 {
|
||||
return
|
||||
}
|
||||
// Disabled-source sets are per user; cache within a pass so a user with
|
||||
// several devices only parses prefs once.
|
||||
disabledByUser := make(map[string]map[string]bool)
|
||||
sent, pruned := 0, 0
|
||||
|
||||
for _, sub := range subs {
|
||||
stories, err := storage.NewClassifiedSince(sub.LastNotifiedAt, digestScan)
|
||||
if err != nil {
|
||||
slog.Error("push: scan new stories failed", "sub", sub.UserSub, "err", err)
|
||||
continue
|
||||
}
|
||||
if len(stories) == 0 {
|
||||
continue
|
||||
}
|
||||
disabled, ok := disabledByUser[sub.UserSub]
|
||||
if !ok {
|
||||
disabled = disabledSourcesFor(sub.UserSub)
|
||||
disabledByUser[sub.UserSub] = disabled
|
||||
}
|
||||
count, top := 0, ""
|
||||
for _, st := range stories {
|
||||
if disabled[st.Source] {
|
||||
continue
|
||||
}
|
||||
if count == 0 {
|
||||
top = st.Headline
|
||||
}
|
||||
count++
|
||||
}
|
||||
if count < s.cfg.Push.MinStories {
|
||||
// If a full scan window filled entirely with stories the user has
|
||||
// hidden, the non-hidden count can stay below the threshold forever
|
||||
// while the watermark never advances — the same hidden window is
|
||||
// re-scanned every pass and the subscriber is permanently starved of
|
||||
// digests. When the window was capped (a genuine glut, not a quiet
|
||||
// spell), step the watermark past it so the next pass sees fresh
|
||||
// stories. A non-full window is a real lull; leave it to accumulate.
|
||||
if len(stories) == digestScan {
|
||||
if derr := storage.TouchPushSubscription(sub.Endpoint, stories[0].SeenAt); derr != nil {
|
||||
slog.Error("push: advance watermark past hidden glut failed", "err", derr)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
payload := buildDigestPayload(count, top)
|
||||
gone, err := s.sendPush(sub, payload)
|
||||
if gone {
|
||||
if derr := storage.RemovePushSubscription(sub.Endpoint); derr != nil {
|
||||
slog.Error("push: prune gone subscription failed", "err", derr)
|
||||
} else {
|
||||
pruned++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
slog.Warn("push: send failed", "sub", sub.UserSub, "err", err)
|
||||
continue
|
||||
}
|
||||
// Advance to the newest story this digest actually accounted for, not a
|
||||
// pass-start "now": the loop can run for minutes (one slow endpoint per
|
||||
// send), so stories arriving mid-pass would otherwise be re-counted next
|
||||
// pass. stories[0] is the newest in the scanned window (seen_at DESC).
|
||||
if derr := storage.TouchPushSubscription(sub.Endpoint, stories[0].SeenAt); derr != nil {
|
||||
slog.Error("push: advance watermark failed", "err", derr)
|
||||
}
|
||||
sent++
|
||||
}
|
||||
if sent > 0 || pruned > 0 {
|
||||
slog.Info("push: digest pass complete", "sent", sent, "pruned", pruned, "subscriptions", len(subs))
|
||||
}
|
||||
}
|
||||
|
||||
// buildDigestPayload renders the notification JSON the service worker expects.
|
||||
func buildDigestPayload(count int, top string) []byte {
|
||||
body := fmt.Sprintf("%d new stories", count)
|
||||
if count == 1 {
|
||||
body = "1 new story"
|
||||
}
|
||||
if top != "" {
|
||||
body += ": " + top
|
||||
}
|
||||
b, _ := json.Marshal(map[string]string{
|
||||
"title": "Pete has fresh news",
|
||||
"body": body,
|
||||
"url": "/",
|
||||
"tag": "pete-digest",
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
// sendPush encrypts and delivers one notification. It reports gone=true when the
|
||||
// push service says the endpoint no longer exists (404/410) so the caller can
|
||||
// prune it; err is set for other, likely-transient failures.
|
||||
func (s *Server) sendPush(sub storage.PushSubscription, payload []byte) (gone bool, err error) {
|
||||
resp, err := webpush.SendNotification(payload, &webpush.Subscription{
|
||||
Endpoint: sub.Endpoint,
|
||||
Keys: webpush.Keys{P256dh: sub.P256dh, Auth: sub.Auth},
|
||||
}, &webpush.Options{
|
||||
// The endpoint URL comes from the browser and is attacker-influenceable,
|
||||
// so deliver through the SSRF-guarded client (blocks loopback/RFC1918/
|
||||
// link-local/metadata targets) with a hard timeout — never the library's
|
||||
// unguarded default http.Client.
|
||||
HTTPClient: s.pushClient(),
|
||||
Subscriber: s.cfg.Push.Subject,
|
||||
VAPIDPublicKey: s.cfg.Push.VAPIDPublicKey,
|
||||
VAPIDPrivateKey: s.cfg.Push.VAPIDPrivateKey,
|
||||
TTL: 24 * 60 * 60,
|
||||
Urgency: webpush.UrgencyNormal,
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == 404 || resp.StatusCode == 410 {
|
||||
return true, nil
|
||||
}
|
||||
if resp.StatusCode >= 400 {
|
||||
return false, fmt.Errorf("push service returned %d", resp.StatusCode)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// disabledSourcesFor returns the set of source names a user has hidden, read
|
||||
// from their stored prefs blob. The blob mirrors localStorage: a JSON object
|
||||
// whose "pete.disabledSources.v1" value is itself a JSON string encoding a
|
||||
// {sourceName: true} map. Any parse failure yields an empty (deny-nothing) set.
|
||||
func disabledSourcesFor(sub string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
blob, err := storage.GetUserPrefs(sub)
|
||||
if err != nil || blob == "" {
|
||||
return out
|
||||
}
|
||||
var prefs map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(blob), &prefs); err != nil {
|
||||
return out
|
||||
}
|
||||
raw, ok := prefs["pete.disabledSources.v1"]
|
||||
if !ok {
|
||||
return out
|
||||
}
|
||||
// The value is normally a JSON *string* containing JSON; unwrap that first,
|
||||
// but tolerate a bare object too.
|
||||
inner := []byte(raw)
|
||||
var asStr string
|
||||
if err := json.Unmarshal(raw, &asStr); err == nil {
|
||||
inner = []byte(asStr)
|
||||
}
|
||||
var set map[string]bool
|
||||
if err := json.Unmarshal(inner, &set); err != nil {
|
||||
return out
|
||||
}
|
||||
for name, on := range set {
|
||||
if on {
|
||||
out[name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// pushClient returns the shared SSRF-guarded, timeout-bounded HTTP client used
|
||||
// for Web Push delivery, building it once on first use. The digest loop is a
|
||||
// single goroutine, so the lazy init needs no lock.
|
||||
func (s *Server) pushClient() *http.Client {
|
||||
if s.pushHTTP == nil {
|
||||
s.pushHTTP = safehttp.NewClient(pushSendTimeout)
|
||||
}
|
||||
return s.pushHTTP
|
||||
}
|
||||
|
||||
// StartPushSender launches the digest loop if push is enabled. Safe to call
|
||||
// unconditionally; it's a no-op when push is off.
|
||||
func (s *Server) StartPushSender(ctx context.Context) {
|
||||
if !s.cfg.Push.Enabled || s.auth == nil {
|
||||
return
|
||||
}
|
||||
go s.runPushSender(ctx)
|
||||
}
|
||||
58
internal/web/push_sender_test.go
Normal file
58
internal/web/push_sender_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
func TestDisabledSourcesFor(t *testing.T) {
|
||||
storage.Close()
|
||||
if err := storage.Init(filepath.Join(t.TempDir(), "push.db")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { storage.Close() })
|
||||
|
||||
// The stored blob mirrors localStorage: the disabledSources value is itself a
|
||||
// JSON *string* encoding a {source: true} map (see prefs.js snapshot()).
|
||||
blob := `{"pete.disabledSources.v1":"{\"Feed A\":true,\"Feed B\":false}","pete.weather.loc.v1":"1000-100"}`
|
||||
if err := storage.PutUserPrefs("sub-1", blob, "u", "u@x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := disabledSourcesFor("sub-1")
|
||||
if !got["Feed A"] {
|
||||
t.Error("Feed A should be disabled")
|
||||
}
|
||||
if got["Feed B"] {
|
||||
t.Error("Feed B is false in prefs and must not be treated as disabled")
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Errorf("disabled set = %v, want just {Feed A}", got)
|
||||
}
|
||||
|
||||
// A user with no prefs disables nothing.
|
||||
if len(disabledSourcesFor("nobody")) != 0 {
|
||||
t.Error("unknown user should have an empty disabled set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDigestPayload(t *testing.T) {
|
||||
single := string(buildDigestPayload(1, "Only Story"))
|
||||
if !strings.Contains(single, "1 new story") || strings.Contains(single, "1 new stories") {
|
||||
t.Errorf("singular wording wrong: %s", single)
|
||||
}
|
||||
if !strings.Contains(single, "Only Story") {
|
||||
t.Errorf("top headline missing: %s", single)
|
||||
}
|
||||
|
||||
many := string(buildDigestPayload(7, "Big One"))
|
||||
if !strings.Contains(many, "7 new stories") || !strings.Contains(many, "Big One") {
|
||||
t.Errorf("plural payload wrong: %s", many)
|
||||
}
|
||||
if !strings.Contains(many, `"tag":"pete-digest"`) {
|
||||
t.Errorf("expected digest tag in payload: %s", many)
|
||||
}
|
||||
}
|
||||
115
internal/web/pwa.go
Normal file
115
internal/web/pwa.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// maxPushBodyBytes caps a subscription payload. A PushSubscription JSON is an
|
||||
// endpoint URL plus two short base64 keys — a few hundred bytes — so 4 KiB is
|
||||
// generous headroom for long endpoint URLs.
|
||||
const maxPushBodyBytes = 4096
|
||||
|
||||
// handlePushSubscribe stores the caller's Web Push subscription. The body is the
|
||||
// browser's PushSubscription.toJSON() shape: {endpoint, keys:{p256dh, auth}}.
|
||||
func (s *Server) handlePushSubscribe(w http.ResponseWriter, r *http.Request) {
|
||||
u := s.requireUser(w, r)
|
||||
if u == nil {
|
||||
return
|
||||
}
|
||||
if !s.cfg.Push.Enabled {
|
||||
http.Error(w, `{"error":"push disabled"}`, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Keys struct {
|
||||
P256dh string `json:"p256dh"`
|
||||
Auth string `json:"auth"`
|
||||
} `json:"keys"`
|
||||
}
|
||||
if !decodeStateBodyN(w, r, &req, maxPushBodyBytes) {
|
||||
return
|
||||
}
|
||||
if req.Endpoint == "" || req.Keys.P256dh == "" || req.Keys.Auth == "" {
|
||||
http.Error(w, `{"error":"incomplete subscription"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// The endpoint is delivered to server-side; reject non-http(s) schemes here so
|
||||
// a client can't stash a file:// or gopher:// target. The digest sender's
|
||||
// SSRF-guarded client blocks non-public hosts at dial time, but keeping bad
|
||||
// endpoints out of the table avoids storing garbage in the first place.
|
||||
if err := safehttp.ValidateURL(req.Endpoint); err != nil {
|
||||
http.Error(w, `{"error":"invalid endpoint"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := storage.AddPushSubscription(u.Sub, req.Endpoint, req.Keys.P256dh, req.Keys.Auth); err != nil {
|
||||
slog.Error("push: subscribe failed", "sub", u.Sub, "err", err)
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handlePushUnsubscribe drops the caller's own stored subscription by endpoint.
|
||||
// The delete is scoped to the signed-in user so one account can't remove
|
||||
// another's subscription by presenting its endpoint string.
|
||||
func (s *Server) handlePushUnsubscribe(w http.ResponseWriter, r *http.Request) {
|
||||
u := s.requireUser(w, r)
|
||||
if u == nil {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
}
|
||||
if !decodeStateBodyN(w, r, &req, maxPushBodyBytes) {
|
||||
return
|
||||
}
|
||||
if req.Endpoint == "" {
|
||||
http.Error(w, `{"error":"missing endpoint"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := storage.RemovePushSubscriptionForUser(u.Sub, req.Endpoint); err != nil {
|
||||
slog.Error("push: unsubscribe failed", "sub", u.Sub, "err", err)
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleManifest serves the web app manifest from the embedded static tree. It
|
||||
// lives at the root so the installable scope covers the whole origin.
|
||||
func (s *Server) handleManifest(w http.ResponseWriter, r *http.Request) {
|
||||
s.serveEmbedded(w, r, "manifest.webmanifest", "application/manifest+json; charset=utf-8", "public, max-age=3600")
|
||||
}
|
||||
|
||||
// handleServiceWorker serves /sw.js from the root. Serving it here rather than
|
||||
// under /static/ lets its scope be the whole origin (a worker's default scope
|
||||
// is its own path), and we set Service-Worker-Allowed as a belt-and-braces in
|
||||
// case it's ever moved. no-cache keeps updated workers from being pinned by the
|
||||
// HTTP cache — the browser still byte-compares to decide whether to install.
|
||||
func (s *Server) handleServiceWorker(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Service-Worker-Allowed", "/")
|
||||
s.serveEmbedded(w, r, "sw.js", "text/javascript; charset=utf-8", "no-cache")
|
||||
}
|
||||
|
||||
// serveEmbedded writes a file from the embedded static FS with explicit headers.
|
||||
func (s *Server) serveEmbedded(w http.ResponseWriter, _ *http.Request, name, contentType, cacheControl string) {
|
||||
sub, err := fs.Sub(staticFS, "static")
|
||||
if err != nil {
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
b, err := fs.ReadFile(sub, name)
|
||||
if err != nil {
|
||||
http.NotFound(w, nil)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Cache-Control", cacheControl)
|
||||
_, _ = w.Write(b)
|
||||
}
|
||||
95
internal/web/reader_test.go
Normal file
95
internal/web/reader_test.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// TestReaderCardDataAndArticleAPI exercises the reader-mode path end to end: a
|
||||
// classified story renders a card carrying its id + headline data attributes,
|
||||
// and /api/article returns the stored full text for that id.
|
||||
func TestReaderCardDataAndArticleAPI(t *testing.T) {
|
||||
storage.Close()
|
||||
if err := storage.Init(filepath.Join(t.TempDir(), "reader.db")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { storage.Close() })
|
||||
|
||||
story := &storage.Story{
|
||||
GUID: "reader-e2e",
|
||||
Headline: "A Distinctive Reader Headline",
|
||||
Lede: "The lede.",
|
||||
Content: "Opening paragraph of the piece.\n\nA second paragraph with more detail.",
|
||||
ArticleURL: "https://example.com/story",
|
||||
Source: "Example Wire",
|
||||
Channel: "tech",
|
||||
Classified: true,
|
||||
SeenAt: time.Now().Unix(),
|
||||
}
|
||||
if err := storage.InsertStory(story); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var id int64
|
||||
if err := storage.Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, story.GUID).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s, err := New(config.WebConfig{SiteTitle: "Pete", ListenAddr: ":0"}, nil, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Index page renders the card with the reader data attributes.
|
||||
rw := httptest.NewRecorder()
|
||||
s.handleIndex(rw, httptest.NewRequest("GET", "/", nil))
|
||||
body := rw.Body.String()
|
||||
if rw.Code != 200 {
|
||||
t.Fatalf("index status = %d", rw.Code)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`data-id="` + strconv.FormatInt(id, 10) + `"`,
|
||||
`data-headline="A Distinctive Reader Headline"`,
|
||||
`data-story-card`,
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("index HTML missing %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
// The article endpoint returns the stored content for that id.
|
||||
rw2 := httptest.NewRecorder()
|
||||
s.handleArticle(rw2, httptest.NewRequest("GET", "/api/article?id="+strconv.FormatInt(id, 10), nil))
|
||||
if rw2.Code != 200 {
|
||||
t.Fatalf("article status = %d body=%s", rw2.Code, rw2.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Content string `json:"content"`
|
||||
Lede string `json:"lede"`
|
||||
}
|
||||
if err := json.Unmarshal(rw2.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.Content != story.Content {
|
||||
t.Errorf("content = %q, want %q", got.Content, story.Content)
|
||||
}
|
||||
|
||||
// A bad id is a 400, an unknown id is a 404.
|
||||
for _, tc := range []struct {
|
||||
q string
|
||||
code int
|
||||
}{{"id=0", 400}, {"id=abc", 400}, {"id=999999", 404}} {
|
||||
w := httptest.NewRecorder()
|
||||
s.handleArticle(w, httptest.NewRequest("GET", "/api/article?"+tc.q, nil))
|
||||
if w.Code != tc.code {
|
||||
t.Errorf("article?%s status = %d, want %d", tc.q, w.Code, tc.code)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
@@ -36,20 +37,43 @@ var channels = []Channel{
|
||||
{Slug: "politics", Title: "Politics", Theme: "politics", Emoji: "🏛️", Blurb: "Policy, power, and the news of the day."},
|
||||
{Slug: "eu", Title: "EU", Theme: "eu", Emoji: "🇪🇺", Blurb: "Portugal and the wider European beat. Read-only — these stories don't post to Matrix."},
|
||||
{Slug: "music", Title: "Music", Theme: "music", Emoji: "🎵", Blurb: "Records, scenes, and the artists shaping the sound."},
|
||||
{Slug: "anime", Title: "Anime", Theme: "anime", Emoji: "🌸", Blurb: "Series, studios, and the wider world of anime and manga."},
|
||||
{Slug: "foss", Title: "FOSS", Theme: "foss", Emoji: "🐧", Blurb: "Kernel, distros, and the wider free and open source software world."},
|
||||
{Slug: "kids", Title: "Kids", Theme: "kids", Emoji: "🧒", Blurb: "World news written for younger readers — short, clear, and curious."},
|
||||
{Slug: "finance", Title: "Finance", Theme: "finance", Emoji: "💰", Blurb: "Markets, money, and the business of the world. Read-only — these stories don't post to Matrix."},
|
||||
{Slug: "lego", Title: "LEGO", Theme: "lego", Emoji: "🧱", Blurb: "Sets, releases, and the wider brick-building world."},
|
||||
}
|
||||
|
||||
// SourceInfo is the trimmed view of a configured feed for the settings panel.
|
||||
type SourceInfo struct {
|
||||
Name string `json:"name"`
|
||||
Channel string `json:"channel"`
|
||||
}
|
||||
|
||||
// Server is the HTTP server for Pete's web UI.
|
||||
type Server struct {
|
||||
cfg config.WebConfig
|
||||
srv *http.Server
|
||||
tpls map[string]*template.Template // keyed by page name (e.g. "index", "channel")
|
||||
cfg config.WebConfig
|
||||
sources []SourceInfo
|
||||
postingEnabled bool // false = web-only mode; hides Matrix-posting UI
|
||||
srv *http.Server
|
||||
tpls map[string]*template.Template // keyed by page name (e.g. "index", "channel")
|
||||
auth *Authenticator // nil when sign-in is disabled or unavailable
|
||||
tts *ttsService // nil when server-side read-aloud is disabled
|
||||
adminSubs map[string]bool // OIDC subjects allowed to view /status
|
||||
pushHTTP *http.Client // SSRF-guarded client for Web Push delivery; built lazily by pushClient()
|
||||
|
||||
// Daily-rotated salt for the privacy-preserving unique-visitor estimate.
|
||||
// Guarded by metricsMu; never persisted (see metrics.go).
|
||||
metricsMu sync.Mutex
|
||||
saltDay int64
|
||||
salt [16]byte
|
||||
}
|
||||
|
||||
// New builds the server. Templates are parsed once at startup. Each page
|
||||
// gets its own template set sharing layout.html + _card.html, which avoids
|
||||
// `main` define collisions between pages.
|
||||
func New(cfg config.WebConfig) (*Server, error) {
|
||||
pages := []string{"index", "channel", "weather"}
|
||||
func New(cfg config.WebConfig, sources []config.SourceConfig, postingEnabled bool) (*Server, error) {
|
||||
pages := []string{"index", "channel", "weather", "bookmarks", "for-you", "status"}
|
||||
tpls := make(map[string]*template.Template, len(pages))
|
||||
for _, p := range pages {
|
||||
t, err := template.New("").Funcs(funcs).ParseFS(templateFS,
|
||||
@@ -62,7 +86,48 @@ func New(cfg config.WebConfig) (*Server, error) {
|
||||
}
|
||||
tpls[p] = t
|
||||
}
|
||||
s := &Server{cfg: cfg, tpls: tpls}
|
||||
infos := make([]SourceInfo, 0, len(sources))
|
||||
for _, sc := range sources {
|
||||
if !sc.Enabled || sc.Name == "" {
|
||||
continue
|
||||
}
|
||||
infos = append(infos, SourceInfo{Name: sc.Name, Channel: sc.DirectRoute})
|
||||
}
|
||||
adminSubs := make(map[string]bool, len(cfg.AdminSubs))
|
||||
for _, sub := range cfg.AdminSubs {
|
||||
if sub != "" {
|
||||
adminSubs[sub] = true
|
||||
}
|
||||
}
|
||||
s := &Server{cfg: cfg, sources: infos, postingEnabled: postingEnabled, tpls: tpls, adminSubs: adminSubs}
|
||||
|
||||
// Optional OIDC sign-in (Authentik). Discovery is a network call; if the
|
||||
// provider is unreachable at boot we log and serve anonymously rather than
|
||||
// refusing to start the whole site.
|
||||
if cfg.Auth.Enabled {
|
||||
dctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
auth, err := newAuthenticator(dctx, cfg.Auth)
|
||||
cancel()
|
||||
if err != nil {
|
||||
slog.Error("web: OIDC auth init failed; sign-in disabled", "err", err)
|
||||
} else {
|
||||
s.auth = auth
|
||||
slog.Info("web: OIDC sign-in enabled", "issuer", cfg.Auth.Issuer)
|
||||
}
|
||||
}
|
||||
|
||||
// Optional server-side neural read-aloud (Piper). Signed-in only, so it is
|
||||
// only useful alongside auth; newTTS logs and disables itself on any misconfig.
|
||||
if s.auth != nil {
|
||||
tts, err := newTTS(cfg.TTS)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.tts = tts
|
||||
} else if cfg.TTS.Enabled {
|
||||
slog.Warn("web: TTS enabled but auth is off; read-aloud is signed-in only, so it stays disabled")
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
staticSub, err := fs.Sub(staticFS, "static")
|
||||
@@ -72,20 +137,54 @@ func New(cfg config.WebConfig) (*Server, error) {
|
||||
mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServer(http.FS(staticSub))))
|
||||
mux.HandleFunc("GET /img/{name}", s.handleImg)
|
||||
|
||||
mux.HandleFunc("GET /manifest.webmanifest", s.handleManifest)
|
||||
mux.HandleFunc("GET /sw.js", s.handleServiceWorker)
|
||||
|
||||
mux.HandleFunc("GET /{$}", s.handleIndex)
|
||||
mux.HandleFunc("GET /weather", s.handleWeatherDemo)
|
||||
mux.HandleFunc("GET /api/search", s.handleSearch)
|
||||
mux.HandleFunc("GET /api/article", s.handleArticle)
|
||||
mux.HandleFunc("GET /api/related", s.handleRelated)
|
||||
mux.HandleFunc("GET /feed.xml", func(w http.ResponseWriter, r *http.Request) { s.handleFeedXML(w, r, "") })
|
||||
mux.HandleFunc("GET /feed.json", func(w http.ResponseWriter, r *http.Request) { s.handleFeedJSON(w, r, "") })
|
||||
for _, ch := range channels {
|
||||
ch := ch
|
||||
mux.HandleFunc("GET /"+ch.Slug, func(w http.ResponseWriter, r *http.Request) {
|
||||
s.handleChannel(w, r, ch)
|
||||
})
|
||||
mux.HandleFunc("GET /"+ch.Slug+"/feed.xml", func(w http.ResponseWriter, r *http.Request) { s.handleFeedXML(w, r, ch.Slug) })
|
||||
mux.HandleFunc("GET /"+ch.Slug+"/feed.json", func(w http.ResponseWriter, r *http.Request) { s.handleFeedJSON(w, r, ch.Slug) })
|
||||
}
|
||||
// Public source-health page. The handler renders a trimmed reader view for
|
||||
// everyone and the full diagnostic view only for admins, so it lives outside
|
||||
// the auth block.
|
||||
mux.HandleFunc("GET /status", s.handleStatus)
|
||||
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
})
|
||||
|
||||
if s.auth != nil {
|
||||
mux.HandleFunc("GET /auth/login", s.auth.handleLogin)
|
||||
mux.HandleFunc("GET /auth/callback", s.auth.handleCallback)
|
||||
mux.HandleFunc("GET /auth/logout", s.auth.handleLogout)
|
||||
mux.HandleFunc("GET /api/preferences", s.handlePrefs)
|
||||
mux.HandleFunc("PUT /api/preferences", s.handlePrefs)
|
||||
mux.HandleFunc("POST /api/read", s.handleRead)
|
||||
mux.HandleFunc("POST /api/bookmark", s.handleBookmark)
|
||||
mux.HandleFunc("GET /api/state", s.handleState)
|
||||
mux.HandleFunc("GET /bookmarks", s.handleBookmarks)
|
||||
mux.HandleFunc("GET /for-you", s.handleForYou)
|
||||
if s.cfg.Push.Enabled {
|
||||
mux.HandleFunc("POST /api/push/subscribe", s.handlePushSubscribe)
|
||||
mux.HandleFunc("POST /api/push/unsubscribe", s.handlePushUnsubscribe)
|
||||
}
|
||||
if s.tts != nil {
|
||||
mux.HandleFunc("POST /api/tts", s.handleTTS)
|
||||
}
|
||||
}
|
||||
|
||||
s.srv = &http.Server{
|
||||
Addr: cfg.ListenAddr,
|
||||
Handler: mux,
|
||||
|
||||
@@ -44,6 +44,20 @@ html[data-phase="night"] {
|
||||
@layer base {
|
||||
body { font-family: "Nunito", system-ui, sans-serif; }
|
||||
html { scroll-behavior: smooth; }
|
||||
/* Belt-and-braces: never let stray wide content push horizontal scroll on
|
||||
the whole page (the channel nav scrolls inside itself instead). */
|
||||
html, body { overflow-x: hidden; }
|
||||
}
|
||||
|
||||
@layer components {
|
||||
/* Channel pill row may exceed viewport on mobile once enough channels are
|
||||
configured. Let it scroll inside its own rounded container and hide the
|
||||
scrollbar chrome so it still reads as a clean pill. */
|
||||
.pete-channel-nav {
|
||||
scrollbar-width: none;
|
||||
-ms-overflow-style: none;
|
||||
}
|
||||
.pete-channel-nav::-webkit-scrollbar { display: none; }
|
||||
}
|
||||
|
||||
@layer utilities {
|
||||
@@ -60,24 +74,44 @@ html[data-phase="night"] {
|
||||
.bg-theme-politics { background-color: #e07a5f; }
|
||||
.bg-theme-eu { background-color: #003399; }
|
||||
.bg-theme-music { background-color: #b079d6; }
|
||||
.bg-theme-anime { background-color: #ec5e8a; }
|
||||
.bg-theme-foss { background-color: #d97706; }
|
||||
.bg-theme-kids { background-color: #14b8a6; }
|
||||
.bg-theme-finance { background-color: #10b981; }
|
||||
.bg-theme-lego { background-color: #d01012; }
|
||||
|
||||
.text-theme-gaming { color: #2d8a5a; }
|
||||
.text-theme-tech { color: #2f7fb8; }
|
||||
.text-theme-politics { color: #b8523a; }
|
||||
.text-theme-eu { color: #003399; }
|
||||
.text-theme-music { color: #8a4fb8; }
|
||||
.text-theme-anime { color: #c33a6a; }
|
||||
.text-theme-foss { color: #b8530a; }
|
||||
.text-theme-kids { color: #0f766e; }
|
||||
.text-theme-finance { color: #059669; }
|
||||
.text-theme-lego { color: #b00d0e; }
|
||||
|
||||
.decoration-theme-gaming { text-decoration-color: #4caf7d; }
|
||||
.decoration-theme-tech { text-decoration-color: #5aa9e6; }
|
||||
.decoration-theme-politics { text-decoration-color: #e07a5f; }
|
||||
.decoration-theme-eu { text-decoration-color: #003399; }
|
||||
.decoration-theme-music { text-decoration-color: #b079d6; }
|
||||
.decoration-theme-anime { text-decoration-color: #ec5e8a; }
|
||||
.decoration-theme-foss { text-decoration-color: #d97706; }
|
||||
.decoration-theme-kids { text-decoration-color: #14b8a6; }
|
||||
.decoration-theme-finance { text-decoration-color: #10b981; }
|
||||
.decoration-theme-lego { text-decoration-color: #d01012; }
|
||||
|
||||
.border-theme-gaming { border-color: #4caf7d; }
|
||||
.border-theme-tech { border-color: #5aa9e6; }
|
||||
.border-theme-politics { border-color: #e07a5f; }
|
||||
.border-theme-eu { border-color: #003399; }
|
||||
.border-theme-music { border-color: #b079d6; }
|
||||
.border-theme-anime { border-color: #ec5e8a; }
|
||||
.border-theme-foss { border-color: #d97706; }
|
||||
.border-theme-kids { border-color: #14b8a6; }
|
||||
.border-theme-finance { border-color: #10b981; }
|
||||
.border-theme-lego { border-color: #d01012; }
|
||||
|
||||
/* "Posted to Matrix" glow — soft pulsing aura in the channel's theme color. */
|
||||
.glow-theme-gaming { box-shadow: 0 0 0 2px rgba(76,175,125,0.35), 0 0 24px 4px rgba(76,175,125,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
|
||||
@@ -85,6 +119,11 @@ html[data-phase="night"] {
|
||||
.glow-theme-politics { box-shadow: 0 0 0 2px rgba(224,122,95,0.35), 0 0 24px 4px rgba(224,122,95,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
|
||||
.glow-theme-eu { box-shadow: 0 0 0 2px rgba(0,51,153,0.35), 0 0 24px 4px rgba(0,51,153,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
|
||||
.glow-theme-music { box-shadow: 0 0 0 2px rgba(176,121,214,0.35), 0 0 24px 4px rgba(176,121,214,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
|
||||
.glow-theme-anime { box-shadow: 0 0 0 2px rgba(236,94,138,0.35), 0 0 24px 4px rgba(236,94,138,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
|
||||
.glow-theme-foss { box-shadow: 0 0 0 2px rgba(217,119,6,0.35), 0 0 24px 4px rgba(217,119,6,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
|
||||
.glow-theme-kids { box-shadow: 0 0 0 2px rgba(20,184,166,0.35), 0 0 24px 4px rgba(20,184,166,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
|
||||
.glow-theme-finance { box-shadow: 0 0 0 2px rgba(16,185,129,0.35), 0 0 24px 4px rgba(16,185,129,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
|
||||
.glow-theme-lego { box-shadow: 0 0 0 2px rgba(208,16,18,0.35), 0 0 24px 4px rgba(208,16,18,0.45); animation: pete-glow-pulse 2.4s ease-in-out infinite; }
|
||||
|
||||
@keyframes pete-glow-pulse {
|
||||
0%, 100% { filter: brightness(1); }
|
||||
@@ -130,3 +169,357 @@ html[data-phase="night"] {
|
||||
filter: contrast(1.05);
|
||||
}
|
||||
}
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Feed / reader mode. A focused, one-story-at-a-time overlay driven by
|
||||
reader.js. Left/right arrows page through the stories currently on screen,
|
||||
marking each read as it's shown. Styling leans on the phase palette vars so
|
||||
it recolours with day/night like everything else.
|
||||
---------------------------------------------------------------------------- */
|
||||
@layer components {
|
||||
.pete-reader-backdrop {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: rgba(20, 14, 6, 0.55);
|
||||
backdrop-filter: blur(6px);
|
||||
}
|
||||
html[data-phase="night"] .pete-reader-backdrop { background: rgba(6, 8, 20, 0.65); }
|
||||
|
||||
.pete-reader-shell {
|
||||
position: relative;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
max-width: 44rem;
|
||||
margin: 0 auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding: max(env(safe-area-inset-top), 0.75rem) 1rem 0.75rem;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.pete-reader-bar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 0.75rem;
|
||||
color: #fff;
|
||||
}
|
||||
.pete-reader-progress {
|
||||
font-family: "Fredoka", "Nunito", system-ui, sans-serif;
|
||||
font-weight: 600;
|
||||
font-size: 0.85rem;
|
||||
letter-spacing: 0.02em;
|
||||
text-shadow: 0 1px 2px rgba(0, 0, 0, 0.4);
|
||||
}
|
||||
.pete-reader-nav { display: flex; align-items: center; gap: 0.4rem; }
|
||||
.pete-reader-btn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-width: 2.25rem;
|
||||
height: 2.25rem;
|
||||
padding: 0 0.7rem;
|
||||
border-radius: 9999px;
|
||||
background: rgba(255, 255, 255, 0.14);
|
||||
color: #fff;
|
||||
font-weight: 700;
|
||||
font-size: 0.95rem;
|
||||
border: 2px solid rgba(255, 255, 255, 0.18);
|
||||
cursor: pointer;
|
||||
transition: background 0.15s ease, transform 0.15s ease, opacity 0.15s ease;
|
||||
}
|
||||
.pete-reader-btn:hover { background: rgba(255, 255, 255, 0.28); transform: translateY(-1px); }
|
||||
.pete-reader-btn:disabled { opacity: 0.35; cursor: default; transform: none; }
|
||||
.pete-reader-btn-open { background: var(--accent); border-color: transparent; color: #1c1305; text-decoration: none; }
|
||||
.pete-reader-btn-open:hover { background: var(--accent); filter: brightness(1.08); }
|
||||
/* Active/pressed toolbar toggle (Listen while speaking, Aa while menu open). */
|
||||
.pete-reader-btn[aria-pressed="true"],
|
||||
.pete-reader-btn[aria-expanded="true"] {
|
||||
background: var(--accent);
|
||||
border-color: transparent;
|
||||
color: #1c1305;
|
||||
}
|
||||
|
||||
/* Text-options popover, anchored under the Aa button in the reader bar. */
|
||||
.pete-reader-type { position: relative; display: inline-flex; }
|
||||
.pete-reader-typemenu[hidden] { display: none; }
|
||||
.pete-reader-typemenu {
|
||||
position: absolute;
|
||||
top: calc(100% + 0.5rem);
|
||||
right: 0;
|
||||
z-index: 10;
|
||||
width: 15rem;
|
||||
padding: 0.75rem;
|
||||
border-radius: 1rem;
|
||||
background: var(--card);
|
||||
color: var(--ink);
|
||||
border: 2px solid rgba(0, 0, 0, 0.08);
|
||||
box-shadow: 0 10px 30px rgba(20, 14, 6, 0.28);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.6rem;
|
||||
}
|
||||
html[data-phase="night"] .pete-reader-typemenu { border-color: rgba(255, 255, 255, 0.12); }
|
||||
.pete-reader-typerow { display: flex; align-items: center; justify-content: space-between; gap: 0.6rem; }
|
||||
.pete-reader-typelabel { font-size: 0.8rem; font-weight: 700; opacity: 0.7; }
|
||||
.pete-reader-typebtns { display: inline-flex; gap: 0.25rem; }
|
||||
.pete-reader-typebtns button {
|
||||
min-width: 2rem;
|
||||
height: 2rem;
|
||||
padding: 0 0.55rem;
|
||||
border-radius: 0.6rem;
|
||||
background: rgba(20, 14, 6, 0.06);
|
||||
color: var(--ink);
|
||||
font-weight: 700;
|
||||
font-size: 0.82rem;
|
||||
border: 2px solid transparent;
|
||||
cursor: pointer;
|
||||
transition: background 0.15s ease;
|
||||
}
|
||||
html[data-phase="night"] .pete-reader-typebtns button { background: rgba(255, 255, 255, 0.08); }
|
||||
.pete-reader-typebtns button:hover { background: rgba(20, 14, 6, 0.12); }
|
||||
.pete-reader-typebtns button.is-active {
|
||||
background: var(--accent);
|
||||
color: #1c1305;
|
||||
border-color: transparent;
|
||||
}
|
||||
.pete-reader-voicerow[hidden] { display: none; }
|
||||
.pete-reader-voice {
|
||||
font: inherit;
|
||||
font-size: 0.8rem;
|
||||
font-weight: 600;
|
||||
padding: 0.25rem 0.4rem;
|
||||
border-radius: 0.55rem;
|
||||
border: 1px solid rgba(20, 14, 6, 0.18);
|
||||
background: rgba(20, 14, 6, 0.06);
|
||||
color: inherit;
|
||||
max-width: 11rem;
|
||||
cursor: pointer;
|
||||
}
|
||||
html[data-phase="night"] .pete-reader-voice {
|
||||
border-color: rgba(255, 255, 255, 0.14);
|
||||
background: rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
|
||||
.pete-reader-scroll {
|
||||
flex: 1 1 auto;
|
||||
overflow-y: auto;
|
||||
-webkit-overflow-scrolling: touch;
|
||||
border-radius: 1.75rem;
|
||||
}
|
||||
.pete-reader-scroll:focus { outline: none; }
|
||||
|
||||
.pete-reader-article {
|
||||
background: var(--card);
|
||||
color: var(--ink);
|
||||
border-radius: 1.75rem;
|
||||
border: 2px solid rgba(0, 0, 0, 0.06);
|
||||
box-shadow: 0 6px 0 rgba(60, 40, 20, 0.12), 0 16px 32px rgba(60, 40, 20, 0.18);
|
||||
padding: clamp(1.25rem, 4vw, 2.5rem);
|
||||
}
|
||||
html[data-phase="night"] .pete-reader-article { border-color: rgba(255, 255, 255, 0.08); }
|
||||
|
||||
.pete-reader-eyebrow {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.5rem;
|
||||
font-size: 0.75rem;
|
||||
margin-bottom: 0.85rem;
|
||||
}
|
||||
.pete-reader-chip {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.35rem;
|
||||
border-radius: 9999px;
|
||||
padding: 0.15rem 0.65rem;
|
||||
color: #fff;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
}
|
||||
.pete-reader-source { font-weight: 700; opacity: 0.75; }
|
||||
.pete-reader-time { opacity: 0.55; }
|
||||
|
||||
.pete-reader-title {
|
||||
font-family: "Fredoka", "Nunito", system-ui, sans-serif;
|
||||
font-weight: 700;
|
||||
line-height: 1.15;
|
||||
letter-spacing: -0.01em;
|
||||
font-size: clamp(1.6rem, 4.5vw, 2.4rem);
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
.pete-reader-hero {
|
||||
width: 100%;
|
||||
border-radius: 1.1rem;
|
||||
margin-bottom: 1.25rem;
|
||||
background: rgba(0, 0, 0, 0.05);
|
||||
}
|
||||
|
||||
/* Reader typography, driven by the Aa popover and persisted per-device. The
|
||||
size lives as a CSS var on the scroll container; font + paper are classes. */
|
||||
.pete-reader-scroll { --reader-fs: 1.08rem; }
|
||||
.pete-reader-body { font-size: var(--reader-fs); line-height: 1.75; }
|
||||
.pete-reader-body p { margin-bottom: 1.05em; }
|
||||
.pete-reader-body p:last-child { margin-bottom: 0; }
|
||||
.pete-reader-scroll.is-size-s { --reader-fs: 0.98rem; }
|
||||
.pete-reader-scroll.is-size-m { --reader-fs: 1.08rem; }
|
||||
.pete-reader-scroll.is-size-l { --reader-fs: 1.22rem; }
|
||||
.pete-reader-scroll.is-size-xl { --reader-fs: 1.4rem; }
|
||||
.pete-reader-scroll.is-serif .pete-reader-body {
|
||||
font-family: Georgia, "Iowan Old Style", "Times New Roman", serif;
|
||||
}
|
||||
/* Sepia paper: warm background + ink regardless of day/night phase. */
|
||||
.pete-reader-scroll.is-sepia .pete-reader-article {
|
||||
background: #f6ecd6;
|
||||
color: #4a3a28;
|
||||
border-color: rgba(74, 58, 40, 0.16);
|
||||
}
|
||||
.pete-reader-scroll.is-sepia .pete-reader-note { border-top-color: rgba(74, 58, 40, 0.18); opacity: 0.8; }
|
||||
.pete-reader-scroll.is-sepia .pete-reader-hero { background: rgba(74, 58, 40, 0.08); }
|
||||
/* Currently-spoken paragraph highlight while reading aloud. */
|
||||
.pete-reader-body p.is-speaking {
|
||||
background: color-mix(in srgb, var(--accent) 32%, transparent);
|
||||
border-radius: 0.4rem;
|
||||
box-shadow: 0 0 0 0.35rem color-mix(in srgb, var(--accent) 32%, transparent);
|
||||
}
|
||||
|
||||
.pete-reader-note {
|
||||
margin-top: 1.25rem;
|
||||
padding-top: 1rem;
|
||||
border-top: 1px solid rgba(0, 0, 0, 0.1);
|
||||
font-size: 0.85rem;
|
||||
opacity: 0.7;
|
||||
}
|
||||
html[data-phase="night"] .pete-reader-note { border-color: rgba(255, 255, 255, 0.12); }
|
||||
.pete-reader-note a { color: var(--accent); font-weight: 700; }
|
||||
|
||||
.pete-reader-empty {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
text-align: center;
|
||||
gap: 0.75rem;
|
||||
padding: clamp(2rem, 8vw, 4rem) 1rem;
|
||||
color: var(--ink);
|
||||
}
|
||||
.pete-reader-empty-emoji { font-size: 2.5rem; }
|
||||
|
||||
/* Transient confirmation toast (e.g. "Link copied") shown over the reader. */
|
||||
.pete-reader-toast {
|
||||
position: absolute;
|
||||
left: 50%;
|
||||
bottom: 4.5rem;
|
||||
transform: translate(-50%, 0.5rem);
|
||||
z-index: 20;
|
||||
padding: 0.5rem 0.9rem;
|
||||
border-radius: 9999px;
|
||||
background: rgba(20, 14, 6, 0.9);
|
||||
color: #fff;
|
||||
font-weight: 700;
|
||||
font-size: 0.85rem;
|
||||
box-shadow: 0 8px 24px rgba(0, 0, 0, 0.3);
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
transition: opacity 0.2s ease, transform 0.2s ease;
|
||||
}
|
||||
.pete-reader-toast.is-shown { opacity: 1; transform: translate(-50%, 0); }
|
||||
|
||||
.pete-reader-hint {
|
||||
text-align: center;
|
||||
color: rgba(255, 255, 255, 0.85);
|
||||
font-size: 0.75rem;
|
||||
text-shadow: 0 1px 2px rgba(0, 0, 0, 0.4);
|
||||
}
|
||||
.pete-reader-hint kbd {
|
||||
font-family: ui-monospace, monospace;
|
||||
background: rgba(255, 255, 255, 0.16);
|
||||
border-radius: 0.3rem;
|
||||
padding: 0.05rem 0.3rem;
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.pete-reader-hint { display: none; }
|
||||
/* With Listen/Share/Aa added, let the toolbar wrap instead of overflowing. */
|
||||
.pete-reader-bar { flex-wrap: wrap; }
|
||||
.pete-reader-nav { flex-wrap: wrap; justify-content: flex-end; row-gap: 0.4rem; }
|
||||
}
|
||||
|
||||
/* "You might also like" rail, shown under the article in feed mode. Lives
|
||||
inside the reader's scroll area, below the article card. */
|
||||
.pete-reader-related { width: 100%; margin: 0.85rem auto 0; }
|
||||
.pete-reader-related-title {
|
||||
font-family: "Fredoka", "Nunito", system-ui, sans-serif;
|
||||
font-weight: 700;
|
||||
font-size: 0.95rem;
|
||||
color: #fff;
|
||||
margin: 0 0 0.6rem;
|
||||
text-shadow: 0 1px 2px rgba(0, 0, 0, 0.4);
|
||||
}
|
||||
.pete-reader-related-grid { display: grid; gap: 0.6rem; }
|
||||
.pete-reader-related-card {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
background: var(--card);
|
||||
color: var(--ink);
|
||||
border-radius: 1rem;
|
||||
border: 2px solid rgba(0, 0, 0, 0.06);
|
||||
padding: 0.6rem;
|
||||
text-decoration: none;
|
||||
transition: transform 0.15s ease, box-shadow 0.15s ease;
|
||||
}
|
||||
.pete-reader-related-card:hover {
|
||||
transform: translateY(-1px);
|
||||
box-shadow: 0 6px 16px rgba(60, 40, 20, 0.16);
|
||||
}
|
||||
html[data-phase="night"] .pete-reader-related-card { border-color: rgba(255, 255, 255, 0.08); }
|
||||
.pete-reader-related-thumb {
|
||||
width: 4.5rem;
|
||||
height: 3.25rem;
|
||||
flex-shrink: 0;
|
||||
object-fit: cover;
|
||||
border-radius: 0.6rem;
|
||||
background: rgba(0, 0, 0, 0.06);
|
||||
}
|
||||
.pete-reader-related-meta { min-width: 0; display: flex; flex-direction: column; gap: 0.25rem; }
|
||||
.pete-reader-related-eyebrow {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.4rem;
|
||||
font-size: 0.7rem;
|
||||
}
|
||||
.pete-reader-related-source { font-weight: 700; opacity: 0.7; }
|
||||
.pete-reader-related-headline {
|
||||
font-weight: 700;
|
||||
line-height: 1.25;
|
||||
font-size: 0.92rem;
|
||||
display: -webkit-box;
|
||||
-webkit-line-clamp: 2;
|
||||
-webkit-box-orient: vertical;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* Grid treatment for stories already read in feed mode: dimmed, with a small
|
||||
corner check. Hovering restores full opacity so nothing feels lost. */
|
||||
[data-story-card][data-read="1"] { opacity: 0.5; transition: opacity 0.2s ease; }
|
||||
[data-story-card][data-read="1"]:hover { opacity: 1; }
|
||||
[data-story-card][data-read="1"]::after {
|
||||
content: "✓";
|
||||
position: absolute;
|
||||
top: 0.6rem;
|
||||
right: 0.6rem;
|
||||
z-index: 6;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
height: 1.5rem;
|
||||
width: 1.5rem;
|
||||
border-radius: 9999px;
|
||||
background: rgba(20, 14, 6, 0.72);
|
||||
color: #fff;
|
||||
font-size: 0.8rem;
|
||||
font-weight: 700;
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
BIN
internal/web/static/img/apple-touch-icon.png
Normal file
BIN
internal/web/static/img/apple-touch-icon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
BIN
internal/web/static/img/icon-192.png
Normal file
BIN
internal/web/static/img/icon-192.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 37 KiB |
BIN
internal/web/static/img/icon-512.png
Normal file
BIN
internal/web/static/img/icon-512.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 154 KiB |
BIN
internal/web/static/img/maskable-512.png
Normal file
BIN
internal/web/static/img/maskable-512.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 100 KiB |
70
internal/web/static/js/prefs.js
Normal file
70
internal/web/static/js/prefs.js
Normal file
@@ -0,0 +1,70 @@
|
||||
// Preference sync. Anonymous visitors keep using localStorage exactly as
|
||||
// before — this file is a no-op for them. When a user is signed in (Authentik
|
||||
// via OIDC), the server is the source of truth: their stored blob is injected
|
||||
// as window.PETE_PREFS and seeded into localStorage *synchronously* here, before
|
||||
// the feature scripts (settings.js, weather*.js) read it. Those scripts then
|
||||
// call PetePrefs.push() after every write to mirror the change back up.
|
||||
//
|
||||
// This script must run before the others — it's loaded first in layout.html,
|
||||
// and all the feature scripts are `defer`, so document order is guaranteed.
|
||||
(function () {
|
||||
// The localStorage keys we sync. The weather *cache* is deliberately excluded:
|
||||
// it's transient and per-device.
|
||||
var SYNCED = ["pete.disabledSources.v1", "pete.weather.loc.v1", "pete-weather-off"];
|
||||
|
||||
var user = window.PETE_USER || null;
|
||||
var serverPrefs = window.PETE_PREFS || null;
|
||||
|
||||
function seed(prefs) {
|
||||
SYNCED.forEach(function (k) {
|
||||
if (!Object.prototype.hasOwnProperty.call(prefs, k)) return;
|
||||
var v = prefs[k];
|
||||
try {
|
||||
if (v === null || v === undefined) localStorage.removeItem(k);
|
||||
else localStorage.setItem(k, String(v));
|
||||
} catch (e) {}
|
||||
});
|
||||
}
|
||||
|
||||
function snapshot() {
|
||||
var out = {};
|
||||
SYNCED.forEach(function (k) {
|
||||
try { var v = localStorage.getItem(k); if (v !== null) out[k] = v; } catch (e) {}
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
var timer = null;
|
||||
function push() {
|
||||
if (!user) return; // anonymous: localStorage only
|
||||
if (timer) clearTimeout(timer);
|
||||
timer = setTimeout(function () {
|
||||
timer = null;
|
||||
try {
|
||||
fetch("/api/preferences", {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(snapshot()),
|
||||
credentials: "same-origin",
|
||||
keepalive: true
|
||||
}).catch(function () {});
|
||||
} catch (e) {}
|
||||
}, 600);
|
||||
}
|
||||
|
||||
window.PetePrefs = { push: push, loggedIn: !!user, syncedKeys: SYNCED };
|
||||
|
||||
if (user) {
|
||||
if (serverPrefs && typeof serverPrefs === "object") {
|
||||
seed(serverPrefs); // cross-device: server wins on load
|
||||
} else {
|
||||
push(); // first sign-in: migrate this browser's prefs to the account
|
||||
}
|
||||
// Swap "saved in this browser" copy for the synced story.
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
document.querySelectorAll("[data-storage-note]").forEach(function (el) {
|
||||
el.textContent = "Synced to your account ✓";
|
||||
});
|
||||
});
|
||||
}
|
||||
})();
|
||||
116
internal/web/static/js/pwa.js
Normal file
116
internal/web/static/js/pwa.js
Normal file
@@ -0,0 +1,116 @@
|
||||
// PWA glue: register the service worker, and (for signed-in users on a
|
||||
// push-enabled server) drive the notification opt-in toggle in the settings
|
||||
// panel. Anonymous visitors still get the offline reader — only the push
|
||||
// controls are gated behind sign-in + a configured VAPID key.
|
||||
(function () {
|
||||
if (!("serviceWorker" in navigator)) return;
|
||||
|
||||
var CFG = window.PETE_PUSH || null; // { enabled, publicKey } or null
|
||||
var reg = null;
|
||||
|
||||
navigator.serviceWorker.register("/sw.js").then(function (r) {
|
||||
reg = r;
|
||||
if (canPush()) initPushUI();
|
||||
}).catch(function () {});
|
||||
|
||||
function canPush() {
|
||||
return !!(CFG && CFG.enabled && CFG.publicKey && window.PETE_USER &&
|
||||
"PushManager" in window && "Notification" in window);
|
||||
}
|
||||
|
||||
// ---- push subscription ----------------------------------------------------
|
||||
function urlBase64ToUint8Array(base64String) {
|
||||
var padding = "=".repeat((4 - (base64String.length % 4)) % 4);
|
||||
var base64 = (base64String + padding).replace(/-/g, "+").replace(/_/g, "/");
|
||||
var raw = atob(base64);
|
||||
var out = new Uint8Array(raw.length);
|
||||
for (var i = 0; i < raw.length; i++) out[i] = raw.charCodeAt(i);
|
||||
return out;
|
||||
}
|
||||
|
||||
function currentSub() {
|
||||
if (!reg) return Promise.resolve(null);
|
||||
return reg.pushManager.getSubscription();
|
||||
}
|
||||
|
||||
function subscribe() {
|
||||
return reg.pushManager.subscribe({
|
||||
userVisibleOnly: true,
|
||||
applicationServerKey: urlBase64ToUint8Array(CFG.publicKey),
|
||||
}).then(function (sub) {
|
||||
return fetch("/api/push/subscribe", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(sub.toJSON()),
|
||||
credentials: "same-origin",
|
||||
}).then(function (res) {
|
||||
if (!res.ok) throw new Error("subscribe rejected");
|
||||
return sub;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function unsubscribe() {
|
||||
return currentSub().then(function (sub) {
|
||||
if (!sub) return;
|
||||
var endpoint = sub.endpoint;
|
||||
return sub.unsubscribe().then(function () {
|
||||
return fetch("/api/push/unsubscribe", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ endpoint: endpoint }),
|
||||
credentials: "same-origin",
|
||||
}).catch(function () {});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// ---- settings-panel toggle ------------------------------------------------
|
||||
function initPushUI() {
|
||||
var slot = document.querySelector("[data-push-section]");
|
||||
if (!slot) return;
|
||||
slot.hidden = false;
|
||||
|
||||
var btn = slot.querySelector("[data-push-toggle]");
|
||||
var note = slot.querySelector("[data-push-note]");
|
||||
if (!btn) return;
|
||||
var busy = false;
|
||||
|
||||
function paint(on, text) {
|
||||
btn.setAttribute("aria-pressed", on ? "true" : "false");
|
||||
btn.textContent = on ? "Notifications on" : "Turn on notifications";
|
||||
if (note && text != null) note.textContent = text;
|
||||
}
|
||||
|
||||
function refresh() {
|
||||
if (Notification.permission === "denied") {
|
||||
btn.disabled = true;
|
||||
paint(false, "Notifications are blocked in your browser settings.");
|
||||
return;
|
||||
}
|
||||
currentSub().then(function (sub) {
|
||||
paint(!!sub, sub ? "You'll get a nudge when new stories land." : "Get a nudge when new stories land.");
|
||||
});
|
||||
}
|
||||
|
||||
btn.addEventListener("click", function () {
|
||||
if (busy) return;
|
||||
busy = true;
|
||||
btn.disabled = true;
|
||||
currentSub().then(function (sub) {
|
||||
if (sub) return unsubscribe().then(function () { paint(false, "Notifications off."); });
|
||||
return Notification.requestPermission().then(function (perm) {
|
||||
if (perm !== "granted") { paint(false, "Permission denied."); return; }
|
||||
return subscribe().then(function () { paint(true, "You're all set. New stories will nudge you."); });
|
||||
});
|
||||
}).catch(function () {
|
||||
paint(false, "Something went wrong. Try again.");
|
||||
}).finally(function () {
|
||||
busy = false;
|
||||
btn.disabled = Notification.permission === "denied";
|
||||
});
|
||||
});
|
||||
|
||||
refresh();
|
||||
}
|
||||
})();
|
||||
807
internal/web/static/js/reader.js
Normal file
807
internal/web/static/js/reader.js
Normal file
@@ -0,0 +1,807 @@
|
||||
// Feed / reader mode. Presents the stories currently on the page one at a
|
||||
// time in a focused overlay, marking each read as it's shown. Left/right arrow
|
||||
// keys (or the on-screen buttons) page through them; the full article text is
|
||||
// whatever Pete captured at ingest (content:encoded or the scraped body),
|
||||
// fetched on demand from /api/article and cached for the session.
|
||||
//
|
||||
// Read state lives in localStorage under pete.read.v1. It's deliberately
|
||||
// device-local — unlike the small prefs blob, the read set grows unbounded, so
|
||||
// it isn't mirrored through PetePrefs to the account.
|
||||
(function () {
|
||||
var overlay = document.getElementById("pete-reader");
|
||||
if (!overlay) return;
|
||||
|
||||
var READ_KEY = "pete.read.v1";
|
||||
var articleEl = overlay.querySelector("[data-reader-article]");
|
||||
var scrollEl = overlay.querySelector("[data-reader-scroll]");
|
||||
var progressEl = overlay.querySelector("[data-reader-progress]");
|
||||
var linkEl = overlay.querySelector("[data-reader-link]");
|
||||
var prevBtn = overlay.querySelector("[data-reader-prev]");
|
||||
var nextBtn = overlay.querySelector("[data-reader-next]");
|
||||
var closeBtn = overlay.querySelector("[data-reader-close]");
|
||||
var backdrop = overlay.querySelector("[data-reader-backdrop]");
|
||||
var readerBookmarkBtn = overlay.querySelector("[data-reader-bookmark]");
|
||||
var listenBtn = overlay.querySelector("[data-reader-listen]");
|
||||
var shareBtn = overlay.querySelector("[data-reader-share]");
|
||||
var typeBtn = overlay.querySelector("[data-reader-type]");
|
||||
var typeMenu = overlay.querySelector("[data-reader-typemenu]");
|
||||
var voiceRow = overlay.querySelector("[data-reader-voicerow]");
|
||||
var voiceSel = overlay.querySelector("[data-reader-voice]");
|
||||
var relatedEl = overlay.querySelector("[data-reader-related]");
|
||||
var relatedCache = {}; // id -> results array
|
||||
|
||||
// Signed-in users (Authentik) get read + bookmark state synced server-side;
|
||||
// window.PETE_USER is non-null for them. Anonymous visitors keep the
|
||||
// localStorage-only behaviour and never see the bookmark controls.
|
||||
var SIGNED_IN = !!(window.PETE_USER);
|
||||
var bookmarkSet = Object.create(null); // id -> 1 for bookmarked stories
|
||||
|
||||
// Read-aloud is a signed-in-only perk, backed by server-side Piper voices
|
||||
// (window.PETE_TTS) rather than the browser's robotic Web Speech voice. Share
|
||||
// is available to everyone (native sheet where present, clipboard otherwise).
|
||||
var TTS_CFG = (window.PETE_TTS && window.PETE_TTS.enabled &&
|
||||
window.PETE_TTS.voices && window.PETE_TTS.voices.length) ? window.PETE_TTS : null;
|
||||
var TTS_OK = SIGNED_IN && !!TTS_CFG;
|
||||
|
||||
var items = []; // {id, url, headline, lede, image, time, source, chTitle, chEmoji, chTheme, posted, paywalled}
|
||||
var index = 0;
|
||||
var open = false;
|
||||
var contentCache = {}; // id -> {content, lede}
|
||||
var inflight = null;
|
||||
var bodyReady = false; // true once the real article body is rendered (not the loading placeholder)
|
||||
|
||||
// ---- read-state store -----------------------------------------------------
|
||||
function loadRead() {
|
||||
try {
|
||||
var raw = localStorage.getItem(READ_KEY);
|
||||
var obj = raw ? JSON.parse(raw) : {};
|
||||
return obj && typeof obj === "object" ? obj : {};
|
||||
} catch (e) { return {}; }
|
||||
}
|
||||
function saveRead(set) {
|
||||
try { localStorage.setItem(READ_KEY, JSON.stringify(set)); } catch (e) {}
|
||||
}
|
||||
var readSet = loadRead();
|
||||
|
||||
function isRead(id) { return !!readSet[id]; }
|
||||
function setRead(id, on) {
|
||||
if (on) readSet[id] = 1; else delete readSet[id];
|
||||
saveRead(readSet);
|
||||
paintCard(id, on);
|
||||
if (SIGNED_IN) postState("/api/read", { id: Number(id), read: !!on });
|
||||
}
|
||||
|
||||
// ---- server sync (signed-in only) -----------------------------------------
|
||||
function postState(url, body) {
|
||||
try {
|
||||
fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
credentials: "same-origin",
|
||||
keepalive: true
|
||||
}).catch(function () {});
|
||||
} catch (e) {}
|
||||
}
|
||||
|
||||
function isBookmarked(id) { return !!bookmarkSet[id]; }
|
||||
|
||||
// setBookmark updates memory, paints every matching control, and persists.
|
||||
function setBookmark(id, on) {
|
||||
if (on) bookmarkSet[id] = 1; else delete bookmarkSet[id];
|
||||
paintBookmark(id, on);
|
||||
postState("/api/bookmark", { id: Number(id), on: !!on });
|
||||
// On the bookmarks page, an un-bookmark should drop the card immediately.
|
||||
if (!on && location.pathname === "/bookmarks") {
|
||||
document.querySelectorAll('[data-story-card][data-id="' + cssEsc(id) + '"]').forEach(function (c) {
|
||||
c.parentNode && c.parentNode.removeChild(c);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// setBookmarkQuiet applies server-provided state without echoing it back.
|
||||
function setBookmarkQuiet(id, on) {
|
||||
if (on) bookmarkSet[id] = 1; else delete bookmarkSet[id];
|
||||
paintBookmark(id, on);
|
||||
}
|
||||
|
||||
function paintBookmark(id, on) {
|
||||
document.querySelectorAll('[data-bookmark-btn][data-story-id="' + cssEsc(id) + '"]').forEach(function (b) {
|
||||
applyCardBookmark(b, on);
|
||||
});
|
||||
if (readerBookmarkBtn && items[index] && String(items[index].id) === String(id)) {
|
||||
applyReaderBookmark(on);
|
||||
}
|
||||
}
|
||||
|
||||
function applyCardBookmark(btn, on) {
|
||||
btn.setAttribute("aria-pressed", on ? "true" : "false");
|
||||
var svg = btn.querySelector("svg");
|
||||
if (on) {
|
||||
btn.style.background = "var(--accent)";
|
||||
btn.style.color = "#1c1305";
|
||||
if (svg) svg.setAttribute("fill", "currentColor");
|
||||
} else {
|
||||
btn.style.background = "rgba(20,14,6,.62)";
|
||||
btn.style.color = "#fff";
|
||||
if (svg) svg.setAttribute("fill", "none");
|
||||
}
|
||||
}
|
||||
|
||||
function applyReaderBookmark(on) {
|
||||
if (!readerBookmarkBtn) return;
|
||||
readerBookmarkBtn.setAttribute("aria-pressed", on ? "true" : "false");
|
||||
readerBookmarkBtn.textContent = on ? "🔖 Saved" : "🔖 Save";
|
||||
}
|
||||
|
||||
// initUserState reveals the bookmark controls and pulls the signed-in user's
|
||||
// read + bookmark state for the stories on this page, painting them and
|
||||
// migrating any device-local reads the account doesn't have yet.
|
||||
function initUserState() {
|
||||
if (!SIGNED_IN) return;
|
||||
document.querySelectorAll("[data-bookmark-btn]").forEach(function (b) {
|
||||
b.style.display = "inline-flex";
|
||||
});
|
||||
var ids = [];
|
||||
document.querySelectorAll("[data-story-card]").forEach(function (c) {
|
||||
var id = c.getAttribute("data-id");
|
||||
if (id) ids.push(id);
|
||||
});
|
||||
if (!ids.length) return;
|
||||
fetch("/api/state?ids=" + encodeURIComponent(ids.join(",")), { credentials: "same-origin" })
|
||||
.then(function (r) { return r.ok ? r.json() : null; })
|
||||
.then(function (data) {
|
||||
if (!data) return;
|
||||
var serverRead = Object.create(null);
|
||||
(data.read || []).forEach(function (id) {
|
||||
serverRead[id] = 1; readSet[id] = 1; paintCard(id, true);
|
||||
});
|
||||
(data.bookmarked || []).forEach(function (id) { setBookmarkQuiet(id, true); });
|
||||
// Migrate device-local reads the account doesn't have yet — but only
|
||||
// once per device. After the first sync the server is authoritative, so
|
||||
// a story the user later marks unread on another device stays unread
|
||||
// instead of being perpetually resurrected from this device's stale
|
||||
// local set on every page load.
|
||||
var MIGRATED_KEY = "pete.readMigrated.v1";
|
||||
var migrated = false;
|
||||
try { migrated = localStorage.getItem(MIGRATED_KEY) === "1"; } catch (e) {}
|
||||
if (!migrated) {
|
||||
ids.forEach(function (id) {
|
||||
if (readSet[id] && !serverRead[id]) postState("/api/read", { id: Number(id), read: true });
|
||||
});
|
||||
try { localStorage.setItem(MIGRATED_KEY, "1"); } catch (e) {}
|
||||
}
|
||||
saveRead(readSet);
|
||||
})
|
||||
.catch(function () {});
|
||||
}
|
||||
// Reflect read state onto every matching card on the page (a story can appear
|
||||
// in more than one section on the home page).
|
||||
function paintCard(id, on) {
|
||||
document.querySelectorAll('[data-story-card][data-id="' + cssEsc(id) + '"]').forEach(function (c) {
|
||||
if (on) c.setAttribute("data-read", "1");
|
||||
else c.removeAttribute("data-read");
|
||||
});
|
||||
}
|
||||
function cssEsc(s) {
|
||||
return String(s).replace(/["\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
// ---- collecting the page's stories ----------------------------------------
|
||||
function isVisible(el) {
|
||||
return window.getComputedStyle(el).display !== "none";
|
||||
}
|
||||
function collect() {
|
||||
var seen = Object.create(null);
|
||||
var out = [];
|
||||
document.querySelectorAll("[data-story-card]").forEach(function (c) {
|
||||
var id = c.getAttribute("data-id");
|
||||
if (!id || seen[id] || !isVisible(c)) return;
|
||||
var url = c.getAttribute("data-url") || c.getAttribute("href") || "";
|
||||
if (!/^https?:\/\//i.test(url)) url = "";
|
||||
seen[id] = true;
|
||||
out.push({
|
||||
id: id,
|
||||
url: url,
|
||||
headline: c.getAttribute("data-headline") || "",
|
||||
lede: c.getAttribute("data-lede") || "",
|
||||
image: c.getAttribute("data-image") || "",
|
||||
time: c.getAttribute("data-time") || "",
|
||||
source: c.getAttribute("data-source") || "",
|
||||
chTitle: c.getAttribute("data-ch-title") || "",
|
||||
chEmoji: c.getAttribute("data-ch-emoji") || "",
|
||||
chTheme: c.getAttribute("data-ch-theme") || "",
|
||||
paywalled: c.getAttribute("data-paywalled") === "1"
|
||||
});
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- rendering ------------------------------------------------------------
|
||||
function escapeHTML(s) {
|
||||
return String(s == null ? "" : s)
|
||||
.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">")
|
||||
.replace(/"/g, """).replace(/'/g, "'");
|
||||
}
|
||||
function safeURL(s) {
|
||||
var raw = String(s == null ? "" : s).trim();
|
||||
return /^https?:\/\//i.test(raw) ? raw : "";
|
||||
}
|
||||
// Turn stored plain text (paragraphs separated by blank lines) into escaped
|
||||
// <p> blocks. Single newlines inside a paragraph become spaces.
|
||||
function paragraphs(text) {
|
||||
var blocks = String(text || "").split(/\n{2,}/);
|
||||
var html = "";
|
||||
for (var i = 0; i < blocks.length; i++) {
|
||||
var t = blocks[i].replace(/\s*\n\s*/g, " ").trim();
|
||||
if (t) html += "<p>" + escapeHTML(t) + "</p>";
|
||||
}
|
||||
return html;
|
||||
}
|
||||
|
||||
function eyebrow(it) {
|
||||
var chip = it.chTheme
|
||||
? '<span class="pete-reader-chip bg-theme-' + escapeHTML(it.chTheme) + '">' +
|
||||
(it.chEmoji ? '<span aria-hidden="true">' + escapeHTML(it.chEmoji) + "</span>" : "") +
|
||||
escapeHTML(it.chTitle) + "</span>"
|
||||
: "";
|
||||
var src = it.source ? '<span class="pete-reader-source">' + escapeHTML(it.source) + "</span>" : "";
|
||||
var time = it.time ? '<span class="pete-reader-time">' + escapeHTML(it.time) + "</span>" : "";
|
||||
return '<div class="pete-reader-eyebrow">' + chip + src + time + "</div>";
|
||||
}
|
||||
|
||||
function renderBody(it, bodyHTML) {
|
||||
var hero = it.image
|
||||
? '<img class="pete-reader-hero" src="' + escapeHTML(it.image) + '" alt="' + escapeHTML(it.headline) + '" loading="lazy" decoding="async">'
|
||||
: "";
|
||||
var href = safeURL(it.url);
|
||||
var note = '<p class="pete-reader-note">' +
|
||||
(it.paywalled ? "This source is paywalled, so the text above may be partial. " : "") +
|
||||
(href ? 'Read it at the source: <a href="' + escapeHTML(href) + '" target="_blank" rel="noopener noreferrer">' +
|
||||
escapeHTML(it.source || "original article") + " ↗</a>." : "") +
|
||||
"</p>";
|
||||
articleEl.innerHTML =
|
||||
eyebrow(it) +
|
||||
'<h1 class="pete-reader-title">' + escapeHTML(it.headline) + "</h1>" +
|
||||
hero +
|
||||
'<div class="pete-reader-body">' + bodyHTML + "</div>" +
|
||||
note;
|
||||
}
|
||||
|
||||
function loadingBody(it) {
|
||||
bodyReady = false; // read-aloud waits for the real body, not this placeholder
|
||||
renderBody(it, '<p style="opacity:.55">Loading the full story…</p>');
|
||||
}
|
||||
|
||||
function fillContent(it, data) {
|
||||
var text = (data && data.content) ? data.content : ((data && data.lede) || it.lede);
|
||||
var html = paragraphs(text);
|
||||
if (!html) {
|
||||
html = '<p style="opacity:.7">No article text was captured for this story. Open the original to read it.</p>';
|
||||
}
|
||||
renderBody(it, html);
|
||||
bodyReady = true;
|
||||
}
|
||||
|
||||
function fetchContent(it) {
|
||||
if (contentCache[it.id]) { fillContent(it, contentCache[it.id]); return; }
|
||||
loadingBody(it);
|
||||
if (inflight) inflight.abort();
|
||||
var ctrl = new AbortController();
|
||||
inflight = ctrl;
|
||||
var reqId = it.id;
|
||||
fetch("/api/article?id=" + encodeURIComponent(it.id), { signal: ctrl.signal, credentials: "same-origin" })
|
||||
.then(function (r) { if (!r.ok) throw new Error("status " + r.status); return r.json(); })
|
||||
.then(function (data) {
|
||||
contentCache[reqId] = data;
|
||||
if (ctrl !== inflight) return; // superseded by a newer navigation
|
||||
if (items[index] && items[index].id === reqId) fillContent(it, data);
|
||||
})
|
||||
.catch(function (err) {
|
||||
if (err.name === "AbortError") return;
|
||||
if (items[index] && items[index].id === reqId) fillContent(it, null); // fall back to lede
|
||||
})
|
||||
.finally(function () { if (ctrl === inflight) inflight = null; });
|
||||
}
|
||||
|
||||
// ---- related ("you might also like") --------------------------------------
|
||||
function clearRelated() {
|
||||
if (!relatedEl) return;
|
||||
relatedEl.innerHTML = "";
|
||||
relatedEl.hidden = true;
|
||||
}
|
||||
|
||||
function renderRelated(reqId, results) {
|
||||
if (!relatedEl) return;
|
||||
// Ignore a response that arrived after the user moved on.
|
||||
if (!items[index] || String(items[index].id) !== String(reqId)) return;
|
||||
if (!results || !results.length) { clearRelated(); return; }
|
||||
var html = '<h2 class="pete-reader-related-title">You might also like</h2>' +
|
||||
'<div class="pete-reader-related-grid">';
|
||||
for (var i = 0; i < results.length; i++) {
|
||||
var it = results[i];
|
||||
var href = safeURL(it.article_url);
|
||||
if (!href) continue;
|
||||
var chip = it.channel_theme
|
||||
? '<span class="pete-reader-chip bg-theme-' + escapeHTML(it.channel_theme) + '">' +
|
||||
(it.channel_emoji ? '<span aria-hidden="true">' + escapeHTML(it.channel_emoji) + "</span>" : "") +
|
||||
escapeHTML(it.channel_title) + "</span>"
|
||||
: "";
|
||||
var thumb = it.thumb_url
|
||||
? '<img class="pete-reader-related-thumb" src="' + escapeHTML(it.thumb_url) + '" alt="' + escapeHTML(it.headline || "") + '" loading="lazy" decoding="async">'
|
||||
: '<div class="pete-reader-related-thumb pete-reader-related-thumb-empty"></div>';
|
||||
html += '<a class="pete-reader-related-card" href="' + escapeHTML(href) + '" target="_blank" rel="noopener noreferrer">' +
|
||||
thumb +
|
||||
'<div class="pete-reader-related-meta">' +
|
||||
'<div class="pete-reader-related-eyebrow">' + chip +
|
||||
(it.source ? '<span class="pete-reader-related-source">' + escapeHTML(it.source) + "</span>" : "") +
|
||||
"</div>" +
|
||||
'<div class="pete-reader-related-headline">' + escapeHTML(it.headline) + "</div>" +
|
||||
"</div></a>";
|
||||
}
|
||||
html += "</div>";
|
||||
relatedEl.innerHTML = html;
|
||||
relatedEl.hidden = false;
|
||||
}
|
||||
|
||||
function fetchRelated(it) {
|
||||
if (!relatedEl) return;
|
||||
clearRelated();
|
||||
var reqId = it.id;
|
||||
if (relatedCache[reqId]) { renderRelated(reqId, relatedCache[reqId]); return; }
|
||||
fetch("/api/related?id=" + encodeURIComponent(it.id), { credentials: "same-origin" })
|
||||
.then(function (r) { return r.ok ? r.json() : null; })
|
||||
.then(function (data) {
|
||||
var results = (data && data.results) || [];
|
||||
relatedCache[reqId] = results;
|
||||
renderRelated(reqId, results);
|
||||
})
|
||||
.catch(function () {});
|
||||
}
|
||||
|
||||
// ---- navigation -----------------------------------------------------------
|
||||
function show(i) {
|
||||
stopSpeak(); // never keep reading a story the user navigated away from
|
||||
index = i;
|
||||
if (index >= items.length) { renderDone(); return; }
|
||||
var it = items[index];
|
||||
progressEl.textContent = (index + 1) + " / " + items.length;
|
||||
var href = safeURL(it.url);
|
||||
if (href) { linkEl.href = href; linkEl.style.display = ""; }
|
||||
else linkEl.style.display = "none";
|
||||
prevBtn.disabled = index === 0;
|
||||
nextBtn.disabled = false;
|
||||
nextBtn.textContent = index === items.length - 1 ? "done ✓" : "→";
|
||||
if (scrollEl) scrollEl.scrollTop = 0;
|
||||
if (readerBookmarkBtn) {
|
||||
readerBookmarkBtn.style.display = SIGNED_IN ? "" : "none";
|
||||
applyReaderBookmark(isBookmarked(it.id));
|
||||
}
|
||||
fetchContent(it);
|
||||
fetchRelated(it);
|
||||
setRead(it.id, true); // presenting a story marks it read
|
||||
}
|
||||
|
||||
function renderDone() {
|
||||
stopSpeak();
|
||||
clearRelated();
|
||||
progressEl.textContent = items.length + " / " + items.length;
|
||||
linkEl.style.display = "none";
|
||||
prevBtn.disabled = items.length === 0;
|
||||
nextBtn.disabled = true;
|
||||
nextBtn.textContent = "→";
|
||||
if (scrollEl) scrollEl.scrollTop = 0;
|
||||
articleEl.innerHTML =
|
||||
'<div class="pete-reader-empty">' +
|
||||
'<span class="pete-reader-empty-emoji" aria-hidden="true">🎉</span>' +
|
||||
'<h1 class="pete-reader-title" style="margin:0">You\'re all caught up</h1>' +
|
||||
'<p style="opacity:.7;max-width:28rem">That\'s every story on this page. ' +
|
||||
"Head back to browse more, or press ← to revisit the last one.</p>" +
|
||||
"</div>";
|
||||
}
|
||||
|
||||
function next() { if (index < items.length) show(index + 1); }
|
||||
function prev() { if (index > 0) show(index - 1); }
|
||||
|
||||
function firstUnread() {
|
||||
for (var i = 0; i < items.length; i++) if (!isRead(items[i].id)) return i;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---- typography (Aa popover) ----------------------------------------------
|
||||
// Per-device reading preferences: text size, sans/serif, default/sepia paper.
|
||||
// Kept in localStorage (like the read set) rather than the synced prefs blob,
|
||||
// since they're a device-local reading-comfort choice.
|
||||
var TYPE_KEY = "pete.reader.type.v1";
|
||||
var SIZES = ["s", "m", "l", "xl"];
|
||||
var typePrefs = { size: "m", font: "sans", paper: "default" };
|
||||
(function loadType() {
|
||||
try {
|
||||
var raw = JSON.parse(localStorage.getItem(TYPE_KEY) || "{}");
|
||||
if (raw && typeof raw === "object") {
|
||||
if (SIZES.indexOf(raw.size) >= 0) typePrefs.size = raw.size;
|
||||
if (raw.font === "serif" || raw.font === "sans") typePrefs.font = raw.font;
|
||||
if (raw.paper === "sepia" || raw.paper === "default") typePrefs.paper = raw.paper;
|
||||
}
|
||||
} catch (e) {}
|
||||
})();
|
||||
function saveType() {
|
||||
try { localStorage.setItem(TYPE_KEY, JSON.stringify(typePrefs)); } catch (e) {}
|
||||
}
|
||||
function applyType() {
|
||||
if (!scrollEl) return;
|
||||
SIZES.forEach(function (s) { scrollEl.classList.remove("is-size-" + s); });
|
||||
scrollEl.classList.add("is-size-" + typePrefs.size);
|
||||
scrollEl.classList.toggle("is-serif", typePrefs.font === "serif");
|
||||
scrollEl.classList.toggle("is-sepia", typePrefs.paper === "sepia");
|
||||
paintType();
|
||||
}
|
||||
function paintType() {
|
||||
if (!typeMenu) return;
|
||||
typeMenu.querySelectorAll("[data-size]").forEach(function (b) {
|
||||
b.classList.toggle("is-active", b.getAttribute("data-size") === typePrefs.size);
|
||||
});
|
||||
typeMenu.querySelectorAll("[data-font]").forEach(function (b) {
|
||||
b.classList.toggle("is-active", b.getAttribute("data-font") === typePrefs.font);
|
||||
});
|
||||
typeMenu.querySelectorAll("[data-paper]").forEach(function (b) {
|
||||
b.classList.toggle("is-active", b.getAttribute("data-paper") === typePrefs.paper);
|
||||
});
|
||||
}
|
||||
function toggleTypeMenu(force) {
|
||||
if (!typeMenu || !typeBtn) return;
|
||||
var openNow = force != null ? force : typeMenu.hidden;
|
||||
typeMenu.hidden = !openNow;
|
||||
typeBtn.setAttribute("aria-expanded", openNow ? "true" : "false");
|
||||
}
|
||||
|
||||
// ---- share ----------------------------------------------------------------
|
||||
var toastEl = null, toastTimer = null;
|
||||
function toast(msg) {
|
||||
if (!toastEl) {
|
||||
toastEl = document.createElement("div");
|
||||
toastEl.className = "pete-reader-toast";
|
||||
overlay.appendChild(toastEl);
|
||||
}
|
||||
toastEl.textContent = msg;
|
||||
toastEl.classList.add("is-shown");
|
||||
if (toastTimer) clearTimeout(toastTimer);
|
||||
toastTimer = setTimeout(function () { toastEl.classList.remove("is-shown"); }, 1800);
|
||||
}
|
||||
function shareCurrent() {
|
||||
var it = items[index];
|
||||
if (!it) return;
|
||||
var url = safeURL(it.url);
|
||||
if (!url) { toast("No link to share"); return; }
|
||||
var data = { title: it.headline || "Pete", text: it.headline || "", url: url };
|
||||
if (navigator.share) {
|
||||
navigator.share(data).catch(function () {});
|
||||
return;
|
||||
}
|
||||
if (navigator.clipboard && navigator.clipboard.writeText) {
|
||||
navigator.clipboard.writeText(url).then(
|
||||
function () { toast("Link copied ✓"); },
|
||||
function () { toast("Couldn't copy link"); }
|
||||
);
|
||||
return;
|
||||
}
|
||||
window.open(url, "_blank", "noopener");
|
||||
}
|
||||
|
||||
// ---- read aloud (signed-in only, server-side Piper) -----------------------
|
||||
// Each paragraph is synthesized on demand by POSTing its text to /api/tts and
|
||||
// playing the returned WAV. We fetch the next paragraph while the current one
|
||||
// plays, so the model-load + synthesis latency is hidden behind playback.
|
||||
// speakGen is bumped on every stop/start/voice-change; any async callback that
|
||||
// fires afterwards checks it against its captured gen and no-ops if stale.
|
||||
var VOICE_KEY = "pete.reader.voice.v1";
|
||||
var speaking = false;
|
||||
var speakGen = 0;
|
||||
var speakParas = []; // <p> elements currently highlighted
|
||||
var speakItems = []; // [{el, text}] for the whole article
|
||||
var speakIdx = 0; // index into speakItems currently playing
|
||||
var speakReq = null; // in-flight fetch for the current paragraph (has .ctrl)
|
||||
var prefetch = null; // { idx, gen, promise, abort } for the next paragraph
|
||||
var audioEl = null; // shared <audio> element
|
||||
var currentURL = null; // object URL currently loaded into audioEl
|
||||
|
||||
// Device-local voice choice, like the type prefs (see loadType).
|
||||
var ttsVoice = "";
|
||||
(function loadVoice() {
|
||||
if (!TTS_CFG) return;
|
||||
var saved = "";
|
||||
try { saved = localStorage.getItem(VOICE_KEY) || ""; } catch (e) {}
|
||||
var ok = TTS_CFG.voices.some(function (v) { return v.id === saved; });
|
||||
ttsVoice = ok ? saved : TTS_CFG.default;
|
||||
})();
|
||||
function saveVoice() { try { localStorage.setItem(VOICE_KEY, ttsVoice); } catch (e) {} }
|
||||
|
||||
function speakingText() {
|
||||
if (!articleEl) return [];
|
||||
var out = [];
|
||||
var h = articleEl.querySelector(".pete-reader-title");
|
||||
if (h && h.textContent.trim()) out.push({ el: null, text: h.textContent.trim() });
|
||||
articleEl.querySelectorAll(".pete-reader-body p").forEach(function (p) {
|
||||
var t = p.textContent.trim();
|
||||
if (t) out.push({ el: p, text: t });
|
||||
});
|
||||
return out;
|
||||
}
|
||||
function clearSpeakHighlight() {
|
||||
speakParas.forEach(function (p) { p.classList.remove("is-speaking"); });
|
||||
speakParas = [];
|
||||
}
|
||||
function highlight(item) {
|
||||
clearSpeakHighlight();
|
||||
if (item && item.el) {
|
||||
item.el.classList.add("is-speaking");
|
||||
speakParas.push(item.el);
|
||||
try { item.el.scrollIntoView({ block: "center", behavior: "smooth" }); } catch (e) {}
|
||||
}
|
||||
}
|
||||
|
||||
// ttsFetch kicks off synthesis of one paragraph. Returns { ctrl, promise },
|
||||
// where promise resolves to an object URL for the audio blob.
|
||||
function ttsFetch(text) {
|
||||
var ctrl = new AbortController();
|
||||
var promise = fetch("/api/tts", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ voice: ttsVoice, text: text }),
|
||||
signal: ctrl.signal,
|
||||
}).then(function (r) {
|
||||
if (!r.ok) throw new Error("tts " + r.status);
|
||||
return r.blob();
|
||||
}).then(function (b) { return URL.createObjectURL(b); });
|
||||
return { ctrl: ctrl, promise: promise };
|
||||
}
|
||||
function abortPrefetch() {
|
||||
if (!prefetch) return;
|
||||
try { prefetch.ctrl.abort(); } catch (e) {}
|
||||
// If it already resolved to a URL, reclaim it.
|
||||
prefetch.promise.then(function (u) { URL.revokeObjectURL(u); }, function () {});
|
||||
prefetch = null;
|
||||
}
|
||||
function startPrefetch(i, gen) {
|
||||
var item = speakItems[i];
|
||||
if (!item) { prefetch = null; return; }
|
||||
var req = ttsFetch(item.text);
|
||||
prefetch = { idx: i, gen: gen, ctrl: req.ctrl, promise: req.promise };
|
||||
}
|
||||
function playURL(url, gen) {
|
||||
if (!audioEl) audioEl = new Audio();
|
||||
if (currentURL) URL.revokeObjectURL(currentURL);
|
||||
currentURL = url;
|
||||
audioEl.src = url;
|
||||
audioEl.onended = function () { if (speaking && gen === speakGen) playIdx(speakIdx + 1); };
|
||||
audioEl.onerror = function () { if (speaking && gen === speakGen) playIdx(speakIdx + 1); };
|
||||
var pr = audioEl.play();
|
||||
if (pr && pr.catch) pr.catch(function () {}); // ignore autoplay/abort rejections
|
||||
}
|
||||
function playIdx(i) {
|
||||
if (!speaking) return;
|
||||
var gen = speakGen;
|
||||
var item = speakItems[i];
|
||||
if (!item) { stopSpeak(); return; }
|
||||
speakIdx = i;
|
||||
highlight(item);
|
||||
var urlP;
|
||||
if (prefetch && prefetch.idx === i && prefetch.gen === gen) {
|
||||
urlP = prefetch.promise;
|
||||
prefetch = null; // hand off; do not abort/revoke this one
|
||||
} else {
|
||||
abortPrefetch();
|
||||
var req = ttsFetch(item.text);
|
||||
speakReq = req;
|
||||
urlP = req.promise;
|
||||
}
|
||||
urlP.then(function (url) {
|
||||
if (!speaking || gen !== speakGen) { URL.revokeObjectURL(url); return; }
|
||||
speakReq = null;
|
||||
startPrefetch(i + 1, gen); // synthesize the next paragraph during playback
|
||||
playURL(url, gen);
|
||||
}).catch(function () {
|
||||
if (!speaking || gen !== speakGen) return;
|
||||
speakReq = null;
|
||||
playIdx(i + 1); // skip a paragraph that failed rather than stalling
|
||||
});
|
||||
}
|
||||
function teardownAudio() {
|
||||
if (speakReq) { try { speakReq.ctrl.abort(); } catch (e) {} speakReq = null; }
|
||||
abortPrefetch();
|
||||
if (audioEl) { try { audioEl.pause(); } catch (e) {} audioEl.onended = null; audioEl.onerror = null; audioEl.removeAttribute("src"); }
|
||||
if (currentURL) { URL.revokeObjectURL(currentURL); currentURL = null; }
|
||||
}
|
||||
function stopSpeak() {
|
||||
if (!TTS_OK) return;
|
||||
speakGen++;
|
||||
speaking = false;
|
||||
teardownAudio();
|
||||
speakItems = [];
|
||||
clearSpeakHighlight();
|
||||
if (listenBtn) { listenBtn.setAttribute("aria-pressed", "false"); listenBtn.textContent = "🔊 Listen"; }
|
||||
}
|
||||
function startSpeak() {
|
||||
if (!TTS_OK) return;
|
||||
if (!bodyReady) { toast("Still loading…"); return; }
|
||||
var parts = speakingText();
|
||||
if (!parts.length) { toast("Nothing to read yet"); return; }
|
||||
speakGen++;
|
||||
teardownAudio();
|
||||
speaking = true;
|
||||
speakItems = parts;
|
||||
speakIdx = 0;
|
||||
if (listenBtn) { listenBtn.setAttribute("aria-pressed", "true"); listenBtn.textContent = "⏹ Stop"; }
|
||||
playIdx(0);
|
||||
}
|
||||
function toggleSpeak() {
|
||||
if (!TTS_OK) return;
|
||||
if (speaking) stopSpeak(); else startSpeak();
|
||||
}
|
||||
// Switching voice restarts the current paragraph with the new voice so the
|
||||
// change is audible immediately, keeping our place in the article.
|
||||
function changeVoice(id) {
|
||||
if (!TTS_CFG) return;
|
||||
ttsVoice = id;
|
||||
saveVoice();
|
||||
if (voiceSel) voiceSel.value = ttsVoice;
|
||||
if (speaking) {
|
||||
speakGen++;
|
||||
teardownAudio();
|
||||
playIdx(speakIdx);
|
||||
}
|
||||
}
|
||||
function buildVoiceMenu() {
|
||||
if (!voiceSel || !TTS_CFG) return;
|
||||
voiceSel.innerHTML = "";
|
||||
TTS_CFG.voices.forEach(function (v) {
|
||||
var o = document.createElement("option");
|
||||
o.value = v.id;
|
||||
o.textContent = v.label;
|
||||
voiceSel.appendChild(o);
|
||||
});
|
||||
voiceSel.value = ttsVoice;
|
||||
voiceSel.addEventListener("change", function () { changeVoice(voiceSel.value); });
|
||||
if (voiceRow) voiceRow.hidden = false;
|
||||
}
|
||||
|
||||
// ---- open / close ---------------------------------------------------------
|
||||
function openReader() {
|
||||
items = collect();
|
||||
if (items.length === 0) return;
|
||||
open = true;
|
||||
overlay.classList.remove("hidden");
|
||||
document.body.classList.add("overflow-hidden");
|
||||
applyType();
|
||||
show(firstUnread());
|
||||
// Move focus into the scroll region so Space / PageUp-Down / arrow keys
|
||||
// scroll the story, not the page behind it, without a click first.
|
||||
if (scrollEl) { try { scrollEl.focus({ preventScroll: true }); } catch (e) { scrollEl.focus(); } }
|
||||
}
|
||||
function closeReader() {
|
||||
open = false;
|
||||
stopSpeak();
|
||||
toggleTypeMenu(false);
|
||||
if (inflight) { inflight.abort(); inflight = null; }
|
||||
clearRelated();
|
||||
overlay.classList.add("hidden");
|
||||
document.body.classList.remove("overflow-hidden");
|
||||
}
|
||||
|
||||
// ---- wiring ---------------------------------------------------------------
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
// Dim stories already read on this device.
|
||||
document.querySelectorAll("[data-story-card]").forEach(function (c) {
|
||||
var id = c.getAttribute("data-id");
|
||||
if (id && isRead(id)) c.setAttribute("data-read", "1");
|
||||
});
|
||||
|
||||
document.querySelectorAll("[data-reader-open]").forEach(function (b) {
|
||||
b.addEventListener("click", function (e) { e.preventDefault(); openReader(); });
|
||||
});
|
||||
if (prevBtn) prevBtn.addEventListener("click", prev);
|
||||
if (nextBtn) nextBtn.addEventListener("click", next);
|
||||
if (closeBtn) closeBtn.addEventListener("click", closeReader);
|
||||
if (backdrop) backdrop.addEventListener("click", closeReader);
|
||||
if (readerBookmarkBtn) readerBookmarkBtn.addEventListener("click", function () {
|
||||
var it = items[index];
|
||||
if (it) setBookmark(it.id, !isBookmarked(it.id));
|
||||
});
|
||||
|
||||
// Read-aloud: signed-in only, and only when server-side TTS is configured.
|
||||
if (listenBtn) {
|
||||
if (TTS_OK) { listenBtn.style.display = ""; listenBtn.addEventListener("click", toggleSpeak); buildVoiceMenu(); }
|
||||
else listenBtn.style.display = "none";
|
||||
}
|
||||
// Share: available to everyone.
|
||||
if (shareBtn) { shareBtn.style.display = ""; shareBtn.addEventListener("click", shareCurrent); }
|
||||
|
||||
// Text-options popover.
|
||||
if (typeBtn) typeBtn.addEventListener("click", function (e) { e.stopPropagation(); toggleTypeMenu(); });
|
||||
if (typeMenu) {
|
||||
typeMenu.addEventListener("click", function (e) {
|
||||
var b = e.target.closest("button");
|
||||
if (!b) return;
|
||||
e.stopPropagation();
|
||||
if (b.hasAttribute("data-size")) typePrefs.size = b.getAttribute("data-size");
|
||||
else if (b.hasAttribute("data-font")) typePrefs.font = b.getAttribute("data-font");
|
||||
else if (b.hasAttribute("data-paper")) typePrefs.paper = b.getAttribute("data-paper");
|
||||
else return;
|
||||
saveType();
|
||||
applyType();
|
||||
});
|
||||
}
|
||||
// Close the type popover on any outside click.
|
||||
document.addEventListener("click", function (e) {
|
||||
if (!typeMenu || typeMenu.hidden) return;
|
||||
if (e.target.closest("[data-reader-type]")) return;
|
||||
toggleTypeMenu(false);
|
||||
});
|
||||
applyType();
|
||||
|
||||
initUserState();
|
||||
});
|
||||
|
||||
// Bookmark buttons live inside the card's <a>; intercept so a tap toggles the
|
||||
// bookmark instead of following the link. Delegated so it also covers cards
|
||||
// that are added or removed after load.
|
||||
document.addEventListener("click", function (e) {
|
||||
var btn = e.target.closest && e.target.closest("[data-bookmark-btn]");
|
||||
if (!btn) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
var id = btn.getAttribute("data-story-id");
|
||||
if (id) setBookmark(id, !isBookmarked(id));
|
||||
});
|
||||
document.addEventListener("keydown", function (e) {
|
||||
if (e.key !== "Enter" && e.key !== " ") return;
|
||||
var btn = e.target.closest && e.target.closest("[data-bookmark-btn]");
|
||||
if (!btn) return;
|
||||
e.preventDefault();
|
||||
var id = btn.getAttribute("data-story-id");
|
||||
if (id) setBookmark(id, !isBookmarked(id));
|
||||
});
|
||||
|
||||
document.addEventListener("keydown", function (e) {
|
||||
// Enter feed mode with `f` when nothing is focused and no other overlay is up.
|
||||
if (!open && (e.key === "f" || e.key === "F") && !e.metaKey && !e.ctrlKey && !e.altKey) {
|
||||
var tag = (document.activeElement && document.activeElement.tagName) || "";
|
||||
if (tag === "INPUT" || tag === "TEXTAREA") return;
|
||||
var searchOpen = document.getElementById("pete-search");
|
||||
if (searchOpen && !searchOpen.classList.contains("hidden")) return;
|
||||
e.preventDefault();
|
||||
openReader();
|
||||
return;
|
||||
}
|
||||
if (!open) return;
|
||||
switch (e.key) {
|
||||
case "ArrowRight": case "l": case " ": e.preventDefault(); next(); break;
|
||||
case "ArrowLeft": case "h": e.preventDefault(); prev(); break;
|
||||
case "Escape":
|
||||
e.preventDefault();
|
||||
if (typeMenu && !typeMenu.hidden) toggleTypeMenu(false); // esc closes the popover first
|
||||
else closeReader();
|
||||
break;
|
||||
case "s": case "S": e.preventDefault(); shareCurrent(); break;
|
||||
case "t": case "T": e.preventDefault(); toggleTypeMenu(); break;
|
||||
case "r": case "R": if (TTS_OK) { e.preventDefault(); toggleSpeak(); } break;
|
||||
case "o": case "Enter": {
|
||||
var it = items[index];
|
||||
var href = it && safeURL(it.url);
|
||||
if (href) { e.preventDefault(); window.open(href, "_blank", "noopener"); }
|
||||
break;
|
||||
}
|
||||
case "u": {
|
||||
var cur = items[index];
|
||||
if (cur) setRead(cur.id, false); // let the user undo an accidental read
|
||||
break;
|
||||
}
|
||||
case "b": case "B": {
|
||||
if (!SIGNED_IN) break;
|
||||
var it = items[index];
|
||||
if (it) { e.preventDefault(); setBookmark(it.id, !isBookmarked(it.id)); }
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
})();
|
||||
@@ -40,6 +40,15 @@
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
// safeURL rejects any href whose scheme isn't http(s). Stored article_url
|
||||
// comes from feeds we don't control; a hostile feed publishing
|
||||
// `javascript:fetch(...)` would otherwise execute on click.
|
||||
function safeURL(s) {
|
||||
const raw = String(s == null ? "" : s).trim();
|
||||
if (/^https?:\/\//i.test(raw)) return raw;
|
||||
return "";
|
||||
}
|
||||
|
||||
function render(results) {
|
||||
items = results || [];
|
||||
activeIndex = items.length > 0 ? 0 : -1;
|
||||
@@ -50,13 +59,15 @@
|
||||
const html = items.map((r, i) => {
|
||||
const thumb = r.thumb_url
|
||||
? `<div class="hidden sm:block w-20 h-20 shrink-0 overflow-hidden rounded-2xl bg-[color:var(--ink)]/5">
|
||||
<img src="${escapeHTML(r.thumb_url)}" alt="" loading="lazy" decoding="async"
|
||||
<img src="${escapeHTML(r.thumb_url)}" alt="${escapeHTML(r.headline || "")}" loading="lazy" decoding="async"
|
||||
class="h-full w-full object-cover">
|
||||
</div>`
|
||||
: `<div class="hidden sm:flex w-20 h-20 shrink-0 items-center justify-center rounded-2xl bg-[color:var(--ink)]/5 text-2xl">${escapeHTML(r.channel_emoji || "·")}</div>`;
|
||||
const postedRing = r.posted ? ` border-theme-${escapeHTML(r.channel_theme)}` : "";
|
||||
const href = safeURL(r.article_url);
|
||||
if (!href) return "";
|
||||
return `
|
||||
<a href="${escapeHTML(r.article_url)}" target="_blank" rel="noopener noreferrer"
|
||||
<a href="${escapeHTML(href)}" target="_blank" rel="noopener noreferrer"
|
||||
data-idx="${i}"
|
||||
class="search-result group flex items-start gap-4 rounded-2xl p-3 transition border-2 border-transparent hover:bg-[color:var(--ink)]/5${postedRing}">
|
||||
${thumb}
|
||||
@@ -138,7 +149,8 @@
|
||||
} else if (e.key === "Enter") {
|
||||
if (activeIndex >= 0 && items[activeIndex]) {
|
||||
e.preventDefault();
|
||||
window.open(items[activeIndex].article_url, "_blank", "noopener");
|
||||
const href = safeURL(items[activeIndex].article_url);
|
||||
if (href) window.open(href, "_blank", "noopener");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
145
internal/web/static/js/settings.js
Normal file
145
internal/web/static/js/settings.js
Normal file
@@ -0,0 +1,145 @@
|
||||
// Per-feed visibility, stored in localStorage. Server still ships every story;
|
||||
// we just hide cards whose data-source is in the disabled set.
|
||||
(function () {
|
||||
var KEY = "pete.disabledSources.v1";
|
||||
|
||||
function load() {
|
||||
try {
|
||||
var raw = localStorage.getItem(KEY);
|
||||
if (!raw) return {};
|
||||
var obj = JSON.parse(raw);
|
||||
return obj && typeof obj === "object" ? obj : {};
|
||||
} catch (e) { return {}; }
|
||||
}
|
||||
function save(set) {
|
||||
try { localStorage.setItem(KEY, JSON.stringify(set)); } catch (e) {}
|
||||
if (window.PetePrefs) window.PetePrefs.push();
|
||||
}
|
||||
|
||||
var disabled = load();
|
||||
|
||||
function apply() {
|
||||
var cards = document.querySelectorAll("[data-story-card]");
|
||||
cards.forEach(function (c) {
|
||||
var src = c.getAttribute("data-source") || "";
|
||||
c.style.display = disabled[src] ? "none" : "";
|
||||
});
|
||||
}
|
||||
|
||||
function knownSources() {
|
||||
// Prefer the server-provided full list (grouped by channel below).
|
||||
var server = Array.isArray(window.PETE_SOURCES) ? window.PETE_SOURCES : [];
|
||||
if (server.length > 0) {
|
||||
var copy = server.slice();
|
||||
// Surface any persisted-disabled sources that aren't in the config anymore.
|
||||
var present = Object.create(null);
|
||||
copy.forEach(function (s) { present[s.name] = true; });
|
||||
Object.keys(disabled).forEach(function (n) {
|
||||
if (!present[n]) copy.push({ name: n, channel: "" });
|
||||
});
|
||||
copy.sort(function (a, b) {
|
||||
var ca = (a.channel || "~").toLowerCase();
|
||||
var cb = (b.channel || "~").toLowerCase();
|
||||
if (ca !== cb) return ca.localeCompare(cb);
|
||||
return a.name.toLowerCase().localeCompare(b.name.toLowerCase());
|
||||
});
|
||||
return copy;
|
||||
}
|
||||
// Fallback: derive from DOM if the server didn't ship a list.
|
||||
var seen = Object.create(null);
|
||||
document.querySelectorAll("[data-story-card]").forEach(function (c) {
|
||||
var src = c.getAttribute("data-source");
|
||||
var ch = c.getAttribute("data-channel") || "";
|
||||
if (src && !seen[src]) seen[src] = { name: src, channel: ch };
|
||||
});
|
||||
Object.keys(disabled).forEach(function (s) {
|
||||
if (!seen[s]) seen[s] = { name: s, channel: "" };
|
||||
});
|
||||
return Object.keys(seen).map(function (k) { return seen[k]; }).sort(function (a, b) {
|
||||
return a.name.toLowerCase().localeCompare(b.name.toLowerCase());
|
||||
});
|
||||
}
|
||||
|
||||
function render() {
|
||||
var list = document.querySelector("[data-settings-list]");
|
||||
var meta = document.querySelector("[data-settings-meta]");
|
||||
if (!list) return;
|
||||
var sources = knownSources();
|
||||
list.innerHTML = "";
|
||||
if (sources.length === 0) {
|
||||
list.innerHTML = '<p class="px-2 py-4 text-sm text-center text-[color:var(--ink)]/50">No feeds configured.</p>';
|
||||
} else {
|
||||
var lastChannel = null;
|
||||
sources.forEach(function (src, i) {
|
||||
if (src.channel !== lastChannel) {
|
||||
lastChannel = src.channel;
|
||||
var hdr = document.createElement("div");
|
||||
hdr.className = "px-3 pt-3 pb-1 text-[11px] uppercase tracking-wider text-[color:var(--ink)]/40 font-bold";
|
||||
hdr.textContent = src.channel || "other";
|
||||
list.appendChild(hdr);
|
||||
}
|
||||
var id = "pete-feed-" + i;
|
||||
var row = document.createElement("label");
|
||||
row.setAttribute("for", id);
|
||||
row.className = "flex items-center justify-between gap-3 rounded-2xl px-3 py-2 hover:bg-[color:var(--ink)]/5 cursor-pointer";
|
||||
var nameEl = document.createElement("span");
|
||||
nameEl.className = "text-sm font-semibold truncate";
|
||||
nameEl.textContent = src.name;
|
||||
var box = document.createElement("input");
|
||||
box.type = "checkbox";
|
||||
box.id = id;
|
||||
box.className = "h-4 w-4 shrink-0 accent-[color:var(--accent)]";
|
||||
box.checked = !disabled[src.name];
|
||||
box.addEventListener("change", function () {
|
||||
if (box.checked) delete disabled[src.name];
|
||||
else disabled[src.name] = true;
|
||||
save(disabled);
|
||||
apply();
|
||||
updateMeta();
|
||||
});
|
||||
row.appendChild(nameEl);
|
||||
row.appendChild(box);
|
||||
list.appendChild(row);
|
||||
});
|
||||
}
|
||||
updateMeta();
|
||||
|
||||
function updateMeta() {
|
||||
if (!meta) return;
|
||||
var off = Object.keys(disabled).length;
|
||||
meta.textContent = off === 0 ? "All feeds enabled" : off + " hidden";
|
||||
}
|
||||
}
|
||||
|
||||
function open() {
|
||||
render();
|
||||
var dlg = document.getElementById("pete-settings");
|
||||
if (dlg) dlg.classList.remove("hidden");
|
||||
}
|
||||
function close() {
|
||||
var dlg = document.getElementById("pete-settings");
|
||||
if (dlg) dlg.classList.add("hidden");
|
||||
}
|
||||
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
apply();
|
||||
var trigger = document.querySelector("[data-settings-trigger]");
|
||||
if (trigger) trigger.addEventListener("click", open);
|
||||
var closeBtn = document.querySelector("[data-settings-close]");
|
||||
if (closeBtn) closeBtn.addEventListener("click", close);
|
||||
var dlg = document.getElementById("pete-settings");
|
||||
if (dlg) dlg.addEventListener("click", function (e) {
|
||||
if (e.target === dlg) close();
|
||||
});
|
||||
var reset = document.querySelector("[data-settings-reset]");
|
||||
if (reset) reset.addEventListener("click", function () {
|
||||
disabled = {};
|
||||
save(disabled);
|
||||
apply();
|
||||
render();
|
||||
});
|
||||
document.addEventListener("keydown", function (e) {
|
||||
if (e.key === "Escape") close();
|
||||
});
|
||||
});
|
||||
})();
|
||||
487
internal/web/static/js/weather-2d.js
Normal file
487
internal/web/static/js/weather-2d.js
Normal file
@@ -0,0 +1,487 @@
|
||||
// Canvas2D weather engine — the fallback when WebGL2 isn't available. This is
|
||||
// the original hand-drawn renderer wrapped as an engine factory; weather.js
|
||||
// owns the toggle, storage and the PeteWeather API and just calls
|
||||
// set/start/stop here. New GPU-only variants (hail, haze, wind, aurora) alias
|
||||
// to their nearest 2D look so the fallback never renders a blank page.
|
||||
(function () {
|
||||
window.PeteWeatherEngines = window.PeteWeatherEngines || {};
|
||||
|
||||
window.PeteWeatherEngines.canvas2d = function (canvas) {
|
||||
var ctx = canvas.getContext("2d");
|
||||
if (!ctx) return null;
|
||||
var root = document.documentElement;
|
||||
|
||||
var W = 0, H = 0, DPR = 1;
|
||||
function resize() {
|
||||
DPR = Math.min(window.devicePixelRatio || 1, 2);
|
||||
W = canvas.clientWidth || window.innerWidth;
|
||||
H = canvas.clientHeight || window.innerHeight;
|
||||
canvas.width = Math.floor(W * DPR);
|
||||
canvas.height = Math.floor(H * DPR);
|
||||
ctx.setTransform(DPR, 0, 0, DPR, 0, 0);
|
||||
}
|
||||
resize();
|
||||
window.addEventListener("resize", resize);
|
||||
|
||||
var aliases = { hail: "snow", haze: "motes", wind: "leaves", aurora: "clear" };
|
||||
|
||||
var counts = {
|
||||
rain: 160, petals: 50, jacaranda: 55, motes: 70, leaves: 38,
|
||||
snow: 150, clouds: 7, fog: 7, clear: 0, storm: 200
|
||||
};
|
||||
var mults = { light: 0.6, medium: 1.0, heavy: 1.4 };
|
||||
|
||||
function rand(a, b) { return a + Math.random() * (b - a); }
|
||||
|
||||
var variant = null; // current rendered variant
|
||||
var intensity = "heavy"; // light | medium | heavy
|
||||
var particles = [];
|
||||
var flash = 0; // lightning flash decay (storm only)
|
||||
var nextBolt = 2; // seconds until next lightning bolt (storm only)
|
||||
|
||||
function spawn(initial) {
|
||||
var p = {};
|
||||
p.x = rand(0, W);
|
||||
p.y = initial ? rand(0, H) : rand(-40, -10);
|
||||
p.z = rand(0.7, 1.3); // depth — affects size + speed + alpha
|
||||
if (variant === "rain" || variant === "storm") {
|
||||
p.vy = rand(700, 1100) * p.z;
|
||||
p.vx = -rand(60, 120);
|
||||
p.len = rand(10, 18) * p.z;
|
||||
p.alpha = rand(0.25, 0.55);
|
||||
} else if (variant === "snow") {
|
||||
p.vy = rand(35, 75) * p.z;
|
||||
p.swayAmp = rand(12, 34);
|
||||
p.swayFreq = rand(0.3, 0.8);
|
||||
p.swayPhase = rand(0, Math.PI * 2);
|
||||
p.size = rand(2, 4.6) * p.z;
|
||||
p.alpha = rand(0.65, 0.95);
|
||||
} else if (variant === "clouds") {
|
||||
// Soft puffs drifting across the upper sky.
|
||||
p.y = initial ? rand(0, H * 0.5) : rand(-H * 0.1, H * 0.45);
|
||||
p.x = initial ? rand(0, W) : -rand(120, 320);
|
||||
p.vx = rand(8, 22);
|
||||
p.size = rand(80, 170) * p.z;
|
||||
p.alpha = rand(0.32, 0.55);
|
||||
p.sprite = Math.floor(rand(0, 3));
|
||||
} else if (variant === "fog") {
|
||||
// Wide translucent bands creeping sideways.
|
||||
p.y = rand(H * 0.1, H);
|
||||
p.x = initial ? rand(0, W) : -rand(200, 500);
|
||||
p.vx = rand(6, 16);
|
||||
p.w = rand(260, 520);
|
||||
p.h = rand(70, 150);
|
||||
p.alpha = rand(0.05, 0.14);
|
||||
} else if (variant === "petals" || variant === "jacaranda") {
|
||||
p.vy = rand(60, 120);
|
||||
p.swayAmp = rand(20, 50);
|
||||
p.swayFreq = rand(0.4, 0.9);
|
||||
p.swayPhase = rand(0, Math.PI * 2);
|
||||
p.rot = rand(0, Math.PI * 2);
|
||||
p.vrot = rand(-1.2, 1.2);
|
||||
p.size = rand(8, 16) * p.z;
|
||||
p.alpha = rand(0.75, 1.0);
|
||||
if (variant === "jacaranda") {
|
||||
p.hue = rand(265, 285); // blue-violet — jacaranda uses the petal silhouette in purple
|
||||
p.sat = rand(55, 75);
|
||||
p.light = rand(70, 82);
|
||||
} else {
|
||||
p.hue = rand(335, 355); // pink (almond/cherry)
|
||||
p.sat = rand(60, 80);
|
||||
p.light = rand(78, 88);
|
||||
}
|
||||
} else if (variant === "motes") {
|
||||
p.vx = rand(-15, 25);
|
||||
p.vy = rand(8, 25);
|
||||
p.size = rand(1.2, 2.8) * p.z;
|
||||
p.alpha = rand(0.35, 0.7);
|
||||
p.twinklePhase = rand(0, Math.PI * 2);
|
||||
p.twinkleFreq = rand(0.5, 1.5);
|
||||
} else if (variant === "leaves") {
|
||||
p.vy = rand(70, 130);
|
||||
p.swayAmp = rand(30, 70);
|
||||
p.swayFreq = rand(0.3, 0.6);
|
||||
p.swayPhase = rand(0, Math.PI * 2);
|
||||
p.rot = rand(0, Math.PI * 2);
|
||||
p.vrot = rand(-2.0, 2.0);
|
||||
p.size = rand(10, 18) * p.z;
|
||||
p.alpha = rand(0.8, 1.0);
|
||||
p.hue = rand(18, 42); // orange→amber→brown
|
||||
p.light = rand(38, 55);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
var night = root.dataset.phase === "night";
|
||||
function refreshNight() { night = root.dataset.phase === "night"; }
|
||||
|
||||
function drawRain(p) {
|
||||
// Day/dawn/dusk backgrounds are warm and pale, so the rain needs a darker,
|
||||
// more saturated blue plus a slightly thicker line to stay visible. Night
|
||||
// keeps a paler tone so streaks read against the dark palette.
|
||||
if (night) {
|
||||
ctx.strokeStyle = "rgba(200,215,240," + p.alpha + ")";
|
||||
ctx.lineWidth = 1;
|
||||
} else {
|
||||
ctx.strokeStyle = "rgba(60,95,150," + Math.min(1, p.alpha + 0.25) + ")";
|
||||
ctx.lineWidth = 1.3;
|
||||
}
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(p.x, p.y);
|
||||
ctx.lineTo(p.x - p.len * 0.12, p.y + p.len);
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
function drawSnow(p) {
|
||||
// Soft round flake with a faint glow so it reads on light and dark palettes.
|
||||
var col = night ? "235,242,255" : "255,255,255";
|
||||
ctx.fillStyle = "rgba(" + col + "," + p.alpha + ")";
|
||||
ctx.beginPath();
|
||||
ctx.arc(p.x, p.y, p.size, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
if (!night) {
|
||||
// A thin cool outline keeps white flakes visible against cream backgrounds.
|
||||
ctx.strokeStyle = "rgba(150,180,220," + (p.alpha * 0.5) + ")";
|
||||
ctx.lineWidth = 0.6;
|
||||
ctx.stroke();
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-baked cloud sprites: clustered blobs softened with a heavy blur so each
|
||||
// cloud reads as one fluffy mass. Built once per palette (day/night) onto
|
||||
// offscreen canvases, then just drifted — cheap to animate.
|
||||
var cloudSprites = null;
|
||||
var cloudSpritesNight = null;
|
||||
|
||||
// Three silhouettes for variety. Each is a list of blobs [cx, cy, r] in a
|
||||
// 0..1 box (x across, y down) with a flattish base around y≈0.66.
|
||||
var cloudShapes = [
|
||||
[[0.50, 0.50, 0.26], [0.34, 0.56, 0.20], [0.66, 0.56, 0.20], [0.42, 0.40, 0.18],
|
||||
[0.58, 0.38, 0.17], [0.22, 0.60, 0.15], [0.78, 0.60, 0.15], [0.50, 0.62, 0.22]],
|
||||
[[0.46, 0.52, 0.24], [0.30, 0.58, 0.18], [0.62, 0.50, 0.22], [0.74, 0.58, 0.16],
|
||||
[0.40, 0.40, 0.16], [0.56, 0.36, 0.15], [0.18, 0.62, 0.13], [0.84, 0.62, 0.12]],
|
||||
[[0.52, 0.50, 0.28], [0.36, 0.56, 0.19], [0.68, 0.56, 0.18], [0.48, 0.36, 0.16],
|
||||
[0.26, 0.60, 0.14], [0.74, 0.60, 0.15], [0.60, 0.62, 0.18], [0.40, 0.62, 0.18]]
|
||||
];
|
||||
|
||||
function makeCloudSprite(col, shape) {
|
||||
var c = document.createElement("canvas");
|
||||
var w = 320, h = 224;
|
||||
c.width = w; c.height = h;
|
||||
var cx = c.getContext("2d");
|
||||
cx.filter = "blur(18px)"; // the softness that turns blobs into a cloud
|
||||
cx.fillStyle = "rgba(" + col + ",1)";
|
||||
for (var i = 0; i < shape.length; i++) {
|
||||
cx.beginPath();
|
||||
cx.arc(shape[i][0] * w, shape[i][1] * h, shape[i][2] * w, 0, Math.PI * 2);
|
||||
cx.fill();
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
function ensureCloudSprites() {
|
||||
if (cloudSprites && cloudSpritesNight === night) return;
|
||||
// Day sky is warm cream so clouds need a cool slate-grey; night uses a pale
|
||||
// tone against the dark palette.
|
||||
var col = night ? "206,216,236" : "150,164,190";
|
||||
cloudSprites = cloudShapes.map(function (s) { return makeCloudSprite(col, s); });
|
||||
cloudSpritesNight = night;
|
||||
}
|
||||
|
||||
function drawCloud(p) {
|
||||
ensureCloudSprites();
|
||||
var spr = cloudSprites[p.sprite % cloudSprites.length];
|
||||
var w = p.size * 2.6;
|
||||
var h = w * (spr.height / spr.width);
|
||||
ctx.globalAlpha = p.alpha;
|
||||
ctx.drawImage(spr, p.x - w / 2, p.y - h / 2, w, h);
|
||||
ctx.globalAlpha = 1;
|
||||
}
|
||||
|
||||
function drawFog(p) {
|
||||
var col = night ? "150,160,180" : "230,228,224";
|
||||
var g = ctx.createLinearGradient(p.x - p.w / 2, 0, p.x + p.w / 2, 0);
|
||||
g.addColorStop(0, "rgba(" + col + ",0)");
|
||||
g.addColorStop(0.5, "rgba(" + col + "," + p.alpha + ")");
|
||||
g.addColorStop(1, "rgba(" + col + ",0)");
|
||||
ctx.fillStyle = g;
|
||||
ctx.beginPath();
|
||||
ctx.ellipse(p.x, p.y, p.w / 2, p.h / 2, 0, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
|
||||
function drawMoon(cx, cy, mr) {
|
||||
var TAU = Math.PI * 2;
|
||||
ctx.save();
|
||||
// Clip everything to the lunar disc so shading stays inside the sphere.
|
||||
ctx.beginPath();
|
||||
ctx.arc(cx, cy, mr, 0, TAU);
|
||||
ctx.clip();
|
||||
|
||||
// Spherical body shading — light source upper-right, terminator lower-left.
|
||||
var lx = cx + mr * 0.45, ly = cy - mr * 0.45;
|
||||
var body = ctx.createRadialGradient(lx, ly, mr * 0.1, cx, cy, mr * 1.35);
|
||||
body.addColorStop(0, "rgba(249,251,255,0.97)");
|
||||
body.addColorStop(0.55, "rgba(214,222,240,0.92)");
|
||||
body.addColorStop(1, "rgba(140,151,180,0.85)");
|
||||
ctx.fillStyle = body;
|
||||
ctx.fillRect(cx - mr, cy - mr, mr * 2, mr * 2);
|
||||
|
||||
// Limb darkening — fade the edge so the disc reads as a ball, not a coin.
|
||||
var limb = ctx.createRadialGradient(cx, cy, mr * 0.62, cx, cy, mr);
|
||||
limb.addColorStop(0, "rgba(18,24,46,0)");
|
||||
limb.addColorStop(1, "rgba(18,24,46,0.4)");
|
||||
ctx.fillStyle = limb;
|
||||
ctx.fillRect(cx - mr, cy - mr, mr * 2, mr * 2);
|
||||
ctx.restore();
|
||||
|
||||
// Soft outer halo, drawn unclipped around the disc.
|
||||
var halo = ctx.createRadialGradient(cx, cy, mr, cx, cy, mr * 2.4);
|
||||
halo.addColorStop(0, "rgba(220,230,255,0.28)");
|
||||
halo.addColorStop(1, "rgba(220,230,255,0)");
|
||||
ctx.fillStyle = halo;
|
||||
ctx.beginPath();
|
||||
ctx.arc(cx, cy, mr * 2.4, 0, TAU);
|
||||
ctx.fill();
|
||||
}
|
||||
|
||||
function drawClearGlow(t) {
|
||||
// No particles — render a single calm sun (day) or moon (night) glow that
|
||||
// breathes very slowly, plus a few stars at night.
|
||||
var breathe = 0.5 + 0.5 * Math.sin(t * 0.25);
|
||||
var cx = W * 0.82, cy = H * 0.18, r = Math.min(W, H) * 0.5;
|
||||
if (night) {
|
||||
// Ambient sky glow around the moon.
|
||||
var g = ctx.createRadialGradient(cx, cy, 0, cx, cy, r);
|
||||
g.addColorStop(0, "rgba(220,230,255," + (0.12 + 0.05 * breathe) + ")");
|
||||
g.addColorStop(1, "rgba(220,230,255,0)");
|
||||
ctx.fillStyle = g;
|
||||
ctx.fillRect(0, 0, W, H);
|
||||
drawMoon(cx, cy, r * 0.16);
|
||||
} else {
|
||||
var gd = ctx.createRadialGradient(cx, cy, 0, cx, cy, r);
|
||||
gd.addColorStop(0, "rgba(255,224,150," + (0.30 + 0.10 * breathe) + ")");
|
||||
gd.addColorStop(0.5, "rgba(255,214,120," + (0.10 + 0.04 * breathe) + ")");
|
||||
gd.addColorStop(1, "rgba(255,214,120,0)");
|
||||
ctx.fillStyle = gd;
|
||||
ctx.fillRect(0, 0, W, H);
|
||||
}
|
||||
}
|
||||
|
||||
// Deterministic star field for clear nights (seeded so they don't jitter).
|
||||
var stars = [];
|
||||
function buildStars() {
|
||||
stars = [];
|
||||
var n = 60;
|
||||
for (var i = 0; i < n; i++) {
|
||||
// Cheap LCG-ish spread keyed by index — stable across frames.
|
||||
var sx = ((i * 73 + 11) % 100) / 100;
|
||||
var sy = ((i * 37 + 7) % 100) / 100;
|
||||
stars.push({ x: sx, y: sy * 0.7, r: 0.6 + (i % 3) * 0.5, ph: (i % 7) });
|
||||
}
|
||||
}
|
||||
function drawStars(t) {
|
||||
for (var i = 0; i < stars.length; i++) {
|
||||
var s = stars[i];
|
||||
var tw = 0.4 + 0.6 * (0.5 + 0.5 * Math.sin(t * 0.8 + s.ph));
|
||||
ctx.fillStyle = "rgba(255,255,255," + (0.5 * tw) + ")";
|
||||
ctx.beginPath();
|
||||
ctx.arc(s.x * W, s.y * H, s.r, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
|
||||
function drawPetals(p) {
|
||||
// Five-petal almond/cherry blossom viewed face-on, with a yellow center.
|
||||
var s = p.size;
|
||||
ctx.save();
|
||||
ctx.translate(p.x, p.y);
|
||||
ctx.rotate(p.rot);
|
||||
ctx.fillStyle = "hsla(" + p.hue + "," + p.sat + "%," + p.light + "%," + p.alpha + ")";
|
||||
for (var i = 0; i < 5; i++) {
|
||||
var a = (i / 5) * Math.PI * 2 - Math.PI / 2;
|
||||
var cx = Math.cos(a) * s * 0.32;
|
||||
var cy = Math.sin(a) * s * 0.32;
|
||||
ctx.beginPath();
|
||||
ctx.ellipse(cx, cy, s * 0.38, s * 0.24, a, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
ctx.fillStyle = "hsla(48,90%,68%," + p.alpha + ")";
|
||||
ctx.beginPath();
|
||||
ctx.arc(0, 0, s * 0.13, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function drawLeaf(p) {
|
||||
var s = p.size;
|
||||
ctx.save();
|
||||
ctx.translate(p.x, p.y);
|
||||
ctx.rotate(p.rot);
|
||||
var g = ctx.createLinearGradient(-s * 0.5, 0, s * 0.5, 0);
|
||||
g.addColorStop(0, "hsla(" + p.hue + ",70%," + (p.light - 10) + "%," + p.alpha + ")");
|
||||
g.addColorStop(1, "hsla(" + p.hue + ",75%," + (p.light + 10) + "%," + p.alpha + ")");
|
||||
ctx.fillStyle = g;
|
||||
// Almond/lozenge leaf — pointed at both ends.
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(-s * 0.55, 0);
|
||||
ctx.quadraticCurveTo(0, -s * 0.38, s * 0.55, 0);
|
||||
ctx.quadraticCurveTo(0, s * 0.38, -s * 0.55, 0);
|
||||
ctx.closePath();
|
||||
ctx.fill();
|
||||
// Midrib vein.
|
||||
ctx.strokeStyle = "hsla(" + p.hue + ",60%," + (p.light - 22) + "%," + (p.alpha * 0.7) + ")";
|
||||
ctx.lineWidth = 0.8;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(-s * 0.5, 0);
|
||||
ctx.lineTo(s * 0.5, 0);
|
||||
ctx.stroke();
|
||||
// Small stem nub at the base.
|
||||
ctx.strokeStyle = "hsla(" + p.hue + ",55%," + (p.light - 25) + "%," + (p.alpha * 0.7) + ")";
|
||||
ctx.lineWidth = 1.0;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(-s * 0.55, 0);
|
||||
ctx.lineTo(-s * 0.7, -s * 0.04);
|
||||
ctx.stroke();
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function drawMote(p, t) {
|
||||
var twinkle = 0.5 + 0.5 * Math.sin(t * p.twinkleFreq + p.twinklePhase);
|
||||
// Night keeps a pale glowing mote against the dark palette. Day/dawn/dusk
|
||||
// use a deeper saturated honey so the motes stand out against warm cream
|
||||
// and peach backgrounds; alpha also gets a bump.
|
||||
var color, a;
|
||||
if (night) {
|
||||
color = "255,240,180";
|
||||
a = p.alpha * (0.45 + 0.55 * twinkle);
|
||||
} else {
|
||||
color = "130,80,180";
|
||||
a = Math.min(1, (p.alpha + 0.2) * (0.6 + 0.4 * twinkle));
|
||||
}
|
||||
// Soft glow halo.
|
||||
var g = ctx.createRadialGradient(p.x, p.y, 0, p.x, p.y, p.size * 5);
|
||||
g.addColorStop(0, "rgba(" + color + "," + (a * 0.55) + ")");
|
||||
g.addColorStop(1, "rgba(" + color + ",0)");
|
||||
ctx.fillStyle = g;
|
||||
ctx.beginPath();
|
||||
ctx.arc(p.x, p.y, p.size * 5, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
// Bright core.
|
||||
ctx.fillStyle = "rgba(" + color + "," + a + ")";
|
||||
ctx.beginPath();
|
||||
ctx.arc(p.x, p.y, p.size, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
|
||||
var last = performance.now();
|
||||
var raf = 0;
|
||||
|
||||
function step(now) {
|
||||
var dt = Math.min(0.05, (now - last) / 1000);
|
||||
last = now;
|
||||
var t = now / 1000;
|
||||
ctx.clearRect(0, 0, W, H);
|
||||
refreshNight();
|
||||
|
||||
if (variant === "clear") {
|
||||
drawClearGlow(t);
|
||||
if (night) drawStars(t);
|
||||
raf = requestAnimationFrame(step);
|
||||
return;
|
||||
}
|
||||
|
||||
// Storm: drive the lightning flash before drawing rain over it.
|
||||
if (variant === "storm") {
|
||||
nextBolt -= dt;
|
||||
if (nextBolt <= 0) {
|
||||
flash = 1;
|
||||
nextBolt = rand(2.5, 7);
|
||||
}
|
||||
if (flash > 0) {
|
||||
ctx.fillStyle = "rgba(235,240,255," + (flash * 0.5) + ")";
|
||||
ctx.fillRect(0, 0, W, H);
|
||||
flash = Math.max(0, flash - dt * 4); // ~0.25s decay
|
||||
}
|
||||
}
|
||||
|
||||
for (var i = 0; i < particles.length; i++) {
|
||||
var p = particles[i];
|
||||
if (variant === "rain" || variant === "storm") {
|
||||
p.x += p.vx * dt;
|
||||
p.y += p.vy * dt;
|
||||
if (p.y > H + 20 || p.x < -40) {
|
||||
particles[i] = spawn(false);
|
||||
particles[i].x = rand(0, W + 60);
|
||||
continue;
|
||||
}
|
||||
drawRain(p);
|
||||
} else if (variant === "snow") {
|
||||
p.y += p.vy * dt;
|
||||
var sxOff = Math.sin(t * p.swayFreq + p.swayPhase) * p.swayAmp;
|
||||
if (p.y > H + 10) { particles[i] = spawn(false); continue; }
|
||||
var ox = p.x; p.x = ox + sxOff;
|
||||
drawSnow(p);
|
||||
p.x = ox;
|
||||
} else if (variant === "clouds") {
|
||||
p.x += p.vx * dt;
|
||||
if (p.x - p.size * 1.3 > W + 40) { particles[i] = spawn(false); continue; }
|
||||
drawCloud(p);
|
||||
} else if (variant === "fog") {
|
||||
p.x += p.vx * dt;
|
||||
if (p.x - p.w / 2 > W + 40) { particles[i] = spawn(false); continue; }
|
||||
drawFog(p);
|
||||
} else if (variant === "motes") {
|
||||
p.x += p.vx * dt;
|
||||
p.y += p.vy * dt;
|
||||
if (p.y > H + 10 || p.x < -10 || p.x > W + 10) {
|
||||
particles[i] = spawn(false);
|
||||
continue;
|
||||
}
|
||||
drawMote(p, t);
|
||||
} else {
|
||||
// Drifting variants: petals, jacaranda, leaves.
|
||||
p.y += p.vy * dt;
|
||||
p.rot += p.vrot * dt;
|
||||
var sway = Math.sin(t * p.swayFreq + p.swayPhase) * p.swayAmp;
|
||||
var origX = p.x;
|
||||
p.x = origX + sway;
|
||||
if (p.y > H + 30) {
|
||||
particles[i] = spawn(false);
|
||||
continue;
|
||||
}
|
||||
if (variant === "jacaranda" || variant === "petals") drawPetals(p);
|
||||
else drawLeaf(p);
|
||||
p.x = origX;
|
||||
}
|
||||
}
|
||||
raf = requestAnimationFrame(step);
|
||||
}
|
||||
|
||||
function start() {
|
||||
if (raf) return;
|
||||
last = performance.now();
|
||||
raf = requestAnimationFrame(step);
|
||||
}
|
||||
function stop() {
|
||||
if (raf) cancelAnimationFrame(raf);
|
||||
raf = 0;
|
||||
ctx.clearRect(0, 0, W, H);
|
||||
}
|
||||
|
||||
// Build the particle pool for a variant. The controller decides whether to
|
||||
// start rendering afterwards.
|
||||
function set(v, inten) {
|
||||
variant = aliases[v] || v || null;
|
||||
intensity = inten || "heavy";
|
||||
flash = 0; nextBolt = rand(1.5, 4);
|
||||
particles = [];
|
||||
if (variant === "clear") { buildStars(); }
|
||||
if (!variant) { stop(); return; }
|
||||
var N = Math.round((counts[variant] || 0) * (mults[intensity] || 1));
|
||||
for (var i = 0; i < N; i++) particles.push(spawn(true));
|
||||
}
|
||||
|
||||
return { name: "canvas2d", set: set, start: start, stop: stop };
|
||||
};
|
||||
})();
|
||||
436
internal/web/static/js/weather-forecast.js
Normal file
436
internal/web/static/js/weather-forecast.js
Normal file
@@ -0,0 +1,436 @@
|
||||
// Live local weather. When the visitor saves a postal code we geocode it once
|
||||
// (Zippopotam — free, no key), then poll Open-Meteo (free, no key) for the
|
||||
// current conditions + a 5-day forecast. The result drives two bits of UI — a
|
||||
// header chip and a home-page card — and overrides the seasonal canvas via
|
||||
// PeteWeather.set(). Everything lives in the browser: no server calls, no keys.
|
||||
//
|
||||
// Politeness: the geocode never changes for a postal code, so it's cached
|
||||
// forever. Forecasts are cached for 2 hours so a reload (or hopping between
|
||||
// pages) doesn't re-hit Open-Meteo.
|
||||
(function () {
|
||||
var LOC_KEY = "pete.weather.loc.v1"; // {country, postal, lat, lon, name, unit}
|
||||
var CACHE_KEY = "pete.weather.cache.v2"; // {key, fetchedAt, current, daily, aqi}
|
||||
var TTL_MS = 2 * 60 * 60 * 1000; // 2 hours
|
||||
|
||||
// ---- storage helpers -----------------------------------------------------
|
||||
function readJSON(key) {
|
||||
try { var r = localStorage.getItem(key); return r ? JSON.parse(r) : null; }
|
||||
catch (e) { return null; }
|
||||
}
|
||||
function writeJSON(key, val) {
|
||||
try { localStorage.setItem(key, JSON.stringify(val)); } catch (e) {}
|
||||
}
|
||||
function getLoc() { return readJSON(LOC_KEY); }
|
||||
function setLoc(loc) {
|
||||
if (loc) writeJSON(LOC_KEY, loc);
|
||||
else { try { localStorage.removeItem(LOC_KEY); } catch (e) {} }
|
||||
if (window.PetePrefs) window.PetePrefs.push();
|
||||
}
|
||||
|
||||
// ---- WMO weather code → canvas variant + label + emoji -------------------
|
||||
// https://open-meteo.com/en/docs — `weather_code` is the WMO code.
|
||||
function describe(code, isDay) {
|
||||
var c = Number(code);
|
||||
function r(variant, intensity, label, dayEmoji, nightEmoji) {
|
||||
return { variant: variant, intensity: intensity, label: label,
|
||||
emoji: (isDay === 0 && nightEmoji) ? nightEmoji : dayEmoji };
|
||||
}
|
||||
if (c === 0) return r("clear", "heavy", "Clear sky", "☀️", "🌙");
|
||||
if (c === 1) return r("clouds", "light", "Mainly clear", "🌤️", "🌙");
|
||||
if (c === 2) return r("clouds", "medium", "Partly cloudy", "⛅", "☁️");
|
||||
if (c === 3) return r("clouds", "heavy", "Overcast", "☁️", "☁️");
|
||||
if (c === 45 || c === 48) return r("fog", "heavy", "Fog", "🌫️", "🌫️");
|
||||
if (c === 51) return r("rain", "light", "Light drizzle", "🌦️", "🌧️");
|
||||
if (c === 53) return r("rain", "medium", "Drizzle", "🌦️", "🌧️");
|
||||
if (c === 55) return r("rain", "heavy", "Heavy drizzle", "🌧️", "🌧️");
|
||||
if (c === 56 || c === 57) return r("rain", "medium", "Freezing drizzle", "🌧️", "🌧️");
|
||||
if (c === 61) return r("rain", "light", "Light rain", "🌦️", "🌧️");
|
||||
if (c === 63) return r("rain", "medium", "Rain", "🌧️", "🌧️");
|
||||
if (c === 65) return r("rain", "heavy", "Heavy rain", "🌧️", "🌧️");
|
||||
if (c === 66 || c === 67) return r("rain", "medium", "Freezing rain", "🌧️", "🌧️");
|
||||
if (c === 71) return r("snow", "light", "Light snow", "🌨️", "🌨️");
|
||||
if (c === 73) return r("snow", "medium", "Snow", "🌨️", "🌨️");
|
||||
if (c === 75) return r("snow", "heavy", "Heavy snow", "❄️", "❄️");
|
||||
if (c === 77) return r("snow", "light", "Snow grains", "🌨️", "🌨️");
|
||||
if (c === 80) return r("rain", "medium", "Light showers", "🌦️", "🌧️");
|
||||
if (c === 81) return r("rain", "medium", "Showers", "🌧️", "🌧️");
|
||||
if (c === 82) return r("rain", "heavy", "Heavy showers", "⛈️", "⛈️");
|
||||
if (c === 85) return r("snow", "medium", "Snow showers", "🌨️", "🌨️");
|
||||
if (c === 86) return r("snow", "heavy", "Heavy snow showers", "❄️", "❄️");
|
||||
if (c === 95) return r("storm", "heavy", "Thunderstorm", "⛈️", "⛈️");
|
||||
if (c === 96 || c === 99) return r("storm", "heavy", "Thunderstorm + hail", "⛈️", "⛈️");
|
||||
return r("clouds", "medium", "—", "☁️", "☁️");
|
||||
}
|
||||
|
||||
// ---- US EPA AQI → label + color -----------------------------------------
|
||||
// https://www.airnow.gov/aqi/aqi-basics/ — standard six-band scale.
|
||||
function describeAqi(v) {
|
||||
if (v == null) return null;
|
||||
if (v <= 50) return { label: "Good", color: "#16a34a" };
|
||||
if (v <= 100) return { label: "Moderate", color: "#ca8a04" };
|
||||
if (v <= 150) return { label: "Unhealthy for sensitive", color: "#ea580c" };
|
||||
if (v <= 200) return { label: "Unhealthy", color: "#dc2626" };
|
||||
if (v <= 300) return { label: "Very unhealthy", color: "#9333ea" };
|
||||
return { label: "Hazardous", color: "#7f1d1d" };
|
||||
}
|
||||
|
||||
// ---- API calls -----------------------------------------------------------
|
||||
function geocode(country, postal) {
|
||||
var url = "https://api.zippopotam.us/" + encodeURIComponent(country.toLowerCase()) +
|
||||
"/" + encodeURIComponent(postal);
|
||||
return fetch(url).then(function (res) {
|
||||
if (!res.ok) throw new Error("Unknown postal code");
|
||||
return res.json();
|
||||
}).then(function (data) {
|
||||
var place = data.places && data.places[0];
|
||||
if (!place) throw new Error("Unknown postal code");
|
||||
return {
|
||||
lat: parseFloat(place.latitude),
|
||||
lon: parseFloat(place.longitude),
|
||||
name: place["place name"] + (place["state abbreviation"] ? ", " + place["state abbreviation"] : "")
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function fetchForecast(loc) {
|
||||
var unit = loc.unit === "fahrenheit" ? "fahrenheit" : "celsius";
|
||||
var url = "https://api.open-meteo.com/v1/forecast" +
|
||||
"?latitude=" + loc.lat + "&longitude=" + loc.lon +
|
||||
"¤t=temperature_2m,weather_code,is_day" +
|
||||
"&daily=weather_code,temperature_2m_max,temperature_2m_min" +
|
||||
"&temperature_unit=" + unit +
|
||||
"&timezone=auto&forecast_days=5";
|
||||
return fetch(url).then(function (res) {
|
||||
if (!res.ok) throw new Error("Forecast unavailable");
|
||||
return res.json();
|
||||
});
|
||||
}
|
||||
|
||||
// Air quality rides on the same lat/lon. Separate Open-Meteo host, no key,
|
||||
// returns a pre-computed US EPA AQI so we don't reimplement the breakpoints.
|
||||
function fetchAqi(loc) {
|
||||
var url = "https://air-quality-api.open-meteo.com/v1/air-quality" +
|
||||
"?latitude=" + loc.lat + "&longitude=" + loc.lon +
|
||||
"¤t=us_aqi&timezone=auto";
|
||||
return fetch(url).then(function (res) {
|
||||
if (!res.ok) throw new Error("AQI unavailable");
|
||||
return res.json();
|
||||
});
|
||||
}
|
||||
|
||||
// Cache key ties a forecast to one location; changing location invalidates it.
|
||||
function cacheKey(loc) { return loc.country + ":" + loc.postal + ":" + (loc.unit || "celsius"); }
|
||||
|
||||
function loadForecast(loc, force) {
|
||||
var key = cacheKey(loc);
|
||||
var cached = readJSON(CACHE_KEY);
|
||||
var fresh = cached && cached.key === key && cached.fetchedAt &&
|
||||
(nowMs() - cached.fetchedAt) < TTL_MS;
|
||||
if (fresh && !force) return Promise.resolve(cached);
|
||||
return Promise.all([
|
||||
fetchForecast(loc),
|
||||
fetchAqi(loc).catch(function () { return null; }) // AQI is best-effort
|
||||
]).then(function (results) {
|
||||
var data = results[0], air = results[1];
|
||||
var entry = {
|
||||
key: key,
|
||||
fetchedAt: nowMs(),
|
||||
current: data.current,
|
||||
daily: data.daily,
|
||||
unitSymbol: (data.current_units && data.current_units.temperature_2m) || "°",
|
||||
aqi: (air && air.current && typeof air.current.us_aqi === "number") ? air.current.us_aqi : null
|
||||
};
|
||||
writeJSON(CACHE_KEY, entry);
|
||||
return entry;
|
||||
}).catch(function (err) {
|
||||
// Stale-but-present beats nothing if the network blips.
|
||||
if (cached && cached.key === key) return cached;
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
|
||||
function nowMs() { return new Date().getTime(); }
|
||||
|
||||
// ---- rendering -----------------------------------------------------------
|
||||
var chipEl = document.querySelector("[data-weather-chip]");
|
||||
var chipBtn = document.querySelector("[data-weather-loc]");
|
||||
var aqiChipEl = document.querySelector("[data-aqi-chip]");
|
||||
var cardMount = document.querySelector("[data-weather-card]");
|
||||
|
||||
function setChip(html, title) {
|
||||
if (chipEl) chipEl.innerHTML = html;
|
||||
if (chipBtn && title) chipBtn.title = title;
|
||||
}
|
||||
|
||||
// The AQI chip hides itself when there's no reading (no location, or the
|
||||
// air-quality fetch failed) so it never shows an empty pill.
|
||||
function setAqiChip(aqi) {
|
||||
if (!aqiChipEl) return;
|
||||
var air = describeAqi(aqi);
|
||||
if (!air) { aqiChipEl.classList.add("hidden"); aqiChipEl.classList.remove("inline-flex"); return; }
|
||||
aqiChipEl.innerHTML =
|
||||
'<span class="inline-block w-2.5 h-2.5 rounded-full" style="background:' + air.color + '"></span>' +
|
||||
'AQI ' + aqi;
|
||||
aqiChipEl.title = "Air quality · " + air.label;
|
||||
aqiChipEl.classList.remove("hidden");
|
||||
aqiChipEl.classList.add("inline-flex");
|
||||
}
|
||||
|
||||
function temp(v, sym) { return Math.round(v) + (sym || "°"); }
|
||||
|
||||
function weekday(iso) {
|
||||
var d = new Date(iso + "T00:00:00");
|
||||
return ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"][d.getDay()];
|
||||
}
|
||||
|
||||
// The /weather page is a variant preview — never let the live forecast
|
||||
// hijack the canvas the user is inspecting.
|
||||
var isDemoPage = location.pathname === "/weather";
|
||||
|
||||
function applyBackground(desc) {
|
||||
if (isDemoPage) return;
|
||||
if (window.PeteWeather && typeof window.PeteWeather.set === "function") {
|
||||
window.PeteWeather.set(desc.variant, desc.intensity);
|
||||
}
|
||||
}
|
||||
|
||||
function renderForecast(loc, entry) {
|
||||
var cur = entry.current;
|
||||
var sym = entry.unitSymbol;
|
||||
var desc = describe(cur.weather_code, cur.is_day);
|
||||
applyBackground(desc);
|
||||
|
||||
setChip(desc.emoji + " <span class=\"tabular-nums\">" + temp(cur.temperature_2m, sym) + "</span>",
|
||||
loc.name + " · " + desc.label);
|
||||
setAqiChip(entry.aqi);
|
||||
|
||||
if (!cardMount) return;
|
||||
var air = describeAqi(entry.aqi);
|
||||
var aqiRow = air
|
||||
? '<div class="flex items-center gap-2 border-t border-[color:var(--ink)]/10 px-5 py-3 text-sm">' +
|
||||
'<span class="inline-block w-2.5 h-2.5 rounded-full shrink-0" style="background:' + air.color + '"></span>' +
|
||||
'<span class="font-bold tabular-nums">AQI ' + entry.aqi + '</span>' +
|
||||
'<span class="text-[color:var(--ink)]/60">' + air.label + '</span>' +
|
||||
'</div>'
|
||||
: '';
|
||||
var daily = entry.daily;
|
||||
var days = "";
|
||||
for (var i = 0; i < daily.time.length; i++) {
|
||||
var dd = describe(daily.weather_code[i], 1);
|
||||
days +=
|
||||
'<div class="flex flex-col items-center gap-1 rounded-2xl px-2 py-2 min-w-0">' +
|
||||
'<span class="text-xs font-semibold text-[color:var(--ink)]/60">' + (i === 0 ? "Today" : weekday(daily.time[i])) + '</span>' +
|
||||
'<span class="text-2xl leading-none" aria-hidden="true">' + dd.emoji + '</span>' +
|
||||
'<span class="text-sm font-bold tabular-nums">' + Math.round(daily.temperature_2m_max[i]) + '°</span>' +
|
||||
'<span class="text-xs tabular-nums text-[color:var(--ink)]/50">' + Math.round(daily.temperature_2m_min[i]) + '°</span>' +
|
||||
'</div>';
|
||||
}
|
||||
cardMount.innerHTML =
|
||||
'<div class="rounded-3xl bg-[color:var(--card)] border-2 border-[color:var(--ink)]/10 shadow-pete overflow-hidden">' +
|
||||
'<div class="flex flex-wrap items-center justify-between gap-4 p-5">' +
|
||||
'<div class="flex items-center gap-4 min-w-0">' +
|
||||
'<span class="text-5xl leading-none" aria-hidden="true">' + desc.emoji + '</span>' +
|
||||
'<div class="min-w-0">' +
|
||||
'<div class="flex items-baseline gap-2">' +
|
||||
'<span class="font-display text-4xl font-bold tabular-nums">' + temp(cur.temperature_2m, sym) + '</span>' +
|
||||
'<span class="text-sm font-semibold text-[color:var(--ink)]/70 truncate">' + desc.label + '</span>' +
|
||||
'</div>' +
|
||||
'<div class="text-sm text-[color:var(--ink)]/60 truncate">📍 ' + escapeHtml(loc.name) + '</div>' +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'<button type="button" data-weather-loc class="shrink-0 rounded-full px-3 py-1.5 text-xs font-semibold text-[color:var(--ink)]/60 hover:bg-[color:var(--ink)]/5 border-2 border-[color:var(--ink)]/10 transition">Change location</button>' +
|
||||
'</div>' +
|
||||
'<div class="grid grid-cols-5 gap-1 border-t border-[color:var(--ink)]/10 bg-[color:var(--ink)]/[0.03] px-2 py-3">' + days + '</div>' +
|
||||
aqiRow +
|
||||
'</div>';
|
||||
// The injected "Change location" button needs the same handler.
|
||||
var changeBtn = cardMount.querySelector("[data-weather-loc]");
|
||||
if (changeBtn) changeBtn.addEventListener("click", openPopover);
|
||||
}
|
||||
|
||||
function escapeHtml(s) {
|
||||
return String(s).replace(/[&<>"']/g, function (c) {
|
||||
return { "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c];
|
||||
});
|
||||
}
|
||||
|
||||
function showError(msg) {
|
||||
setChip("📍 Weather", "Set your location");
|
||||
setAqiChip(null);
|
||||
if (cardMount && getLoc()) {
|
||||
cardMount.innerHTML =
|
||||
'<div class="rounded-3xl bg-[color:var(--card)] border-2 border-dashed border-[color:var(--ink)]/15 p-5 text-center text-sm text-[color:var(--ink)]/60">' +
|
||||
escapeHtml(msg) + ' · <button type="button" data-weather-loc class="font-semibold underline">try again</button>' +
|
||||
'</div>';
|
||||
var b = cardMount.querySelector("[data-weather-loc]");
|
||||
if (b) b.addEventListener("click", openPopover);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- location popover ----------------------------------------------------
|
||||
// Built in JS so layout.html only needs the chip button. Class strings are
|
||||
// full literals so Tailwind's content scan keeps them.
|
||||
var popover = null;
|
||||
var selectedUnit = "celsius"; // default for everyone; °F is opt-in
|
||||
|
||||
function setUnit(u) {
|
||||
selectedUnit = u === "fahrenheit" ? "fahrenheit" : "celsius";
|
||||
if (!popover) return;
|
||||
popover.querySelectorAll("[data-loc-unit]").forEach(function (b) {
|
||||
var on = b.getAttribute("data-loc-unit") === selectedUnit;
|
||||
b.classList.toggle("bg-[color:var(--accent)]", on);
|
||||
b.classList.toggle("text-white", on);
|
||||
b.classList.toggle("text-[color:var(--ink)]/60", !on);
|
||||
});
|
||||
}
|
||||
|
||||
function defaultCountry() {
|
||||
var lang = (navigator.language || "en-US");
|
||||
var m = lang.match(/[-_]([A-Za-z]{2})$/);
|
||||
return m ? m[1].toUpperCase() : "US";
|
||||
}
|
||||
|
||||
function buildPopover() {
|
||||
var dlg = document.createElement("div");
|
||||
dlg.id = "pete-weather-loc";
|
||||
dlg.className = "hidden fixed inset-0 z-50 bg-[color:var(--ink)]/40 backdrop-blur-sm flex items-start justify-center pt-[12vh] px-4";
|
||||
dlg.setAttribute("role", "dialog");
|
||||
dlg.setAttribute("aria-modal", "true");
|
||||
dlg.innerHTML =
|
||||
'<div class="w-full max-w-sm rounded-3xl bg-[color:var(--card)] shadow-pete-lg border-2 border-[color:var(--ink)]/10 overflow-hidden">' +
|
||||
'<div class="flex items-center justify-between gap-3 px-5 py-4 border-b border-[color:var(--ink)]/10">' +
|
||||
'<h2 class="font-display text-lg font-bold">Weather location</h2>' +
|
||||
'<button type="button" data-loc-close class="rounded-full px-2 py-1 text-sm text-[color:var(--ink)]/60 hover:bg-[color:var(--ink)]/5">esc</button>' +
|
||||
'</div>' +
|
||||
'<div class="p-5 space-y-3">' +
|
||||
'<p class="text-xs text-[color:var(--ink)]/60">Enter your postal code to see the local forecast. The background follows the weather. Saved in this browser only.</p>' +
|
||||
'<div class="flex gap-2">' +
|
||||
'<select data-loc-country class="rounded-2xl border-2 border-[color:var(--ink)]/10 bg-[color:var(--bg)] px-3 py-2 text-sm font-semibold outline-none focus:border-[color:var(--accent)]"></select>' +
|
||||
'<input data-loc-postal type="text" inputmode="text" autocomplete="postal-code" placeholder="Postal code" class="flex-1 min-w-0 rounded-2xl border-2 border-[color:var(--ink)]/10 bg-[color:var(--bg)] px-3 py-2 text-sm outline-none focus:border-[color:var(--accent)]">' +
|
||||
'</div>' +
|
||||
'<div class="flex items-center justify-between gap-2">' +
|
||||
'<span class="text-xs font-semibold text-[color:var(--ink)]/60">Units</span>' +
|
||||
'<div class="inline-flex rounded-full border-2 border-[color:var(--ink)]/10 p-0.5 text-xs font-bold">' +
|
||||
'<button type="button" data-loc-unit="celsius" class="rounded-full px-3 py-1 transition">°C</button>' +
|
||||
'<button type="button" data-loc-unit="fahrenheit" class="rounded-full px-3 py-1 transition">°F</button>' +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'<p data-loc-error class="hidden text-xs font-semibold text-red-500"></p>' +
|
||||
'<div class="flex items-center justify-between gap-2 pt-1">' +
|
||||
'<button type="button" data-loc-clear class="rounded-full px-3 py-2 text-xs font-semibold text-[color:var(--ink)]/50 hover:text-[color:var(--ink)]">Clear</button>' +
|
||||
'<button type="button" data-loc-save class="rounded-full bg-[color:var(--accent)] px-5 py-2 text-sm font-bold text-white shadow-pete hover:brightness-105 transition disabled:opacity-50">Save</button>' +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'</div>';
|
||||
document.body.appendChild(dlg);
|
||||
|
||||
// A small, friendly country list (Zippopotam supports ~60). Common ones up
|
||||
// top; the user can type any postal code that matches their selected country.
|
||||
var countries = [
|
||||
["US", "🇺🇸 United States"], ["PT", "🇵🇹 Portugal"], ["GB", "🇬🇧 United Kingdom"],
|
||||
["ES", "🇪🇸 Spain"], ["FR", "🇫🇷 France"], ["DE", "🇩🇪 Germany"], ["IT", "🇮🇹 Italy"],
|
||||
["NL", "🇳🇱 Netherlands"], ["BE", "🇧🇪 Belgium"], ["CH", "🇨🇭 Switzerland"],
|
||||
["AT", "🇦🇹 Austria"], ["IE", "🇮🇪 Ireland"], ["DK", "🇩🇰 Denmark"],
|
||||
["SE", "🇸🇪 Sweden"], ["PL", "🇵🇱 Poland"], ["CZ", "🇨🇿 Czechia"],
|
||||
["CA", "🇨🇦 Canada"], ["AU", "🇦🇺 Australia"], ["NZ", "🇳🇿 New Zealand"],
|
||||
["BR", "🇧🇷 Brazil"], ["MX", "🇲🇽 Mexico"], ["JP", "🇯🇵 Japan"], ["IN", "🇮🇳 India"]
|
||||
];
|
||||
var sel = dlg.querySelector("[data-loc-country]");
|
||||
countries.forEach(function (c) {
|
||||
var o = document.createElement("option");
|
||||
o.value = c[0]; o.textContent = c[1];
|
||||
sel.appendChild(o);
|
||||
});
|
||||
|
||||
dlg.addEventListener("click", function (e) { if (e.target === dlg) closePopover(); });
|
||||
dlg.querySelector("[data-loc-close]").addEventListener("click", closePopover);
|
||||
dlg.querySelector("[data-loc-clear]").addEventListener("click", function () {
|
||||
setLoc(null);
|
||||
try { localStorage.removeItem(CACHE_KEY); } catch (e) {}
|
||||
closePopover();
|
||||
// Hand the background back to the server's seasonal default.
|
||||
if (window.PeteWeather) window.PeteWeather.set(seasonalDefault.variant, seasonalDefault.intensity);
|
||||
setChip("📍 Weather", "Set your location");
|
||||
setAqiChip(null);
|
||||
if (cardMount) cardMount.innerHTML = "";
|
||||
});
|
||||
dlg.querySelectorAll("[data-loc-unit]").forEach(function (b) {
|
||||
b.addEventListener("click", function () { setUnit(b.getAttribute("data-loc-unit")); });
|
||||
});
|
||||
dlg.querySelector("[data-loc-save]").addEventListener("click", onSave);
|
||||
dlg.querySelector("[data-loc-postal]").addEventListener("keydown", function (e) {
|
||||
if (e.key === "Enter") onSave();
|
||||
});
|
||||
return dlg;
|
||||
}
|
||||
|
||||
function showLocError(msg) {
|
||||
var el = popover.querySelector("[data-loc-error]");
|
||||
if (!el) return;
|
||||
if (msg) { el.textContent = msg; el.classList.remove("hidden"); }
|
||||
else { el.classList.add("hidden"); }
|
||||
}
|
||||
|
||||
function onSave() {
|
||||
var country = popover.querySelector("[data-loc-country]").value;
|
||||
var postal = popover.querySelector("[data-loc-postal]").value.trim();
|
||||
var saveBtn = popover.querySelector("[data-loc-save]");
|
||||
if (!postal) { showLocError("Enter a postal code."); return; }
|
||||
showLocError("");
|
||||
saveBtn.disabled = true; saveBtn.textContent = "…";
|
||||
geocode(country, postal).then(function (geo) {
|
||||
var loc = {
|
||||
country: country, postal: postal,
|
||||
lat: geo.lat, lon: geo.lon, name: geo.name,
|
||||
unit: selectedUnit
|
||||
};
|
||||
setLoc(loc);
|
||||
try { localStorage.removeItem(CACHE_KEY); } catch (e) {}
|
||||
closePopover();
|
||||
run(true);
|
||||
}).catch(function (err) {
|
||||
showLocError((err && err.message) || "Couldn't find that postal code.");
|
||||
}).then(function () {
|
||||
saveBtn.disabled = false; saveBtn.textContent = "Save";
|
||||
});
|
||||
}
|
||||
|
||||
function openPopover() {
|
||||
if (!popover) popover = buildPopover();
|
||||
var loc = getLoc();
|
||||
popover.querySelector("[data-loc-country]").value = loc ? loc.country : defaultCountry();
|
||||
popover.querySelector("[data-loc-postal]").value = loc ? loc.postal : "";
|
||||
setUnit(loc && loc.unit ? loc.unit : "celsius");
|
||||
showLocError("");
|
||||
popover.classList.remove("hidden");
|
||||
var inp = popover.querySelector("[data-loc-postal]");
|
||||
if (inp) setTimeout(function () { inp.focus(); }, 30);
|
||||
}
|
||||
function closePopover() {
|
||||
if (popover) popover.classList.add("hidden");
|
||||
}
|
||||
|
||||
// ---- boot ----------------------------------------------------------------
|
||||
// Remember the server's seasonal pick so "Clear" can restore it.
|
||||
var seasonalDefault = {
|
||||
variant: document.documentElement.dataset.weather || "",
|
||||
intensity: document.documentElement.dataset.intensity || "heavy"
|
||||
};
|
||||
|
||||
function run(force) {
|
||||
var loc = getLoc();
|
||||
if (!loc) { setChip("📍 Weather", "Set your location"); return; }
|
||||
setChip("📍 " + escapeHtml(loc.name.split(",")[0]) + "…", "Loading forecast…");
|
||||
loadForecast(loc, force)
|
||||
.then(function (entry) { renderForecast(loc, entry); })
|
||||
.catch(function () { showError("Forecast unavailable"); });
|
||||
}
|
||||
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
if (chipBtn) chipBtn.addEventListener("click", openPopover);
|
||||
document.addEventListener("keydown", function (e) {
|
||||
if (e.key === "Escape") closePopover();
|
||||
});
|
||||
run(false);
|
||||
});
|
||||
})();
|
||||
1028
internal/web/static/js/weather-gl.js
Normal file
1028
internal/web/static/js/weather-gl.js
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,246 +1,84 @@
|
||||
// Canvas-based seasonal weather. The server picks variant + intensity from
|
||||
// the Lisbon-local date; this script reads those off <html data-*> and
|
||||
// renders the appropriate particles. Each variant has a hand-drawn shape
|
||||
// so silhouettes are recognizable instead of a generic blob.
|
||||
// Weather controller. Two drivers feed the background canvas:
|
||||
//
|
||||
// 1. The server picks a *seasonal* variant + intensity from the Lisbon-local
|
||||
// date and writes them to <html data-weather / data-intensity>. This is the
|
||||
// default when the visitor hasn't set a location.
|
||||
// 2. weather-forecast.js, when a postal code is saved, fetches the real
|
||||
// forecast and calls PeteWeather.set(variant, intensity) to override the
|
||||
// background with live conditions.
|
||||
//
|
||||
// Rendering is delegated to an engine: the WebGL2 one (weather-gl.js) when the
|
||||
// browser supports it — GPU sprites, shader fog/aurora, real lightning — with
|
||||
// the original Canvas2D renderer (weather-2d.js) as the fallback. This file
|
||||
// only owns the toggle button, the on/off preference and the public API.
|
||||
(function () {
|
||||
var root = document.documentElement;
|
||||
var canvas = document.getElementById("pete-weather");
|
||||
if (!canvas) return;
|
||||
var variant = root.dataset.weather;
|
||||
if (!variant) return;
|
||||
if (window.matchMedia && window.matchMedia("(prefers-reduced-motion: reduce)").matches) return;
|
||||
var reducedMotion = window.matchMedia && window.matchMedia("(prefers-reduced-motion: reduce)").matches;
|
||||
|
||||
var ctx = canvas.getContext("2d");
|
||||
var W = 0, H = 0, DPR = 1;
|
||||
function resize() {
|
||||
DPR = Math.min(window.devicePixelRatio || 1, 2);
|
||||
W = canvas.clientWidth || window.innerWidth;
|
||||
H = canvas.clientHeight || window.innerHeight;
|
||||
canvas.width = Math.floor(W * DPR);
|
||||
canvas.height = Math.floor(H * DPR);
|
||||
ctx.setTransform(DPR, 0, 0, DPR, 0, 0);
|
||||
var engines = window.PeteWeatherEngines || {};
|
||||
var engine = (engines.webgl2 && engines.webgl2(canvas)) || null;
|
||||
if (!engine && engines.canvas2d) engine = engines.canvas2d(canvas);
|
||||
if (!engine) return;
|
||||
|
||||
var STORAGE_KEY = "pete-weather-off";
|
||||
var toggleBtn = document.querySelector("[data-weather-toggle]");
|
||||
var star = document.querySelector("[data-weather-star]");
|
||||
|
||||
function userDisabled() {
|
||||
try { return localStorage.getItem(STORAGE_KEY) === "1"; } catch (e) { return false; }
|
||||
}
|
||||
resize();
|
||||
window.addEventListener("resize", resize);
|
||||
|
||||
var counts = { rain: 160, petals: 50, jacaranda: 55, motes: 70, leaves: 38 };
|
||||
var mults = { light: 0.6, medium: 1.0, heavy: 1.4 };
|
||||
var N = Math.round((counts[variant] || 0) * (mults[root.dataset.intensity] || 1));
|
||||
|
||||
function rand(a, b) { return a + Math.random() * (b - a); }
|
||||
|
||||
function spawn(initial) {
|
||||
var p = {};
|
||||
p.x = rand(0, W);
|
||||
p.y = initial ? rand(0, H) : rand(-40, -10);
|
||||
p.z = rand(0.7, 1.3); // depth — affects size + speed + alpha
|
||||
if (variant === "rain") {
|
||||
p.vy = rand(700, 1100) * p.z;
|
||||
p.vx = -rand(60, 120);
|
||||
p.len = rand(10, 18) * p.z;
|
||||
p.alpha = rand(0.25, 0.55);
|
||||
} else if (variant === "petals" || variant === "jacaranda") {
|
||||
p.vy = rand(60, 120);
|
||||
p.swayAmp = rand(20, 50);
|
||||
p.swayFreq = rand(0.4, 0.9);
|
||||
p.swayPhase = rand(0, Math.PI * 2);
|
||||
p.rot = rand(0, Math.PI * 2);
|
||||
p.vrot = rand(-1.2, 1.2);
|
||||
p.size = rand(8, 16) * p.z;
|
||||
p.alpha = rand(0.75, 1.0);
|
||||
if (variant === "jacaranda") {
|
||||
p.hue = rand(265, 285); // blue-violet — jacaranda uses the petal silhouette in purple
|
||||
p.sat = rand(55, 75);
|
||||
p.light = rand(70, 82);
|
||||
} else {
|
||||
p.hue = rand(335, 355); // pink (almond/cherry)
|
||||
p.sat = rand(60, 80);
|
||||
p.light = rand(78, 88);
|
||||
}
|
||||
} else if (variant === "motes") {
|
||||
p.vx = rand(-15, 25);
|
||||
p.vy = rand(8, 25);
|
||||
p.size = rand(1.2, 2.8) * p.z;
|
||||
p.alpha = rand(0.35, 0.7);
|
||||
p.twinklePhase = rand(0, Math.PI * 2);
|
||||
p.twinkleFreq = rand(0.5, 1.5);
|
||||
} else if (variant === "leaves") {
|
||||
p.vy = rand(70, 130);
|
||||
p.swayAmp = rand(30, 70);
|
||||
p.swayFreq = rand(0.3, 0.6);
|
||||
p.swayPhase = rand(0, Math.PI * 2);
|
||||
p.rot = rand(0, Math.PI * 2);
|
||||
p.vrot = rand(-2.0, 2.0);
|
||||
p.size = rand(10, 18) * p.z;
|
||||
p.alpha = rand(0.8, 1.0);
|
||||
p.hue = rand(18, 42); // orange→amber→brown
|
||||
p.light = rand(38, 55);
|
||||
function setDisabled(v) {
|
||||
try {
|
||||
if (v) localStorage.setItem(STORAGE_KEY, "1");
|
||||
else localStorage.removeItem(STORAGE_KEY);
|
||||
} catch (e) {}
|
||||
if (window.PetePrefs) window.PetePrefs.push();
|
||||
}
|
||||
function syncBtn(enabled) {
|
||||
if (!toggleBtn) return;
|
||||
toggleBtn.setAttribute("aria-pressed", enabled ? "true" : "false");
|
||||
toggleBtn.title = enabled ? "Turn off weather animation" : "Turn on weather animation";
|
||||
if (star) {
|
||||
star.style.color = "var(--accent)";
|
||||
star.style.opacity = enabled ? "1" : "0.3";
|
||||
star.style.filter = enabled ? "none" : "grayscale(1)";
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
var particles = [];
|
||||
for (var i = 0; i < N; i++) particles.push(spawn(true));
|
||||
var enabled = !userDisabled() && !reducedMotion;
|
||||
|
||||
var night = root.dataset.phase === "night";
|
||||
function refreshNight() { night = root.dataset.phase === "night"; }
|
||||
// Public hook so weather-forecast.js (and the /weather demo) can swap the
|
||||
// seasonal default for other conditions. Passing a falsy variant clears the
|
||||
// canvas.
|
||||
window.PeteWeather = {
|
||||
set: function (v, inten) {
|
||||
root.dataset.weather = v || "";
|
||||
if (inten) root.dataset.intensity = inten;
|
||||
engine.set(v || null, inten || root.dataset.intensity || "heavy");
|
||||
if (enabled && v) engine.start();
|
||||
},
|
||||
isEnabled: function () { return enabled; },
|
||||
renderer: function () { return engine.name; }
|
||||
};
|
||||
|
||||
function drawRain(p) {
|
||||
// Day/dawn/dusk backgrounds are warm and pale, so the rain needs a darker,
|
||||
// more saturated blue plus a slightly thicker line to stay visible. Night
|
||||
// keeps a paler tone so streaks read against the dark palette.
|
||||
if (night) {
|
||||
ctx.strokeStyle = "rgba(200,215,240," + p.alpha + ")";
|
||||
ctx.lineWidth = 1;
|
||||
} else {
|
||||
ctx.strokeStyle = "rgba(60,95,150," + Math.min(1, p.alpha + 0.25) + ")";
|
||||
ctx.lineWidth = 1.3;
|
||||
}
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(p.x, p.y);
|
||||
ctx.lineTo(p.x - p.len * 0.12, p.y + p.len);
|
||||
ctx.stroke();
|
||||
syncBtn(enabled);
|
||||
engine.set(root.dataset.weather || null, root.dataset.intensity || "heavy");
|
||||
if (enabled && root.dataset.weather) engine.start();
|
||||
|
||||
if (toggleBtn) {
|
||||
toggleBtn.addEventListener("click", function () {
|
||||
enabled = !enabled;
|
||||
setDisabled(!enabled);
|
||||
syncBtn(enabled);
|
||||
if (enabled) engine.start(); else engine.stop();
|
||||
});
|
||||
}
|
||||
|
||||
function drawPetals(p) {
|
||||
// Five-petal almond/cherry blossom viewed face-on, with a yellow center.
|
||||
var s = p.size;
|
||||
ctx.save();
|
||||
ctx.translate(p.x, p.y);
|
||||
ctx.rotate(p.rot);
|
||||
ctx.fillStyle = "hsla(" + p.hue + "," + p.sat + "%," + p.light + "%," + p.alpha + ")";
|
||||
for (var i = 0; i < 5; i++) {
|
||||
var a = (i / 5) * Math.PI * 2 - Math.PI / 2;
|
||||
var cx = Math.cos(a) * s * 0.32;
|
||||
var cy = Math.sin(a) * s * 0.32;
|
||||
ctx.beginPath();
|
||||
ctx.ellipse(cx, cy, s * 0.38, s * 0.24, a, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
ctx.fillStyle = "hsla(48,90%,68%," + p.alpha + ")";
|
||||
ctx.beginPath();
|
||||
ctx.arc(0, 0, s * 0.13, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function drawLeaf(p) {
|
||||
var s = p.size;
|
||||
ctx.save();
|
||||
ctx.translate(p.x, p.y);
|
||||
ctx.rotate(p.rot);
|
||||
var g = ctx.createLinearGradient(-s * 0.5, 0, s * 0.5, 0);
|
||||
g.addColorStop(0, "hsla(" + p.hue + ",70%," + (p.light - 10) + "%," + p.alpha + ")");
|
||||
g.addColorStop(1, "hsla(" + p.hue + ",75%," + (p.light + 10) + "%," + p.alpha + ")");
|
||||
ctx.fillStyle = g;
|
||||
// Almond/lozenge leaf — pointed at both ends.
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(-s * 0.55, 0);
|
||||
ctx.quadraticCurveTo(0, -s * 0.38, s * 0.55, 0);
|
||||
ctx.quadraticCurveTo(0, s * 0.38, -s * 0.55, 0);
|
||||
ctx.closePath();
|
||||
ctx.fill();
|
||||
// Midrib vein.
|
||||
ctx.strokeStyle = "hsla(" + p.hue + ",60%," + (p.light - 22) + "%," + (p.alpha * 0.7) + ")";
|
||||
ctx.lineWidth = 0.8;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(-s * 0.5, 0);
|
||||
ctx.lineTo(s * 0.5, 0);
|
||||
ctx.stroke();
|
||||
// Small stem nub at the base.
|
||||
ctx.strokeStyle = "hsla(" + p.hue + ",55%," + (p.light - 25) + "%," + (p.alpha * 0.7) + ")";
|
||||
ctx.lineWidth = 1.0;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(-s * 0.55, 0);
|
||||
ctx.lineTo(-s * 0.7, -s * 0.04);
|
||||
ctx.stroke();
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
function drawMote(p, t) {
|
||||
var twinkle = 0.5 + 0.5 * Math.sin(t * p.twinkleFreq + p.twinklePhase);
|
||||
// Night keeps a pale glowing mote against the dark palette. Day/dawn/dusk
|
||||
// use a deeper saturated honey so the motes stand out against warm cream
|
||||
// and peach backgrounds; alpha also gets a bump.
|
||||
var color, a;
|
||||
if (night) {
|
||||
color = "255,240,180";
|
||||
a = p.alpha * (0.45 + 0.55 * twinkle);
|
||||
} else {
|
||||
color = "130,80,180";
|
||||
a = Math.min(1, (p.alpha + 0.2) * (0.6 + 0.4 * twinkle));
|
||||
}
|
||||
// Soft glow halo.
|
||||
var g = ctx.createRadialGradient(p.x, p.y, 0, p.x, p.y, p.size * 5);
|
||||
g.addColorStop(0, "rgba(" + color + "," + (a * 0.55) + ")");
|
||||
g.addColorStop(1, "rgba(" + color + ",0)");
|
||||
ctx.fillStyle = g;
|
||||
ctx.beginPath();
|
||||
ctx.arc(p.x, p.y, p.size * 5, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
// Bright core.
|
||||
ctx.fillStyle = "rgba(" + color + "," + a + ")";
|
||||
ctx.beginPath();
|
||||
ctx.arc(p.x, p.y, p.size, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
|
||||
var last = performance.now();
|
||||
var raf = 0;
|
||||
|
||||
function step(now) {
|
||||
var dt = Math.min(0.05, (now - last) / 1000);
|
||||
last = now;
|
||||
var t = now / 1000;
|
||||
ctx.clearRect(0, 0, W, H);
|
||||
refreshNight();
|
||||
|
||||
for (var i = 0; i < particles.length; i++) {
|
||||
var p = particles[i];
|
||||
if (variant === "rain") {
|
||||
p.x += p.vx * dt;
|
||||
p.y += p.vy * dt;
|
||||
if (p.y > H + 20 || p.x < -40) {
|
||||
particles[i] = spawn(false);
|
||||
particles[i].x = rand(0, W + 60);
|
||||
continue;
|
||||
}
|
||||
drawRain(p);
|
||||
} else if (variant === "motes") {
|
||||
p.x += p.vx * dt;
|
||||
p.y += p.vy * dt;
|
||||
if (p.y > H + 10 || p.x < -10 || p.x > W + 10) {
|
||||
particles[i] = spawn(false);
|
||||
continue;
|
||||
}
|
||||
drawMote(p, t);
|
||||
} else {
|
||||
// Drifting variants: petals, jacaranda, leaves.
|
||||
p.y += p.vy * dt;
|
||||
p.rot += p.vrot * dt;
|
||||
var sway = Math.sin(t * p.swayFreq + p.swayPhase) * p.swayAmp;
|
||||
var origX = p.x;
|
||||
p.x = origX + sway;
|
||||
if (p.y > H + 30) {
|
||||
particles[i] = spawn(false);
|
||||
continue;
|
||||
}
|
||||
if (variant === "jacaranda" || variant === "petals") drawPetals(p);
|
||||
else drawLeaf(p);
|
||||
p.x = origX;
|
||||
}
|
||||
}
|
||||
raf = requestAnimationFrame(step);
|
||||
}
|
||||
raf = requestAnimationFrame(step);
|
||||
|
||||
document.addEventListener("visibilitychange", function () {
|
||||
if (document.hidden) {
|
||||
cancelAnimationFrame(raf);
|
||||
raf = 0;
|
||||
} else if (!raf) {
|
||||
last = performance.now();
|
||||
raf = requestAnimationFrame(step);
|
||||
}
|
||||
if (!enabled) return;
|
||||
if (document.hidden) engine.stop();
|
||||
else engine.start();
|
||||
});
|
||||
})();
|
||||
|
||||
22
internal/web/static/manifest.webmanifest
Normal file
22
internal/web/static/manifest.webmanifest
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "Pete — friendly news",
|
||||
"short_name": "Pete",
|
||||
"description": "A calm, read-one-at-a-time news reader. Bookmarks, a personalized feed, and offline reading.",
|
||||
"id": "/",
|
||||
"start_url": "/?source=pwa",
|
||||
"scope": "/",
|
||||
"display": "standalone",
|
||||
"orientation": "portrait-primary",
|
||||
"background_color": "#fbf3e3",
|
||||
"theme_color": "#fbf3e3",
|
||||
"categories": ["news", "productivity"],
|
||||
"icons": [
|
||||
{ "src": "/static/img/icon-192.png", "sizes": "192x192", "type": "image/png", "purpose": "any" },
|
||||
{ "src": "/static/img/icon-512.png", "sizes": "512x512", "type": "image/png", "purpose": "any" },
|
||||
{ "src": "/static/img/maskable-512.png", "sizes": "512x512", "type": "image/png", "purpose": "maskable" }
|
||||
],
|
||||
"shortcuts": [
|
||||
{ "name": "For you", "short_name": "For you", "url": "/for-you", "description": "Your personalized feed" },
|
||||
{ "name": "Bookmarks", "short_name": "Saved", "url": "/bookmarks", "description": "Stories you saved" }
|
||||
]
|
||||
}
|
||||
195
internal/web/static/sw.js
Normal file
195
internal/web/static/sw.js
Normal file
@@ -0,0 +1,195 @@
|
||||
// Pete's service worker: an installable-PWA shell, an offline reader, and the
|
||||
// Web Push receiver. Served from the root (/sw.js) so its scope is the whole
|
||||
// origin — it can intercept navigations and /api/article the same as any page.
|
||||
//
|
||||
// Bump CACHE_VERSION whenever the precached shell assets change; activate()
|
||||
// drops every cache that doesn't match the current version.
|
||||
var CACHE_VERSION = "v4";
|
||||
var SHELL_CACHE = "pete-shell-" + CACHE_VERSION;
|
||||
var RUNTIME_CACHE = "pete-runtime-" + CACHE_VERSION;
|
||||
|
||||
// App shell: the static assets every page needs. Versioned by URL content only
|
||||
// loosely, so we lean on network-first for HTML and cache-first for these.
|
||||
var SHELL_ASSETS = [
|
||||
"/static/css/output.css",
|
||||
"/static/js/prefs.js",
|
||||
"/static/js/weather-2d.js",
|
||||
"/static/js/weather-gl.js",
|
||||
"/static/js/weather.js",
|
||||
"/static/js/weather-forecast.js",
|
||||
"/static/js/search.js",
|
||||
"/static/js/settings.js",
|
||||
"/static/js/reader.js",
|
||||
"/static/js/pwa.js",
|
||||
"/static/img/pete.avif",
|
||||
"/static/img/icon-192.png",
|
||||
"/static/img/icon-512.png",
|
||||
];
|
||||
|
||||
// How many visited-article responses to keep for offline reading before the
|
||||
// oldest are evicted. Reader articles are small JSON blobs.
|
||||
var RUNTIME_MAX = 60;
|
||||
|
||||
self.addEventListener("install", function (event) {
|
||||
event.waitUntil(
|
||||
caches.open(SHELL_CACHE).then(function (cache) {
|
||||
// addAll is atomic-ish: if one asset 404s the whole install fails, so keep
|
||||
// this list to assets we know are served. Individual failures are tolerated
|
||||
// by falling back to per-asset puts.
|
||||
return Promise.all(
|
||||
SHELL_ASSETS.map(function (url) {
|
||||
return cache.add(url).catch(function () {});
|
||||
})
|
||||
);
|
||||
}).then(function () {
|
||||
return self.skipWaiting();
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
self.addEventListener("activate", function (event) {
|
||||
event.waitUntil(
|
||||
caches.keys().then(function (keys) {
|
||||
return Promise.all(
|
||||
keys.map(function (k) {
|
||||
if (k !== SHELL_CACHE && k !== RUNTIME_CACHE) return caches.delete(k);
|
||||
})
|
||||
);
|
||||
}).then(function () {
|
||||
return self.clients.claim();
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
// trimCache evicts the oldest entries once a runtime cache passes its cap.
|
||||
function trimCache(cacheName, max) {
|
||||
caches.open(cacheName).then(function (cache) {
|
||||
cache.keys().then(function (keys) {
|
||||
if (keys.length <= max) return;
|
||||
for (var i = 0; i < keys.length - max; i++) cache.delete(keys[i]);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// A minimal offline page for navigations we have nothing cached for.
|
||||
function offlineFallback() {
|
||||
return new Response(
|
||||
"<!doctype html><meta charset=utf-8><meta name=viewport content='width=device-width,initial-scale=1'>" +
|
||||
"<title>Offline · Pete</title>" +
|
||||
"<div style=\"font-family:system-ui,sans-serif;max-width:32rem;margin:20vh auto;padding:0 1.5rem;text-align:center;color:#3a2f1a\">" +
|
||||
"<div style=font-size:3rem>🦆</div>" +
|
||||
"<h1 style=font-size:1.4rem>You're offline</h1>" +
|
||||
"<p style=opacity:.7>Pete can't reach the news right now. Articles you've already opened are still readable, so head back and try one of those.</p>" +
|
||||
"</div>",
|
||||
{ headers: { "Content-Type": "text/html; charset=utf-8" }, status: 503 }
|
||||
);
|
||||
}
|
||||
|
||||
self.addEventListener("fetch", function (event) {
|
||||
var req = event.request;
|
||||
if (req.method !== "GET") return;
|
||||
|
||||
var url = new URL(req.url);
|
||||
if (url.origin !== self.location.origin) return; // never touch cross-origin
|
||||
|
||||
// Visited articles: network-first so text stays fresh, but cache every success
|
||||
// so the reader still works offline for stories the user has already opened.
|
||||
if (url.pathname === "/api/article") {
|
||||
event.respondWith(
|
||||
fetch(req).then(function (res) {
|
||||
if (res && res.ok) {
|
||||
var copy = res.clone();
|
||||
caches.open(RUNTIME_CACHE).then(function (cache) {
|
||||
cache.put(req, copy);
|
||||
trimCache(RUNTIME_CACHE, RUNTIME_MAX);
|
||||
});
|
||||
}
|
||||
return res;
|
||||
}).catch(function () {
|
||||
return caches.match(req).then(function (hit) {
|
||||
return hit || new Response(JSON.stringify({ error: "offline" }), {
|
||||
status: 503,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
});
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Static assets: cache-first (they're versioned by deploy), fill the cache on
|
||||
// first miss so a later offline visit has them.
|
||||
if (url.pathname.indexOf("/static/") === 0) {
|
||||
event.respondWith(
|
||||
caches.match(req).then(function (hit) {
|
||||
return hit || fetch(req).then(function (res) {
|
||||
if (res && res.ok) {
|
||||
var copy = res.clone();
|
||||
caches.open(SHELL_CACHE).then(function (cache) { cache.put(req, copy); });
|
||||
}
|
||||
return res;
|
||||
});
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Page navigations: network-only, falling back to the offline card when the
|
||||
// network is unreachable. We deliberately do NOT cache HTML responses: pages
|
||||
// are personalized (they embed the signed-in user's name/email and a "For you"
|
||||
// rail), and the runtime cache is shared across everyone who uses this
|
||||
// installed PWA. Caching a navigation would let a signed-out visitor — or a
|
||||
// second person on the same device — be served the previous user's identity
|
||||
// and personalized stories offline. Offline reading still works: the reader
|
||||
// fetches cached /api/article JSON on top of the cached static shell.
|
||||
if (req.mode === "navigate") {
|
||||
event.respondWith(
|
||||
fetch(req).catch(function () {
|
||||
return offlineFallback();
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Everything else (other /api/* calls): straight to the network. These are
|
||||
// personalized/stateful and must not be served stale.
|
||||
});
|
||||
|
||||
// ---- Web Push -------------------------------------------------------------
|
||||
// The server sends a JSON payload {title, body, url, tag}. Missing fields fall
|
||||
// back to sensible defaults so a malformed push still shows something useful.
|
||||
self.addEventListener("push", function (event) {
|
||||
var data = {};
|
||||
if (event.data) {
|
||||
try { data = event.data.json(); } catch (e) { data = { body: event.data.text() }; }
|
||||
}
|
||||
var title = data.title || "Pete";
|
||||
var options = {
|
||||
body: data.body || "New stories are waiting.",
|
||||
icon: "/static/img/icon-192.png",
|
||||
badge: "/static/img/icon-192.png",
|
||||
tag: data.tag || "pete-digest",
|
||||
renotify: true,
|
||||
data: { url: data.url || "/" },
|
||||
};
|
||||
event.waitUntil(self.registration.showNotification(title, options));
|
||||
});
|
||||
|
||||
self.addEventListener("notificationclick", function (event) {
|
||||
event.notification.close();
|
||||
var target = (event.notification.data && event.notification.data.url) || "/";
|
||||
event.waitUntil(
|
||||
self.clients.matchAll({ type: "window", includeUncontrolled: true }).then(function (clients) {
|
||||
for (var i = 0; i < clients.length; i++) {
|
||||
var c = clients[i];
|
||||
// Focus an existing Pete tab and route it to the target if we can.
|
||||
if ("focus" in c) {
|
||||
c.focus();
|
||||
if ("navigate" in c && target !== "/") { try { c.navigate(target); } catch (e) {} }
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (self.clients.openWindow) return self.clients.openWindow(target);
|
||||
})
|
||||
);
|
||||
});
|
||||
132
internal/web/status.go
Normal file
132
internal/web/status.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// isAdmin reports whether the request carries a signed-in session whose OIDC
|
||||
// subject is on the admin allowlist. False when auth is off, the allowlist is
|
||||
// empty, or the visitor is anonymous.
|
||||
func (s *Server) isAdmin(r *http.Request) bool {
|
||||
if s.auth == nil || len(s.adminSubs) == 0 {
|
||||
return false
|
||||
}
|
||||
u := s.auth.userFromRequest(r)
|
||||
if u == nil {
|
||||
return false
|
||||
}
|
||||
return s.adminSubs[u.Sub]
|
||||
}
|
||||
|
||||
// sourceStatus is one row of the source-health dashboard: the configured feed
|
||||
// plus its persisted poll health and derived content stats.
|
||||
type sourceStatus struct {
|
||||
Name string
|
||||
Channel string
|
||||
Healthy bool // last poll succeeded (no consecutive failures)
|
||||
NeverRun bool // no poll recorded yet
|
||||
|
||||
LastPollAt time.Time
|
||||
LastSuccessAt time.Time
|
||||
LastError string
|
||||
Failures int
|
||||
LastItemCount int
|
||||
|
||||
Total int
|
||||
Classified int
|
||||
Paywalled int
|
||||
PaywallRate int // percent of retained stories that are gated
|
||||
LastSeenAt time.Time
|
||||
LastPostedAt time.Time
|
||||
}
|
||||
|
||||
type statusPage struct {
|
||||
pageData
|
||||
Sources []sourceStatus
|
||||
DegradedCnt int // sources currently failing
|
||||
Admin bool // viewer is an admin: show the full diagnostic columns
|
||||
}
|
||||
|
||||
// handleStatus renders the source-health page. It's public: everyone sees a
|
||||
// trimmed reader view (per-feed up/stale/idle and when each last updated), while
|
||||
// admins additionally get the operator diagnostics (poll cadence, item counts,
|
||||
// paywall rates, posting times, and raw fetch errors).
|
||||
func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
admin := s.isAdmin(r)
|
||||
s.track(r, "status")
|
||||
|
||||
health, err := storage.ListSourceHealth()
|
||||
if err != nil {
|
||||
slog.Error("web: source health query failed", "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
stats, err := storage.SourceContentStats()
|
||||
if err != nil {
|
||||
slog.Error("web: source content stats failed", "err", err)
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
rows := make([]sourceStatus, 0, len(s.sources))
|
||||
degraded := 0
|
||||
for _, src := range s.sources {
|
||||
h, hasHealth := health[src.Name]
|
||||
st := stats[src.Name]
|
||||
|
||||
row := sourceStatus{
|
||||
Name: src.Name,
|
||||
Channel: src.Channel,
|
||||
NeverRun: !hasHealth || h.LastPollAt == 0,
|
||||
LastError: h.LastError,
|
||||
Failures: h.ConsecutiveFailures,
|
||||
LastItemCount: h.LastItemCount,
|
||||
Total: st.Total,
|
||||
Classified: st.Classified,
|
||||
Paywalled: st.Paywalled,
|
||||
}
|
||||
row.Healthy = hasHealth && h.ConsecutiveFailures == 0
|
||||
// Raw fetch errors leak feed-specific workarounds and upstream URLs, so
|
||||
// they stay out of the public payload entirely (not just hidden in CSS).
|
||||
if !admin {
|
||||
row.LastError = ""
|
||||
}
|
||||
if h.LastPollAt > 0 {
|
||||
row.LastPollAt = time.Unix(h.LastPollAt, 0)
|
||||
}
|
||||
if h.LastSuccessAt > 0 {
|
||||
row.LastSuccessAt = time.Unix(h.LastSuccessAt, 0)
|
||||
}
|
||||
if st.LastSeenAt > 0 {
|
||||
row.LastSeenAt = time.Unix(st.LastSeenAt, 0)
|
||||
}
|
||||
if st.LastPostedAt > 0 {
|
||||
row.LastPostedAt = time.Unix(st.LastPostedAt, 0)
|
||||
}
|
||||
if st.Total > 0 {
|
||||
row.PaywallRate = st.Paywalled * 100 / st.Total
|
||||
}
|
||||
if !row.NeverRun && !row.Healthy {
|
||||
degraded++
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
|
||||
// Failing sources first (most consecutive failures), then healthy ones by
|
||||
// name, so the owner's eye lands on what needs attention.
|
||||
sort.SliceStable(rows, func(i, j int) bool {
|
||||
if rows[i].Failures != rows[j].Failures {
|
||||
return rows[i].Failures > rows[j].Failures
|
||||
}
|
||||
return rows[i].Name < rows[j].Name
|
||||
})
|
||||
|
||||
base := s.base(r)
|
||||
base.Active = "status"
|
||||
s.render(w, "status", statusPage{pageData: base, Sources: rows, DegradedCnt: degraded, Admin: admin})
|
||||
}
|
||||
59
internal/web/status_render_test.go
Normal file
59
internal/web/status_render_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
)
|
||||
|
||||
func TestStatusTemplateExecutes(t *testing.T) {
|
||||
s, err := New(config.WebConfig{SiteTitle: "Pete", ListenAddr: ":0"}, nil, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sources := []sourceStatus{
|
||||
{Name: "Broken Feed", Channel: "tech", Failures: 3, LastError: "dial tcp: timeout",
|
||||
LastPollAt: time.Now(), NeverRun: false, Healthy: false, Total: 4, Paywalled: 2, PaywallRate: 50},
|
||||
{Name: "Good Feed", Channel: "eu", Healthy: true, LastPollAt: time.Now(),
|
||||
LastSuccessAt: time.Now(), LastItemCount: 12, Total: 30, Classified: 28,
|
||||
LastSeenAt: time.Now(), LastPostedAt: time.Now()},
|
||||
{Name: "Idle Feed", Channel: "music", NeverRun: true},
|
||||
}
|
||||
|
||||
render := func(admin bool) string {
|
||||
data := statusPage{
|
||||
pageData: pageData{SiteTitle: "Pete", Channels: channels},
|
||||
DegradedCnt: 1,
|
||||
Admin: admin,
|
||||
Sources: sources,
|
||||
}
|
||||
var b strings.Builder
|
||||
if err := s.tpls["status"].ExecuteTemplate(&b, "layout", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Admin view: every feed plus the operator diagnostics and raw errors.
|
||||
admin := render(true)
|
||||
for _, want := range []string{"Broken Feed", "Good Feed", "dial tcp: timeout", "1 degraded", "Source health"} {
|
||||
if !strings.Contains(admin, want) {
|
||||
t.Errorf("admin status page missing %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
// Public view: same feeds and states, but no error strings or ops columns.
|
||||
pub := render(false)
|
||||
for _, want := range []string{"Broken Feed", "Good Feed", "Idle Feed", "delayed", "Source health"} {
|
||||
if !strings.Contains(pub, want) {
|
||||
t.Errorf("public status page missing %q", want)
|
||||
}
|
||||
}
|
||||
for _, leak := range []string{"dial tcp: timeout", "degraded", "Last posted", "Paywall"} {
|
||||
if strings.Contains(pub, leak) {
|
||||
t.Errorf("public status page leaked operator detail %q", leak)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,26 @@
|
||||
{{define "card"}}
|
||||
{{define "card"}}{{$ch := channel .Story.Channel}}
|
||||
<a href="{{.Story.ArticleURL}}" target="_blank" rel="noopener noreferrer"
|
||||
data-story-card data-source="{{.Story.Source}}" data-channel="{{.Story.Channel}}"
|
||||
data-id="{{.Story.ID}}"
|
||||
data-headline="{{.Story.Headline}}"
|
||||
data-lede="{{.Story.Lede}}"
|
||||
data-url="{{.Story.ArticleURL}}"
|
||||
data-image="{{if .Story.ImageURL}}{{thumb .Story.ImageURL}}{{end}}"
|
||||
data-time="{{timeAgo .Story.SeenAt}}"
|
||||
data-ch-title="{{$ch.Title}}" data-ch-emoji="{{$ch.Emoji}}" data-ch-theme="{{$ch.Theme}}"
|
||||
data-posted="{{if .Story.Posted}}1{{end}}"
|
||||
data-paywalled="{{if .Story.Paywalled}}1{{end}}"
|
||||
class="group relative block rounded-3xl bg-[color:var(--card)] border-2 {{if .Story.Posted}}border-theme-{{.Theme}} glow-theme-{{.Theme}}{{else}}border-[color:var(--ink)]/10{{end}} shadow-pete overflow-hidden hover:-translate-y-1 hover:shadow-pete-lg transition">
|
||||
<span role="button" tabindex="0" data-bookmark-btn data-story-id="{{.Story.ID}}"
|
||||
aria-label="Bookmark this story" aria-pressed="false"
|
||||
style="display:none;position:absolute;top:.6rem;left:.6rem;z-index:6;height:1.9rem;width:1.9rem;align-items:center;justify-content:center;border-radius:9999px;background:rgba(20,14,6,.62);color:#fff;-webkit-backdrop-filter:blur(2px);backdrop-filter:blur(2px)">
|
||||
<svg viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">
|
||||
<path d="M6 2h12a1 1 0 0 1 1 1v18l-7-4-7 4V3a1 1 0 0 1 1-1z"></path>
|
||||
</svg>
|
||||
</span>
|
||||
{{if .Story.ImageURL}}
|
||||
<div class="aspect-[16/10] w-full overflow-hidden bg-[color:var(--ink)]/5">
|
||||
<img src="{{thumb .Story.ImageURL}}" alt="" loading="lazy" decoding="async"
|
||||
<img src="{{thumb .Story.ImageURL}}" alt="{{.Story.Headline}}" loading="lazy" decoding="async"
|
||||
class="h-full w-full object-cover group-hover:scale-105 transition duration-500">
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -20,6 +37,10 @@
|
||||
</span>
|
||||
{{end}}
|
||||
<span class="text-[color:var(--ink)]/50">{{timeAgo .Story.SeenAt}}</span>
|
||||
{{if .Story.ReadMins}}<span class="text-[color:var(--ink)]/45">· {{.Story.ReadMins}} min read</span>{{end}}
|
||||
{{if .Story.Views}}<span class="ml-auto inline-flex items-center gap-1 text-[color:var(--ink)]/45 tabular-nums" title="{{.Story.Views}} reads">
|
||||
<svg viewBox="0 0 24 24" width="13" height="13" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M1 12s4-7 11-7 11 7 11 7-4 7-11 7-11-7-11-7z"></path><circle cx="12" cy="12" r="3"></circle></svg>
|
||||
{{.Story.Views}}</span>{{end}}
|
||||
</div>
|
||||
<h3 class="font-display text-lg font-semibold leading-snug group-hover:underline underline-offset-2 decoration-theme-{{.Theme}}">
|
||||
{{.Story.Headline}}
|
||||
|
||||
37
internal/web/templates/bookmarks.html
Normal file
37
internal/web/templates/bookmarks.html
Normal file
@@ -0,0 +1,37 @@
|
||||
{{define "title"}}Bookmarks — {{.SiteTitle}}{{end}}
|
||||
|
||||
{{define "main"}}
|
||||
<section class="mt-2 mb-8">
|
||||
<div class="rounded-3xl bg-[color:var(--accent)] text-[#1c1305] p-6 sm:p-10 shadow-pete relative overflow-hidden">
|
||||
<div class="absolute -top-6 -right-6 text-[12rem] opacity-20 select-none" aria-hidden="true">🔖</div>
|
||||
<div class="relative">
|
||||
<p class="text-sm uppercase tracking-[0.2em] opacity-70">saved for later</p>
|
||||
<h1 class="font-display text-4xl sm:text-5xl font-bold mt-1">Bookmarks</h1>
|
||||
<p class="mt-2 max-w-xl opacity-80">Stories you've tucked away. Tap the bookmark on any card to add or remove one.</p>
|
||||
<p class="mt-4 text-xs uppercase tracking-wider opacity-75">{{.Total}} saved</p>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{{if .Stories}}
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-5">
|
||||
{{range .Stories}}
|
||||
{{template "card" dict "Story" . "Theme" .Channel "ShowChannel" true}}
|
||||
{{end}}
|
||||
</div>
|
||||
|
||||
<nav class="mt-10 flex items-center justify-between">
|
||||
{{if .HasPrev}}
|
||||
<a href="{{.PrevURL}}" class="rounded-full bg-[color:var(--card)] border-2 border-[color:var(--ink)]/10 px-5 py-2 font-semibold shadow-pete hover:-translate-y-0.5 transition">← newer</a>
|
||||
{{else}}<span></span>{{end}}
|
||||
<span class="text-sm text-[color:var(--ink)]/60">page {{.Page}}</span>
|
||||
{{if .HasNext}}
|
||||
<a href="{{.NextURL}}" class="rounded-full bg-[color:var(--accent)] text-[#1c1305] px-5 py-2 font-semibold shadow-pete hover:-translate-y-0.5 transition">older →</a>
|
||||
{{else}}<span></span>{{end}}
|
||||
</nav>
|
||||
{{else}}
|
||||
<div class="rounded-3xl bg-[color:var(--card)] border-2 border-dashed border-[color:var(--ink)]/15 p-10 text-center text-[color:var(--ink)]/60">
|
||||
Nothing saved yet. Browse the feed and tap the 🔖 on a story to keep it here.
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
26
internal/web/templates/for-you.html
Normal file
26
internal/web/templates/for-you.html
Normal file
@@ -0,0 +1,26 @@
|
||||
{{define "title"}}For you — {{.SiteTitle}}{{end}}
|
||||
|
||||
{{define "main"}}
|
||||
<section class="mt-2 mb-8">
|
||||
<div class="rounded-3xl bg-[color:var(--accent)] text-[#1c1305] p-6 sm:p-10 shadow-pete relative overflow-hidden">
|
||||
<div class="absolute -top-6 -right-6 text-[12rem] opacity-20 select-none" aria-hidden="true">✨</div>
|
||||
<div class="relative">
|
||||
<p class="text-sm uppercase tracking-[0.2em] opacity-70">picked for you</p>
|
||||
<h1 class="font-display text-4xl sm:text-5xl font-bold mt-1">For you</h1>
|
||||
<p class="mt-2 max-w-xl opacity-80">Fresh stories weighted toward the channels and sources you read and bookmark most. The more you read, the sharper this gets.</p>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{{if .Stories}}
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-5">
|
||||
{{range .Stories}}
|
||||
{{template "card" dict "Story" . "Theme" .Channel "ShowChannel" true}}
|
||||
{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="rounded-3xl bg-[color:var(--card)] border-2 border-dashed border-[color:var(--ink)]/15 p-10 text-center text-[color:var(--ink)]/60">
|
||||
Nothing to tune yet. Read a few stories or bookmark the ones you like, and Pete will start building a feed around them.
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
@@ -6,12 +6,55 @@
|
||||
a friendlier news feed.
|
||||
</h1>
|
||||
<p class="mx-auto mt-3 max-w-xl text-base sm:text-lg text-[color:var(--ink)]/70">
|
||||
{{if .PostingEnabled}}
|
||||
Pete reads RSS, sorts stories, and posts them to Matrix.
|
||||
Glowing cards are the ones he's posted.
|
||||
{{else}}
|
||||
Pete reads RSS and sorts stories into channels, fresh from his feeds.
|
||||
{{end}}
|
||||
</p>
|
||||
<p class="mx-auto mt-5 inline-flex items-center gap-2 rounded-full bg-[color:var(--card)] px-4 py-2 text-sm font-semibold shadow-pete border-2 border-[color:var(--ink)]/10">
|
||||
<img src="/static/img/pete.avif" alt="Pete" width="24" height="24"
|
||||
class="h-6 w-6 rounded-full object-cover">
|
||||
<span data-pete-greeting>Hey, I'm Pete. Here's what's new.</span>
|
||||
</p>
|
||||
</section>
|
||||
|
||||
{{if .JustPosted}}
|
||||
<section data-weather-card class="mb-10 empty:hidden"></section>
|
||||
|
||||
{{if .Trending}}
|
||||
<section class="mb-10">
|
||||
<div class="flex items-end justify-between gap-3 mb-4">
|
||||
<h2 class="font-display text-xl sm:text-2xl font-bold">
|
||||
<span aria-hidden="true">🔥</span> Popular this week
|
||||
</h2>
|
||||
<span class="text-xs text-[color:var(--ink)]/50">what readers are opening most</span>
|
||||
</div>
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 gap-4">
|
||||
{{range .Trending}}
|
||||
{{template "card" dict "Story" . "Theme" .Channel "ShowChannel" true}}
|
||||
{{end}}
|
||||
</div>
|
||||
</section>
|
||||
{{end}}
|
||||
|
||||
{{if .ForYou}}
|
||||
<section class="mb-10">
|
||||
<div class="flex items-end justify-between gap-3 mb-4">
|
||||
<h2 class="font-display text-xl sm:text-2xl font-bold">
|
||||
<span aria-hidden="true">✨</span> For you
|
||||
</h2>
|
||||
<a href="/for-you" class="text-xs font-semibold text-[color:var(--ink)]/60 hover:text-[color:var(--ink)]">more →</a>
|
||||
</div>
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-4 gap-4">
|
||||
{{range .ForYou}}
|
||||
{{template "card" dict "Story" . "Theme" .Channel "ShowChannel" true}}
|
||||
{{end}}
|
||||
</div>
|
||||
</section>
|
||||
{{end}}
|
||||
|
||||
{{if and .PostingEnabled .JustPosted}}
|
||||
<section class="mb-10">
|
||||
<div class="flex items-end justify-between gap-3 mb-4">
|
||||
<h2 class="font-display text-xl sm:text-2xl font-bold">
|
||||
@@ -32,7 +75,7 @@
|
||||
<h2 class="font-display text-xl sm:text-2xl font-bold">
|
||||
<span aria-hidden="true">📊</span> Channels
|
||||
</h2>
|
||||
<span class="text-xs text-[color:var(--ink)]/50">last 24 hours</span>
|
||||
<span class="text-xs text-[color:var(--ink)]/50">{{if .PostingEnabled}}last 24 hours{{else}}what Pete's tracking{{end}}</span>
|
||||
</div>
|
||||
<div class="grid grid-cols-2 lg:grid-cols-4 gap-3 sm:gap-4">
|
||||
{{range .Stats}}
|
||||
@@ -45,6 +88,7 @@
|
||||
<span class="font-display text-lg font-semibold">{{.Channel.Title}}</span>
|
||||
</div>
|
||||
<dl class="mt-3 space-y-1 text-xs text-[color:var(--ink)]/70">
|
||||
{{if $.PostingEnabled}}
|
||||
<div class="flex justify-between gap-2">
|
||||
<dt>last post</dt>
|
||||
<dd class="font-semibold text-[color:var(--ink)]">
|
||||
@@ -55,6 +99,7 @@
|
||||
<dt>posted 24h</dt>
|
||||
<dd class="font-semibold text-[color:var(--ink)]">{{.PostsToday}}</dd>
|
||||
</div>
|
||||
{{end}}
|
||||
<div class="flex justify-between gap-2">
|
||||
<dt>stories</dt>
|
||||
<dd class="font-semibold text-[color:var(--ink)]">{{.Total}}</dd>
|
||||
@@ -80,7 +125,7 @@
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="rounded-3xl bg-[color:var(--card)] border-2 border-dashed border-[color:var(--ink)]/15 p-8 text-center text-[color:var(--ink)]/60">
|
||||
Nothing here yet — Pete's still listening.
|
||||
Nothing here yet. Pete's still listening for the first story.
|
||||
</div>
|
||||
{{end}}
|
||||
</section>
|
||||
|
||||
@@ -9,6 +9,17 @@
|
||||
<link href="https://fonts.googleapis.com/css2?family=Fredoka:wght@400;500;600;700&family=Nunito:wght@400;600;700&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="/static/css/output.css">
|
||||
<link rel="icon" href="/static/img/pete.avif" type="image/avif">
|
||||
<link rel="manifest" href="/manifest.webmanifest">
|
||||
<meta name="theme-color" content="#fbf3e3">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="default">
|
||||
<meta name="apple-mobile-web-app-title" content="Pete">
|
||||
<link rel="apple-touch-icon" href="/static/img/apple-touch-icon.png">
|
||||
<link rel="alternate" type="application/rss+xml" title="{{.SiteTitle}} — all stories" href="/feed.xml">
|
||||
<link rel="alternate" type="application/feed+json" title="{{.SiteTitle}} — all stories" href="/feed.json">
|
||||
{{if .Active}}{{if ne .Active "bookmarks"}}
|
||||
<link rel="alternate" type="application/rss+xml" title="{{.SiteTitle}} — {{.Active}}" href="/{{.Active}}/feed.xml">
|
||||
{{end}}{{end}}
|
||||
<script>
|
||||
// Pick a palette phase from the browser clock and update it each minute.
|
||||
(function () {
|
||||
@@ -32,16 +43,98 @@
|
||||
<canvas id="pete-weather" class="pointer-events-none fixed inset-0 -z-[5] h-full w-full" aria-hidden="true"></canvas>
|
||||
|
||||
<header class="mx-auto max-w-6xl px-4 pt-8 pb-4 sm:pt-12">
|
||||
<div class="flex items-center justify-between gap-4">
|
||||
<a href="/" class="flex items-center gap-3 group">
|
||||
<div class="flex items-center justify-between gap-3 sm:gap-4">
|
||||
<a href="/" class="flex items-center gap-3 group min-w-0">
|
||||
<img src="/static/img/pete.avif" alt="Pete" width="48" height="48"
|
||||
class="h-12 w-12 rounded-2xl bg-[color:var(--card)] shadow-pete border-2 border-[color:var(--ink)]/10 object-cover group-hover:-rotate-6 transition-transform">
|
||||
class="h-12 w-12 shrink-0 rounded-2xl bg-[color:var(--card)] shadow-pete border-2 border-[color:var(--ink)]/10 object-cover group-hover:-rotate-6 transition-transform">
|
||||
<span class="font-display text-3xl font-bold tracking-tight">{{.SiteTitle}}</span>
|
||||
<span class="hidden sm:inline rounded-full bg-[color:var(--accent)]/20 px-3 py-1 text-xs font-semibold uppercase tracking-wider text-[color:var(--ink)]/70" data-phase-label>day</span>
|
||||
</a>
|
||||
<div class="flex items-center gap-2">
|
||||
{{if .AuthEnabled}}
|
||||
{{if .User}}
|
||||
<a href="/auth/logout" data-account
|
||||
title="Signed in as {{.User.Display}}{{if .User.Email}} · {{.User.Email}}{{end}} — sign out"
|
||||
class="inline-flex shrink-0 items-center gap-2 rounded-full bg-[color:var(--card)] px-2.5 py-1.5 text-sm font-semibold shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition">
|
||||
<span class="grid h-6 w-6 place-items-center rounded-full bg-[color:var(--accent)] text-white text-xs font-bold">{{.User.Initial}}</span>
|
||||
<span class="hidden sm:inline max-w-[7rem] truncate">{{.User.Display}}</span>
|
||||
</a>
|
||||
{{else}}
|
||||
<a href="/auth/login?next={{.Path}}" data-signin
|
||||
title="Sign in to sync your preferences"
|
||||
class="inline-flex shrink-0 items-center gap-1.5 rounded-full bg-[color:var(--card)] px-3 py-2 text-sm font-semibold shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition">
|
||||
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
stroke-linecap="round" stroke-linejoin="round" class="h-4 w-4 text-[color:var(--ink)]/60">
|
||||
<path d="M15 3h4a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2h-4"></path>
|
||||
<polyline points="10 17 15 12 10 7"></polyline>
|
||||
<line x1="15" y1="12" x2="3" y2="12"></line>
|
||||
</svg>
|
||||
<span class="hidden sm:inline">Sign in</span>
|
||||
</a>
|
||||
{{end}}
|
||||
{{end}}
|
||||
</div>
|
||||
<div class="mt-3 flex flex-wrap items-center gap-2 sm:gap-3">
|
||||
<button type="button" data-settings-trigger
|
||||
title="Feed settings"
|
||||
class="inline-flex shrink-0 items-center justify-center rounded-full bg-[color:var(--card)] p-2 shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition">
|
||||
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
stroke-linecap="round" stroke-linejoin="round" class="h-5 w-5">
|
||||
<circle cx="12" cy="12" r="3"></circle>
|
||||
<path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 1 1-2.83 2.83l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 1 1-4 0v-.09a1.65 1.65 0 0 0-1-1.51 1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 1 1-2.83-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 1 1 0-4h.09a1.65 1.65 0 0 0 1.51-1 1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 1 1 2.83-2.83l.06.06a1.65 1.65 0 0 0 1.82.33h0a1.65 1.65 0 0 0 1-1.51V3a2 2 0 1 1 4 0v.09a1.65 1.65 0 0 0 1 1.51h0a1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 1 1 2.83 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82v0a1.65 1.65 0 0 0 1.51 1H21a2 2 0 1 1 0 4h-.09a1.65 1.65 0 0 0-1.51 1z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Feed settings</span>
|
||||
</button>
|
||||
<button type="button" data-reader-open
|
||||
title="Feed mode — read one story at a time (press f)"
|
||||
class="inline-flex shrink-0 items-center justify-center rounded-full bg-[color:var(--card)] p-2 shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition">
|
||||
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
stroke-linecap="round" stroke-linejoin="round" class="h-5 w-5">
|
||||
<path d="M2 4h6a3 3 0 0 1 3 3v13a2.5 2.5 0 0 0-2.5-2.5H2z"></path>
|
||||
<path d="M22 4h-6a3 3 0 0 0-3 3v13a2.5 2.5 0 0 1 2.5-2.5H22z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Feed mode</span>
|
||||
</button>
|
||||
{{if .Weather.Variant}}
|
||||
<button type="button" data-weather-toggle aria-pressed="true"
|
||||
title="Toggle weather animation"
|
||||
class="inline-flex shrink-0 items-center justify-center rounded-full bg-[color:var(--card)] p-2 shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition">
|
||||
<svg aria-hidden="true" viewBox="0 0 24 24" class="h-5 w-5 transition-colors"
|
||||
data-weather-star fill="currentColor" stroke="currentColor" stroke-width="1.5" stroke-linejoin="round">
|
||||
<polygon points="12,2.7 14.85,9.27 21.95,10.05 16.6,14.83 18.18,21.8 12,18.13 5.82,21.8 7.4,14.83 2.05,10.05 9.15,9.27"/>
|
||||
</svg>
|
||||
<span class="sr-only">Toggle weather animation</span>
|
||||
</button>
|
||||
{{end}}
|
||||
<button type="button" data-weather-loc
|
||||
title="Set your location"
|
||||
class="inline-flex shrink-0 items-center gap-1.5 rounded-full bg-[color:var(--card)] px-3 py-2 text-sm font-semibold shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition">
|
||||
<span data-weather-chip class="tabular-nums">📍 Weather</span>
|
||||
</button>
|
||||
<span data-aqi-chip
|
||||
class="hidden shrink-0 items-center gap-1.5 rounded-full bg-[color:var(--card)] px-3 py-2 text-sm font-semibold shadow-pete border-2 border-[color:var(--ink)]/10 tabular-nums"></span>
|
||||
<a href="/status" data-status-link
|
||||
title="Source health"
|
||||
class="inline-flex shrink-0 items-center justify-center rounded-full bg-[color:var(--card)] p-2 shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition{{if eq .Active "status"}} bg-[color:var(--accent)]/20{{end}}">
|
||||
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
stroke-linecap="round" stroke-linejoin="round" class="h-5 w-5">
|
||||
<path d="M3 12h4l2 6 4-14 2 8h6"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Source health</span>
|
||||
</a>
|
||||
{{if .User}}
|
||||
<a href="/bookmarks" data-bookmarks-link
|
||||
title="Your bookmarks"
|
||||
class="inline-flex shrink-0 items-center justify-center rounded-full bg-[color:var(--card)] p-2 shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition{{if eq .Active "bookmarks"}} bg-[color:var(--accent)]/20{{end}}">
|
||||
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
stroke-linecap="round" stroke-linejoin="round" class="h-5 w-5">
|
||||
<path d="M6 2h12a1 1 0 0 1 1 1v18l-7-4-7 4V3a1 1 0 0 1 1-1z"></path>
|
||||
</svg>
|
||||
<span class="sr-only">Bookmarks</span>
|
||||
</a>
|
||||
{{end}}
|
||||
<div class="flex basis-full items-center gap-2 min-w-0 max-w-full sm:basis-auto sm:ml-auto">
|
||||
<button type="button" data-search-trigger title="Search (⌘K)"
|
||||
class="flex items-center gap-2 rounded-full bg-[color:var(--card)] px-3 py-2 text-sm shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition">
|
||||
class="flex shrink-0 items-center gap-2 rounded-full bg-[color:var(--card)] px-3 py-2 text-sm shadow-pete border-2 border-[color:var(--ink)]/10 hover:bg-[color:var(--ink)]/5 transition">
|
||||
<svg aria-hidden="true" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
|
||||
stroke-linecap="round" stroke-linejoin="round" class="h-4 w-4 text-[color:var(--ink)]/60">
|
||||
<circle cx="11" cy="11" r="7"></circle>
|
||||
@@ -50,11 +143,19 @@
|
||||
<span class="hidden sm:inline text-[color:var(--ink)]/60 font-semibold">Search</span>
|
||||
<kbd class="hidden sm:inline-flex items-center rounded-md bg-[color:var(--ink)]/5 px-1.5 py-0.5 text-[10px] font-mono font-semibold text-[color:var(--ink)]/60">⌘K</kbd>
|
||||
</button>
|
||||
<nav class="flex items-center gap-1 sm:gap-2 rounded-full bg-[color:var(--card)] p-1 shadow-pete border-2 border-[color:var(--ink)]/10">
|
||||
<nav class="pete-channel-nav flex min-w-0 items-center gap-1 sm:gap-2 overflow-x-auto rounded-full bg-[color:var(--card)] p-1 shadow-pete border-2 border-[color:var(--ink)]/10">
|
||||
{{$active := .Active}}
|
||||
{{if .User}}
|
||||
<a href="/for-you"
|
||||
title="Your personalized feed"
|
||||
class="shrink-0 rounded-full px-3 py-2 text-sm font-semibold transition
|
||||
{{if eq $active "for-you"}}bg-[color:var(--accent)] text-[#1c1305] shadow-sm{{else}}hover:bg-[color:var(--ink)]/5{{end}}">
|
||||
<span aria-hidden="true" class="mr-1">✨</span><span class="hidden sm:inline">For you</span>
|
||||
</a>
|
||||
{{end}}
|
||||
{{range .Channels}}
|
||||
<a href="/{{.Slug}}"
|
||||
class="rounded-full px-3 py-2 text-sm font-semibold transition
|
||||
class="shrink-0 rounded-full px-3 py-2 text-sm font-semibold transition
|
||||
{{if eq $active .Slug}}bg-theme-{{.Theme}} text-white shadow-sm{{else}}hover:bg-[color:var(--ink)]/5{{end}}">
|
||||
<span aria-hidden="true" class="mr-1">{{.Emoji}}</span><span class="hidden sm:inline">{{.Title}}</span>
|
||||
</a>
|
||||
@@ -80,6 +181,92 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="pete-settings" class="hidden fixed inset-0 z-50 bg-[color:var(--ink)]/40 backdrop-blur-sm flex items-start justify-center pt-[10vh] px-4" aria-modal="true" role="dialog">
|
||||
<div class="w-full max-w-md rounded-3xl bg-[color:var(--card)] shadow-pete-lg border-2 border-[color:var(--ink)]/10 overflow-hidden">
|
||||
<div class="flex items-center justify-between gap-3 px-5 py-4 border-b border-[color:var(--ink)]/10">
|
||||
<h2 class="font-display text-lg font-bold">Feed settings</h2>
|
||||
<button type="button" data-settings-close class="rounded-full px-2 py-1 text-sm text-[color:var(--ink)]/60 hover:bg-[color:var(--ink)]/5">esc</button>
|
||||
</div>
|
||||
{{if .PushEnabled}}{{if .User}}
|
||||
<div data-push-section hidden class="px-5 pt-4 pb-1">
|
||||
<div class="flex items-center justify-between gap-3 rounded-2xl bg-[color:var(--ink)]/5 px-4 py-3">
|
||||
<div class="min-w-0">
|
||||
<div class="text-sm font-bold">🔔 Notifications</div>
|
||||
<div data-push-note class="text-xs text-[color:var(--ink)]/60">Get a nudge when new stories land.</div>
|
||||
</div>
|
||||
<button type="button" data-push-toggle aria-pressed="false"
|
||||
class="shrink-0 rounded-full bg-[color:var(--accent)] px-3 py-2 text-sm font-semibold text-[#1c1305] shadow-sm hover:brightness-105 transition disabled:opacity-50 disabled:cursor-not-allowed">
|
||||
Turn on notifications
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{{end}}{{end}}
|
||||
<p class="px-5 pt-3 text-xs text-[color:var(--ink)]/60">Uncheck a feed to hide its stories. <span data-storage-note>Saved in this browser.</span></p>
|
||||
<div data-settings-list class="max-h-[55vh] overflow-y-auto p-3 space-y-1"></div>
|
||||
<div class="px-5 py-3 border-t border-[color:var(--ink)]/10 flex items-center justify-between text-xs">
|
||||
<button type="button" data-settings-reset class="text-[color:var(--ink)]/60 hover:text-[color:var(--ink)] font-semibold">Enable all</button>
|
||||
<span data-settings-meta class="text-[color:var(--ink)]/50"></span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="pete-reader" class="hidden fixed inset-0 z-50" aria-modal="true" role="dialog" aria-label="Feed mode reader">
|
||||
<div class="pete-reader-backdrop" data-reader-backdrop></div>
|
||||
<div class="pete-reader-shell">
|
||||
<div class="pete-reader-bar">
|
||||
<span class="pete-reader-progress tabular-nums" data-reader-progress>—</span>
|
||||
<div class="pete-reader-nav">
|
||||
<button type="button" data-reader-prev class="pete-reader-btn" title="Previous (←)" aria-label="Previous article">←</button>
|
||||
<button type="button" data-reader-next class="pete-reader-btn" title="Next (→)" aria-label="Next article">→</button>
|
||||
<button type="button" data-reader-listen class="pete-reader-btn" style="display:none" title="Read aloud (r)" aria-label="Read this story aloud" aria-pressed="false">🔊 Listen</button>
|
||||
<button type="button" data-reader-share class="pete-reader-btn" style="display:none" title="Share (s)" aria-label="Share this story">Share</button>
|
||||
<div class="pete-reader-type">
|
||||
<button type="button" data-reader-type class="pete-reader-btn" title="Text options (t)" aria-label="Text options" aria-expanded="false"><span style="font-size:1.05rem">A</span><span style="font-size:.8rem">a</span></button>
|
||||
<div class="pete-reader-typemenu" data-reader-typemenu hidden>
|
||||
<div class="pete-reader-typerow">
|
||||
<span class="pete-reader-typelabel">Size</span>
|
||||
<div class="pete-reader-typebtns" data-reader-size>
|
||||
<button type="button" data-size="s" title="Small">S</button>
|
||||
<button type="button" data-size="m" title="Medium">M</button>
|
||||
<button type="button" data-size="l" title="Large">L</button>
|
||||
<button type="button" data-size="xl" title="Extra large">XL</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="pete-reader-typerow">
|
||||
<span class="pete-reader-typelabel">Font</span>
|
||||
<div class="pete-reader-typebtns" data-reader-font>
|
||||
<button type="button" data-font="sans">Sans</button>
|
||||
<button type="button" data-font="serif" style="font-family:Georgia,'Times New Roman',serif">Serif</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="pete-reader-typerow">
|
||||
<span class="pete-reader-typelabel">Paper</span>
|
||||
<div class="pete-reader-typebtns" data-reader-paper>
|
||||
<button type="button" data-paper="default">Default</button>
|
||||
<button type="button" data-paper="sepia">Sepia</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="pete-reader-typerow pete-reader-voicerow" data-reader-voicerow hidden>
|
||||
<span class="pete-reader-typelabel">Voice</span>
|
||||
<select class="pete-reader-voice" data-reader-voice aria-label="Read-aloud voice"></select>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<button type="button" data-reader-bookmark class="pete-reader-btn" style="display:none" title="Bookmark (b)" aria-label="Bookmark this story" aria-pressed="false">🔖 Save</button>
|
||||
<a data-reader-link target="_blank" rel="noopener noreferrer" class="pete-reader-btn pete-reader-btn-open" title="Open original (o)">Open ↗</a>
|
||||
<button type="button" data-reader-close class="pete-reader-btn" title="Close (esc)" aria-label="Close reader">esc</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="pete-reader-scroll" data-reader-scroll tabindex="-1">
|
||||
<article class="pete-reader-article" data-reader-article></article>
|
||||
<div class="pete-reader-related" data-reader-related hidden></div>
|
||||
</div>
|
||||
<div class="pete-reader-hint">
|
||||
<span><kbd>←</kbd> <kbd>→</kbd> navigate · <kbd>o</kbd> open · <kbd>s</kbd> share · <kbd>t</kbd> text · <kbd>u</kbd> unread · <kbd>esc</kbd> close</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<main class="mx-auto max-w-6xl px-4 pb-24">
|
||||
{{block "main" .}}{{end}}
|
||||
</main>
|
||||
@@ -91,8 +278,16 @@
|
||||
</footer>
|
||||
|
||||
<script>
|
||||
// Tiny clock + phase label updater for the header pill / footer.
|
||||
// Tiny clock + phase label updater for the header pill / footer, plus Pete's
|
||||
// time-of-day greeting on the home hero (whichever of these exist).
|
||||
(function () {
|
||||
function greeting(h) {
|
||||
if (h >= 5 && h < 8) return "Early start? Here's what came in overnight.";
|
||||
if (h >= 8 && h < 12) return "Morning! Here's what's fresh.";
|
||||
if (h >= 12 && h < 17) return "Afternoon. Here's what's new.";
|
||||
if (h >= 17 && h < 21) return "Evening. Here's what you missed today.";
|
||||
return "Late one? Here's the quiet-hours roundup.";
|
||||
}
|
||||
function tick() {
|
||||
var now = new Date();
|
||||
var hh = String(now.getHours()).padStart(2, "0");
|
||||
@@ -101,13 +296,29 @@
|
||||
if (el) el.textContent = hh + ":" + mm;
|
||||
var label = document.querySelector("[data-phase-label]");
|
||||
if (label) label.textContent = document.documentElement.dataset.phase;
|
||||
var greet = document.querySelector("[data-pete-greeting]");
|
||||
if (greet) greet.textContent = greeting(now.getHours());
|
||||
}
|
||||
tick();
|
||||
setInterval(tick, 30000);
|
||||
})();
|
||||
</script>
|
||||
|
||||
<script>
|
||||
window.PETE_SOURCES = {{.AllSources}};
|
||||
window.PETE_USER = {{if .User}}{ name: {{.User.Display}}, email: {{.User.Email}} }{{else}}null{{end}};
|
||||
window.PETE_PREFS = {{.UserPrefs}};
|
||||
window.PETE_PUSH = {{if .PushEnabled}}{ enabled: true, publicKey: {{.PushPublicKey}} }{{else}}null{{end}};
|
||||
window.PETE_TTS = {{.TTS}};
|
||||
</script>
|
||||
<script src="/static/js/prefs.js" defer></script>
|
||||
<script src="/static/js/weather-2d.js" defer></script>
|
||||
<script src="/static/js/weather-gl.js" defer></script>
|
||||
<script src="/static/js/weather.js" defer></script>
|
||||
<script src="/static/js/weather-forecast.js" defer></script>
|
||||
<script src="/static/js/search.js" defer></script>
|
||||
<script src="/static/js/settings.js" defer></script>
|
||||
<script src="/static/js/reader.js" defer></script>
|
||||
<script src="/static/js/pwa.js" defer></script>
|
||||
</body>
|
||||
</html>{{end}}
|
||||
|
||||
88
internal/web/templates/status.html
Normal file
88
internal/web/templates/status.html
Normal file
@@ -0,0 +1,88 @@
|
||||
{{define "title"}}Source health — {{.SiteTitle}}{{end}}
|
||||
|
||||
{{define "main"}}
|
||||
{{$admin := .Admin}}
|
||||
<section class="mt-2 mb-8">
|
||||
<div class="rounded-3xl bg-[color:var(--card)] border-2 border-[color:var(--ink)]/10 p-6 sm:p-8 shadow-pete">
|
||||
<p class="text-sm uppercase tracking-[0.2em] text-[color:var(--ink)]/50">{{if $admin}}owner view{{else}}live status{{end}}</p>
|
||||
<h1 class="font-display text-3xl sm:text-4xl font-bold mt-1">Source health</h1>
|
||||
<p class="mt-2 max-w-2xl text-[color:var(--ink)]/70">
|
||||
{{if $admin}}Per-feed poll status and content stats. Rows that are currently failing float to the top.
|
||||
{{else}}Every feed Pete follows and whether it's still updating. This runs on a best-effort basis, so a source going quiet for a bit is normal.{{end}}
|
||||
</p>
|
||||
<div class="mt-4 flex flex-wrap gap-2">
|
||||
<span class="inline-flex items-center gap-1.5 rounded-full bg-[color:var(--ink)]/5 px-3 py-1 text-sm font-semibold tabular-nums">
|
||||
{{len .Sources}} sources
|
||||
</span>
|
||||
{{if .DegradedCnt}}
|
||||
<span class="inline-flex items-center gap-1.5 rounded-full bg-red-500/15 text-red-700 dark:text-red-300 px-3 py-1 text-sm font-semibold tabular-nums">
|
||||
⚠ {{.DegradedCnt}} {{if $admin}}degraded{{else}}delayed{{end}}
|
||||
</span>
|
||||
{{else}}
|
||||
<span class="inline-flex items-center gap-1.5 rounded-full bg-emerald-500/15 text-emerald-700 dark:text-emerald-300 px-3 py-1 text-sm font-semibold">
|
||||
✓ all updating
|
||||
</span>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="overflow-x-auto rounded-3xl bg-[color:var(--card)] border-2 border-[color:var(--ink)]/10 shadow-pete">
|
||||
<table class="w-full {{if $admin}}min-w-[52rem]{{else}}min-w-[28rem]{{end}} text-sm">
|
||||
<thead>
|
||||
<tr class="text-left text-[color:var(--ink)]/50 uppercase text-xs tracking-wider border-b-2 border-[color:var(--ink)]/10">
|
||||
<th class="px-4 py-3 font-semibold">Source</th>
|
||||
<th class="px-4 py-3 font-semibold">Channel</th>
|
||||
<th class="px-4 py-3 font-semibold">Status</th>
|
||||
<th class="px-4 py-3 font-semibold">Last update</th>
|
||||
{{if $admin}}
|
||||
<th class="px-4 py-3 font-semibold">Last poll</th>
|
||||
<th class="px-4 py-3 font-semibold text-right">Items</th>
|
||||
<th class="px-4 py-3 font-semibold text-right">Stories</th>
|
||||
<th class="px-4 py-3 font-semibold text-right">Paywall</th>
|
||||
<th class="px-4 py-3 font-semibold">Last story</th>
|
||||
<th class="px-4 py-3 font-semibold">Last posted</th>
|
||||
{{end}}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{range .Sources}}
|
||||
<tr class="border-b border-[color:var(--ink)]/5 last:border-0 align-top">
|
||||
<td class="px-4 py-3 font-semibold whitespace-nowrap">{{.Name}}</td>
|
||||
<td class="px-4 py-3 text-[color:var(--ink)]/70 whitespace-nowrap">{{.Channel}}</td>
|
||||
<td class="px-4 py-3 whitespace-nowrap">
|
||||
{{if .NeverRun}}
|
||||
<span class="inline-flex items-center gap-1.5 rounded-full bg-[color:var(--ink)]/10 px-2.5 py-1 text-xs font-semibold">◌ idle</span>
|
||||
{{else if .Healthy}}
|
||||
<span class="inline-flex items-center gap-1.5 rounded-full bg-emerald-500/15 text-emerald-700 dark:text-emerald-300 px-2.5 py-1 text-xs font-semibold">● live</span>
|
||||
{{else if $admin}}
|
||||
<span class="inline-flex items-center gap-1.5 rounded-full bg-red-500/15 text-red-700 dark:text-red-300 px-2.5 py-1 text-xs font-semibold"
|
||||
title="{{.LastError}}">✕ {{.Failures}} fail{{if ne .Failures 1}}s{{end}}</span>
|
||||
{{else}}
|
||||
<span class="inline-flex items-center gap-1.5 rounded-full bg-red-500/15 text-red-700 dark:text-red-300 px-2.5 py-1 text-xs font-semibold">✕ delayed</span>
|
||||
{{end}}
|
||||
</td>
|
||||
<td class="px-4 py-3 text-[color:var(--ink)]/70 whitespace-nowrap">{{if .LastSuccessAt.IsZero}}never{{else}}{{timeAgo .LastSuccessAt}}{{end}}</td>
|
||||
{{if $admin}}
|
||||
<td class="px-4 py-3 text-[color:var(--ink)]/70 whitespace-nowrap">{{if .LastPollAt.IsZero}}never{{else}}{{timeAgo .LastPollAt}}{{end}}</td>
|
||||
<td class="px-4 py-3 text-right tabular-nums">{{.LastItemCount}}</td>
|
||||
<td class="px-4 py-3 text-right tabular-nums" title="{{.Classified}} classified of {{.Total}}">{{.Total}}</td>
|
||||
<td class="px-4 py-3 text-right tabular-nums {{if gt .PaywallRate 50}}text-amber-600 dark:text-amber-400 font-semibold{{end}}">
|
||||
{{if .Total}}{{.PaywallRate}}%{{else}}—{{end}}
|
||||
</td>
|
||||
<td class="px-4 py-3 text-[color:var(--ink)]/70 whitespace-nowrap">{{if .LastSeenAt.IsZero}}—{{else}}{{timeAgo .LastSeenAt}}{{end}}</td>
|
||||
<td class="px-4 py-3 text-[color:var(--ink)]/70 whitespace-nowrap">{{if .LastPostedAt.IsZero}}—{{else}}{{timeAgo .LastPostedAt}}{{end}}</td>
|
||||
{{end}}
|
||||
</tr>
|
||||
{{if $admin}}{{if .LastError}}{{if not .Healthy}}
|
||||
<tr class="border-b border-[color:var(--ink)]/5">
|
||||
<td colspan="10" class="px-4 pb-3 -mt-1 text-xs text-red-600 dark:text-red-400 font-mono break-all">{{.LastError}}</td>
|
||||
</tr>
|
||||
{{end}}{{end}}{{end}}
|
||||
{{else}}
|
||||
<tr><td colspan="{{if $admin}}10{{else}}4{{end}}" class="px-4 py-8 text-center text-[color:var(--ink)]/50">No sources configured.</td></tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -3,14 +3,21 @@
|
||||
{{define "main"}}
|
||||
<div class="space-y-8">
|
||||
<section class="rounded-3xl bg-[color:var(--card)] p-6 shadow-pete border-2 border-[color:var(--ink)]/10">
|
||||
<h1 class="font-display text-3xl font-bold mb-1">Weather demo</h1>
|
||||
<p class="text-sm text-[color:var(--ink)]/60 mb-5">Pick a variant, intensity, and phase. The canvas reloads on each click.</p>
|
||||
<div class="flex flex-wrap items-start justify-between gap-3 mb-1">
|
||||
<h1 class="font-display text-3xl font-bold">Weather demo</h1>
|
||||
<div class="flex items-center gap-2 text-xs font-semibold">
|
||||
<span data-demo-renderer class="rounded-full border-2 border-[color:var(--ink)]/10 px-3 py-1 uppercase tracking-wider text-[color:var(--ink)]/60">renderer: …</span>
|
||||
<span data-demo-fps class="rounded-full border-2 border-[color:var(--ink)]/10 px-3 py-1 tabular-nums text-[color:var(--ink)]/60">— fps</span>
|
||||
</div>
|
||||
</div>
|
||||
<p class="text-sm text-[color:var(--ink)]/60 mb-5">Pick a variant, intensity, and phase. Switching is instant, no reload.</p>
|
||||
|
||||
<div class="space-y-3 text-sm">
|
||||
<div class="flex flex-wrap items-center gap-2">
|
||||
<span class="w-20 shrink-0 font-semibold uppercase tracking-wider text-[color:var(--ink)]/60">Variant</span>
|
||||
{{range $v := .Variants}}
|
||||
<a href="?variant={{$v}}&intensity={{$.Intensity}}&phase={{$.PhaseSel}}"
|
||||
data-demo-variant="{{$v}}"
|
||||
class="rounded-full px-3 py-1 transition border-2 border-[color:var(--ink)]/10
|
||||
{{if eq $.Variant $v}}bg-[color:var(--accent)] text-white font-semibold{{else}}hover:bg-[color:var(--ink)]/5{{end}}">{{$v}}</a>
|
||||
{{end}}
|
||||
@@ -19,6 +26,7 @@
|
||||
<span class="w-20 shrink-0 font-semibold uppercase tracking-wider text-[color:var(--ink)]/60">Intensity</span>
|
||||
{{range $i := .Intensities}}
|
||||
<a href="?variant={{$.Variant}}&intensity={{$i}}&phase={{$.PhaseSel}}"
|
||||
data-demo-intensity="{{$i}}"
|
||||
class="rounded-full px-3 py-1 transition border-2 border-[color:var(--ink)]/10
|
||||
{{if eq $.Intensity $i}}bg-[color:var(--accent)] text-white font-semibold{{else}}hover:bg-[color:var(--ink)]/5{{end}}">{{$i}}</a>
|
||||
{{end}}
|
||||
@@ -27,6 +35,7 @@
|
||||
<span class="w-20 shrink-0 font-semibold uppercase tracking-wider text-[color:var(--ink)]/60">Phase</span>
|
||||
{{range $p := .Phases}}
|
||||
<a href="?variant={{$.Variant}}&intensity={{$.Intensity}}&phase={{$p}}"
|
||||
data-demo-phase="{{$p}}"
|
||||
class="rounded-full px-3 py-1 transition border-2 border-[color:var(--ink)]/10
|
||||
{{if eq $.PhaseSel $p}}bg-[color:var(--accent)] text-white font-semibold{{else}}hover:bg-[color:var(--ink)]/5{{end}}">{{$p}}</a>
|
||||
{{end}}
|
||||
@@ -34,7 +43,8 @@
|
||||
</div>
|
||||
|
||||
<div class="mt-5 text-xs text-[color:var(--ink)]/50">
|
||||
Current: <code class="font-semibold">{{.Weather.Season}} / {{.Variant}} / {{.Intensity}} / {{.PhaseSel}}</code>
|
||||
Current: <code data-demo-current class="font-semibold">{{.Variant}} / {{.Intensity}} / {{.PhaseSel}}</code>
|
||||
<span class="mx-1">·</span> Tip: aurora and clear night want the night phase; wind and haze are the new autumn gusts and Saharan calima.
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -50,8 +60,76 @@
|
||||
</div>
|
||||
<div class="rounded-3xl bg-[color:var(--card)] p-6 shadow-pete border-2 border-[color:var(--ink)]/10">
|
||||
<p class="font-display">Card B</p>
|
||||
<p class="text-sm text-[color:var(--ink)]/60 mt-1">Move your window taller to give the particles more room.</p>
|
||||
<p class="text-sm text-[color:var(--ink)]/60 mt-1">Make your window taller to give the particles more room.</p>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Instant switching: the pill links stay real URLs for no-JS visitors, but
|
||||
// with JS we swap the effect in place via PeteWeather.set and keep the URL
|
||||
// in sync with replaceState.
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
var root = document.documentElement;
|
||||
var state = {
|
||||
variant: {{.Variant}},
|
||||
intensity: {{.Intensity}},
|
||||
phase: {{.PhaseSel}}
|
||||
};
|
||||
|
||||
function markActive(attr, value) {
|
||||
document.querySelectorAll("[" + attr + "]").forEach(function (el) {
|
||||
var on = el.getAttribute(attr) === value;
|
||||
el.classList.toggle("bg-[color:var(--accent)]", on);
|
||||
el.classList.toggle("text-white", on);
|
||||
el.classList.toggle("font-semibold", on);
|
||||
});
|
||||
}
|
||||
|
||||
function apply() {
|
||||
root.dataset.phase = state.phase;
|
||||
if (window.PeteWeather) window.PeteWeather.set(state.variant, state.intensity);
|
||||
markActive("data-demo-variant", state.variant);
|
||||
markActive("data-demo-intensity", state.intensity);
|
||||
markActive("data-demo-phase", state.phase);
|
||||
var cur = document.querySelector("[data-demo-current]");
|
||||
if (cur) cur.textContent = state.variant + " / " + state.intensity + " / " + state.phase;
|
||||
var label = document.querySelector("[data-phase-label]");
|
||||
if (label) label.textContent = state.phase;
|
||||
history.replaceState(null, "", "?variant=" + state.variant + "&intensity=" + state.intensity + "&phase=" + state.phase);
|
||||
}
|
||||
|
||||
document.addEventListener("click", function (e) {
|
||||
var a = e.target.closest && e.target.closest("[data-demo-variant],[data-demo-intensity],[data-demo-phase]");
|
||||
if (!a || !window.PeteWeather) return;
|
||||
e.preventDefault();
|
||||
if (a.hasAttribute("data-demo-variant")) state.variant = a.getAttribute("data-demo-variant");
|
||||
if (a.hasAttribute("data-demo-intensity")) state.intensity = a.getAttribute("data-demo-intensity");
|
||||
if (a.hasAttribute("data-demo-phase")) state.phase = a.getAttribute("data-demo-phase");
|
||||
apply();
|
||||
});
|
||||
|
||||
var rEl = document.querySelector("[data-demo-renderer]");
|
||||
if (rEl && window.PeteWeather && window.PeteWeather.renderer) {
|
||||
rEl.textContent = "renderer: " + window.PeteWeather.renderer();
|
||||
}
|
||||
|
||||
// Lightweight FPS meter, demo page only. Counts frames on its own rAF and
|
||||
// refreshes the pill once a second.
|
||||
var fpsEl = document.querySelector("[data-demo-fps]");
|
||||
if (fpsEl) {
|
||||
var frames = 0, lastStamp = performance.now();
|
||||
function tick(now) {
|
||||
frames++;
|
||||
if (now - lastStamp >= 1000) {
|
||||
fpsEl.textContent = Math.round(frames * 1000 / (now - lastStamp)) + " fps";
|
||||
frames = 0;
|
||||
lastStamp = now;
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
});
|
||||
</script>
|
||||
{{end}}
|
||||
|
||||
@@ -2,6 +2,7 @@ package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
@@ -16,6 +17,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -23,20 +25,33 @@ import (
|
||||
"golang.org/x/image/draw"
|
||||
_ "golang.org/x/image/webp"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
thumbWidth = 800
|
||||
thumbMaxBytes = 12 << 20 // 12 MiB cap on source download
|
||||
thumbMaxBytes = 12 << 20 // 12 MiB cap on source download for images
|
||||
videoMaxBytes = 64 << 20 // 64 MiB cap when the source is a video file
|
||||
thumbFetchTimeout = 12 * time.Second
|
||||
thumbCacheDir = "./data/img-cache"
|
||||
thumbQuality = "45"
|
||||
thumbSpeed = "8"
|
||||
ffmpegTimeout = 20 * time.Second
|
||||
)
|
||||
|
||||
// videoExtensions are file extensions we route through the ffmpeg
|
||||
// frame-extraction path instead of trying to decode as an image.
|
||||
var videoExtensions = map[string]bool{
|
||||
".mp4": true,
|
||||
".m4v": true,
|
||||
".mov": true,
|
||||
".webm": true,
|
||||
".mkv": true,
|
||||
}
|
||||
|
||||
var (
|
||||
thumbClient = &http.Client{Timeout: thumbFetchTimeout}
|
||||
thumbClient = safehttp.NewClient(thumbFetchTimeout)
|
||||
|
||||
thumbMu sync.Mutex
|
||||
thumbInFlight = map[string]*sync.WaitGroup{}
|
||||
@@ -127,7 +142,8 @@ func serveAvif(w http.ResponseWriter, r *http.Request, path string) {
|
||||
}
|
||||
|
||||
// buildAvifThumb downloads src, decodes + resizes in Go, then shells out to
|
||||
// avifenc to produce a compact AVIF at outPath.
|
||||
// avifenc to produce a compact AVIF at outPath. Video URLs (mp4, webm, ...)
|
||||
// take a parallel path that extracts a frame with ffmpeg before re-encoding.
|
||||
func buildAvifThumb(src, outPath string) error {
|
||||
req, err := http.NewRequest(http.MethodGet, src, nil)
|
||||
if err != nil {
|
||||
@@ -142,12 +158,28 @@ func buildAvifThumb(src, outPath string) error {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("upstream %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, thumbMaxBytes+1))
|
||||
|
||||
// URL extension is enough for the cap decision (Content-Type may be
|
||||
// missing or generic on some CDNs); Content-Type is checked again below
|
||||
// to decide which decode path to take.
|
||||
maxBytes := int64(thumbMaxBytes)
|
||||
urlIsVideo := isVideoURL(src)
|
||||
if urlIsVideo {
|
||||
maxBytes = videoMaxBytes
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(body) > thumbMaxBytes {
|
||||
return fmt.Errorf("image too big (>%d bytes)", thumbMaxBytes)
|
||||
if int64(len(body)) > maxBytes {
|
||||
if urlIsVideo {
|
||||
return fmt.Errorf("video too big (>%d bytes)", maxBytes)
|
||||
}
|
||||
return fmt.Errorf("image too big (>%d bytes)", maxBytes)
|
||||
}
|
||||
|
||||
if urlIsVideo || isVideoContentType(resp.Header.Get("Content-Type")) {
|
||||
return buildAvifFromFFmpeg(body, outPath)
|
||||
}
|
||||
|
||||
if isAvif(body) {
|
||||
@@ -164,6 +196,13 @@ func buildAvifThumb(src, outPath string) error {
|
||||
}
|
||||
src2, _, err := image.Decode(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
// Go's stdlib JPEG decoder rejects some valid-but-rare features
|
||||
// (e.g. 4:1:1 luma/chroma subsampling served by ANN's CDN). Hand
|
||||
// the bytes to ffmpeg as a permissive fallback — it produces the
|
||||
// same shape of output as the video path.
|
||||
if ffErr := buildAvifFromFFmpeg(body, outPath); ffErr == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("decode: %w", err)
|
||||
}
|
||||
b := src2.Bounds()
|
||||
@@ -200,13 +239,17 @@ func buildAvifThumb(src, outPath string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Encode to a sibling temp file then atomically rename so partial outputs
|
||||
// never appear in the cache.
|
||||
return avifEncodeFromPNG(tmpPath, outPath)
|
||||
}
|
||||
|
||||
// avifEncodeFromPNG runs avifenc on pngPath and atomically renames the
|
||||
// result into outPath so partial outputs never appear in the cache.
|
||||
func avifEncodeFromPNG(pngPath, outPath string) error {
|
||||
stagedOut := outPath + ".tmp"
|
||||
cmd := exec.Command("avifenc",
|
||||
"-q", thumbQuality,
|
||||
"-s", thumbSpeed,
|
||||
tmpPath, stagedOut,
|
||||
pngPath, stagedOut,
|
||||
)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
os.Remove(stagedOut)
|
||||
@@ -215,6 +258,121 @@ func buildAvifThumb(src, outPath string) error {
|
||||
return os.Rename(stagedOut, outPath)
|
||||
}
|
||||
|
||||
// isVideoURL reports whether the URL's path extension is one we treat as a
|
||||
// video source for thumbnail extraction.
|
||||
func isVideoURL(src string) bool {
|
||||
u, err := url.Parse(src)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return videoExtensions[strings.ToLower(filepath.Ext(u.Path))]
|
||||
}
|
||||
|
||||
// isVideoContentType reports whether a Content-Type header announces a video.
|
||||
func isVideoContentType(ct string) bool {
|
||||
return strings.HasPrefix(strings.ToLower(strings.TrimSpace(ct)), "video/")
|
||||
}
|
||||
|
||||
// buildAvifFromFFmpeg writes the downloaded bytes to a temp file, asks
|
||||
// ffmpeg to extract a single resized frame as PNG, then hands it to avifenc.
|
||||
// Used both for video sources and as a permissive fallback for images whose
|
||||
// JPEG features Go's stdlib decoder rejects. ffmpeg is optional — if it's
|
||||
// missing we return an error and the caller falls back to redirecting to
|
||||
// the source URL, just like any other build failure.
|
||||
func buildAvifFromFFmpeg(body []byte, outPath string) error {
|
||||
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
||||
return fmt.Errorf("ffmpeg not available: %w", err)
|
||||
}
|
||||
|
||||
in, err := os.CreateTemp("", "pete-vid-*.bin")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inPath := in.Name()
|
||||
defer os.Remove(inPath)
|
||||
if _, err := in.Write(body); err != nil {
|
||||
in.Close()
|
||||
return err
|
||||
}
|
||||
if err := in.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
outPNG, err := os.CreateTemp("", "pete-vid-frame-*.png")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pngPath := outPNG.Name()
|
||||
outPNG.Close()
|
||||
defer os.Remove(pngPath)
|
||||
|
||||
// Aim ~5s in — past the typical fade-in / first-GOP blockiness while
|
||||
// still likely within the clip. For shorter clips we fall back to the
|
||||
// midpoint (when ffprobe is available) so we don't seek off the end.
|
||||
seek := "5"
|
||||
if d := probeVideoDuration(inPath); d > 0 && d < 5 {
|
||||
seek = strconv.FormatFloat(d/2, 'f', 2, 64)
|
||||
}
|
||||
if err := extractVideoFrame(inPath, pngPath, seek); err != nil {
|
||||
// Seek overshot or container's funky — try 2s, then frame 0.
|
||||
if err := extractVideoFrame(inPath, pngPath, "2"); err != nil {
|
||||
if err := extractVideoFrame(inPath, pngPath, "0"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return avifEncodeFromPNG(pngPath, outPath)
|
||||
}
|
||||
|
||||
// probeVideoDuration returns the clip length in seconds, or 0 if ffprobe is
|
||||
// missing or can't read it (image-as-video fallback, malformed containers).
|
||||
func probeVideoDuration(inPath string) float64 {
|
||||
if _, err := exec.LookPath("ffprobe"); err != nil {
|
||||
return 0
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), ffmpegTimeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "ffprobe",
|
||||
"-v", "error",
|
||||
"-show_entries", "format=duration",
|
||||
"-of", "default=noprint_wrappers=1:nokey=1",
|
||||
inPath,
|
||||
)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
d, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64)
|
||||
if err != nil || d <= 0 {
|
||||
return 0
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// extractVideoFrame asks ffmpeg to seek to seekSec, grab one frame, and
|
||||
// downscale it to at most thumbWidth wide before writing pngPath.
|
||||
func extractVideoFrame(inPath, pngPath, seekSec string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), ffmpegTimeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "ffmpeg",
|
||||
"-y",
|
||||
"-ss", seekSec,
|
||||
"-i", inPath,
|
||||
"-frames:v", "1",
|
||||
"-update", "1",
|
||||
"-vf", fmt.Sprintf("scale='min(%d,iw)':-2", thumbWidth),
|
||||
"-f", "image2",
|
||||
pngPath,
|
||||
)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("ffmpeg: %w: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
if fi, err := os.Stat(pngPath); err != nil || fi.Size() == 0 {
|
||||
return fmt.Errorf("ffmpeg produced empty frame")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isAvif reports whether body looks like an AVIF file via the ISO-BMFF "ftyp"
|
||||
// brand at bytes 4..12 ("ftypavif"/"ftypavis").
|
||||
func isAvif(body []byte) bool {
|
||||
|
||||
69
internal/web/trending_test.go
Normal file
69
internal/web/trending_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"net/http/httptest"
|
||||
|
||||
"pete/internal/config"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// TestIndexTrendingAndBadges seeds a story with captured content and some reader
|
||||
// views, then asserts the home page renders the "Popular this week" rail plus the
|
||||
// per-card reading-time chip and read-count badge that decorate() fills in.
|
||||
func TestIndexTrendingAndBadges(t *testing.T) {
|
||||
storage.Close()
|
||||
if err := storage.Init(filepath.Join(t.TempDir(), "trending.db")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { storage.Close() })
|
||||
|
||||
// ~1200 chars of body → about a 1 min read.
|
||||
body := strings.Repeat("word ", 240)
|
||||
story := &storage.Story{
|
||||
GUID: "trend-e2e",
|
||||
Headline: "A Trending Headline",
|
||||
Lede: "Lede.",
|
||||
Content: body,
|
||||
ArticleURL: "https://example.com/trend",
|
||||
Source: "Example Wire",
|
||||
Channel: "tech",
|
||||
Classified: true,
|
||||
SeenAt: time.Now().Unix(),
|
||||
}
|
||||
if err := storage.InsertStory(story); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var id int64
|
||||
if err := storage.Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, story.GUID).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Two reads so the story trends and the badge shows a count.
|
||||
storage.RecordStoryView(id)
|
||||
storage.RecordStoryView(id)
|
||||
|
||||
s, err := New(config.WebConfig{SiteTitle: "Pete", ListenAddr: ":0"}, nil, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rw := httptest.NewRecorder()
|
||||
s.handleIndex(rw, httptest.NewRequest("GET", "/", nil))
|
||||
if rw.Code != 200 {
|
||||
t.Fatalf("index status = %d", rw.Code)
|
||||
}
|
||||
body = rw.Body.String()
|
||||
|
||||
for _, want := range []string{
|
||||
"Popular this week", // trending rail heading
|
||||
"min read", // reading-time chip
|
||||
`title="2 reads"`, // view-count badge
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("index HTML missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
199
internal/web/tts.go
Normal file
199
internal/web/tts.go
Normal file
@@ -0,0 +1,199 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"html/template"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
)
|
||||
|
||||
const (
|
||||
ttsMaxTextLen = 6000 // per-request cap; a single paragraph is ~1-2k chars
|
||||
ttsTimeout = 60 * time.Second
|
||||
ttsMaxConcurrent = 3 // cap simultaneous piper processes so a burst can't pin the box
|
||||
)
|
||||
|
||||
// ttsVoice is one selectable voice, as sent to the client.
|
||||
type ttsVoice struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
}
|
||||
|
||||
// ttsService runs Piper (https://github.com/rhasspy/piper) as a subprocess to
|
||||
// synthesize read-aloud audio server-side. It holds the validated voice
|
||||
// registry and a concurrency semaphore; there is no long-lived process, each
|
||||
// request spawns a short-lived piper that loads its model, synthesizes one
|
||||
// chunk of text to a WAV on stdout, and exits (~0.5s for a paragraph).
|
||||
type ttsService struct {
|
||||
piperBin string
|
||||
voices []ttsVoice // menu order, for the client selector
|
||||
models map[string]string // voice id -> absolute .onnx path
|
||||
def string // default voice id
|
||||
sem chan struct{} // buffered to ttsMaxConcurrent
|
||||
}
|
||||
|
||||
// newTTS validates the Piper install and builds the voice registry. It returns
|
||||
// (nil, nil) when TTS is disabled. A configured voice whose model file is
|
||||
// missing is skipped with a warning rather than failing startup; if that leaves
|
||||
// no usable voices, TTS is disabled.
|
||||
func newTTS(cfg config.TTSConfig) (*ttsService, error) {
|
||||
if !cfg.Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
bin := cfg.PiperBin
|
||||
if bin == "" {
|
||||
bin = "piper"
|
||||
}
|
||||
if p, err := exec.LookPath(bin); err != nil {
|
||||
slog.Error("web: TTS enabled but piper binary not found; read-aloud disabled", "piper_bin", bin, "err", err)
|
||||
return nil, nil
|
||||
} else {
|
||||
bin = p
|
||||
}
|
||||
|
||||
dir := cfg.VoicesDir
|
||||
svc := &ttsService{piperBin: bin, models: make(map[string]string)}
|
||||
|
||||
want := cfg.Voices
|
||||
if len(want) == 0 {
|
||||
// Auto-discover every *.onnx in voices_dir, labelled by filename stem.
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
slog.Error("web: TTS voices_dir unreadable; read-aloud disabled", "voices_dir", dir, "err", err)
|
||||
return nil, nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if e.IsDir() || !strings.HasSuffix(name, ".onnx") {
|
||||
continue
|
||||
}
|
||||
id := strings.TrimSuffix(name, ".onnx")
|
||||
want = append(want, config.VoiceConfig{ID: id, Label: id})
|
||||
}
|
||||
sort.Slice(want, func(i, j int) bool { return want[i].ID < want[j].ID })
|
||||
}
|
||||
|
||||
for _, v := range want {
|
||||
if v.ID == "" {
|
||||
continue
|
||||
}
|
||||
model := filepath.Join(dir, v.ID+".onnx")
|
||||
if _, err := os.Stat(model); err != nil {
|
||||
slog.Warn("web: TTS voice model missing; skipping", "voice", v.ID, "path", model)
|
||||
continue
|
||||
}
|
||||
label := v.Label
|
||||
if label == "" {
|
||||
label = v.ID
|
||||
}
|
||||
svc.models[v.ID] = model
|
||||
svc.voices = append(svc.voices, ttsVoice{ID: v.ID, Label: label})
|
||||
}
|
||||
if len(svc.voices) == 0 {
|
||||
slog.Error("web: TTS enabled but no usable voices found; read-aloud disabled", "voices_dir", dir)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
svc.def = cfg.Default
|
||||
if _, ok := svc.models[svc.def]; !ok {
|
||||
svc.def = svc.voices[0].ID
|
||||
}
|
||||
svc.sem = make(chan struct{}, ttsMaxConcurrent)
|
||||
slog.Info("web: server-side TTS enabled", "piper", bin, "voices", len(svc.voices), "default", svc.def)
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// clientConfig is the JSON handed to the page as window.PETE_TTS so the reader
|
||||
// can build its voice selector and know TTS is available.
|
||||
func (t *ttsService) clientConfig() template.JS {
|
||||
payload := struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Default string `json:"default"`
|
||||
Voices []ttsVoice `json:"voices"`
|
||||
}{Enabled: true, Default: t.def, Voices: t.voices}
|
||||
b, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return template.JS("null")
|
||||
}
|
||||
return jsForScript(b)
|
||||
}
|
||||
|
||||
// handleTTS synthesizes one chunk of text to WAV audio with the requested
|
||||
// voice. It is registered only under the authenticated route group, so callers
|
||||
// are already signed in (read-aloud is a signed-in perk). The request body is
|
||||
// JSON {voice, text}; the response is audio/wav.
|
||||
func (s *Server) handleTTS(w http.ResponseWriter, r *http.Request) {
|
||||
if s.tts == nil {
|
||||
http.Error(w, "tts disabled", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if s.auth == nil || s.auth.userFromRequest(r) == nil {
|
||||
http.Error(w, "sign-in required", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Voice string `json:"voice"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 32*1024)).Decode(&req); err != nil {
|
||||
http.Error(w, "bad request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
text := strings.TrimSpace(req.Text)
|
||||
if text == "" {
|
||||
http.Error(w, "empty text", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if len(text) > ttsMaxTextLen {
|
||||
text = text[:ttsMaxTextLen]
|
||||
}
|
||||
model, ok := s.tts.models[req.Voice]
|
||||
if !ok {
|
||||
model = s.tts.models[s.tts.def] // unknown/blank voice -> default
|
||||
}
|
||||
|
||||
// Bound concurrent piper processes; give up if the client leaves or we wait
|
||||
// too long for a slot rather than queueing unboundedly.
|
||||
slotCtx, cancelSlot := context.WithTimeout(r.Context(), ttsTimeout)
|
||||
defer cancelSlot()
|
||||
select {
|
||||
case s.tts.sem <- struct{}{}:
|
||||
defer func() { <-s.tts.sem }()
|
||||
case <-slotCtx.Done():
|
||||
http.Error(w, "tts busy", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), ttsTimeout)
|
||||
defer cancel()
|
||||
// piper -m <model> -f - : write a full WAV to stdout. Text is fed on stdin,
|
||||
// so there is no shell and nothing user-controlled reaches the arg list
|
||||
// besides the model path, which is looked up from a fixed allowlist above.
|
||||
cmd := exec.CommandContext(ctx, s.tts.piperBin, "-m", model, "-f", "-")
|
||||
cmd.Stdin = strings.NewReader(text)
|
||||
var out, errb bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &errb
|
||||
if err := cmd.Run(); err != nil {
|
||||
slog.Error("web: piper synthesis failed", "err", err, "stderr", strings.TrimSpace(errb.String()))
|
||||
http.Error(w, "synthesis failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
w.Header().Set("Cache-Control", "private, max-age=3600")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(out.Len()))
|
||||
_, _ = w.Write(out.Bytes())
|
||||
}
|
||||
159
internal/web/userstate_api.go
Normal file
159
internal/web/userstate_api.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// maxStateBodyBytes caps read/bookmark request bodies. These carry a single id
|
||||
// and a boolean, so this is generous headroom.
|
||||
const maxStateBodyBytes = 1024
|
||||
|
||||
// maxStateIDs bounds how many ids /api/state will look up in one call. A page
|
||||
// renders a few dozen cards; this is well clear of that.
|
||||
const maxStateIDs = 500
|
||||
|
||||
// requireUser resolves the signed-in user or writes the appropriate error and
|
||||
// returns nil. It mirrors handlePrefs: 404 when auth is disabled entirely, 401
|
||||
// for anonymous callers (the client's cue to stay on localStorage).
|
||||
func (s *Server) requireUser(w http.ResponseWriter, r *http.Request) *SessionUser {
|
||||
if s.auth == nil {
|
||||
http.Error(w, "auth disabled", http.StatusNotFound)
|
||||
return nil
|
||||
}
|
||||
u := s.auth.userFromRequest(r)
|
||||
if u == nil {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
http.Error(w, `{"error":"not signed in"}`, http.StatusUnauthorized)
|
||||
return nil
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// decodeStateBody reads a small JSON body into dst, writing a 400 on failure.
|
||||
func decodeStateBody(w http.ResponseWriter, r *http.Request, dst any) bool {
|
||||
return decodeStateBodyN(w, r, dst, maxStateBodyBytes)
|
||||
}
|
||||
|
||||
// decodeStateBodyN is decodeStateBody with an explicit byte cap, for endpoints
|
||||
// (like push subscribe) whose payloads run larger than a single id + flag.
|
||||
func decodeStateBodyN(w http.ResponseWriter, r *http.Request, dst any, limit int64) bool {
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, limit+1))
|
||||
if err != nil || int64(len(body)) > limit {
|
||||
http.Error(w, `{"error":"bad request"}`, http.StatusBadRequest)
|
||||
return false
|
||||
}
|
||||
if err := json.Unmarshal(body, dst); err != nil {
|
||||
http.Error(w, `{"error":"invalid JSON"}`, http.StatusBadRequest)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// handleRead records or clears a story's read flag for the signed-in user.
|
||||
func (s *Server) handleRead(w http.ResponseWriter, r *http.Request) {
|
||||
u := s.requireUser(w, r)
|
||||
if u == nil {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
ID int64 `json:"id"`
|
||||
Read bool `json:"read"`
|
||||
}
|
||||
if !decodeStateBody(w, r, &req) {
|
||||
return
|
||||
}
|
||||
if req.ID <= 0 {
|
||||
http.Error(w, `{"error":"bad id"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := storage.SetRead(u.Sub, req.ID, req.Read); err != nil {
|
||||
slog.Error("state: set read failed", "sub", u.Sub, "id", req.ID, "err", err)
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleBookmark adds or removes a bookmark for the signed-in user.
|
||||
func (s *Server) handleBookmark(w http.ResponseWriter, r *http.Request) {
|
||||
u := s.requireUser(w, r)
|
||||
if u == nil {
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
ID int64 `json:"id"`
|
||||
On bool `json:"on"`
|
||||
}
|
||||
if !decodeStateBody(w, r, &req) {
|
||||
return
|
||||
}
|
||||
if req.ID <= 0 {
|
||||
http.Error(w, `{"error":"bad id"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := storage.SetBookmark(u.Sub, req.ID, req.On); err != nil {
|
||||
slog.Error("state: set bookmark failed", "sub", u.Sub, "id", req.ID, "err", err)
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleState returns which of the given story ids are read and bookmarked, so
|
||||
// the client can paint a freshly rendered page in one round-trip.
|
||||
func (s *Server) handleState(w http.ResponseWriter, r *http.Request) {
|
||||
u := s.requireUser(w, r)
|
||||
if u == nil {
|
||||
return
|
||||
}
|
||||
ids := parseIDList(r.URL.Query().Get("ids"))
|
||||
read, bookmarked, err := storage.UserStoryState(u.Sub, ids)
|
||||
if err != nil {
|
||||
slog.Error("state: lookup failed", "sub", u.Sub, "err", err)
|
||||
http.Error(w, `{"error":"internal error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"read": keysOf(read),
|
||||
"bookmarked": keysOf(bookmarked),
|
||||
})
|
||||
}
|
||||
|
||||
// parseIDList turns "1,2,3" into a bounded, deduplicated slice of positive ids.
|
||||
func parseIDList(raw string) []int64 {
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[int64]bool)
|
||||
out := make([]int64, 0)
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
id, err := strconv.ParseInt(strings.TrimSpace(part), 10, 64)
|
||||
if err != nil || id <= 0 || seen[id] {
|
||||
continue
|
||||
}
|
||||
seen[id] = true
|
||||
out = append(out, id)
|
||||
if len(out) >= maxStateIDs {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// keysOf returns the keys of a set as a slice (order unspecified).
|
||||
func keysOf(m map[int64]bool) []int64 {
|
||||
out := make([]int64, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
196
main.go
196
main.go
@@ -3,9 +3,12 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -18,17 +21,38 @@ import (
|
||||
"pete/internal/storage"
|
||||
"pete/internal/web"
|
||||
|
||||
webpush "github.com/SherClockHolmes/webpush-go"
|
||||
"maunium.net/go/mautrix/id"
|
||||
)
|
||||
|
||||
// runGenVAPID prints a fresh VAPID keypair for the [web.push] config block.
|
||||
func runGenVAPID() {
|
||||
priv, pub, err := webpush.GenerateVAPIDKeys()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "genvapid: failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("# Add these under [web.push] in your config:")
|
||||
fmt.Printf("vapid_public_key = %q\n", pub)
|
||||
fmt.Printf("vapid_private_key = %q\n", priv)
|
||||
}
|
||||
|
||||
func main() {
|
||||
configPath := flag.String("config", "config.toml", "path to config file")
|
||||
seed := flag.Bool("seed", false, "ingest current feed items as seen without posting, then exit")
|
||||
test := flag.Bool("test", false, "post one story to verify the full pipeline, then exit")
|
||||
testSource := flag.String("test-source", "", "source name to use for -test (default: first enabled)")
|
||||
local := flag.Bool("local", false, "web/RSS-only mode: poll feeds and serve the web UI, no Matrix login or posting")
|
||||
backfillPaywall := flag.Bool("backfill-paywall", false, "re-check paywalled rows with current detection logic; clear false positives and fill missing thumbnails, then exit")
|
||||
genVAPID := flag.Bool("genvapid", false, "generate a VAPID keypair for web.push and print it, then exit")
|
||||
flag.Parse()
|
||||
|
||||
// -genvapid needs neither config nor database; handle it before anything else.
|
||||
if *genVAPID {
|
||||
runGenVAPID()
|
||||
return
|
||||
}
|
||||
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||
Level: slog.LevelInfo,
|
||||
})))
|
||||
@@ -57,6 +81,11 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
if *backfillPaywall {
|
||||
runBackfillPaywall()
|
||||
return
|
||||
}
|
||||
|
||||
// Test mode: post one story to verify the full pipeline, then exit
|
||||
if *test {
|
||||
runTest(cfg, *testSource)
|
||||
@@ -82,11 +111,38 @@ func main() {
|
||||
// Wire reaction handler
|
||||
mx.SetReactionHandler(poster.HandleReaction)
|
||||
|
||||
// Wire !post command: force-publish next queued story for the room's channel.
|
||||
// Allowlist of users permitted to run in-room commands. The channel rooms
|
||||
// are public, so commands are gated to configured admins; an empty list
|
||||
// disables them entirely (fail closed) rather than allowing anyone.
|
||||
admins := make(map[id.UserID]bool, len(cfg.Matrix.Admins))
|
||||
for _, a := range cfg.Matrix.Admins {
|
||||
if a = strings.TrimSpace(a); a != "" {
|
||||
admins[id.UserID(a)] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Wire in-room commands. All are gated to the admin allowlist; the room
|
||||
// rooms are public so anything not from an admin is silently ignored.
|
||||
mx.SetMessageHandler(func(channel string, roomID id.RoomID, eventID id.EventID, sender id.UserID, body string) {
|
||||
if strings.TrimSpace(body) != "!post" {
|
||||
cmd := strings.TrimSpace(body)
|
||||
if cmd != "!post" && cmd != "!petestats" {
|
||||
return
|
||||
}
|
||||
if !admins[sender] {
|
||||
slog.Warn("command ignored: sender not in matrix.admins allowlist", "cmd", cmd, "channel", channel, "sender", sender)
|
||||
return
|
||||
}
|
||||
|
||||
if cmd == "!petestats" {
|
||||
slog.Info("!petestats requested", "channel", channel, "sender", sender)
|
||||
plain, htmlBody := formatStats(storage.GetMetricsSummary())
|
||||
if err := mx.PostThreadedReply(channel, eventID, plain, htmlBody); err != nil {
|
||||
slog.Warn("!petestats: failed to send reply", "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// !post: force-publish the next queued story for the room's channel.
|
||||
slog.Info("!post requested", "channel", channel, "sender", sender)
|
||||
if queue.ForcePost(channel) {
|
||||
return
|
||||
@@ -138,7 +194,11 @@ func main() {
|
||||
go queue.Start(ctx)
|
||||
slog.Info("post queue started")
|
||||
|
||||
postingEnabled := *cfg.Posting.Enabled
|
||||
roundRobinMode := cfg.Posting.RoundRobin.Enabled
|
||||
if !postingEnabled {
|
||||
slog.Info("automatic Matrix posting is disabled (posting.enabled=false); stories will still be ingested and served to the web UI, and !post still works")
|
||||
}
|
||||
|
||||
// Pipeline callback: route the story to its direct_route channel, then
|
||||
// either enqueue immediately or leave it for the round-robin scheduler.
|
||||
@@ -152,6 +212,12 @@ func main() {
|
||||
channel := item.DirectRoute
|
||||
storage.MarkClassified(item.GUID, channel, "[]")
|
||||
|
||||
// Posting disabled: classify and keep for the web UI / manual !post,
|
||||
// but never auto-enqueue to Matrix.
|
||||
if !postingEnabled {
|
||||
return
|
||||
}
|
||||
|
||||
if roundRobinMode {
|
||||
slog.Info("story routed, awaiting round-robin tick",
|
||||
"guid", item.GUID, "source", item.Source, "channel", channel)
|
||||
@@ -179,7 +245,7 @@ func main() {
|
||||
poller.Start(ctx)
|
||||
slog.Info("pollers started")
|
||||
|
||||
if roundRobinMode {
|
||||
if postingEnabled && roundRobinMode {
|
||||
channelNames := make([]string, 0, len(cfg.Matrix.Channels))
|
||||
for name := range cfg.Matrix.Channels {
|
||||
channelNames = append(channelNames, name)
|
||||
@@ -194,11 +260,12 @@ func main() {
|
||||
|
||||
// Start the read-only web UI alongside the Matrix bot.
|
||||
if cfg.Web.Enabled {
|
||||
ws, err := web.New(cfg.Web)
|
||||
ws, err := web.New(cfg.Web, cfg.Sources, postingEnabled)
|
||||
if err != nil {
|
||||
slog.Error("web server init failed", "err", err)
|
||||
} else {
|
||||
go ws.Start(ctx)
|
||||
ws.StartPushSender(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -216,6 +283,69 @@ func main() {
|
||||
slog.Info("pete stopped")
|
||||
}
|
||||
|
||||
// formatStats renders a usage summary for the !petestats reply, returning a
|
||||
// plain-text and an HTML body. Daily uniques are reported per-day only: the
|
||||
// privacy salt rotates each day, so they cannot be honestly summed across days.
|
||||
func formatStats(m storage.MetricsSummary) (plain, htmlBody string) {
|
||||
const topN = 8
|
||||
|
||||
var p, h strings.Builder
|
||||
p.WriteString("📊 Pete usage\n")
|
||||
h.WriteString("📊 <b>Pete usage</b><br>")
|
||||
|
||||
fmt.Fprintf(&p, "Today: %s views · ~%s unique visitors\n",
|
||||
comma(m.ViewsToday), comma(m.UniquesToday))
|
||||
fmt.Fprintf(&h, "Today: <b>%s</b> views · ~<b>%s</b> unique visitors<br>",
|
||||
comma(m.ViewsToday), comma(m.UniquesToday))
|
||||
|
||||
fmt.Fprintf(&p, "All time: %s views\n", comma(m.TotalViews))
|
||||
fmt.Fprintf(&h, "All time: <b>%s</b> views<br>", comma(m.TotalViews))
|
||||
|
||||
if len(m.Pages) > 0 {
|
||||
pages := m.Pages
|
||||
if len(pages) > topN {
|
||||
pages = pages[:topN]
|
||||
}
|
||||
parts := make([]string, 0, len(pages))
|
||||
for _, pg := range pages {
|
||||
parts = append(parts, fmt.Sprintf("%s %s", pg.Path, comma(pg.Total)))
|
||||
}
|
||||
fmt.Fprintf(&p, "Top pages (all time): %s\n", strings.Join(parts, " · "))
|
||||
fmt.Fprintf(&h, "Top pages (all time): %s<br>",
|
||||
template.HTMLEscapeString(strings.Join(parts, " · ")))
|
||||
}
|
||||
|
||||
if len(m.Last7Days) > 0 {
|
||||
parts := make([]string, 0, len(m.Last7Days))
|
||||
for _, d := range m.Last7Days {
|
||||
label := time.Unix(d.Day*86400, 0).UTC().Format("Mon")
|
||||
parts = append(parts, fmt.Sprintf("%s %s", label, comma(d.Uniques)))
|
||||
}
|
||||
fmt.Fprintf(&p, "Daily uniques (7d): %s", strings.Join(parts, " · "))
|
||||
fmt.Fprintf(&h, "Daily uniques (7d): %s",
|
||||
template.HTMLEscapeString(strings.Join(parts, " · ")))
|
||||
}
|
||||
|
||||
return p.String(), h.String()
|
||||
}
|
||||
|
||||
// comma formats a non-negative int with thousands separators (e.g. 12503 →
|
||||
// "12,503").
|
||||
func comma(n int) string {
|
||||
s := strconv.Itoa(n)
|
||||
if n < 0 {
|
||||
return s // not expected for counts; leave untouched
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, c := range s {
|
||||
if i > 0 && (len(s)-i)%3 == 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteRune(c)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func runLocal(cfg *config.Config) {
|
||||
cfg.Web.Enabled = true
|
||||
|
||||
@@ -240,12 +370,17 @@ func runLocal(cfg *config.Config) {
|
||||
poller.Start(ctx)
|
||||
slog.Info("local: pollers started")
|
||||
|
||||
ws, err := web.New(cfg.Web)
|
||||
// Local mode never connects to Matrix, so it's always web-only.
|
||||
ws, err := web.New(cfg.Web, cfg.Sources, false)
|
||||
if err != nil {
|
||||
slog.Error("local: web server init failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
go ws.Start(ctx)
|
||||
// Push only needs the web auth layer, not Matrix, so a web-only deployment
|
||||
// still runs the digest sender — otherwise subscriptions accumulate but no
|
||||
// digest ever fires.
|
||||
ws.StartPushSender(ctx)
|
||||
slog.Info("local: web UI listening", "addr", cfg.Web.ListenAddr)
|
||||
|
||||
storage.RunMaintenance()
|
||||
@@ -263,7 +398,7 @@ func runSeed(cfg *config.Config) {
|
||||
if !src.Enabled {
|
||||
continue
|
||||
}
|
||||
items, err := ingestion.FetchFeed(src.FeedURL)
|
||||
items, err := ingestion.FetchFeed(src.FeedURL, src.UserAgent)
|
||||
if err != nil {
|
||||
slog.Error("seed: feed fetch failed", "source", src.Name, "err", err)
|
||||
continue
|
||||
@@ -303,7 +438,7 @@ func runTest(cfg *config.Config, sourceName string) {
|
||||
if sourceName != "" && s.Name != sourceName {
|
||||
continue
|
||||
}
|
||||
items, err := ingestion.FetchFeed(s.FeedURL)
|
||||
items, err := ingestion.FetchFeed(s.FeedURL, s.UserAgent)
|
||||
if err != nil {
|
||||
slog.Error("test: feed fetch failed", "source", s.Name, "err", err)
|
||||
continue
|
||||
@@ -373,3 +508,50 @@ func runTest(cfg *config.Config, sourceName string) {
|
||||
func timeNow() int64 {
|
||||
return time.Now().Unix()
|
||||
}
|
||||
|
||||
// runBackfillPaywall re-runs the current paywall detection logic against
|
||||
// every row marked paywalled=1. For each row we re-fetch the stored article
|
||||
// URL; if the new logic no longer considers it gated we clear the flag and,
|
||||
// when the row has no thumbnail, populate it from the live og:image.
|
||||
func runBackfillPaywall() {
|
||||
rows, err := storage.Get().Query(`SELECT guid, article_url, image_url FROM stories WHERE paywalled = 1`)
|
||||
if err != nil {
|
||||
slog.Error("backfill: query failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
type rec struct{ guid, url, img string }
|
||||
var pending []rec
|
||||
for rows.Next() {
|
||||
var r rec
|
||||
if err := rows.Scan(&r.guid, &r.url, &r.img); err != nil {
|
||||
slog.Error("backfill: scan failed", "err", err)
|
||||
continue
|
||||
}
|
||||
pending = append(pending, r)
|
||||
}
|
||||
rows.Close()
|
||||
slog.Info("backfill: starting", "candidates", len(pending))
|
||||
|
||||
cleared, imaged, stillGated := 0, 0, 0
|
||||
for _, r := range pending {
|
||||
meta := ingestion.FetchArticleMeta(r.url)
|
||||
gated := meta.Gated() || (meta.Fetched && meta.BodyChars < ingestion.PaywallBodyThreshold)
|
||||
if gated {
|
||||
stillGated++
|
||||
slog.Info("backfill: still gated", "guid", r.guid, "status", meta.Status, "body_chars", meta.BodyChars)
|
||||
continue
|
||||
}
|
||||
newImg := r.img
|
||||
if newImg == "" && meta.ImageURL != "" {
|
||||
newImg = meta.ImageURL
|
||||
imaged++
|
||||
}
|
||||
if _, err := storage.Get().Exec(`UPDATE stories SET paywalled = 0, image_url = ? WHERE guid = ?`, newImg, r.guid); err != nil {
|
||||
slog.Error("backfill: update failed", "guid", r.guid, "err", err)
|
||||
continue
|
||||
}
|
||||
cleared++
|
||||
slog.Info("backfill: cleared paywall flag", "guid", r.guid, "filled_image", newImg != r.img)
|
||||
}
|
||||
slog.Info("backfill complete", "cleared", cleared, "still_gated", stillGated, "thumbnails_filled", imaged)
|
||||
}
|
||||
|
||||
@@ -7,9 +7,14 @@ import (
|
||||
"testing"
|
||||
|
||||
"pete/internal/config"
|
||||
"pete/internal/safehttp"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
func init() {
|
||||
safehttp.AllowPrivate = true
|
||||
}
|
||||
|
||||
func setupTestDB(t *testing.T) string {
|
||||
t.Helper()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
|
||||
BIN
pete_ava.png
BIN
pete_ava.png
Binary file not shown.
|
Before Width: | Height: | Size: 61 KiB |
Reference in New Issue
Block a user