Local mode, Makefile, image fixes, Pete avatar
- Add -local flag: web/RSS-only mode that skips Matrix login and posting, so the web UI can be exercised against live feeds without credentials - Add Makefile (build/local/seed/test/clean) that handles Tailwind + go build in one shot - Fix Guardian thumbnails: NormalizeImageURL was rewriting width=1200 onto signed i.guim.co.uk URLs, invalidating the s= signature and returning 401. Leave signed URLs alone and pick the widest media:content variant up front instead - Use pete.avif as the header logo, favicon, and footer mark; drop the unused leaf.svg
This commit is contained in:
25
Makefile
Normal file
25
Makefile
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
.PHONY: build css go-build local seed test clean
|
||||||
|
|
||||||
|
build: css go-build
|
||||||
|
|
||||||
|
css: node_modules
|
||||||
|
npm run build:css
|
||||||
|
|
||||||
|
node_modules: package.json package-lock.json
|
||||||
|
npm install
|
||||||
|
@touch node_modules
|
||||||
|
|
||||||
|
go-build:
|
||||||
|
go build -tags goolm -o pete .
|
||||||
|
|
||||||
|
local: build
|
||||||
|
./pete -local
|
||||||
|
|
||||||
|
seed: build
|
||||||
|
./pete -seed
|
||||||
|
|
||||||
|
test:
|
||||||
|
go test ./...
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -f pete internal/web/static/css/output.css
|
||||||
@@ -94,6 +94,7 @@ With a 4-hour cadence Pete will post at most 6 stories/day, so set `posting.dail
|
|||||||
| `-config <path>` | Path to config file (default: `config.toml`) |
|
| `-config <path>` | Path to config file (default: `config.toml`) |
|
||||||
| `-seed` | Ingest all current feed items as seen without posting, then exit |
|
| `-seed` | Ingest all current feed items as seen without posting, then exit |
|
||||||
| `-test` | Post one story to verify the full pipeline, then exit |
|
| `-test` | Post one story to verify the full pipeline, then exit |
|
||||||
|
| `-local` | Web/RSS-only mode: poll feeds and serve the web UI on `web.listen_addr`. Skips Matrix login and posting — useful for local testing |
|
||||||
|
|
||||||
## Docker
|
## Docker
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,13 @@ func NormalizeImageURL(raw string) string {
|
|||||||
if q.Get("width") == "" {
|
if q.Get("width") == "" {
|
||||||
return raw
|
return raw
|
||||||
}
|
}
|
||||||
|
// The Guardian signs each (width, quality, fit, …) combination with `s=`.
|
||||||
|
// Changing width without re-signing yields a 401 "invalid signature",
|
||||||
|
// so leave signed URLs alone — the parser is now responsible for picking
|
||||||
|
// the widest media:content variant up front.
|
||||||
|
if q.Get("s") != "" {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
q.Set("width", "1200")
|
q.Set("width", "1200")
|
||||||
u.RawQuery = q.Encode()
|
u.RawQuery = q.Encode()
|
||||||
return u.String()
|
return u.String()
|
||||||
|
|||||||
@@ -6,10 +6,12 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/mmcdole/gofeed"
|
"github.com/mmcdole/gofeed"
|
||||||
|
ext "github.com/mmcdole/gofeed/extensions"
|
||||||
)
|
)
|
||||||
|
|
||||||
const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"
|
const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"
|
||||||
@@ -88,23 +90,21 @@ func extractLede(desc string) string {
|
|||||||
// scraped from content:encoded / description (catches feeds like Ars that
|
// scraped from content:encoded / description (catches feeds like Ars that
|
||||||
// embed the lead image in the article body but not as a media:* element).
|
// embed the lead image in the article body but not as a media:* element).
|
||||||
func extractImageURL(item *gofeed.Item) string {
|
func extractImageURL(item *gofeed.Item) string {
|
||||||
// Check media content (common in RSS 2.0 with media namespace)
|
// Check media content (common in RSS 2.0 with media namespace).
|
||||||
|
// When multiple media:content variants are advertised (Guardian RSS lists
|
||||||
|
// 140/460/700/1200…) we want the widest signed URL we can find.
|
||||||
if item.Extensions != nil {
|
if item.Extensions != nil {
|
||||||
if media, ok := item.Extensions["media"]; ok {
|
if media, ok := item.Extensions["media"]; ok {
|
||||||
if contents, ok := media["content"]; ok {
|
if contents, ok := media["content"]; ok {
|
||||||
for _, c := range contents {
|
if url := widestURL(contents); url != "" {
|
||||||
if url := c.Attrs["url"]; url != "" {
|
|
||||||
return url
|
return url
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if thumbs, ok := media["thumbnail"]; ok {
|
if thumbs, ok := media["thumbnail"]; ok {
|
||||||
for _, t := range thumbs {
|
if url := widestURL(thumbs); url != "" {
|
||||||
if url := t.Attrs["url"]; url != "" {
|
|
||||||
return url
|
return url
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
// media:group wraps nested media:content / media:thumbnail in some feeds
|
// media:group wraps nested media:content / media:thumbnail in some feeds
|
||||||
if groups, ok := media["group"]; ok {
|
if groups, ok := media["group"]; ok {
|
||||||
for _, g := range groups {
|
for _, g := range groups {
|
||||||
@@ -143,6 +143,25 @@ func extractImageURL(item *gofeed.Item) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// widestURL picks the entry with the largest declared width attribute, falling
|
||||||
|
// back to the first URL if none have a parsable width.
|
||||||
|
func widestURL(entries []ext.Extension) string {
|
||||||
|
best := ""
|
||||||
|
bestW := -1
|
||||||
|
for _, e := range entries {
|
||||||
|
url := e.Attrs["url"]
|
||||||
|
if url == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
w, _ := strconv.Atoi(e.Attrs["width"])
|
||||||
|
if w > bestW {
|
||||||
|
best = url
|
||||||
|
bestW = w
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best
|
||||||
|
}
|
||||||
|
|
||||||
func firstImgSrc(htmlBody string) string {
|
func firstImgSrc(htmlBody string) string {
|
||||||
if htmlBody == "" {
|
if htmlBody == "" {
|
||||||
return ""
|
return ""
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -1,4 +0,0 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64">
|
|
||||||
<path d="M48 8C28 8 12 22 12 42c0 6 2 12 6 16 4-12 14-22 26-26-10 8-16 18-18 30 16 0 30-14 30-34 0-8-4-16-8-20z" fill="#6db73c" stroke="#3f6e1f" stroke-width="3" stroke-linejoin="round"/>
|
|
||||||
<path d="M22 52c8-12 18-20 30-26" stroke="#3f6e1f" stroke-width="2" fill="none" stroke-linecap="round"/>
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 365 B |
BIN
internal/web/static/img/pete.avif
Normal file
BIN
internal/web/static/img/pete.avif
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 7.8 KiB |
@@ -8,7 +8,7 @@
|
|||||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||||
<link href="https://fonts.googleapis.com/css2?family=Fredoka:wght@400;500;600;700&family=Nunito:wght@400;600;700&display=swap" rel="stylesheet">
|
<link href="https://fonts.googleapis.com/css2?family=Fredoka:wght@400;500;600;700&family=Nunito:wght@400;600;700&display=swap" rel="stylesheet">
|
||||||
<link rel="stylesheet" href="/static/css/output.css">
|
<link rel="stylesheet" href="/static/css/output.css">
|
||||||
<link rel="icon" href="/static/img/leaf.svg" type="image/svg+xml">
|
<link rel="icon" href="/static/img/pete.avif" type="image/avif">
|
||||||
<script>
|
<script>
|
||||||
// Pick a palette phase from the browser clock and update it each minute.
|
// Pick a palette phase from the browser clock and update it each minute.
|
||||||
(function () {
|
(function () {
|
||||||
@@ -32,7 +32,8 @@
|
|||||||
<header class="mx-auto max-w-6xl px-4 pt-8 pb-4 sm:pt-12">
|
<header class="mx-auto max-w-6xl px-4 pt-8 pb-4 sm:pt-12">
|
||||||
<div class="flex items-center justify-between gap-4">
|
<div class="flex items-center justify-between gap-4">
|
||||||
<a href="/" class="flex items-center gap-3 group">
|
<a href="/" class="flex items-center gap-3 group">
|
||||||
<span class="inline-flex h-12 w-12 items-center justify-center rounded-2xl bg-[color:var(--card)] shadow-pete border-2 border-[color:var(--ink)]/10 text-2xl group-hover:-rotate-6 transition-transform">🌿</span>
|
<img src="/static/img/pete.avif" alt="Pete" width="48" height="48"
|
||||||
|
class="h-12 w-12 rounded-2xl bg-[color:var(--card)] shadow-pete border-2 border-[color:var(--ink)]/10 object-cover group-hover:-rotate-6 transition-transform">
|
||||||
<span class="font-display text-3xl font-bold tracking-tight">{{.SiteTitle}}</span>
|
<span class="font-display text-3xl font-bold tracking-tight">{{.SiteTitle}}</span>
|
||||||
<span class="hidden sm:inline rounded-full bg-[color:var(--accent)]/20 px-3 py-1 text-xs font-semibold uppercase tracking-wider text-[color:var(--ink)]/70" data-phase-label>day</span>
|
<span class="hidden sm:inline rounded-full bg-[color:var(--accent)]/20 px-3 py-1 text-xs font-semibold uppercase tracking-wider text-[color:var(--ink)]/70" data-phase-label>day</span>
|
||||||
</a>
|
</a>
|
||||||
@@ -54,6 +55,8 @@
|
|||||||
</main>
|
</main>
|
||||||
|
|
||||||
<footer class="mx-auto max-w-6xl px-4 pb-10 text-center text-sm text-[color:var(--ink)]/50">
|
<footer class="mx-auto max-w-6xl px-4 pb-10 text-center text-sm text-[color:var(--ink)]/50">
|
||||||
|
<img src="/static/img/pete.avif" alt="" width="20" height="20"
|
||||||
|
class="inline-block h-5 w-5 rounded-full object-cover align-[-4px] mr-1">
|
||||||
served by <span class="font-semibold">Pete</span> · also a Matrix bot · <span data-clock>—</span>
|
served by <span class="font-semibold">Pete</span> · also a Matrix bot · <span data-clock>—</span>
|
||||||
</footer>
|
</footer>
|
||||||
|
|
||||||
|
|||||||
48
main.go
48
main.go
@@ -26,6 +26,7 @@ func main() {
|
|||||||
seed := flag.Bool("seed", false, "ingest current feed items as seen without posting, then exit")
|
seed := flag.Bool("seed", false, "ingest current feed items as seen without posting, then exit")
|
||||||
test := flag.Bool("test", false, "post one story to verify the full pipeline, then exit")
|
test := flag.Bool("test", false, "post one story to verify the full pipeline, then exit")
|
||||||
testSource := flag.String("test-source", "", "source name to use for -test (default: first enabled)")
|
testSource := flag.String("test-source", "", "source name to use for -test (default: first enabled)")
|
||||||
|
local := flag.Bool("local", false, "web/RSS-only mode: poll feeds and serve the web UI, no Matrix login or posting")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||||
@@ -62,6 +63,12 @@ func main() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Local mode: poll feeds + serve web UI only. No Matrix, no posting queue.
|
||||||
|
if *local {
|
||||||
|
runLocal(cfg)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Create Matrix client
|
// Create Matrix client
|
||||||
mx, err := matrix.New(cfg.Matrix)
|
mx, err := matrix.New(cfg.Matrix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -209,6 +216,47 @@ func main() {
|
|||||||
slog.Info("pete stopped")
|
slog.Info("pete stopped")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func runLocal(cfg *config.Config) {
|
||||||
|
cfg.Web.Enabled = true
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
sigCh := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
|
||||||
|
processItem := func(_ context.Context, item *ingestion.FeedItem) {
|
||||||
|
if item.DirectRoute == "" {
|
||||||
|
slog.Info("local: item has no direct_route, discarding", "guid", item.GUID, "source", item.Source)
|
||||||
|
storage.MarkClassified(item.GUID, "_discarded", "[]")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
storage.MarkClassified(item.GUID, item.DirectRoute, "[]")
|
||||||
|
slog.Info("local: story classified",
|
||||||
|
"guid", item.GUID, "source", item.Source, "channel", item.DirectRoute, "headline", item.Headline)
|
||||||
|
}
|
||||||
|
|
||||||
|
poller := ingestion.NewPoller(cfg.Sources, processItem)
|
||||||
|
poller.Start(ctx)
|
||||||
|
slog.Info("local: pollers started")
|
||||||
|
|
||||||
|
ws, err := web.New(cfg.Web)
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("local: web server init failed", "err", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
go ws.Start(ctx)
|
||||||
|
slog.Info("local: web UI listening", "addr", cfg.Web.ListenAddr)
|
||||||
|
|
||||||
|
storage.RunMaintenance()
|
||||||
|
|
||||||
|
sig := <-sigCh
|
||||||
|
slog.Info("local: shutdown signal received", "signal", sig)
|
||||||
|
cancel()
|
||||||
|
poller.Wait()
|
||||||
|
slog.Info("pete stopped")
|
||||||
|
}
|
||||||
|
|
||||||
func runSeed(cfg *config.Config) {
|
func runSeed(cfg *config.Config) {
|
||||||
total := 0
|
total := 0
|
||||||
for _, src := range cfg.Sources {
|
for _, src := range cfg.Sources {
|
||||||
|
|||||||
Reference in New Issue
Block a user