Local mode, Makefile, image fixes, Pete avatar
- Add -local flag: web/RSS-only mode that skips Matrix login and posting, so the web UI can be exercised against live feeds without credentials - Add Makefile (build/local/seed/test/clean) that handles Tailwind + go build in one shot - Fix Guardian thumbnails: NormalizeImageURL was rewriting width=1200 onto signed i.guim.co.uk URLs, invalidating the s= signature and returning 401. Leave signed URLs alone and pick the widest media:content variant up front instead - Use pete.avif as the header logo, favicon, and footer mark; drop the unused leaf.svg
This commit is contained in:
25
Makefile
Normal file
25
Makefile
Normal file
@@ -0,0 +1,25 @@
|
||||
.PHONY: build css go-build local seed test clean
|
||||
|
||||
build: css go-build
|
||||
|
||||
css: node_modules
|
||||
npm run build:css
|
||||
|
||||
node_modules: package.json package-lock.json
|
||||
npm install
|
||||
@touch node_modules
|
||||
|
||||
go-build:
|
||||
go build -tags goolm -o pete .
|
||||
|
||||
local: build
|
||||
./pete -local
|
||||
|
||||
seed: build
|
||||
./pete -seed
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
clean:
|
||||
rm -f pete internal/web/static/css/output.css
|
||||
@@ -94,6 +94,7 @@ With a 4-hour cadence Pete will post at most 6 stories/day, so set `posting.dail
|
||||
| `-config <path>` | Path to config file (default: `config.toml`) |
|
||||
| `-seed` | Ingest all current feed items as seen without posting, then exit |
|
||||
| `-test` | Post one story to verify the full pipeline, then exit |
|
||||
| `-local` | Web/RSS-only mode: poll feeds and serve the web UI on `web.listen_addr`. Skips Matrix login and posting — useful for local testing |
|
||||
|
||||
## Docker
|
||||
|
||||
|
||||
@@ -24,6 +24,13 @@ func NormalizeImageURL(raw string) string {
|
||||
if q.Get("width") == "" {
|
||||
return raw
|
||||
}
|
||||
// The Guardian signs each (width, quality, fit, …) combination with `s=`.
|
||||
// Changing width without re-signing yields a 401 "invalid signature",
|
||||
// so leave signed URLs alone — the parser is now responsible for picking
|
||||
// the widest media:content variant up front.
|
||||
if q.Get("s") != "" {
|
||||
return raw
|
||||
}
|
||||
q.Set("width", "1200")
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
|
||||
@@ -6,10 +6,12 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
ext "github.com/mmcdole/gofeed/extensions"
|
||||
)
|
||||
|
||||
const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"
|
||||
@@ -88,21 +90,19 @@ func extractLede(desc string) string {
|
||||
// scraped from content:encoded / description (catches feeds like Ars that
|
||||
// embed the lead image in the article body but not as a media:* element).
|
||||
func extractImageURL(item *gofeed.Item) string {
|
||||
// Check media content (common in RSS 2.0 with media namespace)
|
||||
// Check media content (common in RSS 2.0 with media namespace).
|
||||
// When multiple media:content variants are advertised (Guardian RSS lists
|
||||
// 140/460/700/1200…) we want the widest signed URL we can find.
|
||||
if item.Extensions != nil {
|
||||
if media, ok := item.Extensions["media"]; ok {
|
||||
if contents, ok := media["content"]; ok {
|
||||
for _, c := range contents {
|
||||
if url := c.Attrs["url"]; url != "" {
|
||||
return url
|
||||
}
|
||||
if url := widestURL(contents); url != "" {
|
||||
return url
|
||||
}
|
||||
}
|
||||
if thumbs, ok := media["thumbnail"]; ok {
|
||||
for _, t := range thumbs {
|
||||
if url := t.Attrs["url"]; url != "" {
|
||||
return url
|
||||
}
|
||||
if url := widestURL(thumbs); url != "" {
|
||||
return url
|
||||
}
|
||||
}
|
||||
// media:group wraps nested media:content / media:thumbnail in some feeds
|
||||
@@ -143,6 +143,25 @@ func extractImageURL(item *gofeed.Item) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// widestURL picks the entry with the largest declared width attribute, falling
|
||||
// back to the first URL if none have a parsable width.
|
||||
func widestURL(entries []ext.Extension) string {
|
||||
best := ""
|
||||
bestW := -1
|
||||
for _, e := range entries {
|
||||
url := e.Attrs["url"]
|
||||
if url == "" {
|
||||
continue
|
||||
}
|
||||
w, _ := strconv.Atoi(e.Attrs["width"])
|
||||
if w > bestW {
|
||||
best = url
|
||||
bestW = w
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func firstImgSrc(htmlBody string) string {
|
||||
if htmlBody == "" {
|
||||
return ""
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,4 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64">
|
||||
<path d="M48 8C28 8 12 22 12 42c0 6 2 12 6 16 4-12 14-22 26-26-10 8-16 18-18 30 16 0 30-14 30-34 0-8-4-16-8-20z" fill="#6db73c" stroke="#3f6e1f" stroke-width="3" stroke-linejoin="round"/>
|
||||
<path d="M22 52c8-12 18-20 30-26" stroke="#3f6e1f" stroke-width="2" fill="none" stroke-linecap="round"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 365 B |
BIN
internal/web/static/img/pete.avif
Normal file
BIN
internal/web/static/img/pete.avif
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 7.8 KiB |
@@ -8,7 +8,7 @@
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Fredoka:wght@400;500;600;700&family=Nunito:wght@400;600;700&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="/static/css/output.css">
|
||||
<link rel="icon" href="/static/img/leaf.svg" type="image/svg+xml">
|
||||
<link rel="icon" href="/static/img/pete.avif" type="image/avif">
|
||||
<script>
|
||||
// Pick a palette phase from the browser clock and update it each minute.
|
||||
(function () {
|
||||
@@ -32,7 +32,8 @@
|
||||
<header class="mx-auto max-w-6xl px-4 pt-8 pb-4 sm:pt-12">
|
||||
<div class="flex items-center justify-between gap-4">
|
||||
<a href="/" class="flex items-center gap-3 group">
|
||||
<span class="inline-flex h-12 w-12 items-center justify-center rounded-2xl bg-[color:var(--card)] shadow-pete border-2 border-[color:var(--ink)]/10 text-2xl group-hover:-rotate-6 transition-transform">🌿</span>
|
||||
<img src="/static/img/pete.avif" alt="Pete" width="48" height="48"
|
||||
class="h-12 w-12 rounded-2xl bg-[color:var(--card)] shadow-pete border-2 border-[color:var(--ink)]/10 object-cover group-hover:-rotate-6 transition-transform">
|
||||
<span class="font-display text-3xl font-bold tracking-tight">{{.SiteTitle}}</span>
|
||||
<span class="hidden sm:inline rounded-full bg-[color:var(--accent)]/20 px-3 py-1 text-xs font-semibold uppercase tracking-wider text-[color:var(--ink)]/70" data-phase-label>day</span>
|
||||
</a>
|
||||
@@ -54,6 +55,8 @@
|
||||
</main>
|
||||
|
||||
<footer class="mx-auto max-w-6xl px-4 pb-10 text-center text-sm text-[color:var(--ink)]/50">
|
||||
<img src="/static/img/pete.avif" alt="" width="20" height="20"
|
||||
class="inline-block h-5 w-5 rounded-full object-cover align-[-4px] mr-1">
|
||||
served by <span class="font-semibold">Pete</span> · also a Matrix bot · <span data-clock>—</span>
|
||||
</footer>
|
||||
|
||||
|
||||
48
main.go
48
main.go
@@ -26,6 +26,7 @@ func main() {
|
||||
seed := flag.Bool("seed", false, "ingest current feed items as seen without posting, then exit")
|
||||
test := flag.Bool("test", false, "post one story to verify the full pipeline, then exit")
|
||||
testSource := flag.String("test-source", "", "source name to use for -test (default: first enabled)")
|
||||
local := flag.Bool("local", false, "web/RSS-only mode: poll feeds and serve the web UI, no Matrix login or posting")
|
||||
flag.Parse()
|
||||
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||
@@ -62,6 +63,12 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
// Local mode: poll feeds + serve web UI only. No Matrix, no posting queue.
|
||||
if *local {
|
||||
runLocal(cfg)
|
||||
return
|
||||
}
|
||||
|
||||
// Create Matrix client
|
||||
mx, err := matrix.New(cfg.Matrix)
|
||||
if err != nil {
|
||||
@@ -209,6 +216,47 @@ func main() {
|
||||
slog.Info("pete stopped")
|
||||
}
|
||||
|
||||
func runLocal(cfg *config.Config) {
|
||||
cfg.Web.Enabled = true
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
processItem := func(_ context.Context, item *ingestion.FeedItem) {
|
||||
if item.DirectRoute == "" {
|
||||
slog.Info("local: item has no direct_route, discarding", "guid", item.GUID, "source", item.Source)
|
||||
storage.MarkClassified(item.GUID, "_discarded", "[]")
|
||||
return
|
||||
}
|
||||
storage.MarkClassified(item.GUID, item.DirectRoute, "[]")
|
||||
slog.Info("local: story classified",
|
||||
"guid", item.GUID, "source", item.Source, "channel", item.DirectRoute, "headline", item.Headline)
|
||||
}
|
||||
|
||||
poller := ingestion.NewPoller(cfg.Sources, processItem)
|
||||
poller.Start(ctx)
|
||||
slog.Info("local: pollers started")
|
||||
|
||||
ws, err := web.New(cfg.Web)
|
||||
if err != nil {
|
||||
slog.Error("local: web server init failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
go ws.Start(ctx)
|
||||
slog.Info("local: web UI listening", "addr", cfg.Web.ListenAddr)
|
||||
|
||||
storage.RunMaintenance()
|
||||
|
||||
sig := <-sigCh
|
||||
slog.Info("local: shutdown signal received", "signal", sig)
|
||||
cancel()
|
||||
poller.Wait()
|
||||
slog.Info("pete stopped")
|
||||
}
|
||||
|
||||
func runSeed(cfg *config.Config) {
|
||||
total := 0
|
||||
for _, src := range cfg.Sources {
|
||||
|
||||
Reference in New Issue
Block a user