35850eaf73
- push digest queries now exclude _duplicate channel like every other visibility query (bookmarks list/count and NewClassifiedSince) - advance push watermark to newest scanned story, not pass-start now, so stories arriving during a long send pass aren't re-counted next pass - replace hand-rolled escapeHTMLText with stdlib html.EscapeString - drop em-dashes from user-facing copy; bump PWA CACHE_VERSION so clients pick up the changed shell assets
293 lines
8.5 KiB
Go
293 lines
8.5 KiB
Go
package web
|
|
|
|
import (
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"html"
|
|
"log/slog"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"pete/internal/storage"
|
|
)
|
|
|
|
// feedItemLimit caps how many stories each outbound feed carries.
|
|
const feedItemLimit = 50
|
|
|
|
// feedCacheControl is a short cache window; feeds refresh a few times an hour.
|
|
const feedCacheControl = "public, max-age=300"
|
|
|
|
const (
|
|
rssContentNS = "http://purl.org/rss/1.0/modules/content/"
|
|
rssAtomNS = "http://www.w3.org/2005/Atom"
|
|
jsonFeedVer = "https://jsonfeed.org/version/1.1"
|
|
)
|
|
|
|
// --- RSS 2.0 ---
|
|
|
|
type rssFeedXML struct {
|
|
XMLName xml.Name `xml:"rss"`
|
|
Version string `xml:"version,attr"`
|
|
ContentNS string `xml:"xmlns:content,attr"`
|
|
AtomNS string `xml:"xmlns:atom,attr"`
|
|
Channel rssChannel `xml:"channel"`
|
|
}
|
|
|
|
type rssChannel struct {
|
|
Title string `xml:"title"`
|
|
Link string `xml:"link"`
|
|
Description string `xml:"description"`
|
|
Language string `xml:"language,omitempty"`
|
|
LastBuildDate string `xml:"lastBuildDate,omitempty"`
|
|
Generator string `xml:"generator,omitempty"`
|
|
AtomLink rssAtomLink `xml:"atom:link"`
|
|
Items []rssItem `xml:"item"`
|
|
}
|
|
|
|
type rssAtomLink struct {
|
|
Href string `xml:"href,attr"`
|
|
Rel string `xml:"rel,attr"`
|
|
Type string `xml:"type,attr"`
|
|
}
|
|
|
|
type rssItem struct {
|
|
Title string `xml:"title"`
|
|
Link string `xml:"link"`
|
|
GUID rssGUID `xml:"guid"`
|
|
PubDate string `xml:"pubDate,omitempty"`
|
|
Category string `xml:"category,omitempty"`
|
|
Description string `xml:"description"`
|
|
Content *rssContent `xml:"content:encoded,omitempty"`
|
|
}
|
|
|
|
type rssGUID struct {
|
|
IsPermaLink string `xml:"isPermaLink,attr"`
|
|
Value string `xml:",chardata"`
|
|
}
|
|
|
|
type rssContent struct {
|
|
Value string `xml:",cdata"`
|
|
}
|
|
|
|
// --- JSON Feed 1.1 ---
|
|
|
|
type jsonFeed struct {
|
|
Version string `json:"version"`
|
|
Title string `json:"title"`
|
|
HomePageURL string `json:"home_page_url"`
|
|
FeedURL string `json:"feed_url"`
|
|
Description string `json:"description,omitempty"`
|
|
Items []jsonFeedItem `json:"items"`
|
|
}
|
|
|
|
type jsonFeedItem struct {
|
|
ID string `json:"id"`
|
|
URL string `json:"url"`
|
|
Title string `json:"title"`
|
|
ContentText string `json:"content_text,omitempty"`
|
|
Summary string `json:"summary,omitempty"`
|
|
Image string `json:"image,omitempty"`
|
|
DatePublished string `json:"date_published,omitempty"`
|
|
Authors []jsonFeedAuthor `json:"authors,omitempty"`
|
|
Tags []string `json:"tags,omitempty"`
|
|
}
|
|
|
|
type jsonFeedAuthor struct {
|
|
Name string `json:"name"`
|
|
}
|
|
|
|
// handleFeedXML serves an RSS 2.0 feed for the given channel ("" = all channels).
|
|
func (s *Server) handleFeedXML(w http.ResponseWriter, r *http.Request, channel string) {
|
|
stories, err := storage.ListForFeed(channel, feedItemLimit)
|
|
if err != nil {
|
|
slog.Error("web: feed query failed", "channel", channel, "err", err)
|
|
http.Error(w, "internal error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
base := s.publicBase(r)
|
|
title, homeURL, desc := s.feedMeta(channel, base)
|
|
selfURL := base + feedPath(channel, "xml")
|
|
|
|
feed := rssFeedXML{
|
|
Version: "2.0",
|
|
ContentNS: rssContentNS,
|
|
AtomNS: rssAtomNS,
|
|
Channel: rssChannel{
|
|
Title: title,
|
|
Link: homeURL,
|
|
Description: desc,
|
|
Language: "en",
|
|
Generator: "Pete",
|
|
AtomLink: rssAtomLink{Href: selfURL, Rel: "self", Type: "application/rss+xml"},
|
|
Items: make([]rssItem, 0, len(stories)),
|
|
},
|
|
}
|
|
if len(stories) > 0 {
|
|
feed.Channel.LastBuildDate = feedTime(stories[0]).UTC().Format(time.RFC1123Z)
|
|
}
|
|
for _, st := range stories {
|
|
link := storyLink(st)
|
|
item := rssItem{
|
|
Title: st.Headline,
|
|
Link: link,
|
|
GUID: rssGUID{IsPermaLink: "false", Value: st.GUID},
|
|
PubDate: feedTime(st).UTC().Format(time.RFC1123Z),
|
|
Category: st.Channel,
|
|
Description: st.Lede,
|
|
}
|
|
if html := contentToHTML(st.Content); html != "" {
|
|
item.Content = &rssContent{Value: html}
|
|
}
|
|
feed.Channel.Items = append(feed.Channel.Items, item)
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/rss+xml; charset=utf-8")
|
|
w.Header().Set("Cache-Control", feedCacheControl)
|
|
if _, err := w.Write([]byte(xml.Header)); err != nil {
|
|
return
|
|
}
|
|
enc := xml.NewEncoder(w)
|
|
enc.Indent("", " ")
|
|
if err := enc.Encode(feed); err != nil {
|
|
slog.Error("web: rss encode failed", "err", err)
|
|
}
|
|
}
|
|
|
|
// handleFeedJSON serves a JSON Feed 1.1 for the given channel ("" = all).
|
|
func (s *Server) handleFeedJSON(w http.ResponseWriter, r *http.Request, channel string) {
|
|
stories, err := storage.ListForFeed(channel, feedItemLimit)
|
|
if err != nil {
|
|
slog.Error("web: feed query failed", "channel", channel, "err", err)
|
|
http.Error(w, "internal error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
base := s.publicBase(r)
|
|
title, homeURL, desc := s.feedMeta(channel, base)
|
|
|
|
feed := jsonFeed{
|
|
Version: jsonFeedVer,
|
|
Title: title,
|
|
HomePageURL: homeURL,
|
|
FeedURL: base + feedPath(channel, "json"),
|
|
Description: desc,
|
|
Items: make([]jsonFeedItem, 0, len(stories)),
|
|
}
|
|
for _, st := range stories {
|
|
item := jsonFeedItem{
|
|
ID: st.GUID,
|
|
URL: storyLink(st),
|
|
Title: st.Headline,
|
|
ContentText: strings.TrimSpace(st.Content),
|
|
Summary: st.Lede,
|
|
Image: st.ImageURL,
|
|
DatePublished: feedTime(st).UTC().Format(time.RFC3339),
|
|
Tags: []string{st.Channel},
|
|
}
|
|
if item.ContentText == "" {
|
|
item.ContentText = st.Lede
|
|
}
|
|
if st.Source != "" {
|
|
item.Authors = []jsonFeedAuthor{{Name: st.Source}}
|
|
}
|
|
feed.Items = append(feed.Items, item)
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/feed+json; charset=utf-8")
|
|
w.Header().Set("Cache-Control", feedCacheControl)
|
|
enc := json.NewEncoder(w)
|
|
enc.SetIndent("", " ")
|
|
if err := enc.Encode(feed); err != nil {
|
|
slog.Error("web: json feed encode failed", "err", err)
|
|
}
|
|
}
|
|
|
|
// feedMeta returns the title, home-page URL, and description for a feed. An
|
|
// empty channel is the site-wide feed; a slug scopes to that channel.
|
|
func (s *Server) feedMeta(channel, base string) (title, homeURL, desc string) {
|
|
site := s.cfg.SiteTitle
|
|
if site == "" {
|
|
site = "Pete"
|
|
}
|
|
if channel == "" {
|
|
return site, base + "/", "The latest across every channel."
|
|
}
|
|
ch := channelBySlug(channel)
|
|
return site + " — " + ch.Title, base + "/" + channel, ch.Blurb
|
|
}
|
|
|
|
// feedPath is the site-relative path of a feed, e.g. "/feed.xml" or
|
|
// "/gaming/feed.json".
|
|
func feedPath(channel, ext string) string {
|
|
if channel == "" {
|
|
return "/feed." + ext
|
|
}
|
|
return "/" + channel + "/feed." + ext
|
|
}
|
|
|
|
// storyLink is the canonical outbound link for a story: its canonical URL when
|
|
// known, otherwise the raw article URL.
|
|
func storyLink(s storage.Story) string {
|
|
if s.URLCanonical != "" {
|
|
return s.URLCanonical
|
|
}
|
|
return s.ArticleURL
|
|
}
|
|
|
|
// feedTime picks the best timestamp for a story: its published date when
|
|
// present, otherwise when Pete first saw it.
|
|
func feedTime(s storage.Story) time.Time {
|
|
if s.PublishedAt > 0 {
|
|
return time.Unix(s.PublishedAt, 0)
|
|
}
|
|
return time.Unix(s.SeenAt, 0)
|
|
}
|
|
|
|
// channelBySlug looks up a channel by slug, falling back to a bare title-cased
|
|
// entry so an unknown slug never panics.
|
|
func channelBySlug(slug string) Channel {
|
|
for _, ch := range channels {
|
|
if ch.Slug == slug {
|
|
return ch
|
|
}
|
|
}
|
|
return Channel{Slug: slug, Title: slug, Theme: slug}
|
|
}
|
|
|
|
// publicBase returns the site's public origin (scheme://host, no trailing
|
|
// slash). It prefers the configured base_url and falls back to the request's
|
|
// host so feeds still carry absolute URLs when base_url is unset.
|
|
func (s *Server) publicBase(r *http.Request) string {
|
|
if b := strings.TrimRight(s.cfg.BaseURL, "/"); b != "" {
|
|
return b
|
|
}
|
|
scheme := "http"
|
|
if r.TLS != nil || strings.EqualFold(r.Header.Get("X-Forwarded-Proto"), "https") {
|
|
scheme = "https"
|
|
}
|
|
return scheme + "://" + r.Host
|
|
}
|
|
|
|
// contentToHTML turns Pete's stored plain-text article body (paragraphs split by
|
|
// blank lines, soft line breaks inside) into simple, escaped HTML for RSS
|
|
// content:encoded. Returns "" when there is no body.
|
|
func contentToHTML(text string) string {
|
|
text = strings.TrimSpace(text)
|
|
if text == "" {
|
|
return ""
|
|
}
|
|
var b strings.Builder
|
|
for _, para := range strings.Split(text, "\n\n") {
|
|
para = strings.TrimSpace(para)
|
|
if para == "" {
|
|
continue
|
|
}
|
|
b.WriteString("<p>")
|
|
b.WriteString(strings.ReplaceAll(html.EscapeString(para), "\n", "<br>"))
|
|
b.WriteString("</p>")
|
|
}
|
|
return b.String()
|
|
}
|