From 298c7bb8f18b562e4fb2406b53dd306d2d975435 Mon Sep 17 00:00:00 2001 From: prosolis <5590409+prosolis@users.noreply.github.com> Date: Sun, 8 Mar 2026 20:46:18 -0700 Subject: [PATCH] Expand sentiment to 10 categories, add DB maintenance job, update README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Expand LLM sentiment classification from 3 to 10 categories: positive, negative, neutral, excited, sarcastic, frustrated, curious, grateful, humorous, supportive — with emoji reactions for each - Add daily DB maintenance job at 03:00 UTC to purge stale caches, expired rate limits, old logs, and run SQLite optimize - Add migrate.sql for upgrading existing databases - Update README: add !sentiment command, maintenance job, rate limit config, in-memory message buffer docs, SDK migration history, remove references to previous private repo Co-Authored-By: Claude Opus 4.6 --- README.md | 13 +++-- internal/db/db.go | 76 ++++++++++++++++++++++++ internal/plugin/howami.go | 10 ++-- internal/plugin/llm_passive.go | 102 ++++++++++++++++++++++----------- main.go | 6 ++ migrate.sql | 15 +++++ 6 files changed, 182 insertions(+), 40 deletions(-) create mode 100644 migrate.sql diff --git a/README.md b/README.md index 755dc63..075eecb 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Matrix community bot with E2EE, 35+ plugins, passive tracking, scheduled posts, and optional LLM features. -Written in Go using [mautrix-go](https://github.com/mautrix/go) for encryption and [modernc.org/sqlite](https://modernc.org/sqlite) for storage. Successor to the TypeScript "Freebee" bot. +Written in Go using [mautrix-go](https://github.com/mautrix/go) for encryption and [modernc.org/sqlite](https://modernc.org/sqlite) for storage. --- @@ -128,7 +128,9 @@ Everything is configured through environment variables or a `.env` file. | Variable | Default | Description | |----------|---------|-------------| +| `RATELIMIT_WEATHER` | `5` | Daily weather lookups per user | | `RATELIMIT_TRANSLATE` | `20` | Daily translation limit per user | +| `RATELIMIT_CONCERTS` | `10` | Daily concert searches per user | --- @@ -286,6 +288,7 @@ Rep is earned when someone thanks you. The bot detects this automatically. | `!howami [@user]` | Roast profile | | `!vibe` | Room energy check | | `!tldr` | Summarize recent chat | +| `!sentiment [@user]` | Sentiment breakdown (10 categories) | | `!potty [@user]` | Profanity count | | `!pottyboard` | Profanity leaderboard | | `!insults [@user]` | Insult stats | @@ -316,7 +319,8 @@ All of these run in the background without any commands: - **Room milestones** - announces at 1k, 5k, 10k, 25k, 50k, 100k, 250k, 500k, 1M messages - **URL previews** - OG tag extraction (feature-flagged, off by default) - **Reactions** - logs all reactions for `!emojiboard` -- **LLM classification** - sentiment, profanity, insults, WOTD usage (needs Ollama) +- **LLM classification** - sentiment (10 categories), profanity, insults, WOTD usage (needs Ollama) +- **Message buffer** - last 50 messages per room held in memory for `!vibe` and `!tldr`. Not persisted to disk; resets on restart. - **Quotes** - star-react any message to save it --- @@ -336,6 +340,7 @@ Uses [robfig/cron](https://github.com/robfig/cron). All times UTC. | 11:00 | Movies | Movie releases today | | 12:00 Sun | Concerts | Weekly concert digest | | Every 30s | Reminders | Fires pending reminders | +| 03:00 | Maintenance | Purges stale caches, old rate limits, expired logs; runs SQLite optimize | --- @@ -455,9 +460,9 @@ gogobee/ ### Why Go? -**E2EE** - The TS version used `matrix-js-sdk` with `fake-indexeddb` for an in-memory crypto store. Every restart wiped device keys and required re-verification in all encrypted rooms. mautrix-go stores crypto state in SQLite. Verify once, it sticks. +**E2EE** - This project went through three SDK iterations: `matrix-bot-sdk` (no E2EE support), `matrix-js-sdk` (E2EE via `fake-indexeddb` with an in-memory crypto store that wiped device keys on every restart), and finally `mautrix-go` which stores crypto state in SQLite with cross-signing bootstrap. Verify once, it sticks. -**Deployment** - Pure Go, no CGo. `go build -tags goolm` gives you a static binary with zero system dependencies. The TS version needed Node.js, npm, a C compiler for better-sqlite3, and libolm. +**Deployment** - Pure Go, no CGo. `go build -tags goolm` gives you a static binary with zero system dependencies. The TypeScript version needed Node.js, npm, a C compiler for better-sqlite3, and libolm. **Scheduler** - Replaced a hand-rolled 60s tick loop with robfig/cron. Standard cron expressions, less code, fewer bugs. diff --git a/internal/db/db.go b/internal/db/db.go index a450652..22f8651 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sync" + "time" _ "modernc.org/sqlite" ) @@ -110,6 +111,74 @@ func CacheSet(key, data string) { } } +// RunMaintenance purges stale data from cache tables, old rate limits, +// expired logs, and runs SQLite optimization. Intended to run daily. +func RunMaintenance() { + d := Get() + now := time.Now().UTC() + today := now.Format("2006-01-02") + cutoff7d := now.AddDate(0, 0, -7).Unix() + cutoff30d := now.AddDate(0, 0, -30).Unix() + date30d := now.AddDate(0, 0, -30).Format("2006-01-02") + date90d := now.AddDate(0, 0, -90).Format("2006-01-02") + + queries := []struct { + label string + sql string + args []interface{} + }{ + // Cache tables — purge entries older than their effective TTL + {"api_cache", `DELETE FROM api_cache WHERE cached_at < ?`, []interface{}{cutoff7d}}, + {"releases_cache", `DELETE FROM releases_cache WHERE cached_at < ?`, []interface{}{cutoff7d}}, + {"hltb_cache", `DELETE FROM hltb_cache WHERE cached_at < ?`, []interface{}{cutoff7d}}, + {"stocks_cache", `DELETE FROM stocks_cache WHERE cached_at < ?`, []interface{}{cutoff7d}}, + {"concerts_cache", `DELETE FROM concerts_cache WHERE cached_at < ?`, []interface{}{cutoff7d}}, + {"anime_cache", `DELETE FROM anime_cache WHERE cached_at < ?`, []interface{}{cutoff30d}}, + {"movie_cache", `DELETE FROM movie_cache WHERE cached_at < ?`, []interface{}{cutoff30d}}, + {"retro_cache", `DELETE FROM retro_cache WHERE cached_at < ?`, []interface{}{cutoff30d}}, + {"urban_cache", `DELETE FROM urban_cache WHERE cached_at < ?`, []interface{}{cutoff30d}}, + {"url_cache", `DELETE FROM url_cache WHERE cached_at < ?`, []interface{}{cutoff30d}}, + + // Rate limits — purge entries older than today + {"rate_limits", `DELETE FROM rate_limits WHERE date < ?`, []interface{}{today}}, + + // Daily prefetch log — keep 30 days + {"daily_prefetch", `DELETE FROM daily_prefetch WHERE date < ?`, []interface{}{date30d}}, + + // Holiday and WOTD logs — keep 90 days + {"holidays_log", `DELETE FROM holidays_log WHERE date < ?`, []interface{}{date90d}}, + {"wotd_log", `DELETE FROM wotd_log WHERE date < ?`, []interface{}{date90d}}, + {"wotd_usage", `DELETE FROM wotd_usage WHERE date < ?`, []interface{}{date90d}}, + + // LLM classifications — keep 30 days + {"llm_classifications", `DELETE FROM llm_classifications WHERE timestamp < ?`, []interface{}{cutoff30d}}, + + // Daily activity older than 1 year + {"daily_activity", `DELETE FROM daily_activity WHERE date < ?`, []interface{}{now.AddDate(-1, 0, 0).Format("2006-01-02")}}, + } + + totalDeleted := int64(0) + for _, q := range queries { + res, err := d.Exec(q.sql, q.args...) + if err != nil { + slog.Error("maintenance: "+q.label, "err", err) + continue + } + n, _ := res.RowsAffected() + if n > 0 { + slog.Info("maintenance: purged", "table", q.label, "rows", n) + totalDeleted += n + } + } + + // SQLite optimization + if _, err := d.Exec(`PRAGMA optimize`); err != nil { + slog.Error("maintenance: pragma optimize", "err", err) + } + + slog.Info("maintenance: complete", "total_purged", totalDeleted) +} + const schema = ` -- Users & XP CREATE TABLE IF NOT EXISTS users ( @@ -508,6 +577,13 @@ CREATE TABLE IF NOT EXISTS sentiment_stats ( positive INTEGER DEFAULT 0, negative INTEGER DEFAULT 0, neutral INTEGER DEFAULT 0, + excited INTEGER DEFAULT 0, + sarcastic INTEGER DEFAULT 0, + frustrated INTEGER DEFAULT 0, + curious INTEGER DEFAULT 0, + grateful INTEGER DEFAULT 0, + humorous INTEGER DEFAULT 0, + supportive INTEGER DEFAULT 0, total_score REAL DEFAULT 0 ); diff --git a/internal/plugin/howami.go b/internal/plugin/howami.go index 0c734e6..c36a310 100644 --- a/internal/plugin/howami.go +++ b/internal/plugin/howami.go @@ -120,11 +120,13 @@ func (p *HowAmIPlugin) gatherProfile(userID id.UserID) string { sb.WriteString(fmt.Sprintf("Achievements unlocked: %d\n", achievementCount)) // Sentiment stats - var positive, negative, neutral int + var positive, negative, neutral, excited, sarcastic, frustrated, curious, grateful, humorous, supportive int if err := d.QueryRow( - `SELECT positive, negative, neutral FROM sentiment_stats WHERE user_id = ?`, uid, - ).Scan(&positive, &negative, &neutral); err == nil { - sb.WriteString(fmt.Sprintf("Sentiment: %d positive, %d negative, %d neutral\n", positive, negative, neutral)) + `SELECT positive, negative, neutral, excited, sarcastic, frustrated, curious, grateful, humorous, supportive + FROM sentiment_stats WHERE user_id = ?`, uid, + ).Scan(&positive, &negative, &neutral, &excited, &sarcastic, &frustrated, &curious, &grateful, &humorous, &supportive); err == nil { + sb.WriteString(fmt.Sprintf("Sentiment: %d positive, %d excited, %d supportive, %d grateful, %d humorous, %d curious, %d neutral, %d sarcastic, %d frustrated, %d negative\n", + positive, excited, supportive, grateful, humorous, curious, neutral, sarcastic, frustrated, negative)) } // Profanity count diff --git a/internal/plugin/llm_passive.go b/internal/plugin/llm_passive.go index 7d813d0..a20e1e3 100644 --- a/internal/plugin/llm_passive.go +++ b/internal/plugin/llm_passive.go @@ -287,26 +287,22 @@ func (p *LLMPassivePlugin) classifyAndProcess(item queueItem) error { } // Aggregate sentiment stats - switch result.Sentiment { - case "positive": - _, _ = d.Exec( - `INSERT INTO sentiment_stats (user_id, positive, total_score) VALUES (?, 1, ?) - ON CONFLICT(user_id) DO UPDATE SET positive = positive + 1, total_score = total_score + ?`, - string(item.UserID), result.SentimentScore, result.SentimentScore, - ) - case "negative": - _, _ = d.Exec( - `INSERT INTO sentiment_stats (user_id, negative, total_score) VALUES (?, 1, ?) - ON CONFLICT(user_id) DO UPDATE SET negative = negative + 1, total_score = total_score + ?`, - string(item.UserID), result.SentimentScore, result.SentimentScore, - ) - default: - _, _ = d.Exec( - `INSERT INTO sentiment_stats (user_id, neutral, total_score) VALUES (?, 1, ?) - ON CONFLICT(user_id) DO UPDATE SET neutral = neutral + 1, total_score = total_score + ?`, - string(item.UserID), result.SentimentScore, result.SentimentScore, - ) + sentimentCol := "neutral" + validSentiments := map[string]bool{ + "positive": true, "negative": true, "neutral": true, + "excited": true, "sarcastic": true, "frustrated": true, + "curious": true, "grateful": true, "humorous": true, "supportive": true, } + if validSentiments[result.Sentiment] { + sentimentCol = result.Sentiment + } + _, _ = d.Exec( + fmt.Sprintf( + `INSERT INTO sentiment_stats (user_id, %s, total_score) VALUES (?, 1, ?) + ON CONFLICT(user_id) DO UPDATE SET %s = %s + 1, total_score = total_score + ?`, + sentimentCol, sentimentCol, sentimentCol), + string(item.UserID), result.SentimentScore, result.SentimentScore, + ) // Track profanity with severity if result.Profanity { @@ -360,12 +356,24 @@ func (p *LLMPassivePlugin) classifyAndProcess(item queueItem) error { ) } - // React with emojis based on classification - if result.Sentiment == "positive" && result.SentimentScore > 0.7 { - _ = p.SendReact(item.RoomID, item.EventID, "\U0001f44d") // thumbsup + // React with emojis based on sentiment + sentimentEmojis := map[string]string{ + "positive": "\U0001f44d", // 👍 + "negative": "\U0001f44e", // 👎 + "excited": "\U0001f525", // 🔥 + "sarcastic": "\U0001f928", // 🤨 + "frustrated": "\U0001f62e\u200d\U0001f4a8", // 😮‍💨 + "curious": "\U0001f9d0", // 🧐 + "grateful": "\U0001f49c", // 💜 + "humorous": "\U0001f602", // 😂 + "supportive": "\U0001f917", // 🤗 } - if result.Sentiment == "negative" && result.SentimentScore < -0.7 { - _ = p.SendReact(item.RoomID, item.EventID, "\U0001f44e") // thumbsdown + if emoji, ok := sentimentEmojis[result.Sentiment]; ok && result.Sentiment != "neutral" { + // Only react to strong sentiments (|score| > 0.5) + score := result.SentimentScore + if score > 0.5 || score < -0.5 { + _ = p.SendReact(item.RoomID, item.EventID, emoji) + } } if result.Profanity { switch result.ProfanitySeverity { @@ -413,7 +421,7 @@ func (p *LLMPassivePlugin) callOllama(messageText string) (*classificationResult JSON schema: { - "sentiment": "positive" | "negative" | "neutral", + "sentiment": "positive" | "negative" | "neutral" | "excited" | "sarcastic" | "frustrated" | "curious" | "grateful" | "humorous" | "supportive", "sentiment_score": number between -1.0 and 1.0, "topics": ["topic1", "topic2"], "profanity": true | false, @@ -644,18 +652,22 @@ func (p *LLMPassivePlugin) handleSentiment(ctx MessageContext) error { } d := db.Get() - var positive, negative, neutral int + var positive, negative, neutral, excited, sarcastic, frustrated, curious, grateful, humorous, supportive int var totalScore float64 err := d.QueryRow( - `SELECT COALESCE(positive, 0), COALESCE(negative, 0), COALESCE(neutral, 0), COALESCE(total_score, 0) + `SELECT COALESCE(positive, 0), COALESCE(negative, 0), COALESCE(neutral, 0), + COALESCE(excited, 0), COALESCE(sarcastic, 0), COALESCE(frustrated, 0), + COALESCE(curious, 0), COALESCE(grateful, 0), COALESCE(humorous, 0), + COALESCE(supportive, 0), COALESCE(total_score, 0) FROM sentiment_stats WHERE user_id = ?`, string(target), - ).Scan(&positive, &negative, &neutral, &totalScore) + ).Scan(&positive, &negative, &neutral, &excited, &sarcastic, &frustrated, + &curious, &grateful, &humorous, &supportive, &totalScore) if err != nil { return p.SendReply(ctx.RoomID, ctx.EventID, fmt.Sprintf("No sentiment data for %s yet.", string(target))) } - total := positive + negative + neutral + total := positive + negative + neutral + excited + sarcastic + frustrated + curious + grateful + humorous + supportive avgScore := 0.0 if total > 0 { avgScore = totalScore / float64(total) @@ -672,7 +684,33 @@ func (p *LLMPassivePlugin) handleSentiment(ctx MessageContext) error { mood = "leaning negative" } - msg := fmt.Sprintf("Sentiment for %s:\n😊 Positive: %s 😐 Neutral: %s 😠 Negative: %s\nAverage mood: %.2f (%s)", - string(target), formatNumber(positive), formatNumber(neutral), formatNumber(negative), avgScore, mood) - return p.SendReply(ctx.RoomID, ctx.EventID, msg) + // Build sentiment breakdown, only showing non-zero counts + type sentEntry struct { + emoji string + label string + count int + } + entries := []sentEntry{ + {"👍", "Positive", positive}, + {"🔥", "Excited", excited}, + {"🤗", "Supportive", supportive}, + {"💜", "Grateful", grateful}, + {"😂", "Humorous", humorous}, + {"🧐", "Curious", curious}, + {"😐", "Neutral", neutral}, + {"🤨", "Sarcastic", sarcastic}, + {"😮\u200d💨", "Frustrated", frustrated}, + {"👎", "Negative", negative}, + } + + var sb strings.Builder + sb.WriteString(fmt.Sprintf("Sentiment for %s:\n", string(target))) + for _, e := range entries { + if e.count > 0 { + sb.WriteString(fmt.Sprintf(" %s %s: %s\n", e.emoji, e.label, formatNumber(e.count))) + } + } + sb.WriteString(fmt.Sprintf("Average mood: %.2f (%s)", avgScore, mood)) + + return p.SendReply(ctx.RoomID, ctx.EventID, sb.String()) } diff --git a/main.go b/main.go index 853de53..4f01cf3 100644 --- a/main.go +++ b/main.go @@ -277,6 +277,12 @@ func setupScheduledJobs( c.AddFunc("@every 30s", func() { plugin.FirePendingReminders(client) }) + + // Database maintenance at 03:00 daily + c.AddFunc("0 3 * * *", func() { + slog.Info("scheduler: running database maintenance") + db.RunMaintenance() + }) } func getRooms() []id.RoomID { diff --git a/migrate.sql b/migrate.sql new file mode 100644 index 0000000..a4d41b5 --- /dev/null +++ b/migrate.sql @@ -0,0 +1,15 @@ +-- Add expanded sentiment columns to sentiment_stats +ALTER TABLE sentiment_stats ADD COLUMN excited INTEGER DEFAULT 0; +ALTER TABLE sentiment_stats ADD COLUMN sarcastic INTEGER DEFAULT 0; +ALTER TABLE sentiment_stats ADD COLUMN frustrated INTEGER DEFAULT 0; +ALTER TABLE sentiment_stats ADD COLUMN curious INTEGER DEFAULT 0; +ALTER TABLE sentiment_stats ADD COLUMN grateful INTEGER DEFAULT 0; +ALTER TABLE sentiment_stats ADD COLUMN humorous INTEGER DEFAULT 0; +ALTER TABLE sentiment_stats ADD COLUMN supportive INTEGER DEFAULT 0; + +-- Create generic API cache table +CREATE TABLE IF NOT EXISTS api_cache ( + cache_key TEXT PRIMARY KEY, + data TEXT NOT NULL, + cached_at INTEGER DEFAULT (unixepoch()) +);