Add from detected spam (#55)

* add ability to add detected spam to samples

this is needed if spam detected by heuristic method, like number of emojis or links but failed to be detected by classifier. in this case it makes sense to add it to dynamic spam samples directly from webui

* render error on spam detection page

* don't convert to int64 separately

app/main.go

@@ -363,15 +363,15 @@ func activateServer(ctx context.Context, opts options, sf *bot.SpamFilter, loc *
 	}
 
 	srv := webapi.Server{Config: webapi.Config{
-		ListenAddr:         opts.Server.ListenAddr,
-		Detector:           sf.Detector,
-		SpamFilter:         sf,
-		Locator:            loc,
-		DetectedSpamReader: detectedSpamStore,
-		AuthPasswd:         authPassswd,
-		Version:            revision,
-		Dbg:                opts.Dbg,
-		Settings:           settings,
+		ListenAddr:   opts.Server.ListenAddr,
+		Detector:     sf.Detector,
+		SpamFilter:   sf,
+		Locator:      loc,
+		DetectedSpam: detectedSpamStore,
+		AuthPasswd:   authPassswd,
+		Version:      revision,
+		Dbg:          opts.Dbg,
+		Settings:     settings,
 	}}
 
 	go func() {

app/storage/detected_spam.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	"github.com/jmoiron/sqlx"
@@ -20,28 +21,37 @@ type DetectedSpam struct {
 
 // DetectedSpamInfo represents information about a detected spam entry.
 type DetectedSpamInfo struct {
+	ID         int64                `db:"id"`
 	Text       string               `db:"text"`
 	UserID     int64                `db:"user_id"`
 	UserName   string               `db:"user_name"`
 	Timestamp  time.Time            `db:"timestamp"`
+	Added      bool                 `db:"added"`  // added to samples
 	ChecksJSON string               `db:"checks"` // Store as JSON
 	Checks     []spamcheck.Response `db:"-"`      // Don't store in DB
 }
 
 // NewDetectedSpam creates a new DetectedSpam storage
 func NewDetectedSpam(db *sqlx.DB) (*DetectedSpam, error) {
 	_, err := db.Exec(`CREATE TABLE IF NOT EXISTS detected_spam (
-    	id INTEGER PRIMARY KEY AUTOINCREMENT,
-        text TEXT,
-        user_id INTEGER,
-        user_name TEXT,
-        timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
-        checks TEXT
-    )`)
+		id INTEGER PRIMARY KEY AUTOINCREMENT,
+		text TEXT,
+		user_id INTEGER,
+		user_name TEXT,
+		timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+		added BOOLEAN DEFAULT 0,
+		checks TEXT
+	)`)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create detected_spam table: %w", err)
 	}
 
+	_, err = db.Exec(`ALTER TABLE detected_spam ADD COLUMN added BOOLEAN DEFAULT 0`)
+	if err != nil {
+		if !strings.Contains(err.Error(), "duplicate column name") {
+			return nil, fmt.Errorf("failed to alter detected_spam table: %w", err)
+		}
+	}
 	// add index on timestamp
 	if _, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_detected_spam_timestamp ON detected_spam(timestamp)`); err != nil {
 		return nil, fmt.Errorf("failed to create index on timestamp: %w", err)
@@ -66,10 +76,19 @@ func (ds *DetectedSpam) Write(entry DetectedSpamInfo, checks []spamcheck.Respons
 	return nil
 }
 
+// SetAddedToSamplesFlag sets the added flag to true for the detected spam entry with the given id
+func (ds *DetectedSpam) SetAddedToSamplesFlag(id int64) error {
+	query := `UPDATE detected_spam SET added = 1 WHERE id = ?`
+	if _, err := ds.db.Exec(query, id); err != nil {
+		return fmt.Errorf("failed to update added to samples flag: %w", err)
+	}
+	return nil
+}
+
 // Read returns all detected spam entries
 func (ds *DetectedSpam) Read() ([]DetectedSpamInfo, error) {
 	var entries []DetectedSpamInfo
-	err := ds.db.Select(&entries, "SELECT text, user_id, user_name, timestamp, checks FROM detected_spam ORDER BY timestamp DESC LIMIT ?", maxDetectedSpamEntries)
+	err := ds.db.Select(&entries, "SELECT * FROM detected_spam ORDER BY timestamp DESC LIMIT ?", maxDetectedSpamEntries)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get detected spam entries: %w", err)
 	}

app/storage/detected_spam_test.go

@@ -58,6 +58,44 @@ func TestDetectedSpam_Write(t *testing.T) {
 	assert.Equal(t, 1, count)
 }
 
+func TestSetAddedToSamplesFlag(t *testing.T) {
+	db, err := sqlx.Open("sqlite", ":memory:")
+	require.NoError(t, err)
+	defer db.Close()
+
+	ds, err := NewDetectedSpam(db)
+	require.NoError(t, err)
+
+	spamEntry := DetectedSpamInfo{
+		Text:      "spam message",
+		UserID:    1,
+		UserName:  "Spammer",
+		Timestamp: time.Now(),
+	}
+
+	checks := []spamcheck.Response{
+		{
+			Name:    "Check1",
+			Spam:    true,
+			Details: "Details 1",
+		},
+	}
+
+	err = ds.Write(spamEntry, checks)
+	require.NoError(t, err)
+	var added bool
+	err = db.Get(&added, "SELECT added FROM detected_spam WHERE text = ?", spamEntry.Text)
+	require.NoError(t, err)
+	assert.False(t, added)
+
+	err = ds.SetAddedToSamplesFlag(1)
+	require.NoError(t, err)
+
+	err = db.Get(&added, "SELECT added FROM detected_spam WHERE text = ?", spamEntry.Text)
+	require.NoError(t, err)
+	assert.True(t, added)
+}
+
 func TestDetectedSpam_Read(t *testing.T) {
 	db, err := sqlx.Open("sqlite", ":memory:")
 	require.NoError(t, err)

app/webapi/assets/components/heads.html

@@ -0,0 +1,4 @@
+<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.0/dist/css/bootstrap.min.css" rel="stylesheet">
+<link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/bootstrap-icons.css" rel="stylesheet">
+<script src="https://unpkg.com/htmx.org"></script>
+<link href="styles.css" rel="stylesheet">

app/webapi/assets/detected_spam.html

@@ -2,17 +2,15 @@
 <html>
 <head>
     <title>Detected Spam - TG-Spam</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.0/dist/css/bootstrap.min.css" rel="stylesheet">
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/bootstrap-icons.css" rel="stylesheet">
-    <script src="https://unpkg.com/htmx.org"></script>
-    <link href="styles.css" rel="stylesheet">
+    {{template "heads.html"}}
 </head>
 <body>
 {{template "navbar.html"}}
 
 <div class="container mt-4">
     <div class="row" id="spam-list">
         <div class="col-md-12">
+            <div id="error-message" class="alert alert-danger d-none" role="alert"></div>
             <h4>Detected Spam ({{.TotalDetectedSpam}})</h4>
             <table class="table table-striped">
                 <thead class="custom-table-header">
@@ -26,17 +24,32 @@ <h4>Detected Spam ({{.TotalDetectedSpam}})</h4>
                 </thead>
                 <tbody>
                 {{range .DetectedSpamEntries}}
+                {{$text := .Text}}
+                {{$id := .ID}}
+                {{$added := .Added}}
                 <tr>
                     <td class="ds-timestamp">{{.Timestamp.Format "2006-01-02 15:04:05"}}</td>
                     <td>{{.UserID}}</td>
                     <td>{{.UserName}}</td>
                     <td class="ds-text">{{.Text}}</td>
                     <td class="ds-checks">
                         {{range .Checks}}
-                        <div class="{{if .Spam}}text-danger{{else}}text-success{{end}}">
-                            <strong>{{.Name}}:</strong> {{.Details}}
+                        <div style="display: flex; align-items: center;">
+                            <div class="{{if .Spam}}text-danger{{else}}text-success{{end}}" style="flex-grow: 1;">
+                                <strong>{{.Name}}:</strong> {{.Details}}
+                            </div>
+                            {{if and (not .Spam) (not $added) (eq .Name "classifier")}}
+                            <button
+                                    hx-post="/detected_spam/add" hx-vals='{"id": {{$id}}, "msg": "{{$text}}"}'
+                                    hx-target="#error-message" hx-error="#error-message" hx-swap="outerHTML"
+                                    class="btn btn-sm btn-warning"
+                                    title="Add this message to spam samples">
+                                Add
+                            </button>
+                            {{end}}
                         </div>
                         {{end}}
+                    </td>
                 </tr>
                 {{else}}
                 <tr>

app/webapi/assets/manage_samples.html

@@ -2,10 +2,7 @@
 <html>
 <head>
     <title>Manage Samples - TG-Spam</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.0/dist/css/bootstrap.min.css" rel="stylesheet">
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/bootstrap-icons.css" rel="stylesheet">
-    <script src="https://unpkg.com/htmx.org"></script>
-    <link href="styles.css" rel="stylesheet">
+    {{template "heads.html"}}
 </head>
 <body>
 {{template "navbar.html"}}

app/webapi/assets/manage_users.html

@@ -2,10 +2,7 @@
 <html>
 <head>
     <title>Manage Users - TG-Spam</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.0/dist/css/bootstrap.min.css" rel="stylesheet">
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/bootstrap-icons.css" rel="stylesheet">
-    <script src="https://unpkg.com/htmx.org"></script>
-    <link href="styles.css" rel="stylesheet">
+    {{template "heads.html"}}
 </head>
 <body>
 {{template "navbar.html"}}

app/webapi/assets/settings.html

@@ -2,9 +2,7 @@
 <html>
 <head>
     <title>Settings - TG-Spam</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.0/dist/css/bootstrap.min.css" rel="stylesheet">
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.5.0/font/bootstrap-icons.css" rel="stylesheet">
-    <link href="styles.css" rel="stylesheet">
+    {{template "heads.html"}}
 </head>
 <body>