apps/quotes/seed.go 4.6 K raw
1
package main
2
3
import (
4
	_ "embed"
5
	"encoding/csv"
6
	"errors"
7
	"fmt"
8
	"io"
9
	"log/slog"
10
	"os"
11
	"strings"
12
	"time"
13
14
	"github.com/stevedylandev/andromeda/pkg/auth"
15
	"github.com/stevedylandev/andromeda/pkg/sqlite"
16
)
17
18
const classicListPath = "classic_authors.txt"
19
20
// embeddedClassicList is baked into the binary so the seed command works inside
21
// a container (or anywhere the txt file is absent) without needing the file on
22
// disk. An on-disk classic_authors.txt still takes precedence, so local edits
23
// apply without a rebuild.
24
//
25
//go:embed classic_authors.txt
26
var embeddedClassicList string
27
28
func parseClassicList(data string) []string {
29
	var out []string
30
	for _, line := range strings.Split(data, "\n") {
31
		line = strings.TrimSpace(line)
32
		if line == "" || strings.HasPrefix(line, "#") {
33
			continue
34
		}
35
		out = append(out, line)
36
	}
37
	return out
38
}
39
40
// loadClassicList reads the match list from disk, falling back to the embedded
41
// copy when the file is absent. Skips blank lines and # comments.
42
func loadClassicList(path string) ([]string, bool, error) {
43
	data, err := os.ReadFile(path)
44
	if errors.Is(err, os.ErrNotExist) {
45
		return parseClassicList(embeddedClassicList), true, nil
46
	}
47
	if err != nil {
48
		return nil, false, err
49
	}
50
	return parseClassicList(string(data)), false, nil
51
}
52
53
// matchesClassic reports whether the author column contains any list entry as a
54
// case-sensitive substring.
55
func matchesClassic(author string, list []string) bool {
56
	for _, name := range list {
57
		if strings.Contains(author, name) {
58
			return true
59
		}
60
	}
61
	return false
62
}
63
64
// splitAttribution splits a CSV author field of the form "Author, Book Title"
65
// at the first comma into author and source. No comma -> empty source.
66
func splitAttribution(field string) (author, source string) {
67
	if i := strings.Index(field, ","); i >= 0 {
68
		return strings.TrimSpace(field[:i]), strings.TrimSpace(field[i+1:])
69
	}
70
	return strings.TrimSpace(field), ""
71
}
72
73
func listSource(embedded bool) string {
74
	if embedded {
75
		return "embedded"
76
	}
77
	return classicListPath
78
}
79
80
func dedupKey(text, author string) string {
81
	return text + "\x00" + author
82
}
83
84
// runSeed imports classic-literature quotes from csvPath into the database at
85
// dbPath. It is idempotent: quotes already present (matched on text+author) are
86
// skipped, so re-running after editing classic_authors.txt only adds new rows.
87
func runSeed(logger *slog.Logger, dbPath, csvPath string) error {
88
	list, embedded, err := loadClassicList(classicListPath)
89
	if err != nil {
90
		return fmt.Errorf("read %s: %w", classicListPath, err)
91
	}
92
	logger.Info("loaded classic list", "entries", len(list), "source", listSource(embedded))
93
94
	db, err := sqlite.Open(dbPath, quotesSchema)
95
	if err != nil {
96
		return err
97
	}
98
	defer db.Close()
99
100
	// Preload existing (text, author) pairs so re-seeds stay idempotent.
101
	seen := map[string]struct{}{}
102
	rows, err := db.Query(`SELECT text, author FROM quotes`)
103
	if err != nil {
104
		return err
105
	}
106
	for rows.Next() {
107
		var t, a string
108
		if err := rows.Scan(&t, &a); err != nil {
109
			rows.Close()
110
			return err
111
		}
112
		seen[dedupKey(t, a)] = struct{}{}
113
	}
114
	rows.Close()
115
116
	f, err := os.Open(csvPath)
117
	if err != nil {
118
		return err
119
	}
120
	defer f.Close()
121
122
	reader := csv.NewReader(f)
123
	reader.FieldsPerRecord = -1
124
	if _, err := reader.Read(); err != nil { // skip header
125
		return err
126
	}
127
128
	tx, err := db.Begin()
129
	if err != nil {
130
		return err
131
	}
132
	stmt, err := tx.Prepare(
133
		`INSERT INTO quotes (short_id, text, author, source, added_at, updated_at)
134
		 VALUES (?, ?, ?, ?, ?, ?)`,
135
	)
136
	if err != nil {
137
		tx.Rollback()
138
		return err
139
	}
140
	defer stmt.Close()
141
142
	now := time.Now().UTC().Unix()
143
	var scanned, inserted, skipped int
144
	for {
145
		rec, err := reader.Read()
146
		if err == io.EOF {
147
			break
148
		}
149
		if err != nil {
150
			logger.Warn("skipping malformed row", "err", err)
151
			continue
152
		}
153
		if len(rec) < 2 {
154
			continue
155
		}
156
		scanned++
157
		text := strings.TrimSpace(rec[0])
158
		rawAuthor := rec[1]
159
		if text == "" || !matchesClassic(rawAuthor, list) {
160
			continue
161
		}
162
		author, source := splitAttribution(rawAuthor)
163
		if author == "" {
164
			continue
165
		}
166
		key := dedupKey(text, author)
167
		if _, ok := seen[key]; ok {
168
			skipped++
169
			continue
170
		}
171
		shortID, err := auth.GenerateShortID(10)
172
		if err != nil {
173
			tx.Rollback()
174
			return err
175
		}
176
		var src any
177
		if source != "" {
178
			src = source
179
		}
180
		if _, err := stmt.Exec(shortID, text, author, src, now, now); err != nil {
181
			tx.Rollback()
182
			return err
183
		}
184
		seen[key] = struct{}{}
185
		inserted++
186
	}
187
188
	if err := tx.Commit(); err != nil {
189
		return err
190
	}
191
	total, _ := countQuotes(db)
192
	logger.Info("seed complete", "scanned", scanned, "inserted", inserted, "skipped_duplicates", skipped, "total_in_db", total)
193
	return nil
194
}