chore: added better logging
adca6aa3
5 file(s) · +94 −12
| 73 | 73 | - `GARGANTUA_CORPUS` - Text used for Markov babbler (default looks for `moby.txt` in the repo) |
|
| 74 | 74 | - `GARGANTUA_HOST`- Host used (default `127.0.0.1`) |
|
| 75 | 75 | - `GARGANTUA_PORT` - Port used (default `8090`) |
|
| 76 | + | - `GARGANTUA_LOG_LEVEL` - Minimum log level: `debug`, `info`, `warn` or `error` (default `info`) |
|
| 77 | + | - `GARGANTUA_LOG_FORMAT` - Log format: `text` or `json` (default `text`) |
|
| 76 | 78 | - Gargantua uses a slow drip method for writing the html that will keep the scraper longer. You can tweak the range of delay used between writes and the amount of data written. |
|
| 77 | 79 | - `GARGANTUA_DELAY_MIN_MS` - Minimum delay in ms (default `150`) |
|
| 78 | 80 | - `GARGANTUA_DELAY_MAX_MS` - Maximum delay in ms (default `250`) |
| 1 | 1 | package main |
|
| 2 | 2 | ||
| 3 | 3 | import ( |
|
| 4 | - | "log" |
|
| 5 | 4 | "math/rand" |
|
| 6 | 5 | "os" |
|
| 7 | 6 | "strings" |
|
| 19 | 18 | func NewBabbler() *MarkovChain { |
|
| 20 | 19 | text, err := os.ReadFile(sourceText) |
|
| 21 | 20 | if err != nil { |
|
| 22 | - | log.Fatal("Could not read source text for babbler", err) |
|
| 21 | + | fatal("could not read source text for babbler", "path", sourceText, "err", err) |
|
| 23 | 22 | } |
|
| 24 | 23 | fields := strings.Fields(string(text)) |
|
| 25 | 24 | ||
| 47 | 46 | keys = append(keys, k) |
|
| 48 | 47 | } |
|
| 49 | 48 | ||
| 49 | + | logger.Debug("babbler corpus loaded", |
|
| 50 | + | "path", sourceText, |
|
| 51 | + | "words", len(fields), |
|
| 52 | + | "unique_words", len(words), |
|
| 53 | + | "chain_entries", len(chain), |
|
| 54 | + | ) |
|
| 55 | + | ||
| 50 | 56 | return &MarkovChain{chain: chain, words: words, keys: keys} |
|
| 51 | 57 | } |
|
| 52 | 58 | ||
| 85 | 91 | for len(links) < n { |
|
| 86 | 92 | link := m.randomLink() |
|
| 87 | 93 | if _, dup := seen[link]; dup { |
|
| 88 | - | continue |
|
| 94 | + | continue |
|
| 89 | 95 | } |
|
| 90 | 96 | seen[link] = struct{}{} |
|
| 91 | 97 | links = append(links, link) |
|
| 1 | + | package main |
|
| 2 | + | ||
| 3 | + | import ( |
|
| 4 | + | "log/slog" |
|
| 5 | + | "os" |
|
| 6 | + | "strings" |
|
| 7 | + | ) |
|
| 8 | + | ||
| 9 | + | var logger = newLogger() |
|
| 10 | + | ||
| 11 | + | func newLogger() *slog.Logger { |
|
| 12 | + | opts := &slog.HandlerOptions{Level: levelFromEnv()} |
|
| 13 | + | ||
| 14 | + | var handler slog.Handler |
|
| 15 | + | if strings.EqualFold(envString("GARGANTUA_LOG_FORMAT", "text"), "json") { |
|
| 16 | + | handler = slog.NewJSONHandler(os.Stdout, opts) |
|
| 17 | + | } else { |
|
| 18 | + | handler = slog.NewTextHandler(os.Stdout, opts) |
|
| 19 | + | } |
|
| 20 | + | ||
| 21 | + | return slog.New(handler) |
|
| 22 | + | } |
|
| 23 | + | ||
| 24 | + | func levelFromEnv() slog.Level { |
|
| 25 | + | switch strings.ToUpper(envString("GARGANTUA_LOG_LEVEL", "info")) { |
|
| 26 | + | case "DEBUG": |
|
| 27 | + | return slog.LevelDebug |
|
| 28 | + | case "WARN": |
|
| 29 | + | return slog.LevelWarn |
|
| 30 | + | case "ERROR": |
|
| 31 | + | return slog.LevelError |
|
| 32 | + | default: |
|
| 33 | + | return slog.LevelInfo |
|
| 34 | + | } |
|
| 35 | + | } |
|
| 36 | + | ||
| 37 | + | func fatal(msg string, args ...any) { |
|
| 38 | + | logger.Error(msg, args...) |
|
| 39 | + | os.Exit(1) |
|
| 40 | + | } |
| 2 | 2 | ||
| 3 | 3 | import ( |
|
| 4 | 4 | "fmt" |
|
| 5 | - | "log" |
|
| 6 | 5 | "net/http" |
|
| 7 | 6 | ) |
|
| 8 | 7 | ||
| 16 | 15 | ) |
|
| 17 | 16 | ||
| 18 | 17 | func main() { |
|
| 18 | + | logger.Info("starting", "addr", fmt.Sprintf("%s:%s", hostEnv, portEnv)) |
|
| 19 | + | ||
| 19 | 20 | http.HandleFunc("/", IndexHandler) |
|
| 20 | 21 | http.HandleFunc("/{slug}", IndexHandler) |
|
| 21 | 22 | http.HandleFunc("/{slug}/{slug2}", IndexHandler) |
|
| 22 | 23 | http.HandleFunc("/robots.txt", RobotsHandler) |
|
| 23 | 24 | ||
| 24 | 25 | port := fmt.Sprintf("%s:%s", hostEnv, portEnv) |
|
| 25 | - | log.Println("Listening on", port) |
|
| 26 | - | log.Fatal(http.ListenAndServe(port, RequestMiddleware(http.DefaultServeMux))) |
|
| 26 | + | if err := http.ListenAndServe(port, RequestMiddleware(http.DefaultServeMux)); err != nil { |
|
| 27 | + | fatal("server failed", "err", err) |
|
| 28 | + | } |
|
| 27 | 29 | } |
|
| 1 | - | package main |
|
| 1 | + | package main |
|
| 2 | 2 | ||
| 3 | 3 | import ( |
|
| 4 | 4 | "bytes" |
|
| 5 | 5 | "embed" |
|
| 6 | 6 | "fmt" |
|
| 7 | 7 | "html/template" |
|
| 8 | - | "log" |
|
| 9 | 8 | "math/rand" |
|
| 10 | 9 | "net/http" |
|
| 11 | 10 | "time" |
|
| 65 | 64 | chunk = len(data) |
|
| 66 | 65 | } |
|
| 67 | 66 | if _, err := writer.Write(data[:chunk]); err != nil { |
|
| 68 | - | return |
|
| 67 | + | return |
|
| 69 | 68 | } |
|
| 70 | 69 | if canFlush { |
|
| 71 | 70 | flusher.Flush() |
|
| 100 | 99 | TarpitWrite(writer, buf.Bytes()) |
|
| 101 | 100 | } |
|
| 102 | 101 | ||
| 102 | + | type statusRecorder struct { |
|
| 103 | + | http.ResponseWriter |
|
| 104 | + | status int |
|
| 105 | + | } |
|
| 106 | + | ||
| 107 | + | func (r *statusRecorder) WriteHeader(code int) { |
|
| 108 | + | r.status = code |
|
| 109 | + | r.ResponseWriter.WriteHeader(code) |
|
| 110 | + | } |
|
| 111 | + | ||
| 112 | + | func (r *statusRecorder) Flush() { |
|
| 113 | + | if flusher, ok := r.ResponseWriter.(http.Flusher); ok { |
|
| 114 | + | flusher.Flush() |
|
| 115 | + | } |
|
| 116 | + | } |
|
| 117 | + | ||
| 103 | 118 | func RequestMiddleware(handler http.Handler) http.Handler { |
|
| 104 | 119 | return http.HandlerFunc(func(writer http.ResponseWriter, req *http.Request) { |
|
| 105 | - | log.Printf("%s %s %s\n", req.Method, req.URL, req.UserAgent()) |
|
| 120 | + | start := time.Now() |
|
| 121 | + | recorder := &statusRecorder{ResponseWriter: writer, status: http.StatusOK} |
|
| 122 | + | ||
| 123 | + | logger.Debug("request started", |
|
| 124 | + | "method", req.Method, |
|
| 125 | + | "path", req.URL.Path, |
|
| 126 | + | "remote", req.RemoteAddr, |
|
| 127 | + | "user_agent", req.UserAgent(), |
|
| 128 | + | ) |
|
| 129 | + | ||
| 106 | 130 | writer.Header().Set("X-Robots-Tag", "noindex, nofollow") |
|
| 107 | - | handler.ServeHTTP(writer, req) |
|
| 131 | + | handler.ServeHTTP(recorder, req) |
|
| 132 | + | ||
| 133 | + | logger.Info("request", |
|
| 134 | + | "method", req.Method, |
|
| 135 | + | "path", req.URL.Path, |
|
| 136 | + | "status", recorder.status, |
|
| 137 | + | "duration", time.Since(start).Round(time.Millisecond), |
|
| 138 | + | "remote", req.RemoteAddr, |
|
| 139 | + | "user_agent", req.UserAgent(), |
|
| 140 | + | ) |
|
| 108 | 141 | }) |
|
| 109 | 142 | } |
|
| 110 | - | ||