chore: finished robots.txt and refactored middleware 9df75bb3
Steve · 2026-11-14 13:03 3 file(s) · +19 −7
main.go +16 −7
10 10
)
11 11
12 12
// Plan
13 -
// - [ ] Serve robots.txt with noindex
13 +
// - [x] Serve robots.txt with noindex
14 14
// - [/] Serve root path html with babbler and links to new paths
15 15
// - [ ] Serve dynamic path that takes in path param to serve babbler and more links
16 16
// - [ ] Slow down responses
19 19
20 20
//go:embed templates/*.html
21 21
var templatesFS embed.FS
22 +
var tmpl = template.Must(template.ParseFS(templatesFS, "templates/*.html"))
23 +
24 +
//go:embed robots.txt
25 +
var robotsTxt string
22 26
23 27
type PageData struct {
24 28
	Title string
25 29
	Items []string
26 30
}
27 31
28 -
var tmpl = template.Must(template.ParseFS(templatesFS, "templates/*.html"))
32 +
func robotsHandler(writer http.ResponseWriter, req *http.Request){
33 +
	writer.Header().Set("Content-Type", "text/plain; charset=utf-8")
34 +
	fmt.Fprint(writer, robotsTxt)
35 +
}
29 36
30 37
func indexHandler(writer http.ResponseWriter, req *http.Request) {
31 38
	data := PageData{
37 44
	}
38 45
}
39 46
40 -
func logRequest(handler http.Handler) http.Handler {
41 -
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
42 -
		log.Printf("%s %s %s\n", r.Method, r.URL, r.RemoteAddr)
43 -
		handler.ServeHTTP(w, r)
47 +
func requestMiddleware(handler http.Handler) http.Handler {
48 +
	return http.HandlerFunc(func(writer http.ResponseWriter, req *http.Request) {
49 +
		log.Printf("%s %s %s\n", req.Method, req.URL, req.RemoteAddr)
50 +
		writer.Header().Set("X-Robots-Tag", "noindex, nofollow")
51 +
		handler.ServeHTTP(writer, req)
44 52
	})
45 53
}
46 54
49 57
		portEnv = "8090"
50 58
	}
51 59
60 +
	http.HandleFunc("/robots.txt", robotsHandler)
52 61
	http.HandleFunc("/", indexHandler)
53 62
54 63
	port := fmt.Sprintf(":%s", portEnv)
55 64
	log.Println("Listening on", port)
56 -
	log.Fatal(http.ListenAndServe(port, logRequest(http.DefaultServeMux)))
65 +
	log.Fatal(http.ListenAndServe(port, requestMiddleware(http.DefaultServeMux)))
57 66
}
robots.txt (added) +2 −0
1 +
User-agent: *
2 +
Disallow: /
templates/index.html +1 −0
2 2
<html>
3 3
	<head>
4 4
		<meta charset="UTF-8">
5 +
    <meta name="robots" content="noindex, nofollow">
5 6
		<title>{{.Title}}</title>
6 7
	</head>
7 8
	<body>