git.stevedylan.dev

feeds.go 11.5 K raw

package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"net/url"
	"slices"
	"strings"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/mmcdole/gofeed"
	"golang.org/x/net/html"
)

type ParsedEntry struct {
	GUID        string
	Title       string
	Link        string
	Author      string
	PublishedAt int64
}

type FetchResult struct {
	Status       int
	ETag         string
	LastModified string
	Title        string
	SiteURL      string
	Entries      []ParsedEntry
}

type FeedPreviewItem struct {
	Title     string
	Link      string
	Author    string
	Published int64
}

const appUserAgent = "feeds/0.1 (+https://github.com/stevedylandev/feeds)"

// fetchSem bounds total concurrent outbound HTTP requests across every handler.
// A single page can reference maxFeedURLs feeds (each a separate fetch), and
// that fan-out times concurrent viral traffic would otherwise open unbounded
// sockets and goroutines. Sized well above renderSem because fetches are
// IO-bound; excess fetches queue until a slot frees or their context expires.
var fetchSem = make(chan struct{}, 64)

// acquireFetch takes an outbound-fetch slot, honoring context cancellation so a
// request that gives up doesn't keep waiting for a slot it no longer needs.
func acquireFetch(ctx context.Context) error {
	select {
	case fetchSem <- struct{}{}:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

func releaseFetch() { <-fetchSem }

func buildHTTPClient() *http.Client {
	return &http.Client{Timeout: 10 * time.Second}
}

func newRequest(ctx context.Context, method, rawURL string) (*http.Request, error) {
	req, err := http.NewRequestWithContext(ctx, method, rawURL, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", appUserAgent)
	return req, nil
}

func fetchFeed(ctx context.Context, feedURL, etag, lastModified string) (*FetchResult, error) {
	client := buildHTTPClient()
	req, err := newRequest(ctx, http.MethodGet, feedURL)
	if err != nil {
		return nil, err
	}
	if etag != "" {
		req.Header.Set("If-None-Match", etag)
	}
	if lastModified != "" {
		req.Header.Set("If-Modified-Since", lastModified)
	}
	if err := acquireFetch(ctx); err != nil {
		return nil, err
	}
	defer releaseFetch()
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("fetch failed: %w", err)
	}
	defer resp.Body.Close()
	result := &FetchResult{
		Status:       resp.StatusCode,
		ETag:         resp.Header.Get("ETag"),
		LastModified: resp.Header.Get("Last-Modified"),
	}
	if resp.StatusCode == http.StatusNotModified {
		if result.ETag == "" {
			result.ETag = etag
		}
		if result.LastModified == "" {
			result.LastModified = lastModified
		}
		return result, nil
	}
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return nil, fmt.Errorf("upstream returned %d", resp.StatusCode)
	}
	parser := gofeed.NewParser()
	feed, err := parser.Parse(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("feed parse failed: %w", err)
	}
	result.Title = strings.TrimSpace(html.UnescapeString(feed.Title))
	result.SiteURL = firstNonEmpty(feed.Link, firstFeedAltLink(feed))
	for _, item := range feed.Items {
		link := strings.TrimSpace(item.Link)
		if link == "" {
			continue
		}
		title := strings.TrimSpace(html.UnescapeString(item.Title))
		if title == "" {
			title = deriveTitleFromHTML(firstNonEmpty(item.Description, item.Content))
		}
		if title == "" {
			title = "Untitled post"
		}
		author := ""
		if item.Author != nil {
			author = strings.TrimSpace(html.UnescapeString(item.Author.Name))
		}
		guid := strings.TrimSpace(item.GUID)
		if guid == "" {
			guid = link
		}
		published := int64(0)
		switch {
		case item.PublishedParsed != nil:
			published = item.PublishedParsed.Unix()
		case item.UpdatedParsed != nil:
			published = item.UpdatedParsed.Unix()
		}
		result.Entries = append(result.Entries, ParsedEntry{
			GUID:        guid,
			Title:       title,
			Link:        link,
			Author:      author,
			PublishedAt: published,
		})
	}
	return result, nil
}

func deriveTitleFromHTML(src string) string {
	txt := strings.Join(strings.Fields(htmlToText(src)), " ")
	if txt == "" {
		return ""
	}
	const maxChars = 80
	if utf8.RuneCountInString(txt) <= maxChars {
		return txt
	}
	runes := []rune(txt)
	return strings.TrimSpace(string(runes[:maxChars])) + "…"
}

func htmlToText(src string) string {
	if strings.TrimSpace(src) == "" {
		return ""
	}
	node, err := html.Parse(strings.NewReader(src))
	if err != nil {
		return src
	}
	var b strings.Builder
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.TextNode {
			b.WriteString(n.Data)
			b.WriteByte(' ')
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(node)
	return html.UnescapeString(b.String())
}

func previewURLs(ctx context.Context, urls []string, perFeed int, cache *feedCache, log *slog.Logger) ([]FeedPreviewItem, map[string]string) {
	var wg sync.WaitGroup
	var mu sync.Mutex
	items := []FeedPreviewItem{}
	titles := map[string]string{}
	for _, raw := range urls {
		feedURL := strings.TrimSpace(raw)
		if feedURL == "" {
			continue
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			res, err := cache.fetch(ctx, feedURL)
			if err != nil {
				log.Warn("preview fetch failed", "url", feedURL, "err", err)
				return
			}
			feedTitle := res.Title
			local := make([]FeedPreviewItem, 0, len(res.Entries))
			for _, entry := range res.Entries {
				if perFeed > 0 && len(local) >= perFeed {
					break
				}
				author := feedTitle
				if entry.Author != "" && feedTitle != "" {
					author = feedTitle + " - " + entry.Author
				} else if entry.Author != "" {
					author = entry.Author
				}
				local = append(local, FeedPreviewItem{Title: entry.Title, Link: entry.Link, Author: author, Published: entry.PublishedAt})
			}
			mu.Lock()
			items = append(items, local...)
			if feedTitle != "" {
				titles[feedURL] = feedTitle
			}
			mu.Unlock()
		}()
	}
	wg.Wait()
	slices.SortFunc(items, func(a, b FeedPreviewItem) int {
		switch {
		case a.Published > b.Published:
			return -1
		case a.Published < b.Published:
			return 1
		default:
			return 0
		}
	})
	return items, titles
}

func discoverFavicon(ctx context.Context, siteURL string) string {
	parsed, err := url.Parse(siteURL)
	if err != nil {
		return ""
	}
	client := buildHTTPClient()
	req, err := newRequest(ctx, http.MethodGet, siteURL)
	if err != nil {
		return ""
	}
	if err := acquireFetch(ctx); err != nil {
		return ""
	}
	defer releaseFetch()
	resp, err := client.Do(req)
	if err == nil {
		defer resp.Body.Close()
		body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
		if href := findLinkHref(string(body), func(rel, typ string) bool {
			rel = strings.ToLower(rel)
			return strings.Contains(rel, "icon")
		}); href != "" {
			if resolved, err := parsed.Parse(href); err == nil {
				return resolved.String()
			}
		}
	}
	if fallback, err := parsed.Parse("/favicon.ico"); err == nil {
		return fallback.String()
	}
	return ""
}

func findLinkHref(doc string, match func(rel, typ string) bool) string {
	node, err := html.Parse(strings.NewReader(doc))
	if err != nil {
		return ""
	}
	var found string
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if found != "" {
			return
		}
		if n.Type == html.ElementNode && strings.EqualFold(n.Data, "link") {
			attrs := attrsMap(n)
			if match(attrs["rel"], attrs["type"]) {
				found = attrs["href"]
				return
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(node)
	return found
}

func discoverFeeds(ctx context.Context, baseURL string) ([]string, error) {
	parsed, err := url.Parse(baseURL)
	if err != nil {
		return nil, fmt.Errorf("invalid URL: %w", err)
	}
	client := buildHTTPClient()

	// Pages to scan for <link rel="alternate"> feed hints. Always include the
	// origin root: a user often pastes a deep or dead feed URL (e.g. an
	// advertised /rss.xml that 404s) while the real feed is advertised on the
	// homepage.
	scanPages := []string{baseURL}
	if root := originRoot(parsed); root != "" && root != baseURL {
		scanPages = append(scanPages, root)
	}

	candidates := []string{}
	addCandidate := func(u string) {
		if u != "" && !slices.Contains(candidates, u) {
			candidates = append(candidates, u)
		}
	}
	for _, page := range scanPages {
		body, status := func() ([]byte, int) {
			req, err := newRequest(ctx, http.MethodGet, page)
			if err != nil {
				return nil, 0
			}
			if err := acquireFetch(ctx); err != nil {
				return nil, 0
			}
			defer releaseFetch()
			resp, err := client.Do(req)
			if err != nil {
				return nil, 0
			}
			defer resp.Body.Close()
			b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
			return b, resp.StatusCode
		}()
		if status < 200 || status >= 300 {
			continue
		}
		for _, href := range findAlternateFeedLinks(string(body)) {
			resolved := href
			if u, err := parsed.Parse(href); err == nil {
				resolved = u.String()
			}
			addCandidate(resolved)
		}
	}

	// Fall back to well-known feed paths only when the pages advertised none.
	if len(candidates) == 0 {
		paths := []string{"/feed", "/feed.xml", "/rss", "/rss.xml", "/atom.xml", "/index.xml", "/feed/rss", "/blog/feed", "/blog/rss"}
		for _, path := range paths {
			if probe, err := parsed.Parse(path); err == nil {
				addCandidate(probe.String())
			}
		}
	}

	// A candidate is only a feed if it actually parses. Content-type is
	// unreliable — many valid feeds serve text/html or send no type at all.
	// Validate concurrently to keep discovery fast.
	valid := make([]bool, len(candidates))
	var wg sync.WaitGroup
	for i, c := range candidates {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if _, err := fetchFeed(ctx, c, "", ""); err == nil {
				valid[i] = true
			}
		}()
	}
	wg.Wait()

	feeds := []string{}
	for i, c := range candidates {
		if valid[i] {
			feeds = append(feeds, c)
		}
	}
	if len(feeds) == 0 {
		return nil, errors.New("no feeds found at this URL")
	}
	return feeds, nil
}

// originRoot returns the scheme://host/ root for a parsed URL.
func originRoot(u *url.URL) string {
	if u == nil || u.Scheme == "" || u.Host == "" {
		return ""
	}
	return u.Scheme + "://" + u.Host + "/"
}

func findAlternateFeedLinks(doc string) []string {
	node, err := html.Parse(strings.NewReader(doc))
	if err != nil {
		return nil
	}
	links := []string{}
	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode && strings.EqualFold(n.Data, "link") {
			attrs := attrsMap(n)
			rel := strings.ToLower(attrs["rel"])
			typ := strings.ToLower(attrs["type"])
			href := attrs["href"]
			if strings.Contains(rel, "alternate") && href != "" && (strings.Contains(typ, "rss") || strings.Contains(typ, "atom") || strings.Contains(typ, "xml")) {
				links = append(links, href)
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(node)
	return links
}

func attrsMap(n *html.Node) map[string]string {
	out := make(map[string]string, len(n.Attr))
	for _, a := range n.Attr {
		out[strings.ToLower(a.Key)] = a.Val
	}
	return out
}

func firstFeedAltLink(feed *gofeed.Feed) string {
	for _, link := range feed.Links {
		if strings.TrimSpace(link) != "" {
			return link
		}
	}
	return ""
}

func firstNonEmpty(values ...string) string {
	for _, v := range values {
		if strings.TrimSpace(v) != "" {
			return strings.TrimSpace(v)
		}
	}
	return ""
}

func formatDate(ts int64) string {
	if ts <= 0 {
		return ""
	}
	return time.Unix(ts, 0).UTC().Format("Jan 2, 2006")
}

func splitAndTrim(s string) []string {
	parts := strings.Split(s, ",")
	out := []string{}
	for _, part := range parts {
		if trimmed := strings.TrimSpace(part); trimmed != "" {
			out = append(out, trimmed)
		}
	}
	return out
}

1	package main
2
3	import (
4	"context"
5	"errors"
6	"fmt"
7	"io"
8	"log/slog"
9	"net/http"
10	"net/url"
11	"slices"
12	"strings"
13	"sync"
14	"time"
15	"unicode/utf8"
16
17	"github.com/mmcdole/gofeed"
18	"golang.org/x/net/html"
19	)
20
21	type ParsedEntry struct {
22	GUID string
23	Title string
24	Link string
25	Author string
26	PublishedAt int64
27	}
28
29	type FetchResult struct {
30	Status int
31	ETag string
32	LastModified string
33	Title string
34	SiteURL string
35	Entries []ParsedEntry
36	}
37
38	type FeedPreviewItem struct {
39	Title string
40	Link string
41	Author string
42	Published int64
43	}
44
45	const appUserAgent = "feeds/0.1 (+https://github.com/stevedylandev/feeds)"
46
47	// fetchSem bounds total concurrent outbound HTTP requests across every handler.
48	// A single page can reference maxFeedURLs feeds (each a separate fetch), and
49	// that fan-out times concurrent viral traffic would otherwise open unbounded
50	// sockets and goroutines. Sized well above renderSem because fetches are
51	// IO-bound; excess fetches queue until a slot frees or their context expires.
52	var fetchSem = make(chan struct{}, 64)
53
54	// acquireFetch takes an outbound-fetch slot, honoring context cancellation so a
55	// request that gives up doesn't keep waiting for a slot it no longer needs.
56	func acquireFetch(ctx context.Context) error {
57	select {
58	case fetchSem <- struct{}{}:
59	return nil
60	case <-ctx.Done():
61	return ctx.Err()
62	}
63	}
64
65	func releaseFetch() { <-fetchSem }
66
67	func buildHTTPClient() *http.Client {
68	return &http.Client{Timeout: 10 * time.Second}
69	}
70
71	func newRequest(ctx context.Context, method, rawURL string) (*http.Request, error) {
72	req, err := http.NewRequestWithContext(ctx, method, rawURL, nil)
73	if err != nil {
74	return nil, err
75	}
76	req.Header.Set("User-Agent", appUserAgent)
77	return req, nil
78	}
79
80	func fetchFeed(ctx context.Context, feedURL, etag, lastModified string) (*FetchResult, error) {
81	client := buildHTTPClient()
82	req, err := newRequest(ctx, http.MethodGet, feedURL)
83	if err != nil {
84	return nil, err
85	}
86	if etag != "" {
87	req.Header.Set("If-None-Match", etag)
88	}
89	if lastModified != "" {
90	req.Header.Set("If-Modified-Since", lastModified)
91	}
92	if err := acquireFetch(ctx); err != nil {
93	return nil, err
94	}
95	defer releaseFetch()
96	resp, err := client.Do(req)
97	if err != nil {
98	return nil, fmt.Errorf("fetch failed: %w", err)
99	}
100	defer resp.Body.Close()
101	result := &FetchResult{
102	Status: resp.StatusCode,
103	ETag: resp.Header.Get("ETag"),
104	LastModified: resp.Header.Get("Last-Modified"),
105	}
106	if resp.StatusCode == http.StatusNotModified {
107	if result.ETag == "" {
108	result.ETag = etag
109	}
110	if result.LastModified == "" {
111	result.LastModified = lastModified
112	}
113	return result, nil
114	}
115	if resp.StatusCode < 200 \|\| resp.StatusCode >= 300 {
116	return nil, fmt.Errorf("upstream returned %d", resp.StatusCode)
117	}
118	parser := gofeed.NewParser()
119	feed, err := parser.Parse(resp.Body)
120	if err != nil {
121	return nil, fmt.Errorf("feed parse failed: %w", err)
122	}
123	result.Title = strings.TrimSpace(html.UnescapeString(feed.Title))
124	result.SiteURL = firstNonEmpty(feed.Link, firstFeedAltLink(feed))
125	for _, item := range feed.Items {
126	link := strings.TrimSpace(item.Link)
127	if link == "" {
128	continue
129	}
130	title := strings.TrimSpace(html.UnescapeString(item.Title))
131	if title == "" {
132	title = deriveTitleFromHTML(firstNonEmpty(item.Description, item.Content))
133	}
134	if title == "" {
135	title = "Untitled post"
136	}
137	author := ""
138	if item.Author != nil {
139	author = strings.TrimSpace(html.UnescapeString(item.Author.Name))
140	}
141	guid := strings.TrimSpace(item.GUID)
142	if guid == "" {
143	guid = link
144	}
145	published := int64(0)
146	switch {
147	case item.PublishedParsed != nil:
148	published = item.PublishedParsed.Unix()
149	case item.UpdatedParsed != nil:
150	published = item.UpdatedParsed.Unix()
151	}
152	result.Entries = append(result.Entries, ParsedEntry{
153	GUID: guid,
154	Title: title,
155	Link: link,
156	Author: author,
157	PublishedAt: published,
158	})
159	}
160	return result, nil
161	}
162
163	func deriveTitleFromHTML(src string) string {
164	txt := strings.Join(strings.Fields(htmlToText(src)), " ")
165	if txt == "" {
166	return ""
167	}
168	const maxChars = 80
169	if utf8.RuneCountInString(txt) <= maxChars {
170	return txt
171	}
172	runes := []rune(txt)
173	return strings.TrimSpace(string(runes[:maxChars])) + "…"
174	}
175
176	func htmlToText(src string) string {
177	if strings.TrimSpace(src) == "" {
178	return ""
179	}
180	node, err := html.Parse(strings.NewReader(src))
181	if err != nil {
182	return src
183	}
184	var b strings.Builder
185	var walk func(*html.Node)
186	walk = func(n *html.Node) {
187	if n.Type == html.TextNode {
188	b.WriteString(n.Data)
189	b.WriteByte(' ')
190	}
191	for c := n.FirstChild; c != nil; c = c.NextSibling {
192	walk(c)
193	}
194	}
195	walk(node)
196	return html.UnescapeString(b.String())
197	}
198
199	func previewURLs(ctx context.Context, urls []string, perFeed int, cache feedCache, log slog.Logger) ([]FeedPreviewItem, map[string]string) {
200	var wg sync.WaitGroup
201	var mu sync.Mutex
202	items := []FeedPreviewItem{}
203	titles := map[string]string{}
204	for _, raw := range urls {
205	feedURL := strings.TrimSpace(raw)
206	if feedURL == "" {
207	continue
208	}
209	wg.Add(1)
210	go func() {
211	defer wg.Done()
212	res, err := cache.fetch(ctx, feedURL)
213	if err != nil {
214	log.Warn("preview fetch failed", "url", feedURL, "err", err)
215	return
216	}
217	feedTitle := res.Title
218	local := make([]FeedPreviewItem, 0, len(res.Entries))
219	for _, entry := range res.Entries {
220	if perFeed > 0 && len(local) >= perFeed {
221	break
222	}
223	author := feedTitle
224	if entry.Author != "" && feedTitle != "" {
225	author = feedTitle + " - " + entry.Author
226	} else if entry.Author != "" {
227	author = entry.Author
228	}
229	local = append(local, FeedPreviewItem{Title: entry.Title, Link: entry.Link, Author: author, Published: entry.PublishedAt})
230	}
231	mu.Lock()
232	items = append(items, local...)
233	if feedTitle != "" {
234	titles[feedURL] = feedTitle
235	}
236	mu.Unlock()
237	}()
238	}
239	wg.Wait()
240	slices.SortFunc(items, func(a, b FeedPreviewItem) int {
241	switch {
242	case a.Published > b.Published:
243	return -1
244	case a.Published < b.Published:
245	return 1
246	default:
247	return 0
248	}
249	})
250	return items, titles
251	}
252
253	func discoverFavicon(ctx context.Context, siteURL string) string {
254	parsed, err := url.Parse(siteURL)
255	if err != nil {
256	return ""
257	}
258	client := buildHTTPClient()
259	req, err := newRequest(ctx, http.MethodGet, siteURL)
260	if err != nil {
261	return ""
262	}
263	if err := acquireFetch(ctx); err != nil {
264	return ""
265	}
266	defer releaseFetch()
267	resp, err := client.Do(req)
268	if err == nil {
269	defer resp.Body.Close()
270	body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
271	if href := findLinkHref(string(body), func(rel, typ string) bool {
272	rel = strings.ToLower(rel)
273	return strings.Contains(rel, "icon")
274	}); href != "" {
275	if resolved, err := parsed.Parse(href); err == nil {
276	return resolved.String()
277	}
278	}
279	}
280	if fallback, err := parsed.Parse("/favicon.ico"); err == nil {
281	return fallback.String()
282	}
283	return ""
284	}
285
286	func findLinkHref(doc string, match func(rel, typ string) bool) string {
287	node, err := html.Parse(strings.NewReader(doc))
288	if err != nil {
289	return ""
290	}
291	var found string
292	var walk func(*html.Node)
293	walk = func(n *html.Node) {
294	if found != "" {
295	return
296	}
297	if n.Type == html.ElementNode && strings.EqualFold(n.Data, "link") {
298	attrs := attrsMap(n)
299	if match(attrs["rel"], attrs["type"]) {
300	found = attrs["href"]
301	return
302	}
303	}
304	for c := n.FirstChild; c != nil; c = c.NextSibling {
305	walk(c)
306	}
307	}
308	walk(node)
309	return found
310	}
311
312	func discoverFeeds(ctx context.Context, baseURL string) ([]string, error) {
313	parsed, err := url.Parse(baseURL)
314	if err != nil {
315	return nil, fmt.Errorf("invalid URL: %w", err)
316	}
317	client := buildHTTPClient()
318
319	// Pages to scan for <link rel="alternate"> feed hints. Always include the
320	// origin root: a user often pastes a deep or dead feed URL (e.g. an
321	// advertised /rss.xml that 404s) while the real feed is advertised on the
322	// homepage.
323	scanPages := []string{baseURL}
324	if root := originRoot(parsed); root != "" && root != baseURL {
325	scanPages = append(scanPages, root)
326	}
327
328	candidates := []string{}
329	addCandidate := func(u string) {
330	if u != "" && !slices.Contains(candidates, u) {
331	candidates = append(candidates, u)
332	}
333	}
334	for _, page := range scanPages {
335	body, status := func() ([]byte, int) {
336	req, err := newRequest(ctx, http.MethodGet, page)
337	if err != nil {
338	return nil, 0
339	}
340	if err := acquireFetch(ctx); err != nil {
341	return nil, 0
342	}
343	defer releaseFetch()
344	resp, err := client.Do(req)
345	if err != nil {
346	return nil, 0
347	}
348	defer resp.Body.Close()
349	b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
350	return b, resp.StatusCode
351	}()
352	if status < 200 \|\| status >= 300 {
353	continue
354	}
355	for _, href := range findAlternateFeedLinks(string(body)) {
356	resolved := href
357	if u, err := parsed.Parse(href); err == nil {
358	resolved = u.String()
359	}
360	addCandidate(resolved)
361	}
362	}
363
364	// Fall back to well-known feed paths only when the pages advertised none.
365	if len(candidates) == 0 {
366	paths := []string{"/feed", "/feed.xml", "/rss", "/rss.xml", "/atom.xml", "/index.xml", "/feed/rss", "/blog/feed", "/blog/rss"}
367	for _, path := range paths {
368	if probe, err := parsed.Parse(path); err == nil {
369	addCandidate(probe.String())
370	}
371	}
372	}
373
374	// A candidate is only a feed if it actually parses. Content-type is
375	// unreliable — many valid feeds serve text/html or send no type at all.
376	// Validate concurrently to keep discovery fast.
377	valid := make([]bool, len(candidates))
378	var wg sync.WaitGroup
379	for i, c := range candidates {
380	wg.Add(1)
381	go func() {
382	defer wg.Done()
383	if _, err := fetchFeed(ctx, c, "", ""); err == nil {
384	valid[i] = true
385	}
386	}()
387	}
388	wg.Wait()
389
390	feeds := []string{}
391	for i, c := range candidates {
392	if valid[i] {
393	feeds = append(feeds, c)
394	}
395	}
396	if len(feeds) == 0 {
397	return nil, errors.New("no feeds found at this URL")
398	}
399	return feeds, nil
400	}
401
402	// originRoot returns the scheme://host/ root for a parsed URL.
403	func originRoot(u *url.URL) string {
404	if u == nil \|\| u.Scheme == "" \|\| u.Host == "" {
405	return ""
406	}
407	return u.Scheme + "://" + u.Host + "/"
408	}
409
410	func findAlternateFeedLinks(doc string) []string {
411	node, err := html.Parse(strings.NewReader(doc))
412	if err != nil {
413	return nil
414	}
415	links := []string{}
416	var walk func(*html.Node)
417	walk = func(n *html.Node) {
418	if n.Type == html.ElementNode && strings.EqualFold(n.Data, "link") {
419	attrs := attrsMap(n)
420	rel := strings.ToLower(attrs["rel"])
421	typ := strings.ToLower(attrs["type"])
422	href := attrs["href"]
423	if strings.Contains(rel, "alternate") && href != "" && (strings.Contains(typ, "rss") \|\| strings.Contains(typ, "atom") \|\| strings.Contains(typ, "xml")) {
424	links = append(links, href)
425	}
426	}
427	for c := n.FirstChild; c != nil; c = c.NextSibling {
428	walk(c)
429	}
430	}
431	walk(node)
432	return links
433	}
434
435	func attrsMap(n *html.Node) map[string]string {
436	out := make(map[string]string, len(n.Attr))
437	for _, a := range n.Attr {
438	out[strings.ToLower(a.Key)] = a.Val
439	}
440	return out
441	}
442
443	func firstFeedAltLink(feed *gofeed.Feed) string {
444	for _, link := range feed.Links {
445	if strings.TrimSpace(link) != "" {
446	return link
447	}
448	}
449	return ""
450	}
451
452	func firstNonEmpty(values ...string) string {
453	for _, v := range values {
454	if strings.TrimSpace(v) != "" {
455	return strings.TrimSpace(v)
456	}
457	}
458	return ""
459	}
460
461	func formatDate(ts int64) string {
462	if ts <= 0 {
463	return ""
464	}
465	return time.Unix(ts, 0).UTC().Format("Jan 2, 2006")
466	}
467
468	func splitAndTrim(s string) []string {
469	parts := strings.Split(s, ",")
470	out := []string{}
471	for _, part := range parts {
472	if trimmed := strings.TrimSpace(part); trimmed != "" {
473	out = append(out, trimmed)
474	}
475	}
476	return out
477	}