git.stevedylan.dev

chore: improved feed discovery and parsing e0c3935e

Steve Simkins · 2026-07-05 11:46 1 file(s) · +65 −28

feeds.go +65 −28

		return nil, fmt.Errorf("invalid URL: %w", err)
	}
	client := buildHTTPClient()
	req, err := newRequest(ctx, http.MethodGet, baseURL)
	if err != nil {
		return nil, fmt.Errorf("invalid URL: %w", err)

	// Pages to scan for <link rel="alternate"> feed hints. Always include the
	// origin root: a user often pastes a deep or dead feed URL (e.g. an
	// advertised /rss.xml that 404s) while the real feed is advertised on the
	// homepage.
	scanPages := []string{baseURL}
	if root := originRoot(parsed); root != "" && root != baseURL {
		scanPages = append(scanPages, root)
	}
	feeds := []string{}
	resp, err := client.Do(req)
	if err == nil {
		defer resp.Body.Close()

	candidates := []string{}
	addCandidate := func(u string) {
		if u != "" && !slices.Contains(candidates, u) {
			candidates = append(candidates, u)
		}
	}
	for _, page := range scanPages {
		req, err := newRequest(ctx, http.MethodGet, page)
		if err != nil {
			continue
		}
		resp, err := client.Do(req)
		if err != nil {
			continue
		}
		body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
		links := findAlternateFeedLinks(string(body))
		for _, href := range links {
		_ = resp.Body.Close()
		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
			continue
		}
		for _, href := range findAlternateFeedLinks(string(body)) {
			resolved := href
			if u, err := parsed.Parse(href); err == nil {
				resolved = u.String()
			}
			if !slices.Contains(feeds, resolved) {
				feeds = append(feeds, resolved)
			}
			addCandidate(resolved)
		}
	}
	if len(feeds) == 0 {

	// Fall back to well-known feed paths only when the pages advertised none.
	if len(candidates) == 0 {
		paths := []string{"/feed", "/feed.xml", "/rss", "/rss.xml", "/atom.xml", "/index.xml", "/feed/rss", "/blog/feed", "/blog/rss"}
		for _, path := range paths {
			probe, err := parsed.Parse(path)
			if err != nil {
				continue
			if probe, err := parsed.Parse(path); err == nil {
				addCandidate(probe.String())
			}
			req, err := newRequest(ctx, http.MethodHead, probe.String())
			if err != nil {
				continue
			}
			resp, err := client.Do(req)
			if err != nil {
				continue
			}
			_ = resp.Body.Close()
			ct := strings.ToLower(resp.Header.Get("Content-Type"))
			if resp.StatusCode >= 200 && resp.StatusCode < 300 && (strings.Contains(ct, "xml") || strings.Contains(ct, "rss") || strings.Contains(ct, "atom")) {
				feeds = append(feeds, probe.String())
		}
	}

	// A candidate is only a feed if it actually parses. Content-type is
	// unreliable — many valid feeds serve text/html or send no type at all.
	// Validate concurrently to keep discovery fast.
	valid := make([]bool, len(candidates))
	var wg sync.WaitGroup
	for i, c := range candidates {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if _, err := fetchFeed(ctx, c, "", ""); err == nil {
				valid[i] = true
			}
		}()
	}
	wg.Wait()

	feeds := []string{}
	for i, c := range candidates {
		if valid[i] {
			feeds = append(feeds, c)
		}
	}
	if len(feeds) == 0 {
		return nil, errors.New("no feeds found at this URL")
	}
	return feeds, nil
}

// originRoot returns the scheme://host/ root for a parsed URL.
func originRoot(u *url.URL) string {
	if u == nil || u.Scheme == "" || u.Host == "" {
		return ""
	}
	return u.Scheme + "://" + u.Host + "/"
}

func findAlternateFeedLinks(doc string) []string {

284	284		return nil, fmt.Errorf("invalid URL: %w", err)
285	285		}
286	286		client := buildHTTPClient()
287		-	req, err := newRequest(ctx, http.MethodGet, baseURL)
288		-	if err != nil {
289		-	return nil, fmt.Errorf("invalid URL: %w", err)
	287	+
	288	+	// Pages to scan for <link rel="alternate"> feed hints. Always include the
	289	+	// origin root: a user often pastes a deep or dead feed URL (e.g. an
	290	+	// advertised /rss.xml that 404s) while the real feed is advertised on the
	291	+	// homepage.
	292	+	scanPages := []string{baseURL}
	293	+	if root := originRoot(parsed); root != "" && root != baseURL {
	294	+	scanPages = append(scanPages, root)
290	295		}
291		-	feeds := []string{}
292		-	resp, err := client.Do(req)
293		-	if err == nil {
294		-	defer resp.Body.Close()
	296	+
	297	+	candidates := []string{}
	298	+	addCandidate := func(u string) {
	299	+	if u != "" && !slices.Contains(candidates, u) {
	300	+	candidates = append(candidates, u)
	301	+	}
	302	+	}
	303	+	for _, page := range scanPages {
	304	+	req, err := newRequest(ctx, http.MethodGet, page)
	305	+	if err != nil {
	306	+	continue
	307	+	}
	308	+	resp, err := client.Do(req)
	309	+	if err != nil {
	310	+	continue
	311	+	}
295	312		body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
296		-	links := findAlternateFeedLinks(string(body))
297		-	for _, href := range links {
	313	+	_ = resp.Body.Close()
	314	+	if resp.StatusCode < 200 \|\| resp.StatusCode >= 300 {
	315	+	continue
	316	+	}
	317	+	for _, href := range findAlternateFeedLinks(string(body)) {
298	318		resolved := href
299	319		if u, err := parsed.Parse(href); err == nil {
300	320		resolved = u.String()
301	321		}
302		-	if !slices.Contains(feeds, resolved) {
303		-	feeds = append(feeds, resolved)
304		-	}
	322	+	addCandidate(resolved)
305	323		}
306	324		}
307		-	if len(feeds) == 0 {
	325	+
	326	+	// Fall back to well-known feed paths only when the pages advertised none.
	327	+	if len(candidates) == 0 {
308	328		paths := []string{"/feed", "/feed.xml", "/rss", "/rss.xml", "/atom.xml", "/index.xml", "/feed/rss", "/blog/feed", "/blog/rss"}
309	329		for _, path := range paths {
310		-	probe, err := parsed.Parse(path)
311		-	if err != nil {
312		-	continue
	330	+	if probe, err := parsed.Parse(path); err == nil {
	331	+	addCandidate(probe.String())
313	332		}
314		-	req, err := newRequest(ctx, http.MethodHead, probe.String())
315		-	if err != nil {
316		-	continue
317		-	}
318		-	resp, err := client.Do(req)
319		-	if err != nil {
320		-	continue
321		-	}
322		-	_ = resp.Body.Close()
323		-	ct := strings.ToLower(resp.Header.Get("Content-Type"))
324		-	if resp.StatusCode >= 200 && resp.StatusCode < 300 && (strings.Contains(ct, "xml") \|\| strings.Contains(ct, "rss") \|\| strings.Contains(ct, "atom")) {
325		-	feeds = append(feeds, probe.String())
	333	+	}
	334	+	}
	335	+
	336	+	// A candidate is only a feed if it actually parses. Content-type is
	337	+	// unreliable — many valid feeds serve text/html or send no type at all.
	338	+	// Validate concurrently to keep discovery fast.
	339	+	valid := make([]bool, len(candidates))
	340	+	var wg sync.WaitGroup
	341	+	for i, c := range candidates {
	342	+	wg.Add(1)
	343	+	go func() {
	344	+	defer wg.Done()
	345	+	if _, err := fetchFeed(ctx, c, "", ""); err == nil {
	346	+	valid[i] = true
326	347		}
	348	+	}()
	349	+	}
	350	+	wg.Wait()
	351	+
	352	+	feeds := []string{}
	353	+	for i, c := range candidates {
	354	+	if valid[i] {
	355	+	feeds = append(feeds, c)
327	356		}
328	357		}
329	358		if len(feeds) == 0 {
330	359		return nil, errors.New("no feeds found at this URL")
331	360		}
332	361		return feeds, nil
	362	+	}
	363	+
	364	+	// originRoot returns the scheme://host/ root for a parsed URL.
	365	+	func originRoot(u *url.URL) string {
	366	+	if u == nil \|\| u.Scheme == "" \|\| u.Host == "" {
	367	+	return ""
	368	+	}
	369	+	return u.Scheme + "://" + u.Host + "/"
333	370		}
334	371
335	372		func findAlternateFeedLinks(doc string) []string {