git.stevedylan.dev

feat: added feed discovery 717cbd05

Steve · 2025-10-29 06:53 3 file(s) · +100 −18

TODO.md +4 −2


- [ ] Add `publishedDate` to `rssPost`
- [ ] Order posts by newest first
- [ ] Instead of "by author" if Author is unavailable, use blog/feed name
- [ ] Add auto-find RSS feed when adding a feed
- [x] Instead of "by author" if Author is unavailable, use blog/feed name
- [x] Add auto-find RSS feed when adding a feed
- [ ] When scrolling through one feed and then selecting another, make the content scroll back to the top
- [ ] Can't open add feed dialog when on mobile
- [ ] Add error handling and checks for adding a feed
- [ ] Add loading states for new feed
- [ ] Find a way to paginate through feeds and get more posts if they only return a few

src/components/app-sidebar.tsx +78 −16

} from "@/lib/evolu";
import { XMLParser } from "fast-xml-parser";
import { useQuery } from "@evolu/react";
import { COMMON_FEED_PATHS } from "@/lib/feed-discovery";
const parser = new XMLParser();

// This is sample data


	async function addFeed() {
		try {
			let xmlData: string;
			try {
				// Try to fetch directly first
				const xmlFetch = await fetch(urlInput);
				xmlData = await xmlFetch.text();
			} catch (corsError) {
				// Fall back to AllOrigins if CORS error occurs
				console.log(corsError);
				const xmlFetch = await fetch(
					`https://api.allorigins.win/raw?url=${urlInput}`,
				);
				xmlData = await xmlFetch.text();
			// Try to discover feeds if the URL doesn't look like a direct feed URL
			let feedUrl = urlInput;
			const looksLikeFeedUrl =
				urlInput.includes("/feed") ||
				urlInput.includes("/rss") ||
				urlInput.includes(".xml") ||
				urlInput.includes("/atom");

			let xmlData: string | null = null;

			if (!looksLikeFeedUrl) {
				// Try common feed paths using CORS proxy
				const urlObj = new URL(urlInput);
				const origin = urlObj.origin;

				console.log("Trying to discover feed from:", origin);

				for (const path of COMMON_FEED_PATHS) {
					const testUrl = `${origin}${path}`;
					console.log("Testing:", testUrl);

					try {
						// Use CORS proxy to avoid CORS issues
						const response = await fetch(
							`https://corsproxy.io/?url=${encodeURIComponent(testUrl)}`,
						);

						if (response.ok) {
							const text = await response.text();
							// Quick check if it looks like XML
							if (
								text.trim().startsWith("<?xml") ||
								text.includes("<rss") ||
								text.includes("<feed")
							) {
								xmlData = text;
								feedUrl = testUrl;
								console.log("Found feed at:", testUrl);
								break;
							}
						}
					} catch (error) {
						console.log("Failed to fetch:", testUrl, error);
						continue;
					}
				}

				if (!xmlData) {
					alert(
						"Could not find an RSS feed at this URL. Please enter a direct feed URL.",
					);
					return;
				}
			} else {
				// Direct feed URL - try to fetch it
				try {
					// Try to fetch directly first
					const xmlFetch = await fetch(feedUrl);
					console.log("Status code: ", xmlFetch.status);
					console.log("Request ok: ", xmlFetch.ok);
					xmlData = await xmlFetch.text();
				} catch (corsError) {
					// Fall back to AllOrigins if CORS error occurs
					console.log(corsError);
					const xmlFetch = await fetch(
						`https://api.allorigins.win/raw?url=${feedUrl}`,
					);
					xmlData = await xmlFetch.text();
				}
			}
			const parsedXmlData = await parser.parse(xmlData);
			console.log(parsedXmlData);

			}

			const result = insert("rssFeed", {
				feedUrl: urlInput,
				feedUrl: feedUrl,
				title: feedData.title,
				description: feedData.description || feedData.subtitle || "",
				category: categoryInput || "Uncategorized",

				insert("rssPost", {
					title: post.title,
					author: isAtom
						? post.author?.name || "Author"
						: post.author || "Author",
						? post.author?.name || feedData.title
						: post.author || feedData.title,
					link: isAtom
						? typeof post.link === "string"
							? post.link || post.id

					<DialogHeader>
						<DialogTitle>Add Feed</DialogTitle>
						<DialogDescription>
							Add a new feed with the RSS URL
							Enter a website URL or direct RSS feed URL
						</DialogDescription>
					</DialogHeader>
					<div className="grid gap-4">

								name="url"
								value={urlInput}
								onChange={(e) => setUrlInput(e.target.value)}
								placeholder="https://example.com"
							/>
							<p className="text-xs text-muted-foreground">
								We'll automatically discover the RSS feed for you
							</p>
						</div>
						<div className="grid gap-3">
							<Label htmlFor="category-input">Category</Label>

src/lib/feed-discovery.ts (added) +18 −0

1	+	/**
2	+	* Discovers RSS/Atom feed URLs by testing common endpoint patterns
3	+	*/
4	+
5	+	export const COMMON_FEED_PATHS = [
6	+	"/feed",
7	+	"/feed.xml",
8	+	"/rss",
9	+	"/rss.xml",
10	+	"/atom.xml",
11	+	"/feed/",
12	+	"/rss/",
13	+	"/index.xml",
14	+	"/feeds/posts/default",
15	+	"/blog/feed",
16	+	"/blog/rss",
17	+	"/blog/feed.xml",
18	+	];

49	49		} from "@/lib/evolu";
50	50		import { XMLParser } from "fast-xml-parser";
51	51		import { useQuery } from "@evolu/react";
	52	+	import { COMMON_FEED_PATHS } from "@/lib/feed-discovery";
52	53		const parser = new XMLParser();
53	54
54	55		// This is sample data

98	99
99	100		async function addFeed() {
100	101		try {
101		-	let xmlData: string;
102		-	try {
103		-	// Try to fetch directly first
104		-	const xmlFetch = await fetch(urlInput);
105		-	xmlData = await xmlFetch.text();
106		-	} catch (corsError) {
107		-	// Fall back to AllOrigins if CORS error occurs
108		-	console.log(corsError);
109		-	const xmlFetch = await fetch(
110		-	`https://api.allorigins.win/raw?url=${urlInput}`,
111		-	);
112		-	xmlData = await xmlFetch.text();
	102	+	// Try to discover feeds if the URL doesn't look like a direct feed URL
	103	+	let feedUrl = urlInput;
	104	+	const looksLikeFeedUrl =
	105	+	urlInput.includes("/feed") \|\|
	106	+	urlInput.includes("/rss") \|\|
	107	+	urlInput.includes(".xml") \|\|
	108	+	urlInput.includes("/atom");
	109	+
	110	+	let xmlData: string \| null = null;
	111	+
	112	+	if (!looksLikeFeedUrl) {
	113	+	// Try common feed paths using CORS proxy
	114	+	const urlObj = new URL(urlInput);
	115	+	const origin = urlObj.origin;
	116	+
	117	+	console.log("Trying to discover feed from:", origin);
	118	+
	119	+	for (const path of COMMON_FEED_PATHS) {
	120	+	const testUrl = `${origin}${path}`;
	121	+	console.log("Testing:", testUrl);
	122	+
	123	+	try {
	124	+	// Use CORS proxy to avoid CORS issues
	125	+	const response = await fetch(
	126	+	`https://corsproxy.io/?url=${encodeURIComponent(testUrl)}`,
	127	+	);
	128	+
	129	+	if (response.ok) {
	130	+	const text = await response.text();
	131	+	// Quick check if it looks like XML
	132	+	if (
	133	+	text.trim().startsWith("<?xml") \|\|
	134	+	text.includes("<rss") \|\|
	135	+	text.includes("<feed")
	136	+	) {
	137	+	xmlData = text;
	138	+	feedUrl = testUrl;
	139	+	console.log("Found feed at:", testUrl);
	140	+	break;
	141	+	}
	142	+	}
	143	+	} catch (error) {
	144	+	console.log("Failed to fetch:", testUrl, error);
	145	+	continue;
	146	+	}
	147	+	}
	148	+
	149	+	if (!xmlData) {
	150	+	alert(
	151	+	"Could not find an RSS feed at this URL. Please enter a direct feed URL.",
	152	+	);
	153	+	return;
	154	+	}
	155	+	} else {
	156	+	// Direct feed URL - try to fetch it
	157	+	try {
	158	+	// Try to fetch directly first
	159	+	const xmlFetch = await fetch(feedUrl);
	160	+	console.log("Status code: ", xmlFetch.status);
	161	+	console.log("Request ok: ", xmlFetch.ok);
	162	+	xmlData = await xmlFetch.text();
	163	+	} catch (corsError) {
	164	+	// Fall back to AllOrigins if CORS error occurs
	165	+	console.log(corsError);
	166	+	const xmlFetch = await fetch(
	167	+	`https://api.allorigins.win/raw?url=${feedUrl}`,
	168	+	);
	169	+	xmlData = await xmlFetch.text();
	170	+	}
113	171		}
114	172		const parsedXmlData = await parser.parse(xmlData);
115	173		console.log(parsedXmlData);

133	191		}
134	192
135	193		const result = insert("rssFeed", {
136		-	feedUrl: urlInput,
	194	+	feedUrl: feedUrl,
137	195		title: feedData.title,
138	196		description: feedData.description \|\| feedData.subtitle \|\| "",
139	197		category: categoryInput \|\| "Uncategorized",

144	202		insert("rssPost", {
145	203		title: post.title,
146	204		author: isAtom
147		-	? post.author?.name \|\| "Author"
148		-	: post.author \|\| "Author",
	205	+	? post.author?.name \|\| feedData.title
	206	+	: post.author \|\| feedData.title,
149	207		link: isAtom
150	208		? typeof post.link === "string"
151	209		? post.link \|\| post.id

221	279		<DialogHeader>
222	280		<DialogTitle>Add Feed</DialogTitle>
223	281		<DialogDescription>
224		-	Add a new feed with the RSS URL
	282	+	Enter a website URL or direct RSS feed URL
225	283		</DialogDescription>
226	284		</DialogHeader>
227	285		<div className="grid gap-4">

232	290		name="url"
233	291		value={urlInput}
234	292		onChange={(e) => setUrlInput(e.target.value)}
	293	+	placeholder="https://example.com"
235	294		/>
	295	+	<p className="text-xs text-muted-foreground">
	296	+	We'll automatically discover the RSS feed for you
	297	+	</p>
236	298		</div>
237	299		<div className="grid gap-3">
238	300		<Label htmlFor="category-input">Category</Label>

4	4
5	5		- [ ] Add `publishedDate` to `rssPost`
6	6		- [ ] Order posts by newest first
7		-	- [ ] Instead of "by author" if Author is unavailable, use blog/feed name
8		-	- [ ] Add auto-find RSS feed when adding a feed
	7	+	- [x] Instead of "by author" if Author is unavailable, use blog/feed name
	8	+	- [x] Add auto-find RSS feed when adding a feed
	9	+	- [ ] When scrolling through one feed and then selecting another, make the content scroll back to the top
	10	+	- [ ] Can't open add feed dialog when on mobile
9	11		- [ ] Add error handling and checks for adding a feed
10	12		- [ ] Add loading states for new feed
11	13		- [ ] Find a way to paginate through feeds and get more posts if they only return a few