feat: added new jetstream-consumer package
56d56bf9
Replaces need for tap to index documents
10 file(s) · +427 −2
| 21 | 21 | "vite": "^5.0.0", |
|
| 22 | 22 | }, |
|
| 23 | 23 | }, |
|
| 24 | + | "packages/jetstream-consumer": { |
|
| 25 | + | "name": "jetstream-consumer", |
|
| 26 | + | "version": "1.0.0", |
|
| 27 | + | "devDependencies": { |
|
| 28 | + | "@types/bun": "^1.3.9", |
|
| 29 | + | "@types/node": "^25.3.0", |
|
| 30 | + | }, |
|
| 31 | + | "peerDependencies": { |
|
| 32 | + | "typescript": "^5", |
|
| 33 | + | }, |
|
| 34 | + | }, |
|
| 24 | 35 | "packages/server": { |
|
| 25 | 36 | "name": "@document-feeds/server", |
|
| 26 | 37 | "version": "1.0.0", |
|
| 257 | 268 | ||
| 258 | 269 | "@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="], |
|
| 259 | 270 | ||
| 271 | + | "@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="], |
|
| 272 | + | ||
| 260 | 273 | "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], |
|
| 261 | 274 | ||
| 262 | - | "@types/node": ["@types/node@25.0.6", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-NNu0sjyNxpoiW3YuVFfNz7mxSQ+S4X2G28uqg2s+CzoqoQjLPsWSbsFFyztIAqt2vb8kfEAsJNepMGPTxFDx3Q=="], |
|
| 275 | + | "@types/node": ["@types/node@25.3.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-4K3bqJpXpqfg2XKGK9bpDTc6xO/xoUP/RBWS7AtRMug6zZFaRekiLzjVtAoZMquxoAbzBvy5nxQ7veS5eYzf8A=="], |
|
| 263 | 276 | ||
| 264 | 277 | "@types/prop-types": ["@types/prop-types@15.7.15", "", {}, "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw=="], |
|
| 265 | 278 | ||
| 280 | 293 | "blake3-wasm": ["blake3-wasm@2.1.5", "", {}, "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g=="], |
|
| 281 | 294 | ||
| 282 | 295 | "browserslist": ["browserslist@4.28.1", "", { "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", "electron-to-chromium": "^1.5.263", "node-releases": "^2.0.27", "update-browserslist-db": "^1.2.0" }, "bin": { "browserslist": "cli.js" } }, "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA=="], |
|
| 296 | + | ||
| 297 | + | "bun-types": ["bun-types@1.3.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="], |
|
| 283 | 298 | ||
| 284 | 299 | "caniuse-lite": ["caniuse-lite@1.0.30001764", "", {}, "sha512-9JGuzl2M+vPL+pz70gtMF9sHdMFbY9FJaQBi186cHKH3pSzDvzoUJUPV6fqiKIMyXbud9ZLg4F3Yza1vJ1+93g=="], |
|
| 285 | 300 | ||
| 333 | 348 | ||
| 334 | 349 | "is-arrayish": ["is-arrayish@0.3.4", "", {}, "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA=="], |
|
| 335 | 350 | ||
| 351 | + | "jetstream-consumer": ["jetstream-consumer@workspace:packages/jetstream-consumer"], |
|
| 352 | + | ||
| 336 | 353 | "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], |
|
| 337 | 354 | ||
| 338 | 355 | "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], |
|
| 411 | 428 | ||
| 412 | 429 | "undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="], |
|
| 413 | 430 | ||
| 414 | - | "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], |
|
| 431 | + | "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], |
|
| 415 | 432 | ||
| 416 | 433 | "unenv": ["unenv@2.0.0-rc.14", "", { "dependencies": { "defu": "^6.1.4", "exsolve": "^1.0.1", "ohash": "^2.0.10", "pathe": "^2.0.3", "ufo": "^1.5.4" } }, "sha512-od496pShMen7nOy5VmVJCnq8rptd45vh6Nx/r2iPbrba6pa6p+tS2ywuIHRZ/OBvSbQZB0kWvpO9XBNVFXHD3Q=="], |
|
| 417 | 434 | ||
| 437 | 454 | ||
| 438 | 455 | "@cspotcode/source-map-support/@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.9", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" } }, "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ=="], |
|
| 439 | 456 | ||
| 457 | + | "bun-types/@types/node": ["@types/node@25.0.6", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-NNu0sjyNxpoiW3YuVFfNz7mxSQ+S4X2G28uqg2s+CzoqoQjLPsWSbsFFyztIAqt2vb8kfEAsJNepMGPTxFDx3Q=="], |
|
| 458 | + | ||
| 440 | 459 | "sharp/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], |
|
| 441 | 460 | ||
| 442 | 461 | "wrangler/esbuild": ["esbuild@0.17.19", "", { "optionalDependencies": { "@esbuild/android-arm": "0.17.19", "@esbuild/android-arm64": "0.17.19", "@esbuild/android-x64": "0.17.19", "@esbuild/darwin-arm64": "0.17.19", "@esbuild/darwin-x64": "0.17.19", "@esbuild/freebsd-arm64": "0.17.19", "@esbuild/freebsd-x64": "0.17.19", "@esbuild/linux-arm": "0.17.19", "@esbuild/linux-arm64": "0.17.19", "@esbuild/linux-ia32": "0.17.19", "@esbuild/linux-loong64": "0.17.19", "@esbuild/linux-mips64el": "0.17.19", "@esbuild/linux-ppc64": "0.17.19", "@esbuild/linux-riscv64": "0.17.19", "@esbuild/linux-s390x": "0.17.19", "@esbuild/linux-x64": "0.17.19", "@esbuild/netbsd-x64": "0.17.19", "@esbuild/openbsd-x64": "0.17.19", "@esbuild/sunos-x64": "0.17.19", "@esbuild/win32-arm64": "0.17.19", "@esbuild/win32-ia32": "0.17.19", "@esbuild/win32-x64": "0.17.19" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw=="], |
|
| 462 | + | ||
| 463 | + | "bun-types/@types/node/undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], |
|
| 443 | 464 | ||
| 444 | 465 | "wrangler/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.17.19", "", { "os": "android", "cpu": "arm" }, "sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A=="], |
|
| 445 | 466 | ||
| 1 | + | # dependencies (bun install) |
|
| 2 | + | node_modules |
|
| 3 | + | ||
| 4 | + | # output |
|
| 5 | + | out |
|
| 6 | + | dist |
|
| 7 | + | *.tgz |
|
| 8 | + | ||
| 9 | + | # code coverage |
|
| 10 | + | coverage |
|
| 11 | + | *.lcov |
|
| 12 | + | ||
| 13 | + | # logs |
|
| 14 | + | logs |
|
| 15 | + | _.log |
|
| 16 | + | report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json |
|
| 17 | + | ||
| 18 | + | # dotenv environment variable files |
|
| 19 | + | .env |
|
| 20 | + | .env.development.local |
|
| 21 | + | .env.test.local |
|
| 22 | + | .env.production.local |
|
| 23 | + | .env.local |
|
| 24 | + | ||
| 25 | + | # caches |
|
| 26 | + | .eslintcache |
|
| 27 | + | .cache |
|
| 28 | + | *.tsbuildinfo |
|
| 29 | + | ||
| 30 | + | # IntelliJ based IDEs |
|
| 31 | + | .idea |
|
| 32 | + | ||
| 33 | + | # Finder (MacOS) folder config |
|
| 34 | + | .DS_Store |
| 1 | + | ||
| 2 | + | Default to using Bun instead of Node.js. |
|
| 3 | + | ||
| 4 | + | - Use `bun <file>` instead of `node <file>` or `ts-node <file>` |
|
| 5 | + | - Use `bun test` instead of `jest` or `vitest` |
|
| 6 | + | - Use `bun build <file.html|file.ts|file.css>` instead of `webpack` or `esbuild` |
|
| 7 | + | - Use `bun install` instead of `npm install` or `yarn install` or `pnpm install` |
|
| 8 | + | - Use `bun run <script>` instead of `npm run <script>` or `yarn run <script>` or `pnpm run <script>` |
|
| 9 | + | - Use `bunx <package> <command>` instead of `npx <package> <command>` |
|
| 10 | + | - Bun automatically loads .env, so don't use dotenv. |
|
| 11 | + | ||
| 12 | + | ## APIs |
|
| 13 | + | ||
| 14 | + | - `Bun.serve()` supports WebSockets, HTTPS, and routes. Don't use `express`. |
|
| 15 | + | - `bun:sqlite` for SQLite. Don't use `better-sqlite3`. |
|
| 16 | + | - `Bun.redis` for Redis. Don't use `ioredis`. |
|
| 17 | + | - `Bun.sql` for Postgres. Don't use `pg` or `postgres.js`. |
|
| 18 | + | - `WebSocket` is built-in. Don't use `ws`. |
|
| 19 | + | - Prefer `Bun.file` over `node:fs`'s readFile/writeFile |
|
| 20 | + | - Bun.$`ls` instead of execa. |
|
| 21 | + | ||
| 22 | + | ## Testing |
|
| 23 | + | ||
| 24 | + | Use `bun test` to run tests. |
|
| 25 | + | ||
| 26 | + | ```ts#index.test.ts |
|
| 27 | + | import { test, expect } from "bun:test"; |
|
| 28 | + | ||
| 29 | + | test("hello world", () => { |
|
| 30 | + | expect(1).toBe(1); |
|
| 31 | + | }); |
|
| 32 | + | ``` |
|
| 33 | + | ||
| 34 | + | ## Frontend |
|
| 35 | + | ||
| 36 | + | Use HTML imports with `Bun.serve()`. Don't use `vite`. HTML imports fully support React, CSS, Tailwind. |
|
| 37 | + | ||
| 38 | + | Server: |
|
| 39 | + | ||
| 40 | + | ```ts#index.ts |
|
| 41 | + | import index from "./index.html" |
|
| 42 | + | ||
| 43 | + | Bun.serve({ |
|
| 44 | + | routes: { |
|
| 45 | + | "/": index, |
|
| 46 | + | "/api/users/:id": { |
|
| 47 | + | GET: (req) => { |
|
| 48 | + | return new Response(JSON.stringify({ id: req.params.id })); |
|
| 49 | + | }, |
|
| 50 | + | }, |
|
| 51 | + | }, |
|
| 52 | + | // optional websocket support |
|
| 53 | + | websocket: { |
|
| 54 | + | open: (ws) => { |
|
| 55 | + | ws.send("Hello, world!"); |
|
| 56 | + | }, |
|
| 57 | + | message: (ws, message) => { |
|
| 58 | + | ws.send(message); |
|
| 59 | + | }, |
|
| 60 | + | close: (ws) => { |
|
| 61 | + | // handle close |
|
| 62 | + | } |
|
| 63 | + | }, |
|
| 64 | + | development: { |
|
| 65 | + | hmr: true, |
|
| 66 | + | console: true, |
|
| 67 | + | } |
|
| 68 | + | }) |
|
| 69 | + | ``` |
|
| 70 | + | ||
| 71 | + | HTML files can import .tsx, .jsx or .js files directly and Bun's bundler will transpile & bundle automatically. `<link>` tags can point to stylesheets and Bun's CSS bundler will bundle. |
|
| 72 | + | ||
| 73 | + | ```html#index.html |
|
| 74 | + | <html> |
|
| 75 | + | <body> |
|
| 76 | + | <h1>Hello, world!</h1> |
|
| 77 | + | <script type="module" src="./frontend.tsx"></script> |
|
| 78 | + | </body> |
|
| 79 | + | </html> |
|
| 80 | + | ``` |
|
| 81 | + | ||
| 82 | + | With the following `frontend.tsx`: |
|
| 83 | + | ||
| 84 | + | ```tsx#frontend.tsx |
|
| 85 | + | import React from "react"; |
|
| 86 | + | import { createRoot } from "react-dom/client"; |
|
| 87 | + | ||
| 88 | + | // import .css files directly and it works |
|
| 89 | + | import './index.css'; |
|
| 90 | + | ||
| 91 | + | const root = createRoot(document.body); |
|
| 92 | + | ||
| 93 | + | export default function Frontend() { |
|
| 94 | + | return <h1>Hello, world!</h1>; |
|
| 95 | + | } |
|
| 96 | + | ||
| 97 | + | root.render(<Frontend />); |
|
| 98 | + | ``` |
|
| 99 | + | ||
| 100 | + | Then, run index.ts |
|
| 101 | + | ||
| 102 | + | ```sh |
|
| 103 | + | bun --hot ./index.ts |
|
| 104 | + | ``` |
|
| 105 | + | ||
| 106 | + | For more information, read the Bun API docs in `node_modules/bun-types/docs/**.mdx`. |
| 1 | + | FROM oven/bun:latest |
|
| 2 | + | ||
| 3 | + | WORKDIR /app |
|
| 4 | + | ||
| 5 | + | COPY package.json bun.lockb* ./ |
|
| 6 | + | RUN bun install --frozen-lockfile || bun install |
|
| 7 | + | ||
| 8 | + | COPY src ./src |
|
| 9 | + | COPY tsconfig.json ./ |
|
| 10 | + | ||
| 11 | + | RUN mkdir -p /app/data |
|
| 12 | + | ||
| 13 | + | CMD ["bun", "run", "src/index.ts"] |
| 1 | + | # jetstream-consumer |
|
| 2 | + | ||
| 3 | + | To install dependencies: |
|
| 4 | + | ||
| 5 | + | ```bash |
|
| 6 | + | bun install |
|
| 7 | + | ``` |
|
| 8 | + | ||
| 9 | + | To run: |
|
| 10 | + | ||
| 11 | + | ```bash |
|
| 12 | + | bun run src/index.ts |
|
| 13 | + | ``` |
|
| 14 | + | ||
| 15 | + | This project was created using `bun init` in bun v1.3.5. [Bun](https://bun.com) is a fast all-in-one JavaScript runtime. |
| 1 | + | 1772039237880654 |
| 1 | + | services: |
|
| 2 | + | jetstream-consumer: |
|
| 3 | + | build: . |
|
| 4 | + | environment: |
|
| 5 | + | - WEBHOOK_URL=https://example.com/webhook/tap/batch |
|
| 6 | + | - WEBHOOK_SECRET= |
|
| 7 | + | - JETSTREAM_INSTANCES=jetstream1.us-east.bsky.network,jetstream2.us-east.bsky.network,jetstream1.us-west.bsky.network,jetstream2.us-west.bsky.network |
|
| 8 | + | - WANTED_COLLECTIONS=site.standard.document |
|
| 9 | + | - CURSOR_FILE=/app/data/cursor.txt |
|
| 10 | + | - BATCH_INTERVAL_MS=5000 |
|
| 11 | + | - INACTIVITY_TIMEOUT_MS=300000 |
|
| 12 | + | volumes: |
|
| 13 | + | - ./data:/app/data |
|
| 14 | + | restart: unless-stopped |
| 1 | + | { |
|
| 2 | + | "name": "jetstream-consumer", |
|
| 3 | + | "version": "1.0.0", |
|
| 4 | + | "private": true, |
|
| 5 | + | "scripts": { |
|
| 6 | + | "start": "bun run src/index.ts", |
|
| 7 | + | "dev": "bun run src/index.ts" |
|
| 8 | + | }, |
|
| 9 | + | "devDependencies": { |
|
| 10 | + | "@types/bun": "^1.3.9", |
|
| 11 | + | "@types/node": "^25.3.0" |
|
| 12 | + | }, |
|
| 13 | + | "module": "src/index.ts", |
|
| 14 | + | "type": "module", |
|
| 15 | + | "peerDependencies": { |
|
| 16 | + | "typescript": "^5" |
|
| 17 | + | } |
|
| 18 | + | } |
| 1 | + | export {}; |
|
| 2 | + | ||
| 3 | + | const WEBHOOK_URL = process.env.WEBHOOK_URL; |
|
| 4 | + | const WEBHOOK_SECRET = process.env.WEBHOOK_SECRET; |
|
| 5 | + | const JETSTREAM_INSTANCES = ( |
|
| 6 | + | process.env.JETSTREAM_INSTANCES ?? |
|
| 7 | + | "jetstream1.us-east.bsky.network,jetstream2.us-east.bsky.network,jetstream1.us-west.bsky.network,jetstream2.us-west.bsky.network" |
|
| 8 | + | ) |
|
| 9 | + | .split(",") |
|
| 10 | + | .map((h) => h.trim()); |
|
| 11 | + | const WANTED_COLLECTIONS = ( |
|
| 12 | + | process.env.WANTED_COLLECTIONS ?? "site.standard.document" |
|
| 13 | + | ) |
|
| 14 | + | .split(",") |
|
| 15 | + | .map((c) => c.trim()); |
|
| 16 | + | const CURSOR_FILE = process.env.CURSOR_FILE ?? "./cursor.txt"; |
|
| 17 | + | const BATCH_INTERVAL_MS = Number(process.env.BATCH_INTERVAL_MS) || 5000; |
|
| 18 | + | const INACTIVITY_TIMEOUT_MS = |
|
| 19 | + | Number(process.env.INACTIVITY_TIMEOUT_MS) || 300_000; |
|
| 20 | + | ||
| 21 | + | const DEFAULT_CURSOR = "1772036746"; |
|
| 22 | + | ||
| 23 | + | if (!WEBHOOK_URL) { |
|
| 24 | + | console.error("WEBHOOK_URL environment variable is required"); |
|
| 25 | + | process.exit(1); |
|
| 26 | + | } |
|
| 27 | + | ||
| 28 | + | let cursor = DEFAULT_CURSOR; |
|
| 29 | + | let currentInstanceIndex = 0; |
|
| 30 | + | let ws: WebSocket | null = null; |
|
| 31 | + | let lastMessageTime = Date.now(); |
|
| 32 | + | let batch: Array<{ |
|
| 33 | + | type: string; |
|
| 34 | + | did: string; |
|
| 35 | + | collection: string; |
|
| 36 | + | rkey: string; |
|
| 37 | + | cid: string; |
|
| 38 | + | record?: Record<string, unknown>; |
|
| 39 | + | }> = []; |
|
| 40 | + | ||
| 41 | + | async function loadCursor(): Promise<void> { |
|
| 42 | + | try { |
|
| 43 | + | const file = Bun.file(CURSOR_FILE); |
|
| 44 | + | const text = await file.text(); |
|
| 45 | + | const trimmed = text.trim(); |
|
| 46 | + | if (trimmed) { |
|
| 47 | + | cursor = trimmed; |
|
| 48 | + | console.log(`Loaded cursor from file: ${cursor}`); |
|
| 49 | + | } |
|
| 50 | + | } catch { |
|
| 51 | + | console.log(`No cursor file found, using default: ${cursor}`); |
|
| 52 | + | } |
|
| 53 | + | } |
|
| 54 | + | ||
| 55 | + | async function saveCursor(): Promise<void> { |
|
| 56 | + | try { |
|
| 57 | + | await Bun.write(CURSOR_FILE, cursor); |
|
| 58 | + | } catch (err) { |
|
| 59 | + | console.error("Failed to save cursor:", err); |
|
| 60 | + | } |
|
| 61 | + | } |
|
| 62 | + | ||
| 63 | + | async function flushBatch(): Promise<void> { |
|
| 64 | + | if (batch.length === 0) return; |
|
| 65 | + | ||
| 66 | + | const events = batch.splice(0, batch.length); |
|
| 67 | + | console.log(`Flushing batch of ${events.length} events`); |
|
| 68 | + | ||
| 69 | + | try { |
|
| 70 | + | const headers: Record<string, string> = { |
|
| 71 | + | "Content-Type": "application/json", |
|
| 72 | + | }; |
|
| 73 | + | if (WEBHOOK_SECRET) { |
|
| 74 | + | headers["Authorization"] = `Bearer ${WEBHOOK_SECRET}`; |
|
| 75 | + | } |
|
| 76 | + | ||
| 77 | + | const res = await fetch(WEBHOOK_URL!, { |
|
| 78 | + | method: "POST", |
|
| 79 | + | headers, |
|
| 80 | + | body: JSON.stringify(events), |
|
| 81 | + | }); |
|
| 82 | + | ||
| 83 | + | if (!res.ok) { |
|
| 84 | + | console.error( |
|
| 85 | + | `Webhook responded with ${res.status}: ${await res.text()}`, |
|
| 86 | + | ); |
|
| 87 | + | return; |
|
| 88 | + | } |
|
| 89 | + | ||
| 90 | + | const result = (await res.json()) as Record<string, unknown>; |
|
| 91 | + | console.log("Batch result:", JSON.stringify(result)); |
|
| 92 | + | await saveCursor(); |
|
| 93 | + | } catch (err) { |
|
| 94 | + | console.error("Failed to send batch:", err); |
|
| 95 | + | } |
|
| 96 | + | } |
|
| 97 | + | ||
| 98 | + | function connect(): void { |
|
| 99 | + | const host = JETSTREAM_INSTANCES[currentInstanceIndex]; |
|
| 100 | + | const collections = WANTED_COLLECTIONS.map( |
|
| 101 | + | (c) => `wantedCollections=${c}`, |
|
| 102 | + | ).join("&"); |
|
| 103 | + | const url = `wss://${host}/subscribe?${collections}&cursor=${cursor}`; |
|
| 104 | + | ||
| 105 | + | console.log(`Connecting to ${host} with cursor ${cursor}`); |
|
| 106 | + | ||
| 107 | + | ws = new WebSocket(url); |
|
| 108 | + | ||
| 109 | + | ws.addEventListener("open", () => { |
|
| 110 | + | console.log(`Connected to ${host}`); |
|
| 111 | + | lastMessageTime = Date.now(); |
|
| 112 | + | }); |
|
| 113 | + | ||
| 114 | + | ws.addEventListener("message", (event) => { |
|
| 115 | + | lastMessageTime = Date.now(); |
|
| 116 | + | ||
| 117 | + | try { |
|
| 118 | + | const data = JSON.parse(String(event.data)); |
|
| 119 | + | ||
| 120 | + | if (data.kind !== "commit") return; |
|
| 121 | + | ||
| 122 | + | cursor = String(data.time_us); |
|
| 123 | + | ||
| 124 | + | batch.push({ |
|
| 125 | + | type: data.commit.operation, |
|
| 126 | + | did: data.did, |
|
| 127 | + | collection: data.commit.collection, |
|
| 128 | + | rkey: data.commit.rkey, |
|
| 129 | + | cid: data.commit.cid, |
|
| 130 | + | record: data.commit.record, |
|
| 131 | + | }); |
|
| 132 | + | } catch (err) { |
|
| 133 | + | console.error("Failed to parse message:", err); |
|
| 134 | + | } |
|
| 135 | + | }); |
|
| 136 | + | ||
| 137 | + | ws.addEventListener("close", (event) => { |
|
| 138 | + | console.log(`WebSocket closed: code=${event.code} reason=${event.reason}`); |
|
| 139 | + | }); |
|
| 140 | + | ||
| 141 | + | ws.addEventListener("error", (event) => { |
|
| 142 | + | console.error("WebSocket error:", event); |
|
| 143 | + | }); |
|
| 144 | + | } |
|
| 145 | + | ||
| 146 | + | function switchInstance(): void { |
|
| 147 | + | currentInstanceIndex = |
|
| 148 | + | (currentInstanceIndex + 1) % JETSTREAM_INSTANCES.length; |
|
| 149 | + | const newHost = JETSTREAM_INSTANCES[currentInstanceIndex]; |
|
| 150 | + | console.log(`Switching to instance: ${newHost}`); |
|
| 151 | + | ||
| 152 | + | if (ws) { |
|
| 153 | + | try { |
|
| 154 | + | ws.close(); |
|
| 155 | + | } catch { |
|
| 156 | + | // ignore close errors |
|
| 157 | + | } |
|
| 158 | + | ws = null; |
|
| 159 | + | } |
|
| 160 | + | ||
| 161 | + | connect(); |
|
| 162 | + | } |
|
| 163 | + | ||
| 164 | + | // --- Main --- |
|
| 165 | + | ||
| 166 | + | await loadCursor(); |
|
| 167 | + | connect(); |
|
| 168 | + | ||
| 169 | + | // Batch flush interval |
|
| 170 | + | setInterval(() => { |
|
| 171 | + | flushBatch(); |
|
| 172 | + | }, BATCH_INTERVAL_MS); |
|
| 173 | + | ||
| 174 | + | // Inactivity check every 30s |
|
| 175 | + | setInterval(() => { |
|
| 176 | + | const elapsed = Date.now() - lastMessageTime; |
|
| 177 | + | if (elapsed > INACTIVITY_TIMEOUT_MS) { |
|
| 178 | + | console.log( |
|
| 179 | + | `No messages for ${Math.round(elapsed / 1000)}s, triggering failover`, |
|
| 180 | + | ); |
|
| 181 | + | switchInstance(); |
|
| 182 | + | } |
|
| 183 | + | }, 30_000); |
|
| 184 | + | ||
| 185 | + | console.log("Jetstream consumer started"); |
|
| 186 | + | console.log(` Webhook URL: ${WEBHOOK_URL}`); |
|
| 187 | + | console.log(` Instances: ${JETSTREAM_INSTANCES.join(", ")}`); |
|
| 188 | + | console.log(` Collections: ${WANTED_COLLECTIONS.join(", ")}`); |
|
| 189 | + | console.log(` Batch interval: ${BATCH_INTERVAL_MS}ms`); |
|
| 190 | + | console.log(` Inactivity timeout: ${INACTIVITY_TIMEOUT_MS}ms`); |
| 1 | + | { |
|
| 2 | + | "compilerOptions": { |
|
| 3 | + | "target": "ES2022", |
|
| 4 | + | "module": "ESNext", |
|
| 5 | + | "moduleResolution": "bundler", |
|
| 6 | + | "types": ["bun-types"], |
|
| 7 | + | "strict": true, |
|
| 8 | + | "esModuleInterop": true, |
|
| 9 | + | "skipLibCheck": true, |
|
| 10 | + | "noEmit": true |
|
| 11 | + | }, |
|
| 12 | + | "include": ["src"] |
|
| 13 | + | } |