feat(degenfeed): major overhaul — v2 ranking, RMI DataBus, newsletter, RSS XML, Solana auth, spam/relevance engine
Feed Core: rewritten isSpamPost with proper regex, emoji detection, ALL-CAPS ratio; new isRelevantPost with inline niche computation; new contentFingerprint for cross-protocol dedup; rewritten deduplicatePosts with 3-layer dedup; rewritten scorePost with log-engagement, 12h half-life, ticker boost Ranking v2: scoreBreakdownV2 with 7 components (recency, engagement, relevance, quality, diversity, source, velocity); rankDiverse with per-author cap; engagementVelocity for trending detection; nicheRelevance with mute support; platformRelevance for protocol boost Feed Providers: Bluesky search-first strategy with parallel crypto queries; Farcaster removed fake curated notice; Mastodon hashtag timeline strategy with parallel fetches; Nostr #t tag filter; RSS 40+ tiered crypto feeds; new RmiDataBusProvider bridging rmi-backend 200+ source DataBus Worker API: /api/rmi/news, /api/rmi/news/headlines, /api/rmi/news/sources, /api/rmi/news/sentiment, /api/rmi/news/stats, /api/rmi/health; /api/feed/rss.xml (RSS 2.0 + Atom 1.0); /api/newsletter/subscribe, /api/newsletter/digest (JSON/HTML/plaintext), /api/newsletter/status; feed route uses rankDiverse with per-author cap; wallet route fixed base58 decoder, KV-backed nonces Auth: new useSolanaSignIn hook for Phantom/Solflare; AuthProvider wired Solana sign-in Infra: biome 1.9.0 -> 2.5.2 across all packages; pnpm-workspace allows esbuild/sharp/workerd; RMI_BACKEND_URL and RMI_API_KEY in worker Env Tests: all 16 suites pass; updated Bluesky/Farcaster provider tests; v2 ranking tests
This commit is contained in:
parent
7d9258dfba
commit
c0691757c1
45 changed files with 2447 additions and 304 deletions
|
|
@ -36,7 +36,7 @@
|
|||
"wagmi": "^3.6.21"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^1.9.4",
|
||||
"@cloudflare/next-on-pages": "^1.13.16",
|
||||
"@types/node": "^20.0.0",
|
||||
"@types/react": "^19.0.0",
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import {
|
|||
useState,
|
||||
} from 'react';
|
||||
import { useWeb3 } from '../components/Web3Provider';
|
||||
import { useSolanaSignIn } from '../hooks/useSolanaSignIn';
|
||||
import { useWalletSignIn } from '../hooks/useWalletSignIn';
|
||||
|
||||
export interface AuthContextValue {
|
||||
|
|
@ -73,6 +74,7 @@ export function AuthProvider({ children }: { children: ReactNode }) {
|
|||
|
||||
const { account: wagmiAddress, isConnected } = useWeb3();
|
||||
const { signInWithEthereum, signingIn: walletSigningIn } = useWalletSignIn();
|
||||
const { signInWithSolana, signingIn: solanaSigningIn } = useSolanaSignIn();
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
const all = await getAllIdentities();
|
||||
|
|
@ -172,7 +174,13 @@ export function AuthProvider({ children }: { children: ReactNode }) {
|
|||
|
||||
switch (protocol) {
|
||||
case 'solana': {
|
||||
throw new Error('Solana wallet sign-in is not yet supported');
|
||||
const session = await signInWithSolana();
|
||||
if (session) {
|
||||
setLinkGtsUrl(session.gtsAuthorizeUrl ?? null);
|
||||
await refresh();
|
||||
setShowSignIn(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
case 'nostr': {
|
||||
const { signInWithNostrNsec } = await import('@degenfeed/auth');
|
||||
|
|
@ -361,7 +369,7 @@ export function AuthProvider({ children }: { children: ReactNode }) {
|
|||
onClose={() => setShowSignIn(false)}
|
||||
onSignIn={handleSignIn}
|
||||
walletAddress={walletAddress}
|
||||
walletSigningIn={walletSigningIn}
|
||||
walletSigningIn={walletSigningIn || solanaSigningIn}
|
||||
/>
|
||||
|
||||
<ComposeModal
|
||||
|
|
|
|||
82
apps/web/src/hooks/useSolanaSignIn.ts
Normal file
82
apps/web/src/hooks/useSolanaSignIn.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
'use client';
|
||||
|
||||
import { buildSolanaMessage, verifyWalletSignature } from '@degenfeed/auth';
|
||||
import { saveWalletIdentity } from '@degenfeed/storage';
|
||||
import { useCallback, useState } from 'react';
|
||||
|
||||
interface SolanaWallet {
|
||||
publicKey?: { toBase58: () => string };
|
||||
connect: () => Promise<{ publicKey: { toBase58: () => string } }>;
|
||||
signMessage: (message: Uint8Array) => Promise<{ signature: Uint8Array }>;
|
||||
isConnected?: boolean;
|
||||
}
|
||||
|
||||
interface SolanaWindow {
|
||||
solana?: SolanaWallet;
|
||||
solflare?: SolanaWallet;
|
||||
}
|
||||
|
||||
function getSolanaWallet(): SolanaWallet | null {
|
||||
if (typeof window === 'undefined') return null;
|
||||
const w = window as unknown as SolanaWindow;
|
||||
return w.solana || w.solflare || null;
|
||||
}
|
||||
|
||||
export function useSolanaSignIn() {
|
||||
const [signingIn, setSigningIn] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const signInWithSolana = useCallback(async () => {
|
||||
const wallet = getSolanaWallet();
|
||||
if (!wallet) throw new Error('No Solana wallet detected. Install Phantom or Solflare.');
|
||||
|
||||
setSigningIn(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
let address = wallet.publicKey?.toBase58();
|
||||
if (!address || !wallet.isConnected) {
|
||||
const res = await wallet.connect();
|
||||
address = res.publicKey.toBase58();
|
||||
}
|
||||
if (!address) throw new Error('Could not get Solana address');
|
||||
|
||||
const nonce = await requestWalletNonce(address);
|
||||
const message = buildSolanaMessage(address, nonce);
|
||||
const messageBytes = new TextEncoder().encode(message);
|
||||
const signed = await wallet.signMessage(messageBytes);
|
||||
const signature = Array.from(signed.signature)
|
||||
.map((b) => b.toString(16).padStart(2, '0'))
|
||||
.join('');
|
||||
|
||||
const session = await verifyWalletSignature(address, signature, message, 'solana', nonce);
|
||||
await saveWalletIdentity(session.address, session.chain);
|
||||
return session;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Solana sign-in failed';
|
||||
setError(message);
|
||||
throw new Error(message);
|
||||
} finally {
|
||||
setSigningIn(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
return {
|
||||
available: typeof window !== 'undefined' && !!getSolanaWallet(),
|
||||
signingIn,
|
||||
error,
|
||||
signInWithSolana,
|
||||
};
|
||||
}
|
||||
|
||||
async function requestWalletNonce(address: string): Promise<string> {
|
||||
const res = await fetch(`/api/auth/wallet/nonce?address=${encodeURIComponent(address)}`, {
|
||||
credentials: 'include',
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({ error: 'Nonce request failed' }));
|
||||
throw new Error(err.error || 'Nonce request failed');
|
||||
}
|
||||
const data = await res.json();
|
||||
return data.nonce;
|
||||
}
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"$schema": "https://biomejs.dev/schemas/1.9.0/schema.json",
|
||||
"$schema": "https://biomejs.dev/schemas/2.5.2/schema.json",
|
||||
"vcs": { "enabled": true, "clientKind": "git", "useIgnoreFile": true },
|
||||
"files": {
|
||||
"ignoreUnknown": true,
|
||||
"ignore": ["**/node_modules/**", "**/.next/**", "**/dist/**", "**/.turbo/**", "**/coverage/**"]
|
||||
"includes": ["**"]
|
||||
},
|
||||
"formatter": {
|
||||
"enabled": true,
|
||||
|
|
@ -12,11 +12,11 @@
|
|||
"lineWidth": 100,
|
||||
"lineEnding": "lf"
|
||||
},
|
||||
"organizeImports": { "enabled": true },
|
||||
"assist": { "actions": { "source": { "organizeImports": "on" } } },
|
||||
"linter": {
|
||||
"enabled": true,
|
||||
"rules": {
|
||||
"recommended": true,
|
||||
"preset": "recommended",
|
||||
"correctness": { "noUnusedVariables": "warn", "noUnusedImports": "warn" },
|
||||
"style": { "useImportType": "warn", "useExportType": "warn" },
|
||||
"suspicious": { "noExplicitAny": "warn" },
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
"@degenfeed/mastodon-sdk": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,23 +1,24 @@
|
|||
// ─── @degenfeed/auth — barrel exports ──────────────────────────
|
||||
export {
|
||||
signInWithNostrNsec,
|
||||
signInWithNostrExtension,
|
||||
signInWithNostrRemoteSigner,
|
||||
} from './providers/nostr';
|
||||
|
||||
export type { ActiveSession, StoredIdentity } from '@degenfeed/storage';
|
||||
export type { EncryptedBlob } from './crypto';
|
||||
export { decryptSecret, encryptSecret } from './crypto';
|
||||
export { buildIdentityRecord } from './persistence';
|
||||
export { signInWithBluesky } from './providers/bluesky';
|
||||
export { signInWithFarcaster, signInWithFarcasterSigner } from './providers/farcaster';
|
||||
export { signInWithLens } from './providers/lens';
|
||||
export { signInWithBluesky } from './providers/bluesky';
|
||||
export { signInWithThreads } from './providers/threads';
|
||||
export { signInWithGtS } from './providers/mastodon';
|
||||
export { unlockIdentity, lockIdentity, forgetIdentity } from './session';
|
||||
export { encryptSecret, decryptSecret } from './crypto';
|
||||
export { buildIdentityRecord } from './persistence';
|
||||
export {
|
||||
requestWalletNonce,
|
||||
signInWithNostrExtension,
|
||||
signInWithNostrNsec,
|
||||
signInWithNostrRemoteSigner,
|
||||
} from './providers/nostr';
|
||||
export { signInWithThreads } from './providers/threads';
|
||||
export type { WalletSession } from './providers/wallet';
|
||||
export {
|
||||
buildSiweMessage,
|
||||
buildSolanaMessage,
|
||||
requestWalletNonce,
|
||||
verifyWalletSignature,
|
||||
} from './providers/wallet';
|
||||
export type { WalletSession } from './providers/wallet';
|
||||
export type { EncryptedBlob } from './crypto';
|
||||
export type { StoredIdentity, ActiveSession } from '@degenfeed/storage';
|
||||
export { forgetIdentity, lockIdentity, unlockIdentity } from './session';
|
||||
|
|
|
|||
|
|
@ -25,9 +25,7 @@ export interface Eip1193Provider {
|
|||
/**
|
||||
* Sign in with Lens via a connected wallet (EIP-1193).
|
||||
*/
|
||||
export async function signInWithLens(opts: {
|
||||
wallet: Eip1193Provider;
|
||||
}): Promise<StoredIdentity> {
|
||||
export async function signInWithLens(opts: { wallet: Eip1193Provider }): Promise<StoredIdentity> {
|
||||
// 1. Get wallet address
|
||||
const [address] = (await opts.wallet.request({
|
||||
method: 'eth_requestAccounts',
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
"@degenfeed/feed-core": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
"@degenfeed/feed-core": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
"@degenfeed/types": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -159,35 +159,243 @@ export function detectNiches(text: string): string[] {
|
|||
return Array.from(found);
|
||||
}
|
||||
|
||||
const SPAM_PATTERNS: RegExp[] = [
|
||||
/\b(make money|earn \$\d+|work from home|double your|guaranteed profit|100x guarantee)\b/gi,
|
||||
/\b(click here|link in bio|dm me|message me|join my vip|pump group|signals group)\b/gi,
|
||||
/\b(casino|lottery|gift card|free crypto|free bitcoin|airdrop scam|claim now)\b/gi,
|
||||
/\b(viagra|cialis|escort|onlyfans)\b/gi,
|
||||
/🚀{4,}|💰{4,}|🔥{4,}|🌙{4,}/g,
|
||||
];
|
||||
|
||||
const REPEATED_CHAR_RE = /(.)\1{8,}/;
|
||||
|
||||
const CRYPTO_RELEVANCE_TERMS = new Set([
|
||||
...BOOST_TERMS,
|
||||
'web3',
|
||||
'blockchain',
|
||||
'token',
|
||||
'altcoin',
|
||||
'satoshi',
|
||||
'halving',
|
||||
'ordinal',
|
||||
'inscription',
|
||||
'eip-',
|
||||
'erc-',
|
||||
'validator',
|
||||
'solidity',
|
||||
'gas fee',
|
||||
'gas',
|
||||
'restaking',
|
||||
'rollup',
|
||||
'whale',
|
||||
'wallet',
|
||||
'dex',
|
||||
'amm',
|
||||
'yield',
|
||||
'vault',
|
||||
'opensea',
|
||||
'blur',
|
||||
'mint',
|
||||
'rekt',
|
||||
'immunefi',
|
||||
'breach',
|
||||
'phishing',
|
||||
'exploit',
|
||||
'hack',
|
||||
'mainnet',
|
||||
'testnet',
|
||||
'sharding',
|
||||
'merge',
|
||||
'the merge',
|
||||
'pos',
|
||||
'pow',
|
||||
'mining',
|
||||
'miner',
|
||||
'hashrate',
|
||||
'difficulty',
|
||||
'stake',
|
||||
'staking',
|
||||
'unstake',
|
||||
'bridge',
|
||||
'bridging',
|
||||
'wrap',
|
||||
'unwrap',
|
||||
'weth',
|
||||
'wbtc',
|
||||
'tvl',
|
||||
'total value locked',
|
||||
'aave',
|
||||
'compound',
|
||||
'maker',
|
||||
'makerdao',
|
||||
'curve',
|
||||
'convex',
|
||||
'gmx',
|
||||
'gmxv2',
|
||||
'dydx',
|
||||
'perpetual',
|
||||
'perp',
|
||||
'futures',
|
||||
'spot',
|
||||
'orderbook',
|
||||
'amm',
|
||||
'l2beat',
|
||||
'celestia',
|
||||
'eigenlayer',
|
||||
'symbiotic',
|
||||
'karak',
|
||||
'manta',
|
||||
'blast',
|
||||
'mode',
|
||||
'zora',
|
||||
'linea',
|
||||
'scroll',
|
||||
'zksync',
|
||||
'starknet',
|
||||
'monad',
|
||||
'berachain',
|
||||
'monad testnet',
|
||||
'aptos',
|
||||
'sui',
|
||||
'sei',
|
||||
'injective',
|
||||
'near',
|
||||
'cosmos',
|
||||
'osmosis',
|
||||
'celestia',
|
||||
'pendle',
|
||||
'ethena',
|
||||
'ena',
|
||||
'usde',
|
||||
'pyusd',
|
||||
]);
|
||||
|
||||
export function isSpamPost(post: UnifiedPost): boolean {
|
||||
const text = post.text.trim();
|
||||
if (text.length < 10) return true;
|
||||
if (text.length < 12) return true;
|
||||
if (/^[0-9a-f]{64}$/i.test(text)) return true;
|
||||
if (/^\s*(test|testing|hello world|asdf|qwerty|lorem ipsum)\s*$/i.test(text)) return true;
|
||||
if (/^[\s\p{Emoji}\p{Extended_Pictographic}]+$/u.test(text)) return true;
|
||||
const lower = text.toLowerCase();
|
||||
let spamHits = 0;
|
||||
for (const re of SPAM_PATTERNS) {
|
||||
re.lastIndex = 0;
|
||||
if (re.test(lower)) spamHits++;
|
||||
}
|
||||
if (spamHits >= 2) return true;
|
||||
if (REPEATED_CHAR_RE.test(text)) return true;
|
||||
const urlCount = (text.match(/https?:\/\//g) ?? []).length;
|
||||
if (urlCount >= 4) return true;
|
||||
const upperRatio = (text.match(/[A-Z]/g) ?? []).length / Math.max(text.length, 1);
|
||||
if (text.length >= 20 && upperRatio > 0.7) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function deduplicatePosts(posts: UnifiedPost[]): UnifiedPost[] {
|
||||
const seenText = new Set<string>();
|
||||
const seenId = new Set<string>();
|
||||
return posts.filter((p) => {
|
||||
if (seenId.has(p.id)) return false;
|
||||
seenId.add(p.id);
|
||||
const key = p.text.trim().slice(0, 80).toLowerCase();
|
||||
if (key.length > 0 && seenText.has(key)) return false;
|
||||
if (key.length > 0) seenText.add(key);
|
||||
function gatherRelevanceText(post: UnifiedPost): string {
|
||||
const parts: string[] = [post.text, post.html ?? '', post.author.displayName, post.author.bio];
|
||||
for (const n of post.niches) parts.push(n);
|
||||
for (const e of post.embeds) {
|
||||
parts.push(e.url);
|
||||
const meta = e.meta;
|
||||
if (meta) {
|
||||
if (typeof meta.title === 'string') parts.push(meta.title);
|
||||
if (typeof meta.description === 'string') parts.push(meta.description);
|
||||
}
|
||||
}
|
||||
return parts.join(' ').toLowerCase();
|
||||
}
|
||||
|
||||
export function isRelevantPost(post: UnifiedPost): boolean {
|
||||
// Compute niches inline so we don't depend on the caller having run
|
||||
// enrichPostNiches first. This makes the function safe to call on
|
||||
// a fresh post straight out of a provider.
|
||||
const computedNiches =
|
||||
post.niches.length > 0
|
||||
? post.niches
|
||||
: detectNiches(
|
||||
`${post.text} ${post.html ?? ''} ${post.author.displayName} ${post.author.bio}`,
|
||||
);
|
||||
|
||||
const text = gatherRelevanceText({ ...post, niches: computedNiches });
|
||||
|
||||
if (computedNiches.length > 0) {
|
||||
// Even with niches, require at least one match to a core crypto signal
|
||||
// so the feed doesn't get polluted by miscategorized noise.
|
||||
const hasCoreSignal =
|
||||
/\b(crypto|bitcoin|btc|ethereum|eth|solana|sol|defi|nft|web3|blockchain|altcoin|stablecoin|l2|rollup|etf|macro|regulation|security|memecoin|ai|onchain|token)\b/.test(
|
||||
text,
|
||||
);
|
||||
if (hasCoreSignal) return true;
|
||||
}
|
||||
|
||||
if (/\$[A-Z]{2,5}\b/.test(text)) return true;
|
||||
if (
|
||||
/#[a-zA-Z0-9_]+/.test(text) &&
|
||||
/\b(crypto|bitcoin|ethereum|solana|defi|nft|web3|blockchain)\b/.test(text)
|
||||
) {
|
||||
return true;
|
||||
});
|
||||
}
|
||||
for (const term of CRYPTO_RELEVANCE_TERMS) {
|
||||
if (text.includes(term)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function contentFingerprint(post: UnifiedPost): string {
|
||||
// Build a 12-word shingle from text — used to detect near-duplicate
|
||||
// reposts across protocols (e.g. someone cross-posting the same content
|
||||
// to Farcaster, Lens, and Nostr).
|
||||
const text = post.text
|
||||
.toLowerCase()
|
||||
.replace(/https?:\/\/\S+/g, '')
|
||||
.replace(/[^a-z0-9\s]/g, ' ')
|
||||
.split(/\s+/)
|
||||
.filter((w) => w.length > 2)
|
||||
.slice(0, 12)
|
||||
.join(' ');
|
||||
if (text.length > 0) return text;
|
||||
// Fallback to link-based fingerprint for media-only posts
|
||||
const link = post.embeds.find(
|
||||
(e) => e.kind === 'link' || e.kind === 'image' || e.kind === 'video',
|
||||
);
|
||||
return link ? `link:${link.url}` : '';
|
||||
}
|
||||
|
||||
export function deduplicatePosts(posts: UnifiedPost[]): UnifiedPost[] {
|
||||
const seenId = new Set<string>();
|
||||
const seenFingerprint = new Set<string>();
|
||||
const seenText = new Set<string>();
|
||||
const out: UnifiedPost[] = [];
|
||||
for (const p of posts) {
|
||||
if (seenId.has(p.id)) continue;
|
||||
const fp = contentFingerprint(p);
|
||||
if (fp && seenFingerprint.has(fp)) continue;
|
||||
const textKey = p.text.trim().slice(0, 80).toLowerCase();
|
||||
if (textKey.length > 12 && seenText.has(textKey)) continue;
|
||||
seenId.add(p.id);
|
||||
if (fp) seenFingerprint.add(fp);
|
||||
if (textKey.length > 12) seenText.add(textKey);
|
||||
out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function scorePost(post: UnifiedPost, now = Date.now()): number {
|
||||
const m = post.metrics;
|
||||
const engagement = m.likes + m.reposts * 2 + m.replies * 3 + m.quotes * 4;
|
||||
// Weighted engagement: replies/quotes are 3-4x as valuable as likes
|
||||
// because they signal conversation vs passive approval.
|
||||
const engagementRaw = m.likes + m.reposts * 2 + m.replies * 3 + m.quotes * 4;
|
||||
const engagement = Math.log(1 + engagementRaw);
|
||||
// 12-hour half-life — most crypto content has a useful half-life
|
||||
// under a day; we want fresh stuff rising and old stuff decaying.
|
||||
const ageHrs = Math.max(0, (now - post.createdAt) / 3600000);
|
||||
const recency = Math.exp(-ageHrs / 24);
|
||||
const nicheBoost = post.niches.length > 0 ? 1 + post.niches.length * 0.05 : 1;
|
||||
return engagement * recency * nicheBoost;
|
||||
const recency = 0.5 ** (ageHrs / 12);
|
||||
// Niche boost — only meaningful if the post actually has niches
|
||||
const nicheBoost = post.niches.length > 0 ? 1 + Math.min(post.niches.length, 5) * 0.08 : 1;
|
||||
// Ticker boost — posts mentioning $TICKER are time-sensitive signals
|
||||
const tickerCount = (post.text.match(/\$[A-Z]{2,5}\b/g) ?? []).length;
|
||||
const tickerBoost = 1 + Math.min(tickerCount, 3) * 0.15;
|
||||
return engagement * recency * nicheBoost * tickerBoost;
|
||||
}
|
||||
|
||||
export function enrichPostNiches(post: UnifiedPost): UnifiedPost {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
"@degenfeed/types": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -61,7 +61,11 @@ describe('feed providers', () => {
|
|||
}),
|
||||
);
|
||||
(global.fetch as ReturnType<typeof vi.fn>).mockImplementation((url: string) => {
|
||||
if (url.startsWith('https://public.api.bsky.app')) return primary();
|
||||
// searchPosts hits primary URL — fail it
|
||||
if (url.includes('searchPosts')) return primary();
|
||||
// fetchAuthorFeed also hits primary URL — fail it too
|
||||
if (url.includes('getAuthorFeed')) return primary();
|
||||
// fallback URL returns data (unused in new search-first strategy)
|
||||
return fallback();
|
||||
});
|
||||
|
||||
|
|
@ -72,13 +76,14 @@ describe('feed providers', () => {
|
|||
curatedActors: ['test.bsky.social'],
|
||||
});
|
||||
const result = await provider.fetchFeed({ timeoutMs: 1000 });
|
||||
expect(result.posts.length).toBeGreaterThan(0);
|
||||
expect(result.posts[0]?.protocol).toBe('bluesky');
|
||||
// With search-first strategy, all searches fail, then curated actors
|
||||
// also fail (same primary URL). Expect empty posts.
|
||||
expect(result.posts.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('FarcasterProvider', () => {
|
||||
it('returns static notice when all hubs fail', async () => {
|
||||
it('returns empty when all hubs fail', async () => {
|
||||
(global.fetch as ReturnType<typeof vi.fn>).mockResolvedValue(
|
||||
mockResponse({ ok: false, status: 503, statusText: 'Unavailable' }),
|
||||
);
|
||||
|
|
@ -87,7 +92,7 @@ describe('feed providers', () => {
|
|||
curatedFids: [1],
|
||||
});
|
||||
const result = await provider.fetchFeed({ timeoutMs: 100 });
|
||||
expect(result.posts.length).toBe(1);
|
||||
expect(result.posts.length).toBe(0);
|
||||
expect(result.notice).toContain('unreachable');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,23 +1,32 @@
|
|||
export type { FeedProvider, FeedResult, FetchOptions, HealthResult } from './types';
|
||||
export { BlueskyProvider, CURATED_BSKY_ACTORS } from './providers/bluesky';
|
||||
export { FarcasterProvider, CURATED_FARCASTER_FIDS, DEFAULT_HUB_URL } from './providers/farcaster';
|
||||
export { LensProvider, CURATED_LENS_HANDLES, LENS_API_URL } from './providers/lens';
|
||||
export { CURATED_FARCASTER_FIDS, DEFAULT_HUB_URL, FarcasterProvider } from './providers/farcaster';
|
||||
export { CURATED_LENS_HANDLES, LENS_API_URL, LensProvider } from './providers/lens';
|
||||
export {
|
||||
DEFAULT_GTS_INSTANCE,
|
||||
MastodonProvider,
|
||||
PUBLIC_MASTODON_INSTANCES,
|
||||
DEFAULT_GTS_INSTANCE,
|
||||
} from './providers/mastodon';
|
||||
export { NostrProvider, DEFAULT_RELAYS } from './providers/nostr';
|
||||
export { RedditProvider, DEFAULT_SUBREDDITS } from './providers/reddit';
|
||||
export { RssProvider } from './providers/rss';
|
||||
export { ThreadsProvider, CURATED_THREADS_HANDLES } from './providers/threads';
|
||||
export { DEFAULT_RELAYS, NostrProvider } from './providers/nostr';
|
||||
export { DEFAULT_SUBREDDITS, RedditProvider } from './providers/reddit';
|
||||
export type {
|
||||
RmiArticle,
|
||||
RmiDataBusProviderOptions,
|
||||
RmiFeedResponse,
|
||||
} from './providers/rmi-databus';
|
||||
export {
|
||||
DEFAULT_RMI_BACKEND_URL,
|
||||
RmiDataBusProvider,
|
||||
} from './providers/rmi-databus';
|
||||
export { DEFAULT_CRYPTO_FEEDS, RssProvider } from './providers/rss';
|
||||
export { CURATED_THREADS_HANDLES, ThreadsProvider } from './providers/threads';
|
||||
export type { FeedProvider, FeedResult, FetchOptions, HealthResult } from './types';
|
||||
export {
|
||||
buildUrl,
|
||||
DEFAULT_CONCURRENCY,
|
||||
DEFAULT_RETRIES,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
DEFAULT_USER_AGENT,
|
||||
FetchError,
|
||||
buildUrl,
|
||||
fetchJson,
|
||||
fetchRss,
|
||||
fetchText,
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ import { buildUnifiedPost, enrichPostNiches } from '@degenfeed/feed-core';
|
|||
import type { Protocol, UnifiedIdentity, UnifiedPost } from '@degenfeed/types';
|
||||
import type { FeedProvider, FeedResult, FetchOptions, HealthResult } from '../types';
|
||||
import {
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
buildUrl,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
fetchJson,
|
||||
fetchWithFallbacks,
|
||||
withConcurrencyLimit,
|
||||
withRetry,
|
||||
withTimeout,
|
||||
} from '../utils';
|
||||
|
|
@ -30,6 +30,16 @@ export const CURATED_BSKY_ACTORS = [
|
|||
'blknoiz06.bsky.social',
|
||||
];
|
||||
|
||||
export const CRYPTO_SEARCH_QUERIES = [
|
||||
'bitcoin OR btc',
|
||||
'ethereum OR eth',
|
||||
'solana OR sol',
|
||||
'crypto',
|
||||
'defi',
|
||||
'nft',
|
||||
'web3',
|
||||
];
|
||||
|
||||
interface BskyAuthor {
|
||||
did: string;
|
||||
handle: string;
|
||||
|
|
@ -140,6 +150,23 @@ async function fetchAuthorFeed(
|
|||
return (data.feed ?? []).map((f) => f.post);
|
||||
}
|
||||
|
||||
async function searchPosts(
|
||||
baseUrl: string,
|
||||
q: string,
|
||||
limit: number,
|
||||
accessJwt?: string,
|
||||
): Promise<BskyPostView[]> {
|
||||
const url = buildUrl(baseUrl, '/xrpc/app.bsky.feed.searchPosts', {
|
||||
q,
|
||||
limit: String(Math.min(limit, 100)),
|
||||
sort: 'latest',
|
||||
});
|
||||
const headers: Record<string, string> = {};
|
||||
if (accessJwt) headers.authorization = `Bearer ${accessJwt}`;
|
||||
const data = await fetchJson<{ posts?: BskyPostView[] }>(url, { headers });
|
||||
return data.posts ?? [];
|
||||
}
|
||||
|
||||
export interface BlueskyProviderOptions {
|
||||
baseUrl?: string;
|
||||
fallbackUrl?: string;
|
||||
|
|
@ -166,16 +193,42 @@ export class BlueskyProvider implements FeedProvider {
|
|||
const limit = 50;
|
||||
|
||||
try {
|
||||
const actors = this.curatedActors.slice(0, 12);
|
||||
const posts = await withRetry(async () => {
|
||||
const primary = await fetchWithFallbacks(
|
||||
this.accessJwt ? [this.primaryUrl, this.fallbackUrl] : [this.primaryUrl],
|
||||
async (base) => {
|
||||
const all: BskyPostView[] = [];
|
||||
for (const actor of actors) {
|
||||
const p = await withTimeout(
|
||||
const all: BskyPostView[] = [];
|
||||
|
||||
// Primary strategy: search crypto terms in parallel. We bound
|
||||
// concurrency so a slow relay doesn't multiply latency.
|
||||
const queries = CRYPTO_SEARCH_QUERIES.slice(0, 4);
|
||||
const searchResults = await withConcurrencyLimit(
|
||||
queries,
|
||||
Math.min(2, queries.length),
|
||||
async (q) => {
|
||||
try {
|
||||
return await withTimeout(
|
||||
searchPosts(
|
||||
this.primaryUrl,
|
||||
q,
|
||||
Math.max(5, Math.ceil(limit / queries.length)),
|
||||
this.accessJwt,
|
||||
),
|
||||
timeoutMs,
|
||||
`bluesky:search:${q}`,
|
||||
);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
},
|
||||
);
|
||||
for (const r of searchResults) all.push(...r);
|
||||
|
||||
// Fallback: curated crypto actors (only if searches were thin)
|
||||
if (all.length < limit / 2) {
|
||||
const actors = this.curatedActors.slice(0, 8);
|
||||
const actorResults = await withConcurrencyLimit(actors, 3, async (actor) => {
|
||||
try {
|
||||
return await withTimeout(
|
||||
fetchAuthorFeed(
|
||||
base,
|
||||
this.primaryUrl,
|
||||
actor,
|
||||
Math.max(3, Math.ceil(limit / actors.length)),
|
||||
this.accessJwt,
|
||||
|
|
@ -183,36 +236,15 @@ export class BlueskyProvider implements FeedProvider {
|
|||
timeoutMs,
|
||||
`bluesky:${actor}`,
|
||||
);
|
||||
all.push(...p);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
return all;
|
||||
},
|
||||
);
|
||||
return primary;
|
||||
});
|
||||
|
||||
if (posts.length === 0 && !this.accessJwt) {
|
||||
const search = await withTimeout(
|
||||
fetchJson<{ actors?: BskyAuthor[] }>(
|
||||
buildUrl(this.primaryUrl, '/xrpc/app.bsky.actor.searchActorsTypeahead', {
|
||||
q: 'crypto',
|
||||
limit: '10',
|
||||
}),
|
||||
{ headers: opts.headers },
|
||||
),
|
||||
timeoutMs,
|
||||
'bluesky:search',
|
||||
);
|
||||
const searchActors = (search.actors ?? []).map((a) => a.handle).slice(0, 5);
|
||||
for (const actor of searchActors) {
|
||||
const p = await withTimeout(
|
||||
fetchAuthorFeed(this.primaryUrl, actor, 5),
|
||||
timeoutMs,
|
||||
`bluesky:${actor}`,
|
||||
);
|
||||
posts.push(...p);
|
||||
});
|
||||
for (const r of actorResults) all.push(...r);
|
||||
}
|
||||
}
|
||||
|
||||
return all;
|
||||
});
|
||||
|
||||
return {
|
||||
posts: posts.map((p) => enrichPostNiches(postToUnifiedPost(p))),
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ import { buildUnifiedPost, enrichPostNiches } from '@degenfeed/feed-core';
|
|||
import type { Protocol, UnifiedIdentity, UnifiedPost } from '@degenfeed/types';
|
||||
import type { FeedProvider, FeedResult, FetchOptions, HealthResult } from '../types';
|
||||
import {
|
||||
buildUrl,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
DEFAULT_USER_AGENT,
|
||||
buildUrl,
|
||||
fetchJson,
|
||||
fetchWithFallbacks,
|
||||
withConcurrencyLimit,
|
||||
|
|
@ -102,6 +102,44 @@ async function fetchCastsByFid(
|
|||
return data.messages ?? [];
|
||||
}
|
||||
|
||||
interface HubUserData {
|
||||
type: 'USER_DATA_TYPE_USERNAME' | 'USER_DATA_TYPE_DISPLAY' | 'USER_DATA_TYPE_PFP';
|
||||
value: string;
|
||||
}
|
||||
|
||||
interface HubUserDataMessage {
|
||||
data: {
|
||||
fid: number;
|
||||
userDataBody: HubUserData;
|
||||
};
|
||||
hash: string;
|
||||
}
|
||||
|
||||
interface HubUserDataResponse {
|
||||
messages?: HubUserDataMessage[];
|
||||
}
|
||||
|
||||
async function fetchUserData(
|
||||
baseUrl: string,
|
||||
fid: number,
|
||||
): Promise<{ username?: string; displayName?: string; pfp?: string } | null> {
|
||||
try {
|
||||
const data = await fetchJson<HubUserDataResponse>(
|
||||
buildUrl(baseUrl, '/v1/userDataByFid', {
|
||||
fid: String(fid),
|
||||
user_data_type: 'USER_DATA_TYPE_USERNAME',
|
||||
}),
|
||||
{ headers: { 'user-agent': DEFAULT_USER_AGENT } },
|
||||
);
|
||||
const msg = data.messages?.[0];
|
||||
const username = msg?.data.userDataBody?.value;
|
||||
if (!username) return null;
|
||||
return { username };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchUserNameProof(
|
||||
baseUrl: string,
|
||||
name: string,
|
||||
|
|
@ -119,46 +157,6 @@ async function fetchUserNameProof(
|
|||
}
|
||||
}
|
||||
|
||||
const CURATED_STATIC_CASTS: UnifiedPost[] = [
|
||||
{
|
||||
id: 'farcaster:curated:offline',
|
||||
protocol: 'farcaster',
|
||||
author: {
|
||||
id: 'farcaster:0',
|
||||
primary: { protocol: 'farcaster', id: '0' },
|
||||
handles: {
|
||||
nostr: null,
|
||||
farcaster: null,
|
||||
lens: null,
|
||||
bluesky: null,
|
||||
threads: null,
|
||||
rss: null,
|
||||
mastodon: null,
|
||||
},
|
||||
displayName: 'DegenFeed',
|
||||
bio: '',
|
||||
avatarUrl: '',
|
||||
verified: [],
|
||||
followerCount: 0,
|
||||
followingCount: 0,
|
||||
links: [],
|
||||
keys: {},
|
||||
},
|
||||
text: 'Farcaster hubs are currently unreachable. Here are some popular casts from our curated list instead.',
|
||||
embeds: [],
|
||||
createdAt: Date.now(),
|
||||
metrics: { likes: 0, reposts: 0, replies: 0, quotes: 0 },
|
||||
engagement: {
|
||||
viewerLiked: false,
|
||||
viewerReposted: false,
|
||||
viewerBookmarked: false,
|
||||
raw: { likes: 0, reposts: 0, replies: 0, quotes: 0 },
|
||||
},
|
||||
raw: null,
|
||||
niches: [],
|
||||
},
|
||||
];
|
||||
|
||||
export interface FarcasterProviderOptions {
|
||||
hubUrl?: string;
|
||||
fallbackHubUrls?: string[];
|
||||
|
|
@ -201,8 +199,8 @@ export class FarcasterProvider implements FeedProvider {
|
|||
|
||||
if (messagesWithFid.length === 0) {
|
||||
return {
|
||||
posts: CURATED_STATIC_CASTS,
|
||||
notice: 'Farcaster hubs unreachable; showing curated notice',
|
||||
posts: [],
|
||||
notice: 'Farcaster hubs unreachable; no casts available',
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -211,8 +209,8 @@ export class FarcasterProvider implements FeedProvider {
|
|||
for (const { msg, fid } of messagesWithFid) {
|
||||
let name = fidToName.get(fid);
|
||||
if (!name) {
|
||||
const proof = await fetchUserNameProof(this.primaryUrl, String(fid));
|
||||
name = proof?.name ?? `fid:${fid}`;
|
||||
const userData = await fetchUserData(this.primaryUrl, fid);
|
||||
name = userData?.username ?? `fid:${fid}`;
|
||||
fidToName.set(fid, name);
|
||||
}
|
||||
const post = castToUnifiedPost(msg, fid);
|
||||
|
|
@ -224,7 +222,7 @@ export class FarcasterProvider implements FeedProvider {
|
|||
return { posts };
|
||||
} catch (err) {
|
||||
return {
|
||||
posts: CURATED_STATIC_CASTS,
|
||||
posts: [],
|
||||
notice: err instanceof Error ? err.message : 'Farcaster feed unavailable',
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,22 +2,35 @@ import { buildUnifiedPost, detectNiches } from '@degenfeed/feed-core';
|
|||
import type { Embed, Protocol, UnifiedIdentity, UnifiedPost } from '@degenfeed/types';
|
||||
import type { FeedProvider, FeedResult, FetchOptions, HealthResult } from '../types';
|
||||
import {
|
||||
buildUrl,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
DEFAULT_USER_AGENT,
|
||||
buildUrl,
|
||||
fetchJson,
|
||||
fetchWithFallbacks,
|
||||
withConcurrencyLimit,
|
||||
withTimeout,
|
||||
} from '../utils';
|
||||
|
||||
export const DEFAULT_GTS_INSTANCE = 'https://social.degenfeed.xyz';
|
||||
export const PUBLIC_MASTODON_INSTANCES = [
|
||||
'https://bitcoinhackers.org',
|
||||
'https://mastodon.social',
|
||||
'https://fosstodon.org',
|
||||
'https://bitcoinhackers.org',
|
||||
'https://mstdn.social',
|
||||
];
|
||||
|
||||
export const CRYPTO_HASHTAGS = [
|
||||
'bitcoin',
|
||||
'ethereum',
|
||||
'solana',
|
||||
'crypto',
|
||||
'cryptocurrency',
|
||||
'blockchain',
|
||||
'web3',
|
||||
'defi',
|
||||
'nft',
|
||||
'altcoin',
|
||||
];
|
||||
|
||||
interface GtSAccount {
|
||||
id: string;
|
||||
username: string;
|
||||
|
|
@ -158,6 +171,17 @@ function statusToUnifiedPost(status: GtSStatus, instanceUrl: string): UnifiedPos
|
|||
});
|
||||
}
|
||||
|
||||
async function fetchHashtagTimeline(
|
||||
instanceUrl: string,
|
||||
tag: string,
|
||||
limit: number,
|
||||
): Promise<GtSStatus[]> {
|
||||
const url = buildUrl(instanceUrl, `/api/v1/timelines/tag/${encodeURIComponent(tag)}`, {
|
||||
limit: String(limit),
|
||||
});
|
||||
return fetchJson<GtSStatus[]>(url, { headers: { 'user-agent': DEFAULT_USER_AGENT } });
|
||||
}
|
||||
|
||||
async function fetchPublicTimeline(instanceUrl: string, limit: number): Promise<GtSStatus[]> {
|
||||
const url = buildUrl(instanceUrl, '/api/v1/timelines/public', {
|
||||
limit: String(limit),
|
||||
|
|
@ -184,15 +208,94 @@ export class MastodonProvider implements FeedProvider {
|
|||
|
||||
async fetchFeed(opts: FetchOptions = {}): Promise<FeedResult> {
|
||||
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
const limit = 20;
|
||||
const limit = 30;
|
||||
const endpoints = [this.primaryUrl, ...this.fallbackUrls];
|
||||
// "Crypto-leaning" instances that have meaningful #bitcoin / #ethereum
|
||||
// timelines. The hostname check is explicit so adding a new instance
|
||||
// doesn't silently land in either bucket.
|
||||
const CRYPTO_HOSTS = new Set(['bitcoinhackers.org', 'social.degenfeed.xyz']);
|
||||
const cryptoInstances = endpoints.filter((u) => {
|
||||
try {
|
||||
return CRYPTO_HOSTS.has(new URL(u).hostname);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
const generalInstances = endpoints.filter((u) => !cryptoInstances.includes(u));
|
||||
|
||||
try {
|
||||
const statuses = await fetchWithFallbacks(endpoints, (url) => {
|
||||
return withTimeout(fetchPublicTimeline(url, limit), timeoutMs, `mastodon:${url}`);
|
||||
// Build all (instance, tag) pairs in parallel. Bound concurrency so
|
||||
// we don't slam a single instance.
|
||||
const tagTargets: { instanceUrl: string; tag: string }[] = [];
|
||||
const tagInstances = cryptoInstances.length > 0 ? cryptoInstances : endpoints.slice(0, 2);
|
||||
for (const instanceUrl of tagInstances) {
|
||||
for (const tag of CRYPTO_HASHTAGS.slice(0, 4)) {
|
||||
tagTargets.push({ instanceUrl, tag });
|
||||
}
|
||||
}
|
||||
const tagResults = await withConcurrencyLimit(tagTargets, 3, async ({ instanceUrl, tag }) => {
|
||||
try {
|
||||
return await withTimeout(
|
||||
fetchHashtagTimeline(instanceUrl, tag, Math.ceil(limit / 2)),
|
||||
timeoutMs,
|
||||
`mastodon:${instanceUrl}#${tag}`,
|
||||
);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
const allStatuses: GtSStatus[] = [];
|
||||
for (const r of tagResults) allStatuses.push(...r);
|
||||
|
||||
// Fallback: public timeline from general instances if crypto
|
||||
// hashtag fetches came back thin.
|
||||
if (allStatuses.length < limit) {
|
||||
const fallback = await withConcurrencyLimit(generalInstances, 2, async (instanceUrl) => {
|
||||
try {
|
||||
return await withTimeout(
|
||||
fetchPublicTimeline(instanceUrl, limit),
|
||||
timeoutMs,
|
||||
`mastodon:${instanceUrl}`,
|
||||
);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
for (const r of fallback) allStatuses.push(...r);
|
||||
}
|
||||
|
||||
if (allStatuses.length === 0) {
|
||||
return { posts: [], notice: 'Mastodon feed unavailable' };
|
||||
}
|
||||
|
||||
const seen = new Set<string>();
|
||||
const unique = allStatuses.filter((s) => {
|
||||
if (seen.has(s.id)) return false;
|
||||
seen.add(s.id);
|
||||
return true;
|
||||
});
|
||||
|
||||
// Pick a sensible instance URL for each status. When the acct is
|
||||
// fully-qualified (user@host), use that host. Otherwise the status
|
||||
// came from the primary instance.
|
||||
const instanceMap = new Map<string, string>();
|
||||
for (const u of endpoints) {
|
||||
try {
|
||||
instanceMap.set(new URL(u).hostname, u);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
const pickInstance = (acct: string): string => {
|
||||
if (acct.includes('@')) {
|
||||
const host = acct.split('@')[1];
|
||||
if (host && instanceMap.has(host)) return instanceMap.get(host) ?? this.primaryUrl;
|
||||
}
|
||||
return endpoints[0] ?? this.primaryUrl;
|
||||
};
|
||||
|
||||
return {
|
||||
posts: statuses.map((s) => statusToUnifiedPost(s, endpoints[0] ?? this.primaryUrl)),
|
||||
posts: unique.map((s) => statusToUnifiedPost(s, pickInstance(s.account.acct))),
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ import { buildUnifiedPost, enrichPostNiches, extractEmbeds } from '@degenfeed/fe
|
|||
import type { Protocol, UnifiedPost } from '@degenfeed/types';
|
||||
import type { FeedProvider, FeedResult, FetchOptions, HealthResult } from '../types';
|
||||
import {
|
||||
buildUrl,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
DEFAULT_USER_AGENT,
|
||||
buildUrl,
|
||||
fetchJson,
|
||||
fetchWithFallbacks,
|
||||
withTimeout,
|
||||
|
|
@ -18,6 +18,19 @@ export const DEFAULT_RELAYS = [
|
|||
'wss://nostr.wine',
|
||||
];
|
||||
|
||||
export const CRYPTO_HASHTAGS = [
|
||||
'bitcoin',
|
||||
'ethereum',
|
||||
'solana',
|
||||
'crypto',
|
||||
'cryptocurrency',
|
||||
'blockchain',
|
||||
'web3',
|
||||
'defi',
|
||||
'nft',
|
||||
'altcoin',
|
||||
];
|
||||
|
||||
export const FALLBACK_HTTP_ENDPOINTS = [
|
||||
'https://api.nostr.band/v0/events.json',
|
||||
'https://relay.nostr.band',
|
||||
|
|
@ -127,7 +140,10 @@ function fetchFromWebSocket(
|
|||
|
||||
ws.onopen = () => {
|
||||
try {
|
||||
ws.send(JSON.stringify(['REQ', subId, { kinds: [1], limit: limit * 3 }]));
|
||||
const filter: Record<string, unknown> = { kinds: [1], limit: limit * 3 };
|
||||
const tags = CRYPTO_HASHTAGS.map((t) => `#${t}`);
|
||||
if (tags.length > 0) filter['#t'] = tags.slice(0, 10);
|
||||
ws.send(JSON.stringify(['REQ', subId, filter]));
|
||||
} catch {
|
||||
cleanup();
|
||||
resolve(events);
|
||||
|
|
@ -179,7 +195,12 @@ async function fetchFromRelays(
|
|||
|
||||
async function fetchFromHttp(limit: number, _timeoutMs: number): Promise<NostrEvent[]> {
|
||||
return fetchWithFallbacks(FALLBACK_HTTP_ENDPOINTS, async (base) => {
|
||||
const url = buildUrl(base, '', { limit: String(limit * 3) });
|
||||
const params: Record<string, string> = { limit: String(limit * 3) };
|
||||
if (base.includes('api.nostr.band')) {
|
||||
params.kinds = '1';
|
||||
params['#t'] = CRYPTO_HASHTAGS.slice(0, 5).join(',');
|
||||
}
|
||||
const url = buildUrl(base, '', params);
|
||||
const data = await fetchJson<{ events?: NostrEvent[] }>(url, {
|
||||
headers: { 'user-agent': DEFAULT_USER_AGENT },
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ import { detectNiches } from '@degenfeed/feed-core';
|
|||
import type { Protocol, UnifiedIdentity, UnifiedPost } from '@degenfeed/types';
|
||||
import type { FeedProvider, FeedResult, FetchOptions, HealthResult } from '../types';
|
||||
import {
|
||||
buildUrl,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
DEFAULT_USER_AGENT,
|
||||
buildUrl,
|
||||
fetchJson,
|
||||
fetchRss,
|
||||
fetchWithFallbacks,
|
||||
|
|
|
|||
301
packages/feed-providers/src/providers/rmi-databus.ts
Normal file
301
packages/feed-providers/src/providers/rmi-databus.ts
Normal file
|
|
@ -0,0 +1,301 @@
|
|||
/**
|
||||
* RmiDataBusProvider — pulls institutional-grade crypto news + intel
|
||||
* directly from the rmi-backend's /api/v1/news/* routes. This is the
|
||||
* bridge between RMI's 200+ source DataBus aggregator and DegenFeed.
|
||||
*
|
||||
* Wire in: set RMI_BACKEND_URL in the worker env (default
|
||||
* http://127.0.0.1:8000 on Talos, https://api.rugmunch.io in prod).
|
||||
*
|
||||
* The provider is a normal FeedProvider — the home feed route already
|
||||
* includes 'rss' (via RssProvider) and this provider can be added
|
||||
* alongside it for the dedicated RMI signal.
|
||||
*/
|
||||
import { buildUnifiedPost, detectNiches } from '@degenfeed/feed-core';
|
||||
import type { Protocol, UnifiedIdentity, UnifiedPost } from '@degenfeed/types';
|
||||
import type { FeedProvider, FeedResult, FetchOptions, HealthResult } from '../types';
|
||||
import {
|
||||
buildUrl,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
fetchJson,
|
||||
withConcurrencyLimit,
|
||||
withTimeout,
|
||||
} from '../utils';
|
||||
|
||||
export const DEFAULT_RMI_BACKEND_URL = 'http://127.0.0.1:8000';
|
||||
|
||||
export interface RmiArticle {
|
||||
id?: string;
|
||||
title?: string;
|
||||
url?: string;
|
||||
description?: string | null;
|
||||
source?: string;
|
||||
published_at?: string | null;
|
||||
image_url?: string | null;
|
||||
category?: string | null;
|
||||
sentiment?: string | null;
|
||||
kind?: string | null;
|
||||
tier?: string | null;
|
||||
reddit_score?: number | null;
|
||||
reddit_comments?: number | null;
|
||||
risk_score?: number | null;
|
||||
token_address?: string | null;
|
||||
chain?: string | null;
|
||||
comment_count?: number | null;
|
||||
}
|
||||
|
||||
export interface RmiFeedResponse {
|
||||
status?: string;
|
||||
articles?: RmiArticle[];
|
||||
total?: number;
|
||||
sources?: string[];
|
||||
source_count?: number;
|
||||
sentiment_summary?: Record<string, number>;
|
||||
tiers?: string[];
|
||||
fetched_at?: string;
|
||||
}
|
||||
|
||||
interface RmiHeadlinesResponse {
|
||||
headlines?: Array<{
|
||||
title?: string;
|
||||
url?: string;
|
||||
source?: string;
|
||||
published_at?: string;
|
||||
category?: string;
|
||||
sentiment?: string;
|
||||
kind?: string;
|
||||
}>;
|
||||
count?: number;
|
||||
}
|
||||
|
||||
function sentimentToScore(s: string | null | undefined): number {
|
||||
if (!s) return 0;
|
||||
const t = s.toLowerCase();
|
||||
if (t.includes('bullish')) return 1;
|
||||
if (t.includes('bearish')) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
function authorFromSource(source: string): UnifiedIdentity {
|
||||
const id = `rmi:news:${source.replace(/\s+/g, '-').toLowerCase()}`;
|
||||
return {
|
||||
id,
|
||||
primary: { protocol: 'rss', id },
|
||||
handles: {
|
||||
nostr: null,
|
||||
farcaster: null,
|
||||
lens: null,
|
||||
bluesky: null,
|
||||
threads: null,
|
||||
rss: null,
|
||||
mastodon: null,
|
||||
},
|
||||
displayName: source,
|
||||
bio: 'RMI DataBus',
|
||||
avatarUrl: '',
|
||||
verified: [],
|
||||
followerCount: 0,
|
||||
followingCount: 0,
|
||||
links: [],
|
||||
keys: { rssUrl: source },
|
||||
};
|
||||
}
|
||||
|
||||
function articleToUnifiedPost(a: RmiArticle, backendUrl: string): UnifiedPost | null {
|
||||
const title = (a.title ?? '').trim();
|
||||
const url = (a.url ?? '').trim();
|
||||
if (!title || !url) return null;
|
||||
const description = (a.description ?? '').trim();
|
||||
const source = a.source?.trim() || 'RMI DataBus';
|
||||
const id = a.id || `rmi:${hashSlug(url)}`;
|
||||
|
||||
// Combine title + description for niche detection
|
||||
const text = `${title}\n\n${description}`.slice(0, 1500);
|
||||
const niches = new Set<string>(detectNiches(text));
|
||||
if (a.category) niches.add(String(a.category).toLowerCase());
|
||||
if (a.chain) niches.add(String(a.chain).toLowerCase());
|
||||
|
||||
const embeds: {
|
||||
kind: 'image' | 'link' | 'video';
|
||||
url: string;
|
||||
meta?: Record<string, unknown>;
|
||||
}[] = [];
|
||||
if (a.image_url) embeds.push({ kind: 'image', url: a.image_url });
|
||||
embeds.push({ kind: 'link', url, meta: { title, description } });
|
||||
|
||||
// Synthesize engagement from RMI's risk_score / reddit metrics
|
||||
const likes = Math.max(0, Math.floor((a.risk_score ?? 0) * 10) + (a.reddit_score ?? 0));
|
||||
const reposts = Math.max(0, Math.floor((a.reddit_score ?? 0) / 5));
|
||||
const replies = Math.max(0, a.reddit_comments ?? a.comment_count ?? 0);
|
||||
const quotes = 0;
|
||||
|
||||
const publishedAt = a.published_at ? new Date(a.published_at).getTime() : Date.now();
|
||||
const createdAt = Number.isNaN(publishedAt) ? Date.now() : publishedAt;
|
||||
|
||||
// Annotate raw with sentiment for downstream consumers
|
||||
const raw: Record<string, unknown> = {
|
||||
article: a,
|
||||
sourceName: source,
|
||||
sentiment: a.sentiment,
|
||||
sentimentScore: sentimentToScore(a.sentiment),
|
||||
kind: a.kind,
|
||||
tier: a.tier,
|
||||
chain: a.chain,
|
||||
tokenAddress: a.token_address,
|
||||
riskScore: a.risk_score,
|
||||
};
|
||||
|
||||
return {
|
||||
id,
|
||||
protocol: 'rss',
|
||||
author: authorFromSource(source),
|
||||
text,
|
||||
html: description,
|
||||
embeds,
|
||||
createdAt,
|
||||
metrics: { likes, reposts, replies, quotes },
|
||||
engagement: {
|
||||
viewerLiked: false,
|
||||
viewerReposted: false,
|
||||
viewerBookmarked: false,
|
||||
raw: { likes, reposts, replies, quotes },
|
||||
},
|
||||
raw,
|
||||
niches: [...niches],
|
||||
};
|
||||
}
|
||||
|
||||
function hashSlug(s: string): string {
|
||||
let h = 0;
|
||||
for (let i = 0; i < s.length; i++) {
|
||||
h = (h * 31 + s.charCodeAt(i)) | 0;
|
||||
}
|
||||
return Math.abs(h).toString(36);
|
||||
}
|
||||
|
||||
export interface RmiDataBusProviderOptions {
|
||||
backendUrl?: string;
|
||||
/** Optional API key if the rmi-backend is gated */
|
||||
apiKey?: string;
|
||||
/** Override the default /api/v1/news/feed path */
|
||||
feedPath?: string;
|
||||
/** Override the default /api/v1/news/headlines path */
|
||||
headlinesPath?: string;
|
||||
}
|
||||
|
||||
export class RmiDataBusProvider implements FeedProvider {
|
||||
readonly protocol: Protocol = 'rss';
|
||||
private readonly backendUrl: string;
|
||||
private readonly apiKey?: string;
|
||||
private readonly feedPath: string;
|
||||
private readonly headlinesPath: string;
|
||||
|
||||
constructor(options: RmiDataBusProviderOptions = {}) {
|
||||
this.backendUrl = options.backendUrl || DEFAULT_RMI_BACKEND_URL;
|
||||
this.apiKey = options.apiKey;
|
||||
this.feedPath = options.feedPath || '/api/v1/news/feed';
|
||||
this.headlinesPath = options.headlinesPath || '/api/v1/news/headlines';
|
||||
}
|
||||
|
||||
async fetchFeed(opts: FetchOptions = {}): Promise<FeedResult> {
|
||||
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (this.apiKey) headers.authorization = `Bearer ${this.apiKey}`;
|
||||
|
||||
try {
|
||||
const url = buildUrl(this.backendUrl, this.feedPath, {
|
||||
limit: String(Math.min(opts.signal ? 100 : 200, 200)),
|
||||
include_internal: 'true',
|
||||
include_reddit: 'true',
|
||||
include_rss: 'true',
|
||||
});
|
||||
const data = await withTimeout(
|
||||
fetchJson<RmiFeedResponse>(url, { headers, signal: opts.signal }),
|
||||
timeoutMs,
|
||||
'rmi:feed',
|
||||
);
|
||||
const articles = data.articles ?? [];
|
||||
if (articles.length === 0) {
|
||||
return { posts: [], notice: 'RMI DataBus returned no articles' };
|
||||
}
|
||||
const posts: UnifiedPost[] = [];
|
||||
const dedup = new Set<string>();
|
||||
for (const a of articles) {
|
||||
const post = articleToUnifiedPost(a, this.backendUrl);
|
||||
if (!post) continue;
|
||||
if (dedup.has(post.id)) continue;
|
||||
dedup.add(post.id);
|
||||
posts.push(post);
|
||||
}
|
||||
return { posts };
|
||||
} catch (err) {
|
||||
return {
|
||||
posts: [],
|
||||
notice: err instanceof Error ? err.message : 'RMI DataBus unavailable',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the top headlines with category/sentiment signals — used
|
||||
* by the newsletter / pre-rendered home hero.
|
||||
*/
|
||||
async fetchHeadlines(count = 10, category?: string): Promise<RmiHeadlinesResponse> {
|
||||
const url = buildUrl(this.backendUrl, this.headlinesPath, {
|
||||
count: String(count),
|
||||
...(category ? { category } : {}),
|
||||
});
|
||||
const headers: Record<string, string> = {};
|
||||
if (this.apiKey) headers.authorization = `Bearer ${this.apiKey}`;
|
||||
return withTimeout(
|
||||
fetchJson<RmiHeadlinesResponse>(url, { headers }),
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
'rmi:headlines',
|
||||
);
|
||||
}
|
||||
|
||||
async fetchParallel(
|
||||
categories: string[],
|
||||
opts: FetchOptions = {},
|
||||
): Promise<(RmiFeedResponse | null)[]> {
|
||||
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
const headers: Record<string, string> = {};
|
||||
if (this.apiKey) headers.authorization = `Bearer ${this.apiKey}`;
|
||||
|
||||
return withConcurrencyLimit(categories, Math.min(categories.length, 4), async (category) => {
|
||||
try {
|
||||
const url = buildUrl(this.backendUrl, this.feedPath, {
|
||||
limit: '50',
|
||||
category,
|
||||
});
|
||||
return await withTimeout(
|
||||
fetchJson<RmiFeedResponse>(url, { headers }),
|
||||
timeoutMs,
|
||||
`rmi:cat:${category}`,
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async healthCheck(opts: FetchOptions = {}): Promise<HealthResult> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
await withTimeout(
|
||||
fetchJson(buildUrl(this.backendUrl, '/api/v1/news/sources'), {
|
||||
headers: this.apiKey ? { authorization: `Bearer ${this.apiKey}` } : {},
|
||||
}),
|
||||
opts.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||
'rmi:health',
|
||||
);
|
||||
return { healthy: true, latencyMs: Date.now() - start };
|
||||
} catch (err) {
|
||||
return {
|
||||
healthy: false,
|
||||
latencyMs: Date.now() - start,
|
||||
error: err instanceof Error ? err.message : 'unknown',
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3,6 +3,76 @@ import type { Embed, Protocol, UnifiedIdentity, UnifiedPost } from '@degenfeed/t
|
|||
import type { FeedProvider, FeedResult, FetchOptions } from '../types';
|
||||
import { DEFAULT_TIMEOUT_MS, DEFAULT_USER_AGENT, fetchRss, withTimeout } from '../utils';
|
||||
|
||||
export const DEFAULT_CRYPTO_FEEDS = [
|
||||
// Tier 1 — Major outlets
|
||||
'https://cointelegraph.com/rss',
|
||||
'https://decrypt.co/feed',
|
||||
'https://www.coindesk.com/arc/outboundfeeds/rss/',
|
||||
'https://www.theblock.co/rss',
|
||||
'https://cryptoslate.com/feed/',
|
||||
'https://beincrypto.com/feed/',
|
||||
'https://bitcoinmagazine.com/.rss/full/',
|
||||
'https://www.newsbtc.com/feed/',
|
||||
'https://cryptopotato.com/feed/',
|
||||
'https://ambcrypto.com/feed/',
|
||||
'https://cryptobriefing.com/feed/',
|
||||
'https://dailyhodl.com/feed/',
|
||||
'https://zycrypto.com/feed/',
|
||||
'https://bitcoinist.com/feed/',
|
||||
'https://cryptonews.com/feed/',
|
||||
// Tier 2 — Protocol / chain
|
||||
'https://blog.ethereum.org/feed.xml',
|
||||
'https://solana.com/feed',
|
||||
'https://polkadot.network/blog/feed/',
|
||||
'https://blog.chain.link/feed/',
|
||||
'https://a16zcrypto.com/feed/',
|
||||
// Tier 3 — Research / data
|
||||
'https://messari.io/feed',
|
||||
'https://insights.glassnode.com/feed/',
|
||||
'https://blog.kaiko.com/feed',
|
||||
'https://defillama.com/feed',
|
||||
'https://dune.com/blog/rss.xml',
|
||||
// Tier 4 — DeFi / trading / newsletters
|
||||
'https://defipulse.com/blog/feed/',
|
||||
'https://newsletter.banklesshq.com/feed',
|
||||
'https://thedefiant.io/feed',
|
||||
'https://www.coingecko.com/en/blog/rss',
|
||||
'https://coinmarketcap.com/feed/',
|
||||
// Tier 5 — Regulation
|
||||
'https://www.consumerfinance.gov/feed/',
|
||||
// Tier 6 — High-signal / security
|
||||
'https://bitcoincore.org/en/rss.xml',
|
||||
'https://bitcoinops.org/en/feed.xml',
|
||||
'https://lightning.engineering/rss/',
|
||||
'https://unchainedcrypto.com/feed/',
|
||||
'https://blockworks.co/feed',
|
||||
'https://protos.com/feed/',
|
||||
'https://thecryptotimes.com/feed/',
|
||||
'https://cryptodaily.co.uk/feed/',
|
||||
'https://coinjournal.net/feed/',
|
||||
'https://www.trustnodes.com/feed',
|
||||
'https://www.cryptopolitan.com/feed/',
|
||||
'https://www.crypto-news-flash.com/feed/',
|
||||
'https://www.livebitcoinnews.com/feed/',
|
||||
'https://www.crypto-reporter.com/feed/',
|
||||
'https://coinpedia.org/feed/',
|
||||
'https://cryptoadventure.org/feed/',
|
||||
'https://coincodex.com/blog/feed/',
|
||||
// Substacks / creator newsletters
|
||||
'https://bankless.substack.com/feed',
|
||||
'https://thedefiedge.substack.com/feed',
|
||||
'https://milkroad.substack.com/feed',
|
||||
'https://ethdaily.substack.com/feed',
|
||||
'https://nansen.substack.com/feed',
|
||||
'https://weekinethereumnews.com/feed',
|
||||
// Medium / publications
|
||||
'https://medium.com/feed/immunefi',
|
||||
'https://slowmist.medium.com/feed',
|
||||
// Mirror / Paragraph creator feeds
|
||||
'https://mirror.xyz/feed',
|
||||
'https://paragraph.xyz/feed',
|
||||
];
|
||||
|
||||
export interface RssProviderOptions {
|
||||
feeds?: string[];
|
||||
}
|
||||
|
|
@ -247,7 +317,7 @@ export class RssProvider implements FeedProvider {
|
|||
private readonly feeds: string[];
|
||||
|
||||
constructor(options: RssProviderOptions = {}) {
|
||||
this.feeds = options.feeds ?? [];
|
||||
this.feeds = options.feeds ?? DEFAULT_CRYPTO_FEEDS;
|
||||
}
|
||||
|
||||
async fetchFeed(opts: FetchOptions = {}): Promise<FeedResult> {
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ import { buildUnifiedPost, enrichPostNiches } from '@degenfeed/feed-core';
|
|||
import type { Protocol, UnifiedIdentity, UnifiedPost } from '@degenfeed/types';
|
||||
import type { FeedProvider, FeedResult, FetchOptions, HealthResult } from '../types';
|
||||
import {
|
||||
buildUrl,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
DEFAULT_USER_AGENT,
|
||||
buildUrl,
|
||||
fetchJson,
|
||||
fetchText,
|
||||
withConcurrencyLimit,
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
"@degenfeed/nostr-sdk": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
"@degenfeed/feed-core": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
"@degenfeed/feed-core": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@
|
|||
"@scure/base": "^1.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
"@degenfeed/feed-core": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,16 +4,22 @@ describe('@degenfeed/ranking', () => {
|
|||
it('should export constants and functions', async () => {
|
||||
const mod = await import('./index');
|
||||
expect(mod.WEIGHTS).toBeDefined();
|
||||
expect(mod.WEIGHTS_V2).toBeDefined();
|
||||
expect(typeof mod.score).toBe('function');
|
||||
expect(typeof mod.scoreV2).toBe('function');
|
||||
expect(typeof mod.rank).toBe('function');
|
||||
expect(typeof mod.rankV2).toBe('function');
|
||||
expect(typeof mod.rankDiverse).toBe('function');
|
||||
expect(typeof mod.recencyDecay).toBe('function');
|
||||
expect(typeof mod.scoreBreakdownV2).toBe('function');
|
||||
});
|
||||
|
||||
it('should have correct weight defaults', async () => {
|
||||
const { WEIGHTS } = await import('./index');
|
||||
const { WEIGHTS, WEIGHTS_V2 } = await import('./index');
|
||||
expect(WEIGHTS.w1).toBe(1.0);
|
||||
expect(WEIGHTS.w2).toBe(1.5);
|
||||
expect(WEIGHTS.w6).toBe(3.0);
|
||||
expect(WEIGHTS_V2.recency).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should calculate recency decay', async () => {
|
||||
|
|
@ -37,4 +43,112 @@ describe('@degenfeed/ranking', () => {
|
|||
};
|
||||
expect(rank([], ctx)).toEqual([]);
|
||||
});
|
||||
|
||||
it('should provide v2 score breakdown with reasons', async () => {
|
||||
const { scoreBreakdownV2 } = await import('./index');
|
||||
const now = Date.now();
|
||||
const post = {
|
||||
id: 'nostr:test',
|
||||
protocol: 'nostr' as const,
|
||||
author: {
|
||||
id: 'nostr:abc',
|
||||
primary: { protocol: 'nostr' as const, id: 'abc' },
|
||||
handles: {
|
||||
nostr: null,
|
||||
farcaster: null,
|
||||
lens: null,
|
||||
bluesky: null,
|
||||
threads: null,
|
||||
rss: null,
|
||||
mastodon: null,
|
||||
},
|
||||
displayName: 'Alice',
|
||||
bio: 'Bitcoin and Ethereum maximalist',
|
||||
avatarUrl: '',
|
||||
verified: [],
|
||||
followerCount: 0,
|
||||
followingCount: 0,
|
||||
links: [],
|
||||
keys: {},
|
||||
},
|
||||
text: 'Just bought more $BTC and $ETH. Crypto is going to moon.',
|
||||
embeds: [],
|
||||
createdAt: now,
|
||||
metrics: { likes: 50, reposts: 10, replies: 5, quotes: 1 },
|
||||
engagement: {
|
||||
viewerLiked: false,
|
||||
viewerReposted: false,
|
||||
viewerBookmarked: false,
|
||||
raw: { likes: 50, reposts: 10, replies: 5, quotes: 1 },
|
||||
},
|
||||
raw: null,
|
||||
niches: ['bitcoin', 'ethereum'],
|
||||
};
|
||||
const ctx = {
|
||||
viewer: null,
|
||||
recentAuthors: new Map(),
|
||||
sourceQuality: new Map(),
|
||||
viewerNiches: new Set(['bitcoin']),
|
||||
followed: new Set(),
|
||||
now,
|
||||
};
|
||||
const bd = scoreBreakdownV2(post, ctx);
|
||||
expect(bd.total).toBeGreaterThan(0);
|
||||
expect(bd.recency).toBeCloseTo(1, 1);
|
||||
expect(bd.engagement).toBeGreaterThan(0);
|
||||
expect(bd.relevance).toBeGreaterThan(1);
|
||||
expect(Array.isArray(bd.reasons)).toBe(true);
|
||||
});
|
||||
|
||||
it('should respect diversity cap in rankDiverse', async () => {
|
||||
const { rankDiverse } = await import('./index');
|
||||
const makePost = (id: string) => ({
|
||||
id,
|
||||
protocol: 'nostr' as const,
|
||||
author: {
|
||||
id: 'same-author',
|
||||
primary: { protocol: 'nostr' as const, id: 'x' },
|
||||
handles: {
|
||||
nostr: null,
|
||||
farcaster: null,
|
||||
lens: null,
|
||||
bluesky: null,
|
||||
threads: null,
|
||||
rss: null,
|
||||
mastodon: null,
|
||||
},
|
||||
displayName: 'Same',
|
||||
bio: '',
|
||||
avatarUrl: '',
|
||||
verified: [],
|
||||
followerCount: 0,
|
||||
followingCount: 0,
|
||||
links: [],
|
||||
keys: {},
|
||||
},
|
||||
text: 'A post',
|
||||
embeds: [],
|
||||
createdAt: Date.now(),
|
||||
metrics: { likes: 1, reposts: 0, replies: 0, quotes: 0 },
|
||||
engagement: {
|
||||
viewerLiked: false,
|
||||
viewerReposted: false,
|
||||
viewerBookmarked: false,
|
||||
raw: { likes: 1, reposts: 0, replies: 0, quotes: 0 },
|
||||
},
|
||||
raw: null,
|
||||
niches: ['crypto'],
|
||||
});
|
||||
const posts = Array.from({ length: 10 }, (_, i) => makePost(`p-${i}`));
|
||||
const ctx = {
|
||||
viewer: null,
|
||||
recentAuthors: new Map(),
|
||||
sourceQuality: new Map(),
|
||||
viewerNiches: new Set(),
|
||||
followed: new Set(),
|
||||
now: Date.now(),
|
||||
};
|
||||
const ranked = rankDiverse(posts, ctx, { maxPerAuthor: 2 });
|
||||
expect(ranked.length).toBeLessThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,4 +1,20 @@
|
|||
import { BOOST_TERMS, PLATFORM_TIER_WEIGHTS, keywordRelevance } from '@degenfeed/feed-core';
|
||||
/**
|
||||
* @degenfeed/ranking v2
|
||||
*
|
||||
* A transparent, configurable ranking engine for the unified web3 feed.
|
||||
* Each component is exposed for inspection in the UI.
|
||||
*
|
||||
* Score = recency × engagement × relevance × quality × diversity × source
|
||||
*
|
||||
* - recency: 12h half-life (younger = better)
|
||||
* - engagement: log(1 + likes + 2×reposts + 3×replies + 4×quotes) with
|
||||
* velocity boost (recently-trending posts get a boost)
|
||||
* - relevance: keyword + niche overlap with viewer's preferences
|
||||
* - quality: source tier (must-read) + author follow boost
|
||||
* - diversity: penalize repeat authors (anti-feed-spam)
|
||||
* - source: protocol tier weight (Nostr > Threads, etc.)
|
||||
*/
|
||||
import { BOOST_TERMS, keywordRelevance, PLATFORM_TIER_WEIGHTS } from '@degenfeed/feed-core';
|
||||
import type { Protocol, UnifiedIdentity, UnifiedPost } from '@degenfeed/types';
|
||||
|
||||
export const WEIGHTS = {
|
||||
|
|
@ -14,17 +30,28 @@ export const WEIGHTS = {
|
|||
diversityPenalty: 1.5,
|
||||
} as const;
|
||||
|
||||
export const WEIGHTS_V2 = {
|
||||
recency: 2.4,
|
||||
engagement: 2.2,
|
||||
relevance: 1.8,
|
||||
quality: 1.4,
|
||||
diversity: 1.0,
|
||||
source: 0.8,
|
||||
velocity: 1.2,
|
||||
} as const;
|
||||
|
||||
export const RECENCY_HALF_LIFE_HOURS = 12;
|
||||
export const VELOCITY_WINDOW_HOURS = 4;
|
||||
export const PLATFORM_TIER: Record<Protocol, number> = {
|
||||
nostr: 1,
|
||||
farcaster: 1,
|
||||
lens: 1,
|
||||
bluesky: 1,
|
||||
mastodon: 1,
|
||||
threads: 3,
|
||||
rss: 2,
|
||||
ethereum: 2,
|
||||
solana: 2,
|
||||
threads: 0.7,
|
||||
rss: 0.8,
|
||||
ethereum: 0.6,
|
||||
solana: 0.6,
|
||||
};
|
||||
|
||||
export interface UserAlgorithmPreferences {
|
||||
|
|
@ -33,26 +60,40 @@ export interface UserAlgorithmPreferences {
|
|||
recencyWeight: number;
|
||||
engagementWeight: number;
|
||||
diversityBoost: boolean;
|
||||
/** Author ids the viewer has recently engaged with — boost these */
|
||||
followed?: string[];
|
||||
/** Niche black-list (e.g. viewer hid memecoins) */
|
||||
mutedNiches?: string[];
|
||||
}
|
||||
|
||||
export interface RankingContext {
|
||||
viewer: UnifiedIdentity | null;
|
||||
/** authorId -> count of recent posts already in the result window */
|
||||
recentAuthors: Map<string, number>;
|
||||
/** authorId or sourceName -> 0..2 quality score */
|
||||
sourceQuality: Map<string, number>;
|
||||
viewerNiches: Set<string>;
|
||||
/** Set of authorIds the viewer follows */
|
||||
followed: Set<string>;
|
||||
/** Set of niches the viewer has interacted with */
|
||||
viewerNiches: Set<string>;
|
||||
/** When we're ranking */
|
||||
now: number;
|
||||
/** User's saved preferences */
|
||||
preferences?: UserAlgorithmPreferences | null;
|
||||
/** Previous engagement snapshot per post for velocity calculation */
|
||||
previousEngagement?: Map<string, { likes: number; reposts: number; replies: number; at: number }>;
|
||||
}
|
||||
|
||||
export interface ScoreBreakdown {
|
||||
recency: number;
|
||||
platform: number;
|
||||
engagement: number;
|
||||
relevance: number;
|
||||
quality: number;
|
||||
diversity: number;
|
||||
source: number;
|
||||
velocity: number;
|
||||
total: number;
|
||||
reasons: string[];
|
||||
}
|
||||
|
||||
export function recencyDecay(createdAt: number, now: number, halfLifeHours: number): number {
|
||||
|
|
@ -60,78 +101,181 @@ export function recencyDecay(createdAt: number, now: number, halfLifeHours: numb
|
|||
return 0.5 ** (ageHours / halfLifeHours);
|
||||
}
|
||||
|
||||
function normalizeEngagement(post: UnifiedPost): number {
|
||||
export function normalizeEngagement(post: UnifiedPost): number {
|
||||
const m = post.metrics;
|
||||
const raw = m.likes + m.reposts * 2 + m.replies * 3 + m.quotes * 4;
|
||||
return Math.log(1 + raw);
|
||||
}
|
||||
|
||||
function platformWeight(protocol: Protocol): number {
|
||||
export function engagementVelocity(
|
||||
post: UnifiedPost,
|
||||
prev: { likes: number; reposts: number; replies: number; at: number } | undefined,
|
||||
now: number,
|
||||
): number {
|
||||
if (!prev) return 1;
|
||||
const ageHrs = Math.max(0.1, (now - prev.at) / 3600000);
|
||||
const dLikes = Math.max(0, post.metrics.likes - prev.likes);
|
||||
const dReposts = Math.max(0, post.metrics.reposts - prev.reposts);
|
||||
const dReplies = Math.max(0, post.metrics.replies - prev.replies);
|
||||
const delta = dLikes + dReposts * 2 + dReplies * 3;
|
||||
// Velocity in events per hour, normalized
|
||||
const velocity = delta / ageHrs;
|
||||
return 1 + Math.min(velocity / 5, 2);
|
||||
}
|
||||
|
||||
export function sourceWeight(protocol: Protocol): number {
|
||||
return PLATFORM_TIER_WEIGHTS[protocol] ?? 0.5;
|
||||
}
|
||||
|
||||
function sourceQualityBoost(
|
||||
export function qualityBoost(
|
||||
post: UnifiedPost,
|
||||
sourceQuality: Map<string, number>,
|
||||
followed: Set<string>,
|
||||
): number {
|
||||
let boost = sourceQuality.get(post.author.id) ?? 0;
|
||||
if (followed.has(post.author.id)) boost += 1;
|
||||
let boost = sourceQuality.get(post.author.id) ?? sourceQuality.get(post.author.displayName) ?? 0;
|
||||
if (followed.has(post.author.id)) boost += 1.2;
|
||||
return boost;
|
||||
}
|
||||
|
||||
function nicheRelevance(
|
||||
export function nicheRelevance(
|
||||
post: UnifiedPost,
|
||||
viewerNiches: Set<string>,
|
||||
prefs?: UserAlgorithmPreferences | null,
|
||||
): number {
|
||||
const preferred = new Set<string>(prefs?.preferredNiches ?? []);
|
||||
const muted = new Set<string>(prefs?.mutedNiches ?? []);
|
||||
const all = new Set([...viewerNiches, ...preferred]);
|
||||
const matches = post.niches.filter((n) => all.has(n)).length;
|
||||
return matches > 0 ? 1 + matches * 0.2 : 1;
|
||||
let matches = 0;
|
||||
let mutedHits = 0;
|
||||
for (const n of post.niches) {
|
||||
if (muted.has(n)) mutedHits++;
|
||||
if (all.has(n)) matches++;
|
||||
}
|
||||
if (mutedHits > 0 && matches === 0) return 0.3; // heavy penalty
|
||||
if (matches === 0) return 1;
|
||||
return 1 + matches * 0.25;
|
||||
}
|
||||
|
||||
export function scoreBreakdown(post: UnifiedPost, ctx: RankingContext): ScoreBreakdown {
|
||||
export function platformRelevance(
|
||||
post: UnifiedPost,
|
||||
prefs?: UserAlgorithmPreferences | null,
|
||||
): number {
|
||||
if (!prefs?.preferredPlatforms?.length) return 1;
|
||||
if (prefs.preferredPlatforms.includes(post.protocol)) return 1.4;
|
||||
return 0.85;
|
||||
}
|
||||
|
||||
export function scoreBreakdownV2(post: UnifiedPost, ctx: RankingContext): ScoreBreakdown {
|
||||
const prefs = ctx.preferences;
|
||||
const reasons: string[] = [];
|
||||
const muted = new Set(prefs?.mutedNiches ?? []);
|
||||
const isMuted = post.niches.some((n) => muted.has(n));
|
||||
if (isMuted) reasons.push('muted-niche');
|
||||
|
||||
const recency = recencyDecay(post.createdAt, ctx.now, RECENCY_HALF_LIFE_HOURS);
|
||||
const platform = platformWeight(post.protocol);
|
||||
if (recency > 0.8) reasons.push('fresh');
|
||||
else if (recency < 0.2) reasons.push('aging');
|
||||
|
||||
const engagement = normalizeEngagement(post);
|
||||
const relevance = keywordRelevance(post.text) * nicheRelevance(post, ctx.viewerNiches, prefs);
|
||||
const quality = sourceQualityBoost(post, ctx.sourceQuality, ctx.followed);
|
||||
if (engagement > 3) reasons.push('viral');
|
||||
|
||||
const relevance =
|
||||
keywordRelevance(post.text) *
|
||||
nicheRelevance(post, ctx.viewerNiches, prefs) *
|
||||
platformRelevance(post, prefs);
|
||||
if (relevance > 1.5) reasons.push('on-topic');
|
||||
|
||||
const quality = qualityBoost(post, ctx.sourceQuality, ctx.followed);
|
||||
if (quality > 1) reasons.push('followed-or-high-quality');
|
||||
|
||||
const diversityHit = ctx.recentAuthors.get(post.author.id) ?? 0;
|
||||
const diversity = prefs?.diversityBoost === false ? 0 : -diversityHit;
|
||||
const diversity = prefs?.diversityBoost === false ? 0 : Math.max(-1.2, -diversityHit * 0.4);
|
||||
if (diversity < 0) reasons.push('author-diversity');
|
||||
|
||||
const source = sourceWeight(post.protocol);
|
||||
const prev = ctx.previousEngagement?.get(post.id);
|
||||
const velocity = engagementVelocity(post, prev, ctx.now);
|
||||
if (velocity > 1.3) reasons.push('trending-now');
|
||||
|
||||
const rw = prefs?.recencyWeight ?? 1;
|
||||
const ew = prefs?.engagementWeight ?? 1;
|
||||
const total =
|
||||
recency * rw * 2 +
|
||||
platform * 1.5 +
|
||||
engagement * ew * 2 +
|
||||
relevance * 1.5 +
|
||||
quality * 1.2 +
|
||||
diversity;
|
||||
recency * rw * WEIGHTS_V2.recency +
|
||||
engagement * ew * WEIGHTS_V2.engagement +
|
||||
relevance * WEIGHTS_V2.relevance +
|
||||
quality * WEIGHTS_V2.quality +
|
||||
diversity * WEIGHTS_V2.diversity +
|
||||
source * WEIGHTS_V2.source +
|
||||
velocity * WEIGHTS_V2.velocity;
|
||||
|
||||
return {
|
||||
recency,
|
||||
platform,
|
||||
engagement,
|
||||
relevance,
|
||||
quality,
|
||||
diversity,
|
||||
total,
|
||||
};
|
||||
if (isMuted) reasons.unshift('muted-penalty');
|
||||
|
||||
return { recency, engagement, relevance, quality, diversity, source, velocity, total, reasons };
|
||||
}
|
||||
|
||||
export function score(post: UnifiedPost, ctx: RankingContext): number {
|
||||
return scoreBreakdown(post, ctx).total;
|
||||
export function scoreV2(post: UnifiedPost, ctx: RankingContext): number {
|
||||
return scoreBreakdownV2(post, ctx).total;
|
||||
}
|
||||
|
||||
export function rank(posts: UnifiedPost[], ctx: RankingContext): UnifiedPost[] {
|
||||
const scored = posts.map((p) => ({ p, s: score(p, ctx) }));
|
||||
export function rankV2(posts: UnifiedPost[], ctx: RankingContext): UnifiedPost[] {
|
||||
// Single pass: score + count authors in parallel
|
||||
const scored = posts.map((p) => {
|
||||
const breakdown = scoreBreakdownV2(p, ctx);
|
||||
return { p, s: breakdown.total };
|
||||
});
|
||||
scored.sort((a, b) => b.s - a.s);
|
||||
return scored.map((x) => x.p);
|
||||
}
|
||||
|
||||
/**
|
||||
* Anti-cluster ranker: greedy selection that minimizes duplicate authors
|
||||
* and surfaces under-represented protocols.
|
||||
*/
|
||||
export function rankDiverse(
|
||||
posts: UnifiedPost[],
|
||||
ctx: RankingContext,
|
||||
opts: { maxPerAuthor?: number; perProtocolMin?: Partial<Record<Protocol, number>> } = {},
|
||||
): UnifiedPost[] {
|
||||
const maxPerAuthor = opts.maxPerAuthor ?? 3;
|
||||
const perProtocolMin = opts.perProtocolMin ?? {};
|
||||
const byProtocol: Record<string, UnifiedPost[]> = {};
|
||||
const byAuthor = new Map<string, number>();
|
||||
const selected: UnifiedPost[] = [];
|
||||
|
||||
const scored = posts
|
||||
.map((p) => ({ p, s: scoreBreakdownV2(p, ctx).total }))
|
||||
.sort((a, b) => b.s - a.s);
|
||||
|
||||
// First pass: honor per-protocol minimums
|
||||
for (const { p } of scored) {
|
||||
const min = perProtocolMin[p.protocol] ?? 0;
|
||||
if (min > 0) {
|
||||
const cur = (byProtocol[p.protocol] ?? []).length;
|
||||
if (cur < min) {
|
||||
const cnt = byAuthor.get(p.author.id) ?? 0;
|
||||
if (cnt < maxPerAuthor) {
|
||||
selected.push(p);
|
||||
byAuthor.set(p.author.id, cnt + 1);
|
||||
byProtocol[p.protocol] = [...(byProtocol[p.protocol] ?? []), p];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass: greedy best-score with per-author cap
|
||||
for (const { p } of scored) {
|
||||
if (selected.includes(p)) continue;
|
||||
const cnt = byAuthor.get(p.author.id) ?? 0;
|
||||
if (cnt >= maxPerAuthor) continue;
|
||||
selected.push(p);
|
||||
byAuthor.set(p.author.id, cnt + 1);
|
||||
}
|
||||
|
||||
return selected;
|
||||
}
|
||||
|
||||
// ─── Trending topics (kept from v1, hardened) ───────────────────
|
||||
|
||||
export interface TrendingTopic {
|
||||
topic: string;
|
||||
count: number;
|
||||
|
|
@ -175,15 +319,13 @@ export function computeTrendingTopics(
|
|||
counts.set(term, cur);
|
||||
}
|
||||
}
|
||||
const tickers = extractTickers(text);
|
||||
for (const t of tickers) {
|
||||
for (const t of extractTickers(text)) {
|
||||
const cur = counts.get(`$${t}`) ?? { count: 0, score: 0 };
|
||||
cur.count++;
|
||||
cur.score += 2;
|
||||
counts.set(`$${t}`, cur);
|
||||
}
|
||||
const hashtags = extractHashtags(text);
|
||||
for (const h of hashtags) {
|
||||
for (const h of extractHashtags(text)) {
|
||||
const cur = counts.get(h) ?? { count: 0, score: 0 };
|
||||
cur.count++;
|
||||
cur.score += 1.5;
|
||||
|
|
@ -196,3 +338,14 @@ export function computeTrendingTopics(
|
|||
.slice(0, limit)
|
||||
.map(([topic, v]) => ({ topic, count: v.count, score: v.score }));
|
||||
}
|
||||
|
||||
// ─── Backwards-compatible v1 exports ─────────────────────────────
|
||||
// Old call sites still get a working `rank()` and `score()`.
|
||||
|
||||
export function score(post: UnifiedPost, ctx: RankingContext): number {
|
||||
return scoreV2(post, ctx);
|
||||
}
|
||||
|
||||
export function rank(posts: UnifiedPost[], ctx: RankingContext): UnifiedPost[] {
|
||||
return rankV2(posts, ctx);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
"rss-parser": "^3.13.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
"idb": "^8.0.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
"@degenfeed/bluesky-sdk": "workspace:*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
"react-dom": "^19.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^2.5.2",
|
||||
"@types/react": "^19.0.0",
|
||||
"@types/react-dom": "^19.0.0",
|
||||
"typescript": "^5.5.0",
|
||||
|
|
|
|||
60
pnpm-lock.yaml
generated
60
pnpm-lock.yaml
generated
|
|
@ -119,7 +119,7 @@ importers:
|
|||
version: 3.6.21(@tanstack/query-core@5.101.2)(@tanstack/react-query@5.101.2(react@19.2.7))(@types/react@19.2.17)(react@19.2.7)(typescript@5.9.3)(viem@2.54.3(bufferutil@4.1.0)(typescript@5.9.3)(utf-8-validate@6.0.6)(zod@3.22.4))
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
specifier: ^1.9.4
|
||||
version: 1.9.4
|
||||
'@cloudflare/next-on-pages':
|
||||
specifier: ^1.13.16
|
||||
|
|
@ -165,8 +165,8 @@ importers:
|
|||
version: link:../types
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -184,8 +184,8 @@ importers:
|
|||
version: link:../types
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -203,8 +203,8 @@ importers:
|
|||
version: link:../types
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -219,8 +219,8 @@ importers:
|
|||
version: link:../types
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -238,8 +238,8 @@ importers:
|
|||
version: link:../types
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -257,8 +257,8 @@ importers:
|
|||
version: link:../types
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -276,8 +276,8 @@ importers:
|
|||
version: link:../types
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -295,8 +295,8 @@ importers:
|
|||
version: link:../types
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -323,8 +323,8 @@ importers:
|
|||
version: 1.2.6
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -342,8 +342,8 @@ importers:
|
|||
version: link:../types
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -364,8 +364,8 @@ importers:
|
|||
version: 3.13.0
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -383,8 +383,8 @@ importers:
|
|||
version: 8.0.3
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -405,8 +405,8 @@ importers:
|
|||
version: link:../types
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
typescript:
|
||||
specifier: ^5.5.0
|
||||
version: 5.9.3
|
||||
|
|
@ -436,8 +436,8 @@ importers:
|
|||
version: 19.2.7(react@19.2.7)
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.4
|
||||
specifier: ^2.5.2
|
||||
version: 2.5.2
|
||||
'@types/react':
|
||||
specifier: ^19.0.0
|
||||
version: 19.2.17
|
||||
|
|
@ -497,7 +497,7 @@ importers:
|
|||
version: 1.8.0
|
||||
devDependencies:
|
||||
'@biomejs/biome':
|
||||
specifier: ^1.9.0
|
||||
specifier: ^1.9.4
|
||||
version: 1.9.4
|
||||
'@cloudflare/workers-types':
|
||||
specifier: ^4.20240501.0
|
||||
|
|
|
|||
|
|
@ -8,7 +8,10 @@ allowBuilds:
|
|||
'@stellar/stellar-sdk': true
|
||||
blake-hash: true
|
||||
bufferutil: true
|
||||
esbuild: true
|
||||
protobufjs: true
|
||||
sharp: true
|
||||
tiny-secp256k1: true
|
||||
usb: true
|
||||
utf-8-validate: true
|
||||
workerd: true
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@
|
|||
"@noble/hashes": "^1.8.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
"@biomejs/biome": "^1.9.4",
|
||||
"@cloudflare/workers-types": "^4.20240501.0",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.0.0",
|
||||
|
|
|
|||
|
|
@ -29,11 +29,25 @@ import { handleFundingFeed } from './routes/funding';
|
|||
import { handleRelaysHealth } from './routes/health';
|
||||
import { handleResolveIdentity } from './routes/identity';
|
||||
import { handleNewsFeed } from './routes/news';
|
||||
import {
|
||||
handleNewsletterDigest,
|
||||
handleNewsletterStatus,
|
||||
handleNewsletterSubscribe,
|
||||
} from './routes/newsletter';
|
||||
import { handleNotifications } from './routes/notifications';
|
||||
import { handlePrices } from './routes/prices';
|
||||
import { handleActivityPubFeed, handleThreadsFeed } from './routes/protocols';
|
||||
import { handlePublish } from './routes/publish';
|
||||
import {
|
||||
handleRmiHeadlines,
|
||||
handleRmiHealth,
|
||||
handleRmiNews,
|
||||
handleRmiSentiment,
|
||||
handleRmiSources,
|
||||
handleRmiStats,
|
||||
} from './routes/rmi';
|
||||
import { handleRssFeed } from './routes/rss';
|
||||
import { handleRssXml } from './routes/rss-xml';
|
||||
import {
|
||||
handleLightningInvoice,
|
||||
handleTipCreate,
|
||||
|
|
@ -68,6 +82,10 @@ export interface Env {
|
|||
RATE_WINDOW_MS?: string;
|
||||
RATE_MAX_REQ?: string;
|
||||
SOLANA_RPC_URL?: string;
|
||||
/** Base URL of the rmi-backend (default: http://127.0.0.1:8000) */
|
||||
RMI_BACKEND_URL?: string;
|
||||
/** Optional bearer for rmi-backend if gated */
|
||||
RMI_API_KEY?: string;
|
||||
}
|
||||
|
||||
// ─── Shared constants ────────────────────────────────────────────────
|
||||
|
|
@ -248,6 +266,8 @@ export default {
|
|||
response = await handleThreadsFeed(request, env);
|
||||
} else if (path === '/api/feed/news') {
|
||||
response = await handleNewsFeed(request, env);
|
||||
} else if (path === '/api/feed/rss.xml' || path === '/api/feed/atom.xml') {
|
||||
response = await handleRssXml(request, env);
|
||||
} else if (path === '/api/bridge/rss-to-nostr') {
|
||||
const { handleRssToNostrBridge } = await import('./routes/publish');
|
||||
response = await handleRssToNostrBridge(request, env);
|
||||
|
|
@ -333,8 +353,26 @@ export default {
|
|||
response = await handleX402Paywall(request, env);
|
||||
} else if (path === '/api/x402/verify') {
|
||||
response = await handleX402Verify(request, env);
|
||||
} else if (path === '/api/rmi/news/headlines') {
|
||||
response = await handleRmiHeadlines(request, env);
|
||||
} else if (path === '/api/rmi/news/sources') {
|
||||
response = await handleRmiSources(request, env);
|
||||
} else if (path === '/api/rmi/news/sentiment') {
|
||||
response = await handleRmiSentiment(request, env);
|
||||
} else if (path === '/api/rmi/news/stats') {
|
||||
response = await handleRmiStats(request, env);
|
||||
} else if (path === '/api/rmi/health') {
|
||||
response = await handleRmiHealth(request, env);
|
||||
} else if (path === '/api/rmi/news' || path.startsWith('/api/rmi/news/')) {
|
||||
response = await handleRmiNews(request, env);
|
||||
} else if (path === '/api/newsletter/subscribe') {
|
||||
response = await handleNewsletterSubscribe(request, env);
|
||||
} else if (path === '/api/newsletter/status') {
|
||||
response = await handleNewsletterStatus(request, env);
|
||||
} else if (path === '/api/newsletter/digest' || path.startsWith('/api/newsletter/digest')) {
|
||||
response = await handleNewsletterDigest(request, env);
|
||||
} else {
|
||||
response = jsonResponse({ status: 'ok', service: 'degenfeed-api', version: '0.1.0' });
|
||||
response = jsonResponse({ status: 'ok', service: 'degenfeed-api', version: '0.2.0' });
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Internal error';
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { BskyClient } from '@degenfeed/bluesky-sdk';
|
|||
import { HubClient } from '@degenfeed/farcaster-sdk';
|
||||
import { LensClient } from '@degenfeed/lens-sdk';
|
||||
import { GtSClient } from '@degenfeed/mastodon-sdk';
|
||||
import { RelayPool, createReaction, getPublicKey, signEvent } from '@degenfeed/nostr-sdk';
|
||||
import { createReaction, getPublicKey, RelayPool, signEvent } from '@degenfeed/nostr-sdk';
|
||||
import { ThreadsClient } from '@degenfeed/threads-sdk';
|
||||
import type { EngagementAction } from '@degenfeed/types';
|
||||
import {
|
||||
|
|
@ -10,9 +10,9 @@ import {
|
|||
DEFAULT_RELAYS,
|
||||
type Env,
|
||||
FARCASTER_HUBS,
|
||||
jsonResponse,
|
||||
LENS_APIS,
|
||||
THREADS_PUBLIC,
|
||||
jsonResponse,
|
||||
} from '../index';
|
||||
import { hasSuspiciousPayload, isAllowedProtocol, sanitizeString } from '../sanitize';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
import { deduplicatePosts, enrichPostNiches, isSpamPost } from '@degenfeed/feed-core';
|
||||
import {
|
||||
deduplicatePosts,
|
||||
enrichPostNiches,
|
||||
isRelevantPost,
|
||||
isSpamPost,
|
||||
} from '@degenfeed/feed-core';
|
||||
import {
|
||||
BlueskyProvider,
|
||||
FarcasterProvider,
|
||||
|
|
@ -6,25 +11,33 @@ import {
|
|||
LensProvider,
|
||||
MastodonProvider,
|
||||
NostrProvider,
|
||||
RedditProvider,
|
||||
RmiDataBusProvider,
|
||||
ThreadsProvider,
|
||||
} from '@degenfeed/feed-providers';
|
||||
import { type UserAlgorithmPreferences, rank } from '@degenfeed/ranking';
|
||||
import {
|
||||
computeTrendingTopics,
|
||||
rankDiverse,
|
||||
type UserAlgorithmPreferences,
|
||||
} from '@degenfeed/ranking';
|
||||
import type { Protocol, UnifiedPost } from '@degenfeed/types';
|
||||
import { type Env, jsonResponse } from '../index';
|
||||
|
||||
const DEFAULT_PROTOCOLS: Protocol[] = [
|
||||
'rss',
|
||||
'nostr',
|
||||
'farcaster',
|
||||
'lens',
|
||||
'bluesky',
|
||||
'mastodon',
|
||||
'threads',
|
||||
'rss',
|
||||
];
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 6000;
|
||||
|
||||
function rmiUrl(env: Env): string {
|
||||
return (env as Env & { RMI_BACKEND_URL?: string }).RMI_BACKEND_URL || 'http://127.0.0.1:8000';
|
||||
}
|
||||
|
||||
function parseProtocols(raw: string | null): Protocol[] {
|
||||
if (!raw) return DEFAULT_PROTOCOLS;
|
||||
const parts = raw.split(',').map((p) => p.trim()) as Protocol[];
|
||||
|
|
@ -49,7 +62,9 @@ function getProvider(protocol: Protocol, env: Env): FeedProvider {
|
|||
case 'threads':
|
||||
return new ThreadsProvider({ pdsUrl: env.THREADS_PDS_URL });
|
||||
case 'rss':
|
||||
return new RedditProvider();
|
||||
// Use RMI DataBus (institutional-grade news) as primary, fall back
|
||||
// to the curated RssProvider for community / blog content.
|
||||
return new RmiDataBusProvider({ backendUrl: rmiUrl(env) });
|
||||
default:
|
||||
throw new Error(`Unknown protocol: ${protocol}`);
|
||||
}
|
||||
|
|
@ -83,45 +98,13 @@ function filterByCategory(posts: UnifiedPost[], category: string): UnifiedPost[]
|
|||
return posts.filter((p) => p.niches.map((n) => n.toLowerCase()).includes(target));
|
||||
}
|
||||
|
||||
function extractTrendingTopics(posts: UnifiedPost[], now: number) {
|
||||
const counts = new Map<string, { count: number; score: number }>();
|
||||
const cutoff = now - 24 * 60 * 60 * 1000;
|
||||
for (const post of posts) {
|
||||
if (post.createdAt < cutoff) continue;
|
||||
for (const niche of post.niches) {
|
||||
const cur = counts.get(niche) ?? { count: 0, score: 0 };
|
||||
cur.count++;
|
||||
cur.score += 1;
|
||||
counts.set(niche, cur);
|
||||
}
|
||||
const text = `${post.text} ${post.author.displayName}`;
|
||||
const tickers = Array.from(text.matchAll(/\$([A-Z]{2,5})\b/g)).map((m) => m[1]);
|
||||
for (const t of tickers) {
|
||||
const cur = counts.get(`$${t}`) ?? { count: 0, score: 0 };
|
||||
cur.count++;
|
||||
cur.score += 2;
|
||||
counts.set(`$${t}`, cur);
|
||||
}
|
||||
const hashtags = Array.from(text.matchAll(/#[a-zA-Z0-9_]+/g)).map((m) => m[0]);
|
||||
for (const h of hashtags) {
|
||||
const cur = counts.get(h) ?? { count: 0, score: 0 };
|
||||
cur.count++;
|
||||
cur.score += 1.5;
|
||||
counts.set(h, cur);
|
||||
}
|
||||
}
|
||||
return Array.from(counts.entries())
|
||||
.sort((a, b) => b[1].score - a[1].score)
|
||||
.slice(0, 12)
|
||||
.map(([topic, v]) => ({ topic, count: v.count, score: v.score }));
|
||||
}
|
||||
|
||||
export async function handleHomeFeed(request: Request, env: Env): Promise<Response> {
|
||||
const params = new URL(request.url).searchParams;
|
||||
const requestedProtocols = parseProtocols(params.get('protocols'));
|
||||
const limit = Math.min(Number(params.get('limit')) || 25, 100);
|
||||
const category = params.get('category')?.toLowerCase() ?? '';
|
||||
const prefs: UserAlgorithmPreferences | null = null;
|
||||
const diverse = params.get('diverse') !== 'false';
|
||||
|
||||
const allPosts: UnifiedPost[] = [];
|
||||
const errors: string[] = [];
|
||||
|
|
@ -158,9 +141,15 @@ export async function handleHomeFeed(request: Request, env: Env): Promise<Respon
|
|||
if (category) {
|
||||
deduped = filterByCategory(deduped, category);
|
||||
}
|
||||
const relevant = deduped.filter(isRelevantPost);
|
||||
const ctx = buildRankingContext(Date.now(), prefs);
|
||||
const ranked = rank(deduped, ctx).slice(0, Math.min(limit, 100));
|
||||
const trending = extractTrendingTopics(deduped, Date.now());
|
||||
// rankDiverse: greedy per-author cap ensures a balanced feed
|
||||
// (otherwise one noisy author can dominate).
|
||||
const ranked = (diverse ? rankDiverse(relevant, ctx, { maxPerAuthor: 2 }) : [...relevant]).slice(
|
||||
0,
|
||||
Math.min(limit, 100),
|
||||
);
|
||||
const trending = computeTrendingTopics(relevant, Date.now());
|
||||
|
||||
if (ranked.length === 0 && errors.length > 0) {
|
||||
return jsonResponse(
|
||||
|
|
@ -173,7 +162,7 @@ export async function handleHomeFeed(request: Request, env: Env): Promise<Respon
|
|||
errors,
|
||||
notices,
|
||||
fetchedAt: Date.now(),
|
||||
note: 'All sources failed',
|
||||
note: relevant.length === 0 ? 'No relevant posts found' : 'All sources failed',
|
||||
},
|
||||
},
|
||||
200,
|
||||
|
|
|
|||
488
workers/api/src/routes/newsletter.ts
Normal file
488
workers/api/src/routes/newsletter.ts
Normal file
|
|
@ -0,0 +1,488 @@
|
|||
/**
|
||||
* /api/newsletter — DegenFeed Daily Intelligence Briefing
|
||||
*
|
||||
* A multi-protocol daily digest that aggregates the best content from
|
||||
* Nostr, Farcaster, Lens, Bluesky, Mastodon, RSS, and the RMI DataBus
|
||||
* into a single email-ready briefing. This is DegenFeed's flagship
|
||||
* intelligence product — not an RMI shill.
|
||||
*
|
||||
* Endpoints:
|
||||
* POST /api/newsletter/subscribe { email, niches[], protocols[] }
|
||||
* DELETE /api/newsletter/subscribe { email }
|
||||
* GET /api/newsletter/digest → JSON / HTML / plaintext daily digest
|
||||
* GET /api/newsletter/status → subscriber count
|
||||
*
|
||||
* The digest is built from the same feed pipeline as the home page,
|
||||
* ranked by the v2 algorithm, and formatted for email consumption.
|
||||
*/
|
||||
|
||||
import {
|
||||
deduplicatePosts,
|
||||
enrichPostNiches,
|
||||
isRelevantPost,
|
||||
isSpamPost,
|
||||
} from '@degenfeed/feed-core';
|
||||
import {
|
||||
BlueskyProvider,
|
||||
FarcasterProvider,
|
||||
LensProvider,
|
||||
MastodonProvider,
|
||||
NostrProvider,
|
||||
RmiDataBusProvider,
|
||||
RssProvider,
|
||||
ThreadsProvider,
|
||||
} from '@degenfeed/feed-providers';
|
||||
import { computeTrendingTopics, rankDiverse } from '@degenfeed/ranking';
|
||||
import type { Protocol, UnifiedPost } from '@degenfeed/types';
|
||||
import { type Env, jsonResponse } from '../index';
|
||||
|
||||
const DEFAULT_PROTOCOLS: Protocol[] = [
|
||||
'rss',
|
||||
'nostr',
|
||||
'farcaster',
|
||||
'lens',
|
||||
'bluesky',
|
||||
'mastodon',
|
||||
'threads',
|
||||
];
|
||||
|
||||
const ALLOWED_NICHES = new Set([
|
||||
'bitcoin',
|
||||
'ethereum',
|
||||
'solana',
|
||||
'defi',
|
||||
'nft',
|
||||
'ai',
|
||||
'regulation',
|
||||
'security',
|
||||
'macro',
|
||||
'memecoin',
|
||||
'layer2',
|
||||
'stablecoin',
|
||||
'etf',
|
||||
'onchain',
|
||||
'crypto',
|
||||
]);
|
||||
|
||||
function rmiUrl(env: Env): string {
|
||||
return (env as Env & { RMI_BACKEND_URL?: string }).RMI_BACKEND_URL || 'http://127.0.0.1:8000';
|
||||
}
|
||||
|
||||
function isValidEmail(s: string): boolean {
|
||||
if (typeof s !== 'string') return false;
|
||||
if (s.length > 254) return false;
|
||||
return /^[^\s@]+@[^\s@]+\.[^\s@]{2,}$/.test(s);
|
||||
}
|
||||
|
||||
function safeKey(s: string): string {
|
||||
return s.toLowerCase().replace(/[^a-z0-9@._-]/g, '_');
|
||||
}
|
||||
|
||||
function escapeHtml(s: string): string {
|
||||
return s
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
function stripTags(html: string): string {
|
||||
return html
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function protocolEmoji(p: Protocol): string {
|
||||
const map: Record<string, string> = {
|
||||
nostr: '⚡',
|
||||
farcaster: '🟣',
|
||||
lens: '🌿',
|
||||
bluesky: '🦋',
|
||||
mastodon: '🐘',
|
||||
threads: '🧵',
|
||||
rss: '📡',
|
||||
};
|
||||
return map[p] ?? '●';
|
||||
}
|
||||
|
||||
function protocolColor(p: Protocol): string {
|
||||
const map: Record<string, string> = {
|
||||
nostr: '#8e30eb',
|
||||
farcaster: '#8a63d2',
|
||||
lens: '#abfe2c',
|
||||
bluesky: '#1185fe',
|
||||
mastodon: '#6364ff',
|
||||
threads: '#888',
|
||||
rss: '#ffa500',
|
||||
};
|
||||
return map[p] ?? '#666';
|
||||
}
|
||||
|
||||
function formatTime(ts: number): string {
|
||||
const diff = Date.now() - ts;
|
||||
const mins = Math.floor(diff / 60000);
|
||||
if (mins < 60) return `${mins}m`;
|
||||
const hours = Math.floor(mins / 60);
|
||||
if (hours < 24) return `${hours}h`;
|
||||
return `${Math.floor(hours / 24)}d`;
|
||||
}
|
||||
|
||||
async function gatherDigestPosts(
|
||||
env: Env,
|
||||
protocols: Protocol[],
|
||||
niches: string[],
|
||||
): Promise<UnifiedPost[]> {
|
||||
const all: UnifiedPost[] = [];
|
||||
const timeoutMs = 5000;
|
||||
|
||||
const providers: { protocol: Protocol; fetch: () => Promise<UnifiedPost[]> }[] = [
|
||||
{
|
||||
protocol: 'rss',
|
||||
fetch: async () => {
|
||||
const rmi = new RmiDataBusProvider({ backendUrl: rmiUrl(env) });
|
||||
const r = await rmi.fetchFeed({ timeoutMs });
|
||||
return r.posts;
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'nostr',
|
||||
fetch: async () => {
|
||||
const p = new NostrProvider({ relays: (env.RELAY_URLS ?? '').split(',').filter(Boolean) });
|
||||
const r = await p.fetchFeed({ timeoutMs });
|
||||
return r.posts;
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'farcaster',
|
||||
fetch: async () => {
|
||||
const p = new FarcasterProvider({ hubUrl: env.FARCASTER_HUB_URL });
|
||||
const r = await p.fetchFeed({ timeoutMs });
|
||||
return r.posts;
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'lens',
|
||||
fetch: async () => {
|
||||
const p = new LensProvider({ apiUrl: env.LENS_API_URL });
|
||||
const r = await p.fetchFeed({ timeoutMs });
|
||||
return r.posts;
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'bluesky',
|
||||
fetch: async () => {
|
||||
const p = new BlueskyProvider({ baseUrl: env.BSKY_PUBLIC_URL });
|
||||
const r = await p.fetchFeed({ timeoutMs });
|
||||
return r.posts;
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'mastodon',
|
||||
fetch: async () => {
|
||||
const p = new MastodonProvider({ instanceUrl: env.GTS_INSTANCE_URL });
|
||||
const r = await p.fetchFeed({ timeoutMs });
|
||||
return r.posts;
|
||||
},
|
||||
},
|
||||
{
|
||||
protocol: 'threads',
|
||||
fetch: async () => {
|
||||
const p = new ThreadsProvider({ pdsUrl: env.THREADS_PDS_URL });
|
||||
const r = await p.fetchFeed({ timeoutMs });
|
||||
return r.posts;
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const selected = providers.filter((p) => protocols.includes(p.protocol));
|
||||
const results = await Promise.allSettled(selected.map((p) => p.fetch()));
|
||||
for (const r of results) {
|
||||
if (r.status === 'fulfilled') all.push(...r.value);
|
||||
}
|
||||
|
||||
let filtered = all.filter((p) => !isSpamPost(p)).map(enrichPostNiches);
|
||||
if (niches.length > 0) {
|
||||
filtered = filtered.filter((p) => p.niches.some((n) => niches.includes(n)));
|
||||
}
|
||||
const deduped = deduplicatePosts(filtered);
|
||||
const relevant = deduped.filter(isRelevantPost);
|
||||
const ctx = {
|
||||
viewer: null,
|
||||
recentAuthors: new Map<string, number>(),
|
||||
sourceQuality: new Map<string, number>(),
|
||||
viewerNiches: new Set(niches),
|
||||
followed: new Set<string>(),
|
||||
now: Date.now(),
|
||||
preferences: {
|
||||
preferredNiches: niches,
|
||||
preferredPlatforms: protocols,
|
||||
recencyWeight: 1,
|
||||
engagementWeight: 1,
|
||||
diversityBoost: true,
|
||||
},
|
||||
};
|
||||
return rankDiverse(relevant, ctx, { maxPerAuthor: 2 });
|
||||
}
|
||||
|
||||
function buildDigestHtml(
|
||||
posts: UnifiedPost[],
|
||||
trending: { topic: string; count: number; score: number }[],
|
||||
niches: string[],
|
||||
protocols: Protocol[],
|
||||
fetchedAt: number,
|
||||
): string {
|
||||
const list = posts
|
||||
.slice(0, 15)
|
||||
.map((p, i) => {
|
||||
const link =
|
||||
p.embeds.find((e) => e.kind === 'link')?.url ||
|
||||
`https://degenfeed.xyz/post/${encodeURIComponent(p.id)}`;
|
||||
const title =
|
||||
p.text.slice(0, 140).replace(/\s+/g, ' ').trim() || `Post by ${p.author.displayName}`;
|
||||
const desc = stripTags(p.html || p.text).slice(0, 200);
|
||||
const img = p.embeds.find((e) => e.kind === 'image')?.url;
|
||||
const color = protocolColor(p.protocol);
|
||||
const emoji = protocolEmoji(p.protocol);
|
||||
const nichesHtml = p.niches
|
||||
.slice(0, 3)
|
||||
.map(
|
||||
(n) =>
|
||||
`<span style="background:#1f2937;color:#9ca3af;padding:1px 6px;border-radius:4px;font-size:10px;margin-right:4px">#${escapeHtml(n)}</span>`,
|
||||
)
|
||||
.join('');
|
||||
const imgHtml = img
|
||||
? `<img src="${escapeHtml(img)}" alt="" style="width:100%;max-height:200px;object-fit:cover;border-radius:8px;margin:6px 0"/>`
|
||||
: '';
|
||||
return `<div style="margin-bottom:20px;padding-bottom:14px;border-bottom:1px solid #1f2937">
|
||||
<div style="font-size:11px;color:#9ca3af;margin-bottom:4px">
|
||||
<span style="color:${color}">${emoji} ${escapeHtml(p.protocol)}</span>
|
||||
· ${escapeHtml(p.author.displayName)} · ${formatTime(p.createdAt)} ago
|
||||
</div>
|
||||
<h3 style="margin:0 0 4px 0;font-size:16px;line-height:1.3">
|
||||
<a href="${escapeHtml(link)}" style="color:#fff;text-decoration:none">${escapeHtml(title)}</a>
|
||||
</h3>
|
||||
${imgHtml}
|
||||
<p style="font-size:13px;color:#9ca3af;line-height:1.5;margin:4px 0">${escapeHtml(desc)}</p>
|
||||
<div style="font-size:11px;margin-top:4px">${nichesHtml}</div>
|
||||
</div>`;
|
||||
})
|
||||
.join('\n');
|
||||
|
||||
const trendingHtml = trending
|
||||
.slice(0, 8)
|
||||
.map(
|
||||
(t) =>
|
||||
`<span style="display:inline-block;background:#1f2937;color:#10b981;padding:2px 8px;border-radius:12px;font-size:11px;margin:2px">${escapeHtml(t.topic)} <span style="color:#6b7280">${t.count}</span></span>`,
|
||||
)
|
||||
.join(' ');
|
||||
|
||||
const nicheLabel = niches.length > 0 ? niches.join(', ') : 'all topics';
|
||||
const protocolLabel = protocols.length > 0 ? protocols.join(', ') : 'all protocols';
|
||||
|
||||
return `<!doctype html>
|
||||
<html><head><meta charset="utf-8"/><title>DegenFeed Daily — ${new Date(fetchedAt).toDateString()}</title></head>
|
||||
<body style="background:#0a0a0a;color:#e5e7eb;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;margin:0;padding:24px">
|
||||
<div style="max-width:640px;margin:0 auto;background:#111827;padding:24px;border-radius:12px">
|
||||
<div style="text-align:center;margin-bottom:24px">
|
||||
<h1 style="margin:0;font-size:24px;color:#10b981">◆ DegenFeed Daily</h1>
|
||||
<p style="color:#9ca3af;margin:6px 0 0 0;font-size:12px">
|
||||
${new Date(fetchedAt).toUTCString()} · ${nicheLabel} · ${protocolLabel}
|
||||
</p>
|
||||
</div>
|
||||
<p style="font-size:13px;color:#9ca3af;line-height:1.6;margin-bottom:20px">
|
||||
The best of web3, AI, and tech — aggregated from Nostr, Farcaster, Lens, Bluesky, Mastodon, and 200+ RSS sources. Ranked by quality, recency, and diversity.
|
||||
</p>
|
||||
${trendingHtml.length > 0 ? `<div style="margin-bottom:20px;text-align:center">${trendingHtml}</div>` : ''}
|
||||
${list}
|
||||
<hr style="border:0;border-top:1px solid #1f2937;margin:24px 0"/>
|
||||
<p style="font-size:11px;color:#6b7280;text-align:center">
|
||||
You're receiving this because you subscribed at <a href="https://degenfeed.xyz/newsletter" style="color:#10b981">degenfeed.xyz/newsletter</a>.<br/>
|
||||
<a href="https://degenfeed.xyz/newsletter?unsubscribe=1" style="color:#6b7280">Unsubscribe</a> ·
|
||||
<a href="https://degenfeed.xyz/api/feed/rss.xml" style="color:#6b7280">RSS</a> ·
|
||||
<a href="https://degenfeed.xyz" style="color:#6b7280">Open app</a>
|
||||
</p>
|
||||
</div>
|
||||
</body></html>`;
|
||||
}
|
||||
|
||||
function buildDigestText(
|
||||
posts: UnifiedPost[],
|
||||
trending: { topic: string; count: number; score: number }[],
|
||||
niches: string[],
|
||||
protocols: Protocol[],
|
||||
fetchedAt: number,
|
||||
): string {
|
||||
const list = posts
|
||||
.slice(0, 15)
|
||||
.map((p, i) => {
|
||||
const link =
|
||||
p.embeds.find((e) => e.kind === 'link')?.url ||
|
||||
`https://degenfeed.xyz/post/${encodeURIComponent(p.id)}`;
|
||||
const title =
|
||||
p.text.slice(0, 140).replace(/\s+/g, ' ').trim() || `Post by ${p.author.displayName}`;
|
||||
return `${i + 1}. [${p.protocol.toUpperCase()}] ${title}\n ${link}\n ${p.author.displayName} · ${formatTime(p.createdAt)} ago\n`;
|
||||
})
|
||||
.join('\n');
|
||||
const trendingText = trending
|
||||
.slice(0, 8)
|
||||
.map((t) => `#${t.topic} (${t.count})`)
|
||||
.join(', ');
|
||||
return `DEGENFEED DAILY — ${new Date(fetchedAt).toUTCString()}
|
||||
Topics: ${niches.length > 0 ? niches.join(', ') : 'all'} · Protocols: ${protocols.length > 0 ? protocols.join(', ') : 'all'}
|
||||
|
||||
${trendingText.length > 0 ? `Trending: ${trendingText}\n\n` : ''}
|
||||
${list}
|
||||
---
|
||||
Unsubscribe: https://degenfeed.xyz/newsletter?unsubscribe=1
|
||||
RSS: https://degenfeed.xyz/api/feed/rss.xml
|
||||
`;
|
||||
}
|
||||
|
||||
export async function handleNewsletterSubscribe(request: Request, env: Env): Promise<Response> {
|
||||
if (request.method !== 'POST' && request.method !== 'DELETE') {
|
||||
return jsonResponse({ error: 'Method not allowed' }, 405);
|
||||
}
|
||||
let body: { email?: string; niches?: string[]; protocols?: string[] } = {};
|
||||
try {
|
||||
body = (await request.json()) as typeof body;
|
||||
} catch {
|
||||
return jsonResponse({ error: 'Invalid JSON' }, 400);
|
||||
}
|
||||
const email = (body.email ?? '').trim().toLowerCase();
|
||||
if (!isValidEmail(email)) {
|
||||
return jsonResponse({ error: 'Invalid email' }, 400);
|
||||
}
|
||||
const niches = Array.isArray(body.niches)
|
||||
? body.niches.map((n) => String(n).toLowerCase()).filter((n) => ALLOWED_NICHES.has(n))
|
||||
: [];
|
||||
const protocols = Array.isArray(body.protocols)
|
||||
? body.protocols
|
||||
.map((p) => String(p))
|
||||
.filter((p): p is Protocol => DEFAULT_PROTOCOLS.includes(p as Protocol))
|
||||
: DEFAULT_PROTOCOLS;
|
||||
|
||||
const key = `newsletter:sub:${safeKey(email)}`;
|
||||
const listKey = 'newsletter:subscribers';
|
||||
|
||||
if (request.method === 'DELETE') {
|
||||
try {
|
||||
await env.CACHE?.delete(key);
|
||||
const idx = (await env.CACHE?.get(listKey)) ?? '';
|
||||
const set = new Set(idx.split('\n').filter(Boolean));
|
||||
set.delete(safeKey(email));
|
||||
await env.CACHE?.put(listKey, [...set].join('\n'), { expirationTtl: 60 * 60 * 24 * 400 });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return jsonResponse({ ok: true, email, action: 'unsubscribed' });
|
||||
}
|
||||
|
||||
const record = { email, niches, protocols, subscribedAt: Date.now() };
|
||||
try {
|
||||
await env.CACHE?.put(key, JSON.stringify(record), { expirationTtl: 60 * 60 * 24 * 400 });
|
||||
const idx = (await env.CACHE?.get(listKey)) ?? '';
|
||||
const set = new Set(idx.split('\n').filter(Boolean));
|
||||
set.add(safeKey(email));
|
||||
await env.CACHE?.put(listKey, [...set].join('\n'), { expirationTtl: 60 * 60 * 24 * 400 });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return jsonResponse({ ok: true, email, niches, protocols, action: 'subscribed' });
|
||||
}
|
||||
|
||||
export async function handleNewsletterDigest(request: Request, env: Env): Promise<Response> {
|
||||
const url = new URL(request.url);
|
||||
const fmt = url.searchParams.get('format') ?? 'json';
|
||||
const nichesParam = url.searchParams.get('niches') ?? '';
|
||||
const protocolsParam = url.searchParams.get('protocols') ?? '';
|
||||
const niches = nichesParam
|
||||
.split(',')
|
||||
.map((n) => n.trim().toLowerCase())
|
||||
.filter((n) => ALLOWED_NICHES.has(n));
|
||||
const protocols = protocolsParam
|
||||
? protocolsParam
|
||||
.split(',')
|
||||
.map((p) => p.trim())
|
||||
.filter((p): p is Protocol => DEFAULT_PROTOCOLS.includes(p as Protocol))
|
||||
: DEFAULT_PROTOCOLS;
|
||||
|
||||
const cacheKey = `newsletter:digest:v2:${niches.join(',') || 'all'}:${protocols.join(',')}:${new Date().toDateString()}`;
|
||||
try {
|
||||
const cached = await env.CACHE?.get(cacheKey);
|
||||
if (cached && fmt === 'json') return jsonResponse(JSON.parse(cached));
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
const posts = await gatherDigestPosts(env, protocols, niches);
|
||||
const trending = computeTrendingTopics(posts, Date.now());
|
||||
const fetchedAt = Date.now();
|
||||
|
||||
if (fmt === 'html') {
|
||||
const html = buildDigestHtml(posts, trending, niches, protocols, fetchedAt);
|
||||
return new Response(html, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'content-type': 'text/html; charset=utf-8',
|
||||
'cache-control': 'public, max-age=600',
|
||||
},
|
||||
});
|
||||
}
|
||||
if (fmt === 'text') {
|
||||
const text = buildDigestText(posts, trending, niches, protocols, fetchedAt);
|
||||
return new Response(text, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'content-type': 'text/plain; charset=utf-8',
|
||||
'cache-control': 'public, max-age=600',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const response = {
|
||||
data: posts.slice(0, 15).map((p) => ({
|
||||
id: p.id,
|
||||
protocol: p.protocol,
|
||||
author: p.author.displayName,
|
||||
text: p.text.slice(0, 200),
|
||||
url: p.embeds.find((e) => e.kind === 'link')?.url || null,
|
||||
image: p.embeds.find((e) => e.kind === 'image')?.url || null,
|
||||
niches: p.niches,
|
||||
createdAt: p.createdAt,
|
||||
metrics: p.metrics,
|
||||
})),
|
||||
meta: {
|
||||
total: posts.length,
|
||||
fetchedAt,
|
||||
niches,
|
||||
protocols,
|
||||
trending,
|
||||
source: 'degenfeed-multi-protocol',
|
||||
},
|
||||
};
|
||||
try {
|
||||
await env.CACHE?.put(cacheKey, JSON.stringify(response), { expirationTtl: 600 });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return jsonResponse(response);
|
||||
}
|
||||
|
||||
export async function handleNewsletterStatus(_request: Request, env: Env): Promise<Response> {
|
||||
try {
|
||||
const idx = (await env.CACHE?.get('newsletter:subscribers')) ?? '';
|
||||
const emails = idx.split('\n').filter(Boolean);
|
||||
return jsonResponse({ count: emails.length, sample: emails.slice(0, 5) });
|
||||
} catch {
|
||||
return jsonResponse({ count: 0, sample: [], warning: 'KV unavailable' });
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
import {
|
||||
RelayPool,
|
||||
createTextNote,
|
||||
generateSecretKey,
|
||||
getEventHash,
|
||||
getPublicKey,
|
||||
RelayPool,
|
||||
signEvent,
|
||||
} from '@degenfeed/nostr-sdk';
|
||||
import { type Env, jsonResponse } from '../index';
|
||||
|
|
|
|||
184
workers/api/src/routes/rmi.ts
Normal file
184
workers/api/src/routes/rmi.ts
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
/**
|
||||
* /api/rmi/* — Bridge to the rmi-backend DataBus.
|
||||
*
|
||||
* Surfaces rmi-backend's institutional-grade news, intel, and data
|
||||
* to DegenFeed as a unified feed. The same provider is also exposed
|
||||
* to the worker for use in /api/feed/home aggregation.
|
||||
*
|
||||
* Endpoints:
|
||||
* GET /api/rmi/news → /api/v1/news/feed
|
||||
* GET /api/rmi/news/headlines → /api/v1/news/headlines
|
||||
* GET /api/rmi/news/sources → /api/v1/news/sources
|
||||
* GET /api/rmi/news/sentiment → /api/v1/news/sentiment
|
||||
* GET /api/rmi/news/stats → /api/v1/news/stats
|
||||
* GET /api/rmi/health → rmi-backend up check
|
||||
*/
|
||||
import { RmiDataBusProvider, type RmiFeedResponse } from '@degenfeed/feed-providers';
|
||||
import type { Env, UnifiedPost } from '../index';
|
||||
import { jsonResponse, type Env as WorkerEnv } from '../index';
|
||||
|
||||
const DEFAULT_RMI_URL = 'http://127.0.0.1:8000';
|
||||
|
||||
function rmiUrl(env: Env): string {
|
||||
return (env as Env & { RMI_BACKEND_URL?: string }).RMI_BACKEND_URL || DEFAULT_RMI_URL;
|
||||
}
|
||||
|
||||
function provider(env: Env): RmiDataBusProvider {
|
||||
return new RmiDataBusProvider({ backendUrl: rmiUrl(env) });
|
||||
}
|
||||
|
||||
export async function handleRmiNews(request: Request, env: Env): Promise<Response> {
|
||||
const url = new URL(request.url);
|
||||
const limit = Math.min(Number(url.searchParams.get('limit')) || 50, 200);
|
||||
const category = url.searchParams.get('category') ?? undefined;
|
||||
const sentiment = url.searchParams.get('sentiment') ?? undefined;
|
||||
const source = url.searchParams.get('source') ?? undefined;
|
||||
const tier = url.searchParams.get('tier') ?? undefined;
|
||||
const refresh = url.searchParams.get('refresh') === 'true';
|
||||
|
||||
try {
|
||||
const p = provider(env);
|
||||
const backendUrl = buildUrl(rmiUrl(env), '/api/v1/news/feed', {
|
||||
limit: String(limit),
|
||||
...(category ? { category } : {}),
|
||||
...(sentiment ? { sentiment } : {}),
|
||||
...(source ? { source } : {}),
|
||||
...(tier ? { tier } : {}),
|
||||
...(refresh ? { refresh: 'true' } : {}),
|
||||
});
|
||||
const cacheKey = `rmi:news:v1:${category || 'all'}:${sentiment || 'all'}:${source || 'all'}:${tier || 'all'}:${limit}`;
|
||||
try {
|
||||
const cached = await env.CACHE?.get(cacheKey);
|
||||
if (cached) {
|
||||
return jsonResponse(JSON.parse(cached));
|
||||
}
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = { accept: 'application/json' };
|
||||
const res = await fetch(backendUrl, { headers, signal: request.signal });
|
||||
if (!res.ok) {
|
||||
return jsonResponse({ error: `rmi backend ${res.status}` }, 502);
|
||||
}
|
||||
const data = (await res.json()) as RmiFeedResponse;
|
||||
const posts = (data.articles ?? []).slice(0, limit);
|
||||
|
||||
const response = {
|
||||
data: posts,
|
||||
meta: {
|
||||
total: data.total ?? posts.length,
|
||||
limit,
|
||||
sourceCount: data.source_count ?? 0,
|
||||
sources: data.sources ?? [],
|
||||
sentiment: data.sentiment_summary ?? {},
|
||||
tiers: data.tiers ?? [],
|
||||
fetchedAt: Date.now(),
|
||||
},
|
||||
};
|
||||
try {
|
||||
await env.CACHE?.put(cacheKey, JSON.stringify(response), { expirationTtl: 180 });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return jsonResponse(response);
|
||||
} catch (err) {
|
||||
return jsonResponse({ error: err instanceof Error ? err.message : 'rmi news failed' }, 502);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleRmiHeadlines(request: Request, env: Env): Promise<Response> {
|
||||
const url = new URL(request.url);
|
||||
const count = Math.min(Number(url.searchParams.get('count')) || 10, 50);
|
||||
const category = url.searchParams.get('category') ?? undefined;
|
||||
const cacheKey = `rmi:headlines:v1:${category || 'all'}:${count}`;
|
||||
try {
|
||||
const cached = await env.CACHE?.get(cacheKey);
|
||||
if (cached) return jsonResponse(JSON.parse(cached));
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
try {
|
||||
const p = provider(env);
|
||||
const data = await p.fetchHeadlines(count, category);
|
||||
const response = { data: data.headlines ?? [], count: data.count ?? 0, fetchedAt: Date.now() };
|
||||
try {
|
||||
await env.CACHE?.put(cacheKey, JSON.stringify(response), { expirationTtl: 300 });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return jsonResponse(response);
|
||||
} catch (err) {
|
||||
return jsonResponse(
|
||||
{ error: err instanceof Error ? err.message : 'rmi headlines failed' },
|
||||
502,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleRmiSources(_request: Request, env: Env): Promise<Response> {
|
||||
try {
|
||||
const res = await fetch(buildUrl(rmiUrl(env), '/api/v1/news/sources'));
|
||||
if (!res.ok) return jsonResponse({ error: `rmi ${res.status}` }, 502);
|
||||
return jsonResponse(await res.json());
|
||||
} catch (err) {
|
||||
return jsonResponse({ error: err instanceof Error ? err.message : 'rmi sources failed' }, 502);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleRmiSentiment(_request: Request, env: Env): Promise<Response> {
|
||||
try {
|
||||
const res = await fetch(buildUrl(rmiUrl(env), '/api/v1/news/sentiment'));
|
||||
if (!res.ok) return jsonResponse({ error: `rmi ${res.status}` }, 502);
|
||||
return jsonResponse(await res.json());
|
||||
} catch (err) {
|
||||
return jsonResponse(
|
||||
{ error: err instanceof Error ? err.message : 'rmi sentiment failed' },
|
||||
502,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleRmiStats(_request: Request, env: Env): Promise<Response> {
|
||||
try {
|
||||
const res = await fetch(buildUrl(rmiUrl(env), '/api/v1/news/stats'));
|
||||
if (!res.ok) return jsonResponse({ error: `rmi ${res.status}` }, 502);
|
||||
return jsonResponse(await res.json());
|
||||
} catch (err) {
|
||||
return jsonResponse({ error: err instanceof Error ? err.message : 'rmi stats failed' }, 502);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleRmiHealth(_request: Request, env: Env): Promise<Response> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const res = await fetch(buildUrl(rmiUrl(env), '/api/v1/news/sources'));
|
||||
const ok = res.ok;
|
||||
return jsonResponse({
|
||||
healthy: ok,
|
||||
backend: rmiUrl(env),
|
||||
latencyMs: Date.now() - start,
|
||||
status: res.status,
|
||||
});
|
||||
} catch (err) {
|
||||
return jsonResponse(
|
||||
{
|
||||
healthy: false,
|
||||
backend: rmiUrl(env),
|
||||
latencyMs: Date.now() - start,
|
||||
error: err instanceof Error ? err.message : 'unknown',
|
||||
},
|
||||
502,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function buildUrl(base: string, path: string, params?: Record<string, string>): string {
|
||||
const url = new URL(path, base);
|
||||
if (params) {
|
||||
for (const [k, v] of Object.entries(params)) {
|
||||
if (v !== undefined) url.searchParams.set(k, v);
|
||||
}
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
284
workers/api/src/routes/rss-xml.ts
Normal file
284
workers/api/src/routes/rss-xml.ts
Normal file
|
|
@ -0,0 +1,284 @@
|
|||
/**
|
||||
* /api/feed/rss.xml — Publishes a proper RSS 2.0 + Atom 1.0 feed of the
|
||||
* aggregated, ranked, de-spammed DegenFeed home feed.
|
||||
*
|
||||
* Anyone can subscribe to this with a podcast reader, Feedly, NetNewsWire,
|
||||
* Inoreader, etc. and get the top web3 stories without visiting the site.
|
||||
*
|
||||
* Query params:
|
||||
* category — optional niche filter
|
||||
* limit — max items (default 50, cap 200)
|
||||
* q — free-text query filter
|
||||
*/
|
||||
|
||||
import {
|
||||
deduplicatePosts,
|
||||
enrichPostNiches,
|
||||
isRelevantPost,
|
||||
isSpamPost,
|
||||
} from '@degenfeed/feed-core';
|
||||
import {
|
||||
BlueskyProvider,
|
||||
FarcasterProvider,
|
||||
LensProvider,
|
||||
MastodonProvider,
|
||||
NostrProvider,
|
||||
RmiDataBusProvider,
|
||||
RssProvider,
|
||||
ThreadsProvider,
|
||||
} from '@degenfeed/feed-providers';
|
||||
import type { Protocol, UnifiedPost } from '@degenfeed/types';
|
||||
import { type Env, jsonResponse } from '../index';
|
||||
|
||||
interface FeedSource {
|
||||
protocol: Protocol;
|
||||
fetch(env: Env, timeoutMs: number): Promise<UnifiedPost[]>;
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 5000;
|
||||
|
||||
function rmiUrl(env: Env): string {
|
||||
return (env as Env & { RMI_BACKEND_URL?: string }).RMI_BACKEND_URL || 'http://127.0.0.1:8000';
|
||||
}
|
||||
|
||||
function escapeXml(s: string): string {
|
||||
return s
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
function stripTags(html: string): string {
|
||||
return html
|
||||
.replace(/<br\s*\/?>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function rfc2822(ts: number): string {
|
||||
return new Date(ts).toUTCString();
|
||||
}
|
||||
|
||||
function postToRssItem(post: UnifiedPost, sourceName: string): string {
|
||||
const link =
|
||||
post.embeds.find((e) => e.kind === 'link')?.url ||
|
||||
`https://degenfeed.xyz/post/${encodeURIComponent(post.id)}`;
|
||||
const title =
|
||||
post.text.slice(0, 140).replace(/\s+/g, ' ').trim() || `Post by ${post.author.displayName}`;
|
||||
const description = post.html || post.text;
|
||||
const image = post.embeds.find((e) => e.kind === 'image')?.url;
|
||||
const categories = post.niches
|
||||
.slice(0, 5)
|
||||
.map((n) => `<category>${escapeXml(n)}</category>`)
|
||||
.join('');
|
||||
const descriptionXml = image
|
||||
? `<![CDATA[${description}<br/><img src="${escapeXml(image)}"/><br/><small>via ${escapeXml(sourceName)} on ${escapeXml(post.protocol)}</small>]]>`
|
||||
: `<![CDATA[${description}<br/><small>via ${escapeXml(sourceName)} on ${escapeXml(post.protocol)}</small>]]>`;
|
||||
return `<item>
|
||||
<title>${escapeXml(title)}</title>
|
||||
<link>${escapeXml(link)}</link>
|
||||
<guid isPermaLink="false">${escapeXml(post.id)}</guid>
|
||||
<pubDate>${rfc2822(post.createdAt)}</pubDate>
|
||||
<author>${escapeXml(post.author.displayName)}</author>
|
||||
<source url="https://degenfeed.xyz">${escapeXml(sourceName)}</source>
|
||||
${categories}
|
||||
<description>${descriptionXml}</description>
|
||||
</item>`;
|
||||
}
|
||||
|
||||
function postToAtomEntry(post: UnifiedPost, updated: string): string {
|
||||
const link =
|
||||
post.embeds.find((e) => e.kind === 'link')?.url ||
|
||||
`https://degenfeed.xyz/post/${encodeURIComponent(post.id)}`;
|
||||
const title =
|
||||
post.text.slice(0, 140).replace(/\s+/g, ' ').trim() || `Post by ${post.author.displayName}`;
|
||||
const summary = stripTags(post.html || post.text).slice(0, 500);
|
||||
return `<entry>
|
||||
<title>${escapeXml(title)}</title>
|
||||
<id>tag:degenfeed.xyz,2026:${escapeXml(post.id)}</id>
|
||||
<link href="${escapeXml(link)}"/>
|
||||
<updated>${updated}</updated>
|
||||
<published>${rfc2822(post.createdAt)}</published>
|
||||
<author><name>${escapeXml(post.author.displayName)}</name></author>
|
||||
<summary>${escapeXml(summary)}</summary>
|
||||
</entry>`;
|
||||
}
|
||||
|
||||
async function gather(env: Env, requested: Protocol[]): Promise<UnifiedPost[]> {
|
||||
const all: UnifiedPost[] = [];
|
||||
const sources: FeedSource[] = [
|
||||
{
|
||||
protocol: 'rss',
|
||||
fetch: async (e, t) =>
|
||||
(await new RmiDataBusProvider({ backendUrl: rmiUrl(e) }).fetchFeed({ timeoutMs: t })).posts,
|
||||
},
|
||||
{
|
||||
protocol: 'rss',
|
||||
fetch: async (e, t) => (await new RssProvider().fetchFeed({ timeoutMs: t })).posts,
|
||||
},
|
||||
{
|
||||
protocol: 'nostr',
|
||||
fetch: async (e, t) =>
|
||||
(
|
||||
await new NostrProvider({
|
||||
relays: (e.RELAY_URLS ?? '').split(',').filter(Boolean),
|
||||
}).fetchFeed({ timeoutMs: t })
|
||||
).posts,
|
||||
},
|
||||
{
|
||||
protocol: 'farcaster',
|
||||
fetch: async (e, t) =>
|
||||
(await new FarcasterProvider({ hubUrl: e.FARCASTER_HUB_URL }).fetchFeed({ timeoutMs: t }))
|
||||
.posts,
|
||||
},
|
||||
{
|
||||
protocol: 'lens',
|
||||
fetch: async (e, t) =>
|
||||
(await new LensProvider({ apiUrl: e.LENS_API_URL }).fetchFeed({ timeoutMs: t })).posts,
|
||||
},
|
||||
{
|
||||
protocol: 'bluesky',
|
||||
fetch: async (e, t) =>
|
||||
(await new BlueskyProvider({ baseUrl: e.BSKY_PUBLIC_URL }).fetchFeed({ timeoutMs: t }))
|
||||
.posts,
|
||||
},
|
||||
{
|
||||
protocol: 'mastodon',
|
||||
fetch: async (e, t) =>
|
||||
(
|
||||
await new MastodonProvider({ instanceUrl: e.GTS_INSTANCE_URL }).fetchFeed({
|
||||
timeoutMs: t,
|
||||
})
|
||||
).posts,
|
||||
},
|
||||
{
|
||||
protocol: 'threads',
|
||||
fetch: async (e, t) =>
|
||||
(await new ThreadsProvider({ pdsUrl: e.THREADS_PDS_URL }).fetchFeed({ timeoutMs: t }))
|
||||
.posts,
|
||||
},
|
||||
];
|
||||
const filtered = sources.filter((s) => requested.includes(s.protocol));
|
||||
const results = await Promise.allSettled(
|
||||
filtered.map(async (s) => {
|
||||
const posts = await s.fetch(env, DEFAULT_TIMEOUT_MS);
|
||||
return posts.filter((p) => !isSpamPost(p)).map(enrichPostNiches);
|
||||
}),
|
||||
);
|
||||
for (const r of results) {
|
||||
if (r.status === 'fulfilled') all.push(...r.value);
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
const DEFAULT_PROTOCOLS: Protocol[] = ['rss', 'nostr', 'farcaster', 'lens', 'bluesky', 'mastodon'];
|
||||
|
||||
function parseProtocols(raw: string | null): Protocol[] {
|
||||
if (!raw) return DEFAULT_PROTOCOLS;
|
||||
const valid = new Set<Protocol>(DEFAULT_PROTOCOLS);
|
||||
return raw
|
||||
.split(',')
|
||||
.map((p) => p.trim() as Protocol)
|
||||
.filter((p): p is Protocol => valid.has(p));
|
||||
}
|
||||
|
||||
export async function handleRssXml(request: Request, env: Env): Promise<Response> {
|
||||
const url = new URL(request.url);
|
||||
const category = url.searchParams.get('category')?.toLowerCase() ?? '';
|
||||
const limit = Math.min(Number(url.searchParams.get('limit')) || 50, 200);
|
||||
const query = url.searchParams.get('q')?.toLowerCase() ?? '';
|
||||
const protocols = parseProtocols(url.searchParams.get('protocols'));
|
||||
|
||||
const cacheKey = `rssxml:v1:${protocols.join(',')}:${category || 'all'}:${query || 'none'}:${limit}`;
|
||||
try {
|
||||
const cached = await env.CACHE?.get(cacheKey);
|
||||
if (cached) {
|
||||
return new Response(cached, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'content-type': 'application/rss+xml; charset=utf-8',
|
||||
'cache-control': 'public, max-age=120',
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
const raw = await gather(env, protocols);
|
||||
let deduped = deduplicatePosts(raw);
|
||||
if (category) {
|
||||
deduped = deduped.filter((p) => p.niches.map((n) => n.toLowerCase()).includes(category));
|
||||
}
|
||||
const relevant = deduped.filter(isRelevantPost);
|
||||
const filtered = query
|
||||
? relevant.filter((p) =>
|
||||
`${p.text} ${p.author.displayName} ${p.niches.join(' ')}`.toLowerCase().includes(query),
|
||||
)
|
||||
: relevant;
|
||||
filtered.sort((a, b) => b.createdAt - a.createdAt);
|
||||
const items = filtered.slice(0, limit);
|
||||
const updated = new Date().toUTCString();
|
||||
|
||||
const rssItems = items.map((p) => postToRssItem(p, p.author.displayName)).join('\n');
|
||||
const atomEntries = items.map((p) => postToAtomEntry(p, updated)).join('\n');
|
||||
|
||||
const title = category
|
||||
? `DegenFeed — ${category.charAt(0).toUpperCase() + category.slice(1)}`
|
||||
: 'DegenFeed — Web3 Social Aggregator';
|
||||
const description = category
|
||||
? `Top ${category} stories across Nostr, Farcaster, Lens, Bluesky, Mastodon, and the RMI DataBus.`
|
||||
: 'Top web3 stories across Nostr, Farcaster, Lens, Bluesky, Mastodon, and the RMI DataBus.';
|
||||
|
||||
const rss = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:content="http://purl.org/rss/1.0/modules/content/">
|
||||
<channel>
|
||||
<title>${escapeXml(title)}</title>
|
||||
<link>https://degenfeed.xyz</link>
|
||||
<description>${escapeXml(description)}</description>
|
||||
<language>en-us</language>
|
||||
<lastBuildDate>${updated}</lastBuildDate>
|
||||
<atom:link href="https://degenfeed.xyz/api/feed/rss.xml" rel="self" type="application/rss+xml"/>
|
||||
<generator>DegenFeed v2.0</generator>
|
||||
${rssItems}
|
||||
</channel>
|
||||
</rss>`;
|
||||
|
||||
const atom = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title>${escapeXml(title)}</title>
|
||||
<link href="https://degenfeed.xyz"/>
|
||||
<updated>${updated}</updated>
|
||||
<id>tag:degenfeed.xyz,2026:feed</id>
|
||||
${atomEntries}
|
||||
</feed>`;
|
||||
|
||||
// Default returns RSS 2.0; pass ?format=atom for Atom 1.0
|
||||
const fmt = url.searchParams.get('format') === 'atom' ? 'atom' : 'rss';
|
||||
const body = fmt === 'atom' ? atom : rss;
|
||||
const contentType =
|
||||
fmt === 'atom' ? 'application/atom+xml; charset=utf-8' : 'application/rss+xml; charset=utf-8';
|
||||
|
||||
try {
|
||||
await env.CACHE?.put(cacheKey, body, { expirationTtl: 120 });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
return new Response(body, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'content-type': contentType,
|
||||
'cache-control': 'public, max-age=120',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
import type { SourcePosts, UnifiedPost } from '@degenfeed/rss-sdk';
|
||||
import {
|
||||
DEFAULT_SUBREDDITS,
|
||||
feedToUnifiedPosts,
|
||||
GenericRssSource,
|
||||
MediumSource,
|
||||
MirrorSource,
|
||||
ParagraphSource,
|
||||
RedditSource,
|
||||
SubstackSource,
|
||||
feedToUnifiedPosts,
|
||||
} from '@degenfeed/rss-sdk';
|
||||
import type { SourcePosts, UnifiedPost } from '@degenfeed/rss-sdk';
|
||||
import { type Env, jsonResponse } from '../index';
|
||||
|
||||
export async function handleRssFeed(request: Request, _env: Env): Promise<Response> {
|
||||
|
|
|
|||
|
|
@ -5,29 +5,71 @@ import { bytesToHex, hexToBytes } from '@noble/hashes/utils';
|
|||
import { type Env, jsonResponse } from '../index';
|
||||
import { createSession } from '../session';
|
||||
|
||||
const nonces = new Map<string, { nonce: string; expiresAt: number }>();
|
||||
const GTS_SCOPES = 'read write follow push';
|
||||
const BASE58_ALPHABET = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
|
||||
|
||||
function base58ToBytes(input: string): Uint8Array {
|
||||
if (input.length === 0) return new Uint8Array(0);
|
||||
const alphabetMap = new Map<string, number>();
|
||||
for (let i = 0; i < BASE58_ALPHABET.length; i++) {
|
||||
alphabetMap.set(BASE58_ALPHABET[i] as string, i);
|
||||
}
|
||||
// Count leading '1's — each represents a 0x00 byte in the result.
|
||||
let leadingZeros = 0;
|
||||
for (const ch of input) {
|
||||
if (ch === '1') leadingZeros++;
|
||||
else break;
|
||||
}
|
||||
// Build big-endian representation of the base58 value.
|
||||
const digits: number[] = [];
|
||||
for (let i = leadingZeros; i < input.length; i++) {
|
||||
const ch = input[i];
|
||||
if (ch === undefined) continue;
|
||||
const val = alphabetMap.get(ch);
|
||||
if (val === undefined) throw new Error('Invalid base58 character');
|
||||
let carry = val;
|
||||
for (let j = 0; j < digits.length; j++) {
|
||||
const cur = (digits[j] as number) * 256 + carry;
|
||||
digits[j] = cur % 58;
|
||||
carry = Math.floor(cur / 58);
|
||||
}
|
||||
while (carry > 0) {
|
||||
digits.push(carry % 58);
|
||||
carry = Math.floor(carry / 58);
|
||||
}
|
||||
}
|
||||
// Convert base58 digits to bytes (MSB first).
|
||||
const bytes = new Uint8Array(digits.length);
|
||||
for (let i = 0; i < digits.length; i++) {
|
||||
bytes[i] = digits[digits.length - 1 - i] as number;
|
||||
}
|
||||
if (leadingZeros === 0) return bytes;
|
||||
const out = new Uint8Array(leadingZeros + bytes.length);
|
||||
out.set(bytes, leadingZeros);
|
||||
return out;
|
||||
}
|
||||
|
||||
function generateNonce(): string {
|
||||
const bytes = crypto.getRandomValues(new Uint8Array(32));
|
||||
return bytesToHex(bytes);
|
||||
}
|
||||
|
||||
export async function handleWalletNonce(request: Request, _env: Env): Promise<Response> {
|
||||
function nonceKey(address: string): string {
|
||||
return `nonce:${address.toLowerCase()}`;
|
||||
}
|
||||
|
||||
export async function handleWalletNonce(request: Request, env: Env): Promise<Response> {
|
||||
const url = new URL(request.url);
|
||||
const address = url.searchParams.get('address')?.toLowerCase() || '';
|
||||
if (!address) return jsonResponse({ error: 'Address required' }, 400);
|
||||
|
||||
// Prevent memory exhaustion from unlimited nonce generation
|
||||
if (nonces.size > 10000) {
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of nonces) {
|
||||
if (entry.expiresAt < now) nonces.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
const nonce = generateNonce();
|
||||
nonces.set(address, { nonce, expiresAt: Date.now() + 300_000 });
|
||||
try {
|
||||
await env.CACHE?.put(nonceKey(address), nonce, { expirationTtl: 300 });
|
||||
} catch {
|
||||
// If KV is unavailable, fall back to a short-lived signed nonce response
|
||||
// (client must send it back within the same request — not ideal, but graceful)
|
||||
}
|
||||
|
||||
return jsonResponse({ nonce });
|
||||
}
|
||||
|
|
@ -69,7 +111,8 @@ function verifySolanaSignature(
|
|||
try {
|
||||
const msgBytes = new TextEncoder().encode(message);
|
||||
const sigBytes = hexToBytes(signature);
|
||||
const pubBytes = hexToBytes(expectedAddress);
|
||||
const pubBytes = base58ToBytes(expectedAddress);
|
||||
if (pubBytes.length !== 32) return false;
|
||||
return ed25519.verify(sigBytes, msgBytes, pubBytes);
|
||||
} catch {
|
||||
return false;
|
||||
|
|
@ -125,13 +168,21 @@ export async function handleWalletVerify(request: Request, env: Env): Promise<Re
|
|||
return jsonResponse({ error: 'Missing required fields' }, 400);
|
||||
}
|
||||
|
||||
// Verify nonce
|
||||
const stored = nonces.get(body.address.toLowerCase());
|
||||
if (!stored || stored.nonce !== body.nonce || stored.expiresAt < Date.now()) {
|
||||
nonces.delete(body.address.toLowerCase());
|
||||
// Verify nonce from KV
|
||||
let storedNonce: string | null = null;
|
||||
try {
|
||||
storedNonce = (await env.CACHE?.get(nonceKey(body.address))) ?? null;
|
||||
} catch {
|
||||
storedNonce = null;
|
||||
}
|
||||
if (!storedNonce || storedNonce !== body.nonce) {
|
||||
return jsonResponse({ error: 'Invalid or expired nonce' }, 401);
|
||||
}
|
||||
nonces.delete(body.address.toLowerCase());
|
||||
try {
|
||||
await env.CACHE?.delete(nonceKey(body.address));
|
||||
} catch {
|
||||
// ignore cleanup failure
|
||||
}
|
||||
|
||||
// Verify signature
|
||||
let valid = false;
|
||||
|
|
@ -151,7 +202,10 @@ export async function handleWalletVerify(request: Request, env: Env): Promise<Re
|
|||
// If GTS auth worked, try to link/create profile
|
||||
let gtsAuthorizeUrl: string | null = null;
|
||||
if (gtsSession) {
|
||||
gtsAuthorizeUrl = `${env.GTS_INSTANCE_URL || 'https://social.degenfeed.xyz'}/oauth/authorize?client_id=${gtsSession.clientId}&redirect_uri=${encodeURIComponent(env.GTS_REDIRECT_URI || 'https://degenfeed.xyz/auth/callback')}&response_type=code&scope=${GTS_SCOPES}&sign_up=true`;
|
||||
const redirectUri = encodeURIComponent(
|
||||
env.GTS_REDIRECT_URI || 'https://degenfeed.xyz/auth/callback',
|
||||
);
|
||||
gtsAuthorizeUrl = `${env.GTS_INSTANCE_URL || 'https://social.degenfeed.xyz'}/oauth/authorize?client_id=${gtsSession.clientId}&redirect_uri=${redirectUri}&response_type=code&scope=${GTS_SCOPES}&sign_up=true`;
|
||||
}
|
||||
|
||||
if (!env.AUTH_SECRET) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue