From 76c467689fc4916ef07077a9c025a05fbf7c5719 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 24 Jun 2026 16:21:40 +0000 Subject: [PATCH] Block crawler-driven compute egress on dynamic explorer routes Search/AI crawlers were walking the /transactions, /address and /block pages as an effectively infinite spider trap (each page cross-links to the others), running uncached server functions on every hit. This generated large compute->CDN egress around the clock even with zero human users (production logs showed 100% Applebot traffic on these routes). - robots.ts: disallow /transactions, /address/ and /block/ for all bots and Googlebot (removed from allow lists). Compliant crawlers stay out. - middleware.ts (new): edge-level enforcement for bots that ignore robots.txt. Matches the three route prefixes (catching both the page GET and the Server Action POST) and short-circuits known crawler UAs with a tiny 403 before the heavy function runs. Real users pass through; all responses on these routes carry X-Robots-Tag: noindex, nofollow. - next.config.ts: edge-cache the explorer page shells (s-maxage=300, SWR) so repeat human GETs are CDN hits, not fresh compute. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01EHCQFhXU1zen3gmq9ZMbHJ --- next.config.ts | 18 ++++++++-- src/app/robots.ts | 15 ++++---- src/middleware.ts | 87 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 src/middleware.ts diff --git a/next.config.ts b/next.config.ts index 02e0c3b2..7f60462d 100644 --- a/next.config.ts +++ b/next.config.ts @@ -130,8 +130,7 @@ const nextConfig: NextConfig = { // auth gating happens client-side from localStorage. The shells are // therefore safe to edge-cache (short TTL + SWR). NOTE: only the static // shell is cached; the per-user data requests are separate POSTs that - // always reach the origin. Parameterised routes (address/[address] etc.) - // are intentionally excluded. + // always reach the origin. source: '/(dashboard|analysis|security|chat|report|coin-control)', headers: [ { @@ -140,6 +139,21 @@ const nextConfig: NextConfig = { }, ], }, + { + // Dynamic explorer routes (transactions/address/block). Like the app + // shells above, the SSR'd HTML is identical per route — all per-item + // data loads client-side via Server Actions. Vercel only edge-caches + // GET/HEAD, so the Server Action POSTs are never cached and always reach + // the origin; this rule only makes repeat *page* (GET) hits a cache HIT + // instead of a fresh server function, cutting compute->CDN egress. + source: '/:section(transactions|address|block)/:id*', + headers: [ + { + key: 'Cache-Control', + value: 'public, max-age=0, s-maxage=300, stale-while-revalidate=3600', + }, + ], + }, ]; }, // Updated experimental flags for Next.js 16 diff --git a/src/app/robots.ts b/src/app/robots.ts index c331fb0a..2fed6b70 100644 --- a/src/app/robots.ts +++ b/src/app/robots.ts @@ -13,9 +13,6 @@ export default function robots(): MetadataRoute.Robots { '/mempool', '/discover', '/feedback', - '/block/*', - '/transactions/*', - '/address/*', ], disallow: [ '/dashboard/', @@ -24,7 +21,12 @@ export default function robots(): MetadataRoute.Robots { '/chat/', '/report/', '/coin-control/', - '/transactions', // Disallow the generic transaction list page + // Dynamic explorer pages form an effectively infinite crawl trap + // (tx -> address -> tx -> ...) with negligible SEO value and run + // uncached server functions on every hit. Keep crawlers out. + '/transactions', // Covers the list page and all /transactions/* details + '/address/', + '/block/', '/api/', ], }, @@ -37,9 +39,6 @@ export default function robots(): MetadataRoute.Robots { '/mempool', '/discover', '/feedback', - '/block/*', - '/transactions/*', - '/address/*', ], disallow: [ '/dashboard/', @@ -49,6 +48,8 @@ export default function robots(): MetadataRoute.Robots { '/report/', '/coin-control/', '/transactions', + '/address/', + '/block/', '/api/', ], }, diff --git a/src/middleware.ts b/src/middleware.ts new file mode 100644 index 00000000..ea24528a --- /dev/null +++ b/src/middleware.ts @@ -0,0 +1,87 @@ +import { NextResponse, type NextRequest } from 'next/server'; + +/** + * Edge middleware to protect the dynamic blockchain-explorer routes + * (`/transactions/[id]`, `/address/[address]`, `/block/[id]`) from crawler + * traffic. + * + * These pages cross-link densely (a transaction links to every input/output + * address, each address links back to every transaction, blocks link to prior + * blocks + transactions), forming an effectively infinite "spider trap". Every + * crawl runs an uncached server function (SSR shell + a `'use server'` data + * fetch), so unchecked bots generate large compute->CDN egress around the clock + * even with zero human users. + * + * robots.txt (see `src/app/robots.ts`) tells well-behaved bots to stay out. + * This middleware is the enforcement layer for bots that ignore it: it matches + * the path (catching both the GET page request and the Server Action POST that + * posts back to the same path) and short-circuits known crawler User-Agents at + * the edge with a tiny `403` before the expensive Node function ever runs. + * + * Real users pass through untouched, but every response on these routes carries + * `X-Robots-Tag: noindex, nofollow` as defense in depth so the pages are never + * indexed even if discovered through other links. + */ + +// Curated list of common search-engine, social, AI and SEO crawler User-Agents, +// plus a generic fallback. Normal browsers do not match these tokens. +const BOT_UA_REGEX = new RegExp( + [ + 'applebot', + 'googlebot', + 'bingbot', + 'gptbot', + 'oai-searchbot', + 'chatgpt', + 'claudebot', + 'claude-web', + 'anthropic', + 'ccbot', + 'perplexitybot', + 'amazonbot', + 'bytespider', + 'yandex(bot)?', + 'baiduspider', + 'duckduckbot', + 'slurp', // Yahoo + 'facebookexternalhit', + 'meta-externalagent', + 'semrushbot', + 'ahrefsbot', + 'mj12bot', + 'dotbot', + 'petalbot', + 'dataforseobot', + // Generic fallback for anything self-identifying as automated. + 'bot\\b', + 'crawler', + 'spider', + ].join('|'), + 'i' +); + +export function middleware(request: NextRequest) { + const userAgent = request.headers.get('user-agent') || ''; + + if (BOT_UA_REGEX.test(userAgent)) { + return new NextResponse(null, { + status: 403, + headers: { + 'X-Robots-Tag': 'noindex, nofollow', + 'Cache-Control': 'no-store', + }, + }); + } + + const response = NextResponse.next(); + response.headers.set('X-Robots-Tag', 'noindex, nofollow'); + return response; +} + +export const config = { + matcher: [ + '/transactions/:path*', + '/address/:path*', + '/block/:path*', + ], +};