Block crawler access to dynamic explorer routes and optimize caching

next.config.ts

@@ -130,8 +130,7 @@ const nextConfig: NextConfig = {
         // auth gating happens client-side from localStorage. The shells are
         // therefore safe to edge-cache (short TTL + SWR). NOTE: only the static
         // shell is cached; the per-user data requests are separate POSTs that
-        // always reach the origin. Parameterised routes (address/[address] etc.)
-        // are intentionally excluded.
+        // always reach the origin.
         source: '/(dashboard|analysis|security|chat|report|coin-control)',
         headers: [
           {
@@ -140,6 +139,21 @@ const nextConfig: NextConfig = {
           },
         ],
       },
+      {
+        // Dynamic explorer routes (transactions/address/block). Like the app
+        // shells above, the SSR'd HTML is identical per route — all per-item
+        // data loads client-side via Server Actions. Vercel only edge-caches
+        // GET/HEAD, so the Server Action POSTs are never cached and always reach
+        // the origin; this rule only makes repeat *page* (GET) hits a cache HIT
+        // instead of a fresh server function, cutting compute->CDN egress.
+        source: '/:section(transactions|address|block)/:id*',
+        headers: [
+          {
+            key: 'Cache-Control',
+            value: 'public, max-age=0, s-maxage=300, stale-while-revalidate=3600',
+          },
+        ],
+      },
     ];
   },
   // Updated experimental flags for Next.js 16

src/app/robots.ts

@@ -13,9 +13,6 @@ export default function robots(): MetadataRoute.Robots {
           '/mempool',
           '/discover',
           '/feedback',
-          '/block/*',
-          '/transactions/*',
-          '/address/*',
         ],
         disallow: [
           '/dashboard/',
@@ -24,7 +21,12 @@ export default function robots(): MetadataRoute.Robots {
           '/chat/',
           '/report/',
           '/coin-control/',
-          '/transactions', // Disallow the generic transaction list page
+          // Dynamic explorer pages form an effectively infinite crawl trap
+          // (tx -> address -> tx -> ...) with negligible SEO value and run
+          // uncached server functions on every hit. Keep crawlers out.
+          '/transactions', // Covers the list page and all /transactions/* details
+          '/address/',
+          '/block/',
           '/api/',
         ],
       },
@@ -37,9 +39,6 @@ export default function robots(): MetadataRoute.Robots {
           '/mempool',
           '/discover',
           '/feedback',
-          '/block/*',
-          '/transactions/*',
-          '/address/*',
         ],
         disallow: [
           '/dashboard/',
@@ -49,6 +48,8 @@ export default function robots(): MetadataRoute.Robots {
           '/report/',
           '/coin-control/',
           '/transactions',
+          '/address/',
+          '/block/',
           '/api/',
         ],
       },

src/middleware.ts

@@ -0,0 +1,87 @@
+import { NextResponse, type NextRequest } from 'next/server';
+
+/**
+ * Edge middleware to protect the dynamic blockchain-explorer routes
+ * (`/transactions/[id]`, `/address/[address]`, `/block/[id]`) from crawler
+ * traffic.
+ *
+ * These pages cross-link densely (a transaction links to every input/output
+ * address, each address links back to every transaction, blocks link to prior
+ * blocks + transactions), forming an effectively infinite "spider trap". Every
+ * crawl runs an uncached server function (SSR shell + a `'use server'` data
+ * fetch), so unchecked bots generate large compute->CDN egress around the clock
+ * even with zero human users.
+ *
+ * robots.txt (see `src/app/robots.ts`) tells well-behaved bots to stay out.
+ * This middleware is the enforcement layer for bots that ignore it: it matches
+ * the path (catching both the GET page request and the Server Action POST that
+ * posts back to the same path) and short-circuits known crawler User-Agents at
+ * the edge with a tiny `403` before the expensive Node function ever runs.
+ *
+ * Real users pass through untouched, but every response on these routes carries
+ * `X-Robots-Tag: noindex, nofollow` as defense in depth so the pages are never
+ * indexed even if discovered through other links.
+ */
+
+// Curated list of common search-engine, social, AI and SEO crawler User-Agents,
+// plus a generic fallback. Normal browsers do not match these tokens.
+const BOT_UA_REGEX = new RegExp(
+  [
+    'applebot',
+    'googlebot',
+    'bingbot',
+    'gptbot',
+    'oai-searchbot',
+    'chatgpt',
+    'claudebot',
+    'claude-web',
+    'anthropic',
+    'ccbot',
+    'perplexitybot',
+    'amazonbot',
+    'bytespider',
+    'yandex(bot)?',
+    'baiduspider',
+    'duckduckbot',
+    'slurp', // Yahoo
+    'facebookexternalhit',
+    'meta-externalagent',
+    'semrushbot',
+    'ahrefsbot',
+    'mj12bot',
+    'dotbot',
+    'petalbot',
+    'dataforseobot',
+    // Generic fallback for anything self-identifying as automated.
+    'bot\\b',
+    'crawler',
+    'spider',
+  ].join('|'),
+  'i'
+);
+
+export function middleware(request: NextRequest) {
+  const userAgent = request.headers.get('user-agent') || '';
+
+  if (BOT_UA_REGEX.test(userAgent)) {
+    return new NextResponse(null, {
+      status: 403,
+      headers: {
+        'X-Robots-Tag': 'noindex, nofollow',
+        'Cache-Control': 'no-store',
+      },
+    });
+  }
+
+  const response = NextResponse.next();
+  response.headers.set('X-Robots-Tag', 'noindex, nofollow');
+  return response;
+}
+
+export const config = {
+  matcher: [
+    '/transactions/:path*',
+    '/address/:path*',
+    '/block/:path*',
+  ],
+};