chore: edge cache headers, agent-aware robots, UA bot-block middleware

Cuts Vercel hobby edge-request burn on both skillkit (Vite static)
and skillkit-docs (Next.js fumadocs) projects. Site stays
human-first AND agent-first — Firecrawl, Context7, Crawl4AI,
OpenClaw, Hermes, ChatGPT-User, Claude-User, PerplexityBot
explicitly allowed; training crawlers and SEO scrapers blocked.

skillkit (Vite, docs/skillkit):
- vercel.json: Cache-Control on assets (1d browser, 7d edge,
  30d SWR; immutable for hashed /assets/*), HTML (5min browser,
  1d edge, 7d SWR), and /api JSON
- redirects with has user-agent: deflect known training+SEO
  bots to /robots.txt; negative lookahead on source prevents
  /robots.txt redirect loop
- public/robots.txt: explicit allow + deny lists

skillkit-docs (Next.js, docs/fumadocs):
- next.config.mjs: headers() block for /_next/static (1y
  immutable), assets (1d/7d/30d), /docs/* and / (5min/1d/7d)
- src/middleware.ts: UA-aware allow/deny pipeline; allowed
  agents pass through, blocked bots get 403 with cache header
  so Vercel edge serves the rejection cheaply
- public/robots.txt: same allow/deny list as marketing site
This commit is contained in:
Rohit Ghumare
2026-04-28 11:16:20 +01:00
parent bf6ec9008b
commit e42ae9d87a
5 changed files with 368 additions and 0 deletions
+28
View File
@@ -8,6 +8,34 @@ const config = {
trailingSlash: false,
skipTrailingSlashRedirect: true,
assetPrefix: 'https://skillkit-docs.vercel.app',
async headers() {
return [
{
source: '/:path*.(css|js|png|jpg|jpeg|svg|webp|woff2|woff|ttf|ico)',
headers: [
{ key: 'Cache-Control', value: 'public, max-age=86400, s-maxage=604800, stale-while-revalidate=2592000' },
],
},
{
source: '/_next/static/:path*',
headers: [
{ key: 'Cache-Control', value: 'public, max-age=31536000, immutable' },
],
},
{
source: '/docs/:path*',
headers: [
{ key: 'Cache-Control', value: 'public, max-age=300, s-maxage=86400, stale-while-revalidate=604800' },
],
},
{
source: '/',
headers: [
{ key: 'Cache-Control', value: 'public, max-age=300, s-maxage=86400, stale-while-revalidate=604800' },
],
},
];
},
};
export default withMDX(config);
+138
View File
@@ -0,0 +1,138 @@
# Humans + agents welcome. Training crawlers + SEO scrapers blocked.
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Applebot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /
User-agent: FirecrawlAgent
Allow: /
User-agent: firecrawl
Allow: /
User-agent: Context7Bot
Allow: /
User-agent: Crawl4AI
Allow: /
User-agent: Clawdbot
Allow: /
User-agent: OpenClaw
Allow: /
User-agent: Hermes
Allow: /
User-agent: GPTBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: Meta-ExternalAgent
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: peer39_crawler
Disallow: /
User-agent: YouBot
Disallow: /
User-agent: Timpibot
Disallow: /
User-agent: ICC-Crawler
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: MegaIndex
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: *
Allow: /
Sitemap: https://skillkit-docs.vercel.app/sitemap.xml
+22
View File
@@ -0,0 +1,22 @@
import { NextResponse } from 'next/server';
import type { NextRequest } from 'next/server';
const BLOCK = /GPTBot|ClaudeBot|anthropic-ai|CCBot|Google-Extended|Applebot-Extended|Bytespider|Amazonbot|Meta-ExternalAgent|cohere-ai|Diffbot|ImagesiftBot|Omgilibot|peer39_crawler|YouBot|Timpibot|ICC-Crawler|AhrefsBot|SemrushBot|MJ12bot|DotBot|PetalBot|BLEXBot|MegaIndex|SeznamBot|DataForSeoBot/i;
const ALLOW = /Googlebot|Bingbot|DuckDuckBot|Applebot(?!-Extended)|ChatGPT-User|OAI-SearchBot|PerplexityBot|Perplexity-User|Claude-User|Claude-SearchBot|FirecrawlAgent|firecrawl|Context7Bot|Crawl4AI|Clawdbot|OpenClaw|Hermes/i;
export function middleware(req: NextRequest) {
const ua = req.headers.get('user-agent') || '';
if (ALLOW.test(ua)) return NextResponse.next();
if (BLOCK.test(ua)) {
return new NextResponse('disallowed by robots.txt', {
status: 403,
headers: { 'Cache-Control': 'public, max-age=86400' },
});
}
return NextResponse.next();
}
export const config = {
matcher: '/((?!_next/static|_next/image|favicon|robots\\.txt|sitemap\\.xml).*)',
};
+138
View File
@@ -0,0 +1,138 @@
# Humans + agents welcome. Training crawlers + SEO scrapers blocked.
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Applebot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /
User-agent: FirecrawlAgent
Allow: /
User-agent: firecrawl
Allow: /
User-agent: Context7Bot
Allow: /
User-agent: Crawl4AI
Allow: /
User-agent: Clawdbot
Allow: /
User-agent: OpenClaw
Allow: /
User-agent: Hermes
Allow: /
User-agent: GPTBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: Meta-ExternalAgent
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: peer39_crawler
Disallow: /
User-agent: YouBot
Disallow: /
User-agent: Timpibot
Disallow: /
User-agent: ICC-Crawler
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: MegaIndex
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: *
Allow: /
Sitemap: https://skillkit.dev/sitemap.xml
+42
View File
@@ -21,5 +21,47 @@
"source": "/docs",
"destination": "https://skillkit-docs.vercel.app/docs"
}
],
"headers": [
{
"source": "/(.*)\\.(css|js|png|jpg|jpeg|svg|webp|woff2|woff|ttf|ico|mp4)",
"headers": [
{ "key": "Cache-Control", "value": "public, max-age=86400, s-maxage=604800, stale-while-revalidate=2592000" }
]
},
{
"source": "/assets/(.*)",
"headers": [
{ "key": "Cache-Control", "value": "public, max-age=31536000, immutable" }
]
},
{
"source": "/(.*)\\.html",
"headers": [
{ "key": "Cache-Control", "value": "public, max-age=300, s-maxage=86400, stale-while-revalidate=604800" }
]
},
{
"source": "/",
"headers": [
{ "key": "Cache-Control", "value": "public, max-age=300, s-maxage=86400, stale-while-revalidate=604800" }
]
},
{
"source": "/api",
"headers": [
{ "key": "Cache-Control", "value": "public, max-age=300, s-maxage=86400, stale-while-revalidate=604800" }
]
}
],
"redirects": [
{
"source": "/((?!robots\\.txt$).*)",
"has": [
{ "type": "header", "key": "user-agent", "value": "(?i).*(GPTBot|ClaudeBot|anthropic-ai|CCBot|Google-Extended|Applebot-Extended|Bytespider|Amazonbot|Meta-ExternalAgent|cohere-ai|Diffbot|ImagesiftBot|Omgilibot|peer39_crawler|YouBot|Timpibot|ICC-Crawler|AhrefsBot|SemrushBot|MJ12bot|DotBot|PetalBot|BLEXBot|MegaIndex|SeznamBot|DataForSeoBot).*" }
],
"destination": "/robots.txt",
"permanent": false
}
]
}