mirror of
https://github.com/rohitg00/skillkit.git
synced 2026-11-18 18:04:26 +00:00
chore: edge cache headers, agent-aware robots, UA bot-block middleware
Cuts Vercel hobby edge-request burn on both skillkit (Vite static) and skillkit-docs (Next.js fumadocs) projects. Site stays human-first AND agent-first — Firecrawl, Context7, Crawl4AI, OpenClaw, Hermes, ChatGPT-User, Claude-User, PerplexityBot explicitly allowed; training crawlers and SEO scrapers blocked. skillkit (Vite, docs/skillkit): - vercel.json: Cache-Control on assets (1d browser, 7d edge, 30d SWR; immutable for hashed /assets/*), HTML (5min browser, 1d edge, 7d SWR), and /api JSON - redirects with has user-agent: deflect known training+SEO bots to /robots.txt; negative lookahead on source prevents /robots.txt redirect loop - public/robots.txt: explicit allow + deny lists skillkit-docs (Next.js, docs/fumadocs): - next.config.mjs: headers() block for /_next/static (1y immutable), assets (1d/7d/30d), /docs/* and / (5min/1d/7d) - src/middleware.ts: UA-aware allow/deny pipeline; allowed agents pass through, blocked bots get 403 with cache header so Vercel edge serves the rejection cheaply - public/robots.txt: same allow/deny list as marketing site
This commit is contained in:
@@ -8,6 +8,34 @@ const config = {
|
||||
trailingSlash: false,
|
||||
skipTrailingSlashRedirect: true,
|
||||
assetPrefix: 'https://skillkit-docs.vercel.app',
|
||||
async headers() {
|
||||
return [
|
||||
{
|
||||
source: '/:path*.(css|js|png|jpg|jpeg|svg|webp|woff2|woff|ttf|ico)',
|
||||
headers: [
|
||||
{ key: 'Cache-Control', value: 'public, max-age=86400, s-maxage=604800, stale-while-revalidate=2592000' },
|
||||
],
|
||||
},
|
||||
{
|
||||
source: '/_next/static/:path*',
|
||||
headers: [
|
||||
{ key: 'Cache-Control', value: 'public, max-age=31536000, immutable' },
|
||||
],
|
||||
},
|
||||
{
|
||||
source: '/docs/:path*',
|
||||
headers: [
|
||||
{ key: 'Cache-Control', value: 'public, max-age=300, s-maxage=86400, stale-while-revalidate=604800' },
|
||||
],
|
||||
},
|
||||
{
|
||||
source: '/',
|
||||
headers: [
|
||||
{ key: 'Cache-Control', value: 'public, max-age=300, s-maxage=86400, stale-while-revalidate=604800' },
|
||||
],
|
||||
},
|
||||
];
|
||||
},
|
||||
};
|
||||
|
||||
export default withMDX(config);
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
# Humans + agents welcome. Training crawlers + SEO scrapers blocked.
|
||||
|
||||
User-agent: Googlebot
|
||||
Allow: /
|
||||
|
||||
User-agent: Bingbot
|
||||
Allow: /
|
||||
|
||||
User-agent: DuckDuckBot
|
||||
Allow: /
|
||||
|
||||
User-agent: Applebot
|
||||
Allow: /
|
||||
|
||||
User-agent: ChatGPT-User
|
||||
Allow: /
|
||||
|
||||
User-agent: OAI-SearchBot
|
||||
Allow: /
|
||||
|
||||
User-agent: PerplexityBot
|
||||
Allow: /
|
||||
|
||||
User-agent: Perplexity-User
|
||||
Allow: /
|
||||
|
||||
User-agent: Claude-User
|
||||
Allow: /
|
||||
|
||||
User-agent: Claude-SearchBot
|
||||
Allow: /
|
||||
|
||||
User-agent: FirecrawlAgent
|
||||
Allow: /
|
||||
|
||||
User-agent: firecrawl
|
||||
Allow: /
|
||||
|
||||
User-agent: Context7Bot
|
||||
Allow: /
|
||||
|
||||
User-agent: Crawl4AI
|
||||
Allow: /
|
||||
|
||||
User-agent: Clawdbot
|
||||
Allow: /
|
||||
|
||||
User-agent: OpenClaw
|
||||
Allow: /
|
||||
|
||||
User-agent: Hermes
|
||||
Allow: /
|
||||
|
||||
User-agent: GPTBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: ClaudeBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: anthropic-ai
|
||||
Disallow: /
|
||||
|
||||
User-agent: CCBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Google-Extended
|
||||
Disallow: /
|
||||
|
||||
User-agent: Applebot-Extended
|
||||
Disallow: /
|
||||
|
||||
User-agent: Bytespider
|
||||
Disallow: /
|
||||
|
||||
User-agent: Amazonbot
|
||||
Disallow: /
|
||||
|
||||
User-agent: FacebookBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Meta-ExternalAgent
|
||||
Disallow: /
|
||||
|
||||
User-agent: cohere-ai
|
||||
Disallow: /
|
||||
|
||||
User-agent: Diffbot
|
||||
Disallow: /
|
||||
|
||||
User-agent: ImagesiftBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Omgilibot
|
||||
Disallow: /
|
||||
|
||||
User-agent: peer39_crawler
|
||||
Disallow: /
|
||||
|
||||
User-agent: YouBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Timpibot
|
||||
Disallow: /
|
||||
|
||||
User-agent: ICC-Crawler
|
||||
Disallow: /
|
||||
|
||||
User-agent: AhrefsBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: SemrushBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: MJ12bot
|
||||
Disallow: /
|
||||
|
||||
User-agent: DotBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: PetalBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: BLEXBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: MegaIndex
|
||||
Disallow: /
|
||||
|
||||
User-agent: SeznamBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: DataForSeoBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: *
|
||||
Allow: /
|
||||
|
||||
Sitemap: https://skillkit-docs.vercel.app/sitemap.xml
|
||||
@@ -0,0 +1,22 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import type { NextRequest } from 'next/server';
|
||||
|
||||
const BLOCK = /GPTBot|ClaudeBot|anthropic-ai|CCBot|Google-Extended|Applebot-Extended|Bytespider|Amazonbot|Meta-ExternalAgent|cohere-ai|Diffbot|ImagesiftBot|Omgilibot|peer39_crawler|YouBot|Timpibot|ICC-Crawler|AhrefsBot|SemrushBot|MJ12bot|DotBot|PetalBot|BLEXBot|MegaIndex|SeznamBot|DataForSeoBot/i;
|
||||
|
||||
const ALLOW = /Googlebot|Bingbot|DuckDuckBot|Applebot(?!-Extended)|ChatGPT-User|OAI-SearchBot|PerplexityBot|Perplexity-User|Claude-User|Claude-SearchBot|FirecrawlAgent|firecrawl|Context7Bot|Crawl4AI|Clawdbot|OpenClaw|Hermes/i;
|
||||
|
||||
export function middleware(req: NextRequest) {
|
||||
const ua = req.headers.get('user-agent') || '';
|
||||
if (ALLOW.test(ua)) return NextResponse.next();
|
||||
if (BLOCK.test(ua)) {
|
||||
return new NextResponse('disallowed by robots.txt', {
|
||||
status: 403,
|
||||
headers: { 'Cache-Control': 'public, max-age=86400' },
|
||||
});
|
||||
}
|
||||
return NextResponse.next();
|
||||
}
|
||||
|
||||
export const config = {
|
||||
matcher: '/((?!_next/static|_next/image|favicon|robots\\.txt|sitemap\\.xml).*)',
|
||||
};
|
||||
@@ -0,0 +1,138 @@
|
||||
# Humans + agents welcome. Training crawlers + SEO scrapers blocked.
|
||||
|
||||
User-agent: Googlebot
|
||||
Allow: /
|
||||
|
||||
User-agent: Bingbot
|
||||
Allow: /
|
||||
|
||||
User-agent: DuckDuckBot
|
||||
Allow: /
|
||||
|
||||
User-agent: Applebot
|
||||
Allow: /
|
||||
|
||||
User-agent: ChatGPT-User
|
||||
Allow: /
|
||||
|
||||
User-agent: OAI-SearchBot
|
||||
Allow: /
|
||||
|
||||
User-agent: PerplexityBot
|
||||
Allow: /
|
||||
|
||||
User-agent: Perplexity-User
|
||||
Allow: /
|
||||
|
||||
User-agent: Claude-User
|
||||
Allow: /
|
||||
|
||||
User-agent: Claude-SearchBot
|
||||
Allow: /
|
||||
|
||||
User-agent: FirecrawlAgent
|
||||
Allow: /
|
||||
|
||||
User-agent: firecrawl
|
||||
Allow: /
|
||||
|
||||
User-agent: Context7Bot
|
||||
Allow: /
|
||||
|
||||
User-agent: Crawl4AI
|
||||
Allow: /
|
||||
|
||||
User-agent: Clawdbot
|
||||
Allow: /
|
||||
|
||||
User-agent: OpenClaw
|
||||
Allow: /
|
||||
|
||||
User-agent: Hermes
|
||||
Allow: /
|
||||
|
||||
User-agent: GPTBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: ClaudeBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: anthropic-ai
|
||||
Disallow: /
|
||||
|
||||
User-agent: CCBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Google-Extended
|
||||
Disallow: /
|
||||
|
||||
User-agent: Applebot-Extended
|
||||
Disallow: /
|
||||
|
||||
User-agent: Bytespider
|
||||
Disallow: /
|
||||
|
||||
User-agent: Amazonbot
|
||||
Disallow: /
|
||||
|
||||
User-agent: FacebookBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Meta-ExternalAgent
|
||||
Disallow: /
|
||||
|
||||
User-agent: cohere-ai
|
||||
Disallow: /
|
||||
|
||||
User-agent: Diffbot
|
||||
Disallow: /
|
||||
|
||||
User-agent: ImagesiftBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Omgilibot
|
||||
Disallow: /
|
||||
|
||||
User-agent: peer39_crawler
|
||||
Disallow: /
|
||||
|
||||
User-agent: YouBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Timpibot
|
||||
Disallow: /
|
||||
|
||||
User-agent: ICC-Crawler
|
||||
Disallow: /
|
||||
|
||||
User-agent: AhrefsBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: SemrushBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: MJ12bot
|
||||
Disallow: /
|
||||
|
||||
User-agent: DotBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: PetalBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: BLEXBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: MegaIndex
|
||||
Disallow: /
|
||||
|
||||
User-agent: SeznamBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: DataForSeoBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: *
|
||||
Allow: /
|
||||
|
||||
Sitemap: https://skillkit.dev/sitemap.xml
|
||||
@@ -21,5 +21,47 @@
|
||||
"source": "/docs",
|
||||
"destination": "https://skillkit-docs.vercel.app/docs"
|
||||
}
|
||||
],
|
||||
"headers": [
|
||||
{
|
||||
"source": "/(.*)\\.(css|js|png|jpg|jpeg|svg|webp|woff2|woff|ttf|ico|mp4)",
|
||||
"headers": [
|
||||
{ "key": "Cache-Control", "value": "public, max-age=86400, s-maxage=604800, stale-while-revalidate=2592000" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"source": "/assets/(.*)",
|
||||
"headers": [
|
||||
{ "key": "Cache-Control", "value": "public, max-age=31536000, immutable" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"source": "/(.*)\\.html",
|
||||
"headers": [
|
||||
{ "key": "Cache-Control", "value": "public, max-age=300, s-maxage=86400, stale-while-revalidate=604800" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"source": "/",
|
||||
"headers": [
|
||||
{ "key": "Cache-Control", "value": "public, max-age=300, s-maxage=86400, stale-while-revalidate=604800" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"source": "/api",
|
||||
"headers": [
|
||||
{ "key": "Cache-Control", "value": "public, max-age=300, s-maxage=86400, stale-while-revalidate=604800" }
|
||||
]
|
||||
}
|
||||
],
|
||||
"redirects": [
|
||||
{
|
||||
"source": "/((?!robots\\.txt$).*)",
|
||||
"has": [
|
||||
{ "type": "header", "key": "user-agent", "value": "(?i).*(GPTBot|ClaudeBot|anthropic-ai|CCBot|Google-Extended|Applebot-Extended|Bytespider|Amazonbot|Meta-ExternalAgent|cohere-ai|Diffbot|ImagesiftBot|Omgilibot|peer39_crawler|YouBot|Timpibot|ICC-Crawler|AhrefsBot|SemrushBot|MJ12bot|DotBot|PetalBot|BLEXBot|MegaIndex|SeznamBot|DataForSeoBot).*" }
|
||||
],
|
||||
"destination": "/robots.txt",
|
||||
"permanent": false
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user