# ==============================================================
# ROBOTS.TXT - SFI Group
# Generated automatically by Next.js Route Handler
# Reference: https://www.robotstxt.org/
# ==============================================================

# ── DEFAULT RULES (untuk semua crawlers) ──
User-agent: *
Allow: /
Allow: /sitemap.xml

# ── DISALLOW PRIVATE PATHS ──
# Pages yang tidak ingin di-index
Disallow: /admin
Disallow: /private
Disallow: /api
Disallow: /maintenance
Disallow: /under-maintenance

# ── DISALLOW FILE TYPES ──
# Jangan crawl generated files (sudah di-compress)
Disallow: /*.json$
Disallow: /*.js$
Disallow: /*.css$

# ── GOOGLE SPECIFIC ──
# Google bot punya akses penuh
User-agent: Googlebot
Allow: /

# ── BING SPECIFIC ──
# Bing bot punya akses penuh
User-agent: Bingbot
Allow: /

# ── CRAWL DELAY ──
# Delay 1 detik antara requests (opsional)
# Mencegah server overload dari crawlers
Crawl-delay: 1

# ── SITEMAPS ──
# Location of sitemap files
Sitemap: https://sfigroup.co.id/sitemap.xml