# =========================
# forustone.com robots.txt
# SEO + AI Search Friendly + Server Protection
# =========================

# --- High-frequency crawler restrictions ---
User-agent: meta-webindexer
Crawl-delay: 10
Disallow: /wp-json/
Disallow: /wp-admin/
Disallow: /wp-login.php

# --- Other aggressive crawlers to throttle ---
User-agent: Baiduspider
Crawl-delay: 5
Disallow: /wp-json/

User-agent: SemrushBot
Crawl-delay: 5

User-agent: AhrefsBot
Crawl-delay: 5

# --- Major search engines ---
User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php

User-agent: Bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php

User-agent: Applebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# --- OpenAI Search ---
User-agent: OAI-SearchBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# --- Perplexity Search ---
User-agent: PerplexityBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# --- Anthropic / Claude Search ---
User-agent: Claude-SearchBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php

User-agent: Claude-User
Allow: /

# --- AI training crawlers blocked ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

# --- Conservative blocks for unwanted crawlers ---
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: crawler4j
Disallow: /

User-agent: Scrapy
Disallow: /

# --- Default rules ---
User-agent: *
Crawl-delay: 3
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /cdn-cgi/
Allow: /wp-admin/admin-ajax.php

Sitemap: https://forustone.com/sitemap_index.xml