User-agent: *
Disallow: /auth/login-otp/
Disallow: /auth/login-password/
Disallow: /auth/oauth-login/
Disallow: /dashboard/
Disallow: /ev
Disallow: /api/csrf/
Disallow: /api/messages/
# Episode permalinks are noindex and outside the sitemap; crawlers spent
# ~800k requests a week on them (2026-09).
Disallow: /e/
# Episode-list pages and sorts on a show page (?page=, ?sort=, ?q=) are noindex
# with a canonical back to the show; crawlers spent ~340k requests a week on
# them (2026-10). The directory's ?category= and ?letter= pages stay crawlable.
Disallow: /podcasts/*/?

# Training-only and zero-referral crawlers. Search-facing agents that send
# visitors (OAI-SearchBot, ChatGPT-User, Claude-SearchBot, PerplexityBot) stay
# allowed: PerplexityBot indexes for Perplexity search, not model training
# (docs.perplexity.ai/guides/bots), and sent 58 visits in 90 days (2026-10).
User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ExaSearchBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: PetalBot
Disallow: /

# AI search/data APIs and SEO-tool link indexes: zero visitors in 91 days
# against ~1.25M requests a week, 60% of all traffic (2026-10).
User-agent: ShapBot
Disallow: /

User-agent: KeenableBot
Disallow: /

User-agent: Keenable-SearchBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

User-agent: Reflectionbot
Disallow: /

User-agent: AionBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SERankingBacklinksBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: AwarioBot
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

User-agent: IbouBot
Disallow: /

Sitemap: https://grep.fm/sitemap.xml