# ============================================================
#  robots.txt  —  ingestlabs.com
#  Updated for AI search / GEO visibility (July 2026)
# ============================================================
# Notes for maintainers:
#   • The wildcard group below is the source of truth for
#     thin / duplicate / internal paths.
#   • Named AI groups repeat the sensitive Disallows (/admin/,
#     /api/) so a named crawler never gets broader access than
#     the default. If you also want AI crawlers to skip the
#     thin-content paths, copy the full Disallow list from the
#     wildcard group into each AI group.
# ============================================================

Sitemap: https://ingestlabs.com/sitemap.xml

# ------------------------------------------------------------
# Default rules for all bots
# ------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /discovering-and-managing-audience-insights/
Disallow: /services-style-01/
Disallow: /author/rohan/
Disallow: /author/dhiren/
Disallow: /author/puneeth_mp/
Disallow: /blog-style-01/
Disallow: /landing-page-2/
Disallow: /testimonials/
Disallow: /idl/
Disallow: /c-sdk/
Disallow: /how-to-obtain-valid-consent-under-the-gdpr-or-apps/
Disallow: /portfolio/server-maintenance/
Disallow: /service/ingestt-iq/
Disallow: /complete-guide-to-facebook-retargeting-strategies/

# ------------------------------------------------------------
# AI TRAINING CRAWLERS — allowed to learn from our content
#   Google-Extended  : Gemini / Vertex AI training & grounding
#   Applebot-Extended: Apple Intelligence training
#   (These default to "allowed" if unlisted; naming them makes
#    our intent explicit and auditable.)
# ------------------------------------------------------------
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/
Disallow: /api/

# ------------------------------------------------------------
# AI SEARCH & ASSISTANT FETCHERS — surface us in AI answers
#   OAI-SearchBot   : OpenAI search index (distinct from GPTBot)
#   ChatGPT-User    : live fetch when a user asks ChatGPT
#   PerplexityBot   : Perplexity search index
#   Perplexity-User : live fetch for a Perplexity user query
#   Claude-User     : live fetch for a Claude user query
# ------------------------------------------------------------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-User
User-agent: Claude-Web
Allow: /
Disallow: /admin/
Disallow: /api/

# ------------------------------------------------------------
# OPTIONAL — to BLOCK a crawler (e.g. ByteDance / TikTok),
# uncomment the two lines below. Left commented = allowed via
# the wildcard group above.
# ------------------------------------------------------------
# User-agent: Bytespider
# Disallow: /