# robots.txt — apimatic.io
#
# Groups:
#   1. All crawlers      — legacy blog-tag / apidoc exclusions
#   2. AI training bots  — blocked (no model-training use of this site)
#   3. AI search + user-fetch bots — allowed (AI-search visibility retained)


# ---------------------------------------------------------------------------
# 1. All crawlers
# ---------------------------------------------------------------------------


User-agent: *

# Paginated archives — invalid page indexes
Disallow: /blog/page/0/
Disallow: /events/page/0/
Disallow: /events/page/-1
Disallow: /art-of-developer-experience/page/-1

# Blog tag archives
Disallow: /blog/tag/api-blueprint/
Disallow: /blog/tag/api-description/
Disallow: /blog/tag/api-description/feed/
Disallow: /blog/tag/api-description-formats
Disallow: /blog/tag/api-event/
Disallow: /blog/tag/api-event/feed/
Disallow: /blog/tag/api-fortress/
Disallow: /blog/tag/api-lifecycle/
Disallow: /blog/tag/api-portal/page/2/
Disallow: /blog/tag/api-transformer
Disallow: /blog/tag/api-transformer/
Disallow: /blog/tag/api-transformer/page/1/
Disallow: /blog/tag/api
Disallow: /blog/tag/api/
Disallow: /blog/tag/api/page/1/
Disallow: /blog/tag/api/page/2/
Disallow: /blog/tag/api/page/4/
Disallow: /blog/tag/api/page/5/
Disallow: /blog/tag/apimatic
Disallow: /blog/tag/apimatic/
Disallow: /blog/tag/apimatic/page/1/
Disallow: /blog/tag/apimatic/page/2/
Disallow: /blog/tag/apimatic/page/3/
Disallow: /blog/tag/apimatic-features/
Disallow: /blog/tag/apimatic-features/page/2/
Disallow: /blog/tag/apimatic-partners/
Disallow: /blog/tag/apis/
Disallow: /blog/tag/auto-generated-sdks/
Disallow: /blog/tag/auto-generated-sdks/page/1/
Disallow: /blog/tag/auto-generated-sdks/page/2/
Disallow: /blog/tag/automated-sdks/
Disallow: /blog/tag/code-generation
Disallow: /blog/tag/code-generation/feed/
Disallow: /blog/tag/code-generation/page/2/
Disallow: /blog/tag/code-sample/
Disallow: /blog/tag/codegen
Disallow: /blog/tag/coding-standards/
Disallow: /blog/tag/crypto-apis/
Disallow: /blog/tag/crypto-websocket-api/
Disallow: /blog/tag/custom-exception-classes/
Disallow: /blog/tag/customer/
Disallow: /blog/tag/customer-support/
Disallow: /blog/tag/developer-experience
Disallow: /blog/tag/developer-experience/
Disallow: /blog/tag/developer-experience/page/1/
Disallow: /blog/tag/developer-experience/page/2/
Disallow: /blog/tag/developer-relations/
Disallow: /blog/tag/developer-tools/
Disallow: /blog/tag/developers/
Disallow: /blog/tag/dude-solutions/
Disallow: /blog/tag/dx/
Disallow: /blog/tag/github
Disallow: /blog/tag/http-docs/
Disallow: /blog/tag/iaas/
Disallow: /blog/tag/kubernetes/
Disallow: /blog/tag/mistakes/
Disallow: /blog/tag/mobile-apps/
Disallow: /blog/tag/multiple-base-uri-support/
Disallow: /blog/tag/oauth2
Disallow: /blog/tag/open-api/
Disallow: /blog/tag/organizational-culture/
Disallow: /blog/tag/restful-api/
Disallow: /blog/tag/sdk/feed/
Disallow: /blog/tag/sdk-timeouts/
Disallow: /blog/tag/sdks/
Disallow: /blog/tag/sdks/feed/
Disallow: /blog/tag/sdks/page/1/
Disallow: /blog/tag/sdks/page/2/
Disallow: /blog/tag/sdks/page/3/
Disallow: /blog/tag/sms/
Disallow: /blog/tag/sms-marketing/
Disallow: /blog/tag/software/
Disallow: /blog/tag/startup
Disallow: /blog/tag/startup-support-program/
Disallow: /blog/tag/swagger/
Disallow: /blog/tag/team-culture/
Disallow: /blog/tag/transformer/
Disallow: /blog/tag/trends/
Disallow: /blog/tag/usage-trends/
Disallow: /blog/tag/user-interface/
Disallow: /blog/tag/wtd-san-francisco/

# Legacy blog posts
Disallow: /blog/2015/07/how-voxbone-dealt-with-api-changes-and-seamless-developers-transition-4ed68dca020c/
Disallow: /blog/2015/07/how-voxbone-dealt-with-api-changes-and-seamless-developers-transition/?source=post_page-----4ed68dca020c----------------------
Disallow: /blog/2016/02/rapidly-expand-reach-and-reduce-time-to-market-with-auto-generated-skds-for-private-apis-d4f9e659eb23/
Disallow: /blog/2016/02/rapidly-expand-reach-and-reduce-time-to-market-with-auto-generated-skds-for-private-apis/
Disallow: /blog/2017/02/do-more-with-apimatic-and-azure-integration/support@devapi.dream.press

# Podcast
Disallow: /art-of-developer-experience/podcast-brian-rinaldi
Disallow: /art-of-developer-experience/podcast-brian-rinaldi/

# API docs — /api-docs/
Disallow: /api-docs/apimatic/v/1_0
Disallow: /api-docs/bankaccount-data/v/1_0
Disallow: /api-docs/bankaccount-data/v/2_0
Disallow: /api-docs/data-compliance/v/1_0
Disallow: /api-docs/data-curation/v/2_0
Disallow: /api-docs/data-exchange/v/4_0
Disallow: /api-docs/data-validation/v/2_0
Disallow: /api-docs/docsmorev16/v/
Disallow: /api-docs/effect-group/v/
Disallow: /api-docs/ftlightning/v/
Disallow: /api-docs/keithics-rest-api/v/
Disallow: /api-docs/treks-billing-api/v/

# API docs — /apidocs/
Disallow: /apidocs/apimatic
Disallow: /apidocs/apimatic/
Disallow: /apidocs/apimatic/v/
Disallow: /apidocs/apimatic/v/1_0
Disallow: /apidocs/brytemap/v/
Disallow: /apidocs/cgaas-api
Disallow: /apidocs/cgaas-api/
Disallow: /apidocs/connex-platform/v/
Disallow: /apidocs/d7api/v/
Disallow: /apidocs/data-curation
Disallow: /apidocs/data-curation/v/2_0
Disallow: /apidocs/data-exchange
Disallow: /apidocs/globaltillapi/v/
Disallow: /apidocs/neutrinoapi/v/
Disallow: /apidocs/poc-erwin/v/
Disallow: /apidocs/stagingpro/v/
Disallow: /apidocs/unifonic-authenticate
Disallow: /apidocs/unifonic-notice
Disallow: /apidocs/whatsapp-api/v/
Disallow: /apidocs/xionglobal/v/

# Other
Disallow: /apimatic-playbook/


# ---------------------------------------------------------------------------
# 2. AI training crawlers — blocked
#    Named groups do NOT inherit the "*" rules, so each states Disallow: /
# ---------------------------------------------------------------------------

# OpenAI — model training
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: GPTBot
Disallow: /

# Anthropic — model training
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: ClaudeBot
Disallow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: anthropic-ai
Disallow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Claude-Web
Disallow: /

# Common Crawl — corpus widely used for model training
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: CCBot
Disallow: /

# Meta — model training
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: meta-externalagent
Disallow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: FacebookBot
Disallow: /

# ByteDance — model training
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Bytespider
Disallow: /

# Amazon — model training
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Amazonbot
Disallow: /

# Training opt-out flags. These tokens gate training use only; they are not
# crawlers, so Disallow here does NOT affect Google Search or Apple search.
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Google-Extended
Disallow: /

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Applebot-Extended
Disallow: /


# ---------------------------------------------------------------------------
# 3. AI search + user-triggered fetch — allowed
#    Not training crawlers. These power AI-search visibility and render pages
#    a user explicitly pasted into a chat. Blocking them costs reach.
# ---------------------------------------------------------------------------

# OpenAI — ChatGPT search index
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: OAI-SearchBot
Allow: /

# OpenAI — fetches a link a user pasted
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: ChatGPT-User
Allow: /

# Anthropic — Claude search index
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Claude-SearchBot
Allow: /

# Anthropic — fetches a link a user pasted
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Claude-User
Allow: /

# Perplexity — search index
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: PerplexityBot
Allow: /


Sitemap: http://www.apimatic.io/sitemap.xml
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*