# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
# ┃ Robots policy                                                ┃
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
# ┃ We block sensitive system endpoints.                         ┃
# ┃                                                              ┃
# ┃ While keeping checkout/account/ protected.                   ┃
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛

# ========== Default rules (apply to all crawlers) ==========
User-agent: *
Allow: /
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /*/checkouts
Disallow: /*/orders
Disallow: /account
Disallow: /account/register
Disallow: /a/downloads/-/*
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /cdn/wpm/*.js
Disallow: /sf_private_access_tokens
Disallow: /recommendations/products
Disallow: /*/recommendations/products
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
# Duplicate/parameterized catalog pages
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: */collections/*filter*&*filter*
# Duplicate blog param variants
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*
# Remote product preview URLs
Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

# ========== Explicit AI crawler allowlist ==========
# (They still inherit the same Disallow protections as above.)

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

# Anthropic
User-agent: ClaudeBot
User-agent: Claude-Web
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Common Crawl (consumed by many AI systems)
User-agent: CCBot
Allow: /

# Google's generative AI control (not a crawler, but respected)
User-agent: Google-Extended
Allow: /

# Apple AI
User-agent: Applebot-Extended
Allow: /

# Meta AI fetchers
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
Allow: /

# ByteDance / Bytespider (used by several AI products)
User-agent: Bytespider
Allow: /

# (Optional) keep Google AdsBot explicit, mirroring safe blocks
User-agent: AdsBot-Google
Allow: /
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /*/checkouts
Disallow: /*/orders
Disallow: /account
Disallow: /account/register
Disallow: /cdn/wpm/*.js
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /recommendations/products
Disallow: /*/recommendations/products

# ========== Sitemap ==========
Sitemap: https://obsidia-tcg.store/sitemap.xml