# robots.txt — Lagos Flavors
# Nigerian restaurant, 10514 111 Street NW, Edmonton, AB
# https://www.lagosflavours.com/

# ------------------------------------------------------------
# All crawlers
# This is the catch-all group: it covers Googlebot, Bingbot,
# DuckDuckBot and every other bot that has no dedicated group
# of its own below (per the robots.txt spec, a bot always
# matches its own named group first, and only falls back to
# "*" when none exists).
# ------------------------------------------------------------
User-agent: *
Allow: /
Allow: /assets/

# Build partials and internal files — not standalone pages
Disallow: /components/
Disallow: /Store%20Information.txt
Disallow: /update-menu.js

# llms.txt is written for AI agents, not human searchers, so we
# ask general search engines (Google, Bing, DuckDuckGo, etc. —
# everything matching this "*" group) to keep it out of their
# index. Note this Noindex directive is unofficial: Google
# stopped honoring it in robots.txt in September 2019, though
# some other engines still respect it, so treat it as a best
# effort, not a guarantee. AI crawlers are unaffected by this
# line — they match their own group below instead, which
# explicitly Allows llms.txt with no Noindex.
Noindex: /llms.txt

# Avoid duplicate content from tracking / filter parameters
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*author=
Disallow: /*?*tag=
Disallow: /*?*month=
Disallow: /*?*view=
Disallow: /*?*format=

# ------------------------------------------------------------
# AI / LLM crawlers — full access to content, llms.txt included
# ------------------------------------------------------------
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: cohere-ai
User-agent: Bytespider
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: FacebookBot
Allow: /
Allow: /llms.txt
Disallow: /components/
Disallow: /Store%20Information.txt
Disallow: /privacy-policy.html
Disallow: /terms-of-use.html

# ------------------------------------------------------------
# Google Ads
# ------------------------------------------------------------
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google-Mobile-Apps
Allow: /

# ------------------------------------------------------------
# Crawl delay for heavy / low-value bots
# ------------------------------------------------------------
User-agent: Baiduspider
Crawl-delay: 10

User-agent: YandexBot
Crawl-delay: 10

# ------------------------------------------------------------
# Sitemap
# ------------------------------------------------------------
Sitemap: https://www.lagosflavours.com/sitemap.xml