# OneFindMe — onefindme.com
# AI-powered multilingual AliExpress product search engine.
# AI assistants & answer engines are explicitly WELCOME (LLMO / GEO).

# Content signals (advisory): we permit search indexing AND use of our content as
# input to AI answer/search systems (RAG, grounding). Training is also permitted so
# the brand becomes known to models over time.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
# Only the admin surface is withheld. The service worker and the web app
# manifests used to be blocked here, and since every page links its manifest and
# registers the worker, Google kept discovering files it was then forbidden to
# fetch — which is what "blocked by robots.txt" was reporting. Nothing was
# gained by hiding them: they are public files, and Google reads the manifest
# for install and mobile signals.
Disallow: /admin.html
Disallow: /js/admin.js
Disallow: /js/admin-behavior.js
Disallow: /deals/admin.html
Disallow: /go/admin.html

# ── AI assistants, answer engines & search crawlers — explicitly allowed ──
# ChatGPT (search + browse + training)
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: GPTBot
Allow: /
# Claude — Anthropic now splits its crawlers by purpose. Claude-SearchBot is the one
# that fetches a page in order to CITE it in an answer, so it is the one that matters
# here; ClaudeBot is training and Claude-Web is user-initiated browsing.
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: Claude-SearchBot
Allow: /
User-agent: anthropic-ai
Allow: /
# Perplexity
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /
# Google (Gemini / AI Overviews) + Common Crawl (feeds many models) + others
User-agent: Google-Extended
Allow: /
User-agent: CCBot
Allow: /
User-agent: cohere-ai
Allow: /
User-agent: YouBot
Allow: /
User-agent: DuckAssistBot
Allow: /
User-agent: Amazonbot
Allow: /
User-agent: Applebot-Extended
Allow: /
User-agent: Meta-ExternalAgent
Allow: /
User-agent: Bytespider
Allow: /

Sitemap: https://onefindme.com/sitemap.xml
# AI guide: https://onefindme.com/llms.txt

# OneFindMe — AI-powered multilingual product search engine · https://onefindme.com
# Languages: Hebrew, English, Arabic, Russian, Spanish, Portuguese, Turkish, French, German, Italian, Polish