# ============================================================
# robots.txt — Optimized for SEO, AEO, LLMs & AI Crawlers
# Site: https://www.sanalifeenergy.com
# Updated: 2026-02-24
# ============================================================


# ============================================================
# GLOBAL DEFAULT — Allow all well-behaved crawlers
# ============================================================
User-agent: *
Allow: /


# ============================================================
# GOOGLE — Full Crawl Access
# ============================================================
User-agent: Googlebot
User-agent: Googlebot-News
User-agent: Googlebot-Image
User-agent: Google-InspectionTool
User-agent: GoogleOther
User-agent: Google-Extended
User-agent: Feedfetcher-Google
User-agent: AdsBot-Google
User-agent: APIs-Google
User-agent: Google-CloudVertexBot
User-agent: Google-Safety
User-agent: Storebot-Google
User-agent: Mediapartners-Google
User-agent: Google-Read-Aloud
Allow: /


# ============================================================
# AI & LLM CRAWLERS — Answer Engine Optimization (AEO)
# ============================================================
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: AppleBot-Extended
User-agent: cohere-ai
User-agent: YouBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Diffbot
User-agent: PhindBot
User-agent: Seekr
User-agent: MistralAI-User
User-agent: CCBot
User-agent: ProRataInc
User-agent: Terracotta
User-agent: Novellum
User-agent: Manus-User
Allow: /


# ============================================================
# GLOBAL SEARCH ENGINES
# ============================================================
User-agent: Bingbot
User-agent: BingPreview
User-agent: AdIdxBot
User-agent: DuckDuckBot
User-agent: DuckAssistBot
User-agent: Amazonbot
User-agent: Slurp
User-agent: YandexBot
User-agent: YandexImages
User-agent: YandexVideo
User-agent: Baiduspider
User-agent: Sogou Spider
User-agent: Yeti
User-agent: Qwantify
User-agent: PetalBot
User-agent: Timpibot
User-agent: MojeekBot
User-agent: SeznamBot
User-agent: archive.org_bot
Allow: /


# ============================================================
# SOCIAL MEDIA & LINK PREVIEW BOTS
# ============================================================
User-agent: LinkedInBot
User-agent: facebookexternalhit
User-agent: FacebookBot
User-agent: Instagrambot
User-agent: Twitterbot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: Slackbot
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: PinterestBot
User-agent: Snap URL Preview Service
User-agent: Anchor Browser
Allow: /


# ============================================================
# RSS, FEED READERS & CONTENT AGGREGATION
# ============================================================
User-agent: Feedly
User-agent: FeedlyBot
User-agent: FlipboardProxy
User-agent: NewsBlur
User-agent: NewsBlurBot
User-agent: Inoreader
User-agent: Feedbin
User-agent: The Old Reader
User-agent: BazQux
User-agent: CommaFeed
User-agent: NetNewsWire
User-agent: Feedspot
User-agent: FeedspotBot
User-agent: Bloglovin
User-agent: Superfeedr
User-agent: Pocket
User-agent: Instapaper
User-agent: Blogtrottr
User-agent: SimplePie
User-agent: UniversalFeedParser
User-agent: fivefilters
User-agent: RSSingBot
User-agent: ia_archiver
Allow: /


# ============================================================
# BLOCK LIST — SEO Audit, Competitive Intelligence & Scrapers
# ============================================================

# Competitive Intelligence & SEO Audit Tools
User-agent: SimilarWebBot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: SEOkicks-robot
User-agent: Rogerbot
User-agent: Sistrix
User-agent: MegaIndex.ru
User-agent: Serpstatbot
User-agent: CognitiveSEOBot
User-agent: OnPageBot
Disallow: /

# B2B Data Harvesters
User-agent: Zoominfobot
User-agent: BomboraBot
User-agent: PiplBot
User-agent: LeadInfoBot
Disallow: /

# Technology Profilers
User-agent: BuiltWith
User-agent: NetcraftSurveyAgent
User-agent: NerdyBot
Disallow: /

# Content Scrapers
User-agent: TurnitinBot
User-agent: HTTrack
User-agent: Webzio
User-agent: webzio-extended
User-agent: IonCrawl
Disallow: /

# Ad Categorization
User-agent: GrapeshotCrawler
Disallow: /

# Aggressive / Abusive Crawlers
User-agent: MauiBot
User-agent: AlphaBot
User-agent: Riddler
User-agent: Seekport
User-agent: ltx71
User-agent: linkdexbot
User-agent: Cliqzbot
User-agent: Cocolyzebot
User-agent: VelenPublicWebCrawler
User-agent: ICC-Crawler
User-agent: SEOENGBot
Disallow: /

# Site Copiers & Rippers
User-agent: BlackWidow
User-agent: WebCopier
User-agent: WebStripper
User-agent: WebZIP
User-agent: SiteSnagger
User-agent: TeleportPro
User-agent: Sucker
User-agent: Grabber
User-agent: ExtractorPro
Disallow: /

# Email Harvesters
User-agent: EmailCollector
User-agent: EmailSiphon
User-agent: EmailWolf
User-agent: Harvest
Disallow: /

# Generic Scraping Frameworks
User-agent: Scrapy
User-agent: Python-urllib
User-agent: python-requests
User-agent: libwww-perl
User-agent: Go-http-client
Disallow: /

# AI Training Scrapers (Non-Reciprocal)
User-agent: img2dataset
User-agent: Kangaroo Bot
User-agent: PanguBot
Disallow: /


# ============================================================
# SITEMAP
# ============================================================
Sitemap: https://www.sanalifeenergy.com/sitemap.xml

Sitemap: https://www.sanalifeenergy.com/sitemap.xml