# ==============================================================================
# ETERNAL HOSTING - ROBOTS.TXT
# Optimized for SEO and Security
# ==============================================================================

# Allow all major search engines to crawl the site
User-agent: *
Allow: /

# ==============================================================================
# BLOCKED DIRECTORIES AND FILES
# ==============================================================================

# Block access to sensitive directories
Disallow: /components/
Disallow: /data/
Disallow: /assets/css/
Disallow: /assets/js/

# Block sensitive files
Disallow: /*.php$
Disallow: /*.inc$
Disallow: /*.conf$
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.bak$

# Block admin and system directories (even if they don't exist)
Disallow: /admin/
Disallow: /administrator/
Disallow: /wp-admin/
Disallow: /cpanel/
Disallow: /plesk/

# Block temporary and backup files
Disallow: /*?
Disallow: /*.tmp
Disallow: /*.backup
Disallow: /*~

# ==============================================================================
# EXPLICITLY ALLOWED IMPORTANT PAGES
# ==============================================================================

# Main pages (ensure these are crawled)
Allow: /
Allow: /minecraft-server-hosting
Allow: /blog
Allow: /privacy-policy
Allow: /terms-of-service

# Blog posts (high-value content)
Allow: /how-to-set-up-your-first-minecraft-server
Allow: /choosing-a-server-host-and-a-server-plan
Allow: /how-to-reduce-server-lag-of-your-minecraft-server
Allow: /how-to-protect-your-minecraft-server-from-hackers-and-griefers
Allow: /securing-your-game-server-against-ddos-attacks
Allow: /how-to-monetize-your-game-servers
Allow: /the-economics-of-running-a-minecraft-server
Allow: /10-minecraft-plugins-to-take-your-server-to-the-next-level

# Important assets for SEO
Allow: /assets/images/
Allow: /*.webp
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.ico

# ==============================================================================
# SEARCH ENGINE SPECIFIC RULES
# ==============================================================================

# Google Bot - Full access with high crawl rate
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Bing Bot - Full access
User-agent: Bingbot
Allow: /
Crawl-delay: 2

# Yahoo Bot - Full access
User-agent: Slurp
Allow: /
Crawl-delay: 2

# DuckDuckGo Bot - Full access
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 2

# Yandex Bot - Full access
User-agent: YandexBot
Allow: /
Crawl-delay: 3

# ==============================================================================
# BLOCK MALICIOUS BOTS AND SCRAPERS
# ==============================================================================

# Block common scrapers and malicious bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# Block AI training bots (optional - uncomment if you want to block)
# User-agent: GPTBot
# Disallow: /

# User-agent: ChatGPT-User
# Disallow: /

# User-agent: CCBot
# Disallow: /

# User-agent: anthropic-ai
# Disallow: /

# User-agent: Claude-Web
# Disallow: /

# ==============================================================================
# SOCIAL MEDIA CRAWLERS (Allow for social sharing)
# ==============================================================================

# Facebook crawler
User-agent: facebookexternalhit
Allow: /

# Twitter crawler
User-agent: Twitterbot
Allow: /

# LinkedIn crawler
User-agent: LinkedInBot
Allow: /

# Discord crawler (for link previews)
User-agent: Discordbot
Allow: /

# ==============================================================================
# SITEMAPS
# ==============================================================================

# XML Sitemap location
Sitemap: https://eternalhosting.cloud/sitemap.xml

# Additional sitemaps (create these files)
Sitemap: https://eternalhosting.cloud/sitemap-blog.xml
Sitemap: https://eternalhosting.cloud/sitemap-pages.xml
Sitemap: https://eternalhosting.cloud/sitemap-images.xml

# ==============================================================================
# CRAWL OPTIMIZATION
# ==============================================================================

# Default crawl delay for unlisted bots
User-agent: *
Crawl-delay: 5

# ==============================================================================
# HOSTING-SPECIFIC OPTIMIZATIONS
# ==============================================================================

# Allow crawling of pricing and plan information
Allow: /*minecraft*
Allow: /*hosting*
Allow: /*server*
Allow: /*plan*
Allow: /*price*

# Allow crawling of help and support content
Allow: /*help*
Allow: /*support*
Allow: /*guide*
Allow: /*tutorial*
Allow: /*how-to*

# Block crawling of potential customer data
Disallow: /*customer*
Disallow: /*billing*
Disallow: /*account*
Disallow: /*login*
Disallow: /*panel*

# ==============================================================================
# PERFORMANCE CONSIDERATIONS
# ==============================================================================

# Prevent crawling of resource-heavy operations
Disallow: /*search*
Disallow: /*filter*
Disallow: /*sort*
Disallow: /*ajax*
Disallow: /*api*

# Block crawling of duplicate content
Disallow: /*?*
Disallow: /*&*
Disallow: /*utm_*
Disallow: /*ref=*
Disallow: /*source=*