# robots.txt for Super Iron Foundry
# superironfoundry.com

User-agent: *
Allow: /
Allow: /wp-admin/admin-ajax.php

# Allow essential resources
Allow: /*.js$
Allow: /*.css$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$

# Block thin/irrelevant/unnecessary sections
Disallow: /wp-admin/
Disallow: /author/
Disallow: /category/
Disallow: /tags/
Disallow: /*?replytocom
Disallow: /*?utm_*
Disallow: /*?amp

# -------------------------------------
# AI & Generative Search Bots
# -------------------------------------

User-agent: GPTBot
Allow: /
Crawl-delay: 10

User-agent: ChatGPT
Allow: /
Crawl-delay: 10

User-agent: OpenAI
Allow: /
Crawl-delay: 10

User-agent: OAI-Search
Allow: /
Crawl-delay: 10

User-agent: Bard
Allow: /
Crawl-delay: 10

User-agent: Gemini
Allow: /
Crawl-delay: 10

User-agent: ClaudeBot
Allow: /
Crawl-delay: 10

User-agent: PerplexityBot
Allow: /
Crawl-delay: 10

# -------------------------------------
# Major Search Engines
# -------------------------------------

User-agent: Googlebot
Allow: /
Disallow: /category/
Crawl-delay: 5

User-agent: Googlebot-Image
Allow: /
Crawl-delay: 5

User-agent: Bingbot
Allow: /
Disallow: /category/
Crawl-delay: 15

User-agent: YandexBot
Allow: /
Crawl-delay: 25

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 20

User-agent: Baiduspider
Allow: /
Crawl-delay: 30

User-agent: Slurp
Allow: /
Crawl-delay: 20

# -------------------------------------
# Social Media Preview Bots
# -------------------------------------

User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: Instagram
Allow: /

# -------------------------------------
# Apple & Amazon Bots
# -------------------------------------

User-agent: Applebot
Allow: /
Crawl-delay: 10

User-agent: AlexaCrawler
Allow: /
Crawl-delay: 35

# -------------------------------------
# Sitemaps
# -------------------------------------
Sitemap: https://superironfoundry.com/sitemap_index.xml
Sitemap: https://superironfoundry.com/post-sitemap.xml