Sitemap: https://www.homebrewfinds.com/sitemap.xml
Sitemap: https://www.homebrewfinds.com/news-sitemap.xml
# ===============================================================
# Homebrew Finds Robots.txt
# Purpose: Optimize crawl efficiency and direct bots to valuable content.
# Last updated: 2025-10-29
# ===============================================================

User-agent: *

# --- Block low-value or duplicate content ---
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /?s=
Disallow: *&s=
Disallow: /search
Disallow: /author/
Disallow: *?attachment_id=
Disallow: */feed
Disallow: */rss
Disallow: */embed

# --- Allow critical assets and uploads ---
Allow: /wp-content/uploads/
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.pdf$

# ===============================================================
# Major search engines (don’t block these)
# ===============================================================
User-agent: Googlebot
Disallow:
User-agent: Googlebot-Image
Disallow:
User-agent: AdsBot-Google
Disallow:
User-agent: AdsBot-Google-Mobile
Disallow:
User-agent: Mediapartners-Google
Disallow:
User-agent: Bingbot
Disallow:

# ===============================================================
# Block known aggressive or non-essential crawlers
# (add/remove to taste; case-insensitive matching)
# ===============================================================
User-agent: AhrefsBot
Disallow: /
User-agent: AhrefsSiteAudit
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: IstellaBot
Disallow: /
User-agent: MegaIndex.ru
Disallow: /
User-agent: MJ12Bot
Disallow: /
User-agent: MauiBot
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: OAI-SearchBot
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: SeekportBot
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: trendictionbot
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: ZoominfoBot
Disallow: /
User-agent: Google-Extended
Disallow: /

# --- Social preview bots (block ONLY if you do NOT want rich previews) ---
# User-agent: FacebookExternalHit
# Disallow: /
# User-agent: Twitterbot
# Disallow: /
# User-agent: LinkedInBot
# Disallow: /

# Optional throttle for scrapers that respect it:
# Crawl-delay: 10

# ===============================================================
# Sitemap
# ===============================================================
Sitemap: https://www.homebrewfinds.com/sitemap.xml

User-agent: *
Disallow: /wp-content/uploads/wpo/wpo-plugins-tables-list.json

# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow:

Sitemap: https://www.homebrewfinds.com/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK