# START YOAST SEO ROBOTS.TXT RULES
# You can learn more about robots.txt at https://yoast.com/robots-txt/
# ---------------------------------------------------------------------

# Allow all reputable crawlers full access
User-agent: *

# Standard WordPress admin and system directories
# Prevents crawling of sensitive and non-public areas
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php # Essential for theme/plugin functionality

Disallow: /wp-includes/ # Core WordPress files, no SEO value
Disallow: /wp-login.php # Login page, no SEO value
Disallow: /xmlrpc.php # API endpoint, often targeted by bots, no SEO value

# Block internal search results (prevents duplicate content issues)
Disallow: /*?s=*

# Block common query parameters that can create duplicate content
# Use this with caution if your site relies on specific query parameters for unique content.
# For most blogs/news sites, these are safe to disallow.
Disallow: /*?p=* # May apply to some post permalinks, if using old/non-pretty permalinks
Disallow: /*?attachment_id=* # Prevents indexing of attachment pages (often thin content)
Disallow: /*?replytocom=* # Comment reply links
Disallow: /*?author=* # Author archives (often duplicate of posts)
Disallow: /*?cat=* # Category archives (if you use clean permalinks, this isn't usually an issue)
Disallow: /*?tag=* # Tag archives (if you use clean permalinks, this isn't usually an issue)

# Block various feeds (often duplicate of main content, uses crawl budget)
# If you explicitly want your feeds indexed, remove these lines.
# For most sites, search engines use feeds for content discovery, not direct indexing of the feed URL.
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /*/feed/
Disallow: /*/comments/feed/

# Block trackbacks/pingbacks (legacy feature, no SEO value)
Disallow: /trackback/

# Block WordPress embeds (can create duplicate or low-value content)
Disallow: /*/embed/
Disallow: /*/embed/?*
Disallow: /*/?embed=true

# Block specific file types (if you have many that are not content related)
# Example: Disallow: /*.pdf$
# Example: Disallow: /*.zip$

# Sitemap location - CRUCIAL for search engine discovery of your content
# Ensure this path is correct for your Yoast SEO sitemap index
Sitemap: https://thepressunited.com/sitemap_index.xml

# END YOAST SEO ROBOTS.TXT RULES