#****************************************************************************
# robots.txt
#     : Robots, spiders, and search engines use this file to determine which 
#       content they should *not* crawl while indexing your website.
#     : This system is called "The Standard Robots Exclusion."
#****************************************************************************

User-agent: *
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /xmlrpc.php

# Block common junk URL parameters (duplicates, tracking, replytocom)
Disallow: /*?replytocom=
Disallow: /*?p=0
Disallow: /*?cat=-1
Disallow: /*?amp
Disallow: /*?fbclid
Disallow: /*?utm_source
Disallow: /*?utm_medium
Disallow: /*?utm_campaign
Disallow: /*?utm_term
Disallow: /*?utm_content

# Block plugin-generated or trash folders
Disallow: /wpmc-trash/
Disallow: /wpo/

# Allow specific bots full access
User-agent: Googlebot
Allow: /

User-agent: NinjaBot
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: Googlebot-Image
Allow: /wp-content/uploads/

User-agent: Adsbot-Google
Allow: /

User-agent: Googlebot-Mobile
Allow: /

# Sitemap reference
Sitemap: https://www.astronomytrek.com/sitemap_index.xml