# Global settings for all bots
User-agent: *
# Disallow access to sensitive directories
Disallow: /admin/
Disallow: /private/
Disallow: /tmp/
Disallow: /cart/
Disallow: /checkout/
Disallow: /user-profile/
Disallow: /config/
Disallow: /scripts/
Disallow: /backup/
Disallow: /logs/
Disallow: /howdy/

# Prevent access to files that are typically unnecessary for search engines
Disallow: /*.json$
Disallow: /*.csv$
Disallow: /*.zip$
Disallow: /*.tar$
Disallow: /*.gz$

# Allow Googlebot to crawl everything important
User-agent: Googlebot
Allow: /

# Allow specific assets that enhance page performance and user experience
Allow: /images/
Allow: /css/
Allow: /js/

# Sitemap to guide search engines to important pages
Sitemap: https://dclouds.in/sitemap_index.xml