# =====================================
# Global rules – allow search engines
# =====================================
User-agent: *
Allow: /

# Block sensitive/system directories
Disallow: /cgi-bin/
Disallow: /admin_new/
Disallow: /Fckeditor/
Disallow: /FCKeditor1/
Disallow: /annonces/cgi-bin/
Disallow: /dev/
Disallow: /site/
Disallow: /site2/
Disallow: /site2.maliweb.net/

# =====================================
# Explicit major bots (optional but clear)
# =====================================
User-agent: bingbot
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: meta-externalagent
Allow: /

# =====================================
# Aggressive bots – rate limiting
# =====================================
User-agent: AhrefsBot
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 5

User-agent: MJ12bot
Crawl-delay: 10

# =====================================
# Sitemaps
# =====================================
Sitemap: https://www.maliweb.net/sitemap.xml
Sitemap: https://www.maliweb.net/sitemap-1.xml
Sitemap: https://www.maliweb.net/sitemap-2.xml
Sitemap: https://www.maliweb.net/sitemap-3.xml
Sitemap: https://www.maliweb.net/sitemap-4.xml
Sitemap: https://www.maliweb.net/sitemap-5.xml
Sitemap: https://www.maliweb.net/sitemap-6.xml