# =========================================== # robots.txt for cimco.com # =========================================== User-agent: * Disallow: /wp-admin/ Disallow: /wp-includes/ Disallow: /wp-content/plugins/ Disallow: /wp-json/ Disallow: /xmlrpc.php Disallow: /?s= Disallow: /*?wc-ajax= Disallow: /*add-to-cart=* Disallow: /*trp-ajax.php Disallow: /*?cimco_region=* Crawl-delay: 10 # Allow essential assets (CSS, JS, images) Allow: /wp-content/uploads/ Allow: /wp-content/themes/ Allow: /wp-content/plugins/*.css$ Allow: /wp-content/plugins/*.js$ # =========================================== # Known bots - specific throttling or blocking # =========================================== # Matchory bot (too aggressive) User-agent: MatchorySearch Disallow: / Crawl-delay: 20 # Ahrefs (SEO bot - allow but slow down) User-agent: AhrefsBot Crawl-delay: 10 # Baidu spider (China) - block User-agent: Baiduspider Disallow: / # Sogou (China) - block User-agent: Sogou web spider Disallow: / # PetalBot (Huawei) - block User-agent: PetalBot Disallow: / # MJ12bot - block (aggressive) User-agent: MJ12bot Disallow: / # SemrushBot - allow but throttle User-agent: SemrushBot Crawl-delay: 10 # =========================================== # Sitemap # =========================================== Sitemap: https://www.cimco.com/sitemap.xml