# AhrefsBot 
User-agent: AhrefsBot 
Disallow: / 

# BLEXBot (http://webmeup-crawler.com/) 
User-agent: BLEXBot 
Crawl-delay: 60 

# Common Crawl (https://commoncrawl.org/faq/) 
User-agent: CCBot 
Crawl-delay: 30 

# DotBot (https://opensiteexplorer.org/dotbot) 
User-agent: dotbot 
Disallow: / 

# Grapeshot crawler (making malformed URL requests) 
User-Agent: grapeshot 
Disallow: / 

# Linguee (https://www.linguee.com/bot) 
User-agent: Linguee 
Disallow: / 

# MJ12bot 
User-Agent: MJ12bot 
Disallow: / 

# MSNBot 
User-Agent: msnbot 
Crawl-delay: 4 

# PetalBot (http://aspiegel.com/petalbot) 
User-agent: PetalBot 
Disallow: / 

# SEMrushBot (crawling ancient links and not respecting limits) 
User-Agent: SemrushBot 
Disallow: / 

# Yahoo (Slurp) 
User-agent: Slurp 
Crawl-delay: 5 

# Yahoo Slurp 
User-agent: Yahoo! Slurp 
Crawl-delay: 5 

# Yandex 
User-agent: Yandex 
Crawl-delay: 4 

Disallow: /policies/returns
Disallow: /policies/shipping

Sitemap: https://www.martechcnc.com/sitemap.xml