# Block aggressive SEO crawlers

# AhrefsBot
User-agent: AhrefsBot

# DotBot (https://opensiteexplorer.org/dotbot)
User-agent: dotbot

# Grapeshot crawler (making malformed URL requests)
User-agent: grapeshot

# Linguee (https://www.linguee.com/bot)
User-agent: Linguee

# MJ12bot
User-agent: MJ12bot

# PetalBot (http://aspiegel.com/petalbot)
User-agent: PetalBot

# SEMrushBot (crawling ancient links and not respecting limits)
User-agent: SemrushBot

Disallow: /

# Rate-limit selected crawlers

# BLEXBot (http://webmeup-crawler.com/) 
User-agent: BLEXBot 
Crawl-delay: 60 
Disallow: /policies/returns
Disallow: /policies/shipping

# Common Crawl (https://commoncrawl.org/faq/) 
User-agent: CCBot 
Crawl-delay: 30 
Disallow: /policies/returns
Disallow: /policies/shipping

# Bingbot(MSNBot) 
User-Agent: bingbot 
Crawl-delay: 4 
Disallow: /policies/returns
Disallow: /policies/shipping

# Yahoo (Slurp) 
User-agent: Slurp 
Crawl-delay: 5 
Disallow: /policies/returns
Disallow: /policies/shipping

# Yandex 
User-agent: Yandex 
Crawl-delay: 4 
Disallow: /policies/returns
Disallow: /policies/shipping

# OpenAI bots
User-agent: OAI-SearchBot
Allow: / 
Disallow: /policies/returns
Disallow: /policies/shipping

User-agent: OAI-AdsBot
Allow: / 
Disallow: /policies/returns
Disallow: /policies/shipping

User-agent: *
Disallow: /policies/returns
Disallow: /policies/shipping

Sitemap: https://www.martechcnc.com/sitemap.xml