#nlcrawler are very agressive creating mulitple sessions
#nlcrawler has been blocked by IP (38.106.112.254) as they ignored the following instruction
User-agent: nlcrawler/1.0+(+http://northernlight.com/)
User-agent: nlcrawler
Disallow: /

User-agent: *
Disallow: /clients/

# Wait 3 seconds between successive requests. TO keep within the 200 requests per 10 minute window
Crawl-delay: 3

# upped to 4 seconds as vegbot still triggering (150 requests)
User-agent: VegeBot
Crawl-delay: 4
User-agent: Vegi bot
Crawl-delay: 4

User-agent: Yandex
Crawl-delay: 4

User-agent: Twitterbot/1.0
Allow: /