User-agent: *   # for all robots
Disallow: /     # This will prevent all robots from traversing our domain.

#Then we can list those which are allowed to crawl our domain.

#  Planet Direct knows where to go
User-agent: Mozilla/2.0 (Normandy 1.0b3a) Planet Direct          # Planet Direct IRS crawler
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/


# Lycos knows where to go.
User-agent: Lycos_Spider_(T-Rex)/3.0
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# InfoSeek knows where to go.
User-agent: InfoSeek Sidewinder/0.9
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# WebCrawler knows where to go.
User-agent: WebCrawler/3.0_Robot libwww/5.0a
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# WebCrawler knows where to go.
User-agent: WebCrawler-AddURL/2.0
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# WebCrawler knows where to go.
User-agent: WebCrawler-Status/1.0
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# Excite (formerly known as Architext) knows where to go.
User-agent: ArchitextSpider
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# AltaVista (from Digital)knows where to go.
User-agent: Scooter/1.0
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# EuroSeek knows where to go.
User-agent: Arachnoidea
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# HotBot (by Wired) knows where to go.
User-agent: Slurp/2.0
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# Northern Light knows where to go.
User-agent: Gulliver/1.2
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# MOMSpider knows where to go.
User-agent: MOMSpider/1.00
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# SEAQuest! (Southeast Asia Search Engine) http://SEAQuest.kusza.edu.my knows where to go.
User-agent: ANKabut/1.5
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# FreeFind knows where to go.
User-agent: FreeFind
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# NetMechanic knows where to go.
User-agent: NetMechanic
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# LinkAlarm knows where to go.
User-agent: LinkAlarm
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# N2H2-BirdDog knows where to go.
User-agent: N2H2-BirdDog
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# htdig knows where to go.
User-agent: htdig
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# PicoSearch knows where to go.
User-agent: PicoSearch/1.0
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# FashionSpider <http://www.media.mit.edu:80/> knows where to go.
User-agent: FashionSpider
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# KIT-Fireball knows where to go.
User-agent: KIT-Fireball/2.0 libwww/5.0a
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# Australian Cultural Network Robot <http://www.acn.net.au> knows where to go.
User-agent: ACN-Robot
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# Arribavista.com media search engine knows where to go.
User-agent: ArribaPacketRat
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# CyberSpyder.com link verifier knows where to go.
User-agent: CyberSpyder
Disallow: /images/ 
Disallow: /IMG/
Disallow: /js/

# TELEPORT
User-agent: TeleportPro
Disallow: /