# a standard robots exclusion file that excludes all
# but the most significant robots from visiting your site
# the default Disallow rule permits all of your web to be 
# crawled by the allowed agents, you will want to change this
# see http://info.webcrawler.com/mak/projects/robots/exclusion.html
# for further details

User-agent: MSR-ISRCCrawler
Disallow: /

User-agent: Nutch Spider/Nutch-1.0-dev
Disallow: /

User-agent: LTI/LemurProject
Disallow: /

User-agent: archive.org_bot 
Disallow:

User-agent: LiteFinder/1.0
Disallow: /

User-agent: NextGenSearchBot
Disallow: /

User-agent: *					# block the webfiles folder
Disallow: /webfiles/

User-agent: *					# block intranet folder
Disallow: /bcbintra/


# tell all other robots to go away

# User-agent: *
# Disallow: /