Disallow: forums/post-*.html$
Disallow: forums/updates-topic.html*$
Disallow: forums/stop-updates-topic.html*$
Disallow: forums/ptopic*.html$
Disallow: forums/ntopic*.html$
User-agent: Slurp
Crawl-delay: 15
User-Agent: msnbot
crawl-delay: 15
# Disallow Collectors and Spam
User-agent: atSpider
Disallow: /
User-agent: cherrypicker
Disallow: /
User-agent: DSurf
Disallow: /
User-agent: EliteSys Entry
Disallow: /
User-agent: EmailCollector
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: EmailWolf
Disallow: /
User-agent: Mail Sweeper
Disallow: /
User-agent: munky
Disallow: /
User-agent: Roverbot
Disallow: /
User-agent: WebEmailExtrac
Disallow: /
# Disallow Offline Browsers
User-agent: xGet
Disallow: /
User-agent: wGet
Disallow: /
User-agent: WebWalk
Disallow: /
User-agent: webvac
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: WebMirror
Disallow: /
User-agent: WebFetcher
Disallow: /
User-agent: WebCopy
Disallow: /
User-agent: webcopier
Disallow: /
User-agent: WebCatcher
Disallow: /
User-agent: WebBandit
Disallow: /
User-agent: w3mir
Disallow: /
User-agent: vobsub
Disallow: /
User-agent: Templeton
Disallow: /
User-agent: ssearcher100
Disallow: /
User-agent: SpiderBot
Disallow: /
User-agent: Shai'Hulud
Disallow: /
User-agent: PBWF
Disallow: /
User-agent: LightningDownload
Disallow: /
User-agent: KDD Exploror
Disallow: /
User-agent: Jeeves
Disallow: /
User-agent: InfoSpiders
Disallow: /
User-agent: httrack
Disallow: /
User-agent: HavIndex
Disallow: /
User-agent: GetUrl
Disallow: /
User-agent: GetBot
Disallow: /
User-agent: ESIRover
Disallow: /
User-agent: Download Wonder
Disallow: /
User-agent: Collage
Disallow: /
User-agent: SBIder
Disallow: /
User-agent: W3C-checklink
Disallow: /
User-agent: peerbot
Disallow: /favicon.ico
User-agent: MSIECrawler
Disallow: /
User-agent: psbot
Disallow: /
User-agent: NetMechanic
Disallow: /
User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: LinkWalker
Disallow: /
User-agent: FreeFind
Disallow: /
User-agent: MondoSearch
Disallow: /
User-agent: Spiderline
Disallow: /
User-agent: fusionbot
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: NPBot
Disallow: /
User-agent: Teleport Pro/1.29
Disallow: /
User-agent: Wget/1.8.2
Disallow: /
User-agent: fast
Disallow: /
User-agent: grub-client
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent:EmailCollector
Disallow: /
User-agent: appie
Disallow: /
User-agent: psbot
Disallow: /
User-agent: Atomz
Disallow: /
User-agent: RufusBot
Disallow: /
User-agent: NextGenSearchBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: ConveraCrawler
Disallow: /
User-agent: Sproose
Disallow: /
User-agent: BecomeBot
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: WebTrends
Disallow: /
User-agent: digout4u
Disallow: /
User-agent: extractorpro
Disallow: /
User-agent: GetRight
Disallow: /
User-agent: go-ahead-got-it
Disallow: /
User-agent: grub
Disallow: /
User-agent: HTTPClient
Disallow: /
User-agent: LinkWalker
Disallow: /
User-agent: nearsite
Disallow: /
User-agent: netattache
Disallow: /
User-agent: NEWT ActiveX
Disallow: /
User-agent: sitesnagger
Disallow: /
User-agent: teleport
Disallow: /
User-agent: TovekTools Web Indexer
Disallow: /
User-agent: UbiCrawler
Disallow: /
User-agent: Web Downloader
Disallow: /
User-agent: webwhacker
Disallow: /
User-agent: webzip
Disallow: /
User-agent: GagaRobot
Disallow: /
User-agent:WISENutbot
Disallow:/
User-agent:ISSpider
Disallow:/
User-agent:AIBOT
Disallow:/
User-agent: *
Disallow: /lab-book/
User-agent: UbiCrawler
Disallow: /
User-agent: DOC
Disallow: /
User-agent: Zao
Disallow: /
User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: linko
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Microsoft.URL.Control
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: larbin
Disallow: /
User-agent: libwww
Disallow: /
User-agent: ZyBORG
Disallow: /
User-agent: Download Ninja
Disallow: /
User-agent: wget
Disallow: /
User-agent: grub-client
Disallow: /
User-agent: k2spider
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: QuepasaCreep
Disallow: /
User-agent: Feedster Crawler/1.0
Disallow: /
User-agent: MS-Catapult/0.9
Disallow: /
User-agent: Whacker
Disallow: /
User-agent: FlashSite
Disallow: /
User-agent: NetJet
Disallow: /
User-agent: Catalog-Robot
Disallow: /
User-agent: Snarf
Disallow: /
User-agent: infoGear
Disallow: /
User-agent: DittoSpyder
Disallow: /
User-agent: Ask Jeeves/Teoma
Disallow: /
User-agent: VoilaBot
Disallow: /
User-agent: genieBot
Disallow: /