# The robots.txt file for http:///learn.utoronto.ca/ #Altavista Spiders User-agent: Scooter Disallow: /hello.html Disallow: /insurance/home/auto/insurance/start.html Disallow: /nl/ Disallow: /welcomeframe1.html User-agent: Top1000 altavista Disallow: /hello.html Disallow: /insurance/home/auto/insurance/start.html Disallow: /nl/ Disallow: /welcomeframe1.html # Excite Spider User-agent: ArchitextSpider Disallow: /hello.html Disallow: /insurance/home/auto/insurance/start.html Disallow: /nl/ Disallow: /welcome1.html Disallow: /welcome2.html Disallow: /welcome3.html #Webcrawler Spider User-agent: WebCrawler Disallow: /hello.html Disallow: /insurance/home/auto/insurance/start.html Disallow: /nl/ Disallow: /welcome1.html Disallow: /welcome2.html Disallow: /welcome3.html # Inktomi agent - hotbot et al User-agent: Slurp Disallow: /hello.html Disallow: /nl/ Disallow: /welcomeframe1.html Disallow: /welcome1.html Disallow: /welcome2.html Disallow: /welcome3.html #InfoSeek Robots User-agent: InfoSeek Sidewinder Disallow: /hello.html Disallow: /insurance/home/auto/insurance/start.html Disallow: /nl/ Disallow: /welcomeframe1.html Disallow: /welcome1.html Disallow: /welcome2.html Disallow: /welcome3.html User-agent: InfoSeek Robot Disallow: /hello.html Disallow: /insurance/home/auto/insurance/start.html Disallow: /nl/ Disallow: /welcomeframe1.html Disallow: /welcome1.html Disallow: /welcome2.html Disallow: /welcome3.html User-agent: Mozilla Disallow: /hello.html Disallow: /insurance/home/auto/insurance/start.html Disallow: /nl/ Disallow: /welcomeframe1.html Disallow: /welcome1.html Disallow: /welcome2.html Disallow: /welcome3.html #Lycos Spiders User-agent: Lycos_Spider_(T-Rex) Disallow: /insurance/home/auto/insurance/start.html Disallow: /nl/ Disallow: /welcomeframe1.html Disallow: /welcome1.html Disallow: /welcome2.html Disallow: /welcome3.html User-agent: Lycos Disallow: /insurance/home/auto/insurance/start.html Disallow: /nl/ Disallow: /welcomeframe1.html Disallow: /welcome1.html Disallow: /welcome2.html Disallow: /welcome3.html #Northern Light Spider User-agent: Gulliver Disallow: