# robots.txt for http://peterjanes.ca/ # https://gizmodo.com/google-says-itll-scrape-everything-you-post-online-for-1850601486 User-agent: Googlebot Disallow: / User-agent: Google-Extended Disallow: / User-agent: GoogleOther Disallow: / User-agent: GPTBot Disallow: / User-agent: ChatGPT-User Disallow: / User-agent: Charlotte Disallow: / User-agent: uw_cse_xwc Disallow: / User-agent: discobot Disallow: / User-agent: VadixBot Disallow: / User-agent: MSRBOT Disallow: / User-agent: twiceler Disallow: / User-agent: WebAlta Disallow: / User-agent: ilial Disallow: / User-agent: Snapbot Disallow: / User-agent: ichiro Disallow: / User-agent: Snapbot Disallow: / User-agent: DataSpear Disallow: / User-agent: RufusBot Disallow: / User-agent: heritrix Disallow: / User-agent: ConveraCrawler Disallow: / User-agent: Gaisbot Disallow: / User-agent: DeepIndex Disallow: / User-agent: boitho.com-robot Disallow: / User-agent: Lachesis Disallow: / # I'll take my search results ranking based on content, not a "boost", thanks. User-agent: Gigabot Disallow: / User-agent: augurnfind Disallow: / User-agent: boitho Disallow: / User-agent: asterias Disallow: / User-agent: girafa Disallow: / User-agent: timboBot Disallow: / User-agent: sitecheck.internetseer.com Disallow: / User-agent: BlogPulse Disallow: / User-agent: FeedBucket Disallow: / User-agent: NPBot Disallow: / User-agent: NutchCVS Disallow: / User-agent: Turnitinbot Disallow: / User-agent: NaverRobot Disallow: / User-agent: MSNBOT Disallow: / User-agent: MSIECrawler Disallow: / User-agent: MicrosoftPrototypeCrawler Disallow: / # Lots of fast requests User-agent: http://www.almaden.ibm.com/cs/crawler Disallow: / # Doesn't understand mixed case or special characters User-agent: QuepasaCreep Disallow: / User-agent: Rika Disallow: / # divx.com media search engine User-agent: FlickBot Disallow: / User-agent: VoilaBot Disallow: / User-agent: Intelliseek Disallow: / User-agent: WebZIP Disallow: / User-agent: larbin Disallow: / User-agent: openfind Disallow: / User-agent: Openbot Disallow: / User-agent: psbot Disallow: / User-agent: ZyBorg Disallow: / ## The next three are AltaVista spiders. After 800 requests they still only ## have 2 pages listed in their index. What a waste of bandwidth... goodbye! #User-agent: Scooter #Disallow: / # Experimental AV spider User-agent: Mercator Disallow: / # ??? User-agent: vscooter Disallow: / User-agent: Liberate Disallow: / User-agent: lmspider Disallow: / User-agent: Vagabondo Disallow: / User-agent: Zao Disallow: / # If a user-agent matches, the * rule is no longer in effect User-agent: Googlebot Disallow: /bot-redirect Disallow: /xhtml-redirect Disallow: /(none) Disallow: /~peterj/CBP/ Disallow: /~peterj/CBParchive/ Disallow: /CBP/ Disallow: /CBParchive/ Disallow: /mt/ Disallow: /blog/comment Disallow: /blog/trackback Disallow: /blog/trackback/ Disallow: /blog/search Disallow: /blog/page/ Disallow: /blog/tag/ Disallow: /blog/wp-admin/ Disallow: /stats/ Disallow: /blog/archives/categories/ Disallow: /blog/archives/category/ Disallow: /cgi-bin Disallow: /cgi-bin/ Disallow: /cgi-bin/show Disallow: /dav/ Disallow: /id/ Disallow: /mp3/ Disallow: /include/ Disallow: /LenniFan/ Disallow: /blog/2002/04/22/lenni-jabour-cest-what/ Disallow: /styles/ Disallow: /~peterj/comics/ Disallow: /~peterj/personal Disallow: /~peterj/pyblagg Disallow: /~peterj/spycyroll/ Disallow: /~peterj/newSong/ Disallow: /~peterj/WhoIsThat/ Disallow: /peterj/comics/ Disallow: /peterj/personal Disallow: /peterj/pyblagg Disallow: /peterj/spycyroll/ Disallow: /peterj/newSong/ Disallow: /peterj/WhoIsThat/ Disallow: /personal Disallow: /personal/ Disallow: /newblog/ Disallow: /family/images/ Disallow: /family/media/ Disallow: /family/themes/ Disallow: /family/timeline.php Disallow: /family/calendar.php Disallow: /family/login.php Disallow: /family/reportengine.php Disallow: /family/search.php Disallow: /family/fanchart.php Disallow: /family/clippings.php Disallow: /family/sosabook.php Disallow: /family/aliveinyear.php Disallow: /family/ancestry.php Disallow: /family/descendancy.php Disallow: /family/famlist.php Disallow: /family/hourglass.php Disallow: /family/indilist.php Disallow: /family/patriarchlist.php Disallow: /family/pedigree.php Disallow: /family/placelist.php Disallow: /post Disallow: /blog/post Disallow: /resume Disallow: /resume.xhtml Disallow: /u/ # Limit requests by Slurp # http://help.yahoo.com/help/us/ysearch/slurp/slurp-03.html User-agent: Slurp Crawl-delay: 30 Disallow: /bot-redirect Disallow: /xhtml-redirect Disallow: /(none) Disallow: /~peterj/CBP/ Disallow: /~peterj/CBParchive/ Disallow: /CBP/ Disallow: /CBParchive/ Disallow: /mt/ Disallow: /stats/ Disallow: /blog/archives/categories/ Disallow: /blog/archives/category/ Disallow: /blog/category/ Disallow: /blog/comment Disallow: /blog/trackback Disallow: /blog/trackback/ Disallow: /blog/search Disallow: /blog/page/ Disallow: /blog/tag/ Disallow: /blog/wp-admin/ Disallow: /blog/2007/12/28/upcoming-karla/feed/ Disallow: /cgi-bin Disallow: /cgi-bin/ Disallow: /cgi-bin/show Disallow: /dav/ Disallow: /id/ Disallow: /mp3/ Disallow: /include/ Disallow: /LenniFan/ Disallow: /blog/2002/04/22/lenni-jabour-cest-what/ Disallow: /styles/ Disallow: /~peterj/comics/ Disallow: /~peterj/personal Disallow: /~peterj/pyblagg Disallow: /~peterj/spycyroll/ Disallow: /~peterj/newSong/ Disallow: /~peterj/WhoIsThat/ Disallow: /peterj/comics/ Disallow: /peterj/personal Disallow: /peterj/pyblagg Disallow: /peterj/spycyroll/ Disallow: /peterj/newSong/ Disallow: /peterj/WhoIsThat/ Disallow: /personal Disallow: /personal/ Disallow: /newblog/ Disallow: /family/images/ Disallow: /family/media/ Disallow: /family/themes/ Disallow: /family/timeline.php Disallow: /family/calendar.php Disallow: /family/login.php Disallow: /family/reportengine.php Disallow: /family/search.php Disallow: /family/fanchart.php Disallow: /family/clippings.php Disallow: /family/sosabook.php Disallow: /family/aliveinyear.php Disallow: /family/ancestry.php Disallow: /family/descendancy.php Disallow: /family/family.php Disallow: /family/famlist.php Disallow: /family/hourglass.php Disallow: /family/indilist.php Disallow: /family/patriarchlist.php Disallow: /family/pedigree.php Disallow: /family/placelist.php Disallow: /post Disallow: /blog/post Disallow: /resume Disallow: /resume.xhtml Disallow: /u/ # Anything that doesn't match a user-agent above User-agent: * Disallow: /bot-redirect Disallow: /xhtml-redirect Disallow: /(none) Disallow: /mt/ Disallow: /stats/ Disallow: /blog/archives/categories/ Disallow: /blog/archives/category/ Disallow: /blog/category/ Disallow: /blog/comment Disallow: /blog/trackback Disallow: /blog/trackback/ Disallow: /blog/search Disallow: /blog/page/ Disallow: /blog/tag/ Disallow: /blog/wp-admin/ Disallow: /cgi-bin Disallow: /cgi-bin/ Disallow: /cgi-bin/show Disallow: /dav/ Disallow: /id/ Disallow: /mp3/ Disallow: /include/ Disallow: /LenniFan/ Disallow: /blog/2002/04/22/lenni-jabour-cest-what/ Disallow: /styles/ Disallow: /~peterj/comics/ Disallow: /~peterj/personal Disallow: /~peterj/pyblagg Disallow: /~peterj/spycyroll/ Disallow: /~peterj/newSong/ Disallow: /~peterj/WhoIsThat/ Disallow: /peterj/comics/ Disallow: /peterj/personal Disallow: /peterj/pyblagg Disallow: /peterj/spycyroll/ Disallow: /peterj/newSong/ Disallow: /peterj/WhoIsThat/ Disallow: /personal Disallow: /personal/ Disallow: /newblog/ Disallow: /family/images/ Disallow: /family/media/ Disallow: /family/themes/ Disallow: /family/timeline.php Disallow: /family/calendar.php Disallow: /family/login.php Disallow: /family/reportengine.php Disallow: /family/search.php Disallow: /family/fanchart.php Disallow: /family/clippings.php Disallow: /family/sosabook.php Disallow: /family/aliveinyear.php Disallow: /family/ancestry.php Disallow: /family/descendancy.php Disallow: /family/family.php Disallow: /family/famlist.php Disallow: /family/hourglass.php Disallow: /family/indilist.php Disallow: /family/patriarchlist.php Disallow: /family/pedigree.php Disallow: /family/placelist.php Disallow: /post Disallow: /blog/post Disallow: /resume Disallow: /resume.xhtml Disallow: /u/