#normal option allow bots as usual

#stop spamming us please
#User-agent: msnbot
#Disallow: /

#stop spamming us please
User-agent: Slurp
Disallow: /

#stop spamming us please
User-agent: TurnitinBot
Disallow: /

#stop spamming us please
User-Agent: panscient.com
Disallow: /

#stop spamming us please
User-agent: 008
Disallow: / 

#stop spamming us please
User-agent: SemrushBot
Disallow: / 

#stop spamming us please
User-agent: SemrushBot-SA
Disallow: / 



#Robots are welcome to crawl the html parts of the site, but not the various ebook formats
User-agent: *
Disallow: /etexts/Salient23011960/
Disallow: /etexts/Salient23021960/
Disallow: /etexts/Salient23031960/
Disallow: /etexts/Salient23041960/
Disallow: /etexts/Salient23051960/
Disallow: /etexts/Salient23061960/
Disallow: /etexts/Salient23071960/
Disallow: /etexts/Salient23081960/
Disallow: /etexts/Salient23091960/
Disallow: /etexts/Salient24011961/
Disallow: /etexts/Salient24021961/
Disallow: /etexts/Salient24031961/
Disallow: /etexts/Salient24041961/
Disallow: /etexts/Salient24051961/
Disallow: /etexts/Salient24061961/
Disallow: /etexts/Salient24071961/
Disallow: /etexts/Salient24081961/
Disallow: /etexts/Salient24091961/
Disallow: /etexts/Salient24101961/
Disallow: /etexts/Salient24111961/
Disallow: /etexts/Salient24121961/
Disallow: /etexts/Salient24131961/
Disallow: /etexts/Salient24141961/
Disallow: /etexts/Salient24151961/
Disallow: /etexts/Salient25011962/
Disallow: /etexts/Salient25021962/
Disallow: /etexts/Salient25031962/
Disallow: /etexts/Salient25041962/
Disallow: /etexts/Salient25051962/
Disallow: /etexts/Salient25061962/
Disallow: /etexts/Salient25071962/
Disallow: /etexts/Salient25081962/
Disallow: /etexts/Salient25091962/
Disallow: /etexts/Salient25101962/
Disallow: /etexts/Salient25111962/
Disallow: /etexts/Salient25121962/
Disallow: /etexts/Salient25131962/
Disallow: /etexts/Salient26011963/
Disallow: /etexts/Salient26021963/
Disallow: /etexts/Salient26031963/
Disallow: /etexts/Salient26041963/
Disallow: /etexts/Salient26051963/
Disallow: /etexts/Salient26061963/
Disallow: /etexts/Salient26071963/
Disallow: /etexts/Salient26081963/
Disallow: /etexts/Salient26091963/
Disallow: /etexts/Salient26101963/
Disallow: /etexts/Salient26112963/
Disallow: /etexts/Salient26123963/
Disallow: /etexts/Salient27011964/
Disallow: /etexts/Salient27021964/
Disallow: /etexts/Salient27031964/
Disallow: /etexts/Salient27041964/
Disallow: /etexts/Salient27051964/
Disallow: /etexts/Salient27061964/
Disallow: /etexts/Salient27071964/
Disallow: /etexts/Salient27081964/
Disallow: /etexts/Salient27091964/
Disallow: /etexts/Salient27101964/
Disallow: /etexts/Salient27111964/
Disallow: /etexts/Salient27121964/
Disallow: /etexts/Salient27131964/
Disallow: /etexts/Salient27141964/
Disallow: /etexts/Salient27151964/
Disallow: /etexts/Salient28011965/
Disallow: /etexts/Salient28021965/
Disallow: /etexts/Salient28031965/
Disallow: /etexts/Salient28041965/
Disallow: /etexts/Salient28051965/
Disallow: /etexts/Salient28061965/
Disallow: /etexts/Salient28071965/
Disallow: /etexts/Salient28081965/
Disallow: /etexts/Salient28091965/
Disallow: /etexts/Salient28101965/
Disallow: /etexts/Salient28111965/
Disallow: /etexts/Salient28121965/
Disallow: /etexts/Salient29011966/
Disallow: /etexts/Salient29021966/
Disallow: /etexts/Salient29031966/
Disallow: /etexts/Salient29041966/
Disallow: /etexts/Salient29051966/
Disallow: /etexts/Salient29061966/
Disallow: /etexts/Salient29071966/
Disallow: /etexts/Salient29081966/
Disallow: /etexts/Salient29091966/
Disallow: /etexts/Salient29101966/
Disallow: /etexts/Salient29111966/
Disallow: /etexts/Salient29121966/
Disallow: /etexts/Salient29131966/
Disallow: /etexts/Salient29141966/
Disallow: /etexts/Salient30011967/
Disallow: /etexts/Salient30021967/
Disallow: /etexts/Salient30031967/
Disallow: /etexts/Salient30041967/
Disallow: /etexts/Salient30051967/
Disallow: /etexts/Salient30061967/
Disallow: /etexts/Salient30071967/
Disallow: /etexts/Salient30081967/
Disallow: /etexts/Salient30091967/
Disallow: /etexts/Salient30101967/
Disallow: /etexts/Salient30111967/
Disallow: /etexts/Salient30121967/
Disallow: /etexts/Salient30131967/
Disallow: /etexts/Salient30141967/
Disallow: /etexts/Salient31011968/
Disallow: /etexts/Salient31021968/
Disallow: /etexts/Salient31031968/
Disallow: /etexts/Salient31041968/
Disallow: /etexts/Salient31051968/
Disallow: /etexts/Salient31061968/
Disallow: /etexts/Salient31071968/
Disallow: /etexts/Salient31081968/
Disallow: /etexts/Salient31091968/
Disallow: /etexts/Salient31101968/
Disallow: /etexts/Salient31111968/
Disallow: /etexts/Salient31121968/
Disallow: /etexts/Salient31131968/
Disallow: /etexts/Salient32011969/
Disallow: /etexts/Salient32021969/
Disallow: /etexts/Salient32031969/
Disallow: /etexts/Salient32041969/
Disallow: /etexts/Salient32051969/
Disallow: /etexts/Salient32061969/
Disallow: /etexts/Salient32071969/
Disallow: /etexts/Salient32081969/
Disallow: /etexts/Salient32091969/
Disallow: /etexts/Salient32101969/
Disallow: /etexts/Salient32111969/
Disallow: /etexts/Salient32121969/
Disallow: /etexts/Salient32131969/
Disallow: /etexts/Salient32141969/
Disallow: /etexts/Salient32151969/
Disallow: /etexts/Salient32161969/
Disallow: /etexts/Salient32171969/
Disallow: /etexts/Salient32181969/
Disallow: /etexts/Salient32191969/
Disallow: /etexts/Salient32201969/
Disallow: /etexts/Salient32211969/
Disallow: /etexts/Salient32221969/
Disallow: /etexts/Salient32231969/
Disallow: /etexts/Salient32241969/
Disallow: /etexts/Salient32251969/
Disallow: /etexts/SalientSpecial1969/
Disallow: /etexts/Salient33011970/
Disallow: /etexts/Salient33021970/
Disallow: /etexts/Salient33031970/
Disallow: /etexts/Salient33041970/
Disallow: /etexts/Salient33051970/
Disallow: /etexts/Salient33061970/
Disallow: /etexts/Salient33071970/
Disallow: /etexts/Salient33081970/
Disallow: /etexts/Salient33091970/
Disallow: /etexts/Salient33101970/
Disallow: /etexts/Salient33111970/
Disallow: /etexts/Salient33121970/
Disallow: /etexts/Salient33131970/
Disallow: /etexts/Salient33141970/
Disallow: /etexts/Salient33151970/
Disallow: /etexts/Salient34011971/
Disallow: /etexts/Salient34021971/
Disallow: /etexts/Salient34031971/
Disallow: /etexts/Salient34041971/
Disallow: /etexts/Salient34051971/
Disallow: /etexts/Salient34061971/
Disallow: /etexts/Salient34071971/
Disallow: /etexts/Salient34081971/
Disallow: /etexts/Salient34091971/
Disallow: /etexts/Salient34101971/
Disallow: /etexts/Salient34121971/
Disallow: /etexts/Salient34131971/
Disallow: /etexts/Salient34141971/
Disallow: /etexts/Salient34151971/
Disallow: /etexts/Salient34161971/
Disallow: /etexts/Salient34171971/
Disallow: /etexts/Salient34181971/
Disallow: /etexts/Salient35011972/
Disallow: /etexts/Salient35021972/
Disallow: /etexts/Salient35031972/
Disallow: /etexts/Salient35041972/
Disallow: /etexts/Salient35051972/
Disallow: /etexts/Salient35061972/
Disallow: /etexts/Salient35071972/
Disallow: /etexts/Salient35081972/
Disallow: /etexts/Salient35091972/
Disallow: /etexts/Salient35101972/
Disallow: /etexts/Salient35111972/
Disallow: /etexts/Salient35121972/
Disallow: /etexts/Salient35131972/
Disallow: /etexts/Salient35141972/
Disallow: /etexts/Salient35151972/
Disallow: /etexts/Salient35161972/
Disallow: /etexts/Salient35171972/
Disallow: /etexts/Salient35181972/
Disallow: /etexts/Salient35191972/
Disallow: /etexts/Salient35201972/
Disallow: /etexts/Salient35221972/
Disallow: /etexts/Salient35231972/
Disallow: /etexts/Salient35241972/
Disallow: /etexts/Salient35251972/
Disallow: /etexts/Salient36011973/
Disallow: /etexts/Salient36021973/
Disallow: /etexts/Salient36031973/
Disallow: /etexts/Salient36041973/
Disallow: /etexts/Salient36051973/
Disallow: /etexts/Salient36061973/
Disallow: /etexts/Salient36071973/
Disallow: /etexts/Salient36081973/
Disallow: /etexts/Salient36091973/
Disallow: /etexts/Salient36101973/
Disallow: /etexts/Salient36111973/
Disallow: /etexts/Salient36121973/
Disallow: /etexts/Salient36151973/
Disallow: /etexts/Salient36161973/
Disallow: /etexts/Salient36171973/
Disallow: /etexts/Salient36181973/
Disallow: /etexts/Salient36191973/
Disallow: /etexts/Salient36201973/
Disallow: /etexts/Salient36211973/
Disallow: /etexts/Salient36221973/
Disallow: /etexts/Salient36231973/
Disallow: /etexts/Salient36241973/
Disallow: /etexts/Salient36251973/
Disallow: /etexts/Salient37011974/
Disallow: /etexts/Salient37021974/
Disallow: /etexts/Salient37031974/
Disallow: /etexts/Salient37051974/
Disallow: /etexts/Salient37061974/
Disallow: /etexts/Salient37071974/
Disallow: /etexts/Salient37081974/
Disallow: /etexts/Salient37091974/
Disallow: /etexts/Salient37101974/
Disallow: /etexts/Salient37111974/
Disallow: /etexts/Salient37131974/
Disallow: /etexts/Salient37151974/
Disallow: /etexts/Salient37161974/
Disallow: /etexts/Salient37171974/
Disallow: /etexts/Salient37181974/
Disallow: /etexts/Salient37191974/
Disallow: /etexts/Salient37201974/
Disallow: /etexts/Salient37211974/
Disallow: /etexts/Salient37221974/
Disallow: /etexts/Salient37231974/
Disallow: /etexts/Salient37241974/
Disallow: /etexts/Salient37251974/
Disallow: /etexts/Salient37261974/
Disallow: /etexts/Salient38011975/
Disallow: /etexts/Salient38021975/
Disallow: /etexts/Salient38031975/
Disallow: /etexts/Salient38041975/
Disallow: /etexts/Salient38051975/
Disallow: /etexts/Salient38061975/
Disallow: /etexts/Salient38071975/
Disallow: /etexts/Salient38081975/
Disallow: /etexts/Salient38091975/
Disallow: /etexts/Salient38101975/
Disallow: /etexts/Salient38111975/
Disallow: /etexts/Salient38121975/
Disallow: /etexts/Salient38131975/
Disallow: /etexts/Salient38141975/
Disallow: /etexts/Salient38151975/
Disallow: /etexts/Salient38161975/
Disallow: /etexts/Salient38171975/
Disallow: /etexts/Salient38181975/
Disallow: /etexts/Salient38191975/
Disallow: /etexts/Salient38201975/
Disallow: /etexts/Salient38211975/
Disallow: /etexts/Salient38221975/
Disallow: /etexts/Salient38231975/
Disallow: /etexts/Salient38241975/
Disallow: /etexts/Salient38251975/
Disallow: /etexts/Salient38261975/
Disallow: /etexts/Salient39011976/
Disallow: /etexts/Salient39021976/
Disallow: /etexts/Salient39031976/
Disallow: /etexts/Salient39041976/
Disallow: /etexts/Salient39051976/
Disallow: /etexts/Salient39061976/
Disallow: /etexts/Salient39071976/
Disallow: /etexts/Salient39081976/
Disallow: /etexts/Salient39091976/
Disallow: /etexts/Salient39101976/
Disallow: /etexts/Salient39111976/
Disallow: /etexts/Salient39121976/
Disallow: /etexts/Salient39141976/
Disallow: /etexts/Salient39161976/
Disallow: /etexts/Salient39171976/
Disallow: /etexts/Salient39181976/
Disallow: /etexts/Salient39191976/
Disallow: /etexts/Salient39211976/
Disallow: /etexts/Salient39231976/
Disallow: /etexts/Salient39241976/
Disallow: /etexts/Salient39251976/
Disallow: /etexts/Salient40011977/
Disallow: /etexts/Salient40021977/
Disallow: /etexts/Salient40031977/
Disallow: /etexts/Salient40041977/
Disallow: /etexts/Salient40051977/
Disallow: /etexts/Salient40061977/
Disallow: /etexts/Salient40071977/
Disallow: /etexts/Salient40081977/
Disallow: /etexts/Salient40091977/
Disallow: /etexts/Salient40101977/
Disallow: /etexts/Salient40111977/
Disallow: /etexts/Salient40121977/
Disallow: /etexts/Salient40131977/
Disallow: /etexts/Salient40141977/
Disallow: /etexts/Salient40151977/
Disallow: /etexts/Salient40161977/
Disallow: /etexts/Salient40171977/
Disallow: /etexts/Salient40181977/
Disallow: /etexts/Salient40191977/
Disallow: /etexts/Salient40201977/
Disallow: /etexts/Salient40211977/
Disallow: /etexts/Salient40221977/
Disallow: /etexts/Salient40231977/
Disallow: /etexts/Salient40251977/
Disallow: /etexts/Salient40261977/
Disallow: /etexts/Salient41011978/
Disallow: /etexts/Salient41021978/
Disallow: /etexts/Salient41031978/
Disallow: /etexts/Salient41041978/
Disallow: /etexts/Salient41051978/
Disallow: /etexts/Salient41061978/
Disallow: /etexts/Salient41071978/
Disallow: /etexts/Salient41081978/
Disallow: /etexts/Salient41091978/
Disallow: /etexts/Salient41101978/
Disallow: /etexts/Salient41121978/
Disallow: /etexts/Salient41131978/
Disallow: /etexts/Salient41141978/
Disallow: /etexts/Salient41151978/
Disallow: /etexts/Salient41181978/
Disallow: /etexts/Salient41191978/
Disallow: /etexts/Salient41211978/
Disallow: /etexts/Salient41221978/
Disallow: /etexts/Salient41231978/
Disallow: /etexts/Salient41261978/
Disallow: /etexts/Salient42011979/
Disallow: /etexts/Salient42021979/
Disallow: /etexts/Salient42031979/
Disallow: /etexts/Salient42041979/
Disallow: /etexts/Salient42051979/
Disallow: /etexts/Salient42061979/
Disallow: /etexts/Salient42071979/
Disallow: /etexts/Salient42081979/
Disallow: /etexts/Salient42091979/
Disallow: /etexts/Salient42101979/
Disallow: /etexts/Salient42111979/
Disallow: /etexts/Salient42121979/
Disallow: /etexts/Salient42131979/
Disallow: /etexts/Salient42141979/
Disallow: /etexts/Salient42151979/
Disallow: /etexts/Salient42161979/
Disallow: /etexts/Salient42181979/
Disallow: /etexts/Salient42191979/
Disallow: /etexts/Salient42201979/
Disallow: /etexts/Salient42211979/
Disallow: /etexts/Salient42221979/
Disallow: /etexts/Salient42231979/
Disallow: /etexts/Salient42251979/
Disallow: /etexts/Salient42Spec1979/
Disallow: /etext
Disallow: /pdf/
Disallow: /facsimile/
Disallow: /msreader/
Disallow: /oeb/
Disallow: /shop/
Disallow: /solr/
Disallow: /lucene/
Disallow: /daisy/
Disallow: /html-package/
Disallow: /tm/basic/
Disallow: /tm/scholarly/search/
Disallow: /tm/scholarly/facets/
Crawl-delay: 60
#smart robots only crawl the content
#Sitemap: http://www.nzetc.org/sitemapindex.xml



#special stanza for archive.org bot
User-agent: archive.org_bot
Disallow: /solr/
Disallow: /lucene/
Disallow: /daisy/
Disallow: /html-package/
Disallow: /tm/basic/
Disallow: /tm/scholarly/search/
Disallow: /tm/scholarly/facets/
Crawl-delay: 60


# custom config for the VUW-GSA machine
# =====================================
# keep the GSA crawler out of everything but the homepage
User-agent: gsa-crawler
Disallow: /tm/
Disallow: /downloads/
Disallow: /tei-source/
Disallow: /etexts/