User-agent: *
#Request-rate: 1/10
Sitemap: http://disc.sci.gsfc.nasa.gov/sitemap.xml
Crawl-delay: 20
Visit-time: 0200-1100
Disallow: /ftp/
Disallow: /_mm/
Disallow: /_notes/
Disallow: /_baks/
Disallow: /MMWIP/
Disallow: /MODIS/
Disallow: /rssnews/Connections/
Disallow: /opendap/
Disallow: /OPENDAP/
Disallow: /MDISC/
Disallow: /daac-bin/
Disallow: /data/datapool/
Disallow: *.csi
Disallow: /WWW-TMP/
Disallow: /*at_download$
Disallow: /*sendto_form$
Disallow: /*news_summary_view$
Disallow: /*folder_factories$
Disallow: /*topdoc_view$
## Disallowing of Plone News spidering from a portal
Disallow: /AIRS/gesNews/
Disallow: /atdd/gesNews/
Disallow: /geomorphology/gesNews/
Disallow: /hydrology/gesNews/
Disallow: /precipitation/gesNews/
Disallow: /giovanni/gesNews/
Disallow: /neespi/gesNews/
Disallow: /nimbus/gesNews/
Disallow: /uwg/gesNews/
Disallow: /TRMM/gesNews/
Disallow: /landbio/gesNews/
Disallow: /awips/gesNews/
Disallow: /googleearth/gesNews/
Disallow: /nrt/gesNews/
Disallow: /UARS/gesNews/
Disallow: /acdisc/gesNews/
Disallow: /aerosols/gesNews/
Disallow: /airquality/gesNews/
Disallow: /tracegases/gesNews/
Disallow: /ozone/gesNews/
Disallow: /SORCE/gesNews/
Disallow: /Aura/gesNews/
Disallow: /oceans/gesNews/
Disallow: /oceancolor/gesNews/
Disallow: /hurricane/gesNews/
Disallow: /agriculture/gesNews/
Disallow: /data-holdings/gesNews
Disallow: /mairs/gesNews
Disallow: /mdisc/gesNews
Disallow: /malaria_modeling/gesNews

User-Agent: check_http
Disallow: /atdd/

User-Agent: twiceler
Crawl-delay: 20
Disallow: /AIRS/
Disallow: /atdd/
Disallow: /geomorphology/
Disallow: /gesNews/
Disallow: /hydrology/
Disallow: /precipitation/
Disallow: /giovanni/
Disallow: /neespi/
Disallow: /nimbus
Disallow: /uwg/
Disallow: /TRMM/
Disallow: /landbio/
Disallow: /awips/
Disallow: /googleearth/
Disallow: /nrt/
Disallow: /UARS/
Disallow: /acdisc/
Disallow: /aerosols/
Disallow: /airquality/
Disallow: /tracegases/
Disallow: /ozone/
Disallow: /SORCE/
Disallow: /Aura/
Disallow: /oceans/
Disallow: /oceancolor/
Disallow: /data-holdings/
Disallow: /mairs/
Disallow: /mdisc/
Disallow: /malaria_modeling/

User-Agent: webvac
Disallow: /AIRS/
Disallow: /atdd/
Disallow: /geomorphology/
Disallow: /gesNews/
Disallow: /hydrology/
Disallow: /precipitation/
Disallow: /giovanni/
Disallow: /neespi/
Disallow: /nimbus
Disallow: /uwg/
Disallow: /TRMM/
Disallow: /landbio/
Disallow: /awips/
Disallow: /googleearth/
Disallow: /nrt/
Disallow: /UARS/
Disallow: /acdisc/
Disallow: /aerosols/
Disallow: /airquality/
Disallow: /tracegases/
Disallow: /ozone/
Disallow: /SORCE/
Disallow: /Aura/
Disallow: /oceans/
Disallow: /oceancolor/
Disallow: /data-holdings/
Disallow: /mairs/
Disallow: /mdisc/
Disallow: /malaria_modeling/

User-Agent: findfiles.net
Disallow: /AIRS/
Disallow: /atdd/
Disallow: /geomorphology/
Disallow: /gesNews/
Disallow: /hydrology/
Disallow: /precipitation/
Disallow: /giovanni/
Disallow: /neespi/
Disallow: /nimbus
Disallow: /uwg/
Disallow: /TRMM/
Disallow: /landbio/
Disallow: /awips/
Disallow: /googleearth/
Disallow: /nrt/
Disallow: /UARS/
Disallow: /acdisc/
Disallow: /aerosols/
Disallow: /airquality/
Disallow: /tracegases/
Disallow: /ozone/
Disallow: /SORCE/
Disallow: /Aura/
Disallow: /oceans/
Disallow: /oceancolor/
Disallow: /data-holdings/
Disallow: /mairs/
Disallow: /mdisc/
Disallow: /malaria_modeling/

User-Agent: gsa-crawler
Disallow: /AIRS/
Disallow: /atdd/
Disallow: /agriculture/
Disallow: /geomorphology/
Disallow: /gesNews/
Disallow: /hydrology/
Disallow: /precipitation/
Disallow: /giovanni/
Disallow: /neespi/
Disallow: /nimbus
Disallow: /uwg/
Disallow: /TRMM/
Disallow: /landbio/
Disallow: /awips/
Disallow: /googleearth/
Disallow: /nrt/
Disallow: /UARS/
Disallow: /acdisc/
Disallow: /aerosols/
Disallow: /airquality/
Disallow: /tracegases/
Disallow: /ozone/
Disallow: /SORCE/
Disallow: /Aura/
Disallow: /oceans/
Disallow: /oceancolor/
Disallow: /data-holdings/
Disallow: /mairs/
Disallow: /mdisc/
Disallow: /malaria_modeling/

User-Agent: nasagsacrawler
Disallow: /AIRS/
Disallow: /atdd/
Disallow: /agriculture/
Disallow: /geomorphology/
Disallow: /gesNews/
Disallow: /hydrology/
Disallow: /precipitation/
Disallow: /giovanni/
Disallow: /neespi/
Disallow: /nimbus
Disallow: /uwg/
Disallow: /TRMM/
Disallow: /landbio/
Disallow: /awips/
Disallow: /googleearth/
Disallow: /nrt/
Disallow: /UARS/
Disallow: /acdisc/
Disallow: /aerosols/
Disallow: /airquality/
Disallow: /tracegases/
Disallow: /ozone/
Disallow: /SORCE/
Disallow: /Aura/
Disallow: /oceans/
Disallow: /oceancolor/
Disallow: /data-holdings/
Disallow: /mairs/
Disallow: /mdisc/
Disallow: /malaria_modeling/

#User-Agent: mirasearch
#Disallow: /AIRS/
#Disallow: /atdd/
#Disallow: /agriculture/
#Disallow: /geomorphology/
#Disallow: /gesNews/
#Disallow: /hydrology/
#Disallow: /precipitation/
#Disallow: /giovanni/
#Disallow: /neespi/
#Disallow: /nimbus
#Disallow: /uwg/
#Disallow: /TRMM/
#Disallow: /landbio/
#Disallow: /awips/
#Disallow: /googleearth/
#Disallow: /nrt/
#Disallow: /UARS/
#Disallow: /acdisc/
#Disallow: /aerosols/
#Disallow: /airquality/
#Disallow: /tracegases/
#Disallow: /ozone/
#Disallow: /SORCE/
#Disallow: /Aura/
#Disallow: /oceans/
#Disallow: /oceancolor/
#Disallow: /data-holdings/
#Disallow: /malaria_modeling/


#User-Agent: discsearch
#Disallow: /AIRS/
#Disallow: /atdd/
#Disallow: /agriculture/
#Disallow: /geomorphology/
#Disallow: /gesNews/
#Disallow: /hydrology/
#Disallow: /precipitation/
#Disallow: /giovanni/
#Disallow: /neespi/
#Disallow: /nimbus
#Disallow: /uwg/
#Disallow: /TRMM/
#Disallow: /landbio/
#Disallow: /awips/
#Disallow: /googleearth/
#Disallow: /nrt/
#Disallow: /UARS/
#Disallow: /acdisc/
#Disallow: /aerosols/
#Disallow: /airquality/
#Disallow: /tracegases/
#Disallow: /ozone/
#Disallow: /SORCE/
#Disallow: /Aura/
#Disallow: /oceans/
#Disallow: /oceancolor/
#Disallow: /data-holdings/


User-Agent: gsfcbot
Disallow: /AIRS/
Disallow: /atdd/
Disallow: /agriculture/
Disallow: /geomorphology/
Disallow: /gesNews/
Disallow: /hydrology/
Disallow: /precipitation/
Disallow: /giovanni/
Disallow: /neespi/
Disallow: /nimbus
Disallow: /uwg/
Disallow: /TRMM/
Disallow: /landbio/
Disallow: /googleearth/
Disallow: /nrt/
Disallow: /UARS/
Disallow: /acdisc/
Disallow: /aerosols/
Disallow: /airquality/
Disallow: /tracegases/
Disallow: /ozone/
Disallow: /SORCE/
Disallow: /Aura/
Disallow: /oceans/
Disallow: /oceancolor/
Disallow: /data-holdings/
Disallow: /mairs/
Disallow: /mdisc/
Disallow: /malaria_modeling/