User-agent: *
#Crawl-delay: 20
#Visit-time: 0200-1100
Disallow: /ftp/
Disallow: /_mm/
Disallow: /_notes/
Disallow: /_baks/
Disallow: /MMWIP/
Disallow: /MODIS/
Disallow: /rssnews/Connections/
Disallow: /opendap/
Disallow: /OPENDAP/
Disallow: /MDISC/
Disallow: /daac-bin/
Disallow: /data/datapool/
Disallow: *.csi
Disallow: /WWW-TMP/
Disallow: /*at_download$
Disallow: /*sendto_form$
Disallow: /*news_summary_view$
Disallow: /*folder_factories$
Disallow: /*topdoc_view$
## Disallowing of Plone News spidering from a portal
Disallow: /about-us/gesNews/
Disallow: /acdisc/gesNews/
Disallow: /additional/gesNews/
Disallow: /aerosols/gesNews/
Disallow: /agriculture/gesNews/
Disallow: /airquality/gesNews/
Disallow: /AIRS/gesNews/
Disallow: /archetype_tool/gesNews/
Disallow: /atdd/gesNews/
Disallow: /atmosphere/gesNews/
Disallow: /Aura/gesNews/
Disallow: /awips/gesNews/
Disallow: /data-holdings/gesNews/
Disallow: /dust/gesNews/
Disallow: /events/gesNews/
Disallow: /featured-items/gesNews/
Disallow: /field-campaigns/gesNews/
Disallow: /geomorphology/gesNews/
Disallow: /giovanni/gesNews/
Disallow: /googleearth/
Disallow: /goreplace-collection/gesNews/
Disallow: /guides/gesNews/
Disallow: /hurricane/gesNews/
Disallow: /hydrology/gesNews/
Disallow: /images/gesNews/
Disallow: /landbio/gesNews/
Disallow: /legacydata/gesNews/
Disallow: /mairs/gesNews/
Disallow: /malaria_modeling/gesNews/
Disallow: /mdisc/gesNews/
Disallow: /MDSA/gesNews/
Disallow: /measures/gesNews/
Disallow: /Members/gesNews/
Disallow: /mirador-news-items/gesNews/
Disallow: /neespi/gesNews/
Disallow: /news_old/gesNews/
Disallow: /nimbus/gesNews/
Disallow: /nimbus-ii/gesNews/
Disallow: /nrt/gesNews/
Disallow: /ocean-color-history/gesNews/
Disallow: /oceancolor/gesNews/
Disallow: /oceans/gesNews/
Disallow: /ozone/gesNews/
Disallow: /precipitation/gesNews/
Disallow: /services/gesNews/
Disallow: /SORCE/gesNews/
Disallow: /tracegases/gesNews/
Disallow: /TRMM/gesNews/
Disallow: /UARS/gesNews/
Disallow: /uwg/gesNews/
Disallow: /water-cycle/gesNews/
Disallow: /whom-string-search-rashid/gesNews/
Disallow: /YOTC/gesNews/
Sitemap: http://disc.sci.gsfc.nasa.gov/sitemap.xml

User-Agent: msnbot/2.0b
Crawl-delay: 100
Sitemap: http://disc.sci.gsfc.nasa.gov/sitemap.xml

User-Agent: Slurp
Crawl-delay: 100
Sitemap: http://disc.sci.gsfc.nasa.gov/sitemap.xml