# robots.txt for https://www.nsf.gov/

Sitemap:  https://www.nsf.gov/sitemap.xml

User-agent: bingbot
Disallow: /cgi-bin/
Disallow: /sbe/sbe_2020/2020_pdfs/
Disallow: /awardsearch/advancedSearchNoScript*
Disallow: /awards/award_visualization.jsp*
Disallow: /awards/award_visualization_noscript.jsp*
Disallow: /email.jsp*
Disallow: /helpful.jsp*
Disallow: /events/event_calendar.jsp*
Disallow: /news/news_results.jsp*
Disallow: /discoveries/disc_search_results.jsp*
Disallow: /funding/funding_results.jsp*
Disallow: /staff/results.jsp*
Disallow: /grfp*
Disallow: /od/oia/activities/cov/bio/2001/GPRA_Rpt.doc

User-agent: Googlebot  
Disallow: /cgi-bin/
Disallow: /sbe/sbe_2020/2020_pdfs/
Disallow: /awardsearch/advancedSearchNoScript*
Disallow: /awards/award_visualization.jsp*
Disallow: /awards/award_visualization_noscript.jsp*
Disallow: /email.jsp*
Disallow: /helpful.jsp*
Disallow: /events/event_calendar.jsp*
Disallow: /news/news_results.jsp*
Disallow: /discoveries/disc_search_results.jsp*
Disallow: /funding/funding_results.jsp*
Disallow: /staff/results.jsp*
Disallow: /grfp*
Disallow: /od/oia/activities/cov/bio/2001/GPRA_Rpt.doc

User-agent: Slurp
Disallow: /cgi-bin/
Disallow: /awardsearch/
Disallow: /sbe/sbe_2020/2020_pdfs/
Disallow: /awardsearch/advancedSearchNoScript*
Disallow: /awards/award_visualization.jsp*
Disallow: /awards/award_visualization_noscript.jsp*
Disallow: /email.jsp*
Disallow: /helpful.jsp*
Disallow: /events/event_calendar.jsp*
Disallow: /news/news_results.jsp*
Disallow: /discoveries/disc_search_results.jsp*
Disallow: /funding/funding_results.jsp*
Disallow: /staff/results.jsp*
Disallow: /grfp*
Disallow: /od/oia/activities/cov/bio/2001/GPRA_Rpt.doc
Crawl-delay: 1

User-agent: ia_archiver
Disallow: /cgi-bin/
Disallow: /awardsearch/
Disallow: /sbe/sbe_2020/2020_pdfs/
Disallow: /awardsearch/advancedSearchNoScript*
Disallow: /awards/award_visualization.jsp*
Disallow: /awards/award_visualization_noscript.jsp*
Disallow: /email.jsp*
Disallow: /helpful.jsp*
Disallow: /events/event_calendar.jsp*
Disallow: /news/news_results.jsp*
Disallow: /discoveries/disc_search_results.jsp*
Disallow: /funding/funding_results.jsp*
Disallow: /staff/results.jsp*
Disallow: /grfp*
Disallow: /od/oia/activities/cov/bio/2001/GPRA_Rpt.doc
Crawl-delay: 1 

User-agent: LinkTiger
Disallow: /cgi-bin/
Disallow: /awardsearch/
Disallow: /sbe/sbe_2020/2020_pdfs/
Disallow: /awardsearch/advancedSearchNoScript*
Disallow: /awards/award_visualization.jsp*
Disallow: /awards/award_visualization_noscript.jsp*
Disallow: /email.jsp*
Disallow: /helpful.jsp*
Disallow: /events/event_calendar.jsp*
Disallow: /news/news_results.jsp*
Disallow: /discoveries/disc_search_results.jsp*
Disallow: /funding/funding_results.jsp*
Disallow: /staff/results.jsp*
Disallow: /grfp*
Disallow: /od/oia/activities/cov/bio/2001/GPRA_Rpt.doc
Crawl-delay: 1

User-agent: Hosted Compliance Sheriff
Disallow: /cgi-bin/
Disallow: /awardsearch/
Disallow: /sbe/sbe_2020/2020_pdfs/
Disallow: /awardsearch/advancedSearchNoScript*
Disallow: /awards/award_visualization.jsp*
Disallow: /awards/award_visualization_noscript.jsp*
Disallow: /email.jsp*
Disallow: /helpful.jsp*
Disallow: /events/event_calendar.jsp*
Disallow: /news/news_results.jsp*
Disallow: /discoveries/disc_search_results.jsp*
Disallow: /funding/funding_results.jsp*
Disallow: /staff/results.jsp*
Disallow: /grfp*
Disallow: /od/oia/activities/cov/bio/2001/GPRA_Rpt.doc
Crawl-delay: 1

# Statistics bot
User-agent: archive.org_bot
Disallow: /cgi-bin/
Crawl-delay: 1

# Everyone else
User-agent: *                  
Disallow: /cgi-bin/
Disallow: /awardsearch/
Disallow: /sbe/sbe_2020/2020_pdfs/
Disallow: /awardsearch/advancedSearchNoScript*
Disallow: /awards/award_visualization.jsp*
Disallow: /awards/award_visualization_noscript.jsp*
Disallow: /email.jsp*
Disallow: /helpful.jsp*
Disallow: /events/event_calendar.jsp*
Disallow: /news/news_results.jsp*
Disallow: /discoveries/disc_search_results.jsp*
Disallow: /funding/funding_results.jsp*
Disallow: /staff/results.jsp*
Disallow: /grfp*
Disallow: /od/oia/activities/cov/bio/2001/GPRA_Rpt.doc
Crawl-delay: 10