# www.robotstxt.org
# Allow crawling of all content
User-agent: *


# Hand-authored pages. Built by Hugo from this repo.
Sitemap: https://www.virtualflybrain.org/sitemap.xml

# Generated term pages. sitemap-terms.xml is a sitemap index over
# sitemap1.xml .. sitemapN.xml, written by the update_vfb_terms_to_static_site
# Jenkins job after the Hugo build. Advertising the index rather than each shard
# keeps the shard count out of this file: it was pinned at 38 here while the job
# was producing 189, so roughly four fifths of the term corpus was never
# advertised to crawlers.
Sitemap: https://www.virtualflybrain.org/sitemap-terms.xml