/usr/share/moin/htdocs/robots.txt is in python-moinmoin 1.9.7-1ubuntu2.2.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 | # if you want to add own robot rules, do it BEFORE the final rule matching *
User-agent: *
Crawl-delay: 20
# This has to match script url + cfg.url_prefix_action - it
# saves lots of search engine load and traffic by disallowing crawlers
# to request action related URLs.
#
# NOTE - in order to make this have any effect, you have to set
# url_prefix_action to "action", cf. HelpOnConfiguration
Disallow: /action/
|