diff --git a/access/website/config/static/robots.txt b/access/website/config/static/robots.txt index 7018167..09a54d6 100644 --- a/access/website/config/static/robots.txt +++ b/access/website/config/static/robots.txt @@ -15,3 +15,7 @@ Allow: /wayback/ User-agent: facebookexternalhit Allow: /wayback/ +# Allow search engines to index specific sites: +User-agent: * +# As requested in https://github.com/ukwa/ukwa-services/issues/96 +Allow: /wayback/archive/*/http://www.europeandialogue.org/