Robots.txt allow files

This commit is contained in:
Ben Grant 2021-05-25 02:21:38 +10:00
parent 6ed9c1e226
commit 053b0fa6fb
2 changed files with 11 additions and 3 deletions

View file

@ -3,7 +3,16 @@ User-agent: *
Allow: /$
Allow: /how-to$
Allow: /privacy$
Allow: /sitemap.xml$
Allow: /*.xml$
Allow: /*.js$
Allow: /*.css$
Allow: /*.json$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ttf$
Allow: /*.png$
Allow: /*.ico$
Allow: /*.svg$
Disallow: *
sitemap: https://crab.fit/sitemap.xml
Sitemap: https://crab.fit/sitemap.xml

View file

@ -1,5 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://crab.fit/</loc>