scott/crabfit
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Robots.txt allow files

Browse source
This commit is contained in:
Ben Grant 2021-05-25 02:21:38 +10:00
parent 6ed9c1e226
commit 053b0fa6fb
2 changed files with 11 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
crabfit-frontend/public/robots.txt
View file

@ -3,7 +3,16 @@ User-agent: *
Allow: /$ Allow: /$
Allow: /how-to$ Allow: /how-to$
Allow: /privacy$ Allow: /privacy$
Allow: /sitemap.xml$ Allow: /*.xml$
Allow: /*.js$
Allow: /*.css$
Allow: /*.json$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ttf$
Allow: /*.png$
Allow: /*.ico$
Allow: /*.svg$
Disallow: * Disallow: *
sitemap: https://crab.fit/sitemap.xml Sitemap: https://crab.fit/sitemap.xml

1
crabfit-frontend/public/sitemap.xml
View file

@ -1,5 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url> <url>
<loc>https://crab.fit/</loc> <loc>https://crab.fit/</loc>