Robots.txt allow files
This commit is contained in:
parent
6ed9c1e226
commit
053b0fa6fb
|
|
@ -3,7 +3,16 @@ User-agent: *
|
|||
Allow: /$
|
||||
Allow: /how-to$
|
||||
Allow: /privacy$
|
||||
Allow: /sitemap.xml$
|
||||
Allow: /*.xml$
|
||||
Allow: /*.js$
|
||||
Allow: /*.css$
|
||||
Allow: /*.json$
|
||||
Allow: /*.woff$
|
||||
Allow: /*.woff2$
|
||||
Allow: /*.ttf$
|
||||
Allow: /*.png$
|
||||
Allow: /*.ico$
|
||||
Allow: /*.svg$
|
||||
Disallow: *
|
||||
|
||||
sitemap: https://crab.fit/sitemap.xml
|
||||
Sitemap: https://crab.fit/sitemap.xml
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
<url>
|
||||
<loc>https://crab.fit/</loc>
|
||||
|
|
|
|||
Loading…
Reference in a new issue