19 lines
304 B
Plaintext
19 lines
304 B
Plaintext
# https://www.robotstxt.org/robotstxt.html
|
|
User-agent: *
|
|
Allow: /$
|
|
Allow: /how-to$
|
|
Allow: /privacy$
|
|
Allow: /*.xml$
|
|
Allow: /*.js$
|
|
Allow: /*.css$
|
|
Allow: /*.json$
|
|
Allow: /*.woff$
|
|
Allow: /*.woff2$
|
|
Allow: /*.ttf$
|
|
Allow: /*.png$
|
|
Allow: /*.ico$
|
|
Allow: /*.svg$
|
|
Disallow: *
|
|
|
|
Sitemap: https://crab.fit/sitemap.xml
|