added sitemap & robots.txt for SEO

2025-01-25 11:43:48 +01:00
parent 43c1349e12
commit 13233794eb
2 changed files with 46 additions and 0 deletions
@@ -0,0 +1,28 @@
+Sitemap: https://bytedice.net/sitemap.xml
+
+User-agent: Googlebot
+Allow: /
+User-agent: googlebot-image
+Allow: /
+User-agent: googlebot-mobile
+Allow: /
+User-agent: MSNBot
+Allow: /
+
+User-agent: Nutch
+Disallow: /
+User-agent: baiduspider
+Disallow: /
+User-agent: naverbot
+Disallow: /
+User-agent: yeti
+Disallow: /
+User-agent: ia_archiver
+Disallow: /
+
+User-agent: *
+Disallow: /
+
+Disallow: /cgi-bin/
+Disallow: /ssi/
+Disallow: /data/