20 lines
370 B
Plaintext
20 lines
370 B
Plaintext
# robots.txt for Ningxia Smart Farming Platform
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
# Sitemap location
|
|
Sitemap: https://nxfarm.com/sitemap.xml
|
|
|
|
# Disallow crawling of admin and sensitive areas
|
|
Disallow: /admin/
|
|
Disallow: /api/
|
|
Disallow: /private/
|
|
|
|
# Allow all other content
|
|
Allow: /assets/
|
|
Allow: /css/
|
|
Allow: /js/
|
|
Allow: /images/
|
|
|
|
# Crawl delay (requests per second)
|
|
Crawl-delay: 1 |