User-agent: * # Block system and private folders Disallow: /application/ Disallow: /system/ Disallow: /vendor/ Disallow: /tests/ Disallow: /logs/ Disallow: /admin/ Disallow: /admin/login Disallow: /login/ Disallow: /register/ # Allow public pages Allow: / # Sitemaps for SEO Sitemap: https://www.rsbc.uk/page-sitemap.xml Sitemap: https://www.rsbc.uk/services-sitemap.xml Sitemap: https://www.rsbc.uk/who-we-are-sitemap.xml Sitemap: https://www.rsbc.uk/who-we-work-with-sitemap.xml Sitemap: https://www.rsbc.uk/prices-sitemap.xml Sitemap: https://www.rsbc.uk/solutions-sitemap.xml Sitemap: https://www.rsbc.uk/rsbc-guides-sitemap.xml Sitemap: https://www.rsbc.uk/category-sitemap.xml Sitemap: https://www.rsbc.uk/blog-sitemap.xml Sitemap: https://www.rsbc.uk/calculator-sitemap.xml Sitemap: https://www.rsbc.uk/location-sitemap.xml Sitemap: https://www.rsbc.uk/accounts-location-sitemap.xml Sitemap: https://www.rsbc.uk/who-we-work-with-location-sitemap.xml Sitemap: https://www.rsbc.uk/bookkeeping-location-sitemap.xml User-agent: * Allow: / # --- AI / LLM Bot Permissions (General Allow Policy) --- Allow-Training: yes Allow-Retrieval: yes # --- Explicit AI Bots Allowed --- User-agent: gptbot Allow: / Allow-Training: yes Allow-Retrieval: yes User-agent: claudebot Allow: / Allow-Training: yes Allow-Retrieval: yes User-agent: google-extended Allow: / Allow-Training: yes Allow-Retrieval: yes User-agent: perplexitybot Allow: / Allow-Training: yes Allow-Retrieval: yes User-agent: neevabot Allow: / Allow-Training: yes Allow-Retrieval: yes User-agent: youbot Allow: / Allow-Training: yes Allow-Retrieval: yes User-agent: ccbot Allow: / Allow-Training: yes Allow-Retrieval: yes User-agent: duckduckbot Allow: / Allow-Training: yes Allow-Retrieval: yes User-agent: yandexbot Allow: / Allow-Training: yes Allow-Retrieval: yes User-agent: archive.org_bot Allow: / Allow-Training: yes Allow-Retrieval: yes # Metadata files for search engines & AI systems Reference: https://www.rsbc.uk/brand-metadata.txt Reference: https://www.rsbc.uk/brand-metadata.json Reference: https://www.rsbc.uk/llms.txt