# robots.txt for cunningtons.co.uk # Reviewed 24 July 2026 # Strategy: AI crawlers are welcomed, in line with the firm's AI visibility work. User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /cgi-bin/ Disallow: /temp/ Disallow: /dev/ Disallow: /jet/ Disallow: /checkout/ Disallow: /login/ Disallow: /confirmation/ Disallow: /webform/ Disallow: /test/ Disallow: /downloads/ Disallow: /download/ Disallow: /my-account/ Disallow: /thankyou/ Disallow: /locations.kml Disallow: /?post_type=locations&p=556 Disallow: /?post_type=services&p=350 # --- Removed 24 July 2026, pages no longer exist so the blocks guarded nothing: # /locations/ilford/ and variants (branch moved to Hornchurch) # /cunningtons-solicitors-ilford # /services/family-matrimonial/mediation/ (service discontinued) # /our-services/debt-collection/ (service not offered) # /about-us/aymer-hutton/ (former staff, page gone) # /about-us/mohammed-moula/ (former staff, page gone) # /residential-property-solicitor-in-brighton/ # /about-us/notices (legal disclaimer page, should be public) # # FLAG FOR JULIA: /category/commercial-property/ block also removed. It is an # empty category (no commercial conveyancing posts) but the page returns 200. # Better handled by noindex or deletion than a robots block. Left unblocked # for now; decide whether to noindex or delete the category itself. # Semrush: throttle rather than block (was an empty, non-functional group before) User-agent: SemrushBot Crawl-delay: 10 # --- AI crawlers ----------------------------------------------------------- # All explicitly allowed. Crawl-delay is a courtesy only; Google and most # major AI crawlers ignore it, but it does no harm to state it. # Note: allowing GPTBot etc. means content may be used for training as well # as live citation. That is intended, given the firm's AI visibility strategy. User-agent: GPTBot Allow: / Crawl-delay: 2 User-agent: ChatGPT-User Allow: / Crawl-delay: 2 User-agent: OAI-SearchBot Allow: / Crawl-delay: 2 User-agent: ClaudeBot Allow: / Crawl-delay: 2 User-agent: Claude-Web Allow: / Crawl-delay: 2 User-agent: anthropic-ai Allow: / Crawl-delay: 2 User-agent: Google-Extended Allow: / Crawl-delay: 2 User-agent: PerplexityBot Allow: / Crawl-delay: 2 User-agent: Perplexity-User Allow: / Crawl-delay: 2 User-agent: CCBot Allow: / Crawl-delay: 2 User-agent: Applebot-Extended Allow: / Crawl-delay: 2 User-agent: Meta-ExternalAgent Allow: / Crawl-delay: 2 User-agent: Amazonbot Allow: / Crawl-delay: 2 User-agent: Bytespider Allow: / Crawl-delay: 2 User-agent: cohere-ai Allow: / Crawl-delay: 2 User-agent: Timpibot Allow: / Crawl-delay: 2 Sitemap: https://cunningtons.co.uk/sitemap_index.xml