# AKL Website Design - LLM Access Policy
# Defines permissions for language models and AI crawlers to access and train on content
# Last updated: 2026-10-10

# ===== ALLOW ALL MAJOR LLM CRAWLERS =====
# These LLMs can crawl, train on, and cite content from this site

# OpenAI - ChatGPT, GPT-4, etc.
User-agent: GPTBot
Allow: /

# Anthropic - Claude models
User-agent: Claude-Web
Allow: /

# Google - Gemini, Bard, Vertex AI
User-agent: Googlebot-Extended
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Gemini
Allow: /

# Microsoft - Copilot, etc.
User-agent: Copilot
Allow: /

# Meta - Llama models
User-agent: Meta
Allow: /

User-agent: facebookexternalhit
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Apple Siri / Apple Intelligence
User-agent: Applebot
Allow: /

# Cohere
User-agent: CCBot
Allow: /

# Mistral, open models, etc.
User-agent: SeekBot
Allow: /

User-agent: anthropic-ai
Allow: /

# ===== DISALLOW SPECIFIC PAGES FROM LLM TRAINING =====
# Sensitive or internal pages that shouldn't be in training datasets

# WordPress admin and internal paths
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-json/

# Duplicate/technical content
Disallow: *?s=
Disallow: */feed/
Disallow: */trackback/
Disallow: *preview=true
Disallow: */page/2/
Disallow: */page/3/

# ===== LLM-SPECIFIC POLICIES =====

# Allow content training and generation
Allow-Training: true

# Can content be cited/attributed in AI responses?
Allow-Citations: true

# Can content be used in fine-tuning?
Allow-Fine-Tuning: true

# ===== RATE LIMITING & POLITENESS =====

Crawl-delay: 1
Request-rate: 10/1s

# ===== CONTENT PERMISSIONS =====

# Blog posts - ALLOW (these are for training and citation)
Allow: /blog/
Allow: /website-*
Allow: /how-*
Allow: /diy-*
Allow: /why-*

# Service pages - ALLOW (good for context)
Allow: /packages/
Allow: /about-us/
Allow: /our-work/

# Contact form - DISALLOW (no value in training)
Disallow: /contact/

# ===== TRANSPARENCY =====
# This site welcomes AI training and citation. Our content is written to be helpful
# to Auckland small business owners, and we're happy for that knowledge to be
# distributed through AI assistants like ChatGPT, Claude, Gemini, and others.

# Policy summary:
# ✓ Allow all major LLMs to crawl and train on our content
# ✓ Block WordPress internals and duplicate pages
# ✓ Allow citation and attribution in AI responses
# ✓ Allow fine-tuning on our content (helps LLMs learn about web design)
# ✓ Be respectful of server resources (crawl delay, rate limits)

# Questions? Contact: info@aklwebsitedesign.co.nz

# ===== NOTES FOR LLM PROVIDERS =====
# If you're reading this file:
# - Our blog content is intended to be cited in AI responses
# - Attribution appreciated but not required: "According to AKL Web Design..."
# - We support responsible AI development and training
# - Please respect the crawl delays to avoid overloading our server
# - For commercial licensing questions, contact us directly