# =============================================
# robots.txt for wonderinn.no
# WonderInn AS — Luxury Glamping in Norway
# Last updated: 2026-03-31
# =============================================



# =============================================
# DEFAULT RULES — All Crawlers
# =============================================

User-agent: *
Allow: /

# WordPress core (block admin, allow ajax for booking widgets)
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /xmlrpc.php

# Block theme/plugin/cache assets (allow uploads for images)
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Allow: /wp-content/uploads/

# WooCommerce transactional pages (zero SEO value)
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /no/my-account/
Disallow: /nl/mijn-account/
Disallow: /fr/mon-compte/
Disallow: /de/mein-konto/
Disallow: /es/mi-cuenta/

# Internal/partner pages (thin content, not for public indexing)
Disallow: /faq-partner/
Disallow: /*/faq-partner/
Disallow: /listing/
Disallow: /*/listing/
Disallow: /item/
Disallow: /collaboration/

# Search results and query strings
Disallow: /?s=
Disallow: /*?s=
Disallow: /*?replytocom
Disallow: /*?attachment_id=

# Feeds (not useful for crawl budget)
Disallow: /feed/
Disallow: /*/feed/
Disallow: /comments/

# Author archive (thin — all posts are by one account)
Disallow: /author/

# Tag archives (high volume, thin pages — 100+ tag URLs across 6 languages)
# Individual post tags are already in article schema keywords
Disallow: /tag/
Disallow: /*/tag/

# "Uncategorized" category pages (thin, no value)
Disallow: /category/uncategorized/
Disallow: /*/category/uncategorized/
Disallow: /no/category/ukategorisert/
Disallow: /nl/category/geen-onderdeel-van-een-categorie/
Disallow: /fr/category/non-classifiee/
Disallow: /de/category/unkategorisiert/
Disallow: /es/category/sin-categorizar/

# WooCommerce shop/cart pages across languages (gift cards are fine via /product/)
Disallow: /shop/
Disallow: /*/shop/
Disallow: /nl/winkel-op/
Disallow: /nl/winkelwagen/
Disallow: /fr/boutique/
Disallow: /es/tienda/



# =============================================
# AI CRAWLERS — Selectively Allowed
# =============================================
# Strategy: Allow AI retrieval (RAG/grounding) for search visibility.
# Block AI training to protect content rights per EU Directive 2019/790.
# The Content-Signal header below communicates this intent.

# OpenAI (ChatGPT search, browsing)
User-agent: GPTBot
Allow: /
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Google AI (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# Microsoft / Bing AI (Copilot)
User-agent: Bingbot
Allow: /

# Apple AI (Siri, Apple Intelligence)
User-agent: Applebot-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Amazon AI (Alexa, product recommendations)
User-agent: Amazonbot
Allow: /
Disallow: /wp-admin/
Disallow: /cart/
Disallow: /checkout/

# Cohere AI
User-agent: cohere-ai
Allow: /

# Common Crawl (feeds open datasets used by many AI systems)
User-agent: CCBot
Allow: /



# =============================================
# BLOCKED CRAWLERS — Known scrapers / low-value bots
# =============================================

# ByteDance (aggressive crawling, primarily AI training)
User-agent: Bytespider
Disallow: /

# Various aggressive scrapers
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /



# =============================================
# CONTENT SIGNALS (Cloudflare Specification)
# =============================================
# search=yes → Allow search indexing and snippets
# ai-input=yes → Allow real-time RAG/grounding for AI-generated answers
# ai-train=no → Block use of content for AI model training/fine-tuning
#
# This preserves visibility in AI search (ChatGPT, Perplexity, Google AI
# Overviews) while protecting content rights under EU Directive 2019/790.

Content-Signal: search=yes, ai-input=yes, ai-train=no



# =============================================
# SITEMAPS
# =============================================

Sitemap: https://wonderinn.no/sitemap_index.xml