tma-mark2/robots.txt at main · hyperpolymath/tma-mark2 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# robots.txt for eTMA Handler repository
# This file primarily applies to hosted documentation/GitHub Pages

User-agent: *
Allow: /

# AI crawlers - permitted with conditions
# See /.well-known/aibdp.json for full boundary declaration
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Rate limiting hint (advisory)
Crawl-delay: 1

# Sitemap (if hosted documentation exists)
# Sitemap: https://hyperpolymath.github.io/tma-mark2/sitemap.xml

# Related declarations
# AI Boundary Declaration: /.well-known/aibdp.json
# LLMs context: /llms.txt
# Security: /.well-known/security.txt