Initialize rails app
This commit is contained in:
parent
fd82adc73e
commit
64fb080e96
234 changed files with 1398 additions and 14539 deletions
|
|
@ -1,29 +1 @@
|
|||
User-agent: *
|
||||
Disallow: /admin
|
||||
Disallow: /login
|
||||
Disallow: /js
|
||||
Disallow: /css
|
||||
|
||||
User-Agent: GPTBot
|
||||
Disallow: /
|
||||
|
||||
User-Agent: ChatGPT-User
|
||||
Disallow: /
|
||||
|
||||
User-Agent: Google-Extended
|
||||
Disallow: /
|
||||
|
||||
User-Agent: CCBot
|
||||
Disallow: /
|
||||
|
||||
User-Agent: PerplexityBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: anthropic-ai
|
||||
Disallow: /
|
||||
|
||||
User-agent: Claude-Web
|
||||
Disallow: /
|
||||
|
||||
User-agent: ClaudeBot
|
||||
Disallow: /
|
||||
# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue