caddy and forbidden for robots

nano /etc/caddy/Caddyfile

your-domain.com {
        #Your additional configuration here
        encode gzip zstd

        @botForbidden header_regexp User-Agent "(?i)AdsBot-Google|Amazonbot|anthropic-ai|Applebot|Applebot-Extended|AwarioRssBot|AwarioSmartBot|Bytespider|CCBot|ChatGPT|ChatGPT-User|Claude-Web|ClaudeBot|cohere-ai|DataForSeoBot|Diffbot|FacebookBot|Google-Extended|GPTBot|ImagesiftBot|magpie-crawler|omgili|Omgilibot|peer39_crawler|PerplexityBot|YouBot"

        handle @botForbidden   {
            respond /* "Access denied" 403 {
                close
            }
        }

        file_server
        root * /path/to/your/website
        log {
                output file /path/to/your/logs/your-blog-access.log {
                        roll_size 20mb
                        roll_keep 10
                        roll_keep_for 720h
                }
        }
}