# # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host: # Used: http://example.com/robots.txt # Ignored: http://example.com/site/robots.txt # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/robotstxt.html User-agent: * # CSS, JS, Images Allow: /core/*.css$ Allow: /core/*.css? Allow: /core/*.js$ Allow: /core/*.js? Allow: /core/*.gif Allow: /core/*.jpg Allow: /core/*.jpeg Allow: /core/*.png Allow: /core/*.svg Allow: /profiles/*.css$ Allow: /profiles/*.css? Allow: /profiles/*.js$ Allow: /profiles/*.js? Allow: /profiles/*.gif Allow: /profiles/*.jpg Allow: /profiles/*.jpeg Allow: /profiles/*.png Allow: /profiles/*.svg # Directories Disallow: /core/ Disallow: /profiles/ # Files Disallow: /README.md Disallow: /composer/Metapackage/README.txt Disallow: /composer/Plugin/ProjectMessage/README.md Disallow: /composer/Plugin/Scaffold/README.md Disallow: /composer/Plugin/VendorHardening/README.txt Disallow: /composer/Template/README.txt Disallow: /modules/README.txt Disallow: /sites/README.txt Disallow: /themes/README.txt Disallow: /web.config # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /filter/tips Disallow: /node/add/ Disallow: /search/ Disallow: /user/register Disallow: /user/password Disallow: /user/login Disallow: /user/logout Disallow: /media/oembed Disallow: /*/media/oembed # Paths (no clean URLs) Disallow: /index.php/admin/ Disallow: /index.php/comment/reply/ Disallow: /index.php/filter/tips Disallow: /index.php/node/add/ Disallow: /index.php/search/ Disallow: /index.php/user/password Disallow: /index.php/user/register Disallow: /index.php/user/login Disallow: /index.php/user/logout Disallow: /index.php/media/oembed Disallow: /index.php/*/media/oembed # For all user agents to block search pgaes User-agent: * Disallow: *?f%5B* Disallow: *&f%5B* User-agent: GPTBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DotBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: Baiduspider Disallow: / User-agent: YandexBot Disallow: / User-agent: PetalBot Disallow: / User-agent: Sogou web spider Disallow: / User-agent: SeznamBot Disallow: / User-agent: CCBot Disallow: / User-agent: MegaIndex.ru Disallow: / User-agent: Cliqzbot Disallow: / User-agent: Exabot Disallow: / User-agent: ChatGPT Disallow: / User-agent: OpenAI Disallow: / User-agent: Google-Extended Disallow: / User-agent: ChatGPT-User Disallow: / User-agent: PerplexityBot Disallow: / User-agent: Amazonbot Disallow: / User-Agent: FacebookBot Disallow: / User-Agent: Applebot Disallow: / User-agent: anthropic-ai Disallow: / User-agent: Claude-Web Disallow: / User-agent: Diffbot Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: Omgilibot Disallow: / User-agent: Omgili Disallow: / User-agent: YouBot Disallow: / # Additional unwanted bots User-agent: ZumBot Disallow: / User-agent: wotbox Disallow: / User-agent: spbot Disallow: / User-agent: BLEXBot Disallow: / User-agent: Exabot Disallow: / # Custombot sigs User-agent: X-MS-Client-Application: Disallow: / User-agent: python-requests/2.24.0 Disallow: / User-agent: Nuclei Disallow: / User-agent: DotBot/1.1 Disallow: / User-agent: aria2/1.34.0 Disallow: / User-agent: Bytespider Disallow: / User-agent: dotbot Disallow: / User-agent|RSS Disallow: / User-agent: Golden Disallow: / User-agent: Gulper Disallow: / User-agent: Magus Disallow: / User-agent: Miralinks Disallow: / User-agent: Piepmatz Disallow: / User-agent: SeobilityBot Disallow: / User-agent: Snap Disallow: / User-agent: Space Disallow: / User-agent: Superfeedr Disallow: / User-agent: Xing Disallow: / User-agent: mediacloud Disallow: / User-agent: rss2tg Disallow: / User-agent: wp.com Disallow: / User-agent: yacybot Disallow: / User-agent: BomboraBot Disallow: / User-agent: RyteBot Disallow: / User-agent: rogerbot Disallow: / User-agent: _zbot Disallow: / User-agent: Adsbot/3.1 Disallow: / User-agent: AhrefsBot Disallow: / User-agent: AndersPinkBot Disallow: / User-agent: Atomseobot Disallow: / User-agent: BLEXBot Disallow: / User-agent: CLTbot Disallow: / User-agent: ChatGPT Disallow: / User-agent: Checkbot Disallow: / User-agent: Cincraw Disallow: / User-agent: Citation_bot Disallow: / User-agent: CodeGetterbot Disallow: / User-agent: CrowdTanglebot Disallow: / User-agent: CrsspxlBot Disallow: / User-agent: CubeRssReader Disallow: / User-agent: DaniBot Disallow: / User-agent: Dark_Matter_Bot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: Deskyobot Disallow: / User-agent: DiffeoBot Disallow: / User-agent: DingTalkBot-LinkService Disallow: / User-agent: Discordbot Disallow: / User-agent: Dubbotbot Disallow: / User-agent: DuckDuckBot Disallow: / User-agent: Exabot Disallow: / User-agent: Folkbot Disallow: / User-agent: Fuze_Bot Disallow: / User-agent: Gaisbot Disallow: / User-agent: GetDigestBot Disallow: / User-agent: Gigabot Disallow: / User-agent: Girafabot Disallow: / User-agent: GnowitNewsbot Disallow: / User-agent: Go-http-client Disallow: / User-agent: HatenaBlog-bot Disallow: / User-agent: HotJava Disallow: / User-agent: InfoTigerBot Disallow: / User-agent: Java-http-client Disallow: / User-agent: LinkisBot Disallow: / User-agent: LinkpadBot Disallow: / User-agent: LivelapBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: MagiBot Disallow: / User-agent: Mail.RU_Bot Disallow: / User-agent: Mediumbot Disallow: / User-agent: MojeekBot Disallow: / User-agent: Monsidobot Disallow: / User-agent: MoodleBot Disallow: / User-agent: Neevabot Disallow: / User-agent: NetpeakCheckerBot Disallow: / User-agent: Nimbostratus-Bot Disallow: / User-agent: ONBbot Disallow: / User-agent: OnalyticaBot Disallow: / User-agent: PaperLiBot Disallow: / User-agent: PetalBot Disallow: / User-agent: Phractusbot Disallow: / User-agent: Pleroma Disallow: / User-agent: Quora-Bot Disallow: / User-agent: RasaBot Disallow: / User-agent: RedirectBot Disallow: / User-agent: Refindbot Disallow: / User-agent: ResearchBot Disallow: / User-agent: RightIntelBot Disallow: / User-agent: Riverbot Disallow: / User-agent: SEMrushBot Disallow: / User-agent: SERPtimizerBot Disallow: / User-agent: SMTBot Disallow: / User-agent: SOLOFIELD Disallow: / User-agent: SabsimBot Disallow: / User-agent: Scribbr-Citation-Bot Disallow: / User-agent: SemanticScholarBot Disallow: / User-agent: Semanticbot Disallow: / User-agent: SemrushBot Disallow: / User-agent: SerendeputyBot Disallow: / User-agent: SeznamBot Disallow: / User-agent: SirdataBot Disallow: / User-agent: SiteAnalyzerbot Disallow: / User-agent: SiteAuditBot Disallow: / User-agent: SiteCheckerBotCrawler Disallow: / User-agent: Slack-ImgProxy Disallow: / User-agent: Slackbot Disallow: / User-agent: Sottopop Disallow: / User-agent: SuperBot Disallow: / User-agent: SurdotlyBot Disallow: / User-agent: SynologyChatBot Disallow: / User-agent: TMMBot Disallow: / User-agent: TSMbot Disallow: / User-agent: TelegramBot Disallow: / User-agent: Terkkobot Disallow: / User-agent: TinEye-bot Disallow: / User-agent: TkBot Disallow: / User-agent: TripleCheckerRobot Disallow: / User-agent: TulipChain Disallow: / User-agent: UptimeRobot Disallow: / User-agent: V-BOT Disallow: / User-agent: WebBot Disallow: / User-agent: WellKnownBot Disallow: / User-agent: XLinkBotReverter Disallow: / User-agent: YandexBot Disallow: / User-agent: YandexRenderResourcesBot Disallow: / User-agent: YandexNews Disallow: / User-agent: YandexRCA Disallow: / User-agent: Yellowbrandprotectionbot Disallow: / User-agent: ZoomBot Disallow: / User-agent: ZoominfoBot Disallow: / User-agent: archive.org_bot Disallow: / User-agent: awesome_bot Disallow: / User-agent: badbot Disallow: / User-agent: bl.uk_lddc_bot Disallow: / User-agent: bnf.fr_bot Disallow: / User-agent: coccocbot Disallow: / User-agent: cubebot Disallow: / User-agent: curl-all Disallow: / User-agent: digitalshadowsbot Disallow: / User-agent: equellaurlbot Disallow: / User-agent: harsilbot Disallow: / User-agent: i2kconnect Disallow: / User-agent: iSec_Bot Disallow: / User-agent: libcurl Disallow: / User-agent: linkdexbot Disallow: / User-agent: magpie-crawler Disallow: / User-agent: moatbot Disallow: / User-agent: monitorapp-robot-parser Disallow: / User-agent: notestock Disallow: / User-agent: online-webceo-bot Disallow: / User-agent: opengraph-bot Disallow: / User-agent: pagepeeker Disallow: / User-agent: prophy-bot Disallow: / User-agent: python-requests Disallow: / User-agent: repology-linkchecker Disallow: / User-agent: sc_bot Disallow: / User-agent: seo-audit-check-bot Disallow: / User-agent: startmebot Disallow: / User-agent: strutbot Disallow: / User-agent: trendictionbot0 Disallow: / User-agent: undefined Disallow: / User-agent: unirest-java Disallow: / User-agent: urlchecker Disallow: / User-agent: YandexAccessibilityBot Disallow: / User-agent: YandexImages Disallow: / User-agent: python Disallow: /