# ============================================================ # robots.txt for https://www.365education.org # Generated: 2026-06-16 # Purpose: Maximum crawlability, indexing & SEO performance # ============================================================ # ============================================================ # SEARCH ENGINES — Full Access # ============================================================ User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Video Allow: / User-agent: Googlebot-News Allow: / User-agent: AdsBot-Google Allow: / User-agent: Bingbot Allow: / User-agent: AdIdxBot Allow: / User-agent: BingPreview Allow: / User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / User-agent: Baiduspider Allow: / User-agent: YandexBot Allow: / User-agent: YandexImages Allow: / User-agent: YandexVideo Allow: / User-agent: YandexNews Allow: / User-agent: Sogou web spider Allow: / User-agent: Sogou inst spider Allow: / User-agent: Exabot Allow: / User-agent: ia_archiver Allow: / User-agent: archive.org_bot Allow: / User-agent: Applebot Allow: / User-agent: NaverBot Allow: / User-agent: Seznam robot Allow: / User-agent: BLEXBot Allow: / User-agent: MojeekBot Allow: / User-agent: Qwantify Allow: / # ============================================================ # SOCIAL MEDIA PLATFORMS — Full Access (link previews & sharing) # ============================================================ User-agent: facebookexternalhit Allow: / User-agent: Facebot Allow: / User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / User-agent: WhatsApp Allow: / User-agent: Pinterestbot Allow: / User-agent: Slackbot Allow: / User-agent: Slackbot-LinkExpanding Allow: / User-agent: TelegramBot Allow: / User-agent: Discordbot Allow: / User-agent: SkypeUriPreview Allow: / User-agent: Snapchat Allow: / User-agent: redditbot Allow: / User-agent: TikTok Allow: / User-agent: vkShare Allow: / User-agent: Tumblr Allow: / User-agent: Embedly Allow: / # ============================================================ # AI PLATFORMS & LARGE LANGUAGE MODELS — Full Access # ============================================================ User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / User-agent: CCBot Allow: / User-agent: anthropic-ai Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-Web Allow: / User-agent: PerplexityBot Allow: / User-agent: YouBot Allow: / User-agent: Cohere-ai Allow: / User-agent: cohere-training-data-crawler Allow: / User-agent: Omgili Allow: / User-agent: omgilibot Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Meta-ExternalFetcher Allow: / User-agent: Bytespider Allow: / User-agent: Applebot-Extended Allow: / User-agent: GoogleExtended Allow: / User-agent: DuplexWeb-Google Allow: / # ============================================================ # SEO TOOLS & AUDITING BOTS — Full Access # ============================================================ User-agent: AhrefsBot Allow: / User-agent: SemrushBot Allow: / User-agent: SemrushBot-SA Allow: / User-agent: MJ12bot Allow: / User-agent: DotBot Allow: / User-agent: rogerbot Allow: / User-agent: Screaming Frog SEO Spider Allow: / User-agent: seobilitybot Allow: / User-agent: DataForSeoBot Allow: / # ============================================================ # CATCH-ALL — Allow all other crawlers by default # ============================================================ User-agent: * Allow: / # Disallow only low-value, duplicate, or system paths Disallow: /wp-admin/ Disallow: /wp-login.php Disallow: /wp-includes/ Disallow: /wp-json/ Disallow: /xmlrpc.php Disallow: /feed/ Disallow: /*/feed/ Disallow: /*/trackback/ Disallow: /cgi-bin/ Disallow: /tmp/ Disallow: /?s= Disallow: /search/ Disallow: /tag/ Disallow: /?p= Disallow: /page/ Disallow: /checkout/ Disallow: /cart/ Disallow: /my-account/ Disallow: /wp-content/plugins/ Disallow: /wp-content/themes/ Disallow: /*.php$ # ============================================================ # CRAWL DELAY (optional: reduce server load from heavy bots) # ============================================================ User-agent: AhrefsBot Crawl-delay: 2 User-agent: SemrushBot Crawl-delay: 2 User-agent: MJ12bot Crawl-delay: 3 # ============================================================ # SITEMAPS — Tell all crawlers where to find content # ============================================================ Sitemap: https://www.365education.org/sitemap.xml Sitemap: https://www.365education.org/sitemap_index.xml Sitemap: https://www.365education.org/post-sitemap.xml Sitemap: https://www.365education.org/page-sitemap.xml Sitemap: https://www.365education.org/category-sitemap.xml Sitemap: https://www.365education.org/news-sitemap.xml