# robots.txt for PageForge # Domain: https://pageforge.pro/ # Strategy: Allow search engines and AI discovery crawlers. # Block only admin, login, internal, duplicate, and crawl-waste URLs. # BEGIN AI Content Signals # Generated on 2026-06-25 14:39:58 UTC # As a condition of accessing this website, you agree to abide by the following content signals: # (a) If a content-signal = yes, you may collect content for the corresponding use. # (b) If a content-signal = no, you may not collect content for the corresponding use. # (c) If the website operator does not include a content signal for a corresponding use, # the website operator neither grants nor restricts permission via content signal # with respect to the corresponding use. # # The content signals and their meanings are: # search: building a search index and providing search results # (e.g., returning hyperlinks and short excerpts from your website's contents). # Search does not include providing AI-generated search summaries. # ai-input: inputting content into one or more AI models # (e.g., retrieval augmented generation, grounding, or other real-time # taking of content for generative AI search answers). # ai-train: training or fine-tuning AI models. # # ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF RIGHTS # UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED # RIGHTS IN THE DIGITAL SINGLE MARKET. User-Agent: * Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: GPTBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: OAI-SearchBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: ChatGPT-User Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: ClaudeBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: Claude-Web Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: anthropic-ai Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: PerplexityBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: Perplexity-User Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: Google-Extended Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: GoogleOther Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: CCBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: cohere-ai Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: Bytespider Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: Amazonbot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: Applebot-Extended Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: meta-externalagent Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: FacebookBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: DuckAssistBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: AI2Bot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: MistralAI-User Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: Diffbot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: ImagesiftBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: Omgilibot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: YouBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: Timpibot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: DeepSeekBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: GrokBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / User-Agent: PanguBot Content-Signal: search=yes, ai-input=yes, ai-train=yes Allow: / # ----------------------------- # Major Search + AI Crawlers # ----------------------------- User-agent: Googlebot User-agent: Bingbot User-agent: Slurp User-agent: DuckDuckBot User-agent: Applebot User-agent: YandexBot User-agent: Baiduspider User-agent: OAI-SearchBot User-agent: GPTBot User-agent: ChatGPT-User User-agent: PerplexityBot User-agent: Perplexity-User User-agent: ClaudeBot User-agent: Claude-User User-agent: Claude-SearchBot User-agent: Google-Extended User-agent: CCBot User-agent: Amazonbot User-agent: FacebookBot User-agent: Meta-ExternalAgent User-agent: Bytespider User-agent: * Allow: / # END AI Content Signals # WordPress admin and private areas Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /cgi-bin/ # Internal WordPress files and low-value crawl areas Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Allow: /wp-content/uploads/ Allow: /wp-content/themes/ # Search result pages and duplicate internal searches Disallow: /?s= Disallow: /search/ Disallow: /*?s= Disallow: /*&s= # Reply/comment duplicate URLs Disallow: /*?replytocom= Disallow: /*&replytocom= # Tracking and campaign parameter duplicates Disallow: /*?utm_ Disallow: /*&utm_ Disallow: /*?fbclid= Disallow: /*&fbclid= Disallow: /*?gclid= Disallow: /*&gclid= Disallow: /*?msclkid= Disallow: /*&msclkid= # WordPress duplicate utility URLs Disallow: /trackback/ Disallow: /*/trackback/ Disallow: /comments/feed/ Disallow: /*/comments/feed/ # Prevent crawl waste on temporary preview URLs Disallow: /*?preview= Disallow: /*&preview= Disallow: /*?p= Disallow: /*&p= # ----------------------------- # XML Sitemaps # ----------------------------- Sitemap: https://pageforge.pro/sitemap_index.xml Sitemap: https://pageforge.pro/post-sitemap.xml Sitemap: https://pageforge.pro/page-sitemap.xml Sitemap: https://pageforge.pro/tools-sitemap.xml Sitemap: https://pageforge.pro/knowledge_base-sitemap.xml Sitemap: https://pageforge.pro/category-sitemap.xml Sitemap: https://pageforge.pro/post_tag-sitemap.xml Sitemap: https://pageforge.pro/kb_category-sitemap.xml Sitemap: https://pageforge.pro/pfpp_partner-sitemap.xml Sitemap: https://pageforge.pro/pfpp_service-sitemap.xml Sitemap: https://pageforge.pro/pfpp_industry-sitemap.xml Sitemap: https://pageforge.pro/pfpp_technology-sitemap.xml Sitemap: https://pageforge.pro/pfpp_country-sitemap.xml Sitemap: https://pageforge.pro/pfpp_language-sitemap.xml # VigIA LLMs # llms.txt: https://pageforge.pro/llms.txt