{"id":606,"date":"2026-11-20T14:03:42","date_gmt":"2026-11-20T03:03:42","guid":{"rendered":"https:\/\/onlinewebdirectory.net\/blog\/research\/qaimat-zawahif-al-dhakaa-al-istinai\/"},"modified":"2026-11-20T14:03:45","modified_gmt":"2026-11-20T03:03:45","slug":"qaimat-zawahif-al-dhakaa-al-istinai","status":"publish","type":"post","link":"https:\/\/onlinewebdirectory.net\/blog\/ar\/ai-crawlers-ar\/crawler-list-ar\/qaimat-zawahif-al-dhakaa-al-istinai\/","title":{"rendered":"\u0642\u0627\u0626\u0645\u0629 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a 2026: \u0643\u0644 \u0631\u0648\u0628\u0648\u062a \u0648\u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0648\u0645\u0647\u0645\u062a\u0647"},"content":{"rendered":"<p>\u0645\u0639\u0638\u0645 &#8220;\u0642\u0648\u0627\u0626\u0645 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a&#8221; \u0627\u0644\u062a\u064a \u0633\u062a\u062c\u062f\u0647\u0627 \u0645\u0646\u0633\u0648\u062e\u0629 \u0645\u0646 \u0628\u0639\u0636\u0647\u0627\u060c \u0648\u0643\u062b\u064a\u0631 \u0645\u0646\u0647\u0627 \u0642\u062f\u064a\u0645. \u0641\u0647\u064a \u0644\u0627 \u062a\u0632\u0627\u0644 \u062a\u0630\u0643\u0631 <code>Claude-Web<\/code> \u0643\u0632\u0627\u062d\u0641 Anthropic\u060c \u0623\u0648 \u062a\u062e\u062a\u0631\u0639 \u0631\u0648\u0628\u0648\u062a\u064b\u0627 \u0627\u0633\u0645\u0647 &#8220;Googlebot-Extended&#8221;\u060c \u0623\u0648 \u062a\u0642\u0648\u0644 \u0625\u0646 Google-Extended \u064a\u0632\u062d\u0641 \u0625\u0644\u0649 \u0635\u0641\u062d\u0627\u062a\u0643. \u0647\u0630\u0627 \u063a\u064a\u0631 \u0635\u062d\u064a\u062d. \u0644\u064a\u0633 \u0644\u0640 Google-Extended \u0648\u0643\u064a\u0644 \u0645\u0633\u062a\u062e\u062f\u0645 \u062e\u0627\u0635 \u0628\u0647\u061b \u0625\u0646\u0647 \u0645\u062c\u0631\u062f \u0631\u0645\u0632 \u0641\u064a robots.txt.<\/p>\n<p>\u062a\u062e\u062a\u0644\u0641 \u0647\u0630\u0647 \u0627\u0644\u0642\u0627\u0626\u0645\u0629 \u0641\u064a \u0623\u0645\u0631\u064a\u0646. \u0643\u0644 \u0648\u0643\u064a\u0644 \u0645\u0633\u062a\u062e\u062f\u0645 \u0623\u062f\u0646\u0627\u0647 \u062a\u0645 \u0627\u0644\u062a\u062d\u0642\u0642 \u0645\u0646\u0647 \u0645\u0642\u0627\u0628\u0644 \u0648\u062b\u0627\u0626\u0642 \u0627\u0644\u0645\u0634\u063a\u0651\u0644 \u0646\u0641\u0633\u0647 \u0628\u062a\u0627\u0631\u064a\u062e 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026. \u0643\u0645\u0627 \u0642\u0633\u0646\u0627 \u0643\u064a\u0641 \u062a\u062a\u0639\u0627\u0645\u0644 \u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u0627\u0644\u062d\u0642\u064a\u0642\u064a\u0629 \u0645\u0639 \u0647\u0630\u0647 \u0627\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a: \u062c\u0644\u0628\u0646\u0627 \u0645\u0644\u0641 robots.txt \u0645\u0646 3,000 \u0645\u0648\u0642\u0639 \u0627\u062e\u062a\u0631\u0646\u0627\u0647\u0627 \u0639\u0634\u0648\u0627\u0626\u064a\u064b\u0627 \u0645\u0646 \u0633\u062c\u0644 Online Web Directory\u060c \u0648\u0623\u062d\u0635\u064a\u0646\u0627 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0627\u0644\u062a\u064a \u062a\u0630\u0643\u0631\u0647\u0627 \u0648\u062a\u062d\u0638\u0631\u0647\u0627. \u0633\u062a\u062c\u062f \u0623\u064a\u0636\u064b\u0627 \u0642\u0648\u0627\u0639\u062f \u0645\u064f\u062e\u062a\u0628\u0631\u0629 \u0644\u0640 robots.txt \u0648Nginx \u0648Apache\u060c \u0648\u0633\u0643\u0631\u0628\u062a\u064b\u0627 \u064a\u062a\u062d\u0642\u0642 \u0645\u0645\u0627 \u0625\u0630\u0627 \u0643\u0627\u0646 \u0632\u0627\u0626\u0631 &#8220;GPTBot&#8221; \u0642\u0627\u062f\u0645\u064b\u0627 \u0641\u0639\u0644\u064b\u0627 \u0645\u0646 OpenAI.<\/p>\n<div class=\"owd-definition\">\n<p><strong>\u0645\u0627 \u0647\u064a \u0642\u0627\u0626\u0645\u0629 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a\u061f<\/strong> \u0642\u0627\u0626\u0645\u0629 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0645\u0631\u062c\u0639 \u0644\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a \u0627\u0644\u062a\u064a \u062a\u0633\u062a\u062e\u062f\u0645\u0647\u0627 \u0634\u0631\u0643\u0627\u062a \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0644\u062c\u0644\u0628 \u0635\u0641\u062d\u0627\u062a \u0627\u0644\u0648\u064a\u0628\u060c \u0645\u0639 \u0631\u0645\u0632 robots.txt \u0644\u0643\u0644 \u0631\u0648\u0628\u0648\u062a \u0648\u0633\u0644\u0633\u0644\u0629 \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0648\u0645\u0647\u0645\u062a\u0647: \u062c\u0645\u0639 \u0628\u064a\u0627\u0646\u0627\u062a \u0627\u0644\u062a\u062f\u0631\u064a\u0628\u060c \u0623\u0648 \u0641\u0647\u0631\u0633\u0629 \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0644\u0644\u0628\u062d\u062b \u0628\u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a\u060c \u0623\u0648 \u062c\u0644\u0628 \u0635\u0641\u062d\u0629 \u0645\u0628\u0627\u0634\u0631\u0629 \u0644\u0623\u0646 \u0645\u0633\u062a\u062e\u062f\u0645\u064b\u0627 \u0637\u0644\u0628\u0647\u0627. \u062a\u0633\u062a\u062e\u062f\u0645\u0647\u0627 \u0644\u0643\u062a\u0627\u0628\u0629 \u0642\u0648\u0627\u0639\u062f robots.txt \u0648\u0642\u0631\u0627\u0621\u0629 \u0633\u062c\u0644\u0627\u062a \u0627\u0644\u062e\u0627\u062f\u0645 \u0648\u0636\u0628\u0637 \u0633\u064a\u0627\u0633\u0627\u062a \u062c\u062f\u0627\u0631 \u0627\u0644\u062d\u0645\u0627\u064a\u0629.<\/p>\n<\/div>\n<h2 id=\"at-a-glance\">\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0641\u064a \u0644\u0645\u062d\u0629<\/h2>\n<figure class=\"wp-block-table owd-table\">\n<table>\n<thead>\n<tr>\n<th>\u0627\u0644\u0633\u0624\u0627\u0644<\/th>\n<th>\u0625\u062c\u0627\u0628\u0629 \u0645\u062e\u062a\u0635\u0631\u0629<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>\u0643\u0645 \u0639\u062f\u062f \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0627\u0644\u0645\u0647\u0645\u0629\u061f<\/td>\n<td>\u0646\u062d\u0648 20 \u0648\u0643\u064a\u0644 \u0645\u0633\u062a\u062e\u062f\u0645 \u0645\u0648\u062b\u0651\u0642\u064b\u0627 \u0645\u0646 9 \u0645\u0634\u063a\u0651\u0644\u064a\u0646 (OpenAI \u0648Anthropic \u0648Google \u0648Perplexity \u0648Apple \u0648Meta \u0648Amazon \u0648Common Crawl \u0648Mistral)\u060c \u0625\u0636\u0627\u0641\u0629 \u0625\u0644\u0649 DuckDuckGo \u0648\u0639\u062f\u062f \u0643\u0628\u064a\u0631 \u0645\u0646 \u0627\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a \u0627\u0644\u0635\u063a\u064a\u0631\u0629.<\/td>\n<\/tr>\n<tr>\n<td>\u0627\u0644\u0623\u0646\u0648\u0627\u0639 \u0627\u0644\u0631\u0626\u064a\u0633\u064a\u0629<\/td>\n<td>\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062a\u062f\u0631\u064a\u0628\u060c \u0648\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0628\u062d\u062b \u0628\u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a\u060c \u0648\u0627\u0644\u062c\u0627\u0644\u0628\u0627\u062a \u0623\u0648 \u0627\u0644\u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0630\u064a\u0646 \u064a\u0634\u063a\u0651\u0644\u0647\u0645 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645. Google-Extended \u0648Applebot-Extended \u0631\u0645\u0648\u0632 \u062a\u062d\u0643\u0645 \u0648\u0644\u064a\u0633\u062a \u0632\u0648\u0627\u062d\u0641.<\/td>\n<\/tr>\n<tr>\n<td>\u0647\u0644 \u062a\u0644\u062a\u0632\u0645 \u0628\u0645\u0644\u0641 robots.txt\u061f<\/td>\n<td>\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u0648\u0627\u0644\u0628\u062d\u062b \u0627\u0644\u0645\u0648\u062b\u0651\u0642\u0629 \u062a\u0642\u0648\u0644 \u0625\u0646\u0647\u0627 \u062a\u0644\u062a\u0632\u0645. \u0623\u0645\u0627 \u0627\u0644\u062c\u0627\u0644\u0628\u0627\u062a \u0627\u0644\u062a\u064a \u064a\u0634\u063a\u0651\u0644\u0647\u0627 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 (ChatGPT-User \u0648Perplexity-User \u0648\u062c\u0627\u0644\u0628\u0627\u062a Google \u0627\u0644\u062a\u064a \u064a\u0634\u063a\u0651\u0644\u0647\u0627 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0648Meta-ExternalFetcher) \u0641\u062a\u0642\u0648\u0644 \u0625\u0646 robots.txt \u0642\u062f \u0644\u0627 \u064a\u0646\u0637\u0628\u0642 \u0639\u0644\u064a\u0647\u0627.<\/td>\n<\/tr>\n<tr>\n<td>\u0647\u0644 \u064a\u0645\u0643\u0646 \u0644\u0645\u0644\u0641 robots.txt \u0623\u0646 \u064a\u064f\u0631\u062c\u0639 403\u061f<\/td>\n<td>\u0644\u0627. \u0645\u0644\u0641 robots.txt \u0637\u0644\u0628 \u0648\u0644\u064a\u0633 \u0642\u0641\u0644\u064b\u0627. \u0631\u0645\u0632 403 \u064a\u0623\u062a\u064a \u0645\u0646 \u062e\u0627\u062f\u0645\u0643 \u0623\u0648 \u0634\u0628\u0643\u0629 CDN \u0623\u0648 \u062c\u062f\u0627\u0631 \u0627\u0644\u062d\u0645\u0627\u064a\u0629.<\/td>\n<\/tr>\n<tr>\n<td>\u0643\u0645 \u0645\u0631\u0629 \u062a\u062d\u0638\u0631\u0647\u0627 \u0627\u0644\u0645\u0648\u0627\u0642\u0639\u061f<\/td>\n<td>\u0641\u064a \u0639\u064a\u0651\u0646\u062a\u0646\u0627 \u0628\u062a\u0627\u0631\u064a\u062e 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026\u060c \u062a\u062d\u0638\u0631 16.9% \u0645\u0646 \u0645\u0644\u0641\u0627\u062a robots.txt \u0631\u0648\u0628\u0648\u062a GPTBot \u062d\u0638\u0631\u064b\u0627 \u0643\u0627\u0645\u0644\u064b\u0627\u060c \u0628\u064a\u0646\u0645\u0627 \u062a\u062d\u0638\u0631 2.8% \u0641\u0642\u0637 OAI-SearchBot \u062d\u0638\u0631\u064b\u0627 \u0643\u0627\u0645\u0644\u064b\u0627.<\/td>\n<\/tr>\n<tr>\n<td>\u0643\u064a\u0641 \u0623\u062a\u062d\u0642\u0642 \u0645\u0646 \u0631\u0648\u0628\u0648\u062a\u061f<\/td>\n<td>\u0637\u0627\u0628\u0650\u0642 \u0639\u0646\u0648\u0627\u0646 IP \u0644\u0644\u0637\u0644\u0628 \u0645\u0639 \u0642\u0627\u0626\u0645\u0629 \u0639\u0646\u0627\u0648\u064a\u0646 IP \u0627\u0644\u062a\u064a \u064a\u0646\u0634\u0631\u0647\u0627 \u0627\u0644\u0645\u0634\u063a\u0651\u0644 (\u0645\u062b\u0644 <code>https:\/\/openai.com\/gptbot.json<\/code>). \u062a\u0632\u064a\u064a\u0641 \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0623\u0645\u0631 \u0633\u0647\u0644 \u0644\u0644\u063a\u0627\u064a\u0629.<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/figure>\n<h2 id=\"the-list\">\u0642\u0627\u0626\u0645\u0629 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a (\u062a\u0645 \u0627\u0644\u062a\u062d\u0642\u0642 \u0641\u064a \u0646\u0648\u0641\u0645\u0628\u0631 2026)<\/h2>\n<p>\u064a\u0639\u0631\u0636 \u0627\u0644\u062c\u062f\u0648\u0644 \u0623\u062f\u0646\u0627\u0647 \u0643\u0644 \u0631\u0645\u0632 \u0641\u064a robots.txt\u060c \u0648\u0645\u0646 \u064a\u0634\u063a\u0651\u0644\u0647\u060c \u0648\u0645\u0627 \u064a\u0641\u0639\u0644\u0647 \u0648\u0641\u0642\u064b\u0627 \u0644\u0644\u0645\u0634\u063a\u0651\u0644\u060c \u0648\u0645\u0627 \u0625\u0630\u0627 \u0643\u0627\u0646 \u0627\u0644\u0645\u0634\u063a\u0651\u0644 \u064a\u0642\u0648\u0644 \u0625\u0646\u0647 \u064a\u0644\u062a\u0632\u0645 \u0628\u0645\u0644\u0641 robots.txt. \u064a\u0633\u062a\u062e\u062f\u0645 \u0639\u0645\u0648\u062f &#8220;\u0627\u0644\u0646\u0648\u0639&#8221; \u062b\u0644\u0627\u062b \u0641\u0626\u0627\u062a: <strong>\u062a\u062f\u0631\u064a\u0628<\/strong> (\u064a\u062c\u0645\u0639 \u0645\u062d\u062a\u0648\u0649 \u0642\u062f \u064a\u064f\u0633\u062a\u062e\u062f\u0645 \u0644\u062a\u062f\u0631\u064a\u0628 \u0627\u0644\u0646\u0645\u0627\u0630\u062c)\u060c \u0648<strong>\u0628\u062d\u062b<\/strong> (\u064a\u0641\u0647\u0631\u0633 \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0643\u064a \u064a\u0633\u062a\u0634\u0647\u062f \u0628\u0647\u0627 \u0645\u0646\u062a\u062c \u0630\u0643\u0627\u0621 \u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0644\u0627\u062d\u0642\u064b\u0627)\u060c \u0648<strong>\u0645\u0633\u062a\u062e\u062f\u0645<\/strong> (\u064a\u062c\u0644\u0628 \u0635\u0641\u062d\u0629 \u0641\u064a \u0627\u0644\u0648\u0642\u062a \u0627\u0644\u0641\u0639\u0644\u064a \u0644\u0623\u0646 \u0634\u062e\u0635\u064b\u0627 \u0637\u0644\u0628\u0647\u0627). \u0627\u0633\u062a\u0628\u0639\u062f\u0646\u0627 \u0627\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a \u0627\u0644\u062a\u064a \u0644\u0627 \u064a\u0646\u0634\u0631 \u0645\u0634\u063a\u0651\u0644\u0647\u0627 \u0648\u062b\u0627\u0626\u0642 \u062a\u0645\u0643\u0651\u0646\u0627 \u0645\u0646 \u0627\u0644\u062a\u062d\u0642\u0642 \u0645\u0646\u0647\u0627.<\/p>\n<figure class=\"wp-block-table owd-table\">\n<table>\n<thead>\n<tr>\n<th>\u0631\u0645\u0632 robots.txt<\/th>\n<th>\u0627\u0644\u0645\u0634\u063a\u0651\u0644<\/th>\n<th>\u0627\u0644\u0646\u0648\u0639<\/th>\n<th>\u0645\u0627\u0630\u0627 \u064a\u0641\u0639\u0644<\/th>\n<th>\u064a\u0644\u062a\u0632\u0645 \u0628\u0640 robots.txt\u061f<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td><code>GPTBot<\/code><\/td>\n<td>OpenAI<\/td>\n<td>\u062a\u062f\u0631\u064a\u0628<\/td>\n<td>\u064a\u0632\u062d\u0641 \u0625\u0644\u0649 \u0645\u062d\u062a\u0648\u0649 \u0642\u062f \u064a\u064f\u0633\u062a\u062e\u062f\u0645 \u0644\u062a\u062f\u0631\u064a\u0628 \u0627\u0644\u0646\u0645\u0627\u0630\u062c \u0627\u0644\u0623\u0633\u0627\u0633\u064a\u0629 \u0644\u0640 OpenAI.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>OAI-SearchBot<\/code><\/td>\n<td>OpenAI<\/td>\n<td>\u0628\u062d\u062b<\/td>\n<td>\u064a\u0641\u0647\u0631\u0633 \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0643\u064a \u062a\u0638\u0647\u0631 \u0641\u064a \u0646\u062a\u0627\u0626\u062c \u0628\u062d\u062b ChatGPT.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>ChatGPT-User<\/code><\/td>\n<td>OpenAI<\/td>\n<td>\u0645\u0633\u062a\u062e\u062f\u0645<\/td>\n<td>\u064a\u0632\u0648\u0631 \u0635\u0641\u062d\u0629 \u0639\u0646\u062f\u0645\u0627 \u064a\u0637\u0631\u062d \u0645\u0633\u062a\u062e\u062f\u0645 ChatGPT \u0623\u0648 GPT \u0645\u062e\u0635\u0651\u0635 \u0633\u0624\u0627\u0644\u064b\u0627.<\/td>\n<td>&#8220;\u0642\u062f \u0644\u0627 \u064a\u0646\u0637\u0628\u0642&#8221;<\/td>\n<\/tr>\n<tr>\n<td><code>OAI-AdsBot<\/code><\/td>\n<td>OpenAI<\/td>\n<td>\u0641\u062d\u0635 \u0627\u0644\u0625\u0639\u0644\u0627\u0646\u0627\u062a<\/td>\n<td>\u064a\u0641\u062d\u0635 \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0627\u0644\u0645\u0642\u0635\u0648\u062f\u0629 \u0627\u0644\u0645\u0642\u062f\u064e\u0651\u0645\u0629 \u0643\u0625\u0639\u0644\u0627\u0646\u0627\u062a \u0641\u064a ChatGPT\u061b \u0644\u0627 \u064a\u064f\u0633\u062a\u062e\u062f\u0645 \u0644\u0644\u062a\u062f\u0631\u064a\u0628.<\/td>\n<td>\u064a\u0632\u0648\u0631 \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0627\u0644\u0645\u0642\u062f\u064e\u0651\u0645\u0629 \u0641\u0642\u0637<\/td>\n<\/tr>\n<tr>\n<td><code>ClaudeBot<\/code><\/td>\n<td>Anthropic<\/td>\n<td>\u062a\u062f\u0631\u064a\u0628<\/td>\n<td>\u064a\u062c\u0645\u0639 \u0645\u062d\u062a\u0648\u0649 \u0627\u0644\u0648\u064a\u0628 \u0627\u0644\u0630\u064a \u0642\u062f \u064a\u0633\u0647\u0645 \u0641\u064a \u062a\u062f\u0631\u064a\u0628 \u0646\u0645\u0627\u0630\u062c Claude.<\/td>\n<td>\u0646\u0639\u0645\u060c \u0648\u064a\u062d\u062a\u0631\u0645 <code>Crawl-delay<\/code><\/td>\n<\/tr>\n<tr>\n<td><code>Claude-SearchBot<\/code><\/td>\n<td>Anthropic<\/td>\n<td>\u0628\u062d\u062b<\/td>\n<td>\u064a\u0641\u0647\u0631\u0633 \u0627\u0644\u0645\u062d\u062a\u0648\u0649 \u0644\u062a\u062d\u0633\u064a\u0646 \u0646\u062a\u0627\u0626\u062c \u0628\u062d\u062b Claude.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>Claude-User<\/code><\/td>\n<td>Anthropic<\/td>\n<td>\u0645\u0633\u062a\u062e\u062f\u0645<\/td>\n<td>\u064a\u062c\u0644\u0628 \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0639\u0646\u062f\u0645\u0627 \u064a\u0637\u0631\u062d \u0645\u0633\u062a\u062e\u062f\u0645 Claude \u0633\u0624\u0627\u0644\u064b\u0627.<\/td>\n<td>\u0646\u0639\u0645\u060c \u0648\u0641\u0642\u064b\u0627 \u0644\u0640 Anthropic<\/td>\n<\/tr>\n<tr>\n<td><code>Google-Extended<\/code><\/td>\n<td>Google<\/td>\n<td>\u0631\u0645\u0632 \u062a\u062d\u0643\u0645<\/td>\n<td>\u064a\u062d\u062f\u062f \u0645\u0627 \u0625\u0630\u0627 \u0643\u0627\u0646 \u0627\u0644\u0645\u062d\u062a\u0648\u0649 \u0627\u0644\u0630\u064a \u062a\u0632\u062d\u0641 \u0625\u0644\u064a\u0647 Google \u064a\u0645\u0643\u0646 \u0623\u0646 \u064a\u062f\u0631\u0651\u0628 Gemini \u0623\u0648 \u064a\u062f\u0639\u0645 \u0625\u062c\u0627\u0628\u0627\u062a\u0647. \u0644\u0627 \u064a\u0648\u062c\u062f \u0632\u0627\u062d\u0641 \u0645\u0646\u0641\u0635\u0644.<\/td>\n<td>\u0631\u0645\u0632 \u0641\u0642\u0637<\/td>\n<\/tr>\n<tr>\n<td><code>Google-CloudVertexBot<\/code><\/td>\n<td>Google<\/td>\n<td>\u0628\u0637\u0644\u0628 \u0627\u0644\u0645\u0627\u0644\u0643<\/td>\n<td>\u064a\u0632\u062d\u0641 \u0625\u0644\u0649 \u0645\u0648\u0627\u0642\u0639 \u0627\u0644\u0645\u0627\u0644\u0643\u064a\u0646 \u0627\u0644\u0630\u064a\u0646 \u064a\u0628\u0646\u0648\u0646 Vertex AI Agents\u061b \u0644\u0627 \u062a\u0623\u062b\u064a\u0631 \u0644\u0647 \u0639\u0644\u0649 \u0628\u062d\u062b Google.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>Google-Agent<\/code><\/td>\n<td>Google<\/td>\n<td>\u0645\u0633\u062a\u062e\u062f\u0645<\/td>\n<td>\u0648\u0643\u0644\u0627\u0621 \u064a\u0639\u0645\u0644\u0648\u0646 \u0639\u0644\u0649 \u0628\u0646\u064a\u0629 Google \u0648\u064a\u062a\u0635\u0641\u062d\u0648\u0646 \u0648\u064a\u0646\u0641\u0630\u0648\u0646 \u0625\u062c\u0631\u0627\u0621\u0627\u062a \u0628\u0637\u0644\u0628 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645.<\/td>\n<td>\u0628\u062a\u0634\u063a\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645: \u0639\u0645\u0648\u0645\u064b\u0627 \u0644\u0627<\/td>\n<\/tr>\n<tr>\n<td><code>PerplexityBot<\/code><\/td>\n<td>Perplexity<\/td>\n<td>\u0628\u062d\u062b<\/td>\n<td>\u064a\u0641\u0647\u0631\u0633 \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0644\u0646\u062a\u0627\u0626\u062c \u0628\u062d\u062b Perplexity\u061b \u0644\u0627 \u064a\u064f\u0633\u062a\u062e\u062f\u0645 \u0644\u0644\u062a\u062f\u0631\u064a\u0628.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>Perplexity-User<\/code><\/td>\n<td>Perplexity<\/td>\n<td>\u0645\u0633\u062a\u062e\u062f\u0645<\/td>\n<td>\u064a\u062c\u0644\u0628 \u0635\u0641\u062d\u0629 \u0623\u062b\u0646\u0627\u0621 \u062c\u0644\u0633\u0629 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0641\u064a Perplexity.<\/td>\n<td>&#8220;\u064a\u062a\u062c\u0627\u0647\u0644\u0647 \u0639\u0645\u0648\u0645\u064b\u0627&#8221;<\/td>\n<\/tr>\n<tr>\n<td><code>Applebot-Extended<\/code><\/td>\n<td>Apple<\/td>\n<td>\u0631\u0645\u0632 \u062a\u062d\u0643\u0645<\/td>\n<td>\u064a\u062d\u062f\u062f \u0645\u0627 \u0625\u0630\u0627 \u0643\u0627\u0646\u062a \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0627\u0644\u062a\u064a \u0632\u062d\u0641 \u0625\u0644\u064a\u0647\u0627 Applebot \u064a\u0645\u0643\u0646 \u0623\u0646 \u062a\u062f\u0631\u0651\u0628 \u0627\u0644\u0646\u0645\u0627\u0630\u062c \u0627\u0644\u0623\u0633\u0627\u0633\u064a\u0629 \u0644\u0640 Apple. \u0644\u0627 \u064a\u0632\u062d\u0641.<\/td>\n<td>\u0631\u0645\u0632 \u0641\u0642\u0637<\/td>\n<\/tr>\n<tr>\n<td><code>meta-externalagent<\/code><\/td>\n<td>Meta<\/td>\n<td>\u062a\u062f\u0631\u064a\u0628<\/td>\n<td>\u064a\u0632\u062d\u0641 \u0644\u0627\u0633\u062a\u062e\u062f\u0627\u0645\u0627\u062a \u0645\u062b\u0644 \u062a\u062f\u0631\u064a\u0628 \u0646\u0645\u0627\u0630\u062c \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0623\u0648 \u0641\u0647\u0631\u0633\u0629 \u0627\u0644\u0645\u062d\u062a\u0648\u0649.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>meta-externalfetcher<\/code><\/td>\n<td>Meta<\/td>\n<td>\u0645\u0633\u062a\u062e\u062f\u0645<\/td>\n<td>\u064a\u062c\u0644\u0628 \u0631\u0648\u0627\u0628\u0637 \u0641\u0631\u062f\u064a\u0629 \u0628\u0637\u0644\u0628 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645\u060c \u0628\u0645\u0627 \u0641\u064a \u0630\u0644\u0643 \u0627\u0644\u0645\u0647\u0627\u0645 \u0627\u0644\u0648\u0643\u064a\u0644\u064a\u0629.<\/td>\n<td>&#8220;\u0642\u062f \u064a\u062a\u062c\u0627\u0648\u0632\u0647&#8221;<\/td>\n<\/tr>\n<tr>\n<td><code>Amazonbot<\/code><\/td>\n<td>Amazon<\/td>\n<td>\u062a\u062f\u0631\u064a\u0628<\/td>\n<td>\u064a\u062d\u0633\u0651\u0646 \u0645\u0646\u062a\u062c\u0627\u062a Amazon \u0648\u062e\u062f\u0645\u0627\u062a\u0647\u0627\u061b \u0642\u062f \u064a\u064f\u0633\u062a\u062e\u062f\u0645 \u0644\u062a\u062f\u0631\u064a\u0628 \u0646\u0645\u0627\u0630\u062c \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0644\u062f\u0649 Amazon.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>Amzn-SearchBot<\/code><\/td>\n<td>Amazon<\/td>\n<td>\u0628\u062d\u062b<\/td>\n<td>\u064a\u0641\u0647\u0631\u0633 \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0644\u062a\u062c\u0627\u0631\u0628 \u0627\u0644\u0628\u062d\u062b \u0645\u062b\u0644 Alexa\u061b \u0644\u0627 \u064a\u064f\u0633\u062a\u062e\u062f\u0645 \u0644\u0644\u062a\u062f\u0631\u064a\u0628.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>Amzn-User<\/code><\/td>\n<td>Amazon<\/td>\n<td>\u0645\u0633\u062a\u062e\u062f\u0645<\/td>\n<td>\u064a\u062c\u0644\u0628 \u0645\u0639\u0644\u0648\u0645\u0627\u062a \u062d\u064a\u0651\u0629 \u0644\u0644\u0625\u062c\u0627\u0628\u0629 \u0639\u0644\u0649 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645\u060c \u0645\u062b\u0644 \u0633\u0624\u0627\u0644 \u0645\u0648\u062c\u0651\u0647 \u0625\u0644\u0649 Alexa.<\/td>\n<td>&#8220;\u0642\u062f \u0644\u0627 \u064a\u0644\u062a\u0632\u0645 \u0628\u0643\u0644 \u0634\u064a\u0621&#8221;<\/td>\n<\/tr>\n<tr>\n<td><code>CCBot<\/code><\/td>\n<td>Common Crawl<\/td>\n<td>\u0628\u064a\u0627\u0646\u0627\u062a \u0627\u0644\u062a\u062f\u0631\u064a\u0628<\/td>\n<td>\u064a\u0628\u0646\u064a \u0623\u0631\u0634\u064a\u0641 \u0627\u0644\u0648\u064a\u0628 \u0627\u0644\u0645\u0641\u062a\u0648\u062d Common Crawl\u060c \u0648\u0647\u0648 \u0645\u0635\u062f\u0631 \u0634\u0627\u0626\u0639 \u0644\u0628\u064a\u0627\u0646\u0627\u062a \u062a\u062f\u0631\u064a\u0628 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>MistralAI-Training<\/code><\/td>\n<td>Mistral AI<\/td>\n<td>\u062a\u062f\u0631\u064a\u0628<\/td>\n<td>\u064a\u0632\u062d\u0641 \u0625\u0644\u0649 \u0627\u0644\u0645\u062d\u062a\u0648\u0649 \u0644\u0628\u0646\u0627\u0621 \u0645\u062c\u0645\u0648\u0639\u0627\u062a \u0628\u064a\u0627\u0646\u0627\u062a \u0644\u062a\u062f\u0631\u064a\u0628 \u0646\u0645\u0627\u0630\u062c Mistral.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>MistralAI-Index<\/code><\/td>\n<td>Mistral AI<\/td>\n<td>\u0628\u062d\u062b<\/td>\n<td>\u064a\u0641\u0647\u0631\u0633 \u0627\u0644\u0645\u062d\u062a\u0648\u0649 \u0644\u0628\u062d\u062b Mistral\u061b \u0644\u0627 \u064a\u064f\u0633\u062a\u062e\u062f\u0645 \u0644\u0644\u062a\u062f\u0631\u064a\u0628.<\/td>\n<td>\u0646\u0639\u0645<\/td>\n<\/tr>\n<tr>\n<td><code>MistralAI-User<\/code><\/td>\n<td>Mistral AI<\/td>\n<td>\u0645\u0633\u062a\u062e\u062f\u0645<\/td>\n<td>\u064a\u0632\u0648\u0631 \u0635\u0641\u062d\u0629 \u0639\u0646\u062f\u0645\u0627 \u064a\u0637\u0631\u062d \u0645\u0633\u062a\u062e\u062f\u0645 \u0633\u0624\u0627\u0644\u064b\u0627 \u0639\u0644\u0649 \u0645\u0633\u0627\u0639\u062f Vibe \u0645\u0646 Mistral.<\/td>\n<td>\u0646\u0639\u0645\u060c \u0644\u0637\u0644\u0628\u0627\u062a \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645\u064a\u0646<\/td>\n<\/tr>\n<tr>\n<td><code>DuckAssistBot<\/code><\/td>\n<td>DuckDuckGo<\/td>\n<td>\u0628\u062d\u062b<\/td>\n<td>\u064a\u0632\u062d\u0641 \u0625\u0644\u0649 \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0641\u064a \u0627\u0644\u0648\u0642\u062a \u0627\u0644\u0641\u0639\u0644\u064a \u0644\u0625\u062c\u0627\u0628\u0627\u062a DuckDuckGo \u0627\u0644\u0645\u062f\u0639\u0648\u0645\u0629 \u0628\u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a.<\/td>\n<td>\u0646\u0639\u0645 (\u064a\u0633\u0631\u064a \u0628\u0639\u062f 72 \u0633\u0627\u0639\u0629)<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/figure>\n<p>\u0645\u0635\u0627\u062f\u0631 \u0643\u0644 \u0635\u0641 \u0645\u0631\u062a\u0628\u0637\u0629 \u0641\u064a \u0646\u0647\u0627\u064a\u0629 \u0647\u0630\u0627 \u0627\u0644\u0645\u0642\u0627\u0644. \u0627\u0644\u0627\u0642\u062a\u0628\u0627\u0633\u0627\u062a \u0641\u064a \u0627\u0644\u0639\u0645\u0648\u062f \u0627\u0644\u0623\u062e\u064a\u0631 \u0645\u0623\u062e\u0648\u0630\u0629 \u0645\u0628\u0627\u0634\u0631\u0629 \u0645\u0646 \u0648\u062b\u0627\u0626\u0642 \u0627\u0644\u0645\u0634\u063a\u0651\u0644. \u0627\u0633\u062a\u0628\u0639\u062f\u0646\u0627 Bytespider (ByteDance) \u0639\u0646 \u0642\u0635\u062f: \u0644\u0645 \u0646\u062c\u062f \u0635\u0641\u062d\u0629 \u0645\u0646 ByteDance \u062a\u0648\u062b\u0651\u0642\u0647\u060c \u0631\u063a\u0645 \u0623\u0646 18.5% \u0645\u0646 \u0645\u0644\u0641\u0627\u062a robots.txt \u0627\u0644\u062a\u064a \u0641\u062d\u0635\u0646\u0627\u0647\u0627 \u062a\u0630\u0643\u0631\u0647. \u0648\u0644\u0644\u0633\u0628\u0628 \u0646\u0641\u0633\u0647 \u0644\u0627 \u0646\u0635\u0641 \u0627\u0644\u0631\u0645\u0648\u0632 \u0627\u0644\u0623\u0642\u0644 \u0634\u064a\u0648\u0639\u064b\u0627 \u0645\u062b\u0644 <code>cohere-ai<\/code> \u0648<code>Diffbot<\/code> \u0648<code>omgili<\/code>\u061b \u0633\u062a\u062c\u062f \u0641\u064a \u0642\u0633\u0645 \u0627\u0644\u0628\u064a\u0627\u0646\u0627\u062a \u0643\u0645 \u0645\u0631\u0629 \u062a\u0630\u0643\u0631\u0647\u0627 \u0627\u0644\u0645\u0648\u0627\u0642\u0639.<\/p>\n<h3>\u0633\u0644\u0627\u0633\u0644 \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0627\u0644\u0643\u0627\u0645\u0644\u0629 \u0627\u0644\u062a\u064a \u0633\u062a\u0631\u0627\u0647\u0627 \u0641\u064a \u0627\u0644\u0633\u062c\u0644\u0627\u062a<\/h3>\n<p>\u064a\u0637\u0627\u0628\u0642 robots.txt \u0627\u0644\u0631\u0645\u0632 \u0627\u0644\u0642\u0635\u064a\u0631\u060c \u0644\u0643\u0646 \u0633\u062c\u0644\u0627\u062a \u0627\u0644\u0648\u0635\u0648\u0644 \u062a\u0639\u0631\u0636 \u0627\u0644\u0633\u0644\u0633\u0644\u0629 \u0627\u0644\u0643\u0627\u0645\u0644\u0629. \u0647\u0630\u0647 \u0647\u064a \u0627\u0644\u0623\u0645\u062b\u0644\u0629 \u0627\u0644\u062a\u064a \u064a\u0646\u0634\u0631\u0647\u0627 \u0627\u0644\u0645\u0634\u063a\u0651\u0644\u0648\u0646. \u0623\u0631\u0642\u0627\u0645 \u0627\u0644\u0625\u0635\u062f\u0627\u0631\u0627\u062a \u062a\u062a\u063a\u064a\u0631\u060c \u0644\u0630\u0627 \u0637\u0627\u0628\u0650\u0642 \u0639\u0644\u0649 \u0627\u0644\u0631\u0645\u0632 \u0644\u0627 \u0639\u0644\u0649 \u0627\u0644\u0633\u0644\u0633\u0644\u0629 \u0628\u0623\u0643\u0645\u0644\u0647\u0627.<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code># GPTBot\nMozilla\/5.0 AppleWebKit\/537.36 (KHTML, like Gecko); compatible; GPTBot\/1.4; +https:\/\/openai.com\/gptbot\n\n# OAI-SearchBot\nMozilla\/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/131.0.0.0 Safari\/537.36; compatible; OAI-SearchBot\/1.4; +https:\/\/openai.com\/searchbot\n\n# ChatGPT-User\nMozilla\/5.0 AppleWebKit\/537.36 (KHTML, like Gecko); compatible; ChatGPT-User\/1.0; +https:\/\/openai.com\/bot\n\n# PerplexityBot\nMozilla\/5.0 (compatible; PerplexityBot\/1.0; +https:\/\/perplexity.ai\/perplexitybot)\n\n# Perplexity-User\nMozilla\/5.0 (compatible; Perplexity-User\/1.0; +https:\/\/perplexity.ai\/perplexity-user)\n\n# Amazonbot\nMozilla\/5.0 AppleWebKit\/537.36 (KHTML, like Gecko; compatible; Amazonbot\/0.1) Chrome\/W.X.Y.Z Safari\/537.36\n\n# CCBot\nCCBot\/2.0 (https:\/\/commoncrawl.org\/faq\/)\n\n# meta-externalagent\nmeta-externalagent\/1.1 (+https:\/\/developers.facebook.com\/docs\/sharing\/webmasters\/crawler)\n\n# MistralAI-User\nMozilla\/5.0 AppleWebKit\/537.36 (KHTML, like Gecko; compatible; MistralAI-User\/1.0; +https:\/\/docs.mistral.ai\/robots)\n\n# DuckAssistBot\nDuckAssistBot\/1.2; (+http:\/\/duckduckgo.com\/duckassistbot.html)<\/code><\/pre>\n<p>\u062a\u0634\u064a\u0631 OpenAI \u0623\u064a\u0636\u064b\u0627 \u0625\u0644\u0649 \u0623\u0646 GPTBot \u0648OAI-SearchBot \u0642\u062f \u064a\u0636\u064a\u0641\u0627\u0646 \u0639\u0644\u0627\u0645\u0629 <code>robots.txt<\/code> \u0625\u0644\u0649 \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0639\u0646\u062f \u062c\u0644\u0628 \u0645\u0644\u0641 robots.txt\u060c \u0644\u062a\u062a\u0645\u0643\u0646 \u0645\u0646 \u062a\u0645\u064a\u064a\u0632 \u062a\u0644\u0643 \u0627\u0644\u0637\u0644\u0628\u0627\u062a \u0641\u064a \u0627\u0644\u0633\u062c\u0644\u0627\u062a \u0627\u0644\u062a\u064a \u0644\u0627 \u062a\u0633\u062c\u0651\u0644 \u0627\u0644\u0645\u0633\u0627\u0631\u0627\u062a.<\/p>\n<h2 id=\"types\">\u062b\u0644\u0627\u062b\u0629 \u0623\u0646\u0648\u0627\u0639 \u0645\u0646 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a\u060c \u0648\u0644\u0645\u0627\u0630\u0627 \u064a\u0647\u0645 \u0627\u0644\u0641\u0631\u0642<\/h2>\n<p>\u062d\u0638\u0631 &#8220;\u0631\u0648\u0628\u0648\u062a\u0627\u062a \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a&#8221; \u0643\u0645\u062c\u0645\u0648\u0639\u0629 \u0648\u0627\u062d\u062f\u0629 \u0647\u0648 \u0623\u0643\u062b\u0631 \u062e\u0637\u0623 \u0646\u0631\u0627\u0647. \u0644\u0644\u0623\u0646\u0648\u0627\u0639 \u0627\u0644\u062b\u0644\u0627\u062b\u0629 \u062a\u0643\u0627\u0644\u064a\u0641 \u0648\u0641\u0648\u0627\u0626\u062f \u0645\u062e\u062a\u0644\u0641\u0629\u060c \u0648\u064a\u062a\u0639\u0627\u0645\u0644 \u0627\u0644\u0645\u0634\u063a\u0651\u0644\u0648\u0646 \u0645\u0639 robots.txt \u0628\u0634\u0643\u0644 \u0645\u062e\u062a\u0644\u0641 \u0644\u0643\u0644 \u0645\u0646\u0647\u0627.<\/p>\n<h3>\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062a\u062f\u0631\u064a\u0628<\/h3>\n<p>\u062a\u062c\u0645\u0639 GPTBot \u0648ClaudeBot \u0648CCBot \u0648meta-externalagent \u0648Amazonbot \u0648MistralAI-Training \u0635\u0641\u062d\u0627\u062a \u0642\u062f \u062a\u0646\u062a\u0647\u064a \u0641\u064a \u0628\u064a\u0627\u0646\u0627\u062a \u062a\u062f\u0631\u064a\u0628 \u0627\u0644\u0646\u0645\u0627\u0630\u062c. \u062d\u0638\u0631\u0647\u0627 \u0644\u0627 \u064a\u064f\u062e\u0631\u062c\u0643 \u0645\u0646 \u0646\u062a\u0627\u0626\u062c \u0627\u0644\u0628\u062d\u062b \u0628\u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0644\u062f\u0649 \u0627\u0644\u0634\u0631\u0643\u0629 \u0646\u0641\u0633\u0647\u0627: \u062a\u0642\u0648\u0644 OpenAI \u0625\u0646 \u0643\u0644 \u0625\u0639\u062f\u0627\u062f \u0641\u064a robots.txt \u0645\u0633\u062a\u0642\u0644\u060c \u0644\u0630\u0627 \u064a\u0645\u0643\u0646\u0643 \u0645\u0646\u0639 GPTBot \u0648\u0627\u0644\u0633\u0645\u0627\u062d \u0644\u0640 OAI-SearchBot. \u0641\u064a \u0639\u064a\u0651\u0646\u062a\u0646\u0627\u060c \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u0647\u064a \u0627\u0644\u0641\u0626\u0629 \u0627\u0644\u0623\u0643\u062b\u0631 \u062d\u0638\u0631\u064b\u0627: 17.3% \u0645\u0646 \u0645\u0644\u0641\u0627\u062a robots.txt \u062a\u0645\u0646\u0639 CCBot \u0643\u0644\u064a\u064b\u0627\u060c \u064816.9% \u062a\u0645\u0646\u0639 GPTBot\u060c \u064815.9% \u062a\u0645\u0646\u0639 ClaudeBot.<\/p>\n<h3>\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0628\u062d\u062b \u0628\u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a<\/h3>\n<p>\u062a\u0641\u0647\u0631\u0633 OAI-SearchBot \u0648Claude-SearchBot \u0648PerplexityBot \u0648Amzn-SearchBot \u0648MistralAI-Index \u0648DuckAssistBot \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0643\u064a \u064a\u062c\u062f\u0647\u0627 \u0627\u0644\u0645\u0633\u0627\u0639\u062f \u0648\u064a\u0633\u062a\u0634\u0647\u062f \u0628\u0647\u0627 \u0644\u0627\u062d\u0642\u064b\u0627. \u062d\u0638\u0631\u0647\u0627 \u0642\u062f \u064a\u064f\u0628\u0639\u062f\u0643 \u0639\u0646 \u062a\u0644\u0643 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a. \u0648\u064a\u0628\u062f\u0648 \u0623\u0646 \u0623\u0635\u062d\u0627\u0628 \u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u064a\u0639\u0631\u0641\u0648\u0646 \u0630\u0644\u0643: 2.8% \u0641\u0642\u0637 \u0645\u0646 \u0627\u0644\u0645\u0644\u0641\u0627\u062a \u062a\u062d\u0638\u0631 OAI-SearchBot \u0643\u0644\u064a\u064b\u0627 \u06485.8% \u062a\u062d\u0638\u0631 PerplexityBot\u060c \u0645\u0642\u0627\u0628\u0644 16.9% \u0644\u0640 GPTBot.<\/p>\n<h3>\u0627\u0644\u062c\u0627\u0644\u0628\u0627\u062a \u0648\u0627\u0644\u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0630\u064a\u0646 \u064a\u0634\u063a\u0651\u0644\u0647\u0645 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645<\/h3>\n<p>\u064a\u0639\u0645\u0644 ChatGPT-User \u0648Claude-User \u0648Perplexity-User \u0648meta-externalfetcher \u0648Amzn-User \u0648MistralAI-User \u0648Google-Agent \u0644\u0635\u0627\u0644\u062d \u0634\u062e\u0635 \u0648\u0627\u062d\u062f \u0641\u064a \u0643\u0644 \u0645\u0631\u0629\u060c \u0643\u0645\u0627 \u064a\u0641\u0639\u0644 \u0627\u0644\u0645\u062a\u0635\u0641\u062d. \u064a\u0642\u0648\u0644 \u0639\u062f\u0629 \u0645\u0634\u063a\u0651\u0644\u064a\u0646 \u0625\u0646 robots.txt \u0642\u062f \u0644\u0627 \u064a\u0646\u0637\u0628\u0642 \u0644\u0623\u0646 \u0625\u0646\u0633\u0627\u0646\u064b\u0627 \u0637\u0644\u0628 \u0627\u0644\u0635\u0641\u062d\u0629. \u0648\u0647\u0630\u0647 \u0623\u064a\u0636\u064b\u0627 \u0627\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a \u0627\u0644\u062a\u064a \u064a\u0633\u062a\u062e\u062f\u0645\u0647\u0627 \u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u062a\u0633\u0648\u0642 \u0648\u0627\u0644\u062d\u062c\u0632\u060c \u0644\u0630\u0627 \u0642\u062f \u064a\u0624\u062f\u064a \u062d\u0638\u0631\u0647\u0627 \u0625\u0644\u0649 \u0635\u062f\u0651 \u0639\u0645\u0644\u0627\u0621 \u062d\u0642\u064a\u0642\u064a\u064a\u0646. 5.9% \u0645\u0646 \u0627\u0644\u0645\u0644\u0641\u0627\u062a \u0641\u064a \u0639\u064a\u0651\u0646\u062a\u0646\u0627 \u062a\u062d\u0638\u0631 ChatGPT-User \u0643\u0644\u064a\u064b\u0627.<\/p>\n<h3>\u0631\u0645\u0648\u0632 \u062a\u062d\u0643\u0645 \u0644\u0627 \u062a\u0632\u062d\u0641<\/h3>\n<p>\u0644\u0627 \u064a\u0632\u0648\u0631 Google-Extended \u0648\u0644\u0627 Applebot-Extended \u0645\u0648\u0642\u0639\u0643 \u0623\u0628\u062f\u064b\u0627. \u062a\u0632\u062d\u0641 Google \u0648Apple \u0628\u0627\u0633\u062a\u062e\u062f\u0627\u0645 Googlebot \u0648Applebot \u0643\u0627\u0644\u0645\u0639\u062a\u0627\u062f\u060c \u062b\u0645 \u062a\u062a\u062d\u0642\u0642 \u0645\u0646 \u0647\u0630\u0647 \u0627\u0644\u0631\u0645\u0648\u0632 \u0644\u062a\u0642\u0631\u0631 \u0645\u0627 \u0625\u0630\u0627 \u0643\u0627\u0646 \u0627\u0644\u0645\u062d\u062a\u0648\u0649 \u064a\u0645\u0643\u0646 \u0623\u0646 \u064a\u062f\u0631\u0651\u0628 \u0646\u0645\u0627\u0630\u062c\u0647\u0627. \u0648\u0641\u0642\u064b\u0627 \u0644\u0640 Google\u060c \u0645\u0646\u0639 Google-Extended \u0644\u0627 \u064a\u0624\u062b\u0631 \u0639\u0644\u0649 \u0627\u0644\u0638\u0647\u0648\u0631 \u0623\u0648 \u0627\u0644\u062a\u0631\u062a\u064a\u0628 \u0641\u064a \u0628\u062d\u062b Google. 14.1% \u0645\u0646 \u0627\u0644\u0645\u0644\u0641\u0627\u062a \u0627\u0644\u0645\u0641\u062d\u0648\u0635\u0629 \u062a\u062d\u0638\u0631 Google-Extended \u0643\u0644\u064a\u064b\u0627 \u064813.5% \u062a\u062d\u0638\u0631 Applebot-Extended. \u0648\u0642\u0627\u0644\u062a Google \u0623\u064a\u0636\u064b\u0627 \u0625\u0646\u0647\u0627 \u062a\u062e\u0637\u0637 \u0644\u062f\u0639\u0645 \u0625\u0634\u0627\u0631\u0629 \u062a\u062f\u0631\u064a\u0628 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0627\u0644\u0645\u0634\u062a\u0631\u0643\u0629 (<code>user-agent: __usage_ai-training<\/code>) \u0641\u064a \u0623\u0648\u0627\u0626\u0644 2027\u061b \u0646\u0634\u0631\u062d \u0630\u0644\u0643 \u0641\u064a <a href=\"\/blog\/ar\/robots-txt-ar\/training-opt-out-ar\/robots-txt-rafd-tadrib-al-dhakaa-al-istinai\/\">\u062f\u0644\u064a\u0644\u0646\u0627 \u0644\u0625\u0644\u063a\u0627\u0621 \u0627\u0644\u0627\u0634\u062a\u0631\u0627\u0643 \u0641\u064a \u062a\u062f\u0631\u064a\u0628 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0639\u0628\u0631 robots.txt<\/a>.<\/p>\n<h3>\u0631\u0645\u0648\u0632 \u0642\u062f\u064a\u0645\u0629 \u0644\u0627 \u062a\u0632\u0627\u0644 \u062a\u0645\u0644\u0623 \u0645\u0644\u0641\u0627\u062a robots.txt<\/h3>\n<p>\u0635\u0641\u062d\u0629 \u0627\u0644\u0645\u0633\u0627\u0639\u062f\u0629 \u0627\u0644\u062d\u0627\u0644\u064a\u0629 \u0644\u0640 Anthropic \u062a\u0630\u0643\u0631 ClaudeBot \u0648Claude-User \u0648Claude-SearchBot \u0641\u0642\u0637. \u0648\u0645\u0639 \u0630\u0644\u0643 \u0644\u0627 \u062a\u0632\u0627\u0644 15.6% \u0645\u0646 \u0627\u0644\u0645\u0644\u0641\u0627\u062a \u0627\u0644\u0645\u0641\u062d\u0648\u0635\u0629 \u062a\u0630\u0643\u0631 <code>anthropic-ai<\/code> \u064813.8% \u062a\u0630\u0643\u0631 <code>Claude-Web<\/code>. \u0627\u0644\u0642\u0648\u0627\u0639\u062f \u0627\u0644\u062e\u0627\u0635\u0629 \u0628\u0647\u0630\u0647 \u0627\u0644\u0623\u0633\u0645\u0627\u0621 \u063a\u064a\u0631 \u0636\u0627\u0631\u0629\u060c \u0644\u0643\u0646 \u0625\u0646 \u0643\u0627\u0646\u062a \u0642\u0648\u0627\u0639\u062f\u0643 \u0627\u0644\u0648\u062d\u064a\u062f\u0629 \u0644\u0640 Anthropic \u0641\u0642\u062f \u0644\u0627 \u062a\u063a\u0637\u064a \u0627\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a \u0627\u0644\u062a\u064a \u062a\u0648\u062b\u0651\u0642\u0647\u0627 Anthropic \u0627\u0644\u064a\u0648\u0645. \u062a\u0623\u0643\u062f \u0645\u0646 \u0630\u0643\u0631 <code>ClaudeBot<\/code> \u0635\u0631\u0627\u062d\u0629\u064b.<\/p>\n<h2 id=\"what-it-looks-like\">\u0643\u064a\u0641 \u064a\u0628\u062f\u0648 \u0648\u0635\u0648\u0644 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0639\u0645\u0644\u064a\u064b\u0627<\/h2>\n<p>\u0648\u062b\u0627\u0626\u0642 \u0627\u0644\u0645\u0634\u063a\u0651\u0644 \u0647\u064a \u0627\u0644\u0645\u0631\u062c\u0639 \u0627\u0644\u0645\u0648\u062b\u0648\u0642 \u0644\u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645. \u0635\u0641\u062d\u0629 OpenAI \u0645\u062b\u0644\u064b\u0627 \u062a\u0639\u0631\u0636 \u0643\u0644 \u0631\u0648\u0628\u0648\u062a \u0645\u0639 \u0645\u062b\u0627\u0644 \u0644\u0633\u0644\u0633\u0644\u0629 \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645\u060c \u0648\u0645\u0644\u0627\u062d\u0638\u0627\u062a \u062d\u0648\u0644 \u0639\u0644\u0627\u0645\u0629 robots.txt\u060c \u0648\u0631\u0627\u0628\u0637 \u0625\u0644\u0649 \u0645\u0644\u0641 \u0639\u0646\u0627\u0648\u064a\u0646 IP \u0627\u0644\u0645\u0646\u0634\u0648\u0631.<\/p>\n<figure class=\"wp-block-image size-full\"><img fetchpriority=\"high\" class=\"wp-image-576\" srcset=\"https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/openai-bots-v3-768w.webp 768w, https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/openai-bots-v3.webp 1200w\" sizes=\"(max-width: 848px) calc(100vw - 48px), 800px\" src=\"https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/openai-bots-v3.webp\" alt=\"\u0648\u062b\u0627\u0626\u0642 \u0632\u0648\u0627\u062d\u0641 OpenAI \u0627\u0644\u062e\u0627\u0635\u0629 \u0628\u0640 OAI-SearchBot\u060c \u0648\u062a\u0639\u0631\u0636 \u0645\u062b\u0627\u0644 \u0633\u0644\u0633\u0644\u0629 \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0648\u0639\u0644\u0627\u0645\u0629 robots.txt \u0648\u0645\u0644\u0641 \u0639\u0646\u0627\u0648\u064a\u0646 IP \u0627\u0644\u0645\u0646\u0634\u0648\u0631\" width=\"1200\" height=\"900\" loading=\"eager\" decoding=\"async\" \/><figcaption>\u0648\u062b\u0627\u0626\u0642 OpenAI \u0627\u0644\u062e\u0627\u0635\u0629 \u0628\u0640 OAI-SearchBot\u060c \u0627\u0644\u062a\u064f\u0642\u0637\u062a \u0641\u064a 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026.<\/figcaption><\/figure>\n<p>\u0645\u0627 \u064a\u0641\u0639\u0644\u0647 \u0627\u0644\u0645\u0648\u0642\u0639 \u0641\u0639\u0644\u064a\u064b\u0627 \u0645\u0639 \u0647\u0630\u0647 \u0627\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a \u0623\u0645\u0631 \u0645\u062e\u062a\u0644\u0641. \u0637\u0644\u0628\u0646\u0627 \u0633\u062a \u0635\u0641\u062d\u0627\u062a \u0631\u0626\u064a\u0633\u064a\u0629 \u0645\u0646 \u062e\u0627\u062f\u0645\u0646\u0627 \u0628\u0648\u0643\u064a\u0644 \u0645\u0633\u062a\u062e\u062f\u0645 Chrome\u060c \u0648\u0648\u0643\u064a\u0644 \u0645\u0633\u062a\u062e\u062f\u0645 GPTBot \u0645\u0646 OpenAI\u060c \u0648\u0648\u0643\u064a\u0644 \u0645\u0633\u062a\u062e\u062f\u0645 ClaudeBot. \u064a\u0642\u0639 \u062e\u0627\u062f\u0645\u0646\u0627 \u0639\u0644\u0649 \u0639\u0646\u0648\u0627\u0646 IP \u0644\u0645\u0631\u0643\u0632 \u0628\u064a\u0627\u0646\u0627\u062a\u060c \u062a\u062a\u0639\u0627\u0645\u0644 \u0645\u0639\u0647 \u0643\u062b\u064a\u0631 \u0645\u0646 \u0623\u0646\u0638\u0645\u0629 \u0627\u0644\u062d\u0645\u0627\u064a\u0629 \u0645\u0646 \u0627\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a \u0628\u0631\u064a\u0628\u0629\u060c \u0644\u0630\u0627 \u0642\u062f \u062a\u062e\u062a\u0644\u0641 \u0627\u0644\u0646\u062a\u0627\u0626\u062c \u0645\u0646 \u0627\u062a\u0635\u0627\u0644 \u0645\u0646\u0632\u0644\u064a.<\/p>\n<figure class=\"wp-block-image size-full\"><img class=\"wp-image-570\" srcset=\"https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/curl-tests-v2-768w.webp 768w, https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/curl-tests-v2.webp 1200w\" sizes=\"auto, (max-width: 848px) calc(100vw - 48px), 800px\" src=\"https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/curl-tests-v2.webp\" alt=\"\u0646\u062a\u0627\u0626\u062c curl \u0644\u0633\u062a\u0629 \u0645\u0648\u0627\u0642\u0639 \u0637\u064f\u0644\u0628\u062a \u0628\u0648\u0643\u0644\u0627\u0621 \u0645\u0633\u062a\u062e\u062f\u0645 Chrome \u0648GPTBot \u0648ClaudeBot \u0641\u064a 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026\" width=\"1200\" height=\"991\" loading=\"lazy\" decoding=\"async\" \/><figcaption>\u0627\u0644\u0631\u0627\u0628\u0637 \u0646\u0641\u0633\u0647\u060c \u062b\u0644\u0627\u062b\u0629 \u0648\u0643\u0644\u0627\u0621 \u0645\u0633\u062a\u062e\u062f\u0645. \u0646\u062a\u0627\u0626\u062c \u0645\u0646 IP \u0644\u0645\u0631\u0643\u0632 \u0628\u064a\u0627\u0646\u0627\u062a\u060c 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026 \u0627\u0644\u0633\u0627\u0639\u0629 08:03 \u0628\u062a\u0648\u0642\u064a\u062a AEDT.<\/figcaption><\/figure>\n<p>\u0638\u0647\u0631\u062a \u062b\u0644\u0627\u062b\u0629 \u0623\u0646\u0645\u0627\u0637. \u062e\u062f\u0645 Reddit \u0645\u062a\u0635\u0641\u062d Chrome \u0644\u0643\u0646\u0647 \u0623\u0639\u0627\u062f 403 \u0644\u0640 GPTBot \u0648429 \u0644\u0640 ClaudeBot\u060c \u0648\u0647\u0630\u0627 \u062a\u0635\u0641\u064a\u0629 \u062d\u0633\u0628 \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0639\u0644\u0649 \u0627\u0644\u062e\u0627\u062f\u0645 \u0623\u0648 CDN. \u0648\u0623\u0639\u0627\u062f\u062a \u0646\u064a\u0648\u064a\u0648\u0631\u0643 \u062a\u0627\u064a\u0645\u0632 403 \u0644\u0644\u062b\u0644\u0627\u062b\u0629\u060c \u0645\u0627 \u064a\u0634\u064a\u0631 \u0625\u0644\u0649 \u0623\u0646 \u0627\u0644\u062d\u0638\u0631 \u0642\u0627\u0626\u0645 \u0639\u0644\u0649 \u0639\u0646\u0648\u0627\u0646 IP \u0644\u062f\u064a\u0646\u0627 \u0623\u0648 \u0628\u0635\u0645\u0629 \u0627\u0644\u0637\u0644\u0628\u060c \u0644\u0627 \u0639\u0644\u0649 \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645. \u0648\u0623\u0639\u0627\u062f bbc.com \u0627\u0644\u0631\u0645\u0632 200 \u0644\u0640 GPTBot \u0631\u063a\u0645 \u0623\u0646 \u0645\u0644\u0641 robots.txt \u0644\u062f\u064a\u0647 \u064a\u0645\u0646\u0639 GPTBot \u0648ClaudeBot \u0648CCBot \u0648PerplexityBot \u0648\u062e\u0645\u0633\u0629 \u0631\u0645\u0648\u0632 \u0630\u0643\u0627\u0621 \u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0623\u062e\u0631\u0649 \u0639\u0644\u0649 \u0627\u0644\u0623\u0642\u0644. \u0647\u0630\u0627 \u0623\u0648\u0636\u062d \u062f\u0644\u064a\u0644 \u0639\u0644\u0649 \u0623\u0646 robots.txt \u064a\u0637\u0644\u0628 \u0648\u0644\u0627 \u064a\u0641\u0631\u0636. \u0648\u0623\u062c\u0627\u0628\u062a Amazon \u0639\u0644\u0649 \u0627\u0644\u062b\u0644\u0627\u062b\u0629 \u0628\u0631\u0645\u0632 202 \u0648\u062c\u0633\u0645 \u0641\u0627\u0631\u063a\u060c \u0623\u064a \u062a\u062d\u062f\u064d\u0651 \u0648\u0633\u064a\u0637 \u0628\u062f\u0644\u064b\u0627 \u0645\u0646 \u0635\u0641\u062d\u0629.<\/p>\n<p>\u064a\u0641\u0635\u0644 \u0633\u062c\u0644 Online Web Directory \u0644\u0643\u0644 \u0645\u0648\u0642\u0639 \u0628\u064a\u0646 \u0647\u0627\u062a\u064a\u0646 \u0627\u0644\u0637\u0628\u0642\u062a\u064a\u0646: \u0647\u0644 \u064a\u0633\u062a\u0637\u064a\u0639 \u0627\u0644\u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0648\u0635\u0648\u0644 \u0625\u0644\u0649 \u0627\u0644\u0645\u0648\u0642\u0639\u060c \u0648\u0645\u0627\u0630\u0627 \u064a\u0642\u0648\u0644 \u0645\u0644\u0641 robots.txt.<\/p>\n<figure class=\"wp-block-image size-full\"><img class=\"wp-image-572\" srcset=\"https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/owd-site-bbc-v2-768w.webp 768w, https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/owd-site-bbc-v2.webp 1200w\" sizes=\"auto, (max-width: 848px) calc(100vw - 48px), 800px\" src=\"https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/owd-site-bbc-v2.webp\" alt=\"\u0633\u062c\u0644 Online Web Directory \u0644\u0645\u0648\u0642\u0639 bbc.com: \u0645\u0641\u062a\u0648\u062d \u0644\u0644\u0648\u0643\u0644\u0627\u0621\u060c \u0628\u0644\u0627 \u062c\u062f\u0627\u0631 \u0631\u0648\u0628\u0648\u062a\u0627\u062a\u060c \u0648robots.txt \u064a\u0642\u064a\u0651\u062f \u0628\u0639\u0636 \u0627\u0644\u0645\u0633\u0627\u0631\u0627\u062a\" width=\"1200\" height=\"1097\" loading=\"lazy\" decoding=\"async\" \/><figcaption>\u0633\u062c\u0644 OWD \u0644\u0645\u0648\u0642\u0639 bbc.com: \u0645\u0641\u062a\u0648\u062d \u0644\u0644\u0648\u0643\u0644\u0627\u0621\u060c \u0645\u0639 \u062a\u0642\u064a\u064a\u062f robots.txt \u0644\u0628\u0639\u0636 \u0627\u0644\u0645\u0633\u0627\u0631\u0627\u062a.<\/figcaption><\/figure>\n<h2 id=\"how-to-check\">\u0643\u064a\u0641 \u062a\u062a\u062d\u0642\u0642 \u0645\u0646 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0627\u0644\u062a\u064a \u064a\u0645\u0643\u0646\u0647\u0627 \u0627\u0644\u0648\u0635\u0648\u0644 \u0625\u0644\u0649 \u0645\u0648\u0642\u0639\u0643<\/h2>\n<h3>1. \u0627\u0642\u0631\u0623 \u0645\u062c\u0645\u0648\u0639\u0627\u062a robots.txt \u0627\u0644\u062e\u0627\u0635\u0629 \u0628\u0631\u0645\u0648\u0632 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a<\/h3>\n<p>\u0627\u0628\u062f\u0623 \u0628\u0645\u0627 \u062a\u0637\u0644\u0628\u0647 \u0645\u0646 \u0627\u0644\u0632\u0648\u0627\u062d\u0641. \u064a\u0637\u0628\u0639 \u0647\u0630\u0627 \u0627\u0644\u0623\u0645\u0631 \u0643\u0644 \u0645\u062c\u0645\u0648\u0639\u0629 \u062a\u0630\u0643\u0631 \u0631\u0645\u0632 \u0630\u0643\u0627\u0621 \u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0634\u0627\u0626\u0639\u064b\u0627\u060c \u0645\u0639 \u0627\u0644\u0633\u0637\u0631 \u0627\u0644\u0630\u064a \u064a\u0644\u064a\u0647\u0627:<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code>curl -s https:\/\/www.bbc.com\/robots.txt | grep -i -A1 -E '^user-agent: *(gptbot|oai-searchbot|chatgpt-user|claudebot|google-extended|perplexitybot|ccbot|bytespider|applebot-extended)'<\/code><\/pre>\n<p>\u0627\u0644\u0645\u062e\u0631\u062c\u0627\u062a \u0627\u0644\u0641\u0639\u0644\u064a\u0629 \u0644\u0645\u0648\u0642\u0639 bbc.com \u0641\u064a 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026 \u0627\u0644\u0633\u0627\u0639\u0629 08:09 \u0628\u062a\u0648\u0642\u064a\u062a AEDT (\u0645\u062e\u062a\u0635\u0631\u0629 \u0625\u0644\u0649 \u0623\u0631\u0628\u0639 \u0645\u0646 \u0627\u0644\u0645\u062c\u0645\u0648\u0639\u0627\u062a \u0627\u0644\u062a\u0633\u0639):<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code>User-agent: CCBot\nDisallow: \/\n--\nUser-agent: ClaudeBot\nDisallow: \/\n--\nUser-agent: GPTBot\nDisallow: \/\n--\nUser-agent: OAI-SearchBot\nDisallow: \/<\/code><\/pre>\n<p>\u062a\u0630\u0643\u0651\u0631 \u0623\u0646 \u0627\u0644\u0632\u0627\u062d\u0641 \u064a\u0644\u062a\u0632\u0645 \u0641\u0642\u0637 \u0628\u0627\u0644\u0645\u062c\u0645\u0648\u0639\u0629 \u0627\u0644\u0623\u0643\u062b\u0631 \u062a\u062d\u062f\u064a\u062f\u064b\u0627 \u0627\u0644\u062a\u064a \u062a\u0637\u0627\u0628\u0642 \u0631\u0645\u0632\u0647. \u0625\u0630\u0627 \u0630\u0643\u0631\u062a <code>GPTBot<\/code> \u0641\u064a \u0645\u062c\u0645\u0648\u0639\u0629 \u062e\u0627\u0635\u0629 \u0628\u0647\u060c \u0641\u0633\u064a\u062a\u062c\u0627\u0647\u0644 \u0642\u0648\u0627\u0639\u062f <code>User-agent: *<\/code> \u062a\u0645\u0627\u0645\u064b\u0627\u060c \u0644\u0630\u0627 \u0643\u0631\u0651\u0631 \u0623\u064a \u0642\u0648\u0627\u0639\u062f \u0645\u0634\u062a\u0631\u0643\u0629 \u062f\u0627\u062e\u0644 \u0645\u062c\u0645\u0648\u0639\u0629 GPTBot.<\/p>\n<h3>2. \u0627\u062e\u062a\u0628\u0631 \u0643\u064a\u0641 \u064a\u0633\u062a\u062c\u064a\u0628 \u062e\u0627\u062f\u0645\u0643 \u0644\u0648\u0643\u0644\u0627\u0621 \u0645\u0633\u062a\u062e\u062f\u0645 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a<\/h3>\n<p>\u0642\u062f \u064a\u0642\u0648\u0644 robots.txt &#8220;\u0644\u0627&#8221; \u0628\u064a\u0646\u0645\u0627 \u064a\u0642\u0648\u0644 \u062e\u0627\u062f\u0645\u0643 &#8220;\u0646\u0639\u0645&#8221;\u060c \u0623\u0648 \u0627\u0644\u0639\u0643\u0633. \u0627\u0637\u0644\u0628 \u0627\u0644\u0635\u0641\u062d\u0629 \u0646\u0641\u0633\u0647\u0627 \u0628\u0648\u0643\u0644\u0627\u0621 \u0645\u0633\u062a\u062e\u062f\u0645 \u0645\u062e\u062a\u0644\u0641\u064a\u0646 \u0648\u0642\u0627\u0631\u0646 \u0631\u0645\u0648\u0632 \u0627\u0644\u062d\u0627\u0644\u0629:<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code>GPT='Mozilla\/5.0 AppleWebKit\/537.36 (KHTML, like Gecko); compatible; GPTBot\/1.4; +https:\/\/openai.com\/gptbot'\nCHROME='Mozilla\/5.0 (X11; Linux x86_64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/130.0.0.0 Safari\/537.36'\nfor ua in \"$CHROME\" \"$GPT\"; do\n  curl -s -o \/dev\/null -A \"$ua\" -w \"%{http_code} %{size_download}B\\n\" https:\/\/www.example-shop.com\/\ndone<\/code><\/pre>\n<p>\u0627\u0633\u062a\u0628\u062f\u0644 \u0627\u0644\u0631\u0627\u0628\u0637 \u0628\u0631\u0627\u0628\u0637 \u0645\u0648\u0642\u0639\u0643. \u0625\u0630\u0627 \u0638\u0647\u0631 403 \u0623\u0648 429 \u0644\u0640 GPTBot \u0648\u062d\u062f\u0647 \u0641\u0647\u0630\u0627 \u064a\u0639\u0646\u064a \u062a\u0635\u0641\u064a\u0629 \u062d\u0633\u0628 \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645. \u0648\u0625\u0630\u0627 \u0638\u0647\u0631 403 \u0644\u0644\u0627\u062b\u0646\u064a\u0646 \u0641\u0627\u0644\u0633\u0628\u0628 \u0639\u0627\u062f\u0629 \u0633\u0645\u0639\u0629 \u0639\u0646\u0648\u0627\u0646 IP \u0623\u0648 \u062a\u062d\u062f\u064d\u0651 \u0644\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a. \u0634\u063a\u0651\u0644\u0647 \u0645\u0646 \u0627\u062a\u0635\u0627\u0644 \u0645\u0646\u0632\u0644\u064a \u0648\u0645\u0646 \u062e\u0627\u062f\u0645 \u0623\u064a\u0636\u064b\u0627: \u0627\u0644\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062d\u0642\u064a\u0642\u064a\u0629 \u062a\u0623\u062a\u064a \u0645\u0646 \u0639\u0646\u0627\u0648\u064a\u0646 IP \u0644\u0645\u0631\u0627\u0643\u0632 \u0627\u0644\u0628\u064a\u0627\u0646\u0627\u062a\u060c \u0644\u0630\u0627 \u0646\u062a\u064a\u062c\u0629 \u0627\u0644\u062e\u0627\u062f\u0645 \u0623\u0642\u0631\u0628 \u0625\u0644\u0649 \u0645\u0627 \u062a\u0631\u0627\u0647.<\/p>\n<h3>3. \u0627\u0639\u062b\u0631 \u0639\u0644\u0649 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0641\u064a \u0633\u062c\u0644\u0627\u062a \u0627\u0644\u0648\u0635\u0648\u0644<\/h3>\n<p>\u0627\u062d\u0633\u0628 \u0627\u0644\u0632\u064a\u0627\u0631\u0627\u062a \u0644\u0643\u0644 \u0631\u0648\u0628\u0648\u062a\u060c \u062b\u0645 \u062a\u062d\u0642\u0642 \u0645\u0646 \u0631\u0645\u0648\u0632 \u0627\u0644\u062d\u0627\u0644\u0629 \u0627\u0644\u062a\u064a \u062a\u0644\u0642\u0627\u0647\u0627 \u0643\u0644 \u0631\u0648\u0628\u0648\u062a. \u0634\u063a\u0651\u0644\u0646\u0627 \u0647\u0630\u0647 \u0627\u0644\u0623\u0648\u0627\u0645\u0631 \u0639\u0644\u0649 \u0633\u062c\u0644 \u0627\u0644\u0648\u0635\u0648\u0644 \u0644\u062e\u0627\u062f\u0645 Nginx \u062a\u062c\u0631\u064a\u0628\u064a \u0641\u064a 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026:<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code>$ grep -o -E 'GPTBot|OAI-SearchBot|ChatGPT-User|ClaudeBot|Claude-User|Claude-SearchBot|PerplexityBot|Perplexity-User|CCBot|Bytespider|Amazonbot|meta-external[a-z]+' access.log | sort | uniq -c | sort -rn\n      2 OAI-SearchBot\n      2 GPTBot\n$ awk -F'\"' '\/GPTBot\/ {split($3,a,\" \"); print a[1]}' access.log | sort | uniq -c\n      1 200\n      1 403<\/code><\/pre>\n<p>\u064a\u0639\u0645\u0644 \u0627\u0644\u0623\u0645\u0631 \u0627\u0644\u062b\u0627\u0646\u064a \u0645\u0639 \u062a\u0646\u0633\u064a\u0642 \u0627\u0644\u0633\u062c\u0644 \u0627\u0644\u0642\u064a\u0627\u0633\u064a &#8220;combined&#8221;\u060c \u062d\u064a\u062b \u064a\u0623\u062a\u064a \u0631\u0645\u0632 \u0627\u0644\u062d\u0627\u0644\u0629 \u0628\u0639\u062f \u0633\u0637\u0631 \u0627\u0644\u0637\u0644\u0628 \u0627\u0644\u0645\u0642\u062a\u0628\u0633. \u0631\u0645\u0632 200 \u0644\u0640 GPTBot \u0639\u0644\u0649 <code>\/robots.txt<\/code> \u0645\u0639 403 \u0644\u0643\u0644 \u0645\u0627 \u0639\u062f\u0627\u0647 \u0647\u0648 \u0628\u0627\u0644\u0636\u0628\u0637 \u0645\u0627 \u062a\u0631\u064a\u062f\u0647 \u0639\u0646\u062f \u062d\u0638\u0631 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u0639\u0644\u0649 \u0627\u0644\u062e\u0627\u062f\u0645.<\/p>\n<h3>4. \u062a\u062d\u0642\u0642 \u0645\u0646 \u0623\u0646 \u0632\u0627\u0626\u0631 &#8220;GPTBot&#8221; \u0647\u0648 OpenAI \u0641\u0639\u0644\u064b\u0627<\/h3>\n<p>\u064a\u0633\u062a\u0637\u064a\u0639 \u0623\u064a \u0634\u062e\u0635 \u0625\u0631\u0633\u0627\u0644 \u0648\u0643\u064a\u0644 \u0645\u0633\u062a\u062e\u062f\u0645 GPTBot. \u062a\u0646\u0634\u0631 OpenAI \u0648Perplexity \u0648DuckDuckGo \u0648Mistral \u0648Google \u0642\u0648\u0627\u0626\u0645 \u0639\u0646\u0627\u0648\u064a\u0646 IP\u060c \u0644\u0630\u0627 \u062a\u062d\u0642\u0642 \u0645\u0646 \u0639\u0646\u0648\u0627\u0646 IP \u0627\u0644\u0645\u0635\u062f\u0631. \u064a\u062e\u062a\u0628\u0631 \u0647\u0630\u0627 \u0627\u0644\u0633\u0643\u0631\u0628\u062a \u0639\u0646\u0648\u0627\u0646 IP \u0645\u0642\u0627\u0628\u0644 \u0645\u0644\u0641\u0627\u062a OpenAI \u0627\u0644\u062b\u0644\u0627\u062b\u0629:<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code># verify_ip.py - usage: python3 verify_ip.py 132.196.86.1\nimport sys, json, ipaddress, urllib.request\n\nip = ipaddress.ip_address(sys.argv[1])\nfor name in [\"gptbot\", \"searchbot\", \"chatgpt-user\"]:\n    data = json.load(urllib.request.urlopen(f\"https:\/\/openai.com\/{name}.json\", timeout=20))\n    nets = [ipaddress.ip_network(p.get(\"ipv4Prefix\") or p.get(\"ipv6Prefix\")) for p in data[\"prefixes\"]]\n    hit = [str(n) for n in nets if ip.version == n.version and ip in n]\n    print(f\"{name:13} {len(nets):3} ranges  {'MATCH ' + hit[0] if hit else 'no match'}\")<\/code><\/pre>\n<p>\u0627\u0644\u0645\u062e\u0631\u062c\u0627\u062a \u0641\u064a 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026 \u0627\u0644\u0633\u0627\u0639\u0629 08:12 \u0628\u062a\u0648\u0642\u064a\u062a AEDT:<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code>$ python3 verify_ip.py 132.196.86.1\ngptbot         18 ranges  MATCH 132.196.86.0\/24\nsearchbot      39 ranges  no match\nchatgpt-user  237 ranges  no match\n$ python3 verify_ip.py 1.1.1.1\ngptbot         18 ranges  no match\nsearchbot      39 ranges  no match\nchatgpt-user  237 ranges  no match<\/code><\/pre>\n<p>\u0648\u0642\u062a \u0627\u062e\u062a\u0628\u0627\u0631\u0646\u0627\u060c \u0627\u062d\u062a\u0648\u0649 \u0645\u0644\u0641 <code>gptbot.json<\/code> \u0645\u0646 OpenAI \u0639\u0644\u0649 18 \u0646\u0637\u0627\u0642 IPv4 \u0628\u062a\u0627\u0631\u064a\u062e \u0625\u0646\u0634\u0627\u0621 22 \u0633\u0628\u062a\u0645\u0628\u0631 2026. \u0627\u062c\u0644\u0628 \u0627\u0644\u0645\u0644\u0641\u0627\u062a \u062d\u062f\u064a\u062b\u0629 \u0628\u062f\u0644\u064b\u0627 \u0645\u0646 \u062a\u062b\u0628\u064a\u062a\u0647\u0627 \u0641\u064a \u0627\u0644\u0643\u0648\u062f.<\/p>\n<h3>5. \u0627\u0628\u062d\u062b \u0639\u0646 \u0645\u0648\u0642\u0639\u0643 \u0641\u064a \u0627\u0644\u0633\u062c\u0644<\/h3>\n<p>\u064a\u062a\u062d\u0642\u0642 Online Web Directory \u0645\u0646 \u0643\u064a\u0641\u064a\u0629 \u0627\u0633\u062a\u062c\u0627\u0628\u0629 \u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u0644\u0639\u0645\u0644\u0627\u0621 HTTP \u0627\u0644\u0628\u0633\u064a\u0637\u064a\u0646 \u0648\u0645\u0645\u0627 \u064a\u0642\u0648\u0644\u0647 \u0645\u0644\u0641 robots.txt. \u0627\u0628\u062d\u062b \u0639\u0646 \u0646\u0637\u0627\u0642\u0643\u060c \u0645\u062b\u0644 <a href=\"https:\/\/onlinewebdirectory.net\/sites\/reddit.com\">reddit.com<\/a> \u0623\u0648 <a href=\"https:\/\/onlinewebdirectory.net\/sites\/nytimes.com\">nytimes.com<\/a>\u060c \u0623\u0648 \u0634\u063a\u0651\u0644 <a href=\"https:\/\/onlinewebdirectory.net\/ar\/check\">\u0641\u062d\u0635 \u0627\u0644\u062c\u0627\u0647\u0632\u064a\u0629<\/a> \u0627\u0644\u0645\u062c\u0627\u0646\u064a. \u0648\u062a\u0639\u0631\u0636 \u0642\u0627\u0626\u0645\u0629 <a href=\"https:\/\/onlinewebdirectory.net\/lists\/blocks-ai-agents\">\u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u0627\u0644\u062a\u064a \u062a\u062d\u0638\u0631 \u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a<\/a> \u0627\u0644\u0646\u0637\u0627\u0642\u0627\u062a \u0627\u0644\u062a\u064a \u062a\u0631\u0641\u0636 \u0627\u0644\u0639\u0645\u0644\u0627\u0621 \u0627\u0644\u0628\u0633\u064a\u0637\u064a\u0646.<\/p>\n<figure class=\"wp-block-image size-full\"><img class=\"wp-image-574\" srcset=\"https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/owd-blocks-list-v2-768w.webp 768w, https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/owd-blocks-list-v2.webp 1200w\" sizes=\"auto, (max-width: 848px) calc(100vw - 48px), 800px\" src=\"https:\/\/onlinewebdirectory.net\/blog\/wp-content\/uploads\/2026\/11\/owd-blocks-list-v2.webp\" alt=\"\u0642\u0627\u0626\u0645\u0629 Online Web Directory \u0644\u0644\u0645\u0648\u0627\u0642\u0639 \u0627\u0644\u062a\u064a \u062a\u062d\u0638\u0631 \u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a: 725 \u0645\u0646 55,144 \u0645\u0648\u0642\u0639\u064b\u0627 \u0645\u0641\u062d\u0648\u0635\u064b\u0627 \u062a\u062d\u0638\u0631 \u0639\u0645\u0644\u0627\u0621 HTTP \u0627\u0644\u0628\u0633\u064a\u0637\u064a\u0646\" width=\"1200\" height=\"960\" loading=\"lazy\" decoding=\"async\" \/><figcaption>\u0642\u0627\u0626\u0645\u0629 Online Web Directory \u0644\u0644\u0645\u0648\u0627\u0642\u0639 \u0627\u0644\u062a\u064a \u062a\u062d\u0638\u0631 \u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a\u060c 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026.<\/figcaption><\/figure>\n<h2 id=\"how-to-control\">\u0643\u064a\u0641 \u062a\u062a\u062d\u0643\u0645 \u0641\u064a \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a: \u0633\u062a \u0637\u0631\u0642<\/h2>\n<p>\u0627\u062e\u062a\u0631 \u062d\u0633\u0628 \u0647\u062f\u0641\u0643. \u0645\u0639\u0638\u0645 \u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u062a\u0631\u064a\u062f \u0627\u0644\u062d\u0641\u0627\u0638 \u0639\u0644\u0649 \u0632\u064a\u0627\u0631\u0627\u062a \u0627\u0644\u0628\u062d\u062b \u0628\u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a\u060c \u0648\u062a\u0642\u0631\u0631 \u0628\u0634\u0623\u0646 \u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u062d\u0627\u0644\u0629\u064b \u0628\u062d\u0627\u0644\u0629\u060c \u0648\u062a\u062a\u0631\u0643 \u0627\u0644\u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0630\u064a\u0646 \u064a\u0634\u063a\u0651\u0644\u0647\u0645 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u062f\u0648\u0646 \u062a\u062f\u062e\u0644.<\/p>\n<h3>\u0627\u0644\u0637\u0631\u064a\u0642\u0629 1: \u0627\u062d\u0638\u0631 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u0648\u0623\u0628\u0642\u0650 \u0627\u0644\u0628\u062d\u062b \u0628\u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a (robots.txt)<\/h3>\n<p>\u0647\u0630\u0647 \u0627\u0644\u0633\u064a\u0627\u0633\u0629 \u0627\u0644\u0623\u0643\u062b\u0631 \u0634\u064a\u0648\u0639\u064b\u0627 \u0628\u064a\u0646 \u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u0627\u0644\u062a\u064a \u0641\u062d\u0635\u0646\u0627\u0647\u0627. \u0623\u0636\u0641 \u0647\u0630\u0627 \u0625\u0644\u0649 <code>\/robots.txt<\/code> \u0641\u064a \u062c\u0630\u0631 \u0643\u0644 \u0646\u0637\u0627\u0642 \u062a\u062e\u062f\u0645\u0647:<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code># Block AI training crawlers\nUser-agent: GPTBot\nUser-agent: ClaudeBot\nUser-agent: CCBot\nUser-agent: Google-Extended\nUser-agent: Applebot-Extended\nUser-agent: meta-externalagent\nUser-agent: Amazonbot\nUser-agent: MistralAI-Training\nUser-agent: Bytespider\nDisallow: \/\n\n# Allow AI search crawlers (so you can be cited in AI answers)\nUser-agent: OAI-SearchBot\nUser-agent: Claude-SearchBot\nUser-agent: PerplexityBot\nUser-agent: Amzn-SearchBot\nUser-agent: DuckAssistBot\nUser-agent: MistralAI-Index\nAllow: \/\n\n# Everyone else\nUser-agent: *\nDisallow: \/wp-admin\/<\/code><\/pre>\n<p>\u0627\u062e\u062a\u0628\u0631\u0646\u0627 \u0647\u0630\u0627 \u0627\u0644\u0645\u0644\u0641 \u0628\u0627\u0633\u062a\u062e\u062f\u0627\u0645 Protego\u060c \u0648\u0647\u0648 \u0645\u062d\u0644\u0651\u0644 \u0645\u062a\u0648\u0627\u0641\u0642 \u0645\u0639 RFC 9309. \u0627\u0644\u0646\u062a\u0627\u0626\u062c \u0644\u0644\u0645\u0633\u0627\u0631 <code>\/blog\/<\/code>:<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code>GPTBot             \/blog\/  blocked\nClaudeBot          \/blog\/  blocked\nCCBot              \/blog\/  blocked\nGoogle-Extended    \/blog\/  blocked\nApplebot-Extended  \/blog\/  blocked\nOAI-SearchBot      \/blog\/  allowed\nClaude-SearchBot   \/blog\/  allowed\nPerplexityBot      \/blog\/  allowed\nGooglebot          \/blog\/  allowed\nBingbot            \/blog\/  allowed<\/code><\/pre>\n<p>\u062a\u0643\u062f\u064a\u0633 \u0639\u062f\u0629 \u0623\u0633\u0637\u0631 <code>User-agent<\/code> \u0641\u0648\u0642 \u0645\u062c\u0645\u0648\u0639\u0629 \u0642\u0648\u0627\u0639\u062f \u0648\u0627\u062d\u062f\u0629 \u0635\u0627\u0644\u062d \u0648\u0641\u0642 RFC 9309. \u0644\u0627\u062d\u0638 \u0623\u0646 \u0645\u062c\u0645\u0648\u0639\u0629 \u0627\u0644\u0628\u062d\u062b \u0628\u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0644\u0627 \u062a\u0631\u062b \u0642\u0627\u0639\u062f\u0629 <code>\/wp-admin\/<\/code> \u0645\u0646 \u0645\u062c\u0645\u0648\u0639\u0629 <code>*<\/code>\u061b \u0623\u0636\u0641\u0647\u0627 \u0647\u0646\u0627\u0643 \u0623\u064a\u0636\u064b\u0627 \u0625\u0646 \u0627\u062d\u062a\u062c\u062a \u0625\u0644\u064a\u0647\u0627.<\/p>\n<h3>\u0627\u0644\u0637\u0631\u064a\u0642\u0629 2: \u0627\u062d\u0638\u0631 \u0643\u0644 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0627\u0644\u0645\u0648\u062b\u0651\u0642\u0629 (robots.txt)<\/h3>\n<p>\u0625\u0630\u0627 \u0643\u0646\u062a \u0644\u0627 \u062a\u0631\u064a\u062f \u0623\u064a \u0627\u0633\u062a\u062e\u062f\u0627\u0645 \u0644\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a\u060c \u0641\u0623\u0636\u0641 \u0631\u0645\u0648\u0632 \u0627\u0644\u0628\u062d\u062b \u0648\u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0625\u0644\u0649 \u0645\u062c\u0645\u0648\u0639\u0629 \u0627\u0644\u0645\u0646\u0639: <code>OAI-SearchBot<\/code> \u0648<code>ChatGPT-User<\/code> \u0648<code>Claude-SearchBot<\/code> \u0648<code>Claude-User<\/code> \u0648<code>PerplexityBot<\/code> \u0648<code>Perplexity-User<\/code> \u0648<code>meta-externalfetcher<\/code> \u0648<code>Amzn-SearchBot<\/code> \u0648<code>Amzn-User<\/code> \u0648<code>MistralAI-Index<\/code> \u0648<code>MistralAI-User<\/code> \u0648<code>DuckAssistBot<\/code>. \u062a\u0648\u0642\u0651\u0639 \u0623\u0646 \u062a\u062e\u062a\u0641\u064a \u0645\u0646 \u0625\u062c\u0627\u0628\u0627\u062a \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a. \u0648\u062a\u0630\u0643\u0651\u0631 \u0623\u0646 \u0639\u062f\u0629 \u062c\u0627\u0644\u0628\u0627\u062a \u064a\u0634\u063a\u0651\u0644\u0647\u0627 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u062a\u0642\u0648\u0644 \u0625\u0646 robots.txt \u0642\u062f \u0644\u0627 \u064a\u0646\u0637\u0628\u0642 \u0639\u0644\u064a\u0647\u0627\u060c \u0644\u0630\u0627 \u0641\u0647\u0630\u0627 \u062a\u0639\u0628\u064a\u0631 \u0639\u0646 \u062a\u0641\u0636\u064a\u0644 \u0648\u0644\u064a\u0633 \u062c\u062f\u0627\u0631\u064b\u0627.<\/p>\n<h3>\u0627\u0644\u0637\u0631\u064a\u0642\u0629 3: \u0641\u0631\u0636 \u0627\u0644\u062d\u0638\u0631 \u0641\u064a Nginx<\/h3>\n<p>\u0644\u0631\u0641\u0636 \u0627\u0644\u0637\u0644\u0628\u0627\u062a \u0641\u0639\u0644\u064a\u064b\u0627\u060c \u0642\u0645 \u0628\u0627\u0644\u062a\u0635\u0641\u064a\u0629 \u0639\u0644\u0649 \u0627\u0644\u062e\u0627\u062f\u0645. \u064a\u064f\u0631\u062c\u0639 \u0647\u0630\u0627 \u0627\u0644\u0625\u0639\u062f\u0627\u062f 403 \u0644\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u0639\u0644\u0649 \u0643\u0644 \u0627\u0644\u0645\u0633\u0627\u0631\u0627\u062a \u0639\u062f\u0627 robots.txt\u060c \u062d\u062a\u0649 \u062a\u0638\u0644 \u0642\u0627\u062f\u0631\u0629 \u0639\u0644\u0649 \u0642\u0631\u0627\u0621\u0629 \u0642\u0648\u0627\u0639\u062f\u0643. \u062a\u0648\u0636\u0639 \u0643\u062a\u0644\u0629 <code>map<\/code> \u0641\u064a \u0633\u064a\u0627\u0642 <code>http<\/code>:<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code># Block AI training crawlers by user agent, but still let them read robots.txt\nmap $http_user_agent $block_ai_crawler {\n    default 0;\n    \"~*(GPTBot|ClaudeBot|CCBot|Bytespider|meta-externalagent|Amazonbot|MistralAI-Training)\" 1;\n}\nserver {\n    listen 80;\n    root \/var\/www\/html;\n    location = \/robots.txt { }\n    location \/ {\n        if ($block_ai_crawler) { return 403; }\n    }\n}<\/code><\/pre>\n<p>\u0627\u062e\u062a\u0628\u0631\u0646\u0627\u0647 \u0639\u0644\u0649 Nginx 1.26.3 \u0641\u064a 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026: \u0646\u062c\u062d <code>nginx -t<\/code>\u060c \u0648\u062d\u0635\u0644 GPTBot \u0639\u0644\u0649 403 \u0641\u064a <code>\/<\/code> \u0648200 \u0641\u064a <code>\/robots.txt<\/code>\u060c \u0628\u064a\u0646\u0645\u0627 \u062d\u0635\u0644 Chrome \u0648OAI-SearchBot \u0639\u0644\u0649 200 \u0641\u064a \u0643\u0644\u064a\u0647\u0645\u0627. \u0627\u0633\u062a\u062e\u062f\u0627\u0645 <code>return<\/code> \u0648\u0627\u062d\u062f \u062f\u0627\u062e\u0644 <code>if<\/code> \u0647\u0648 \u0623\u062d\u062f \u0627\u0644\u0627\u0633\u062a\u062e\u062f\u0627\u0645\u0627\u062a \u0627\u0644\u062a\u064a \u062a\u0648\u062b\u0651\u0642\u0647\u0627 Nginx \u0639\u0644\u0649 \u0623\u0646\u0647\u0627 \u0622\u0645\u0646\u0629.<\/p>\n<h3>\u0627\u0644\u0637\u0631\u064a\u0642\u0629 4: \u0641\u0631\u0636 \u0627\u0644\u062d\u0638\u0631 \u0641\u064a Apache (.htaccess)<\/h3>\n<p>\u0627\u0644\u0642\u0627\u0639\u062f\u0629 \u0646\u0641\u0633\u0647\u0627 \u0644\u0640 Apache \u0628\u0627\u0633\u062a\u062e\u062f\u0627\u0645 <code>mod_rewrite<\/code>:<\/p>\n<pre dir=\"ltr\" class=\"wp-block-code\"><code># Block AI training crawlers by user agent, but still let them read robots.txt\n&lt;IfModule mod_rewrite.c&gt;\nRewriteEngine On\nRewriteCond %{REQUEST_URI} !^\/robots\\.txt$\nRewriteCond %{HTTP_USER_AGENT} (GPTBot|ClaudeBot|CCBot|Bytespider|meta-externalagent|Amazonbot|MistralAI-Training) [NC]\nRewriteRule ^ - [F,L]\n&lt;\/IfModule&gt;<\/code><\/pre>\n<p>\u062a\u0645 \u0627\u0644\u0627\u062e\u062a\u0628\u0627\u0631 \u0639\u0644\u0649 Apache 2.4.71 \u0645\u0639 <code>AllowOverride All<\/code>: \u062d\u0635\u0644 GPTBot \u0639\u0644\u0649 403 \u0641\u064a \u0627\u0644\u0635\u0641\u062d\u0629 \u0627\u0644\u0631\u0626\u064a\u0633\u064a\u0629 \u0648200 \u0641\u064a robots.txt\u061b \u0648\u062d\u0635\u0644 Chrome \u0648OAI-SearchBot \u0639\u0644\u0649 200. \u0627\u0644\u0639\u0644\u0627\u0645\u0629 <code>[F]<\/code> \u062a\u064f\u0631\u0633\u0644 403 Forbidden.<\/p>\n<h3>\u0627\u0644\u0637\u0631\u064a\u0642\u0629 5: \u0627\u0633\u062a\u062e\u062f\u0645 \u0633\u064a\u0627\u0633\u0627\u062a \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0641\u064a \u0634\u0628\u0643\u0629 CDN<\/h3>\n<p>\u0625\u0630\u0627 \u0643\u0646\u062a \u062a\u0633\u062a\u062e\u062f\u0645 Cloudflare \u0641\u0644\u0627 \u062a\u062d\u062a\u0627\u062c \u0625\u0644\u0649 \u0635\u064a\u0627\u0646\u0629 \u0642\u0627\u0626\u0645\u0629. \u0645\u0646 \u0625\u0639\u062f\u0627\u062f\u0627\u062a \u0627\u0644\u0623\u0645\u0627\u0646\u060c \u0635\u0641\u0650\u0651 \u062d\u0633\u0628 &#8220;Bot traffic&#8221; \u0648\u0627\u0641\u062a\u062d <strong>AI Crawler and Agent access policies<\/strong>. \u064a\u062a\u064a\u062d \u0644\u0643 Cloudflare \u062a\u0639\u064a\u064a\u0646 \u0625\u062c\u0631\u0627\u0621\u0627\u062a \u0645\u0646\u0641\u0635\u0644\u0629 \u0644\u062d\u0631\u0643\u0629 Search \u0648Agent \u0648Training. \u062a\u0642\u0648\u0644 \u0648\u062b\u0627\u0626\u0642\u0647 \u0625\u0646 \u0627\u0644\u0646\u0637\u0627\u0642\u0627\u062a \u0627\u0644\u062c\u062f\u064a\u062f\u0629 \u062a\u0643\u0648\u0646 \u0627\u0641\u062a\u0631\u0627\u0636\u064a\u064b\u0627 \u0639\u0644\u0649 &#8220;Allow (do not block)&#8221; \u0644\u0644\u062b\u0644\u0627\u062b\u0629\u060c \u0648\u0625\u0646 &#8220;Disallow&#8221; \u0644\u0644\u062a\u062f\u0631\u064a\u0628 \u064a\u0639\u062f\u0651\u0644 robots.txt \u0641\u0642\u0637 \u0645\u0627 \u0644\u0645 \u064a\u0643\u0646 Bot Preference Sync \u0645\u0641\u0639\u0651\u0644\u064b\u0627. \u0643\u0645\u0627 \u062a\u0639\u0631\u0636 \u0644\u0648\u062d\u0629 AI Crawl Control \u0641\u064a Cloudflare \u0627\u0644\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062a\u064a \u062a\u0632\u0648\u0631 \u0645\u0648\u0642\u0639\u0643 \u0648\u0645\u0627 \u0625\u0630\u0627 \u0643\u0627\u0646\u062a \u062a\u0644\u062a\u0632\u0645 \u0628\u0645\u0644\u0641 robots.txt. \u0645\u0646 \u0628\u064a\u0646 62,088 \u0645\u0648\u0642\u0639\u064b\u0627 \u0641\u064a \u0633\u062c\u0644\u0646\u0627\u060c \u0623\u0638\u0647\u0631 6,966 (11.2%) \u062a\u062d\u062f\u0651\u064a Cloudflare \u0644\u0623\u062f\u0627\u0629 \u0627\u0644\u0641\u062d\u0635 \u0644\u062f\u064a\u0646\u0627.<\/p>\n<h3>\u0627\u0644\u0637\u0631\u064a\u0642\u0629 6: \u062a\u062d\u0642\u0651\u0642 \u062b\u0645 \u0627\u0633\u0645\u062d<\/h3>\n<p>\u0642\u0648\u0627\u0639\u062f \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u062a\u062d\u0638\u0631 \u0627\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a \u0627\u0644\u0635\u0627\u062f\u0642\u0629 \u0648\u062a\u0641\u0648\u0651\u062a \u063a\u064a\u0631 \u0627\u0644\u0635\u0627\u062f\u0642\u0629. \u0625\u0630\u0627 \u0643\u0646\u062a \u062a\u062d\u0638\u0631 \u0639\u0644\u0649 \u0627\u0644\u062d\u0627\u0641\u0629\u060c \u0641\u0627\u0642\u0631\u0646 \u0627\u0644\u0642\u0627\u0639\u062f\u0629 \u0628\u0641\u062d\u0635 \u0639\u0646\u0648\u0627\u0646 IP (\u0627\u0646\u0638\u0631 \u0627\u0644\u062e\u0637\u0648\u0629 4 \u0623\u0639\u0644\u0627\u0647) \u0648\u0627\u0633\u0645\u062d \u0628\u0645\u0631\u0648\u0631 \u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0628\u062d\u062b \u0648\u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0627\u0644\u0645\u0648\u062b\u0651\u0642\u064a\u0646. \u062a\u062d\u0630\u0651\u0631 Anthropic \u0645\u0646 \u0623\u0646 \u062d\u0638\u0631 \u0639\u0646\u0627\u0648\u064a\u0646 IP \u0627\u0644\u062e\u0627\u0635\u0629 \u0628\u0647\u0627 \u0642\u062f \u0644\u0627 \u064a\u0639\u0645\u0644 \u0628\u0634\u0643\u0644 \u0645\u0648\u062b\u0648\u0642 \u0643\u0648\u0633\u064a\u0644\u0629 \u0644\u0625\u0644\u063a\u0627\u0621 \u0627\u0644\u0627\u0634\u062a\u0631\u0627\u0643\u060c \u0644\u0623\u0646\u0647 \u064a\u0645\u0646\u0639 ClaudeBot \u0645\u0646 \u0642\u0631\u0627\u0621\u0629 \u0645\u0644\u0641 robots.txt. \u0627\u0633\u062a\u062e\u062f\u0645 robots.txt \u0644\u0644\u0633\u064a\u0627\u0633\u0629 \u0648\u0641\u062d\u0648\u0635 IP \u0644\u0643\u0634\u0641 \u0627\u0644\u0645\u0646\u062a\u062d\u0644\u064a\u0646.<\/p>\n<h2 id=\"owd-data\">\u0645\u0627\u0630\u0627 \u062a\u0642\u0648\u0644 2,124 \u0645\u0644\u0641 robots.txt \u0639\u0646 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a<\/h2>\n<p>\u0641\u064a 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026 \u0628\u064a\u0646 08:01 \u064808:03 \u0628\u062a\u0648\u0642\u064a\u062a AEDT\u060c \u0627\u062e\u062a\u0631\u0646\u0627 3,000 \u0646\u0637\u0627\u0642 \u0639\u0634\u0648\u0627\u0626\u064a\u064b\u0627 \u0645\u0646 62,088 \u0645\u0648\u0642\u0639\u064b\u0627 \u0641\u064a <a href=\"https:\/\/onlinewebdirectory.net\/exports\/sites.csv\">\u0645\u062c\u0645\u0648\u0639\u0629 \u0627\u0644\u0628\u064a\u0627\u0646\u0627\u062a \u0627\u0644\u0645\u0641\u062a\u0648\u062d\u0629<\/a> \u0644\u0640 Online Web Directory\u060c \u0648\u062c\u0644\u0628\u0646\u0627 <code>\/robots.txt<\/code> \u0645\u0646 \u062e\u0627\u062f\u0645\u0646\u0627. \u0623\u0639\u0627\u062f 2,124 \u0645\u0646\u0647\u0627 \u0645\u0644\u0641\u064b\u0627 \u0642\u0627\u0628\u0644\u064b\u0627 \u0644\u0644\u062a\u062d\u0644\u064a\u0644. \u064a\u064f\u0639\u062f\u0651 \u0627\u0644\u0631\u0645\u0632 &#8220;\u0645\u0630\u0643\u0648\u0631\u064b\u0627&#8221; \u0625\u0630\u0627 \u0643\u0627\u0646 \u0644\u0647 \u0633\u0637\u0631 <code>User-agent<\/code> \u062e\u0627\u0635 \u0628\u0647\u060c \u0648&#8221;\u0645\u062d\u0638\u0648\u0631\u064b\u0627 \u0643\u0644\u064a\u064b\u0627&#8221; \u0625\u0630\u0627 \u0627\u062d\u062a\u0648\u062a \u0645\u062c\u0645\u0648\u0639\u062a\u0647 \u0639\u0644\u0649 <code>Disallow: \/<\/code> \u062f\u0648\u0646 <code>Allow: \/<\/code>.<\/p>\n<figure class=\"wp-block-table owd-table\">\n<table>\n<thead>\n<tr>\n<th>\u0631\u0645\u0632 robots.txt<\/th>\n<th>\u0645\u0630\u0643\u0648\u0631<\/th>\n<th>\u0645\u062d\u0638\u0648\u0631 \u0643\u0644\u064a\u064b\u0627<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td><code>GPTBot<\/code><\/td>\n<td>23.6%<\/td>\n<td>16.9%<\/td>\n<\/tr>\n<tr>\n<td><code>ClaudeBot<\/code><\/td>\n<td>22.0%<\/td>\n<td>15.9%<\/td>\n<\/tr>\n<tr>\n<td><code>CCBot<\/code><\/td>\n<td>20.6%<\/td>\n<td>17.3%<\/td>\n<\/tr>\n<tr>\n<td><code>Google-Extended<\/code><\/td>\n<td>19.4%<\/td>\n<td>14.1%<\/td>\n<\/tr>\n<tr>\n<td><code>Bytespider<\/code><\/td>\n<td>18.5%<\/td>\n<td>16.6%<\/td>\n<\/tr>\n<tr>\n<td><code>Amazonbot<\/code><\/td>\n<td>17.4%<\/td>\n<td>14.3%<\/td>\n<\/tr>\n<tr>\n<td><code>meta-externalagent<\/code><\/td>\n<td>16.7%<\/td>\n<td>14.0%<\/td>\n<\/tr>\n<tr>\n<td><code>Applebot-Extended<\/code><\/td>\n<td>16.6%<\/td>\n<td>13.5%<\/td>\n<\/tr>\n<tr>\n<td><code>anthropic-ai<\/code> (\u0642\u062f\u064a\u0645)<\/td>\n<td>15.6%<\/td>\n<td>13.3%<\/td>\n<\/tr>\n<tr>\n<td><code>Claude-Web<\/code> (\u0642\u062f\u064a\u0645)<\/td>\n<td>13.8%<\/td>\n<td>12.2%<\/td>\n<\/tr>\n<tr>\n<td><code>PerplexityBot<\/code><\/td>\n<td>12.4%<\/td>\n<td>5.8%<\/td>\n<\/tr>\n<tr>\n<td><code>ChatGPT-User<\/code><\/td>\n<td>12.2%<\/td>\n<td>5.9%<\/td>\n<\/tr>\n<tr>\n<td><code>OAI-SearchBot<\/code><\/td>\n<td>9.3%<\/td>\n<td>2.8%<\/td>\n<\/tr>\n<tr>\n<td><code>Claude-User<\/code><\/td>\n<td>6.5%<\/td>\n<td>3.2%<\/td>\n<\/tr>\n<tr>\n<td><code>Perplexity-User<\/code><\/td>\n<td>6.4%<\/td>\n<td>3.2%<\/td>\n<\/tr>\n<tr>\n<td><code>Claude-SearchBot<\/code><\/td>\n<td>6.2%<\/td>\n<td>2.8%<\/td>\n<\/tr>\n<tr>\n<td><code>DuckAssistBot<\/code><\/td>\n<td>4.2%<\/td>\n<td>2.7%<\/td>\n<\/tr>\n<tr>\n<td><code>MistralAI-User<\/code><\/td>\n<td>3.7%<\/td>\n<td>2.6%<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/figure>\n<p>\u0623\u0631\u0628\u0639 \u0646\u062a\u0627\u0626\u062c \u0628\u0627\u0631\u0632\u0629:<\/p>\n<ul>\n<li><strong>\u0645\u0639\u0638\u0645 \u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u0644\u0627 \u062a\u0630\u0643\u0631 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0625\u0637\u0644\u0627\u0642\u064b\u0627.<\/strong> 629 \u0645\u0646 2,124 \u0645\u0644\u0641\u064b\u0627 (29.6%) \u062a\u0630\u0643\u0631 \u0631\u0645\u0632\u064b\u0627 \u0648\u0627\u062d\u062f\u064b\u0627 \u0639\u0644\u0649 \u0627\u0644\u0623\u0642\u0644 \u062e\u0627\u0635\u064b\u0627 \u0628\u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a. \u0648\u0627\u0644\u0645\u0644\u0641\u0627\u062a \u0627\u0644\u062a\u064a \u062a\u0630\u0643\u0631\u0647\u0627 \u062a\u0645\u064a\u0644 \u0625\u0644\u0649 \u0630\u0643\u0631 \u0627\u0644\u0643\u062b\u064a\u0631: \u0627\u0644\u0648\u0633\u064a\u0637 12 \u0631\u0645\u0632\u064b\u0627\u060c \u0648357 \u0645\u0644\u0641\u064b\u0627 \u062a\u0630\u0643\u0631 10 \u0623\u0648 \u0623\u0643\u062b\u0631\u060c \u0648\u0647\u064a \u0639\u0644\u0627\u0645\u0629 \u0639\u0644\u0649 \u0642\u0648\u0627\u0626\u0645 \u062d\u0638\u0631 \u0645\u0646\u0633\u0648\u062e\u0629.<\/li>\n<li><strong>\u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u064a\u064f\u062d\u0638\u0631\u060c \u0648\u0627\u0644\u0628\u062d\u062b \u0644\u0627.<\/strong> \u064a\u064f\u062d\u0638\u0631 GPTBot \u0643\u0644\u064a\u064b\u0627 \u0628\u0645\u0639\u062f\u0644 \u0633\u062a\u0629 \u0623\u0636\u0639\u0627\u0641 OAI-SearchBot (16.9% \u0645\u0642\u0627\u0628\u0644 2.8%).<\/li>\n<li><strong>robots.txt \u0648\u0627\u0644\u0648\u0635\u0648\u0644 \u0627\u0644\u0641\u0639\u0644\u064a \u0644\u0627 \u064a\u062a\u0637\u0627\u0628\u0642\u0627\u0646.<\/strong> \u0645\u0646 \u0628\u064a\u0646 358 \u0645\u0648\u0642\u0639\u064b\u0627 \u062a\u062d\u0638\u0631 GPTBot \u0643\u0644\u064a\u064b\u0627 \u0641\u064a robots.txt\u060c \u0643\u0627\u0646 284 (79%) &#8220;\u0645\u0641\u062a\u0648\u062d\u064b\u0627&#8221; \u0641\u064a \u0641\u062d\u0635 \u0627\u0644\u0648\u0635\u0648\u0644 \u0627\u0644\u062e\u0627\u0635 \u0628\u0640 OWD\u060c \u0623\u064a \u0623\u0646 \u0644\u0627 \u0634\u064a\u0621 \u0639\u0644\u0649 \u0627\u0644\u062e\u0627\u062f\u0645 \u0623\u0648\u0642\u0641 \u0639\u0645\u064a\u0644\u064b\u0627 \u0628\u0633\u064a\u0637\u064b\u0627.<\/li>\n<li><strong>\u0627\u0644\u062d\u0638\u0631 \u0627\u0644\u0634\u0627\u0645\u0644 \u0646\u0627\u062f\u0631.<\/strong> 13 \u0645\u0644\u0641\u064b\u0627 \u0641\u0642\u0637 (0.6%) \u062a\u0645\u0646\u0639 \u0643\u0644 \u0634\u064a\u0621 \u0639\u0646 \u0643\u0644 \u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645.<\/li>\n<\/ul>\n<p>\u0648\u064a\u0631\u0648\u064a \u0627\u0644\u0633\u062c\u0644 \u0627\u0644\u0623\u0648\u0633\u0639 \u0627\u0644\u0642\u0635\u0629 \u0646\u0641\u0633\u0647\u0627 \u0645\u0646 \u0627\u0644\u062c\u0647\u0629 \u0627\u0644\u0623\u062e\u0631\u0649. \u0647\u0630\u0647 \u0646\u062a\u0627\u0626\u062c \u0627\u0644\u0648\u0635\u0648\u0644 \u0644\u062c\u0645\u064a\u0639 \u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u0627\u0644\u0628\u0627\u0644\u063a \u0639\u062f\u062f\u0647\u0627 62,088\u060c \u0648\u0622\u062e\u0631 \u0641\u062d\u0635 \u0644\u0647\u0627 \u0628\u064a\u0646 11 \u0623\u0643\u062a\u0648\u0628\u0631 \u064819 \u0646\u0648\u0641\u0645\u0628\u0631 2026:<\/p>\n<figure class=\"wp-block-table owd-table\">\n<table>\n<thead>\n<tr>\n<th>\u0645\u0627 \u0648\u0627\u062c\u0647\u0647 \u0639\u0645\u064a\u0644 HTTP \u0628\u0633\u064a\u0637<\/th>\n<th>\u0627\u0644\u0645\u0648\u0627\u0642\u0639<\/th>\n<th>\u0627\u0644\u0646\u0633\u0628\u0629<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>\u0645\u0641\u062a\u0648\u062d<\/td>\n<td>50,844<\/td>\n<td>81.9%<\/td>\n<\/tr>\n<tr>\n<td>\u062a\u062d\u062f\u064d\u0651 \u0644\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a<\/td>\n<td>7,821<\/td>\n<td>12.6%<\/td>\n<\/tr>\n<tr>\n<td>\u0645\u062d\u0638\u0648\u0631<\/td>\n<td>3,060<\/td>\n<td>4.9%<\/td>\n<\/tr>\n<tr>\n<td>\u0645\u062d\u062f\u0648\u062f \u0627\u0644\u0645\u0639\u062f\u0644<\/td>\n<td>199<\/td>\n<td>0.3%<\/td>\n<\/tr>\n<tr>\n<td>\u064a\u062a\u0637\u0644\u0628 \u062a\u0633\u062c\u064a\u0644 \u0627\u0644\u062f\u062e\u0648\u0644<\/td>\n<td>150<\/td>\n<td>0.2%<\/td>\n<\/tr>\n<tr>\n<td>\u062e\u0644\u0641 \u062c\u062f\u0627\u0631 \u062f\u0641\u0639<\/td>\n<td>14<\/td>\n<td>0.0%<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/figure>\n<p>\u0646\u0632\u0651\u0644 \u0645\u062c\u0645\u0648\u0639\u0629 \u0627\u0644\u0628\u064a\u0627\u0646\u0627\u062a \u0627\u0644\u0643\u0627\u0645\u0644\u0629 (CC BY 4.0) \u0645\u0646 <a href=\"https:\/\/onlinewebdirectory.net\/exports\/sites.csv\">onlinewebdirectory.net\/exports\/sites.csv<\/a>\u060c \u0623\u0648 \u0627\u0637\u0651\u0644\u0639 \u0639\u0644\u0649 <a href=\"https:\/\/onlinewebdirectory.net\/reports\/ai-agent-access\">\u062a\u0642\u0631\u064a\u0631 \u0648\u0635\u0648\u0644 \u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a<\/a>.<\/p>\n<h2 id=\"related-errors\">\u0623\u062e\u0637\u0627\u0621 \u0648\u0627\u0633\u062a\u062c\u0627\u0628\u0627\u062a \u0630\u0627\u062a \u0635\u0644\u0629<\/h2>\n<ul>\n<li><strong>403 Forbidden.<\/strong> \u0631\u0641\u0636 \u062e\u0627\u062f\u0645\u0643 \u0623\u0648 CDN \u0623\u0648 \u062c\u062f\u0627\u0631 \u0627\u0644\u062d\u0645\u0627\u064a\u0629 \u0627\u0644\u0637\u0644\u0628. \u0644\u0627 \u064a\u0646\u062a\u062c \u0623\u0628\u062f\u064b\u0627 \u0639\u0646 robots.txt \u0646\u0641\u0633\u0647\u060c \u0648\u0644\u064a\u0633 \u062f\u0627\u0626\u0645\u064b\u0627: \u063a\u064a\u0651\u0631 \u0627\u0644\u0642\u0627\u0639\u062f\u0629 \u0648\u0642\u062f \u064a\u0646\u062c\u062d \u0627\u0644\u0637\u0644\u0628 \u0627\u0644\u062a\u0627\u0644\u064a.<\/li>\n<li><strong>429 Too Many Requests.<\/strong> \u062a\u062d\u062f\u064a\u062f \u0627\u0644\u0645\u0639\u062f\u0644. \u0623\u0639\u0627\u062f Reddit \u0647\u0630\u0627 \u0627\u0644\u0631\u0645\u0632 \u0644\u0637\u0644\u0628 ClaudeBot \u0627\u0644\u062e\u0627\u0635 \u0628\u0646\u0627. \u062a\u062f\u0639\u0645 Anthropic \u0627\u0644\u062a\u0648\u062c\u064a\u0647 <code>Crawl-delay<\/code>\u060c \u0648\u0647\u0648 \u0628\u062f\u064a\u0644 \u0623\u0644\u0637\u0641 \u0644\u0640 ClaudeBot.<\/li>\n<li><strong>202 \u0628\u062c\u0633\u0645 \u0641\u0627\u0631\u063a\u060c \u0623\u0648 \u0635\u0641\u062d\u0629 \u062a\u062d\u062f\u064d\u0651 503.<\/strong> \u0639\u0627\u062f\u0629 \u062a\u062d\u062f\u064a JavaScript \u0623\u0648 CAPTCHA. \u0644\u0627 \u062a\u0633\u062a\u0637\u064a\u0639 \u0627\u0644\u0632\u0648\u0627\u062d\u0641 \u062d\u0644\u0651\u0647\u0627\u060c \u0644\u0630\u0627 \u062a\u0643\u0648\u0646 \u0627\u0644\u0635\u0641\u062d\u0629 \u0645\u062d\u0638\u0648\u0631\u0629 \u0628\u0627\u0644\u0646\u0633\u0628\u0629 \u0625\u0644\u064a\u0647\u0627 \u0631\u063a\u0645 \u0639\u062f\u0645 \u0625\u0631\u0633\u0627\u0644 403.<\/li>\n<li><strong>robots.txt \u064a\u064f\u0631\u062c\u0639 4xx.<\/strong> \u0648\u0641\u0642 RFC 9309\u060c \u062a\u0639\u0646\u064a \u0627\u0644\u0627\u0633\u062a\u062c\u0627\u0628\u0629 400-499 \u0623\u0646 \u0627\u0644\u0645\u0644\u0641 &#8220;\u063a\u064a\u0631 \u0645\u062a\u0627\u062d&#8221; \u0648\u064a\u062c\u0648\u0632 \u0644\u0644\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0648\u0635\u0648\u0644 \u0625\u0644\u0649 \u0643\u0644 \u0634\u064a\u0621. \u0641\u064a \u0639\u064a\u0651\u0646\u062a\u0646\u0627\u060c \u0623\u062c\u0627\u0628 314 \u0645\u0646 3,000 \u0645\u0648\u0642\u0639 (10.5%) \u0639\u0644\u0649 \u0637\u0644\u0628 robots.txt \u0628\u0627\u0644\u0631\u0645\u0632 403\u060c \u0648\u0647\u0648 \u0645\u0627 \u064a\u0642\u0631\u0624\u0647 \u0627\u0644\u0632\u0627\u062d\u0641 \u0627\u0644\u0645\u0644\u062a\u0632\u0645 \u0639\u0644\u0649 \u0623\u0646\u0647 \u0644\u0627 \u062a\u0648\u062c\u062f \u0642\u0648\u0627\u0639\u062f \u0625\u0637\u0644\u0627\u0642\u064b\u0627.<\/li>\n<li><strong>robots.txt \u064a\u064f\u0631\u062c\u0639 5xx \u0623\u0648 \u062a\u0646\u062a\u0647\u064a \u0645\u0647\u0644\u062a\u0647.<\/strong> \u064a\u0642\u0648\u0644 RFC 9309 \u0625\u0646 \u0639\u0644\u0649 \u0627\u0644\u0632\u0648\u0627\u062d\u0641 \u062d\u064a\u0646\u0647\u0627 \u0627\u0641\u062a\u0631\u0627\u0636 \u0627\u0644\u0645\u0646\u0639 \u0627\u0644\u0643\u0627\u0645\u0644\u060c \u0644\u0630\u0627 \u0642\u062f \u064a\u0624\u062f\u064a \u062a\u0639\u0637\u0651\u0644 robots.txt \u0625\u0644\u0649 \u062d\u0638\u0631 \u0643\u0644 \u0627\u0644\u0632\u0648\u0627\u062d\u0641 \u0628\u0635\u0645\u062a\u060c \u0628\u0645\u0627 \u0641\u064a\u0647\u0627 Googlebot.<\/li>\n<li><strong>robots.txt \u0623\u0643\u0628\u0631 \u0645\u0646 500 KiB.<\/strong> \u0644\u0627 \u064a\u064f\u0644\u0632\u0645 RFC 9309 \u0627\u0644\u0632\u0648\u0627\u062d\u0641 \u0625\u0644\u0627 \u0628\u062a\u062d\u0644\u064a\u0644 \u0623\u0648\u0644 500 KiB\u060c \u0644\u0630\u0627 \u0642\u062f \u062a\u064f\u062a\u062c\u0627\u0647\u0644 \u0627\u0644\u0642\u0648\u0627\u0639\u062f \u0628\u0639\u062f \u0630\u0644\u0643.<\/li>\n<\/ul>\n<p>\u0625\u0630\u0627 \u0643\u0646\u062a \u062a\u0646\u0634\u0631 \u0645\u0644\u0641 llms.txt \u0644\u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a\u060c \u0641\u062a\u0623\u0643\u062f \u0645\u0646 \u0623\u0646 \u0627\u0644\u0631\u0648\u0628\u0648\u062a\u0627\u062a \u0627\u0644\u062a\u064a \u062a\u0633\u0645\u062d \u0628\u0647\u0627 \u062a\u0633\u062a\u0637\u064a\u0639 \u062c\u0644\u0628\u0647 \u0641\u0639\u0644\u064b\u0627. \u064a\u0648\u0636\u062d <a href=\"\/blog\/ar\/llms-txt-ar\/basics-ar\/ma-huwa-llms-txt\/\">\u062f\u0644\u064a\u0644 llms.txt<\/a> \u0643\u064a\u0641\u064a\u0629 \u0627\u062e\u062a\u0628\u0627\u0631 \u0630\u0644\u0643.<\/p>\n<h2 id=\"summary\">\u0627\u0644\u062e\u0644\u0627\u0635\u0629<\/h2>\n<p>\u062a\u0628\u062f\u0623 \u0642\u0627\u0626\u0645\u0629 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0627\u0644\u062f\u0642\u064a\u0642\u0629 \u0628\u0648\u062b\u0627\u0626\u0642 \u0627\u0644\u0645\u0634\u063a\u0651\u0644\u064a\u0646 \u0623\u0646\u0641\u0633\u0647\u0645\u060c \u0644\u0627 \u0628\u0627\u0644\u0642\u0648\u0627\u0626\u0645 \u0627\u0644\u0623\u062e\u0631\u0649. \u0627\u0644\u064a\u0648\u0645 \u064a\u0639\u0646\u064a \u0630\u0644\u0643 \u0646\u062d\u0648 20 \u0648\u0643\u064a\u0644 \u0645\u0633\u062a\u062e\u062f\u0645 \u0645\u0646 9 \u0634\u0631\u0643\u0627\u062a\u060c \u0641\u064a \u062b\u0644\u0627\u062b\u0629 \u0623\u0646\u0648\u0627\u0639: \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u0648\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0628\u062d\u062b \u0628\u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0648\u0627\u0644\u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0630\u064a\u0646 \u064a\u0634\u063a\u0651\u0644\u0647\u0645 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645\u060c \u0625\u0636\u0627\u0641\u0629 \u0625\u0644\u0649 \u0631\u0645\u0632\u064a \u062a\u062d\u0643\u0645 (Google-Extended \u0648Applebot-Extended) \u0644\u0627 \u064a\u0632\u062d\u0641\u0627\u0646 \u0623\u0628\u062f\u064b\u0627. \u0645\u0639\u0638\u0645 \u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u0627\u0644\u062a\u064a \u062a\u062a\u062e\u0630 \u0625\u062c\u0631\u0627\u0621\u064b \u062a\u062d\u0638\u0631 \u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u0648\u062a\u064f\u0628\u0642\u064a \u0627\u0644\u0628\u062d\u062b\u060c \u0648robots.txt \u0645\u062c\u0631\u062f \u0637\u0644\u0628: \u0625\u0630\u0627 \u0627\u062d\u062a\u062c\u062a \u0625\u0644\u0649 \u062d\u0638\u0631 \u062d\u0642\u064a\u0642\u064a \u0641\u0627\u0641\u0631\u0636\u0647 \u0639\u0644\u0649 \u0627\u0644\u062e\u0627\u062f\u0645 \u0623\u0648 CDN \u0648\u062a\u062d\u0642\u0651\u0642 \u0645\u0646 \u0639\u0646\u0627\u0648\u064a\u0646 IP.<\/p>\n<p>\u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u064a\u0628\u062d\u062b\u0648\u0646 \u0628\u0627\u0644\u0641\u0639\u0644 \u0639\u0646 \u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u0641\u064a Online Web Directory \u0642\u0628\u0644 \u0632\u064a\u0627\u0631\u062a\u0647\u0627. \u0625\u0630\u0627 \u0643\u0627\u0646 \u0633\u062c\u0644 \u0645\u0648\u0642\u0639\u0643 \u062e\u0627\u0637\u0626\u064b\u0627\u060c \u0623\u0648 \u0623\u0631\u062f\u062a \u0623\u0646 \u064a\u0639\u0631\u0641 \u0627\u0644\u0648\u0643\u0644\u0627\u0621 \u0648\u0627\u062c\u0647\u0629 API \u0623\u0648 \u062e\u0627\u062f\u0645 MCP \u0623\u0648 \u0623\u0630\u0648\u0646\u0627\u062a \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0644\u062f\u064a\u0643\u060c <a href=\"https:\/\/onlinewebdirectory.net\/ar\/claim\">\u0641\u0637\u0627\u0644\u0650\u0628 \u0628\u0645\u0648\u0642\u0639\u0643 \u0645\u0642\u0627\u0628\u0644 25 \u062f\u0648\u0644\u0627\u0631\u064b\u0627<\/a>. \u064a\u0645\u0643\u0646\u0643 \u062a\u0634\u063a\u064a\u0644 <a href=\"https:\/\/onlinewebdirectory.net\/ar\/check\">\u0641\u062d\u0635 \u0627\u0644\u062c\u0627\u0647\u0632\u064a\u0629<\/a> \u0627\u0644\u0645\u062c\u0627\u0646\u064a \u0623\u0648\u0644\u064b\u0627.<\/p>\n<h2 id=\"faq\">\u0627\u0644\u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0634\u0627\u0626\u0639\u0629<\/h2>\n<h3>\u0645\u0627 \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0627\u0644\u062e\u0627\u0635 \u0628\u0640 ChatGPT\u061f<\/h3>\n<p>\u064a\u0633\u062a\u062e\u062f\u0645 ChatGPT \u062b\u0644\u0627\u062b\u0629. \u064a\u062c\u0645\u0639 GPTBot \u0628\u064a\u0627\u0646\u0627\u062a \u0627\u0644\u062a\u062f\u0631\u064a\u0628\u060c \u0648\u064a\u0641\u0647\u0631\u0633 OAI-SearchBot \u0627\u0644\u0635\u0641\u062d\u0627\u062a \u0644\u0628\u062d\u062b ChatGPT\u060c \u0648\u064a\u062c\u0644\u0628 ChatGPT-User \u0635\u0641\u062d\u0629 \u0639\u0646\u062f\u0645\u0627 \u064a\u0637\u0644\u0628\u0647\u0627 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645. \u0631\u0645\u0648\u0632\u0647\u0627 \u0641\u064a robots.txt \u0647\u064a <code>GPTBot<\/code> \u0648<code>OAI-SearchBot<\/code> \u0648<code>ChatGPT-User<\/code>.<\/p>\n<h3>\u0647\u0644 \u064a\u0624\u062f\u064a \u062d\u0638\u0631 GPTBot \u0625\u0644\u0649 \u0625\u0632\u0627\u0644\u0629 \u0645\u0648\u0642\u0639\u064a \u0645\u0646 \u0628\u062d\u062b ChatGPT\u061f<\/h3>\n<p>\u0644\u0627. \u062a\u062a\u062d\u0643\u0645 OpenAI \u0641\u064a \u0627\u0644\u0628\u062d\u062b \u0639\u0628\u0631 OAI-SearchBot. \u064a\u0645\u0643\u0646\u0643 \u0645\u0646\u0639 GPTBot \u0644\u0625\u0644\u063a\u0627\u0621 \u0627\u0644\u0627\u0634\u062a\u0631\u0627\u0643 \u0641\u064a \u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u0645\u0639 \u0627\u0644\u0633\u0645\u0627\u062d \u0644\u0640 OAI-SearchBot \u0643\u064a \u062a\u0638\u0647\u0631 \u0635\u0641\u062d\u0627\u062a\u0643 \u0648\u064a\u064f\u0633\u062a\u0634\u0647\u062f \u0628\u0647\u0627 \u0641\u064a \u0628\u062d\u062b ChatGPT.<\/p>\n<h3>\u0647\u0644 Google-Extended \u0632\u0627\u062d\u0641\u061f<\/h3>\n<p>\u0644\u0627. \u062a\u0642\u0648\u0644 Google \u0625\u0646 Google-Extended \u0644\u064a\u0633 \u0644\u0647 \u0648\u0643\u064a\u0644 \u0645\u0633\u062a\u062e\u062f\u0645 \u0645\u0646\u0641\u0635\u0644. \u064a\u062a\u0648\u0644\u0649 Googlebot \u0627\u0644\u0632\u062d\u0641\u060c \u0648\u064a\u062a\u062d\u0643\u0645 \u0631\u0645\u0632 Google-Extended \u0641\u0642\u0637 \u0641\u064a \u0645\u0627 \u0625\u0630\u0627 \u0643\u0627\u0646 \u0647\u0630\u0627 \u0627\u0644\u0645\u062d\u062a\u0648\u0649 \u064a\u0645\u0643\u0646 \u0627\u0633\u062a\u062e\u062f\u0627\u0645\u0647 \u0644\u062a\u062f\u0631\u064a\u0628 Gemini \u0648\u062f\u0639\u0645 \u0625\u062c\u0627\u0628\u0627\u062a\u0647. \u0648\u0644\u0627 \u064a\u0624\u062b\u0631 \u0639\u0644\u0649 \u0627\u0644\u062a\u0631\u062a\u064a\u0628 \u0641\u064a \u0628\u062d\u062b Google.<\/p>\n<h3>\u0647\u0644 \u062a\u0644\u062a\u0632\u0645 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0628\u0645\u0644\u0641 robots.txt\u061f<\/h3>\n<p>\u062a\u0642\u0648\u0644 \u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u062a\u062f\u0631\u064a\u0628 \u0648\u0627\u0644\u0628\u062d\u062b \u0627\u0644\u0645\u0648\u062b\u0651\u0642\u0629 \u0641\u064a \u0647\u0630\u0647 \u0627\u0644\u0642\u0627\u0626\u0645\u0629 \u0625\u0646\u0647\u0627 \u062a\u0644\u062a\u0632\u0645. \u0623\u0645\u0627 \u0627\u0644\u062c\u0627\u0644\u0628\u0627\u062a \u0627\u0644\u062a\u064a \u064a\u0634\u063a\u0651\u0644\u0647\u0627 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0645\u062b\u0644 ChatGPT-User \u0648Perplexity-User \u0648Meta-ExternalFetcher \u0641\u062a\u0642\u0648\u0644 \u0625\u0646 robots.txt \u0642\u062f \u0644\u0627 \u064a\u0646\u0637\u0628\u0642 \u0644\u0623\u0646 \u0634\u062e\u0635\u064b\u0627 \u0637\u0644\u0628 \u0627\u0644\u0635\u0641\u062d\u0629. \u0648\u0642\u062f \u062a\u062a\u062c\u0627\u0647\u0644\u0647 \u0623\u062f\u0648\u0627\u062a \u0627\u0644\u0643\u0634\u0637 \u063a\u064a\u0631 \u0627\u0644\u0645\u0648\u062b\u0651\u0642\u0629 \u062a\u0645\u0627\u0645\u064b\u0627.<\/p>\n<h3>\u0643\u064a\u0641 \u0623\u0639\u0631\u0641 \u0645\u0627 \u0625\u0630\u0627 \u0643\u0627\u0646 \u0627\u0644\u0631\u0648\u0628\u0648\u062a \u0647\u0648 GPTBot \u0623\u0648 ClaudeBot \u0641\u0639\u0644\u064b\u0627\u061f<\/h3>\n<p>\u062a\u062d\u0642\u0651\u0642 \u0645\u0646 \u0639\u0646\u0648\u0627\u0646 IP \u0627\u0644\u0645\u0635\u062f\u0631. \u062a\u0646\u0634\u0631 OpenAI \u0627\u0644\u0646\u0637\u0627\u0642\u0627\u062a \u0641\u064a <code>gptbot.json<\/code> \u0648<code>searchbot.json<\/code> \u0648<code>chatgpt-user.json<\/code>\u061b \u0648\u062a\u0646\u0634\u0631 Perplexity \u0648DuckDuckGo \u0648Mistral \u0648Google \u0646\u0637\u0627\u0642\u0627\u062a\u0647\u0627 \u0627\u0644\u062e\u0627\u0635\u0629. \u0648\u0643\u064a\u0644 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0627\u0644\u0645\u0637\u0627\u0628\u0642 \u0627\u0644\u0642\u0627\u062f\u0645 \u0645\u0646 IP \u063a\u064a\u0631 \u0645\u062f\u0631\u062c \u0647\u0648 \u0645\u0646\u062a\u062d\u0644.<\/p>\n<h2 id=\"sources\">\u0627\u0644\u0645\u0635\u0627\u062f\u0631<\/h2>\n<ol>\n<li>OpenAI\u060c <a href=\"https:\/\/developers.openai.com\/api\/docs\/bots\" target=\"_blank\" rel=\"noopener\">Overview of OpenAI crawlers<\/a> (\u062a\u0627\u0631\u064a\u062e \u0627\u0644\u0627\u0637\u0644\u0627\u0639 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026)<\/li>\n<li>Anthropic, <a href=\"https:\/\/support.claude.com\/en\/articles\/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler\" target=\"_blank\" rel=\"noopener\">Does Anthropic crawl data from the web, and how can site owners block the crawler?<\/a><\/li>\n<li>Google\u060c <a href=\"https:\/\/developers.google.com\/crawling\/docs\/crawlers-fetchers\/google-common-crawlers\" target=\"_blank\" rel=\"noopener\">Google&#8217;s common crawlers<\/a> \u0648<a href=\"https:\/\/developers.google.com\/crawling\/docs\/crawlers-fetchers\/google-user-triggered-fetchers\" target=\"_blank\" rel=\"noopener\">user-triggered fetchers<\/a><\/li>\n<li>Perplexity, <a href=\"https:\/\/docs.perplexity.ai\/docs\/resources\/perplexity-crawlers\" target=\"_blank\" rel=\"noopener\">Perplexity crawlers<\/a><\/li>\n<li>Apple, <a href=\"https:\/\/support.apple.com\/en-us\/119829\" target=\"_blank\" rel=\"noopener\">About Applebot<\/a><\/li>\n<li>Meta, <a href=\"https:\/\/developers.facebook.com\/docs\/sharing\/webmasters\/web-crawlers\/\" target=\"_blank\" rel=\"noopener\">Meta web crawlers<\/a><\/li>\n<li>Amazon, <a href=\"https:\/\/developer.amazon.com\/docs\/amazonbot\/amazonbot.html\" target=\"_blank\" rel=\"noopener\">Amazonbot and Amazon crawlers<\/a><\/li>\n<li>Common Crawl, <a href=\"https:\/\/commoncrawl.org\/ccbot\" target=\"_blank\" rel=\"noopener\">CCBot<\/a><\/li>\n<li>Mistral AI, <a href=\"https:\/\/docs.mistral.ai\/robots\" target=\"_blank\" rel=\"noopener\">Mistral AI robots<\/a><\/li>\n<li>DuckDuckGo, <a href=\"https:\/\/duckduckgo.com\/duckduckgo-help-pages\/results\/duckassistbot\" target=\"_blank\" rel=\"noopener\">About DuckAssistBot<\/a><\/li>\n<li>IETF, <a href=\"https:\/\/www.rfc-editor.org\/rfc\/rfc9309.html\" target=\"_blank\" rel=\"noopener\">RFC 9309: Robots Exclusion Protocol<\/a><\/li>\n<li>Cloudflare\u060c <a href=\"https:\/\/developers.cloudflare.com\/bots\/additional-configurations\/block-ai-bots\/\" target=\"_blank\" rel=\"noopener\">AI Crawler and Agent access policies<\/a> \u0648<a href=\"https:\/\/developers.cloudflare.com\/ai-crawl-control\/\" target=\"_blank\" rel=\"noopener\">AI Crawl Control<\/a><\/li>\n<li>Online Web Directory\u060c <a href=\"https:\/\/onlinewebdirectory.net\/exports\/sites.csv\">\u0645\u062c\u0645\u0648\u0639\u0629 \u0628\u064a\u0627\u0646\u0627\u062a \u0627\u0644\u0645\u0648\u0627\u0642\u0639 \u0627\u0644\u0645\u0641\u062a\u0648\u062d\u0629<\/a> (62,088 \u0645\u0648\u0642\u0639\u064b\u0627\u060c \u0635\u064f\u062f\u0651\u0631\u062a \u0641\u064a 20 \u0646\u0648\u0641\u0645\u0628\u0631 2026) \u0648\u0639\u064a\u0651\u0646\u062a\u0646\u0627 \u0645\u0646 robots.txt \u0644\u0640 3,000 \u0646\u0637\u0627\u0642<\/li>\n<\/ol>\n","protected":false},"excerpt":{"rendered":"<p>\u0642\u0627\u0626\u0645\u0629 \u0645\u0648\u062b\u0651\u0642\u0629 \u0644\u0632\u0648\u0627\u062d\u0641 \u0627\u0644\u0630\u0643\u0627\u0621 \u0627\u0644\u0627\u0635\u0637\u0646\u0627\u0639\u064a \u0648\u0648\u0643\u0644\u0627\u0621 \u0627\u0644\u0645\u0633\u062a\u062e\u062f\u0645 \u0641\u064a 2026 (GPTBot \u0648ClaudeBot \u0648PerplexityBot \u0648\u063a\u064a\u0631\u0647\u0627): \u0645\u0627\u0630\u0627 \u064a\u0641\u0639\u0644 \u0643\u0644 \u0645\u0646\u0647\u0627\u060c \u0648\u0642\u0648\u0627\u0639\u062f \u062d\u0638\u0631 \u0645\u064f\u062e\u062a\u0628\u0631\u0629\u060c \u0648\u0628\u064a\u0627\u0646\u0627\u062a robots.txt \u0645\u0646 2,124 \u0645\u0648\u0642\u0639\u064b\u0627.<\/p>\n","protected":false},"author":2,"featured_media":557,"comment_status":"closed","ping_status":"open","sticky":false,"template":"owd-single-ar","format":"standard","meta":{"footnotes":""},"categories":[1296],"tags":[],"class_list":["post-606","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-crawler-list-ar"],"lang":"ar","translations":{"ar":606,"en":567,"de":580,"es":582,"pt":584,"fr":586,"it":588,"nl":590,"sv":592,"pl":594,"cs":596,"ru":598,"uk":600,"tr":602,"ja":604,"zh":608,"ko":610,"hi":612,"id":614,"vi":616,"th":618},"_links":{"self":[{"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/posts\/606","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/comments?post=606"}],"version-history":[{"count":1,"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/posts\/606\/revisions"}],"predecessor-version":[{"id":607,"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/posts\/606\/revisions\/607"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/media\/557"}],"wp:attachment":[{"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/media?parent=606"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/categories?post=606"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/onlinewebdirectory.net\/blog\/wp-json\/wp\/v2\/tags?post=606"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}