robots.txt

# START YOAST BLOCK

# —————————

User-agent: *

Disallow:

Sitemap: https://www.xxx.com /sitemap_index.xml

# —————————

# END YOAST BLOCK

//robots.txt

# Block all crawlers for /accounts

User-agent: *

Allow: /

Disallow: /accounts

# Allow all crawlers

User-agent: *

Allow: /

Disallow:

Allowing all web crawlers access to all content

#允许所有web爬网程序访问所有内容

User-agent: *

Disallow:

Blocking all web crawlers from all content
#阻止所有网络爬网程序访问所有内容
User-agent: * Disallow: /

Blocking a specific web crawler from a specific folder
#阻止特定文件夹中的特定web爬网程序
User-agent: Googlebot Disallow: /example-subfolder/