forked from UsrSpace-Internal/website
285 lines
2.7 KiB
Plaintext
285 lines
2.7 KiB
Plaintext
Sitemap: sitemap.xml
|
|
|
|
User-Agent: *
|
|
Crawl-Delay: 600
|
|
|
|
User-agent: Screaming
|
|
Disallow: /
|
|
|
|
|
|
User-agent: xovi
|
|
Disallow: /
|
|
User-agent: XoviBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Smager
|
|
Disallow: /
|
|
|
|
|
|
User-agent: SEOkicks
|
|
Disallow: /
|
|
User-agent: SEOkicks-Robot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: sistrix
|
|
Disallow: /
|
|
|
|
|
|
User-agent: MajesticSEO
|
|
Disallow: /
|
|
|
|
|
|
User-agent: BacklinkCrawler
|
|
Disallow: /
|
|
|
|
|
|
User-agent: MJ12bot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: spbot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: SearchmetricsBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: search17
|
|
Disallow: /
|
|
|
|
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: ia_archiver
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Seobility
|
|
Disallow: /
|
|
|
|
|
|
User-agent: TurnitinBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: SlySearch
|
|
Disallow: /
|
|
|
|
|
|
User-agent: findlinks
|
|
Disallow: /
|
|
|
|
|
|
User-agent: magpie-crawler
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Pixray-Seeker
|
|
Disallow: /
|
|
|
|
|
|
User-agent: 008
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Ezooms
|
|
Disallow: /
|
|
|
|
|
|
User-agent: lb-spider
|
|
Disallow: /
|
|
|
|
|
|
User-agent: WBSearchBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: psbot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: HuaweiSymantecSpider
|
|
Disallow: /
|
|
|
|
|
|
User-agent: EC2LinkFinder
|
|
Disallow: /
|
|
|
|
|
|
User-agent: htdig
|
|
Disallow: /
|
|
|
|
|
|
User-agent: SemrushBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: discobot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: linkdex.com
|
|
Disallow: /
|
|
|
|
|
|
User-agent: SeznamBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: EdisterBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: SWEBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: picmole
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Yeti
|
|
Disallow: /
|
|
User-agent: Yeti-Mobile
|
|
Disallow: /
|
|
|
|
|
|
User-agent: PagePeeker
|
|
Disallow: /
|
|
|
|
|
|
User-agent: CatchBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: yacybot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: netEstateNECrawler
|
|
Disallow: /
|
|
|
|
|
|
User-agent: SurveyBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: gonzo
|
|
Disallow: /
|
|
|
|
|
|
User-agent: schrein
|
|
Disallow: /
|
|
|
|
|
|
User-agent: AfiliasWebMiningTool
|
|
Disallow: /
|
|
|
|
|
|
User-agent: suggybot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: bdbrandprotect
|
|
Disallow: /
|
|
|
|
|
|
User-agent: BPImageWalker
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Updownerbot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: lex
|
|
Disallow: /
|
|
|
|
|
|
User-agent: ContentCrawler
|
|
Disallow: /
|
|
|
|
|
|
User-agent: DCPbot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: KaloogaBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: MLBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: iCjobs
|
|
Disallow: /
|
|
|
|
|
|
User-agent: oBot
|
|
Disallow: /
|
|
|
|
|
|
User-agent: WebmasterCoffee
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Qualidator
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Webinator
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Scooter
|
|
Disallow: /
|
|
|
|
|
|
User-agent: thunderstone
|
|
Disallow: /
|
|
|
|
|
|
User-agent: larbin
|
|
Disallow: /
|
|
|
|
|
|
User-agent: OpidooBOT
|
|
Disallow: /
|
|
|
|
|
|
User-agent: ips-agent
|
|
Disallow: /
|
|
|
|
|
|
User-agent: TinEye
|
|
Disallow: /
|
|
|
|
|
|
User-agent: UnisterBot
|
|
Disallow: /
|
|
User-agent: Unister
|
|
Disallow: /
|
|
|
|
|
|
User-agent: ReverseGet
|
|
Disallow: /
|
|
|
|
|
|
User-agent: DotBot
|
|
Disallow: /
|
|
|
|
User-agent: seoscanners.net/1
|
|
Disallow: /
|
|
|
|
User-agent: spider@seoscanners.net
|
|
Disallow: /
|
|
|
|
User-agent: GPTBot
|
|
Disallow: /
|