V4.2021.08.2545 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-ADmantX
-AIBOT
+404checker
+80legs
 ALittle Client
 ASPSeek
-AfD-Verbotsverfahren
+Abonti
+Aboundex
 AhrefsBot
 AiHitBot
-Aipbot
-AllSubmitter
-Ankit
-Anthill
+Alligator
+Anarchie
+Anarchy
 Apexoo
-Aspiegel
-Asterias
 Attach
 AwarioRssBot
-BLEXBot
-BackWeb
+AwarioSmartBot
+BBBike
+BackStreet
+Backlink-Ceck
 BacklinkCrawler
+Barkrowler
 Bigfoot
-BlackWidow
+Black Hole
 Blackboard
-Boardreader
-Brandprotect
+Blow
+BlowFish
+Bolt
+BotALot
 BuiltBotTough
 Bullseye
-BuzzSumo
+CODE87
 Calculon
 CazoodleBot
-Cegbfeieh
-CensysInspect
 CheTeam
-ChinaClaw
-Claritybot
+CheeseBot
+Chlooe
 Cliqzbot
-Cloud mapping
 Cogentbot
-Copier
+Collector
+Copyscape
 Crawling at Home Project
-DBLBot
+Crescent
+CrunchBot
+CyotekWebCopy
+DSearch
+DTS Agent
+DataCha0s
 DatabaseDriverMysqli
-Deusu
 Devil
 Digincore
-Dirbuster
-Disco
+DigitalPebble
+Discobot
+Dispatch
+DnyzBot
 DomainStatsBot
-Domains Project
-Dotbot
+Drip
+EMail Siphon
+EasyDL
 Ecxi
-EirGrabber
+EroCrawler
 Evil
+Exabot
 Express WebPictures
+ExtLinksBot
 Extractor
-ExtractorPro
 Ezooms
 FHscan
 FemtosearchBot
-Firefox/7.0
-Flunky
-Foobot
+FlashGet
+Freeuploader
 Fuzz
-Fyrebot
+G-i-g-a-b-o-t
 GT::WWW
 GalaxyBot
-GetRight
-Getintent
-Gigablast
-Gigabot
-Go-Ahead-Got-It
-GrabNet
-GridBot
+Grabber
+Grafula
+GrapeshotCrawler
 HTMLparser
-HTTP::Lite
-Harvest
-Heritrix
-Humanlinks
-HybridBot
-IDBTE4M
-IRLbot
+HTTrack
+Haansoft
+Havij
+Iblog
 Id-search
-IlseBot
-Image Sucker
-IndeedBot
+Indy Library
 InfoNaviRobot
-InfoTekies
-Intelliseek
-Iskanie
-JOC Web Spider
-Jbrofuzz
-JennyBot
-JikeSpider
+InterGET
+Internet Ninja
+Iria
+JamesBOT
+JetCar
+Jetty
 Joomla
-Jorgee
-Jyxobot
-LNSpiderguy
-Larbin
+JustView
+Kenjin Spider
+Kozmosbot
+LeechFTP
+LeechGet
 LibWeb
+Likse
 LinkWalker
-LinkextractorPro
+Linkdexbot
+LinksManager
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
 Lipperhey Spider
 Litemage_walker
 Lmspider
-Ltx71
 MFC_Tear_Sample
-MTRobot
+MIDown tool
+MQQBrowser
+MSFrontPage
 Mail.RU_Bot
-Majestic-SEO
-Majestic12
+Mass Downloader
 Masscan
 Mata Hari
 Meanpathbot
+MegaIndex.ru
 Metauri
 MicroMessenger
-Microsoft Data Access
-Moblie Safari
-Mojolicious
-Msrabot
-NPbot
-Name Intelligence
-NetAnts
-Netcraft
-Nettrack
-Netvibes
+Microsoft URL Control
+Mojeek
+Morfeus Fucking Scanner
+Mr.4x3
+NICErsPRO
+Nessus
+Net Vampire
+NetSpider
 NextGenSearchBot
-Nibbler
-Nikto
+Niki-bot
 NimbleCrawler
 Nimbostratus
+Nmap
 Nuclei
+Nutch
+Octopus
 Offline Navigator
-OrangeBot
+OnCrawl
+OpenLinkProfiler
 OrangeSpider
-OutfoxBot
+PECL::HTTP
 PHPCrawl
 Page Analyzer
+PageAnalyzer
 PageGrabber
+PageThing.com
+Papa Foto
 Petalbot
+Pi-Monster
+Piepmatz
 Pimonster
-Pixray
-Pump
-PxBroker
-PyCurl
-Quick-Crawler
-RSSingBot
-RankActive
-RankActiveLinkBot
-RankingBot
-ReGet
-RedesScrapy
-Ripper
-RocketCrawler
-SEOkicks-Robot
-SEOlyticsCrawler
+ProPowerBot
+ProWebWalker
+Probethenet
+Psbot
+RankFlex
+RankurBot
+Reaper
+RepoMonkey
+SBIder
+SEOprofiler
 SISTRIX
+SalesIntelligent
+ScoutJet
 Searchestate
-SearchmetricsBot
-Semrush
+SemrushBot
 SeobilityBot
-Shodan
-Site Sucker
-SiteCheckerBotCrawler
-SiteExplorer
-SiteSucker
-Sitevigil
+Siphon
+SiteSnagger
 SlySearch
-Snapbot
-Snoopy
 Sociscraper
-Sosospider
-Sottopop
-SpankBot
+Sogou web spider
 Spanner
+Spinn3r
 Stripper
-Sucuri
-Surfbot
-Teleport
-Telesphoreo
+SuperHTTP
+Suzuran
+Swiftbot
+TeleportPro
 Telesphorep
-TheNomad
-Thumbor
-TightTwatBot
-Tracemyfile
-Trendictionbot
+The Intraformant
+Trendiction
 True_Robot
-Turnitin
-TwengaBot
-Twice
+Turingos
+TurnitinBot
 Typhoeus
-URLy Warning
-URLy.Warning
+UnisterBot
+VB Project
+VelenPublicWebCrawler
 Virusdie
-VoidEYE
+Voil
 Voltron
-WPScan
+WBSearchBot
+WWW-Collector-E
 WWW-Mechanize
-WWW::Mechanize
+WWWOFFLE
 Wallpapers
 Wallpapers/3.0
-WeSEE
+WallpapersHD
+Web Auto
+Web Collage
+Web Fetch
 Web Pix
 Web Sauger
-Web Sucker
-WebBandit
+WebCollage
 WebCopier
-WebLeacher
-WebPix
-WebStripper
-WebZIP
-Webshag
-WebsiteExtractor
+WebFetch
+WebReaper
+Webalta
 Webster
-Whack
-Who.is Bot
-Widow
-WinHTTrack
-WiseGuys Robot
-Xaldon WebSpider
+Whacker
 Xaldon_WebSpider
-YoudaoBot
+Zade
 Zauba
 Zermelo
 ZmEu
+ZoomBot
+ZoominfoBot
 ZyBorg
-arquivo-web-crawler
-clark-crawler
+arquivo.pt
+autoemailspider
+backlink-check
+cah.io.community
+check1.exe
 coccocbot-web
-cognitiveseo
-crawler.feedback
+crawl.sogou.com
+demandbase-bot
+eCatch
 facebookscraper
+gopher
 internetVista monitor
-lwp-trivial
+isitwp.com
+lwp-request
 magpie-crawler
-mediawords
-netEstate NE Crawler
+meanpathbot
+muhstik-scan
 page scorer
+plumanalytics
 ripz
 s1z.ru
+satoristudio.net
 scalaj-http
-seobility
+scan.lol
 seostar
-sexsearcher
+serpstatbot
 siteripz
-sp_auditbot
-sysscan
+spyfu
 tAkeOut
-trendiction.com
-trendiction.de
-ubermetrics-technologies.com
+voyagerx.com
+x09Mozilla
+xpymep1.exe
 zauba.io