V4.2021.11.2699 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404checker
-80legs
-ADmantX
-AIBOT
+ASPSeek
 Abonti
+Acunetix
 AfD-Verbotsverfahren
-Anarchie
+Anarchy
 Ankit
-Anthill
+Apexoo
 Aspiegel
+Asterias
 Attach
-AwarioSmartBot
-BackDoorBot
-BackStreet
-BackWeb
-Backlink-Ceck
-BacklinkCrawler
+BBBike
+BDCbot
 Badass
 Bandit
 Barkrowler
-Battleztar Bazinga
 BetaBot
+Bigfoot
 Bitacle
-Black Hole
-BlackWidow
-BlowFish
+Blackboard
+Boardreader
 Buck
-Bullseye
-CCBot
 CODE87
-Cegbfeieh
-CheTeam
+Calculon
+CazoodleBot
 CheeseBot
 CherryPicker
-ChinaClaw
-Chlooe
+Claritybot
 Cloud mapping
-Cocolyzebot
-Collector
 Copier
 CopyRightCheck
 Copyscape
 Cosmos
-Craftbot
+Crawling at Home Project
+CrazyWebCrawler
 Crescent
-CrunchBot
-CyotekWebCopy
-DIIbot
+Custo
 DSearch
+DataCha0s
 DatabaseDriverMysqli
-Demon
-Deusu
 Discobot
-DittoSpyder
+Discoverybot
+Dispatch
 DnBCrawler-Analytics
+DnyzBot
 DomCopBot
-DomainCrawler
-DomainSigmaCrawler
+DomainStatsBot
+Domains Project
+Dotbot
+Drip
+EMail Siphon
 EMail Wolf
-Ebingbong
-EroCrawler
+Exabot
 Express WebPictures
-Firefox/7.0
-G-i-g-a-b-o-t
+Extractor
+ExtractorPro
+EyeNetIE
+FHscan
+FemtosearchBot
+Fimap
+FlashGet
+Foobot
+Freeuploader
+FrontPage
+Fuzz
+FyberSpider
+Fyrebot
 GT::WWW
-GalaxyBot
-Genieo
+GermCrawler
+GetWeb
 Getintent
 Gigabot
 Go-Ahead-Got-It
-GrapeshotCrawler
-GridBot
+GoZilla
+GrabNet
+Grafula
 HEADMasterSEO
+HTMLparser
+HTTP::Lite
+Haansoft
 Heritrix
-Humanlinks
+HybridBot
 IDBTE4M
-IRLbot
+Id-search
+Image Fetch
+IndeedBot
 Indy Library
-InterGET
-Iskanie
+InfoTekies
+Intelliseek
+Internet Ninja
+Iria
 IstellaBot
-JOC Web Spider
 JamesBOT
-JennyBot
-Jetty
-JikeSpider
 Jorgee
 Kinza
-Larbin
+Kozmosbot
+LWP::Simple
+LeechFTP
 LeechGet
-Libwhisker
-Lightspeedsystems
 Likse
+LinkScan
 Linkbot
+Linkdexbot
 LinkextractorPro
 LinkpadBot
-LinksManager
 LinqiaRSSBot
-Lmspider
-Ltx71
-MFC_Tear_Sample
+LinqiaScrapeBot
+Lipperhey
+Lipperhey Spider
 MIIxpc
-MSFrontPage
+MJ12bot
+MSIECrawler
 MTRobot
 Mag-Net
-MarkMonitor
-Mass Downloader
+Magnet
+Mail.RU_Bot
 MauiBot
-MegaIndex.ru
-MicroMessenger
-Microsoft URL Control
 Moblie Safari
-Morfeus Fucking Scanner
 Mozlila
-NICErsPRO
-Name Intelligence
+Mr.4x3
+Musobot
 NearSite
-Needle
-NetAnts
+Nessus
+Net Vampire
 NetLyzer
-NetZIP
 Nettrack
-Netvibes
 NextGenSearchBot
 Nibbler
+Niki-bot
+Nikto
 NimbleCrawler
-Nimbostratus
 Ninja
 Nutch
-Octopus
 OpenLinkProfiler
-Openfind
-OrangeBot
-OrangeSpider
-OutclicksBot
-POE-Component-Client-HTTP
-PageGrabber
-PageScorer
-PageThing.com
+OutfoxBot
+PHPCrawl
+Page Analyzer
+Pandalytics
 Panscient
+Papa Foto
+Pavuk
+PeoplePal
+Pi-Monster
 Picscout
 ProWebWalker
-Probethenet
+Psbot
 Pu_iN
-Pump
-Quick-Crawler
-RankFlex
-Rankivabot
-Re-re
-ReGet
-RebelMouse
-RepoMonkey
-Ripper
-SMTBot
+PxBroker
+PyCurl
+RSSingBot
+RankActiveLinkBot
+RankingBot
+RankurBot
+RedesScrapy
+SBIder
+SEOkicks
+SEOprofiler
+SISTRIX
 ScanAlert
-Screaming
-Searchestate
-SentiBot
+ScoutJet
+Scrapy
+ScreenerBot
+ScrepyBot
+SeoSiteCheckup
 SeobilityBot
 Seomoz
 Site Sucker
-SiteCheckerBotCrawler
-SiteSucker
+SiteLockSpider
 Siteimprove
-Sitevigil
-Snake
-Snoopy
+Snapbot
 Sociscraper
-Sogou web spider
 Sottopop
-SpankBot
 Spanner
-Sqlmap
+Spinn3r
+SputnikBot
 Steeler
-Sucuri
-SurveyBot
+Stripper
+Sucker
+SuperHTTP
 Suzuran
-Swiftbot
-Szukacz
 T0PHackTeam
 Teleport
 TeleportPro
-Telesoft
+Telesphorep
 TheNomad
-Thumbor
-TightTwatBot
+Titan
+Tracemyfile
 Trendiction
-True_Robot
-Turnitin
-TurnitinBot
-TwengaBot
-VCI
+Trendictionbot
+URLy Warning
+Upflow
+V-BOT
 VelenPublicWebCrawler
-VeriCiteCrawler
-VoidEYE
+VidibleScraper
 Voil
-WWW-Collector-E
-WWW-Mechanize
+WBSearchBot
 WWW::Mechanize
-Wallpapers
-WeSEE
+Web Auto
 Web Fuck
-Web Sucker
+Web Pix
+WebCollage
 WebCopier
-WebEnhancer
+WebFetch
+WebFuck
+WebGo IS
 WebImageCollector
+WebPix
 WebSauger
-WebStripper
-WebSucker
 WebWhacker
-WebmasterWorldForumBot
+WebZIP
+Webalta
+WebsiteExtractor
 Webster
-Whack
+Whacker
 Whatweb
-WinHTTrack
 Wonderbot
-Woobot
+Wotbox
+Wprecon
 Xaldon_WebSpider
-Xenu
 Zade
-Zauba
-ZyBorg
-backlink-check
+ZoomBot
+ZumBot
+arquivo.pt
+autoemailspider
+coccocbot-web
+cognitiveseo
+com.plumanalytics
 crawler4j
-dataforseo.com
-demandbase-bot
-domainsproject.org
-facebookscraper
+evc-batch
 heritrix
-internetVista monitor
-ips-agent
+instabid
+isitwp.com
 lwp-trivial
 magpie-crawler
-pcBrowser
+page scorer
 plumanalytics
-s1z.ru
+polaris version
+probe-image-size
+satoristudio.net
 scalaj-http
 seoscanners
 siteripz
 sp_auditbot
-sysscan
 trendiction.de
 ubermetrics-technologies.com
-webmeup-crawler
-webprosbot
+webpros.com
 x09Mozilla
+x22Mozilla
 xpymep1.exe
-zauba.io
 zgrab