V4.2021.11.2698 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,248 +1,248 @@
-404enemy
+01h4x.com
+404checker
 80legs
-Aboundexbot
-Alexibot
-AllSubmitter
+ADmantX
+AIBOT
+Abonti
+AfD-Verbotsverfahren
 Anarchie
+Ankit
 Anthill
-BDCbot
+Aspiegel
+Attach
+AwarioSmartBot
 BackDoorBot
 BackStreet
 BackWeb
+Backlink-Ceck
 BacklinkCrawler
+Badass
+Bandit
 Barkrowler
 Battleztar Bazinga
 BetaBot
 Bitacle
-Blackboard
-Blow
-BotALot
-Brandprotect
-BuiltWith
+Black Hole
+BlackWidow
+BlowFish
+Buck
 Bullseye
 CCBot
-Calculon
-CazoodleBot
+CODE87
 Cegbfeieh
+CheTeam
+CheeseBot
+CherryPicker
 ChinaClaw
+Chlooe
 Cloud mapping
+Cocolyzebot
+Collector
 Copier
+CopyRightCheck
+Copyscape
 Cosmos
 Craftbot
 Crescent
 CrunchBot
-Custo
-DTS Agent
+CyotekWebCopy
+DIIbot
+DSearch
+DatabaseDriverMysqli
 Demon
 Deusu
 Discobot
-Dispatch
 DittoSpyder
-DnyzBot
-Dotbot
-Download Wonder
-Drip
-EMail Siphon
+DnBCrawler-Analytics
+DomCopBot
+DomainCrawler
+DomainSigmaCrawler
 EMail Wolf
-EasyDL
 Ebingbong
-EirGrabber
-Exabot
-Extractor
-EyeNetIE
-Fimap
+EroCrawler
+Express WebPictures
 Firefox/7.0
-Foobot
-FrontPage
-Fuzz
+G-i-g-a-b-o-t
 GT::WWW
 GalaxyBot
-GetRight
-GetWeb
+Genieo
+Getintent
 Gigabot
-Go!Zilla
 Go-Ahead-Got-It
-GoZilla
-GrabNet
-Grabber
-Grafula
+GrapeshotCrawler
 GridBot
-HaosouSpider
+HEADMasterSEO
 Heritrix
 Humanlinks
-HybridBot
+IDBTE4M
 IRLbot
-IlseBot
 Indy Library
-InfoNaviRobot
-Intelliseek
-Internet Ninja
-InternetSeer
-Iria
+InterGET
 Iskanie
 IstellaBot
-Jbrofuzz
+JOC Web Spider
+JamesBOT
 JennyBot
-JetCar
-JustView
-Kenjin Spider
-Keyword Density
-Kozmosbot
-LNSpiderguy
-LWP::Simple
-Lanshanbot
-LeechFTP
-LexiBot
-LibWeb
-LieBaoFast
+Jetty
+JikeSpider
+Jorgee
+Kinza
+Larbin
+LeechGet
+Libwhisker
 Lightspeedsystems
+Likse
 Linkbot
 LinkextractorPro
-LinqiaScrapeBot
-Lipperhey
-Lipperhey Spider
+LinkpadBot
+LinksManager
+LinqiaRSSBot
 Lmspider
 Ltx71
+MFC_Tear_Sample
+MIIxpc
 MSFrontPage
 MTRobot
 Mag-Net
-Majestic SEO
-Majestic12
-MarkWatch
+MarkMonitor
 Mass Downloader
-Mata Hari
-MeanPath Bot
-Meanpathbot
-Metauri
+MauiBot
+MegaIndex.ru
+MicroMessenger
 Microsoft URL Control
+Moblie Safari
 Morfeus Fucking Scanner
-Msrabot
+Mozlila
 NICErsPRO
 Name Intelligence
-Navroad
+NearSite
+Needle
 NetAnts
 NetLyzer
-NetMechanic
-Netcraft
+NetZIP
+Nettrack
 Netvibes
+NextGenSearchBot
 Nibbler
 NimbleCrawler
-Nuclei
+Nimbostratus
+Ninja
 Nutch
 Octopus
-Offline Explorer
-OpenVAS
+OpenLinkProfiler
+Openfind
+OrangeBot
 OrangeSpider
 OutclicksBot
-Page Analyzer
+POE-Component-Client-HTTP
 PageGrabber
 PageScorer
-Pavuk
-Picsearch
-PleaseCrawl
+PageThing.com
+Panscient
+Picscout
 ProWebWalker
 Probethenet
-Psbot
+Pu_iN
 Pump
-PxBroker
-PyCurl
-RankActive
+Quick-Crawler
 RankFlex
-RankingBot
-RankingBot2
 Rankivabot
-RealDownload
-Reaper
+Re-re
+ReGet
 RebelMouse
-RedesScrapy
-SEOkicks
-SEOkicks-Robot
-SISTRIX
+RepoMonkey
+Ripper
+SMTBot
 ScanAlert
-ScoutJet
+Screaming
 Searchestate
-SearchmetricsBot
-Semrush
-SeoSiteCheckup
+SentiBot
 SeobilityBot
+Seomoz
+Site Sucker
 SiteCheckerBotCrawler
-SiteLockSpider
 SiteSucker
-Sitebeam
-Snapbot
-SocialRankIOBot
+Siteimprove
+Sitevigil
+Snake
+Snoopy
+Sociscraper
 Sogou web spider
-Sosospider
 Sottopop
-SpaceBison
-SputnikBot
-Sqlworm
+SpankBot
+Spanner
+Sqlmap
+Steeler
 Sucuri
-SuperBot
-Surfbot
+SurveyBot
 Suzuran
 Swiftbot
+Szukacz
 T0PHackTeam
-Telesphoreo
-The Intraformant
+Teleport
+TeleportPro
+Telesoft
 TheNomad
-Titan
-Tracemyfile
+Thumbor
+TightTwatBot
+Trendiction
+True_Robot
+Turnitin
 TurnitinBot
-URLy Warning
-URLy.Warning
-VB Project
+TwengaBot
+VCI
 VelenPublicWebCrawler
+VeriCiteCrawler
 VoidEYE
-WEBDAV
+Voil
+WWW-Collector-E
+WWW-Mechanize
 WWW::Mechanize
-WWWOFFLE
-Wallpapers/3.0
-Web Auto
-Web Collage
-Web Enhancer
-Web Sauger
-WebBandit
-WebFetch
-WebFuck
-WebGo IS
+Wallpapers
+WeSEE
+Web Fuck
+Web Sucker
+WebCopier
+WebEnhancer
 WebImageCollector
-WebZIP
+WebSauger
+WebStripper
+WebSucker
+WebWhacker
 WebmasterWorldForumBot
-Website Quester
-WebsiteExtractor
-Whacker
+Webster
+Whack
 Whatweb
-Who.is Bot
-Widow
-WiseGuys Robot
+WinHTTrack
 Wonderbot
 Woobot
 Xaldon_WebSpider
-YoudaoBot
+Xenu
+Zade
 Zauba
-Zermelo
-Zeus
-Zitebot
-arquivo-web-crawler
-check1.exe
-crawler.feedback
+ZyBorg
+backlink-check
 crawler4j
+dataforseo.com
+demandbase-bot
 domainsproject.org
-gopher
+facebookscraper
 heritrix
+internetVista monitor
 ips-agent
-mediawords
+lwp-trivial
+magpie-crawler
 pcBrowser
 plumanalytics
+s1z.ru
 scalaj-http
-sexsearcher
-sitechecker.pro
+seoscanners
 siteripz
-sogouspider
 sp_auditbot
 sysscan
-voyagerx.com
+trendiction.de
+ubermetrics-technologies.com
+webmeup-crawler
 webprosbot
 x09Mozilla
 xpymep1.exe