V4.2021.12.2819 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 360Spider
-404checker
+404enemy
 80legs
-ALittle Client
-ASPSeek
-Aboundexbot
+ADmantX
+Aboundex
+Acunetix
 AfD-Verbotsverfahren
-AiHitBot
-Aipbot
-Alexibot
-AllSubmitter
+AhrefsBot
 Alligator
-Anarchy99
+AlphaBot
+Anarchy
 Anthill
-Aspiegel
-AwarioSmartBot
-BLEXBot
-BackDoorBot
-BackWeb
+AwarioRssBot
+BBBike
+BackStreet
 Backlink-Ceck
-BacklinkCrawler
+Barkrowler
+BatchFTP
 Battleztar Bazinga
 Bigfoot
-BlackWidow
+Bitacle
+Black Hole
+Blackboard
+Blow
 BlowFish
 Bolt
-BotALot
+Brandprotect
 Buddy
-Bullseye
-BunnySlippers
-CCBot
-CSHttp
-CazoodleBot
-Cegbfeieh
-CheeseBot
-CherryPicker
-ChinaClaw
+BuiltBotTough
+CheTeam
+Chlooe
 Citoid
-Claritybot
-Cliqzbot
-Cocolyzebot
+Cogentbot
 CopyRightCheck
 Crescent
-Curious
+Custo
 CyotekWebCopy
-Demon
-Deusu
-Disco
-Discobot
-Discoverybot
+DBLBot
+DSearch
+Digincore
 Dispatch
-DomCopBot
-DomainSigmaCrawler
+DnBCrawler-Analytics
+DomainCrawler
+Domains Project
 Download Wonder
-Drip
-ECCP/1.0
+Dragonfly
+EMail Siphon
 EMail Wolf
+EasyDL
+Ebingbong
 Ecxi
-EroCrawler
 Evil
-ExtLinksBot
-Ezooms
-FlashGet
+ExtractorPro
+EyeNetIE
+FDM
+FemtosearchBot
+Fimap
+Firefox/7.0
 Flunky
-Fyrebot
+Freeuploader
+FyberSpider
+GermCrawler
 GetRight
-GetWeb
-GridBot
-HEADMasterSEO
-HMView
+Gigabot
+GrabNet
+Grabber
+Grafula
 HTTP::Lite
-HTTrack
+Haansoft
+HaosouSpider
 Harvest
 Heritrix
-IDBot
-Id-search
-Image Sucker
-Intelliseek
-InterGET
-InternetSeer
+Hloader
+IDBTE4M
+IlseBot
+Image Fetch
+Indy Library
+InfoTekies
 Iskanie
-IstellaBot
+JOC Web Spider
 JennyBot
 JetCar
-JikeSpider
-Jorgee
-Kenjin Spider
-Keyword Density
-Kozmosbot
-LNSpiderguy
-Lanshanbot
-Larbin
-Leap
-LeechFTP
-LeechGet
-LibWeb
+Joomla
+Kinza
+Lftp
 Libwhisker
-LieBaoFast
-LinkWalker
-Linkdexbot
-LinkextractorPro
-MIIxpc
-MQQBrowser
-Mag-Net
-Mail.RU_Bot
-Majestic-SEO
+Linkbot
+Lipperhey Spider
+Litemage_walker
+Ltx71
+MFC_Tear_Sample
+MSIECrawler
+MTRobot
+Majestic SEO
+MarkWatch
 Mass Downloader
-Masscan
-Mata Hari
-Mb2345Browser
-Metauri
-Microsoft Data Access
-Microsoft URL Control
-Mojeek
+MeanPath Bot
+Mediatoolkitbot
+MicroMessenger
 Mojolicious
-Morfeus Fucking Scanner
-Mozlila
+Mr.4x3
+Msrabot
+NICErsPRO
+NPbot
+Nameprotect
 Needle
+Nessus
+NetAnts
 NetLyzer
-Netcraft
-Nibbler
-Nikto
-Nimbostratus
-Ninja
-Nutch
-Octopus
+NetMechanic
+NetSpider
+Netvibes
+NextGenSearchBot
+NimbleCrawler
 Offline Navigator
+OpenLinkProfiler
 OpenVAS
 Openfind
 Openvas
-OutclicksBot
+OrangeBot
+OrangeSpider
 OutfoxBot
+POE-Component-Client-HTTP
 PageGrabber
 PageThing.com
-Pandalytics
+Papa Foto
+Pavuk
 PeoplePal
-Pi-Monster
 Picsearch
-Piepmatz
-Pimonster
-ProWebWalker
-Probethenet
+Pockey
+ProPowerBot
 Pu_iN
-PxBroker
 PyCurl
+Quick-Crawler
 RSSingBot
-RankFlex
+RankActive
 RankingBot
 RankurBot
-Reaper
-RedesScrapy
-Rogerbot
-SEOkicks
+RealDownload
+Recorder
+RepoMonkey
+RocketCrawler
 SEOkicks-Robot
+SEOstats
 SMTBot
-ScoutJet
+Scrapy
 Screaming
 ScreenerBot
-Searchestate
-SemanticJuice
-SemrushBot
+SearchmetricsBot
 SentiBot
-SeobilityBot
+SeoSiteCheckup
+Shodan
+Siphon
 Site Sucker
-SiteCheckerBotCrawler
-SiteLockSpider
 SiteSnagger
-Siteimprove
-SlySearch
-Snoopy
-Sociscraper
-Sogou web spider
+SiteSucker
+SmartDownload
+Snake
+Snapbot
+SocialRankIOBot
+Sosospider
 SpaceBison
-Spbot
+Spammen
+Spanner
+Spinn3r
 SputnikBot
-Sqlworm
-Sqworm
-Sucker
-SuperBot
-SuperHTTP
+Steeler
+Stripper
 Surfbot
 SurveyBot
+Swiftbot
 Szukacz
-T0PHackTeam
-T8Abot
-Telesoft
-Telesphorep
-The Intraformant
-Thumbor
-Toata
+TheNomad
 Toweyabot
+True_Robot
+Turingos
 Turnitin
-Twice
-V-BOT
+TwengaBot
+UnisterBot
+Upflow
+VCI
 Vacuum
-Vagabondo
-VelenPublicWebCrawler
 VidibleScraper
-Virusdie
-VoidEYE
+Voil
 Voltron
-WASALive-Bot
-WBSearchBot
 WISENutbot
-WWW::Mechanize
+WWW-Collector-E
+WWW-Mechanize
 WWWOFFLE
 Wallpapers
-WebFuck
-WebImageCollector
-WebLeacher
-WebReaper
-WebStripper
+WallpapersHD
+Web Enhancer
+Web Fetch
+Web Fuck
+Web Pix
+Web Sucker
+WebFetch
+WebPix
+WebSauger
+WebSucker
+Webalta
 Website Quester
-WebsiteQuester
-Whatweb
+WebsiteExtractor
+Webster
+Whack
 Who.is Bot
+Widow
+WinHTTrack
+Wonderbot
 Woobot
-Zermelo
-Zeus
+Wprecon
+Xenu
+YoudaoBot
+Zauba
 Zitebot
+ZoomBot
+ZoominfoBot
 ZumBot
 adscanner
-arquivo-web-crawler
-cah.io.community
-check1.exe
-com.plumanalytics
+archive.org_bot
+backlink-check
+clark-crawler
+coccocbot-web
+crawl.sogou.com
+crawler.feedback
+crawler4j
 domainsproject.org
+eCatch
+evc-batch
+facebookscraper
+gopher
+heritrix
+instabid
 internetVista monitor
-ips-agent
-isitwp.com
 lwp-trivial
-magpie-crawler
-mediawords
 muhstik-scan
-pcBrowser
+netEstate NE Crawler
+oBot
 polaris version
-satoristudio.net
-scalaj-http
+probe-image-size
+s1z.ru
 scan.lol
-seobility
 seocompany.store
-seoscanners
 seostar
-sexsearcher
-sogouspider
-sp_auditbot
-webgains-bot
+spyfu
+tAkeOut
 webpros.com
-xpymep1.exe
+webprosbot
+x09Mozilla
 zgrab