V4.2021.07.2463 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 80legs
+ADmantX
 AIBOT
+ASPSeek
+Aboundex
 Aboundexbot
 AfD-Verbotsverfahren
+AhrefsBot
 AiHitBot
+Alligator
 AlphaBot
 Anarchy
-Anarchy99
-Aspiegel
-Attach
-BBBike
+Anthill
+Apexoo
+Asterias
+AwarioRssBot
 BDCbot
-BackDoorBot
+BLEXBot
 BackStreet
 BackWeb
 BacklinkCrawler
-Bandit
 BatchFTP
-Blackboard
+BetaBot
+Bigfoot
 BlowFish
-BotALot
-Brandprotect
-Brandwatch
-BuiltBotTough
-Bullseye
-ChinaClaw
-Chlooe
+Bolt
+Buck
+BunnySlippers
+BuzzSumo
+CCBot
+CODE87
+Calculon
+CensysInspect
+Cliqzbot
 Cloud mapping
+Cocolyzebot
 Cogentbot
-Copier
 CopyRightCheck
-Copyscape
 Cosmos
+Craftbot
 CrazyWebCrawler
-Crescent
-Custo
-DSearch
+Curious
+CyotekWebCopy
+DIIbot
 DTS Agent
+DataCha0s
+DatabaseDriverMysqli
 Demon
 Deusu
-DigitalPebble
+Devil
+Digincore
+Dirbuster
+Disco
 Discobot
-Dispatch
-DomCopBot
-DomainAppender
-Domains Project
-Dotbot
-Download Wonder
-ECCP/1.0
+DittoSpyder
+DomainCrawler
+EMail Siphon
 EMail Wolf
 EasyDL
 Ebingbong
-EroCrawler
+Ecxi
 Evil
-ExtLinksBot
-ExtractorPro
-Extreme Picture Finder
-Ezooms
+Express WebPictures
+Extractor
+EyeNetIE
+FDM
 FemtosearchBot
 Fimap
-Firefox/7.0
-Flunky
-Foobot
-Freeuploader
-Fyrebot
+Fuzz
+FyberSpider
 Genieo
-GetRight
-Getintent
-Gigabot
+GetWeb
 Go!Zilla
-Grafula
-GrapeshotCrawler
-Haansoft
-HaosouSpider
-HybridBot
+Go-Ahead-Got-It
+Gotit
+GrabNet
+GrapeFX
+GridBot
+HMView
+IDBot
 IRLbot
-Id-search
 IlseBot
-Image Fetch
-Image Sucker
-Indy Library
-InfoNaviRobot
-Intelliseek
-InternetSeer
+InfoTekies
+InterGET
 Iria
-Jbrofuzz
-JennyBot
-JetCar
+Jetty
 JikeSpider
 Jyxobot
 Kenjin Spider
-Kinza
-Lanshanbot
-Larbin
-LeechGet
-LibWeb
-LinksManager
-LinqiaMetadataDownloaderBot
-MFC_Tear_Sample
+Kozmosbot
+LNSpiderguy
+LeechFTP
+LexiBot
+Lightspeedsystems
+Likse
+LinkpadBot
+Lipperhey Spider
+Ltx71
 MIDown tool
 MIIxpc
 MQQBrowser
+MSFrontPage
 MSIECrawler
-Mail.RU_Bot
-Majestic SEO
-Majestic12
+MTRobot
+Mag-Net
+Magnet
+Mata Hari
 MauiBot
-Mb2345Browser
-MicroMessenger
+Mediatoolkitbot
+MegaIndex.ru
+Microsoft Data Access
 Mister PiX
-Moblie Safari
-Mojeek
-Morfeus Fucking Scanner
 Mr.4x3
-Msrabot
-NPbot
-Navroad
-NetAnts
-NetZIP
+Name Intelligence
+Net Vampire
+NetLyzer
+NetSpider
 Nettrack
 Nibbler
-Niki-bot
+Nikto
 NimbleCrawler
-Nimbostratus
 Ninja
-Nuclei
-Nutch
+Octopus
 Offline Explorer
+Offline Navigator
 OnCrawl
 OpenLinkProfiler
 Openvas
-PHPCrawl
-PageGrabber
+OrangeSpider
+PECL::HTTP
+Page Analyzer
 PageScorer
-Pandalytics
+PageThing.com
+Panscient
+PeoplePal
+Petalbot
 Pi-Monster
+Picsearch
+Piepmatz
 Pimonster
 Pixray
 PleaseCrawl
-Pockey
-ProPowerBot
 Probethenet
-Pu_iN
+Psbot
 PyCurl
+QueryN Metasearch
+Quick-Crawler
+RSSingBot
+RankActiveLinkBot
 RankFlex
-RankingBot
-RankingBot2
-Rankivabot
+RankurBot
 Reaper
-RedesScrapy
+Recorder
 RepoMonkey
 RocketCrawler
-SEOkicks
-SEOprofiler
-SEOstats
 SalesIntelligent
-ScanAlert
-ScoutJet
-Scrapy
+Scanbot
 Screaming
-ScreenerBot
+ScrepyBot
 SearchmetricsBot
+SemrushBot
 SeoSiteCheckup
+SeobilityBot
 Shodan
+Siphon
 Site Sucker
+Siteimprove
 Sitevigil
-Snapbot
+SlySearch
+Snake
 Sociscraper
-SpaceBison
-Spammen
 SpankBot
-Spbot
-Spinn3r
+SputnikBot
 Sqlworm
 Sqworm
-Stripper
-SuperHTTP
+SuperBot
 Suzuran
 Teleport
-TeleportPro
-Telesphorep
 The Intraformant
-Thumbor
-Titan
-Toata
-Toweyabot
+TheNomad
+TightTwatBot
 Trendictionbot
+True_Robot
 Turingos
-Turnitin
-TurnitinBot
+TwengaBot
 Twice
-URLy.Warning
+Typhoeus
+UnisterBot
+Upflow
 V-BOT
 Vacuum
+Vagabondo
 VelenPublicWebCrawler
 VeriCiteCrawler
-VoidEYE
-Voil
-WISENutbot
-WPScan
+VidibleScraper
+Virusdie
+WEBDAV
 WWW-Mechanize
 WWW::Mechanize
-Web Auto
+Web Collage
+Web Enhancer
 Web Fetch
-WebCollage
+WebAuto
 WebCopier
+WebEnhancer
+WebFetch
+WebFuck
 WebImageCollector
-WebStripper
+WebSauger
+WebWhacker
 WebZIP
-Website Quester
 WebsiteExtractor
-Whack
+WebsiteQuester
+Webster
 Whacker
-Wonderbot
+Widow
+WinHTTrack
+WiseGuys Robot
 Wprecon
-Xaldon WebSpider
-Xenu
-Zade
-Zermelo
-Zeus
 Zitebot
-ZumBot
-backlink-check
+ZmEu
+ZoomBot
+ZoominfoBot
 check1.exe
-coccocbot-web
 cognitiveseo
 crawl.sogou.com
-domainsproject.org
+crawler4j
 eCatch
 evc-batch
 instabid
 internetVista monitor
-ips-agent
-magpie-crawler
-mediawords
-netEstate NE Crawler
+isitwp.com
+lwp-request
+meanpathbot
+oBot
+page scorer
 pcBrowser
 plumanalytics
 polaris version
-satoristudio.net
-scalaj-http
+ripz
 seobility
 seostar
-siteripz
+serpstatbot
+sogouspider
 sp_auditbot
+spyfu
+sysscan
 tAkeOut
-trendiction.de
-webmeup-crawler
-x09Mozilla
-zauba.io
+trendiction.com
+x22Mozilla