V4.2021.11.2709 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-404enemy
-ASPSeek
+360Spider
+404checker
+Aboundexbot
+Acunetix
 AhrefsBot
 AiHitBot
-Alligator
-Anarchie
-Anarchy99
-Aspiegel
-AwarioSmartBot
+Aipbot
+Alexibot
+AllSubmitter
+AlphaBot
+Anarchy
+Apexoo
+Asterias
+Attach
 BackDoorBot
-BackStreet
 BackWeb
 Backlink-Ceck
+BacklinkCrawler
 Badass
+Bandit
 Barkrowler
-Battleztar Bazinga
+BatchFTP
 BetaBot
-Black Hole
-Blackboard
-Boardreader
+Bigfoot
+Bitacle
+Blow
+BlowFish
+BotALot
 Brandwatch
 Buck
-BuiltBotTough
-Bullseye
-CATExplorador
+Buddy
+BuiltWith
+BunnySlippers
 CCBot
-CSHttp
-Calculon
-CazoodleBot
-CheeseBot
-Cloud mapping
-Cogentbot
+CODE87
+ChinaClaw
+Cliqzbot
+Cocolyzebot
+Copier
 Copyscape
-CyotekWebCopy
+Cosmos
+Craftbot
+Curious
 DBLBot
-DIIbot
+DSearch
+DataCha0s
 DatabaseDriverMysqli
-Devil
+Deusu
 Dirbuster
 Disco
-DomCopBot
-DomainAppender
+DnBCrawler-Analytics
+DomainSigmaCrawler
+DomainStatsBot
+Download Wonder
+ECCP/1.0
 EMail Siphon
+EasyDL
+Evil
 Express WebPictures
-ExtractorPro
-Extreme Picture Finder
 Ezooms
+FDM
+FHscan
 FemtosearchBot
-FlashGet
+Fimap
+Flunky
 Foobot
+Freeuploader
+FrontPage
 Fuzz
-Fyrebot
+FyberSpider
+G-i-g-a-b-o-t
 GT::WWW
-Genieo
-Gigabot
+Go!Zilla
 Go-Ahead-Got-It
-GoZilla
-Gotit
-GridBot
-HMView
-HTTrack
+GrabNet
+Grafula
+GrapeshotCrawler
+HTMLparser
 Haansoft
-Hloader
-HybridBot
-IDBot
-IlseBot
+HaosouSpider
+Harvest
+Iblog
+IndeedBot
 Indy Library
+Internet Ninja
 InternetSeer
 Iria
-IstellaBot
-Jbrofuzz
-JetCar
+JOC Web Spider
 Jetty
-JikeSpider
+Jorgee
+Jyxobot
+Keyword Density
 Kinza
+LWP::Simple
 Lanshanbot
-Larbin
-LeechGet
-LexiBot
-Lftp
+Leap
+LeechFTP
 LibWeb
 Libwhisker
-Lightspeedsystems
-LinkWalker
+Likse
+LinkScan
+Linkbot
 Linkdexbot
 LinkextractorPro
-LinksManager
-Lipperhey Spider
+LinkpadBot
+LinqiaMetadataDownloaderBot
 Litemage_walker
-Lmspider
 MIDown tool
-MIIxpc
-MQQBrowser
-MSIECrawler
-Majestic SEO
+MSFrontPage
+Mag-Net
 MarkMonitor
-Masscan
-Mata Hari
-Microsoft URL Control
+Mb2345Browser
+MeanPath Bot
+Meanpathbot
+Mediatoolkitbot
+Microsoft Data Access
+Mister PiX
 Moblie Safari
-Mojeek
+Mojolicious
 Morfeus Fucking Scanner
-Mozlila
-Nameprotect
-Navroad
-Net Vampire
-NetLyzer
+Msrabot
+Musobot
+NPbot
+Nessus
+NetAnts
+NetMechanic
 NetSpider
-Netcraft
-Nettrack
-Nibbler
-Nimbostratus
-Octopus
+Niki-bot
+NimbleCrawler
+Ninja
+Nmap
+Nutch
 Offline Explorer
 Offline Navigator
-OpenLinkProfiler
-OrangeSpider
-OutclicksBot
+Openvas
 PHPCrawl
-Page Analyzer
-PageThing.com
-Pandalytics
 Panscient
+Pavuk
 PeoplePal
 Petalbot
-Pi-Monster
-Picsearch
-PictureFinder
 Pimonster
 Pixray
 ProPowerBot
 Pu_iN
-QueryN Metasearch
+PyCurl
+Quick-Crawler
 RSSingBot
 RankFlex
-RankingBot
-RankurBot
+RankingBot2
+Rankivabot
 Re-re
-ReGet
 Reaper
-RebelMouse
 RedesScrapy
 RepoMonkey
+Ripper
 RocketCrawler
-Rogerbot
 SBIder
-SEOstats
-SISTRIX
-SMTBot
-ScanAlert
+SEOkicks
 Scanbot
 Screaming
-ScreenerBot
-ScrepyBot
-SentiBot
-SeobilityBot
 Seomoz
-Shodan
 Siphon
-Site Sucker
-SiteLockSpider
-SiteSucker
-Sitebeam
+SiteCheckerBotCrawler
+SiteExplorer
+SiteSnagger
 Siteimprove
-Snake
 Snapbot
 Snoopy
-Sogou web spider
 Sosospider
-SputnikBot
+SpankBot
 Stripper
-Suzuran
+SuperHTTP
+Surfbot
+SurveyBot
+Szukacz
 T0PHackTeam
 T8Abot
-TeleportPro
-Telesoft
-Telesphoreo
-Telesphorep
-The Intraformant
-TheNomad
+Teleport
+Thumbor
+Toata
 Trendiction
-Trendictionbot
-True_Robot
 Turingos
 Turnitin
 TurnitinBot
-Twice
+TwengaBot
 Typhoeus
 URLy.Warning
+Upflow
 V-BOT
+VCI
 Vagabondo
-VidibleScraper
+VeriCiteCrawler
 Virusdie
-Voil
+VoidEYE
+Voltron
 WPScan
-WWW-Mechanize
-WWWOFFLE
-Web Collage
+WWW-Collector-E
+WWW::Mechanize
+Wallpapers/3.0
+WeSEE
+Web Auto
+Web Enhancer
+Web Fuck
 Web Pix
-Web Sauger
 Web Sucker
-WebBandit
-WebEnhancer
-WebGo IS
+WebFetch
 WebPix
-WebStripper
-WebWhacker
-Webalta
-WebmasterWorldForumBot
+WebReaper
+WebZIP
 Webshag
+WebsiteExtractor
 Webster
 Whacker
 Whatweb
-Wprecon
+WinHTTrack
+WiseGuys Robot
+Woobot
+Xaldon WebSpider
 Xenu
+Zade
 Zauba
 Zermelo
+Zeus
+ZmEu
 ZoominfoBot
+adscanner
 archive.org_bot
-autoemailspider
-cah.io.community
-check1.exe
-crawl.sogou.com
-crawler.feedback
+arquivo-web-crawler
+backlink-check
+com.plumanalytics
 evc-batch
 gopher
-heritrix
+instabid
 isitwp.com
-lwp-request
+lwp-trivial
 magpie-crawler
-muhstik-scan
-netEstate NE Crawler
-pcBrowser
-plumanalytics
+page scorer
 polaris version
-s1z.ru
 satoristudio.net
-scalaj-http
-scan.lol
+seobility
 serpstatbot
 sitechecker.pro
-sogouspider
+siteripz
 sp_auditbot
-spyfu
-sysscan
-webmeup-crawler
+trendiction.com
+trendiction.de
+voyagerx.com
+webpros.com
 x09Mozilla
-xpymep1.exe
-zauba.io
+zgrab