V4.2021.08.2523 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-360Spider
+404checker
+404enemy
+80legs
 ADmantX
+AIBOT
+ALittle Client
 ASPSeek
+Abonti
 Aboundex
 Aboundexbot
-Acunetix
-AlphaBot
-Anarchy
-Anthill
+AhrefsBot
+AiHitBot
+Anarchie
+Apexoo
+Aspiegel
 Asterias
 AwarioRssBot
+BBBike
 BDCbot
-BDFetch
-BackWeb
-Backlink-Ceck
-BacklinkCrawler
+BLEXBot
+BackDoorBot
 Badass
-Bandit
-BatchFTP
-BetaBot
-Bitacle
-BlackWidow
-BlowFish
-Brandwatch
+Black Hole
+Blackboard
+Blow
+Boardreader
+Bolt
+BotALot
+Buck
+Buddy
 BuiltBotTough
-Bullseye
-BunnySlippers
-BuzzSumo
+BuiltWith
 CATExplorador
 CCBot
+CODE87
 Calculon
-CensysInspect
+CazoodleBot
+CheTeam
+CherryPicker
 ChinaClaw
-Chlooe
-Cliqzbot
-Cogentbot
-Collector
-Copier
 Copyscape
-CrazyWebCrawler
-CrunchBot
+Cosmos
+Craftbot
+Curious
+DBLBot
+DIIbot
+DTS Agent
+Demon
 Devil
-Digincore
-Disco
-DnyzBot
-DomainStatsBot
+DigitalPebble
+Discoverybot
+Dispatch
+DittoSpyder
+DomCopBot
+DomainSigmaCrawler
+Dotbot
+Download Wonder
 Drip
-Ebingbong
+EMail Wolf
+Ecxi
 EirGrabber
+EroCrawler
 Exabot
-Express WebPictures
-Extractor
-Ezooms
+ExtLinksBot
 FDM
-FHscan
 Fimap
-FlashGet
-Foobot
-Fyrebot
-GalaxyBot
+Freeuploader
+FrontPage
+GT::WWW
+GermCrawler
+GetRight
 Gigablast
+Gigabot
 Go-Ahead-Got-It
-GrabNet
-HTTP::Lite
-HaosouSpider
+Grabber
+GrapeFX
+GridBot
 Havij
-Heritrix
+Hloader
 Humanlinks
 IDBot
 IlseBot
-IndeedBot
-Indy Library
+Image Sucker
+InfoNaviRobot
 InterGET
 InternetSeer
-Iskanie
-JamesBOT
-Jbrofuzz
-Jetty
 JikeSpider
-Joomla
-Keyword Density
-LNSpiderguy
-LeechFTP
+LWP::Simple
+Lanshanbot
 LeechGet
-Lftp
-LieBaoFast
-Lightspeedsystems
-Linkbot
-Linkdexbot
+LexiBot
+LinkWalker
 LinkextractorPro
+LinqiaMetadataDownloaderBot
 LinqiaRSSBot
-Lipperhey
-Lmspider
 Ltx71
 MFC_Tear_Sample
 MIDown tool
+MJ12bot
 MQQBrowser
-MSFrontPage
-MTRobot
-Mag-Net
-Magnet
-Mail.RU_Bot
 Majestic SEO
-Majestic12
+Majestic-SEO
 MarkWatch
-Mass Downloader
-MauiBot
+Mediatoolkitbot
 MegaIndex.ru
 Metauri
-Mister PiX
+MicroMessenger
+Microsoft Data Access
 Moblie Safari
-Mojeek
-Mojolicious
+Morfeus Fucking Scanner
+Mozlila
 Mr.4x3
+NICErsPRO
+NPbot
 Navroad
-NetAnts
+Needle
+Net Vampire
 NetSpider
-NetZIP
-Netcraft
+Nettrack
 Netvibes
-Nibbler
-Niki-bot
 Nikto
-OnCrawl
-OpenLinkProfiler
-OpenVAS
-Openvas
+Nutch
+Octopus
+Offline Explorer
+Openfind
 OrangeSpider
 OutfoxBot
-Panscient
-PeoplePal
+PageAnalyzer
+PageGrabber
+PageScorer
+Pandalytics
+Papa Foto
 Petalbot
-Pi-Monster
+Picscout
 Picsearch
-PictureFinder
 Pimonster
-Pixray
-PleaseCrawl
+Probethenet
+Psbot
 Pump
 PxBroker
-PyCurl
-Quick-Crawler
-RankActive
+QueryN Metasearch
+RSSingBot
 RankActiveLinkBot
 RankFlex
-RankingBot2
-Re-re
 ReGet
-RealDownload
-RebelMouse
 Recorder
 RedesScrapy
+Ripper
+RocketCrawler
 Rogerbot
-SEOkicks
-SEOkicks-Robot
+SBIder
 SEOlyticsCrawler
-SEOstats
-SISTRIX
 SMTBot
-SalesIntelligent
+ScanAlert
 ScoutJet
-Searchestate
+Scrapy
+Screaming
+ScreenerBot
+ScrepyBot
+SearchmetricsBot
 SentiBot
 Seomoz
 Siphon
 Site Sucker
 SiteExplorer
-SiteSnagger
-SiteSucker
+SiteLockSpider
+Siteimprove
 Sitevigil
+SlySearch
+Snapbot
+Snoopy
+SocialRankIOBot
+Sociscraper
 Sosospider
-Spammen
+Sottopop
+SpaceBison
+SpankBot
 Spanner
+Spbot
+Spinn3r
+SputnikBot
 Sqlworm
+Sqworm
 Sucker
-SuperBot
-Swiftbot
-Telesoft
+TeleportPro
 Telesphorep
+The Intraformant
 TheNomad
+Titan
+Toata
 Toweyabot
-Trendictionbot
-URLy.Warning
-Upflow
-V-BOT
+True_Robot
+UnisterBot
+VCI
 Vacuum
 Vagabondo
-VeriCiteCrawler
+VelenPublicWebCrawler
+VidibleScraper
+Virusdie
 Voil
-WEBDAV
-WWW-Mechanize
+Voltron
+WWW-Collector-E
 WWWOFFLE
 Wallpapers
 Wallpapers/3.0
-WallpapersHD
 WeSEE
 Web Auto
-Web Collage
 Web Enhancer
+Web Sauger
 Web Sucker
-WebBandit
-WebEnhancer
+WebCopier
 WebFuck
 WebGo IS
-WebSauger
-WebStripper
-WebSucker
+WebLeacher
 WebWhacker
-Webster
+WebZIP
+Webshag
 Whack
-Whacker
+Who.is Bot
 WinHTTrack
-Xenu
+Wonderbot
+Woobot
+Wotbox
+Xaldon_WebSpider
 Zade
 Zauba
-Zitebot
-ZmEu
-ZoomBot
+Zermelo
+Zeus
 ZumBot
-ZyBorg
+autoemailspider
 backlink-check
+cah.io.community
 check1.exe
-coccocbot-web
-crawler4j
-demandbase-bot
-domainsproject.org
+cognitiveseo
+crawler.feedback
+facebookscraper
 gopher
-heritrix
-instabid
-internetVista monitor
-mediawords
+lwp-trivial
+magpie-crawler
 muhstik-scan
-netEstate NE Crawler
 oBot
 page scorer
-ripz
-s1z.ru
-satoristudio.net
 scalaj-http
+serpstatbot
 sexsearcher
-sogouspider
-sp_auditbot
-tAkeOut
-trendiction.com
+sysscan
 trendiction.de
-ubermetrics-technologies.com
+voyagerx.com
 webmeup-crawler
-xpymep1.exe
-zgrab
+x22Mozilla