V4.2021.08.2521 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
 360Spider
-Aboundex
-Aboundexbot
-Acunetix
-AfD-Verbotsverfahren
-AhrefsBot
-AiHitBot
-Aipbot
-Alexibot
-AllSubmitter
-Alligator
+404checker
+404enemy
+80legs
+ALittle Client
+Abonti
+AlphaBot
 Anarchie
+Anarchy99
+Anthill
+Apexoo
+Aspiegel
 Asterias
-BDCbot
-BDFetch
-BackStreet
-BackWeb
+Attach
+AwarioRssBot
+AwarioSmartBot
+BackDoorBot
+BacklinkCrawler
+Badass
+Bandit
 Barkrowler
-BetaBot
-Bigfoot
-Buck
+BatchFTP
+Battleztar Bazinga
+Blackboard
+Boardreader
+BotALot
+Brandwatch
 Buddy
-BuiltBotTough
 BuiltWith
-Bullseye
-BuzzSumo
+BunnySlippers
+CATExplorador
 CCBot
-CSHttp
-CazoodleBot
+Calculon
 CheeseBot
 CherryPicker
 ChinaClaw
-Cliqzbot
 Cloud mapping
 Collector
-CopyRightCheck
-Cosmos
+Copier
+Copyscape
+CrazyWebCrawler
+Crescent
+CrunchBot
+Custo
 CyotekWebCopy
+DIIbot
+DSearch
 DTS Agent
-DataCha0s
+Demon
 Digincore
-Disco
-DnyzBot
-DomCopBot
+DigitalPebble
+Discobot
 DomainCrawler
+DomainSigmaCrawler
 DomainStatsBot
-Domains Project
-Download Wonder
-Dragonfly
 ECCP/1.0
-Ebingbong
-EroCrawler
-Express WebPictures
-Extreme Picture Finder
-EyeNetIE
-Ezooms
+EMail Siphon
+EasyDL
+Ecxi
+EirGrabber
+Evil
+ExtLinksBot
+Extractor
+ExtractorPro
 FDM
-FlashGet
-Foobot
+FemtosearchBot
+Flunky
 FrontPage
-G-i-g-a-b-o-t
 GT::WWW
-GalaxyBot
-Genieo
-GermCrawler
-Getintent
-Go!Zilla
+GetWeb
+Gigablast
 Go-Ahead-Got-It
-Grabber
-Grafula
-GrapeFX
-GrapeshotCrawler
+Gotit
+GrabNet
 HEADMasterSEO
 HMView
+HTMLparser
 HTTP::Lite
+HTTrack
+Haansoft
 Harvest
-Hloader
-IDBot
+HybridBot
+IlseBot
+Image Fetch
+Image Sucker
 IndeedBot
-Internet Ninja
-JOC Web Spider
-JetCar
-Jetty
+Indy Library
+InfoTekies
+InterGET
+InternetSeer
+Iskanie
+IstellaBot
+Jbrofuzz
+JennyBot
 JikeSpider
-Kenjin Spider
-Keyword Density
-Kozmosbot
+Joomla
+JustView
+Kinza
 LNSpiderguy
-Lanshanbot
 LeechFTP
-LeechGet
-LexiBot
-LibWeb
+Likse
 LinkScan
 LinkextractorPro
-LinkpadBot
-LinksManager
-LinqiaMetadataDownloaderBot
-LinqiaRSSBot
-Lipperhey
+LinqiaScrapeBot
+Lipperhey Spider
+Litemage_walker
+MIDown tool
 MIIxpc
 MQQBrowser
 MSFrontPage
-MSIECrawler
-Mail.RU_Bot
-Majestic-SEO
-Majestic12
-Mata Hari
-Mediatoolkitbot
+Mag-Net
+Mass Downloader
+Mb2345Browser
 Metauri
 MicroMessenger
-Microsoft Data Access
+Mister PiX
 Mojeek
-Mojolicious
-Morfeus Fucking Scanner
+Mozlila
+Mr.4x3
 Musobot
 NICErsPRO
-Name Intelligence
+NPbot
 NearSite
-Needle
-Nessus
+Net Vampire
 NetAnts
-Nettrack
-Nimbostratus
-Nmap
-Octopus
-Offline Navigator
-Openfind
-Openvas
-PHPCrawl
+NetLyzer
+Netvibes
+NextGenSearchBot
+Niki-bot
+Nikto
+Ninja
+Nuclei
+Nutch
+Offline Explorer
+OrangeBot
+OrangeSpider
+OutclicksBot
+OutfoxBot
 POE-Component-Client-HTTP
-PageGrabber
+PageAnalyzer
+PageScorer
+PageThing.com
 Pandalytics
-Panscient
 Papa Foto
-Pi-Monster
-Picsearch
-Pixray
+Petalbot
+Picscout
+Piepmatz
+Pimonster
 PleaseCrawl
 Pockey
-ProWebWalker
-Psbot
+Probethenet
+Pump
 PxBroker
 QueryN Metasearch
+RankActive
+RankFlex
+RankingBot
+Rankivabot
 RankurBot
-Re-re
-ReGet
-Recorder
-Ripper
-RocketCrawler
-Rogerbot
-SBIder
-SEOkicks-Robot
-Screaming
+Reaper
+RepoMonkey
+SEOkicks
+ScoutJet
 ScreenerBot
-ScrepyBot
-Searchestate
 SemrushBot
-Site Sucker
-SiteExplorer
+SentiBot
+SeoSiteCheckup
+SiteCheckerBotCrawler
 SiteSnagger
-SlySearch
-Snake
-Snoopy
-SocialRankIOBot
+SiteSucker
+SmartDownload
 Sociscraper
-Spammen
-Spanner
-SputnikBot
-Sqlworm
-Sqworm
-Steeler
-Suzuran
-T8Abot
-TeleportPro
-Telesoft
+Sosospider
+Sottopop
+SpankBot
+Spinn3r
+Sucuri
+SuperBot
+T0PHackTeam
 Telesphoreo
-Telesphorep
-The Intraformant
 TheNomad
-TightTwatBot
+Titan
 Toweyabot
-Tracemyfile
+Trendictionbot
+True_Robot
+Turingos
+Turnitin
 TurnitinBot
-TwengaBot
 Twice
 Typhoeus
-URLy Warning
-V-BOT
-Vacuum
+URLy.Warning
+VB Project
+VCI
 VeriCiteCrawler
-VidibleScraper
 VoidEYE
+Voltron
+WASALive-Bot
 WBSearchBot
-WISENutbot
-WPScan
-WWW-Collector-E
-Wallpapers/3.0
+WWWOFFLE
 Web Auto
 Web Enhancer
-Web Pix
-WebBandit
-WebEnhancer
+Web Fuck
+WebCollage
+WebCopier
+WebFetch
+WebImageCollector
 WebPix
 WebReaper
-WebSucker
-WebmasterWorldForumBot
-Website Quester
+WebWhacker
+Webalta
+Webshag
 WebsiteQuester
-Who.is Bot
+Webster
+Whack
 WiseGuys Robot
-Wotbox
+Wonderbot
 Wprecon
-Xaldon WebSpider
+Zade
 Zauba
+Zermelo
 Zeus
 Zitebot
-ZmEu
 ZoomBot
-arquivo-web-crawler
-arquivo.pt
+ZumBot
+ZyBorg
+autoemailspider
+backlink-check
+check1.exe
 coccocbot-web
-com.plumanalytics
-crawl.sogou.com
-crawler4j
-demandbase-bot
+cognitiveseo
+domainsproject.org
+eCatch
 evc-batch
-ips-agent
-lwp-trivial
+heritrix
+internetVista monitor
 magpie-crawler
-mediawords
-plumanalytics
+meanpathbot
+netEstate NE Crawler
+oBot
+pcBrowser
 probe-image-size
 s1z.ru
-satoristudio.net
 scalaj-http
 scan.lol
-seobility
-seostar
-serpstatbot
-sexsearcher
-sp_auditbot
-spyfu
+siteripz
 sysscan
 tAkeOut
+trendiction.com
 voyagerx.com
-webmeup-crawler
-zauba.io
 zgrab