V4.2022.05.3125 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
 404checker
 404enemy
-80legs
 ADmantX
 AIBOT
-ALittle Client
-ASPSeek
-Abonti
 Aboundexbot
 AfD-Verbotsverfahren
+AhrefsBot
+Aipbot
 Alexibot
 Alligator
-Apexoo
-Atomseobot
-Attach
+Anarchy99
+Ankit
+Anthill
+Asterias
 AwarioRssBot
-AwarioSmartBot
-BDCbot
+BLEXBot
 BackStreet
-BacklinkCrawler
-Badass
+BackWeb
+Backlink-Ceck
 Bandit
-Barkrowler
 Blackboard
 BlowFish
 Boardreader
 BotALot
 Brandwatch
-Bullseye
-BunnySlippers
+Buck
+Buddy
 BuzzSumo
-CATExplorador
-CCBot
 CODE87
-Cegbfeieh
-CensysInspect
-Claritybot
+CazoodleBot
+CheeseBot
+Chlooe
 Cloud mapping
-Collector
+Cocolyzebot
+Cogentbot
 CopyRightCheck
-Copyscape
+Cosmos
 Craftbot
-Crescent
+Crawling at Home Project
+CrunchBot
 Curious
-CyotekWebCopy
-DBLBot
+Custo
 DIIbot
 DTS Agent
 DatabaseDriverMysqli
-Deusu
-DigitalPebble
-Discobot
-DittoSpyder
+Demon
+Devil
+Digincore
+Disco
+Discoverybot
 DnBCrawler-Analytics
-DnyzBot
-DomainAppender
-DomainCrawler
+DomainSigmaCrawler
 DomainStatsBot
-Domains Project
-Download Wonder
-EMail Siphon
 EasyDL
 Ebingbong
-Ecxi
-Evil
-Express WebPictures
+EroCrawler
+ExtLinksBot
+ExtractorPro
+FDM
+FHscan
 FemtosearchBot
 Fimap
-Firefox/7.0
-Flunky
 Foobot
-GT::WWW
-GermCrawler
+Freeuploader
+FrontPage
+FyberSpider
+Fyrebot
+Getintent
 Go!Zilla
+Gotit
+GrabNet
 GrapeFX
-HMView
-HTMLparser
-HTTP::Lite
-HTTrack
-Haansoft
-Harvest
-HonoluluBot
-HybridBot
-IDBTE4M
-Iblog
-Id-search
+GridBot
+HEADMasterSEO
+HaosouSpider
+Heritrix
 IlseBot
-InfoTekies
-Intelliseek
+Image Sucker
+IndeedBot
 InterGET
-InternetSeer
+Internet Ninja
 Iria
 Iskanie
 IstellaBot
-Jbrofuzz
-JetCar
+JamesBOT
+JennyBot
+Jetty
+JikeSpider
+Joomla
 JustView
+Jyxobot
+Kinza
+Kozmosbot
+LNSpiderguy
+LWP::Simple
 Lanshanbot
-Leap
-LeechFTP
+Larbin
+LeechGet
+LexiBot
 Lftp
-LibWeb
 Libwhisker
-Likse
+LinkWalker
+Linkbot
 LinkextractorPro
-LinkpadBot
-Litemage_walker
+LinqiaScrapeBot
+Lipperhey Spider
+Lmspider
 Ltx71
+MIDown tool
+MIIxpc
+MSFrontPage
 MTRobot
+Mail.RU_Bot
 Majestic-SEO
-Mass Downloader
+MarkMonitor
+Masscan
+MeanPath Bot
 Meanpathbot
-Mediatoolkitbot
 MegaIndex.ru
-Microsoft Data Access
-Mojeek
-Mojolicious
-NICErsPRO
-Name Intelligence
-Nameprotect
-NearSite
-Nessus
-Net Vampire
+Metauri
+Microsoft URL Control
+Moblie Safari
+Mozlila
+Mr.4x3
+NPbot
 NetAnts
 NetLyzer
+NetZIP
+Netcraft
 Netvibes
-Nibbler
-Niki-bot
-Nikto
-Ninja
 Nmap
-Not
+Nuclei
 Nutch
+Octopus
+Offline Explorer
 OpenVAS
-Openfind
-OrangeBot
-OutclicksBot
+Openvas
 PHPCrawl
+POE-Component-Client-HTTP
 Page Analyzer
-PageGrabber
-PageThing.com
+Pandalytics
 Panscient
 Pavuk
-Picscout
-Piepmatz
+Petalbot
+Pi-Monster
+Picsearch
+PictureFinder
+Pixray
 PleaseCrawl
-Pockey
-Pump
-PxBroker
+ProPowerBot
 QueryN Metasearch
-RankingBot2
+Quick-Crawler
+RankurBot
 Re-re
-ReGet
+RealDownload
+RebelMouse
+Recorder
 RedesScrapy
 SEOkicks
-SEOkicks-Robot
+SEOprofiler
 SISTRIX
-ScoutJet
+SMTBot
 Scrapy
-Screaming
 Searchestate
-Seekport
-SemanticJuice
 Semrush
 SeobilityBot
-Shodan
 Siphon
-Site Sucker
 SiteSnagger
-SiteSucker
-Sitebeam
+SmartDownload
 Snapbot
-SocialRankIOBot
-Sociscraper
 Sosospider
-Spbot
-Spinn3r
-Sqlmap
-Sqlworm
+Sqworm
 Steeler
-Sucker
-SuperBot
-SurveyBot
-Swiftbot
-Szukacz
-T0PHackTeam
-Telesphoreo
-The Intraformant
+Sucuri
+Surfbot
+Suzuran
+TeleportPro
+TheNomad
 TightTwatBot
-Toata
-Toweyabot
-Trendiction
-Turingos
+Turnitin
+TurnitinBot
 URLy Warning
-VeriCiteCrawler
-VidibleScraper
-WASALive-Bot
-WISENutbot
-WWWOFFLE
-Web Auto
-Web Fetch
-Web Fuck
+URLy.Warning
+VB Project
+Vacuum
+VelenPublicWebCrawler
+Virusdie
+WEBDAV
+WWW-Collector-E
+WWW-Mechanize
+WWW::Mechanize
+WallpapersHD
 Web Sauger
-WebEnhancer
-WebGo IS
+Web Sucker
+WebCopier
+WebImageCollector
 WebLeacher
+WebPix
 WebStripper
 WebSucker
 WebWhacker
-Webshag
+WebZIP
 Website Quester
-Who.is Bot
+WebsiteQuester
+Whacker
+WinHTTrack
 WiseGuys Robot
-Wprecon
-Xaldon WebSpider
-Xaldon_WebSpider
-Zade
-Zeus
-Zitebot
-ZumBot
-autoemailspider
+Wonderbot
+Woobot
+Xenu
+Zermelo
+adscanner
+arquivo-web-crawler
+arquivo.pt
+check1.exe
+clark-crawler
 coccocbot
+com.plumanalytics
 crawl.sogou.com
-crawler.feedback
-demandbase-bot
 domainsproject.org
 eCatch
 evc-batch
 facebookscraper
 gopher
+heritrix
 instabid
+internetVista monitor
 ips-agent
-isitwp.com
-linkdexbot
-lwp-trivial
+iubenda-radar
+lwp-request
+meanpathbot
+mediawords
+muhstik-scan
 netEstate NE Crawler
-page scorer
-pcBrowser
+plumanalytics
 polaris version
 probe-image-size
+s1z.ru
+satoristudio.net
 scalaj-http
-seobility
 seostar
-serpstatbot
+sexsearcher
 sitechecker.pro
-tAkeOut
+sp_auditbot
+sysscan
+trendiction.de
 webmeup-crawler
+webprosbot
+x09Mozilla
 zauba.io
-zgrab