V4.2022.04.3037 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-404enemy
-80legs
-ADmantX
-AIBOT
-ALittle Client
 Abonti
 Acunetix
 AfD-Verbotsverfahren
-AhrefsBot
 AiHitBot
-Alligator
-Anarchie
-Anarchy
-Anarchy99
+Alexibot
+AllSubmitter
 Anthill
 Apexoo
-Asterias
+Aspiegel
 Atomseobot
-Attach
 AwarioRssBot
 BBBike
 BDFetch
 BLEXBot
 BackDoorBot
-BatchFTP
-BetaBot
-BlackWidow
+Bitacle
 Blackboard
-Brandprotect
-Buddy
-BuiltWith
-BunnySlippers
+BlowFish
+Bolt
+BuzzSumo
 CCBot
+CODE87
 CSHttp
 Calculon
-Cegbfeieh
-CherryPicker
-Collector
-Cosmos
-Craftbot
-CrazyWebCrawler
+CheTeam
+Cliqzbot
+Cogentbot
+Copier
+Copyscape
 Crescent
-CrunchBot
-DBLBot
+Curious
+Custo
+CyotekWebCopy
+DIIbot
+DSearch
 DTS Agent
-DatabaseDriverMysqli
-Demon
-Disco
+Digincore
+Dirbuster
+Discobot
+Discoverybot
 Dispatch
-DittoSpyder
-DomainAppender
-DomainStatsBot
-Dragonfly
+DnBCrawler-Analytics
+Dotbot
+Drip
+EasyDL
+Ebingbong
 Ecxi
-Evil
-Express WebPictures
+EroCrawler
+Exabot
+Extractor
+Extreme Picture Finder
 Ezooms
 FDM
-FemtosearchBot
-Fimap
+FHscan
 Flunky
-FrontPage
-Fuzz
+Foobot
+Freeuploader
 FyberSpider
-G-i-g-a-b-o-t
-GT::WWW
+Fyrebot
 GalaxyBot
-GermCrawler
-GetRight
-GetWeb
+Genieo
 Gigabot
 Go-Ahead-Got-It
-GoZilla
 Gotit
-GrabNet
+Grabber
+Grafula
 GrapeFX
 GridBot
+HEADMasterSEO
 HTMLparser
+HTTP::Lite
+HTTrack
 HaosouSpider
+Harvest
+Humanlinks
 IDBTE4M
-Id-search
 Image Sucker
 Indy Library
-InfoNaviRobot
 InterGET
-Internet Ninja
+InternetSeer
+Iskanie
 IstellaBot
-JennyBot
-JetCar
-Joomla
-JustView
-Kinza
-LWP::Simple
-Larbin
-LexiBot
+Kozmosbot
+Lanshanbot
+LeechGet
+LibWeb
 Libwhisker
-LinkScan
+LinkWalker
 Linkbot
-LinkpadBot
+LinkextractorPro
+LinksManager
 Lipperhey
-Lipperhey Spider
 Litemage_walker
-MJ12bot
+Ltx71
+MIDown tool
+MIIxpc
 MSIECrawler
-Mail.RU_Bot
-Majestic SEO
-Majestic12
-MarkMonitor
+MTRobot
+Mag-Net
+Magnet
+Majestic-SEO
 MarkWatch
-Mass Downloader
-Mata Hari
-MauiBot
 MeanPath Bot
+Mediatoolkitbot
 MegaIndex.ru
-Metauri
-Microsoft Data Access
-Minefield
-Mister PiX
-Mojolicious
-Morfeus Fucking Scanner
+MicroMessenger
+Mojeek
+MolokaiBot
+Mr.4x3
+Msrabot
+Musobot
 NPbot
+Nameprotect
+NearSite
+Needle
+NetAnts
 NetLyzer
-NetMechanic
-NetSpider
-Netvibes
+Nettrack
 Nibbler
-Niki-bot
 Nimbostratus
 Ninja
+Not
 Nuclei
 Octopus
 Offline Navigator
 OpenVAS
-Openfind
-OrangeSpider
-OutclicksBot
-OutfoxBot
-PECL::HTTP
-PageGrabber
-PageScorer
-PageThing.com
+OrangeBot
+PHPCrawl
+POE-Component-Client-HTTP
+PageAnalyzer
+Pandalytics
+Panscient
 Papa Foto
 Pavuk
-Petalbot
-Pockey
+PeoplePal
+Picscout
+Picsearch
+Piepmatz
+ProPowerBot
 Psbot
+Pu_iN
 PxBroker
 QueryN Metasearch
+Quick-Crawler
+RankFlex
+Rankivabot
 Re-re
 ReGet
-Recorder
-RedesScrapy
-RepoMonkey
-Rogerbot
 SEOkicks
-SEOprofiler
+SEOlyticsCrawler
 SMTBot
 ScanAlert
-Scanbot
-Scrapy
+ScoutJet
 ScreenerBot
-ScrepyBot
-Searchestate
+SearchmetricsBot
 Seekport
 SemrushBot
-SeoSiteCheckup
-SeobilityBot
-Site Sucker
-SiteCheckerBotCrawler
+SentiBot
+Seomoz
+Shodan
+Siphon
+SiteExplorer
 SiteLockSpider
-Sitebeam
-Siteimprove
+SiteSnagger
+SiteSucker
+SmartDownload
 Snake
 Snoopy
+SocialRankIOBot
+Sogou web spider
+Spammen
+SpankBot
+Spanner
 Spbot
-Spinn3r
-SputnikBot
+Sqworm
 Stripper
-Sucker
+SuperBot
+SuperHTTP
 Surfbot
 Suzuran
-T0PHackTeam
-T8Abot
-Teleport
+Szukacz
 TeleportPro
-Telesphoreo
+TheNomad
 Thumbor
-TightTwatBot
-Toweyabot
+Titan
 Tracemyfile
-Trendictionbot
+Trendiction
 True_Robot
 Turnitin
 URLy Warning
 Upflow
-VelenPublicWebCrawler
-WASALive-Bot
-WBSearchBot
-WISENutbot
+VB Project
+Vacuum
+VeriCiteCrawler
+Virusdie
+Voltron
 WWWOFFLE
-WallpapersHD
-WeSEE
+Wallpapers
+Web Enhancer
 Web Fuck
-Web Sauger
+Web Pix
 Web Sucker
-WebBandit
-WebEnhancer
-WebFetch
-WebImageCollector
-WebPix
+WebAuto
+WebCollage
+WebCopier
 WebReaper
-WebSauger
+WebStripper
 WebSucker
-WebZIP
-Webshag
-WebsiteExtractor
+WebmasterWorldForumBot
 Webster
-Whacker
-Widow
-Xaldon WebSpider
-Xenu
-Zauba
-Zermelo
+WiseGuys Robot
+Woobot
+Wotbox
+Xaldon_WebSpider
+YoudaoBot
+Zeus
+Zitebot
 ZoomBot
+ZumBot
 ZyBorg
-adscanner
-arquivo.pt
+arquivo-web-crawler
 autoemailspider
 backlink-check
+coccocbot
+cognitiveseo
 crawl.sogou.com
-crawler.feedback
+crawler4j
 demandbase-bot
+domainsproject.org
 facebookscraper
-heritrix
-isitwp.com
-linkdexbot
-lwp-request
+gopher
+instabid
+ips-agent
+iubenda-radar
 lwp-trivial
+magpie-crawler
 mediawords
-muhstik-scan
-netEstate NE Crawler
 oBot
-polaris version
-probe-image-size
-scalaj-http
-scan.lol
-serpstatbot
-spyfu
-xpymep1.exe
+page scorer
+ripz
+satoristudio.net
+seocompany.store
+seostar
+sogouspider
+sp_auditbot
+sysscan
+trendiction.com
+voyagerx.com
+webmeup-crawler
+webprosbot