V4.2022.04.3024 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-404enemy
-ADmantX
-AIBOT
+01h4x.com
+404checker
+80legs
 ASPSeek
-Abonti
+Aboundex
+Aboundexbot
 Acunetix
-AhrefsBot
-AiHitBot
-Aipbot
-AlphaBot
 Anarchy
-Anthill
-Aspiegel
-AwarioRssBot
-BBBike
-BLEXBot
+Anarchy99
+Apexoo
+Asterias
+Atomseobot
+BDFetch
 BackWeb
 Backlink-Ceck
 Bandit
 Barkrowler
-BatchFTP
-BetaBot
+BlackWidow
 Blackboard
-Blow
 BlowFish
-Buck
-Bullseye
-Cegbfeieh
-CheTeam
+Boardreader
+BotALot
+Buddy
+BuiltBotTough
+BuiltWith
+BunnySlippers
+BuzzSumo
+CATExplorador
+CCBot
+CSHttp
+CensysInspect
 CheeseBot
 ChinaClaw
+Chlooe
 Citoid
-Cliqzbot
-Cocolyzebot
-Collector
-Copyscape
+Claritybot
+CopyRightCheck
 Cosmos
-Crawling at Home Project
 Crescent
+DBLBot
+DIIbot
 DSearch
-DTS Agent
-Demon
-DigitalPebble
-Discobot
-Discoverybot
+DataCha0s
+DatabaseDriverMysqli
+Devil
+DittoSpyder
 DomCopBot
-DomainStatsBot
-Download Wonder
-ECCP/1.0
+DomainAppender
+DomainCrawler
+Drip
 EMail Wolf
+EasyDL
 Ebingbong
 Ecxi
-EirGrabber
+Exabot
 Express WebPictures
-ExtLinksBot
-ExtractorPro
+Extractor
 Extreme Picture Finder
-Ezooms
-FDM
+EyeNetIE
 FHscan
 FemtosearchBot
 Fimap
-Firefox/7.0
-FlashGet
-Freeuploader
+Flunky
 FrontPage
-FyberSpider
+Fuzz
+Fyrebot
 G-i-g-a-b-o-t
 GalaxyBot
-GetRight
-Gigabot
+GetWeb
+Go-Ahead-Got-It
 Gotit
-GrabNet
 Grabber
-Grafula
 GrapeFX
 GrapeshotCrawler
+HEADMasterSEO
 HMView
+HTTP::Lite
 HTTrack
 Haansoft
-Harvest
-HonoluluBot
-IDBot
-Id-search
-IlseBot
+Hloader
+Humanlinks
+Iblog
+IndeedBot
 Indy Library
-Intelliseek
-InterGET
-JennyBot
+InfoTekies
+Iskanie
+IstellaBot
+JOC Web Spider
+Jbrofuzz
+JetCar
+JikeSpider
+Joomla
 Jorgee
 JustView
-Keyword Density
-Kozmosbot
+Kenjin Spider
+Keybot Translation-Search-Machine
 LNSpiderguy
-LWP::Simple
 Lanshanbot
-Leap
-Lftp
+LeechFTP
+LeechGet
+LexiBot
+LibWeb
+Libwhisker
+LieBaoFast
+Lightspeedsystems
 LinkScan
-Linkbot
-LinkpadBot
+Linkdexbot
 LinksManager
-Lipperhey
-Litemage_walker
-MFC_Tear_Sample
+LinqiaScrapeBot
+Lmspider
+MIDown tool
 MIIxpc
-MJ12bot
-MTRobot
-Mag-Net
+MSIECrawler
+Magnet
 Majestic SEO
-Majestic12
-MarkWatch
-Masscan
-Meanpathbot
-MegaIndex.ru
+Majestic-SEO
+Mata Hari
+Mb2345Browser
+Mediatoolkitbot
 Metauri
-Microsoft URL Control
-Minefield
-Mojeek
+Mister PiX
+Moblie Safari
+Mojolicious
 MolokaiBot
-Morfeus Fucking Scanner
-Mozlila
-Msrabot
-Name Intelligence
-Nameprotect
-NetSpider
-NetZIP
-Nettrack
+Mr.4x3
+NPbot
+NearSite
+Nessus
+NetLyzer
+NetMechanic
+Netcraft
 Nibbler
+Niki-bot
+Nikto
 NimbleCrawler
-Nimbostratus
-Ninja
 Nmap
-Not
 Nuclei
-Offline Navigator
-OnCrawl
+Nutch
 OpenVAS
-Openvas
-POE-Component-Client-HTTP
-PageThing.com
-Papa Foto
-Pi-Monster
-Picscout
+OutfoxBot
+PHPCrawl
+Pavuk
 Picsearch
 Piepmatz
-Pimonster
-PleaseCrawl
-ProWebWalker
-Probethenet
 Psbot
 Pump
-PxBroker
-RankActiveLinkBot
-Reaper
+RSSingBot
+RealDownload
+RebelMouse
+Recorder
+RepoMonkey
+Ripper
+RocketCrawler
 Rogerbot
 SBIder
-SEOkicks
-SMTBot
+SEOkicks-Robot
+SEOstats
+SISTRIX
+SalesIntelligent
+ScanAlert
 Scrapy
+Screaming
 ScreenerBot
 ScrepyBot
+Seekport
+Siphon
+Site Sucker
 SiteExplorer
-SiteLockSpider
-SiteSnagger
-Sitebeam
-Snake
+SiteSucker
+Sitevigil
 Snapbot
-Sosospider
+Sogou web spider
+SpaceBison
 Spammen
-Sucuri
-SuperHTTP
-Surfbot
-Suzuran
-Swiftbot
-TeleportPro
+Sqlmap
+Sqlworm
+Sqworm
+Stripper
+Sucker
+SurveyBot
+Szukacz
 Telesphoreo
-Telesphorep
+The Intraformant
 Thumbor
-Toata
-Trendictionbot
-Turingos
-URLy.Warning
+Titan
+Trendiction
+Turnitin
+TurnitinBot
+Upflow
 V-BOT
-Vacuum
-VeriCiteCrawler
-VidibleScraper
-VoidEYE
-Voil
-Voltron
-WBSearchBot
+VCI
+Virusdie
 WPScan
 WWW-Collector-E
 WWW-Mechanize
-Wallpapers/3.0
-WallpapersHD
+WWW::Mechanize
 WeSEE
-Web Collage
 Web Fetch
-Web Fuck
+Web Pix
 Web Sauger
-Web Sucker
 WebCollage
 WebCopier
-WebEnhancer
 WebFetch
-WebLeacher
-WebPix
-WebSauger
-WebStripper
-WebWhacker
-WebZIP
-Webshag
-Website Quester
-Webster
-Wonderbot
-Wprecon
+WebFuck
+WebSucker
+WebmasterWorldForumBot
+WebsiteExtractor
+WebsiteQuester
+Whack
+Whacker
+Who.is Bot
+WinHTTrack
+WiseGuys Robot
+Woobot
+Wotbox
+Xenu
+Zermelo
 Zeus
 Zitebot
-ZmEu
 ZoomBot
-ZoominfoBot
+ZumBot
 adscanner
 archive.org_bot
-arquivo.pt
-backlink-check
+arquivo-web-crawler
+autoemailspider
 check1.exe
-com.plumanalytics
-crawler.feedback
-dataforseo.com
-domainsproject.org
-eCatch
-heritrix
+clark-crawler
+crawl.sogou.com
+crawler4j
+evc-batch
+gopher
 internetVista monitor
-iubenda-radar
-oBot
-page scorer
-pcBrowser
+ips-agent
+isitwp.com
+plumanalytics
 polaris version
-ripz
-scalaj-http
-scan.lol
 seobility
+seocompany.store
 seoscanners
-sysscan
-tAkeOut
-ubermetrics-technologies.com
-webprosbot
-xpymep1.exe
+serpstatbot
+sexsearcher
+sitechecker.pro
+siteripz
+sogouspider
+spyfu
+trendiction.de
+webmeup-crawler
+webpros.com
+x22Mozilla
+zauba.io