V4.2021.08.2502 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-360Spider
 404checker
 404enemy
 AIBOT
-Abonti
 Aboundexbot
 AfD-Verbotsverfahren
 AhrefsBot
-AiHitBot
-AllSubmitter
 Alligator
 AlphaBot
-Ankit
+Anarchie
+Anarchy99
+Anthill
 Asterias
+Attach
+AwarioRssBot
+BDCbot
+BLEXBot
+BackStreet
+Backlink-Ceck
+Badass
 Barkrowler
-BatchFTP
-Bitacle
+Battleztar Bazinga
+Bigfoot
 Black Hole
-Blackboard
+Blow
 BlowFish
-Bolt
 BotALot
-Brandprotect
 BuiltBotTough
-Bullseye
-BunnySlippers
+BuiltWith
 BuzzSumo
-CODE87
-Calculon
-Cegbfeieh
-CensysInspect
+CATExplorador
+CSHttp
 CheeseBot
-ChinaClaw
+CherryPicker
+Chlooe
+Claritybot
 Cliqzbot
+Cloud mapping
 Cocolyzebot
 Collector
-Copier
-Copyscape
 Cosmos
 Craftbot
-CrazyWebCrawler
-CrunchBot
 Curious
-DataCha0s
-DatabaseDriverMysqli
+CyotekWebCopy
+DSearch
+DTS Agent
 Deusu
 DigitalPebble
 Dirbuster
+Discobot
+Discoverybot
 Dispatch
-DittoSpyder
-DomCopBot
+DnyzBot
+DomainAppender
 Dotbot
 EMail Siphon
 EMail Wolf
-EasyDL
 Ebingbong
-Ezooms
-FDM
-FHscan
+Ecxi
+EroCrawler
+Evil
+ExtLinksBot
+Extreme Picture Finder
 Firefox/7.0
-FlashGet
-Fyrebot
-GermCrawler
+Foobot
+Freeuploader
+FrontPage
+G-i-g-a-b-o-t
+GT::WWW
+GalaxyBot
+Genieo
+GetRight
+GetWeb
+Gigablast
 Go-Ahead-Got-It
-GoZilla
-Gotit
-GrabNet
-GrapeFX
-GridBot
+Grafula
 HEADMasterSEO
-HMView
 HTMLparser
-HTTrack
-Haansoft
-Havij
-Hloader
+HTTP::Lite
+HaosouSpider
+Harvest
+Heritrix
 IDBTE4M
-IlseBot
-Image Fetch
-Image Sucker
-Indy Library
+IDBot
+Id-search
+InfoNaviRobot
 InfoTekies
+Intelliseek
 InterGET
 Internet Ninja
-Iria
+InternetSeer
+Iskanie
+IstellaBot
 Jbrofuzz
 JetCar
+Jetty
 Jorgee
-Jyxobot
-Kinza
-Lanshanbot
-Larbin
+Kenjin Spider
 LeechFTP
-LieBaoFast
+LeechGet
+LexiBot
+LibWeb
+Lightspeedsystems
+Likse
+LinkScan
 LinkWalker
-LinkpadBot
-MJ12bot
+Linkdexbot
+LinqiaRSSBot
+LinqiaScrapeBot
+Lipperhey Spider
+Litemage_walker
+Lmspider
+MFC_Tear_Sample
+MQQBrowser
 MTRobot
+Magnet
 Majestic SEO
 Majestic-SEO
-MeanPath Bot
+MarkMonitor
+MarkWatch
+Mass Downloader
+Mata Hari
+MauiBot
+Mb2345Browser
 Meanpathbot
 Mediatoolkitbot
-Microsoft Data Access
-Microsoft URL Control
-Mister PiX
-Mojeek
+Metauri
+Moblie Safari
 Morfeus Fucking Scanner
-Mr.4x3
 Msrabot
-Musobot
-NPbot
-Nessus
-NetAnts
-Ninja
-Nmap
-Nutch
+NearSite
+Net Vampire
+NetMechanic
+NetZIP
+NimbleCrawler
 Octopus
-Offline Explorer
+OpenLinkProfiler
 OpenVAS
-Openfind
+Openvas
 OrangeSpider
 OutclicksBot
-PageAnalyzer
-Pandalytics
+OutfoxBot
+POE-Component-Client-HTTP
+PageScorer
+PageThing.com
 Papa Foto
-Pi-Monster
+Pavuk
 Picscout
 Picsearch
 PictureFinder
-Pimonster
-Pockey
-ProPowerBot
+PleaseCrawl
 ProWebWalker
-Probethenet
-Psbot
-PxBroker
+Pu_iN
+Pump
+Quick-Crawler
 RSSingBot
-RankActive
 RankActiveLinkBot
 RankingBot
-RankingBot2
+Rankivabot
+RankurBot
+RealDownload
 Reaper
-RocketCrawler
-Rogerbot
+Recorder
 SBIder
-SEOlyticsCrawler
-SEOstats
-SISTRIX
-SalesIntelligent
-Screaming
-ScrepyBot
-SemrushBot
-SentiBot
+SEOprofiler
+ScoutJet
+Scrapy
+ScreenerBot
+SeoSiteCheckup
 SeobilityBot
-Seomoz
-SiteLockSpider
-SiteSnagger
+Site Sucker
+SiteExplorer
 SiteSucker
-Sitebeam
-Snake
+SlySearch
+Snapbot
 Snoopy
+SocialRankIOBot
+Sociscraper
+Sosospider
 Sottopop
-SpaceBison
-Spammen
-SpankBot
-Spinn3r
-SputnikBot
+Spanner
+Sqlmap
+Sqworm
+Steeler
 Stripper
-Sucker
-Surfbot
-Suzuran
-Szukacz
+SuperBot
+Swiftbot
+T0PHackTeam
+T8Abot
 Teleport
 TeleportPro
-Telesphorep
-TightTwatBot
-Tracemyfile
-Trendictionbot
+The Intraformant
+Thumbor
 Turnitin
-TwengaBot
 Twice
 Typhoeus
-URLy Warning
-Upflow
-Voil
+VeriCiteCrawler
+VidibleScraper
+Virusdie
 Voltron
+WBSearchBot
 WWW::Mechanize
-Wallpapers/3.0
 WallpapersHD
-Web Collage
+WeSEE
+Web Auto
 Web Enhancer
-Web Fetch
-Web Fuck
 Web Pix
-Web Sauger
-Web Sucker
 WebAuto
-WebCollage
+WebCopier
 WebFetch
-WebGo IS
-WebImageCollector
-WebPix
-WebSauger
-Webalta
-WebsiteExtractor
-WebsiteQuester
-Who.is Bot
-Widow
-WiseGuys Robot
+WebStripper
+WebWhacker
+WebmasterWorldForumBot
+Webster
+Whatweb
 Woobot
-Wotbox
+Wprecon
 Xaldon WebSpider
 Xaldon_WebSpider
-YoudaoBot
-Zauba
-ZoomBot
+Zeus
+Zitebot
+ZmEu
 ZoominfoBot
 ZumBot
-ZyBorg
-arquivo-web-crawler
-backlink-check
-crawl.sogou.com
+archive.org_bot
+autoemailspider
+check1.exe
+coccocbot-web
+cognitiveseo
 crawler4j
-demandbase-bot
+domainsproject.org
 eCatch
 facebookscraper
-gopher
-heritrix
-instabid
-ips-agent
 isitwp.com
-lwp-request
-lwp-trivial
 mediawords
 netEstate NE Crawler
 page scorer
+pcBrowser
 polaris version
+probe-image-size
+ripz
+satoristudio.net
 scalaj-http
 scan.lol
 seoscanners
 seostar
-sitechecker.pro
-sogouspider
-spyfu
-ubermetrics-technologies.com
+sysscan
+trendiction.com
+voyagerx.com
 x09Mozilla
-x22Mozilla
-zgrab