diff --git a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf index ca8905b0d..66357bbd7 100644 --- a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -35.193.7.13 0; +35.202.145.110 0; diff --git a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf index 453934fe3..19a19648d 100644 --- a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1644 -### Updated: Thu Jun 27 13:12:21 SAST 2019 +### Version: V4.2019.06.1645 +### Updated: Thu Jun 27 13:24:55 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf index ca8905b0d..66357bbd7 100644 --- a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -35.193.7.13 0; +35.202.145.110 0; diff --git a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf index 2f7d1d3a5..9c649adb5 100644 --- a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1644 -### Updated: Thu Jun 27 13:12:23 SAST 2019 +### Version: V3.2019.06.1645 +### Updated: Thu Jun 27 13:24:56 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/_conf_files_testing_changes_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/_conf_files_testing_changes_ip_whitelist/bots.d/blacklist-ips.conf index 479de1217..90fe4f9d3 100644 --- a/.dev-tools/_conf_files_testing_changes_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 1; +104.154.120.187 1; 127.0.0.1 1; diff --git a/.dev-tools/_conf_files_testing_changes_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes_ip_whitelist/bots.d/whitelist-ips.conf index f45bae986..95fbc95fc 100644 --- a/.dev-tools/_conf_files_testing_changes_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,4 +1,4 @@ -35.192.85.2 0; +104.154.120.187 0; 127.0.0.1 1; 127.0.0.1 1; 127.0.0.1 1; diff --git a/.dev-tools/_conf_files_testing_changes_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes_ip_whitelist/conf.d/globalblacklist.conf index 81842727a..9c649adb5 100644 --- a/.dev-tools/_conf_files_testing_changes_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes_ip_whitelist/conf.d/globalblacklist.conf @@ -1,16 +1,21 @@ -### ********************************** -### THE ULTIMATE NGINX BAD BOT BLOCKER -### ********************************** +### ----------------------------------------------------------- +### THE NGINX ULTIMATE BAD BOT, BAD IP AND BAD REFERRER BLOCKER +### ----------------------------------------------------------- ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1643 -### Updated: Thu Jun 27 12:59:42 SAST 2019 +### Version: V3.2019.06.1645 +### Updated: Thu Jun 27 13:24:56 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## +### -------------------------------------------- +### HELP SUPPORT THIS PROJECT - Send Me a Coffee +### https://ko-fi.com/mitchellkrog +### -------------------------------------------- + ############################################################################## # _ __ _ # # / |/ /__ _(_)__ __ __ # @@ -28,12 +33,16 @@ ### and also has rate limiting functionality for bad bots who you only want to rate limit ### and not actually block out entirely. It is very powerful and also very flexible. +### -------------------------------------------------------------------------- ### Created By: https://github.com/mitchellkrogza/ ### Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker ### Copyright Mitchell Krog - ### Contributors: Stuart Cardall - https://github.com/itoffshore +### -------------------------------------------------------------------------- +### -------------------------------------------------------------------------- ### Tested on: nginx/1.10.3 up to latest Mainstream Version (Ubuntu 16.04) +### -------------------------------------------------------------------------- ### This list was developed and is in use on a live Nginx server running some very busy web sites. ### It was built from the ground up using real data from daily logs and is updated almost daily. @@ -41,9 +50,9 @@ ### spam referrers, rogue IP address, scanners, scrapers and domain hijacking sites are extensively checked ### before they are added. It is monitored extensively for any false positives. -### ********* +### --------- ### Features: -### ********* +### --------- ### Clear formatting for Ease of Maintenance. ### Alphabetically ordered lists for Ease of Maintenance. ### Extensive Commenting for Ease of Reference. @@ -54,40 +63,38 @@ ### IP range blocking / whitelisting. ### Rate Limiting Functions. -### ************ +### ------------ ### INSTALLATION -### ************ +### ------------ ### PLEASE use the install, setup and update scripts provided for you to ease your installation. ### This Auto Installation procedure is documented in the README.md and AUTO-CONFIGURATION.md files. ### Installation, Setup and Update Scripts Contributed by Stuart Cardall - https://github.com/itoffshore ### There are also manual configuration instructions provided for those not wishing to do an auto install. -### *********************************************** +### ----------------------------------------------- ### !!!!! PLEASE READ INLINE NOTES ON TESTING !!!!! -### *********************************************** +### ----------------------------------------------- -### Note that: +### SETTINGS: +### --------------------------------------------- ### 0 = allowed - no limits ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### --------------------------------------------- -### ************************************************************ +### ------------------------------------------------------------ ### CONTRIBUTING / PULL REQUESTS / ADDING YOUR OWN BAD REFERRERS -### ************************************************************ +### ------------------------------------------------------------ ### For contributing, corrections or adding bots or referrers to this repo, ### Send a Pull Request (PR) on any of the .list files in the _generator_lists folder ### All Pull Requests will be checked for accuracy before being merged. -# ********************************* -# FIRST BLOCK BY USER-AGENT STRINGS -# ********************************* - -# *********************** +# ----------------------- # !!!!! PLEASE TEST !!!!! -# *********************** +# ----------------------- # ALWAYS test any User-Agent Strings you add here to make sure you have it right # Use a Chrome Extension called "User-Agent Switcher for Chrome" where you can create your @@ -98,36 +105,46 @@ # curl -I http://www.yourdomain.com -A "GoogleBot" ---- GIVES YOU: HTTP/1.1 200 OK (Meaning web page was served to Client) # curl -I http://www.yourdomain.com -A "80legs" ---- GIVES YOU: curl: (52) Empty reply from server (Meaning Nginx gave a 444 Dropped Connection) -# In this section we allow/disallow specific User Agents / Bots. -# ********************************************************************* -# Now we map all good and bad user agents to a variable called $bad_bot -# ********************************************************************* +# ======================= +# START BLOCKER FUNCTIONS +# ======================= + +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# DO NOT EDIT ANYTHING BELOW THIS LINE !!! +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# ============================= +# BEGIN SECTION 1 - USER-AGENTS +# ============================= + +# ALLOW / BLOCK User Agents / Bots + +# ------------------------------------------------------------------- +# Map all GOOD and BAD UA (User Agents) to a variable called $bad_bot +# ------------------------------------------------------------------- map $http_user_agent $bad_bot { - default 0; -# *********************************************** -# Include your Own Custom List of Bad User Agents -# *********************************************** - -# Use the include file below to further customize your own list of additional user-agents you wish to permanently block -# This include file allows whitelisting and blacklisting of anything specified below it, meaning you can over-ride anything in this blocker to your liking. - -# *********************************************************************************** +# ----------------------------------------------------------------------------------- # START CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# *********************************************************************************** +# ----------------------------------------------------------------------------------- + +# Include your Own Custom List of Bad User Agents +# Use the include file below to further customize your own list of additional user-agents you wish to permanently block +# This include file allows whitelisting and blacklisting of anything specified below it. +# This include file alows you to over-ride any Bad / Good UA (Bot) declared in this blocker to your liking. include /etc/nginx/bots.d/blacklist-user-agents.conf; -# ********************************************************************************* +# --------------------------------------------------------------------------------- # END CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ********************************************************************************* +# --------------------------------------------------------------------------------- -# ********************************************* -# Bad User-Agent Strings That We Block Outright -# ********************************************* +# -------------------------------------------------- +# BAD UA (User-Agent) Strings That We Block Outright +# -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)360Spider(?:\b|)" 3; @@ -688,9 +705,16 @@ map $http_user_agent $bad_bot { "~*(?:\b)ZyBorg(?:\b|)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### -# *********************************************** -# Allow Good User-Agent Strings We Know and Trust -# *********************************************** +# -------------------------------------------- +# GOOD UA User-Agent Strings We Know and Trust +# -------------------------------------------- + +# ----------------------------------------------------------------------- +# You can over-ride these in /etc/nginx/bots.d/blacklist-user-agents.conf +# by adding the same UA line there and chaning its value of 1 +# If you think GoogleBot is bad you would simply add them to +# blacklist-user-agents.conf with a value of 1 +# ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)adidxbot(?:\b|)" 0; @@ -727,12 +751,13 @@ map $http_user_agent $bad_bot { "~*(?:\b)yahoo(?:\b|)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### -# *************************************************** -# User-Agent Strings Allowed Through but Rate Limited -# *************************************************** +# -------------------------------------------------------- +# GOOD UA User-Agent Rate Limiting 1 - Disabled by Default +# -------------------------------------------------------- -# Some people block libwww-perl, it used widely in many valid (non rogue) agents -# I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited + # TO ACTIVATE THIS RATE LIMITING Uncomment these two lines in blockbots.conf + #limit_conn bot1_connlimit 100; + #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)jetmon(?:\b|)" 1; @@ -743,9 +768,14 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wget/1.15(?:\b|)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### -# ************************************************************** -# Rate Limited User-Agents who get a bit aggressive on bandwidth -# ************************************************************** +# ------------------------------------------------------- +# GOOD UA User-Agent Rate Limiting 2 - Enabled by Default +# ------------------------------------------------------- + +# ----------------------------------------------------------------------- +# You can over-ride these in /etc/nginx/bots.d/blacklist-user-agents.conf +# by adding the same UA line there and chaning its value of 1 +# ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)Alexa(?:\b|)" 2; @@ -768,163 +798,186 @@ map $http_user_agent $bad_bot { # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } - -# ***************************************** -# SECOND BLOCK BY REFERRER STRINGS AND URLS -# ***************************************** +# =========================== +# END SECTION 1 - USER-AGENTS +# =========================== -# ***************** +# ======================================= +# BEGIN SECTION 2 - REFERRERS AND DOMAINS +# ======================================= + +# ---------------- # PLEASE TEST !!!! -# ***************** - +# ---------------- +# ------------------------------------------------------------------------------------------------------------------------------ # ALWAYS test referrers that you add. This is done manually as follows - +# ------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://anything.adcash.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e http://www.goodwebsite.com/not-adcash --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e http://www.betterwebsite.com/not/adcash --- GIVES YOU: curl: (52) Empty reply from server - +# ------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://www.google.com --- GIVES YOU: full html output of the web page # curl -I http://www.yourdomain.com -e http://www.microsoft.com --- GIVES YOU: full html output of the web page - +# ------------------------------------------------------------------------------------------------------------------------------ # Because of case-insensitive matching any combination of capitilization in the names will all produce a positive hit # make sure you always test thoroughly and monitor logs. This section below also does NOT check for a preceding www. # and it also does not care if the referrer request was sent with http https or even ftp. +# ------------------------------------------------------------------------------------------------------------------------------ -# *********************************************************************** -# Now we map all bad referrer words below to a variable called $bad_words -# *********************************************************************** +# ---------------------------------------------------------------- +# Map all BAD referrer words below to a variable called $bad_words +# ---------------------------------------------------------------- + +# -------------------------------- +# START Bad Referrer Word Scanning +# -------------------------------- map $http_referer $bad_words { default 0; -# ************************** -# Bad Referrer Word Scanning -# ************************** - +# ------------------------------------------------------------------------------------------- # These are Words and Terms often found tagged onto domains or within url query strings. # Create and Customize Your Own Bad Referrer Words Here using the new Include File Method # New Method Uses the include file below so that when pulling future updates your # customized list of bad referrer words are automatically now included for you # Read Comments inside bad-referrer-words.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files - # BE VERY CAREFUL using the bad-referrer-words.conf file - please read the comment and + # BE VERY CAREFUL using this bad-referrer-words.conf file - please read the comments and # examples inside the include file for detailed explanations into how seriously this can # affect your own site from serving assets or other innocent sites from accessing your site - # For safety sake the whitelist-domains.conf file is also loaded here before the bad-referrer-words.conf file. + # For safety sake the whitelist-domains.conf file is also loaded here before the + # bad-referrer-words.conf file is loaded. +# ------------------------------------------------------------------------------------------- -# ************************************************************************ +# ------------------------------------------------------------------------ # START WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************ +# ------------------------------------------------------------------------ include /etc/nginx/bots.d/whitelist-domains.conf; -# ********************************************************************** +# ---------------------------------------------------------------------- # END WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ********************************************************************** +# ---------------------------------------------------------------------- -# ****************************************************************************** +# ------------------------------------------------------------------------------ # START CUSTOM BAD REFERRER WORDS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ****************************************************************************** +# ------------------------------------------------------------------------------ include /etc/nginx/bots.d/bad-referrer-words.conf; -# **************************************************************************** +# ---------------------------------------------------------------------------- # END CUSTOM BAD REFERRER WORDS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# **************************************************************************** +# ---------------------------------------------------------------------------- } +# -------------------------------- +# END Bad Referrer Word Scanning +# -------------------------------- -# ************************* -# Bad Referrer Domain Names -# ************************* -# Now a list of bad referrer urls these domains or any combination of them ie .com .net -# will be blocked out. Doesn't matter if the protocol is http, https or even ftp +# ---------------------------------------- +# START Good and Bad Referrer Domain Names +# ---------------------------------------- +# ------------------------------------------------------------------------------------- +# Good and Bad referrer urls Doesn't matter if the protocol is http, https or even ftp +# ------------------------------------------------------------------------------------- + +# ---------------------- # This section includes: -# ********************** +# ---------------------- +# -------------------------------------------------------------------------------- # Blocking of SEO company Semalt.com (now merged into this one section) # MIRAI Botnet Domains Used for Mass Attacks # Other known bad SEO companies and Ad Hijacking Sites # Sites linked to malware, adware, clickjacking and ransomware +# Domain names and referrers used in referrer spam and seo hijacking +# Whitelisting of your own GOOD domains / referrers +# Whitelisting of any other GOOD domains / referrers you want explicitly NOT block +# -------------------------------------------------------------------------------- -# ***************** +# ---------------- # PLEASE TEST !!!! -# ***************** - +# ---------------- +# ------------------------------------------------------------------------------------------------------------------------------------ # ALWAYS test referrers that you add. This is done manually as follows - +# ------------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://8gold.com --- GIVES YOU: curl: (52) Empty reply from server - +# ------------------------------------------------------------------------------------------------------------------------------------ # Because of case-insensitive matching any combination of capitilization will all produce a positive hit - make sure you always test. - +# ------------------------------------------------------------------------------------------------------------------------------------ # For Example any of the following variations below of 8gold.com will be detected and blocked - +# ------------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://NOT-8gold.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e http://this.is.not8gOlD.net --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e ftp://8gold.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e ftp://www.weare8gold.NET --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e https://subdomain.8gold.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e https://NOT8GolD.org --- GIVES YOU: curl: (52) Empty reply from server - +# ------------------------------------------------------------------------------------------------------------------------------------ # So if you see a bad referrer from wearegoogle.com and you want to block them just add # them as "~*wearegoogle.com" don't ever go and do something like "~*google(-|.)" you will # kill all your SEO in a week. - +# ------------------------------------------------------------------------------------------------------------------------------------ # To add your own custom bad referrers use the custom include file # /etc/nginx/bots.d/custom-bad-referrers.conf # Or send a Pull Request to add it to the global blacklist for other users. - # In the bad referrers section I also include sites that hotlink images without permission. +# ------------------------------------------------------------------------------------------------------------------------------------ -# *********************************************************************** -# Now we map all good & bad referrer urls to variable called #bad_referer -# *********************************************************************** +# -------------------------------------------------------------------- +# Map all good & bad referrer DOMAINS to a variable called bad_referer +# -------------------------------------------------------------------- map $http_referer $bad_referer { hostnames; default 0; -# ************************************* -# GOOD REFERRERS - Spared from Checking -# ************************************* +# -------------------------------------------- +# GOOD REFERRER DOMAINS - Spared from Checking +# -------------------------------------------- +# --------------------------------------------------------------------------------------- # Add all your own web site domain names and server names in this section - # WHITELIST Your Own Domain Names Here using the new Include File Method # New Method Uses the include file below so that when pulling future updates your # whitelisted domain names are automatically now included for you. # Read Comments inside whitelist-domains.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files +# --------------------------------------------------------------------------------------- -# ************************************************************************ +# ------------------------------------------------------------------------ # START WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************ +# ------------------------------------------------------------------------ include /etc/nginx/bots.d/whitelist-domains.conf; -# ********************************************************************** +# ---------------------------------------------------------------------- # END WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ********************************************************************** +# ---------------------------------------------------------------------- -# *********************************** +# ----------------------------------- # CUSTOM BAD REFERRERS - Add your Own -# *********************************** +# ----------------------------------- # Add any extra bad referrers in the following include file to have them # permanently included and blocked - avoid duplicates in your custom file +# custom-bad-referrers.conf is BOTH a BLACKLIST AND WHITELIST +# custom-bad-referrers.conf ALLOWS complete over-riding of anything +# If you think google.com is bad you would simply add them to +# custom-bad-referrers.conf with a value of 1 -# ************************************************************************* +# ------------------------------------------------------------------------- # START CUSTOM BAD REFERRERS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************* +# ------------------------------------------------------------------------- include /etc/nginx/bots.d/custom-bad-referrers.conf; -# *********************************************************************** +# ----------------------------------------------------------------------- # END CUSTOM BAD REFERRERS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# *********************************************************************** +# ----------------------------------------------------------------------- # START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)000free\.us(?:\b|)" 1; @@ -7644,137 +7697,49 @@ map $http_referer $bad_referer { # END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ### } +# ===================================== +# END SECTION 2 - REFERRERS AND DOMAINS +# ===================================== -# *********************************************** -# WHITELISTING AND BLACKLISTING IP ADDRESS RANGES -# *********************************************** +# ======================================================================== +# BEGIN SECTION 3 - WHITELISTING AND BLACKLISTING IP ADDRESSESE AND RANGES +# ======================================================================== -# Geo directive to deny and also whitelist certain ip addresses +# -------------------------------------------------------------------------------------- +# Map all GOOD and BAD IP Addresses and Ranges to a variable called geo $validate_client +# -------------------------------------------------------------------------------------- geo $validate_client { - -# ******************** -# First Our Safety Net -# ******************** - -# Anything not matching our rules is allowed through with default 0; - default 0; -# *********************************** -# Whitelist all your OWN IP addresses -# *********************************** +# ---------------------------------------------- +# Whitelist all your OWN IP addresses and Ranges +# ---------------------------------------------- +# -------------------------------------------------------------------------------------- # WHITELIST all your own IP addresses using the include file below. # New Method Uses the include file below so that when pulling future updates your # whitelisted IP addresses are automatically now included for you. # Read Comments inside whitelist-ips.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files + # whitelist-ips.conf reigns supreme !!! + # Whatever you add to whitelist-ips.conf will be whitelisted FULL STOP + # Anything blacklisted above this line will be over-ridden by whitelist-ips.conf +# -------------------------------------------------------------------------------------- -# ************************************************************************** +# -------------------------------------------------------------------------- # START WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************** +# -------------------------------------------------------------------------- include /etc/nginx/bots.d/whitelist-ips.conf; -# ************************************************************************ +# ------------------------------------------------------------------------ # END WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************ +# ------------------------------------------------------------------------ -# **************** -# Google IP Ranges -# **************** - -# For Safety Sake all Google's Known IP Ranges are all white listed - -# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - 108.177.0.0/17 0; - 172.217.0.0/16 0; - 173.194.0.0/16 0; - 2001:4860:4000::/36 0; - 203.208.60.0/24 0; - 207.126.144.0/20 0; - 209.85.128.0/17 0; - 216.239.32.0/19 0; - 216.58.192.0/19 0; - 2404:6800:4000::/36 0; - 2607:f8b0:4000::/36 0; - 2800:3f0:4000::/36 0; - 2a00:1450:4000::/36 0; - 2c0f:fb50:4000::/36 0; - 35.192.0.0/12 0; - 64.18.0.0/20 0; - 64.233.160.0/19 0; - 64.68.80.0/21 0; - 65.52.0.0/14 0; - 66.102.0.0/20 0; - 66.249.64.0/19 0; - 72.14.192.0/18 0; - 74.125.0.0/16 0; -# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - -# ************** -# Bing IP Ranges -# ************** - -# For Safety Sake all Bing's Known IP Ranges are all white listed - -# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - 131.253.21.0/24 0; - 131.253.22.0/23 0; - 131.253.24.0/21 0; - 131.253.24.0/22 0; - 131.253.32.0/20 0; - 157.54.0.0/15 0; - 157.56.0.0/14 0; - 157.60.0.0/16 0; - 199.30.16.0/24 0; - 199.30.27.0/24 0; - 207.46.0.0/16 0; - 40.112.0.0/13 0; - 40.120.0.0/14 0; - 40.124.0.0/16 0; - 40.125.0.0/17 0; - 40.74.0.0/15 0; - 40.76.0.0/14 0; - 40.80.0.0/12 0; - 40.96.0.0/12 0; -# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - -# ******************** -# Cloudflare IP Ranges -# ******************** - -# For Safety Sake all Cloudflare's Known IP Ranges are all white listed - -# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - 103.21.244.0/22 0; - 103.22.200.0/22 0; - 103.31.4.0/22 0; - 104.16.0.0/12 0; - 108.162.192.0/18 0; - 131.0.72.0/22 0; - 141.101.64.0/18 0; - 162.158.0.0/15 0; - 172.64.0.0/13 0; - 173.245.48.0/20 0; - 188.114.96.0/20 0; - 190.93.240.0/20 0; - 197.234.240.0/22 0; - 198.41.128.0/17 0; - 199.27.128.0/21 0; - 2400:cb00::/32 0; - 2405:8100::/32 0; - 2405:b500::/32 0; - 2606:4700::/32 0; - 2803:f800::/32 0; - 2a06:98c0::/29 0; - 2c0f:f248::/32 0; -# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - -# ************************* -# Wordpress Theme Detectors -# ************************* +# ------------------------------------- +# BLOCK known Wordpress Theme Detectors +# ------------------------------------- # START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ### @@ -7818,10 +7783,11 @@ geo $validate_client { 89.36.223.188 1; #www.cuteseotools.net/wordpress-theme-detector # END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ### -# **************************************** -# NIBBLER - SEO testing and reporting tool -# **************************************** +# ---------------------------------------------- +# BLOCK NIBBLER - SEO testing and reporting tool +# ---------------------------------------------- # See - http://nibbler.silktide.com/ +# ---------------------------------------------- # START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ### 52.201.238.175 1; @@ -7834,33 +7800,10 @@ geo $validate_client { # END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ### -# **************************** -# Known Bad IP's and IP Ranges -# ************************************************* -# Blacklist IP addresses and IP Ranges Customizable -# ************************************************* - -# BLACKLIST all your IP addresses and Ranges using the new include file below. - # New Method Uses the include file below so that when pulling future updates your - # Custom Blacklisted IP addresses are automatically now included for you. - # Read Comments inside blacklist-ips.conf for customization tips. - # Updating the main globalblacklist.conf file will not touch your custom include files - -# ******************************************************************** -# START BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ******************************************************************** - - include /etc/nginx/bots.d/blacklist-ips.conf; - -# ****************************************************************** -# END BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ****************************************************************** - -# ********************************************************************************************* -# KNOWN BAD IP ADDRESSES -# Source: https://github.com/mitchellkrogza/Suspicious.Snooping.Sniffing.Hacking.IP.Addresses -# A Reliable Daily Updated List Included Below of Known Bad IP Addresses, Snoopers and Sniffers -# ********************************************************************************************* +# ----------------------------------------- +# BLOCK KNOWN BAD IP ADDRESSES +# Top known bad IP Adresses from abuseIPDB +# ----------------------------------------- # START KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ### 193.32.163.89 1; @@ -17864,33 +17807,176 @@ geo $validate_client { 46.101.115.65 1; # END KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ### +# -------------------------- +# WHITELIST Google IP Ranges +# -------------------------- + +# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + 108.177.0.0/17 0; + 172.217.0.0/16 0; + 173.194.0.0/16 0; + 2001:4860:4000::/36 0; + 203.208.60.0/24 0; + 207.126.144.0/20 0; + 209.85.128.0/17 0; + 216.239.32.0/19 0; + 216.58.192.0/19 0; + 2404:6800:4000::/36 0; + 2607:f8b0:4000::/36 0; + 2800:3f0:4000::/36 0; + 2a00:1450:4000::/36 0; + 2c0f:fb50:4000::/36 0; + 35.192.0.0/12 0; + 64.18.0.0/20 0; + 64.233.160.0/19 0; + 64.68.80.0/21 0; + 65.52.0.0/14 0; + 66.102.0.0/20 0; + 66.249.64.0/19 0; + 72.14.192.0/18 0; + 74.125.0.0/16 0; +# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + +# ------------------------ +# WHITELIST Bing IP Ranges +# ------------------------ + +# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + 131.253.21.0/24 0; + 131.253.22.0/23 0; + 131.253.24.0/21 0; + 131.253.24.0/22 0; + 131.253.32.0/20 0; + 157.54.0.0/15 0; + 157.56.0.0/14 0; + 157.60.0.0/16 0; + 199.30.16.0/24 0; + 199.30.27.0/24 0; + 207.46.0.0/16 0; + 40.112.0.0/13 0; + 40.120.0.0/14 0; + 40.124.0.0/16 0; + 40.125.0.0/17 0; + 40.74.0.0/15 0; + 40.76.0.0/14 0; + 40.80.0.0/12 0; + 40.96.0.0/12 0; +# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + +# ------------------------------ +# WHITELIST Cloudflare IP Ranges +# ------------------------------ + +# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + 103.21.244.0/22 0; + 103.22.200.0/22 0; + 103.31.4.0/22 0; + 104.16.0.0/12 0; + 108.162.192.0/18 0; + 131.0.72.0/22 0; + 141.101.64.0/18 0; + 162.158.0.0/15 0; + 172.64.0.0/13 0; + 173.245.48.0/20 0; + 188.114.96.0/20 0; + 190.93.240.0/20 0; + 197.234.240.0/22 0; + 198.41.128.0/17 0; + 199.27.128.0/21 0; + 2400:cb00::/32 0; + 2405:8100::/32 0; + 2405:b500::/32 0; + 2606:4700::/32 0; + 2803:f800::/32 0; + 2a06:98c0::/29 0; + 2c0f:f248::/32 0; +# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + +# ------------------------------------------------- +# BLACKLIST IP addresses and IP Ranges Customizable +# ------------------------------------------------- + +# -------------------------------------------------------------------------------------- +# BLACKLIST all your IP addresses and Ranges using the new include file below. + # New Method Uses the include file below so that when pulling future updates your + # Custom Blacklisted IP addresses are automatically now included for you. + # Read Comments inside blacklist-ips.conf for customization tips. + # Updating the main globalblacklist.conf file will not touch your custom include files + # Anything added to blacklist-ips.conf will over-ride anything whitelisted above +# -------------------------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# START BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# -------------------------------------------------------------------- + + include /etc/nginx/bots.d/blacklist-ips.conf; + +# ------------------------------------------------------------------ +# END BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ------------------------------------------------------------------ + +# ---------------------------------------------- +# Whitelist all your OWN IP addresses and Ranges +# ---------------------------------------------- + +# -------------------------------------------------------------------------------------- +# WHITELIST all your own IP addresses using the include file below. + # New Method Uses the include file below so that when pulling future updates your + # whitelisted IP addresses are automatically now included for you. + # Read Comments inside whitelist-ips.conf for customization tips. + # Updating the main globalblacklist.conf file will not touch your custom include files + # whitelist-ips.conf reigns supreme !!! + # Whatever you add to whitelist-ips.conf will be whitelisted FULL STOP + # Anything blacklisted above this line will be over-ridden by whitelist-ips.conf +# -------------------------------------------------------------------------------------- + +# -------------------------------------------------------------------------- +# START WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# -------------------------------------------------------------------------- + + include /etc/nginx/bots.d/whitelist-ips.conf; + +# ------------------------------------------------------------------------ +# END WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ------------------------------------------------------------------------ } +# -------------------------------------------------------------------------------------- # WHITELIST your own IPs from the DDOS Filter # Add your own IP addresses and ranges into the custom include file whitelist-ips.conf # to spare them from the rate limiting DDOS filter. # This section includes the same / single whitelist-ips.conf file so you only # need to edit that include file and have it include here for you too. +# -------------------------------------------------------------------------------------- geo $ratelimited { default 1; -# *************************************************************************** +# --------------------------------------------------------------------------- # START WHITELISTED IP RANGES2 ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# *************************************************************************** +# --------------------------------------------------------------------------- include /etc/nginx/bots.d/whitelist-ips.conf; -# ************************************************************************* +# ------------------------------------------------------------------------- # END WHITELISTED IP RANGES2 ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************* +# ------------------------------------------------------------------------- } -# ***************************************** -# MAP BAD BOTS TO OUR RATE LIMITER FUNCTION -# ***************************************** +# ====================================================================== +# END SECTION 3 - WHITELISTING AND BLACKLISTING IP ADDRESSESE AND RANGES +# ====================================================================== + + +# ============================================ +# BEGIN SECTION 4 - ACTIVATE BLOCKER FUNCTIONS +# ============================================ + +# -------------------------------------------- +# 1. MAP BAD BOTS TO OUR RATE LIMITER FUNCTION +# -------------------------------------------- map $bad_bot $bot_iplimit { 0 ""; @@ -17898,30 +17984,31 @@ geo $ratelimited { 2 $binary_remote_addr; } -# *********************** -# SET RATE LIMITING ZONES -# *********************** +# -------------------------- +# 2. SET RATE LIMITING ZONES +# -------------------------- # BAD BOT RATE LIMITING ZONE -# limits for Zone $bad_bot = 1 -# Nothing Set - you can set a different zone limiter here if you like -# We issue a 444 response instead to all bad bots. +# Rate limiting will only take effect if on any User-Agents with a value of 2 -# limits for Zone $bad_bot = 2 -# this rate limiting will only take effect if you change any of the bots and change -# their block value from 1 to 2. limit_conn_zone $bot_iplimit zone=bot2_connlimit:16m; - limit_req_zone $bot_iplimit zone=bot2_reqlimitip:16m rate=2r/s; + limit_req_zone $bot_iplimit zone=bot2_reqlimitip:16m rate=6r/m; -### *** MAKE SURE TO ADD to your nginx.conf *** -### server_names_hash_bucket_size 64; -### server_names_hash_max_size 4096; -### limit_req_zone $binary_remote_addr zone=flood:50m rate=90r/s; -### limit_conn_zone $binary_remote_addr zone=addr:50m; -### to allow it to load this large set of domains into memory and to set the rate limiting zones for the DDOS filter. +# ========================================== +# END SECTION 4 - ACTIVATE BLOCKER FUNCTIONS +# ========================================== -### THE END of the Long and Winding Road +# ===================== +# END BLOCKER FUNCTIONS +# ===================== -### Also check out my Ultimate Apache Bad Bot Blocker on Github -### https://github.com/mitchellkrogza/apache-ultimate-bad-bot-blocker +### -------------------------------------------- +### HELP SUPPORT THIS PROJECT - Send Me a Coffee +### https://ko-fi.com/mitchellkrog +### -------------------------------------------- + +### FOR APACHE SERVERS +### --------------------------------------------- +### Check out the Ultimate Apache Bad Bot Blocker +### --------------------------------------------- diff --git a/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blacklist-ips.conf index 479de1217..90fe4f9d3 100644 --- a/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 1; +104.154.120.187 1; 127.0.0.1 1; diff --git a/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/whitelist-ips.conf index f45bae986..95fbc95fc 100644 --- a/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/whitelist-ips.conf @@ -1,4 +1,4 @@ -35.192.85.2 0; +104.154.120.187 0; 127.0.0.1 1; 127.0.0.1 1; 127.0.0.1 1; diff --git a/.dev-tools/_conf_files_testing_changes_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes_ratelimiting/conf.d/globalblacklist.conf index 81842727a..9c649adb5 100644 --- a/.dev-tools/_conf_files_testing_changes_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes_ratelimiting/conf.d/globalblacklist.conf @@ -1,16 +1,21 @@ -### ********************************** -### THE ULTIMATE NGINX BAD BOT BLOCKER -### ********************************** +### ----------------------------------------------------------- +### THE NGINX ULTIMATE BAD BOT, BAD IP AND BAD REFERRER BLOCKER +### ----------------------------------------------------------- ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1643 -### Updated: Thu Jun 27 12:59:42 SAST 2019 +### Version: V3.2019.06.1645 +### Updated: Thu Jun 27 13:24:56 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## +### -------------------------------------------- +### HELP SUPPORT THIS PROJECT - Send Me a Coffee +### https://ko-fi.com/mitchellkrog +### -------------------------------------------- + ############################################################################## # _ __ _ # # / |/ /__ _(_)__ __ __ # @@ -28,12 +33,16 @@ ### and also has rate limiting functionality for bad bots who you only want to rate limit ### and not actually block out entirely. It is very powerful and also very flexible. +### -------------------------------------------------------------------------- ### Created By: https://github.com/mitchellkrogza/ ### Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker ### Copyright Mitchell Krog - ### Contributors: Stuart Cardall - https://github.com/itoffshore +### -------------------------------------------------------------------------- +### -------------------------------------------------------------------------- ### Tested on: nginx/1.10.3 up to latest Mainstream Version (Ubuntu 16.04) +### -------------------------------------------------------------------------- ### This list was developed and is in use on a live Nginx server running some very busy web sites. ### It was built from the ground up using real data from daily logs and is updated almost daily. @@ -41,9 +50,9 @@ ### spam referrers, rogue IP address, scanners, scrapers and domain hijacking sites are extensively checked ### before they are added. It is monitored extensively for any false positives. -### ********* +### --------- ### Features: -### ********* +### --------- ### Clear formatting for Ease of Maintenance. ### Alphabetically ordered lists for Ease of Maintenance. ### Extensive Commenting for Ease of Reference. @@ -54,40 +63,38 @@ ### IP range blocking / whitelisting. ### Rate Limiting Functions. -### ************ +### ------------ ### INSTALLATION -### ************ +### ------------ ### PLEASE use the install, setup and update scripts provided for you to ease your installation. ### This Auto Installation procedure is documented in the README.md and AUTO-CONFIGURATION.md files. ### Installation, Setup and Update Scripts Contributed by Stuart Cardall - https://github.com/itoffshore ### There are also manual configuration instructions provided for those not wishing to do an auto install. -### *********************************************** +### ----------------------------------------------- ### !!!!! PLEASE READ INLINE NOTES ON TESTING !!!!! -### *********************************************** +### ----------------------------------------------- -### Note that: +### SETTINGS: +### --------------------------------------------- ### 0 = allowed - no limits ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### --------------------------------------------- -### ************************************************************ +### ------------------------------------------------------------ ### CONTRIBUTING / PULL REQUESTS / ADDING YOUR OWN BAD REFERRERS -### ************************************************************ +### ------------------------------------------------------------ ### For contributing, corrections or adding bots or referrers to this repo, ### Send a Pull Request (PR) on any of the .list files in the _generator_lists folder ### All Pull Requests will be checked for accuracy before being merged. -# ********************************* -# FIRST BLOCK BY USER-AGENT STRINGS -# ********************************* - -# *********************** +# ----------------------- # !!!!! PLEASE TEST !!!!! -# *********************** +# ----------------------- # ALWAYS test any User-Agent Strings you add here to make sure you have it right # Use a Chrome Extension called "User-Agent Switcher for Chrome" where you can create your @@ -98,36 +105,46 @@ # curl -I http://www.yourdomain.com -A "GoogleBot" ---- GIVES YOU: HTTP/1.1 200 OK (Meaning web page was served to Client) # curl -I http://www.yourdomain.com -A "80legs" ---- GIVES YOU: curl: (52) Empty reply from server (Meaning Nginx gave a 444 Dropped Connection) -# In this section we allow/disallow specific User Agents / Bots. -# ********************************************************************* -# Now we map all good and bad user agents to a variable called $bad_bot -# ********************************************************************* +# ======================= +# START BLOCKER FUNCTIONS +# ======================= + +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# DO NOT EDIT ANYTHING BELOW THIS LINE !!! +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# ============================= +# BEGIN SECTION 1 - USER-AGENTS +# ============================= + +# ALLOW / BLOCK User Agents / Bots + +# ------------------------------------------------------------------- +# Map all GOOD and BAD UA (User Agents) to a variable called $bad_bot +# ------------------------------------------------------------------- map $http_user_agent $bad_bot { - default 0; -# *********************************************** -# Include your Own Custom List of Bad User Agents -# *********************************************** - -# Use the include file below to further customize your own list of additional user-agents you wish to permanently block -# This include file allows whitelisting and blacklisting of anything specified below it, meaning you can over-ride anything in this blocker to your liking. - -# *********************************************************************************** +# ----------------------------------------------------------------------------------- # START CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# *********************************************************************************** +# ----------------------------------------------------------------------------------- + +# Include your Own Custom List of Bad User Agents +# Use the include file below to further customize your own list of additional user-agents you wish to permanently block +# This include file allows whitelisting and blacklisting of anything specified below it. +# This include file alows you to over-ride any Bad / Good UA (Bot) declared in this blocker to your liking. include /etc/nginx/bots.d/blacklist-user-agents.conf; -# ********************************************************************************* +# --------------------------------------------------------------------------------- # END CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ********************************************************************************* +# --------------------------------------------------------------------------------- -# ********************************************* -# Bad User-Agent Strings That We Block Outright -# ********************************************* +# -------------------------------------------------- +# BAD UA (User-Agent) Strings That We Block Outright +# -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)360Spider(?:\b|)" 3; @@ -688,9 +705,16 @@ map $http_user_agent $bad_bot { "~*(?:\b)ZyBorg(?:\b|)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### -# *********************************************** -# Allow Good User-Agent Strings We Know and Trust -# *********************************************** +# -------------------------------------------- +# GOOD UA User-Agent Strings We Know and Trust +# -------------------------------------------- + +# ----------------------------------------------------------------------- +# You can over-ride these in /etc/nginx/bots.d/blacklist-user-agents.conf +# by adding the same UA line there and chaning its value of 1 +# If you think GoogleBot is bad you would simply add them to +# blacklist-user-agents.conf with a value of 1 +# ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)adidxbot(?:\b|)" 0; @@ -727,12 +751,13 @@ map $http_user_agent $bad_bot { "~*(?:\b)yahoo(?:\b|)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### -# *************************************************** -# User-Agent Strings Allowed Through but Rate Limited -# *************************************************** +# -------------------------------------------------------- +# GOOD UA User-Agent Rate Limiting 1 - Disabled by Default +# -------------------------------------------------------- -# Some people block libwww-perl, it used widely in many valid (non rogue) agents -# I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited + # TO ACTIVATE THIS RATE LIMITING Uncomment these two lines in blockbots.conf + #limit_conn bot1_connlimit 100; + #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)jetmon(?:\b|)" 1; @@ -743,9 +768,14 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wget/1.15(?:\b|)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### -# ************************************************************** -# Rate Limited User-Agents who get a bit aggressive on bandwidth -# ************************************************************** +# ------------------------------------------------------- +# GOOD UA User-Agent Rate Limiting 2 - Enabled by Default +# ------------------------------------------------------- + +# ----------------------------------------------------------------------- +# You can over-ride these in /etc/nginx/bots.d/blacklist-user-agents.conf +# by adding the same UA line there and chaning its value of 1 +# ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)Alexa(?:\b|)" 2; @@ -768,163 +798,186 @@ map $http_user_agent $bad_bot { # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } - -# ***************************************** -# SECOND BLOCK BY REFERRER STRINGS AND URLS -# ***************************************** +# =========================== +# END SECTION 1 - USER-AGENTS +# =========================== -# ***************** +# ======================================= +# BEGIN SECTION 2 - REFERRERS AND DOMAINS +# ======================================= + +# ---------------- # PLEASE TEST !!!! -# ***************** - +# ---------------- +# ------------------------------------------------------------------------------------------------------------------------------ # ALWAYS test referrers that you add. This is done manually as follows - +# ------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://anything.adcash.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e http://www.goodwebsite.com/not-adcash --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e http://www.betterwebsite.com/not/adcash --- GIVES YOU: curl: (52) Empty reply from server - +# ------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://www.google.com --- GIVES YOU: full html output of the web page # curl -I http://www.yourdomain.com -e http://www.microsoft.com --- GIVES YOU: full html output of the web page - +# ------------------------------------------------------------------------------------------------------------------------------ # Because of case-insensitive matching any combination of capitilization in the names will all produce a positive hit # make sure you always test thoroughly and monitor logs. This section below also does NOT check for a preceding www. # and it also does not care if the referrer request was sent with http https or even ftp. +# ------------------------------------------------------------------------------------------------------------------------------ -# *********************************************************************** -# Now we map all bad referrer words below to a variable called $bad_words -# *********************************************************************** +# ---------------------------------------------------------------- +# Map all BAD referrer words below to a variable called $bad_words +# ---------------------------------------------------------------- + +# -------------------------------- +# START Bad Referrer Word Scanning +# -------------------------------- map $http_referer $bad_words { default 0; -# ************************** -# Bad Referrer Word Scanning -# ************************** - +# ------------------------------------------------------------------------------------------- # These are Words and Terms often found tagged onto domains or within url query strings. # Create and Customize Your Own Bad Referrer Words Here using the new Include File Method # New Method Uses the include file below so that when pulling future updates your # customized list of bad referrer words are automatically now included for you # Read Comments inside bad-referrer-words.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files - # BE VERY CAREFUL using the bad-referrer-words.conf file - please read the comment and + # BE VERY CAREFUL using this bad-referrer-words.conf file - please read the comments and # examples inside the include file for detailed explanations into how seriously this can # affect your own site from serving assets or other innocent sites from accessing your site - # For safety sake the whitelist-domains.conf file is also loaded here before the bad-referrer-words.conf file. + # For safety sake the whitelist-domains.conf file is also loaded here before the + # bad-referrer-words.conf file is loaded. +# ------------------------------------------------------------------------------------------- -# ************************************************************************ +# ------------------------------------------------------------------------ # START WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************ +# ------------------------------------------------------------------------ include /etc/nginx/bots.d/whitelist-domains.conf; -# ********************************************************************** +# ---------------------------------------------------------------------- # END WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ********************************************************************** +# ---------------------------------------------------------------------- -# ****************************************************************************** +# ------------------------------------------------------------------------------ # START CUSTOM BAD REFERRER WORDS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ****************************************************************************** +# ------------------------------------------------------------------------------ include /etc/nginx/bots.d/bad-referrer-words.conf; -# **************************************************************************** +# ---------------------------------------------------------------------------- # END CUSTOM BAD REFERRER WORDS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# **************************************************************************** +# ---------------------------------------------------------------------------- } +# -------------------------------- +# END Bad Referrer Word Scanning +# -------------------------------- -# ************************* -# Bad Referrer Domain Names -# ************************* -# Now a list of bad referrer urls these domains or any combination of them ie .com .net -# will be blocked out. Doesn't matter if the protocol is http, https or even ftp +# ---------------------------------------- +# START Good and Bad Referrer Domain Names +# ---------------------------------------- +# ------------------------------------------------------------------------------------- +# Good and Bad referrer urls Doesn't matter if the protocol is http, https or even ftp +# ------------------------------------------------------------------------------------- + +# ---------------------- # This section includes: -# ********************** +# ---------------------- +# -------------------------------------------------------------------------------- # Blocking of SEO company Semalt.com (now merged into this one section) # MIRAI Botnet Domains Used for Mass Attacks # Other known bad SEO companies and Ad Hijacking Sites # Sites linked to malware, adware, clickjacking and ransomware +# Domain names and referrers used in referrer spam and seo hijacking +# Whitelisting of your own GOOD domains / referrers +# Whitelisting of any other GOOD domains / referrers you want explicitly NOT block +# -------------------------------------------------------------------------------- -# ***************** +# ---------------- # PLEASE TEST !!!! -# ***************** - +# ---------------- +# ------------------------------------------------------------------------------------------------------------------------------------ # ALWAYS test referrers that you add. This is done manually as follows - +# ------------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://8gold.com --- GIVES YOU: curl: (52) Empty reply from server - +# ------------------------------------------------------------------------------------------------------------------------------------ # Because of case-insensitive matching any combination of capitilization will all produce a positive hit - make sure you always test. - +# ------------------------------------------------------------------------------------------------------------------------------------ # For Example any of the following variations below of 8gold.com will be detected and blocked - +# ------------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://NOT-8gold.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e http://this.is.not8gOlD.net --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e ftp://8gold.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e ftp://www.weare8gold.NET --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e https://subdomain.8gold.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e https://NOT8GolD.org --- GIVES YOU: curl: (52) Empty reply from server - +# ------------------------------------------------------------------------------------------------------------------------------------ # So if you see a bad referrer from wearegoogle.com and you want to block them just add # them as "~*wearegoogle.com" don't ever go and do something like "~*google(-|.)" you will # kill all your SEO in a week. - +# ------------------------------------------------------------------------------------------------------------------------------------ # To add your own custom bad referrers use the custom include file # /etc/nginx/bots.d/custom-bad-referrers.conf # Or send a Pull Request to add it to the global blacklist for other users. - # In the bad referrers section I also include sites that hotlink images without permission. +# ------------------------------------------------------------------------------------------------------------------------------------ -# *********************************************************************** -# Now we map all good & bad referrer urls to variable called #bad_referer -# *********************************************************************** +# -------------------------------------------------------------------- +# Map all good & bad referrer DOMAINS to a variable called bad_referer +# -------------------------------------------------------------------- map $http_referer $bad_referer { hostnames; default 0; -# ************************************* -# GOOD REFERRERS - Spared from Checking -# ************************************* +# -------------------------------------------- +# GOOD REFERRER DOMAINS - Spared from Checking +# -------------------------------------------- +# --------------------------------------------------------------------------------------- # Add all your own web site domain names and server names in this section - # WHITELIST Your Own Domain Names Here using the new Include File Method # New Method Uses the include file below so that when pulling future updates your # whitelisted domain names are automatically now included for you. # Read Comments inside whitelist-domains.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files +# --------------------------------------------------------------------------------------- -# ************************************************************************ +# ------------------------------------------------------------------------ # START WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************ +# ------------------------------------------------------------------------ include /etc/nginx/bots.d/whitelist-domains.conf; -# ********************************************************************** +# ---------------------------------------------------------------------- # END WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ********************************************************************** +# ---------------------------------------------------------------------- -# *********************************** +# ----------------------------------- # CUSTOM BAD REFERRERS - Add your Own -# *********************************** +# ----------------------------------- # Add any extra bad referrers in the following include file to have them # permanently included and blocked - avoid duplicates in your custom file +# custom-bad-referrers.conf is BOTH a BLACKLIST AND WHITELIST +# custom-bad-referrers.conf ALLOWS complete over-riding of anything +# If you think google.com is bad you would simply add them to +# custom-bad-referrers.conf with a value of 1 -# ************************************************************************* +# ------------------------------------------------------------------------- # START CUSTOM BAD REFERRERS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************* +# ------------------------------------------------------------------------- include /etc/nginx/bots.d/custom-bad-referrers.conf; -# *********************************************************************** +# ----------------------------------------------------------------------- # END CUSTOM BAD REFERRERS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# *********************************************************************** +# ----------------------------------------------------------------------- # START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)000free\.us(?:\b|)" 1; @@ -7644,137 +7697,49 @@ map $http_referer $bad_referer { # END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ### } +# ===================================== +# END SECTION 2 - REFERRERS AND DOMAINS +# ===================================== -# *********************************************** -# WHITELISTING AND BLACKLISTING IP ADDRESS RANGES -# *********************************************** +# ======================================================================== +# BEGIN SECTION 3 - WHITELISTING AND BLACKLISTING IP ADDRESSESE AND RANGES +# ======================================================================== -# Geo directive to deny and also whitelist certain ip addresses +# -------------------------------------------------------------------------------------- +# Map all GOOD and BAD IP Addresses and Ranges to a variable called geo $validate_client +# -------------------------------------------------------------------------------------- geo $validate_client { - -# ******************** -# First Our Safety Net -# ******************** - -# Anything not matching our rules is allowed through with default 0; - default 0; -# *********************************** -# Whitelist all your OWN IP addresses -# *********************************** +# ---------------------------------------------- +# Whitelist all your OWN IP addresses and Ranges +# ---------------------------------------------- +# -------------------------------------------------------------------------------------- # WHITELIST all your own IP addresses using the include file below. # New Method Uses the include file below so that when pulling future updates your # whitelisted IP addresses are automatically now included for you. # Read Comments inside whitelist-ips.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files + # whitelist-ips.conf reigns supreme !!! + # Whatever you add to whitelist-ips.conf will be whitelisted FULL STOP + # Anything blacklisted above this line will be over-ridden by whitelist-ips.conf +# -------------------------------------------------------------------------------------- -# ************************************************************************** +# -------------------------------------------------------------------------- # START WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************** +# -------------------------------------------------------------------------- include /etc/nginx/bots.d/whitelist-ips.conf; -# ************************************************************************ +# ------------------------------------------------------------------------ # END WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************ +# ------------------------------------------------------------------------ -# **************** -# Google IP Ranges -# **************** - -# For Safety Sake all Google's Known IP Ranges are all white listed - -# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - 108.177.0.0/17 0; - 172.217.0.0/16 0; - 173.194.0.0/16 0; - 2001:4860:4000::/36 0; - 203.208.60.0/24 0; - 207.126.144.0/20 0; - 209.85.128.0/17 0; - 216.239.32.0/19 0; - 216.58.192.0/19 0; - 2404:6800:4000::/36 0; - 2607:f8b0:4000::/36 0; - 2800:3f0:4000::/36 0; - 2a00:1450:4000::/36 0; - 2c0f:fb50:4000::/36 0; - 35.192.0.0/12 0; - 64.18.0.0/20 0; - 64.233.160.0/19 0; - 64.68.80.0/21 0; - 65.52.0.0/14 0; - 66.102.0.0/20 0; - 66.249.64.0/19 0; - 72.14.192.0/18 0; - 74.125.0.0/16 0; -# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - -# ************** -# Bing IP Ranges -# ************** - -# For Safety Sake all Bing's Known IP Ranges are all white listed - -# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - 131.253.21.0/24 0; - 131.253.22.0/23 0; - 131.253.24.0/21 0; - 131.253.24.0/22 0; - 131.253.32.0/20 0; - 157.54.0.0/15 0; - 157.56.0.0/14 0; - 157.60.0.0/16 0; - 199.30.16.0/24 0; - 199.30.27.0/24 0; - 207.46.0.0/16 0; - 40.112.0.0/13 0; - 40.120.0.0/14 0; - 40.124.0.0/16 0; - 40.125.0.0/17 0; - 40.74.0.0/15 0; - 40.76.0.0/14 0; - 40.80.0.0/12 0; - 40.96.0.0/12 0; -# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - -# ******************** -# Cloudflare IP Ranges -# ******************** - -# For Safety Sake all Cloudflare's Known IP Ranges are all white listed - -# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - 103.21.244.0/22 0; - 103.22.200.0/22 0; - 103.31.4.0/22 0; - 104.16.0.0/12 0; - 108.162.192.0/18 0; - 131.0.72.0/22 0; - 141.101.64.0/18 0; - 162.158.0.0/15 0; - 172.64.0.0/13 0; - 173.245.48.0/20 0; - 188.114.96.0/20 0; - 190.93.240.0/20 0; - 197.234.240.0/22 0; - 198.41.128.0/17 0; - 199.27.128.0/21 0; - 2400:cb00::/32 0; - 2405:8100::/32 0; - 2405:b500::/32 0; - 2606:4700::/32 0; - 2803:f800::/32 0; - 2a06:98c0::/29 0; - 2c0f:f248::/32 0; -# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - -# ************************* -# Wordpress Theme Detectors -# ************************* +# ------------------------------------- +# BLOCK known Wordpress Theme Detectors +# ------------------------------------- # START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ### @@ -7818,10 +7783,11 @@ geo $validate_client { 89.36.223.188 1; #www.cuteseotools.net/wordpress-theme-detector # END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ### -# **************************************** -# NIBBLER - SEO testing and reporting tool -# **************************************** +# ---------------------------------------------- +# BLOCK NIBBLER - SEO testing and reporting tool +# ---------------------------------------------- # See - http://nibbler.silktide.com/ +# ---------------------------------------------- # START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ### 52.201.238.175 1; @@ -7834,33 +7800,10 @@ geo $validate_client { # END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ### -# **************************** -# Known Bad IP's and IP Ranges -# ************************************************* -# Blacklist IP addresses and IP Ranges Customizable -# ************************************************* - -# BLACKLIST all your IP addresses and Ranges using the new include file below. - # New Method Uses the include file below so that when pulling future updates your - # Custom Blacklisted IP addresses are automatically now included for you. - # Read Comments inside blacklist-ips.conf for customization tips. - # Updating the main globalblacklist.conf file will not touch your custom include files - -# ******************************************************************** -# START BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ******************************************************************** - - include /etc/nginx/bots.d/blacklist-ips.conf; - -# ****************************************************************** -# END BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ****************************************************************** - -# ********************************************************************************************* -# KNOWN BAD IP ADDRESSES -# Source: https://github.com/mitchellkrogza/Suspicious.Snooping.Sniffing.Hacking.IP.Addresses -# A Reliable Daily Updated List Included Below of Known Bad IP Addresses, Snoopers and Sniffers -# ********************************************************************************************* +# ----------------------------------------- +# BLOCK KNOWN BAD IP ADDRESSES +# Top known bad IP Adresses from abuseIPDB +# ----------------------------------------- # START KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ### 193.32.163.89 1; @@ -17864,33 +17807,176 @@ geo $validate_client { 46.101.115.65 1; # END KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ### +# -------------------------- +# WHITELIST Google IP Ranges +# -------------------------- + +# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + 108.177.0.0/17 0; + 172.217.0.0/16 0; + 173.194.0.0/16 0; + 2001:4860:4000::/36 0; + 203.208.60.0/24 0; + 207.126.144.0/20 0; + 209.85.128.0/17 0; + 216.239.32.0/19 0; + 216.58.192.0/19 0; + 2404:6800:4000::/36 0; + 2607:f8b0:4000::/36 0; + 2800:3f0:4000::/36 0; + 2a00:1450:4000::/36 0; + 2c0f:fb50:4000::/36 0; + 35.192.0.0/12 0; + 64.18.0.0/20 0; + 64.233.160.0/19 0; + 64.68.80.0/21 0; + 65.52.0.0/14 0; + 66.102.0.0/20 0; + 66.249.64.0/19 0; + 72.14.192.0/18 0; + 74.125.0.0/16 0; +# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + +# ------------------------ +# WHITELIST Bing IP Ranges +# ------------------------ + +# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + 131.253.21.0/24 0; + 131.253.22.0/23 0; + 131.253.24.0/21 0; + 131.253.24.0/22 0; + 131.253.32.0/20 0; + 157.54.0.0/15 0; + 157.56.0.0/14 0; + 157.60.0.0/16 0; + 199.30.16.0/24 0; + 199.30.27.0/24 0; + 207.46.0.0/16 0; + 40.112.0.0/13 0; + 40.120.0.0/14 0; + 40.124.0.0/16 0; + 40.125.0.0/17 0; + 40.74.0.0/15 0; + 40.76.0.0/14 0; + 40.80.0.0/12 0; + 40.96.0.0/12 0; +# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + +# ------------------------------ +# WHITELIST Cloudflare IP Ranges +# ------------------------------ + +# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + 103.21.244.0/22 0; + 103.22.200.0/22 0; + 103.31.4.0/22 0; + 104.16.0.0/12 0; + 108.162.192.0/18 0; + 131.0.72.0/22 0; + 141.101.64.0/18 0; + 162.158.0.0/15 0; + 172.64.0.0/13 0; + 173.245.48.0/20 0; + 188.114.96.0/20 0; + 190.93.240.0/20 0; + 197.234.240.0/22 0; + 198.41.128.0/17 0; + 199.27.128.0/21 0; + 2400:cb00::/32 0; + 2405:8100::/32 0; + 2405:b500::/32 0; + 2606:4700::/32 0; + 2803:f800::/32 0; + 2a06:98c0::/29 0; + 2c0f:f248::/32 0; +# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + +# ------------------------------------------------- +# BLACKLIST IP addresses and IP Ranges Customizable +# ------------------------------------------------- + +# -------------------------------------------------------------------------------------- +# BLACKLIST all your IP addresses and Ranges using the new include file below. + # New Method Uses the include file below so that when pulling future updates your + # Custom Blacklisted IP addresses are automatically now included for you. + # Read Comments inside blacklist-ips.conf for customization tips. + # Updating the main globalblacklist.conf file will not touch your custom include files + # Anything added to blacklist-ips.conf will over-ride anything whitelisted above +# -------------------------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# START BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# -------------------------------------------------------------------- + + include /etc/nginx/bots.d/blacklist-ips.conf; + +# ------------------------------------------------------------------ +# END BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ------------------------------------------------------------------ + +# ---------------------------------------------- +# Whitelist all your OWN IP addresses and Ranges +# ---------------------------------------------- + +# -------------------------------------------------------------------------------------- +# WHITELIST all your own IP addresses using the include file below. + # New Method Uses the include file below so that when pulling future updates your + # whitelisted IP addresses are automatically now included for you. + # Read Comments inside whitelist-ips.conf for customization tips. + # Updating the main globalblacklist.conf file will not touch your custom include files + # whitelist-ips.conf reigns supreme !!! + # Whatever you add to whitelist-ips.conf will be whitelisted FULL STOP + # Anything blacklisted above this line will be over-ridden by whitelist-ips.conf +# -------------------------------------------------------------------------------------- + +# -------------------------------------------------------------------------- +# START WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# -------------------------------------------------------------------------- + + include /etc/nginx/bots.d/whitelist-ips.conf; + +# ------------------------------------------------------------------------ +# END WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ------------------------------------------------------------------------ } +# -------------------------------------------------------------------------------------- # WHITELIST your own IPs from the DDOS Filter # Add your own IP addresses and ranges into the custom include file whitelist-ips.conf # to spare them from the rate limiting DDOS filter. # This section includes the same / single whitelist-ips.conf file so you only # need to edit that include file and have it include here for you too. +# -------------------------------------------------------------------------------------- geo $ratelimited { default 1; -# *************************************************************************** +# --------------------------------------------------------------------------- # START WHITELISTED IP RANGES2 ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# *************************************************************************** +# --------------------------------------------------------------------------- include /etc/nginx/bots.d/whitelist-ips.conf; -# ************************************************************************* +# ------------------------------------------------------------------------- # END WHITELISTED IP RANGES2 ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************* +# ------------------------------------------------------------------------- } -# ***************************************** -# MAP BAD BOTS TO OUR RATE LIMITER FUNCTION -# ***************************************** +# ====================================================================== +# END SECTION 3 - WHITELISTING AND BLACKLISTING IP ADDRESSESE AND RANGES +# ====================================================================== + + +# ============================================ +# BEGIN SECTION 4 - ACTIVATE BLOCKER FUNCTIONS +# ============================================ + +# -------------------------------------------- +# 1. MAP BAD BOTS TO OUR RATE LIMITER FUNCTION +# -------------------------------------------- map $bad_bot $bot_iplimit { 0 ""; @@ -17898,30 +17984,31 @@ geo $ratelimited { 2 $binary_remote_addr; } -# *********************** -# SET RATE LIMITING ZONES -# *********************** +# -------------------------- +# 2. SET RATE LIMITING ZONES +# -------------------------- # BAD BOT RATE LIMITING ZONE -# limits for Zone $bad_bot = 1 -# Nothing Set - you can set a different zone limiter here if you like -# We issue a 444 response instead to all bad bots. +# Rate limiting will only take effect if on any User-Agents with a value of 2 -# limits for Zone $bad_bot = 2 -# this rate limiting will only take effect if you change any of the bots and change -# their block value from 1 to 2. limit_conn_zone $bot_iplimit zone=bot2_connlimit:16m; - limit_req_zone $bot_iplimit zone=bot2_reqlimitip:16m rate=2r/s; + limit_req_zone $bot_iplimit zone=bot2_reqlimitip:16m rate=6r/m; -### *** MAKE SURE TO ADD to your nginx.conf *** -### server_names_hash_bucket_size 64; -### server_names_hash_max_size 4096; -### limit_req_zone $binary_remote_addr zone=flood:50m rate=90r/s; -### limit_conn_zone $binary_remote_addr zone=addr:50m; -### to allow it to load this large set of domains into memory and to set the rate limiting zones for the DDOS filter. +# ========================================== +# END SECTION 4 - ACTIVATE BLOCKER FUNCTIONS +# ========================================== -### THE END of the Long and Winding Road +# ===================== +# END BLOCKER FUNCTIONS +# ===================== -### Also check out my Ultimate Apache Bad Bot Blocker on Github -### https://github.com/mitchellkrogza/apache-ultimate-bad-bot-blocker +### -------------------------------------------- +### HELP SUPPORT THIS PROJECT - Send Me a Coffee +### https://ko-fi.com/mitchellkrog +### -------------------------------------------- + +### FOR APACHE SERVERS +### --------------------------------------------- +### Check out the Ultimate Apache Bad Bot Blocker +### --------------------------------------------- diff --git a/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blacklist-ips.conf index 479de1217..90fe4f9d3 100644 --- a/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 1; +104.154.120.187 1; 127.0.0.1 1; diff --git a/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/whitelist-ips.conf index f45bae986..95fbc95fc 100644 --- a/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/whitelist-ips.conf @@ -1,4 +1,4 @@ -35.192.85.2 0; +104.154.120.187 0; 127.0.0.1 1; 127.0.0.1 1; 127.0.0.1 1; diff --git a/.dev-tools/_conf_files_testing_changes_whitelist/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes_whitelist/conf.d/globalblacklist.conf index 81842727a..9c649adb5 100644 --- a/.dev-tools/_conf_files_testing_changes_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes_whitelist/conf.d/globalblacklist.conf @@ -1,16 +1,21 @@ -### ********************************** -### THE ULTIMATE NGINX BAD BOT BLOCKER -### ********************************** +### ----------------------------------------------------------- +### THE NGINX ULTIMATE BAD BOT, BAD IP AND BAD REFERRER BLOCKER +### ----------------------------------------------------------- ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1643 -### Updated: Thu Jun 27 12:59:42 SAST 2019 +### Version: V3.2019.06.1645 +### Updated: Thu Jun 27 13:24:56 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## +### -------------------------------------------- +### HELP SUPPORT THIS PROJECT - Send Me a Coffee +### https://ko-fi.com/mitchellkrog +### -------------------------------------------- + ############################################################################## # _ __ _ # # / |/ /__ _(_)__ __ __ # @@ -28,12 +33,16 @@ ### and also has rate limiting functionality for bad bots who you only want to rate limit ### and not actually block out entirely. It is very powerful and also very flexible. +### -------------------------------------------------------------------------- ### Created By: https://github.com/mitchellkrogza/ ### Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker ### Copyright Mitchell Krog - ### Contributors: Stuart Cardall - https://github.com/itoffshore +### -------------------------------------------------------------------------- +### -------------------------------------------------------------------------- ### Tested on: nginx/1.10.3 up to latest Mainstream Version (Ubuntu 16.04) +### -------------------------------------------------------------------------- ### This list was developed and is in use on a live Nginx server running some very busy web sites. ### It was built from the ground up using real data from daily logs and is updated almost daily. @@ -41,9 +50,9 @@ ### spam referrers, rogue IP address, scanners, scrapers and domain hijacking sites are extensively checked ### before they are added. It is monitored extensively for any false positives. -### ********* +### --------- ### Features: -### ********* +### --------- ### Clear formatting for Ease of Maintenance. ### Alphabetically ordered lists for Ease of Maintenance. ### Extensive Commenting for Ease of Reference. @@ -54,40 +63,38 @@ ### IP range blocking / whitelisting. ### Rate Limiting Functions. -### ************ +### ------------ ### INSTALLATION -### ************ +### ------------ ### PLEASE use the install, setup and update scripts provided for you to ease your installation. ### This Auto Installation procedure is documented in the README.md and AUTO-CONFIGURATION.md files. ### Installation, Setup and Update Scripts Contributed by Stuart Cardall - https://github.com/itoffshore ### There are also manual configuration instructions provided for those not wishing to do an auto install. -### *********************************************** +### ----------------------------------------------- ### !!!!! PLEASE READ INLINE NOTES ON TESTING !!!!! -### *********************************************** +### ----------------------------------------------- -### Note that: +### SETTINGS: +### --------------------------------------------- ### 0 = allowed - no limits ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### --------------------------------------------- -### ************************************************************ +### ------------------------------------------------------------ ### CONTRIBUTING / PULL REQUESTS / ADDING YOUR OWN BAD REFERRERS -### ************************************************************ +### ------------------------------------------------------------ ### For contributing, corrections or adding bots or referrers to this repo, ### Send a Pull Request (PR) on any of the .list files in the _generator_lists folder ### All Pull Requests will be checked for accuracy before being merged. -# ********************************* -# FIRST BLOCK BY USER-AGENT STRINGS -# ********************************* - -# *********************** +# ----------------------- # !!!!! PLEASE TEST !!!!! -# *********************** +# ----------------------- # ALWAYS test any User-Agent Strings you add here to make sure you have it right # Use a Chrome Extension called "User-Agent Switcher for Chrome" where you can create your @@ -98,36 +105,46 @@ # curl -I http://www.yourdomain.com -A "GoogleBot" ---- GIVES YOU: HTTP/1.1 200 OK (Meaning web page was served to Client) # curl -I http://www.yourdomain.com -A "80legs" ---- GIVES YOU: curl: (52) Empty reply from server (Meaning Nginx gave a 444 Dropped Connection) -# In this section we allow/disallow specific User Agents / Bots. -# ********************************************************************* -# Now we map all good and bad user agents to a variable called $bad_bot -# ********************************************************************* +# ======================= +# START BLOCKER FUNCTIONS +# ======================= + +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# DO NOT EDIT ANYTHING BELOW THIS LINE !!! +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# ============================= +# BEGIN SECTION 1 - USER-AGENTS +# ============================= + +# ALLOW / BLOCK User Agents / Bots + +# ------------------------------------------------------------------- +# Map all GOOD and BAD UA (User Agents) to a variable called $bad_bot +# ------------------------------------------------------------------- map $http_user_agent $bad_bot { - default 0; -# *********************************************** -# Include your Own Custom List of Bad User Agents -# *********************************************** - -# Use the include file below to further customize your own list of additional user-agents you wish to permanently block -# This include file allows whitelisting and blacklisting of anything specified below it, meaning you can over-ride anything in this blocker to your liking. - -# *********************************************************************************** +# ----------------------------------------------------------------------------------- # START CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# *********************************************************************************** +# ----------------------------------------------------------------------------------- + +# Include your Own Custom List of Bad User Agents +# Use the include file below to further customize your own list of additional user-agents you wish to permanently block +# This include file allows whitelisting and blacklisting of anything specified below it. +# This include file alows you to over-ride any Bad / Good UA (Bot) declared in this blocker to your liking. include /etc/nginx/bots.d/blacklist-user-agents.conf; -# ********************************************************************************* +# --------------------------------------------------------------------------------- # END CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ********************************************************************************* +# --------------------------------------------------------------------------------- -# ********************************************* -# Bad User-Agent Strings That We Block Outright -# ********************************************* +# -------------------------------------------------- +# BAD UA (User-Agent) Strings That We Block Outright +# -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)360Spider(?:\b|)" 3; @@ -688,9 +705,16 @@ map $http_user_agent $bad_bot { "~*(?:\b)ZyBorg(?:\b|)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### -# *********************************************** -# Allow Good User-Agent Strings We Know and Trust -# *********************************************** +# -------------------------------------------- +# GOOD UA User-Agent Strings We Know and Trust +# -------------------------------------------- + +# ----------------------------------------------------------------------- +# You can over-ride these in /etc/nginx/bots.d/blacklist-user-agents.conf +# by adding the same UA line there and chaning its value of 1 +# If you think GoogleBot is bad you would simply add them to +# blacklist-user-agents.conf with a value of 1 +# ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)adidxbot(?:\b|)" 0; @@ -727,12 +751,13 @@ map $http_user_agent $bad_bot { "~*(?:\b)yahoo(?:\b|)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### -# *************************************************** -# User-Agent Strings Allowed Through but Rate Limited -# *************************************************** +# -------------------------------------------------------- +# GOOD UA User-Agent Rate Limiting 1 - Disabled by Default +# -------------------------------------------------------- -# Some people block libwww-perl, it used widely in many valid (non rogue) agents -# I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited + # TO ACTIVATE THIS RATE LIMITING Uncomment these two lines in blockbots.conf + #limit_conn bot1_connlimit 100; + #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)jetmon(?:\b|)" 1; @@ -743,9 +768,14 @@ map $http_user_agent $bad_bot { "~*(?:\b)Wget/1.15(?:\b|)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### -# ************************************************************** -# Rate Limited User-Agents who get a bit aggressive on bandwidth -# ************************************************************** +# ------------------------------------------------------- +# GOOD UA User-Agent Rate Limiting 2 - Enabled by Default +# ------------------------------------------------------- + +# ----------------------------------------------------------------------- +# You can over-ride these in /etc/nginx/bots.d/blacklist-user-agents.conf +# by adding the same UA line there and chaning its value of 1 +# ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)Alexa(?:\b|)" 2; @@ -768,163 +798,186 @@ map $http_user_agent $bad_bot { # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } - -# ***************************************** -# SECOND BLOCK BY REFERRER STRINGS AND URLS -# ***************************************** +# =========================== +# END SECTION 1 - USER-AGENTS +# =========================== -# ***************** +# ======================================= +# BEGIN SECTION 2 - REFERRERS AND DOMAINS +# ======================================= + +# ---------------- # PLEASE TEST !!!! -# ***************** - +# ---------------- +# ------------------------------------------------------------------------------------------------------------------------------ # ALWAYS test referrers that you add. This is done manually as follows - +# ------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://anything.adcash.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e http://www.goodwebsite.com/not-adcash --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e http://www.betterwebsite.com/not/adcash --- GIVES YOU: curl: (52) Empty reply from server - +# ------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://www.google.com --- GIVES YOU: full html output of the web page # curl -I http://www.yourdomain.com -e http://www.microsoft.com --- GIVES YOU: full html output of the web page - +# ------------------------------------------------------------------------------------------------------------------------------ # Because of case-insensitive matching any combination of capitilization in the names will all produce a positive hit # make sure you always test thoroughly and monitor logs. This section below also does NOT check for a preceding www. # and it also does not care if the referrer request was sent with http https or even ftp. +# ------------------------------------------------------------------------------------------------------------------------------ -# *********************************************************************** -# Now we map all bad referrer words below to a variable called $bad_words -# *********************************************************************** +# ---------------------------------------------------------------- +# Map all BAD referrer words below to a variable called $bad_words +# ---------------------------------------------------------------- + +# -------------------------------- +# START Bad Referrer Word Scanning +# -------------------------------- map $http_referer $bad_words { default 0; -# ************************** -# Bad Referrer Word Scanning -# ************************** - +# ------------------------------------------------------------------------------------------- # These are Words and Terms often found tagged onto domains or within url query strings. # Create and Customize Your Own Bad Referrer Words Here using the new Include File Method # New Method Uses the include file below so that when pulling future updates your # customized list of bad referrer words are automatically now included for you # Read Comments inside bad-referrer-words.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files - # BE VERY CAREFUL using the bad-referrer-words.conf file - please read the comment and + # BE VERY CAREFUL using this bad-referrer-words.conf file - please read the comments and # examples inside the include file for detailed explanations into how seriously this can # affect your own site from serving assets or other innocent sites from accessing your site - # For safety sake the whitelist-domains.conf file is also loaded here before the bad-referrer-words.conf file. + # For safety sake the whitelist-domains.conf file is also loaded here before the + # bad-referrer-words.conf file is loaded. +# ------------------------------------------------------------------------------------------- -# ************************************************************************ +# ------------------------------------------------------------------------ # START WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************ +# ------------------------------------------------------------------------ include /etc/nginx/bots.d/whitelist-domains.conf; -# ********************************************************************** +# ---------------------------------------------------------------------- # END WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ********************************************************************** +# ---------------------------------------------------------------------- -# ****************************************************************************** +# ------------------------------------------------------------------------------ # START CUSTOM BAD REFERRER WORDS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ****************************************************************************** +# ------------------------------------------------------------------------------ include /etc/nginx/bots.d/bad-referrer-words.conf; -# **************************************************************************** +# ---------------------------------------------------------------------------- # END CUSTOM BAD REFERRER WORDS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# **************************************************************************** +# ---------------------------------------------------------------------------- } +# -------------------------------- +# END Bad Referrer Word Scanning +# -------------------------------- -# ************************* -# Bad Referrer Domain Names -# ************************* -# Now a list of bad referrer urls these domains or any combination of them ie .com .net -# will be blocked out. Doesn't matter if the protocol is http, https or even ftp +# ---------------------------------------- +# START Good and Bad Referrer Domain Names +# ---------------------------------------- +# ------------------------------------------------------------------------------------- +# Good and Bad referrer urls Doesn't matter if the protocol is http, https or even ftp +# ------------------------------------------------------------------------------------- + +# ---------------------- # This section includes: -# ********************** +# ---------------------- +# -------------------------------------------------------------------------------- # Blocking of SEO company Semalt.com (now merged into this one section) # MIRAI Botnet Domains Used for Mass Attacks # Other known bad SEO companies and Ad Hijacking Sites # Sites linked to malware, adware, clickjacking and ransomware +# Domain names and referrers used in referrer spam and seo hijacking +# Whitelisting of your own GOOD domains / referrers +# Whitelisting of any other GOOD domains / referrers you want explicitly NOT block +# -------------------------------------------------------------------------------- -# ***************** +# ---------------- # PLEASE TEST !!!! -# ***************** - +# ---------------- +# ------------------------------------------------------------------------------------------------------------------------------------ # ALWAYS test referrers that you add. This is done manually as follows - +# ------------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://8gold.com --- GIVES YOU: curl: (52) Empty reply from server - +# ------------------------------------------------------------------------------------------------------------------------------------ # Because of case-insensitive matching any combination of capitilization will all produce a positive hit - make sure you always test. - +# ------------------------------------------------------------------------------------------------------------------------------------ # For Example any of the following variations below of 8gold.com will be detected and blocked - +# ------------------------------------------------------------------------------------------------------------------------------------ # curl -I http://www.yourdomain.com -e http://NOT-8gold.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e http://this.is.not8gOlD.net --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e ftp://8gold.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e ftp://www.weare8gold.NET --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e https://subdomain.8gold.com --- GIVES YOU: curl: (52) Empty reply from server # curl -I http://www.yourdomain.com -e https://NOT8GolD.org --- GIVES YOU: curl: (52) Empty reply from server - +# ------------------------------------------------------------------------------------------------------------------------------------ # So if you see a bad referrer from wearegoogle.com and you want to block them just add # them as "~*wearegoogle.com" don't ever go and do something like "~*google(-|.)" you will # kill all your SEO in a week. - +# ------------------------------------------------------------------------------------------------------------------------------------ # To add your own custom bad referrers use the custom include file # /etc/nginx/bots.d/custom-bad-referrers.conf # Or send a Pull Request to add it to the global blacklist for other users. - # In the bad referrers section I also include sites that hotlink images without permission. +# ------------------------------------------------------------------------------------------------------------------------------------ -# *********************************************************************** -# Now we map all good & bad referrer urls to variable called #bad_referer -# *********************************************************************** +# -------------------------------------------------------------------- +# Map all good & bad referrer DOMAINS to a variable called bad_referer +# -------------------------------------------------------------------- map $http_referer $bad_referer { hostnames; default 0; -# ************************************* -# GOOD REFERRERS - Spared from Checking -# ************************************* +# -------------------------------------------- +# GOOD REFERRER DOMAINS - Spared from Checking +# -------------------------------------------- +# --------------------------------------------------------------------------------------- # Add all your own web site domain names and server names in this section - # WHITELIST Your Own Domain Names Here using the new Include File Method # New Method Uses the include file below so that when pulling future updates your # whitelisted domain names are automatically now included for you. # Read Comments inside whitelist-domains.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files +# --------------------------------------------------------------------------------------- -# ************************************************************************ +# ------------------------------------------------------------------------ # START WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************ +# ------------------------------------------------------------------------ include /etc/nginx/bots.d/whitelist-domains.conf; -# ********************************************************************** +# ---------------------------------------------------------------------- # END WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ********************************************************************** +# ---------------------------------------------------------------------- -# *********************************** +# ----------------------------------- # CUSTOM BAD REFERRERS - Add your Own -# *********************************** +# ----------------------------------- # Add any extra bad referrers in the following include file to have them # permanently included and blocked - avoid duplicates in your custom file +# custom-bad-referrers.conf is BOTH a BLACKLIST AND WHITELIST +# custom-bad-referrers.conf ALLOWS complete over-riding of anything +# If you think google.com is bad you would simply add them to +# custom-bad-referrers.conf with a value of 1 -# ************************************************************************* +# ------------------------------------------------------------------------- # START CUSTOM BAD REFERRERS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************* +# ------------------------------------------------------------------------- include /etc/nginx/bots.d/custom-bad-referrers.conf; -# *********************************************************************** +# ----------------------------------------------------------------------- # END CUSTOM BAD REFERRERS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# *********************************************************************** +# ----------------------------------------------------------------------- # START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ### "~*(?:\b)000free\.us(?:\b|)" 1; @@ -7644,137 +7697,49 @@ map $http_referer $bad_referer { # END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ### } +# ===================================== +# END SECTION 2 - REFERRERS AND DOMAINS +# ===================================== -# *********************************************** -# WHITELISTING AND BLACKLISTING IP ADDRESS RANGES -# *********************************************** +# ======================================================================== +# BEGIN SECTION 3 - WHITELISTING AND BLACKLISTING IP ADDRESSESE AND RANGES +# ======================================================================== -# Geo directive to deny and also whitelist certain ip addresses +# -------------------------------------------------------------------------------------- +# Map all GOOD and BAD IP Addresses and Ranges to a variable called geo $validate_client +# -------------------------------------------------------------------------------------- geo $validate_client { - -# ******************** -# First Our Safety Net -# ******************** - -# Anything not matching our rules is allowed through with default 0; - default 0; -# *********************************** -# Whitelist all your OWN IP addresses -# *********************************** +# ---------------------------------------------- +# Whitelist all your OWN IP addresses and Ranges +# ---------------------------------------------- +# -------------------------------------------------------------------------------------- # WHITELIST all your own IP addresses using the include file below. # New Method Uses the include file below so that when pulling future updates your # whitelisted IP addresses are automatically now included for you. # Read Comments inside whitelist-ips.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files + # whitelist-ips.conf reigns supreme !!! + # Whatever you add to whitelist-ips.conf will be whitelisted FULL STOP + # Anything blacklisted above this line will be over-ridden by whitelist-ips.conf +# -------------------------------------------------------------------------------------- -# ************************************************************************** +# -------------------------------------------------------------------------- # START WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************** +# -------------------------------------------------------------------------- include /etc/nginx/bots.d/whitelist-ips.conf; -# ************************************************************************ +# ------------------------------------------------------------------------ # END WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************ +# ------------------------------------------------------------------------ -# **************** -# Google IP Ranges -# **************** - -# For Safety Sake all Google's Known IP Ranges are all white listed - -# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - 108.177.0.0/17 0; - 172.217.0.0/16 0; - 173.194.0.0/16 0; - 2001:4860:4000::/36 0; - 203.208.60.0/24 0; - 207.126.144.0/20 0; - 209.85.128.0/17 0; - 216.239.32.0/19 0; - 216.58.192.0/19 0; - 2404:6800:4000::/36 0; - 2607:f8b0:4000::/36 0; - 2800:3f0:4000::/36 0; - 2a00:1450:4000::/36 0; - 2c0f:fb50:4000::/36 0; - 35.192.0.0/12 0; - 64.18.0.0/20 0; - 64.233.160.0/19 0; - 64.68.80.0/21 0; - 65.52.0.0/14 0; - 66.102.0.0/20 0; - 66.249.64.0/19 0; - 72.14.192.0/18 0; - 74.125.0.0/16 0; -# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - -# ************** -# Bing IP Ranges -# ************** - -# For Safety Sake all Bing's Known IP Ranges are all white listed - -# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - 131.253.21.0/24 0; - 131.253.22.0/23 0; - 131.253.24.0/21 0; - 131.253.24.0/22 0; - 131.253.32.0/20 0; - 157.54.0.0/15 0; - 157.56.0.0/14 0; - 157.60.0.0/16 0; - 199.30.16.0/24 0; - 199.30.27.0/24 0; - 207.46.0.0/16 0; - 40.112.0.0/13 0; - 40.120.0.0/14 0; - 40.124.0.0/16 0; - 40.125.0.0/17 0; - 40.74.0.0/15 0; - 40.76.0.0/14 0; - 40.80.0.0/12 0; - 40.96.0.0/12 0; -# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - -# ******************** -# Cloudflare IP Ranges -# ******************** - -# For Safety Sake all Cloudflare's Known IP Ranges are all white listed - -# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - 103.21.244.0/22 0; - 103.22.200.0/22 0; - 103.31.4.0/22 0; - 104.16.0.0/12 0; - 108.162.192.0/18 0; - 131.0.72.0/22 0; - 141.101.64.0/18 0; - 162.158.0.0/15 0; - 172.64.0.0/13 0; - 173.245.48.0/20 0; - 188.114.96.0/20 0; - 190.93.240.0/20 0; - 197.234.240.0/22 0; - 198.41.128.0/17 0; - 199.27.128.0/21 0; - 2400:cb00::/32 0; - 2405:8100::/32 0; - 2405:b500::/32 0; - 2606:4700::/32 0; - 2803:f800::/32 0; - 2a06:98c0::/29 0; - 2c0f:f248::/32 0; -# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### - -# ************************* -# Wordpress Theme Detectors -# ************************* +# ------------------------------------- +# BLOCK known Wordpress Theme Detectors +# ------------------------------------- # START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ### @@ -7818,10 +7783,11 @@ geo $validate_client { 89.36.223.188 1; #www.cuteseotools.net/wordpress-theme-detector # END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ### -# **************************************** -# NIBBLER - SEO testing and reporting tool -# **************************************** +# ---------------------------------------------- +# BLOCK NIBBLER - SEO testing and reporting tool +# ---------------------------------------------- # See - http://nibbler.silktide.com/ +# ---------------------------------------------- # START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ### 52.201.238.175 1; @@ -7834,33 +7800,10 @@ geo $validate_client { # END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ### -# **************************** -# Known Bad IP's and IP Ranges -# ************************************************* -# Blacklist IP addresses and IP Ranges Customizable -# ************************************************* - -# BLACKLIST all your IP addresses and Ranges using the new include file below. - # New Method Uses the include file below so that when pulling future updates your - # Custom Blacklisted IP addresses are automatically now included for you. - # Read Comments inside blacklist-ips.conf for customization tips. - # Updating the main globalblacklist.conf file will not touch your custom include files - -# ******************************************************************** -# START BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ******************************************************************** - - include /etc/nginx/bots.d/blacklist-ips.conf; - -# ****************************************************************** -# END BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ****************************************************************** - -# ********************************************************************************************* -# KNOWN BAD IP ADDRESSES -# Source: https://github.com/mitchellkrogza/Suspicious.Snooping.Sniffing.Hacking.IP.Addresses -# A Reliable Daily Updated List Included Below of Known Bad IP Addresses, Snoopers and Sniffers -# ********************************************************************************************* +# ----------------------------------------- +# BLOCK KNOWN BAD IP ADDRESSES +# Top known bad IP Adresses from abuseIPDB +# ----------------------------------------- # START KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ### 193.32.163.89 1; @@ -17864,33 +17807,176 @@ geo $validate_client { 46.101.115.65 1; # END KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ### +# -------------------------- +# WHITELIST Google IP Ranges +# -------------------------- + +# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + 108.177.0.0/17 0; + 172.217.0.0/16 0; + 173.194.0.0/16 0; + 2001:4860:4000::/36 0; + 203.208.60.0/24 0; + 207.126.144.0/20 0; + 209.85.128.0/17 0; + 216.239.32.0/19 0; + 216.58.192.0/19 0; + 2404:6800:4000::/36 0; + 2607:f8b0:4000::/36 0; + 2800:3f0:4000::/36 0; + 2a00:1450:4000::/36 0; + 2c0f:fb50:4000::/36 0; + 35.192.0.0/12 0; + 64.18.0.0/20 0; + 64.233.160.0/19 0; + 64.68.80.0/21 0; + 65.52.0.0/14 0; + 66.102.0.0/20 0; + 66.249.64.0/19 0; + 72.14.192.0/18 0; + 74.125.0.0/16 0; +# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + +# ------------------------ +# WHITELIST Bing IP Ranges +# ------------------------ + +# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + 131.253.21.0/24 0; + 131.253.22.0/23 0; + 131.253.24.0/21 0; + 131.253.24.0/22 0; + 131.253.32.0/20 0; + 157.54.0.0/15 0; + 157.56.0.0/14 0; + 157.60.0.0/16 0; + 199.30.16.0/24 0; + 199.30.27.0/24 0; + 207.46.0.0/16 0; + 40.112.0.0/13 0; + 40.120.0.0/14 0; + 40.124.0.0/16 0; + 40.125.0.0/17 0; + 40.74.0.0/15 0; + 40.76.0.0/14 0; + 40.80.0.0/12 0; + 40.96.0.0/12 0; +# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + +# ------------------------------ +# WHITELIST Cloudflare IP Ranges +# ------------------------------ + +# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + 103.21.244.0/22 0; + 103.22.200.0/22 0; + 103.31.4.0/22 0; + 104.16.0.0/12 0; + 108.162.192.0/18 0; + 131.0.72.0/22 0; + 141.101.64.0/18 0; + 162.158.0.0/15 0; + 172.64.0.0/13 0; + 173.245.48.0/20 0; + 188.114.96.0/20 0; + 190.93.240.0/20 0; + 197.234.240.0/22 0; + 198.41.128.0/17 0; + 199.27.128.0/21 0; + 2400:cb00::/32 0; + 2405:8100::/32 0; + 2405:b500::/32 0; + 2606:4700::/32 0; + 2803:f800::/32 0; + 2a06:98c0::/29 0; + 2c0f:f248::/32 0; +# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### + +# ------------------------------------------------- +# BLACKLIST IP addresses and IP Ranges Customizable +# ------------------------------------------------- + +# -------------------------------------------------------------------------------------- +# BLACKLIST all your IP addresses and Ranges using the new include file below. + # New Method Uses the include file below so that when pulling future updates your + # Custom Blacklisted IP addresses are automatically now included for you. + # Read Comments inside blacklist-ips.conf for customization tips. + # Updating the main globalblacklist.conf file will not touch your custom include files + # Anything added to blacklist-ips.conf will over-ride anything whitelisted above +# -------------------------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# START BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# -------------------------------------------------------------------- + + include /etc/nginx/bots.d/blacklist-ips.conf; + +# ------------------------------------------------------------------ +# END BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ------------------------------------------------------------------ + +# ---------------------------------------------- +# Whitelist all your OWN IP addresses and Ranges +# ---------------------------------------------- + +# -------------------------------------------------------------------------------------- +# WHITELIST all your own IP addresses using the include file below. + # New Method Uses the include file below so that when pulling future updates your + # whitelisted IP addresses are automatically now included for you. + # Read Comments inside whitelist-ips.conf for customization tips. + # Updating the main globalblacklist.conf file will not touch your custom include files + # whitelist-ips.conf reigns supreme !!! + # Whatever you add to whitelist-ips.conf will be whitelisted FULL STOP + # Anything blacklisted above this line will be over-ridden by whitelist-ips.conf +# -------------------------------------------------------------------------------------- + +# -------------------------------------------------------------------------- +# START WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# -------------------------------------------------------------------------- + + include /etc/nginx/bots.d/whitelist-ips.conf; + +# ------------------------------------------------------------------------ +# END WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ------------------------------------------------------------------------ } +# -------------------------------------------------------------------------------------- # WHITELIST your own IPs from the DDOS Filter # Add your own IP addresses and ranges into the custom include file whitelist-ips.conf # to spare them from the rate limiting DDOS filter. # This section includes the same / single whitelist-ips.conf file so you only # need to edit that include file and have it include here for you too. +# -------------------------------------------------------------------------------------- geo $ratelimited { default 1; -# *************************************************************************** +# --------------------------------------------------------------------------- # START WHITELISTED IP RANGES2 ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# *************************************************************************** +# --------------------------------------------------------------------------- include /etc/nginx/bots.d/whitelist-ips.conf; -# ************************************************************************* +# ------------------------------------------------------------------------- # END WHITELISTED IP RANGES2 ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### -# ************************************************************************* +# ------------------------------------------------------------------------- } -# ***************************************** -# MAP BAD BOTS TO OUR RATE LIMITER FUNCTION -# ***************************************** +# ====================================================================== +# END SECTION 3 - WHITELISTING AND BLACKLISTING IP ADDRESSESE AND RANGES +# ====================================================================== + + +# ============================================ +# BEGIN SECTION 4 - ACTIVATE BLOCKER FUNCTIONS +# ============================================ + +# -------------------------------------------- +# 1. MAP BAD BOTS TO OUR RATE LIMITER FUNCTION +# -------------------------------------------- map $bad_bot $bot_iplimit { 0 ""; @@ -17898,30 +17984,31 @@ geo $ratelimited { 2 $binary_remote_addr; } -# *********************** -# SET RATE LIMITING ZONES -# *********************** +# -------------------------- +# 2. SET RATE LIMITING ZONES +# -------------------------- # BAD BOT RATE LIMITING ZONE -# limits for Zone $bad_bot = 1 -# Nothing Set - you can set a different zone limiter here if you like -# We issue a 444 response instead to all bad bots. +# Rate limiting will only take effect if on any User-Agents with a value of 2 -# limits for Zone $bad_bot = 2 -# this rate limiting will only take effect if you change any of the bots and change -# their block value from 1 to 2. limit_conn_zone $bot_iplimit zone=bot2_connlimit:16m; - limit_req_zone $bot_iplimit zone=bot2_reqlimitip:16m rate=2r/s; + limit_req_zone $bot_iplimit zone=bot2_reqlimitip:16m rate=6r/m; -### *** MAKE SURE TO ADD to your nginx.conf *** -### server_names_hash_bucket_size 64; -### server_names_hash_max_size 4096; -### limit_req_zone $binary_remote_addr zone=flood:50m rate=90r/s; -### limit_conn_zone $binary_remote_addr zone=addr:50m; -### to allow it to load this large set of domains into memory and to set the rate limiting zones for the DDOS filter. +# ========================================== +# END SECTION 4 - ACTIVATE BLOCKER FUNCTIONS +# ========================================== -### THE END of the Long and Winding Road +# ===================== +# END BLOCKER FUNCTIONS +# ===================== -### Also check out my Ultimate Apache Bad Bot Blocker on Github -### https://github.com/mitchellkrogza/apache-ultimate-bad-bot-blocker +### -------------------------------------------- +### HELP SUPPORT THIS PROJECT - Send Me a Coffee +### https://ko-fi.com/mitchellkrog +### -------------------------------------------- + +### FOR APACHE SERVERS +### --------------------------------------------- +### Check out the Ultimate Apache Bad Bot Blocker +### --------------------------------------------- diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 2f7d1d3a5..9c649adb5 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1644 -### Updated: Thu Jun 27 13:12:23 SAST 2019 +### Version: V3.2019.06.1645 +### Updated: Thu Jun 27 13:24:56 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 453934fe3..19a19648d 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1644 -### Updated: Thu Jun 27 13:12:21 SAST 2019 +### Version: V4.2019.06.1645 +### Updated: Thu Jun 27 13:24:55 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 479de1217..90fe4f9d3 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.85.2 1; +104.154.120.187 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index abef879ca..cc3f55139 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,50 +1,50 @@ -AiHitBot -Asterias -backlink-check -BlowFish -CheTeam -CrunchBot -Ebingbong -EyeNetIE -GermCrawler -Go-Ahead-Got-It -Gotit -GT::WWW -Humanlinks -Image Sucker -Indy Library -JennyBot -Larbin -Lmspider -Mass Downloader -netEstate NE Crawler -Nettrack -NetZIP -Nutch -OpenVAS -OutclicksBot +Aboundex +Bandit +CherryPicker +Collector +Craftbot +DomainAppender +EasyDL +EMail Siphon +Fimap +Firefox/7.0 +GetWeb +Gigablast +GoZilla +Go!Zilla +HaosouSpider +HTMLparser +InfoNaviRobot +Jbrofuzz +LinkextractorPro +LinqiaMetadataDownloaderBot +LWP::Simple +Masscan +MFC_Tear_Sample +MIDown tool +Ninja PageAnalyzer -PictureFinder -Pixray -scan.lol -SEOkicks-Robot -SiteSnagger -Site Sucker -Snapbot -Sottopop -Surfbot -Suzuran -Szukacz +Psbot +RankActiveLinkBot +RankFlex +RankingBot2 +RankurBot +ScreenerBot +Searchestate +SemrushBot +SISTRIX +SpankBot T8Abot -Titan -Toweyabot -trendiction.com -Webalta -WebAuto -WebImageCollector -WebmasterWorldForumBot -WebWhacker -WISENutbot -WWW::Mechanize -Zeus -ZmEu +Teleport +Tracemyfile +VB Project +VeriCiteCrawler +Wallpapers/3.0 +WebCopier +WEBDAV +WebLeacher +WebsiteQuester +WeSEE +Whatweb +Xaldon_WebSpider +zgrab diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index b0afdfd84..5ec28b272 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -ADmantX +404checker +80legs +Abonti AhrefsBot AIBOT AiHitBot Aipbot Alexibot AllSubmitter -AlphaBot -Anarchie -Apexoo -ASPSeek -Asterias +archive.org_bot +Attach +autoemailspider BacklinkCrawler +BackStreet BackWeb -Badass -BBBike -Bigfoot +Bandit +BetaBot Blackboard -Blow -Boardreader -Bolt +Black Hole +BlackWidow +BLEXBot +BlowFish BotALot -Buddy BuiltBotTough -BuiltWith +BuzzSumo +CATExplorador +CazoodleBot CCBot -Cegbfeieh -CherryPicker -CheTeam -ChinaClaw +CheeseBot +Chlooe Claritybot -Cogentbot +Cliqzbot +cognitiveseo Collector +com.plumanalytics +Copier +CopyRightCheck Copyscape -Cosmos Craftbot crawl.sogou.com -Crescent +CrunchBot CSHttp -Curious +DatabaseDriverMysqli DataCha0s -demandbase-bot Deusu +Digincore DigitalPebble -Dirbuster +DIIbot Disco Discoverybot Dispatch -DittoSpyder -DnyzBot DomainSigmaCrawler -Dragonfly -Drip +DomainStatsBot +Dotbot +DSearch DTS Agent EirGrabber -EMail Siphon -EMail Wolf -EroCrawler evc-batch -Express WebPictures -Extreme Picture Finder +Extractor +ExtractorPro Ezooms FDM FemtosearchBot -FHscan -Fimap -Firefox/7.0 -Flunky +FlashGet Foobot -FyberSpider -Fyrebot GalaxyBot -Genieo -Getintent GetRight +GetWeb +Gigablast +Gigabot G-i-g-a-b-o-t -Gotit GrabNet -GrapeFX +Grafula GrapeshotCrawler -GridBot -Haansoft -HEADMasterSEO -Heritrix -HTMLparser -Humanlinks -HybridBot -Iblog +HaosouSpider +Harvest +Hloader +HTTrack Id-search +IlseBot +Image Fetch Image Sucker +IndeedBot Indy Library -InfoNaviRobot InfoTekies -Intelliseek -Iskanie -JamesBOT +InternetSeer +IRLbot Jbrofuzz +JikeSpider JOC Web Spider -Jorgee JustView -Jyxobot -Keyword Density -Lanshanbot LexiBot Lftp -LinkextractorPro -LinkpadBot +Lightspeedsystems +Likse +LinksManager LinkWalker -LinqiaScrapeBot +LinqiaMetadataDownloaderBot +LinqiaRSSBot +Lipperhey +Lipperhey Spider +Ltx71 +lwp-request +LWP::Simple +Mag-Net +Mail.RU_Bot +Majestic12 Majestic SEO Majestic-SEO MarkWatch -Masscan -Mass Downloader Mata Hari -meanpathbot +Mediatoolkitbot MegaIndex.ru -Metauri +MFC_Tear_Sample Microsoft Data Access -Microsoft URL Control -MIDown tool Mister PiX +MJ12bot Mojeek -Morfeus Fucking Scanner -Mr.4x3 -MSFrontPage -MSIECrawler -muhstik-scan -Musobot -Nameprotect -Navroad +Msrabot +MS Web Services Client Protocol Needle -NetAnts Netcraft -Nettrack -Nibbler +netEstate NE Crawler +NetLyzer +NetSpider +Netvibes NICErsPRO +Niki-bot +NimbleCrawler +Nimbostratus Ninja -Octopus -Offline Explorer -OnCrawl -Openvas +Nutch +oBot +Openfind +OpenVAS +OrangeBot OrangeSpider -page scorer +OutclicksBot +Page Analyzer +PageGrabber Panscient -Papa Foto +Pavuk pcBrowser PeoplePal Picsearch PictureFinder -Pi-Monster +Pimonster +Pixray +PleaseCrawl +plumanalytics Pockey +POE-Component-Client-HTTP Probethenet +PxBroker +QueryN Metasearch Quick-Crawler -RankActiveLinkBot +RankActive RankFlex -RankingBot RankingBot2 -Rankivabot -RebelMouse -RedesScrapy +Reaper RepoMonkey -RocketCrawler -SBIder +Ripper +SalesIntelligent +ScanAlert Scanbot -ScoutJet -SearchmetricsBot +scan.lol Semrush -SemrushBot +SEOkicks +SEOkicks-Robot SEOlyticsCrawler -SEOprofiler +Seomoz seoscanners -SeoSiteCheckup -sexsearcher -SiteExplorer +SEOstats +serpstatbot +Shodan +Siphon +SISTRIX Siteimprove -SiteSucker +SiteLockSpider +SiteSnagger +Site Sucker Sitevigil SlySearch -SMTBot Snake -Snoopy -SocialRankIOBot -sogouspider +Sociscraper +Sogou web spider Sosospider Sottopop -SpaceBison -SpankBot +Spammen +Spanner SputnikBot spyfu -Sqlmap Sqworm Steeler -Surfbot +Sucker +Sucuri SurveyBot +Suzuran Szukacz T0PHackTeam -Teleport +tAkeOut +Telesphoreo Telesphorep The Intraformant -TheNomad -Thumbor -Trendiction +Titan +Toata +Toweyabot Trendictionbot -trendiction.com trendiction.de -Turingos TurnitinBot TwengaBot Twice -UnisterBot -URLy.Warning +Typhoeus +Upflow Vacuum -VB Project +Vagabondo VCI -VoidEYE -Voltron +VidibleScraper +Virusdie +Voil Wallpapers/3.0 -WallpapersHD -WBSearchBot +Webalta +WebBandit WebCollage Web Collage -WebCopier Web Enhancer -Web Fuck +WebFuck WebImageCollector WebLeacher -Web Pix +WebPix WebReaper WebSauger Web Sauger -Webshag +WebsiteQuester Website Quester -Webster -WebStripper -WebZIP -WeSEE -Whacker -Whatweb -WinHTTrack -WISENutbot -Wprecon +Whack +Wonderbot +Woobot +WPScan +WWW-Collector-E +WWW-Mechanize +WWW::Mechanize WWWOFFLE +x09Mozilla x22Mozilla Xaldon_WebSpider -Zade -Zauba +xpymep1.exe zauba.io -Zermelo Zitebot +ZmEu diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index f97702de7..0aa5a5801 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider 404checker -404enemy -Aboundex ADmantX -AfD-Verbotsverfahren AhrefsBot AIBOT -AiHitBot -Aipbot -Apexoo -archive.org_bot -Attach -Badass +Alexibot +AlphaBot +Anarchie +Asterias +Backlink-Ceck +BacklinkCrawler +BackWeb Bandit Barkrowler -Battleztar Bazinga -BBBike -Bitacle +Bigfoot Black Hole -BlackWidow -BotALot -Brandprotect +BLEXBot +Blow +BlowFish +Bolt Brandwatch -Buddy -BuiltWith Bullseye -CCBot +Calculon Cegbfeieh -CheTeam +CheeseBot +CherryPicker +ChinaClaw +Chlooe Claritybot Cloud mapping coccocbot-web -Cogentbot cognitiveseo -com.plumanalytics +Collector Copier -Cosmos +CopyRightCheck +Copyscape +Craftbot crawler4j -crawl.sogou.com -CrazyWebCrawler -Custo +crawler.feedback +Crescent DatabaseDriverMysqli DataCha0s DBLBot -demandbase-bot Demon -Deusu -Devil -DigitalPebble -DIIbot Disco +Discoverybot +Dispatch DnyzBot +DomainAppender DomainSigmaCrawler -Dotbot -Drip -EasyDL Ebingbong eCatch ECCP/1.0 -EirGrabber -EMail Siphon +EMail Wolf EroCrawler -ExtLinksBot +Evil +Express WebPictures Extractor -ExtractorPro Extreme Picture Finder EyeNetIE +Ezooms +facebookscraper +FemtosearchBot +FHscan Firefox/7.0 FlashGet Foobot -FyberSpider -Genieo -Getintent +Freeuploader +Fyrebot +GalaxyBot +GermCrawler GetRight -GetWeb Gotit -Go!Zilla -Grabber +GoZilla GrabNet +Grafula GrapeshotCrawler GridBot Haansoft HaosouSpider Havij -HMView +HTMLparser HTTP::Lite -HTTrack -Humanlinks Iblog -IDBot +Id-search IlseBot -IndeedBot +Indy Library InfoTekies -instabid -InternetSeer -ips-agent +Intelliseek +Internet Ninja +internetVista monitor +Iria IRLbot Iskanie -JamesBOT -JennyBot +Jbrofuzz +JetCar +Jetty Joomla Jorgee JustView -Jyxobot Kenjin Spider -Larbin -LeechFTP -LexiBot -Lightspeedsystems -LinkpadBot +Keyword Density +Kozmosbot +Libwhisker +Linkdexbot +LinkextractorPro LinkScan LinksManager LinqiaMetadataDownloaderBot +LinqiaRSSBot LinqiaScrapeBot +Lipperhey +Lipperhey Spider +Litemage_walker Lmspider +LNSpiderguy +Ltx71 lwp-trivial -Magnet -magpie-crawler -Mail.RU_Bot +Mag-Net +MarkWatch Masscan -Mass Downloader -Mata Hari MauiBot +Meanpathbot MeanPath Bot +Mediatoolkitbot mediawords -Metauri -Microsoft Data Access -MIDown tool +MIIxpc MJ12bot -Mojeek Morfeus Fucking Scanner -Mr.4x3 +muhstik-scan +Musobot +Name Intelligence Nameprotect Navroad Needle -Nessus Netcraft +Nettrack Net Vampire Netvibes -Nibbler NICErsPRO Niki-bot -Nikto -Ninja -Nutch -Openfind -OpenLinkProfiler +Nmap +Octopus Openvas OpenVAS -OutclicksBot +OrangeBot +OrangeSpider OutfoxBot -PageGrabber -PageScorer -Pandalytics -Panscient -PECL::HTTP -PeoplePal +PageAnalyzer +Page Analyzer +Papa Foto +pcBrowser +PHPCrawl Picscout -ProPowerBot +Pixray +plumanalytics +Pockey +POE-Component-Client-HTTP +ProWebWalker Pump -PxBroker -QueryN Metasearch +PyCurl Quick-Crawler RankActive RankFlex RankingBot2 -RankurBot -RealDownload -Reaper +Rankivabot RebelMouse Recorder -s1z.ru +RedesScrapy +ReGet SalesIntelligent -ScanAlert -scan.lol -ScreenerBot -SearchmetricsBot +SBIder +Screaming +Searchestate SemrushBot -SEOkicks-Robot -SEOstats +SEOkicks +SeoSiteCheckup +Siphon SISTRIX -SiteExplorer +Sitebeam +Siteimprove SiteLockSpider -SiteSnagger SiteSucker -Site Sucker SlySearch -Snake -Snapbot Snoopy +SocialRankIOBot Sociscraper -sogouspider -Sogou web spider +Sottopop SpaceBison -Spanner +Spammen +sp_auditbot +SputnikBot +Sqlmap Sqlworm Sqworm -Steeler -SuperHTTP -Surfbot -SurveyBot +Stripper +Sucker +Swiftbot T8Abot -tAkeOut -Teleport -Telesphoreo +TeleportPro Telesphorep -Thumbor -Toweyabot -Tracemyfile -trendiction.de -TurnitinBot -TwengaBot -Twice +The Intraformant +Titan +Trendiction +Trendictionbot UnisterBot Upflow -URLy.Warning +URLy Warning Vacuum -VCI +Vagabondo +VB Project VeriCiteCrawler -Virusdie -WBSearchBot +Voil +Voltron +Web Auto +WebBandit WebCollage Web Collage -WebCopier +WEBDAV WebEnhancer -WebFetch +Web Enhancer Web Fetch WebFuck +WebLeacher +webmeup-crawler Web Pix +WebReaper +WebSauger Web Sauger +Webshag WebsiteExtractor WebsiteQuester -Website Quester -WebStripper WebWhacker -WebZIP -Whatweb +WeSEE +Whacker Who.is Bot +WinHTTrack +WISENutbot +Woobot Wotbox -WWW-Mechanize +WPScan WWW::Mechanize x09Mozilla +x22Mozilla Xaldon_WebSpider Xenu xpymep1.exe -Zauba -zauba.io -Zermelo +YoudaoBot +Zade Zitebot +ZmEu diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index ccff99a78..c0da05ea3 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,100 +1,100 @@ -3th.co.in -5i2.net -academia-nsk.org -adtech.de -advancedmassagebysara.com -ahmedabadwebs.com -akama.com -amt-k.ru -amyfoxfitness.com -anime.dougasouko.com -apifasterlightin-a.akamaihd.net -ap.senai.br -asrvrep-a.akamaihd.net -bannerconnect.net -bear.gotcher.us -bet-prognoz.com -blackhatworth.com -blavia.00author.com -bloglag.com -bluesalt.co -briomotor.co -bugof.gq -buntube.net -cementaresearch.se -chatroulette.life -chatseo.com -cowblog.fr -crynet.cc -demenageur.com -divci-hry.info -domain.webkeyit.com -doublepimp.com -ee77ee.com -elektrozigaretten1.postbit.com -eshop.md -exchanges-bet.com -extlabs.io -freshberry.com.ua -googlemare.com -gotcher.us -gpirate.com -hermesreplica.win -hitsbox.info -hol.es -igadgetsworld.com -immobiliaremassaro.com -interfucks.net -irkutsk.zrus.org -kinohall.ru -klikbonus.com -likrot.com -lottospring.com -monarchfind-a.akamaihd.net -montredemarque.nl -mrbitsandbytes.com -mvpicton.co.uk -myhealthcare.com -myxdate.info -nac-bearings.ru -new-post.tk -nsatc.net -offf.info -oneclickfiles.com -partner-host.men -partnerline.men -pastaleads.com -pawli.eu -pochemychka.net -pornoblood.com -pornosee.info -porno-video-chati.ru -remont-comp-pomosh.ru -ring4rhino.com -royalads.net +00it.com +24x7-server-support.site +4replicawatch.net +7makemoneyonline.com +adult3dgames.com +ad-words.ru +arvut.org +axbocz.net +becuo.com +bildsuche.ru +buyessay3.blogspot.ru +buyfriend.ru +club-musics.ru +collegeessay19.blogspot.ru +compliance-donald.xyz +custom-product-labels.com +downloadeer.net +euroskat.ru +everypony.ru +ezigarettenkaufen2.dreamwidth.org +felizporno.com +flipper.top +flowersbazar.com +fullgirl.ru +geoads.com +gloverid.site +gosreg.amchs.ru +gotwebsite1.com +graphid.com +growshop.es +gsasearchengineranker.top +gsmtlf.ru +hao123.com +hostnow.men +ideibiznesa2015.ru +incanto.in.ua +infogame.name +inspiring-desperate.tk +iomoio.net +iscblog.info +istizanidineanopiate.blogspot.com +joingames.org +journeydownthescale.info +juliadiets.com +kadashihotel.com +krasnodar.zrus.org +lapitec.eu +lockerz.com +lovi-moment.com.ua +mesto-x.com +mirtorrent.net +mmostrike.ru +moonci.ru +mtmtv.info +mypornfree.ru +narutonaruto.ru +notasprensa.info +o-o-6-o-o.ru +pattersonsweb.com +pdamods.ru +pflexads.com +pixelrz.com +pony-business.com +potoideas.us +promodj.com +pskcijdc.bloger.index.hr +razorweb-a.akamaihd.net +richinvestmonitor.com +ric.info +rimedia.org +rockprogblog.com +rucrypt.com +sanatorrii.ru sbricur.com -seojokes.net -socialseet.ru -soc-proof.su -soheavyblog.com -sonata-arctica.wz.cz -soviet-portal.do.am -spb.afora.ru -spidtest.org -stroiminsk.org -tackletarts.co -thefarmergame.com -torrentgamer.net -tracfone.com -trichizobswiv.agddns.net -umityangin.net -urlopener.blogspot.com.au -vertaform.com -vkmusics.ru -webjam.com -whiteproduct.com -xn----itbeirbjbi7bc6bh2d.xn--p1ai -xtube.com -youdao.com -zeroredirect8.com -zverokruh-shop.cz +sharebutton.org +shemalegalls.blogporn.in +siteheart.net +sitevalued.com +skyway24.ru +snow.nvr163.com +sptslmtrafms.com +sugarkun.com +taaaak.com +thebestphotos.eu +thedownloadfreeonlinegames.blogspot.com +topshef.ru +turizmus.us +uggbootsoutletsale.us +unblocksit.es +uptimebot.net +videochat.ph +vzubah.com +winterclassichockeyjerseys.com +winx-play.ru +woman-h.ru +xaijo.com +yellowstonesafaritours.com +zixizop.net.ru +zynax.ua +zzbroya.com.ua diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 5aed84506..d865372a9 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,1000 +1,1000 @@ -00author.com -00go.com -0c47f8422d3f.com -1000n1.ru -100dollars-seo.com +03p.info +0lovespells0.blogspot.com +108shot.com 11pikachu.ru -1-99seo.com -2000k.ru -2015god.org -21h2o.com -24videos.tv +123kuma.com +1688.com +1millionusd.xyz +1qingdao.com +24x7-server-support.site +2daytrendingnews.com +2itech.ru +3rbseyes.com +40cg.com 4kepics.com -4k-player.pl -51.la -57883.net -5tey7463.weebly.com -5u.com -8kisses.eu -8si.ru -aac52006ec82a24e08b665f4db2b5013f7.com -abbanreddy.com -abcdeg.xyz +4kpics.rocks +5elementov.ru +7makemoneyonline.com +7minuteworkout.com +883zy.com +aa08daf7e13b6345e09e92f771507fa5f4.com +aarbur.com +aaronabel.com +aasoldes.fr +abcdefh.xyz +absugars.com academia-nsk.org -acgs.tk +acheterviagrafr24.com adanih.com -adcash.com -adclickservice.com adclickthru.net -add-add.men -addtoadd.men -adman.se -admatic.com.tr -adsref.men -adtech.fr -adtiger.tk -adultgalls.com -advertisingtag.net -aghanyna.com -akita.kz -albamargroup.com -albumsuper.info +adelly.bg +adloads.com +adobereader-free.ru +ads-cool.pro +adultmeetonline.info +adultwebhosting.info +advancedsoftwaresupport.com +adventureparkcostarica.com +advertex.info +adviceforum.info +ad-words.ru +affrh2015.com +agardomains.com +agecheckadult.com +air-edem.ru +akuhni.by +alert-fjg.xyz alert-jdh.xyz -alessandraleone.com alevinefesleri.com -algarveglobal.com -alienwheels.de -alkoravto.ru -all4wap.ru -allcredits.su -alldownload.pw -allkrim.com -allnews24.in +alienwheel.es +alldezservice.kz +allesohnegirls.net +allnews.md allornamenti.com +allpdfmags.net all-streaming-media.com -alltheviews.com -allwomen.info -alphavisions.net -alveris.ru -amanda-porn.ga -amoi.tn -ampower.me -amyfoxfitness.com -anabolics.shop -angel1777.kz -angigreene.com -animaltoplist.com +allwidewallpapers.com +altamayoztourism.com +amigobulls.com +am-se.com +amung.us +anapa-inns.ru +android4fun.org +android-systems.ru +animalrank.com anmysite.com -anonymizeme.pro -anticrawler.org -aosexkontakte.net -apartmentratings.com -apartment.ru -apidigidocketnet-a.akamaihd.net -apilinkswiftco-a.akamaihd.net -apishops.ru -appartement-stumm.at -apper.de +apccargo.com +apelsinnik.website +apiluckyleapnet-a.akamaihd.net +apiportalmorecom-a.akamaihd.net +apispringsmartne-a.akamaihd.net +apiwoodensealcom-a.akamaihd.net +apparel-offer.com +appearance-cool.com +apple.com-cleaner.systems approvedlocal.co.za +appsecurityr.com +arate.ru arcadepages.com -architecturebest.com ardimobileinfo.ml -arvut.org -as5000.com -asmxsatadriverin.aircus.com -asrvvv-a.akamaihd.net -asscenihotosidea.blogspot.com -asseenontvonline.ru -audiofree.ru -autochoixspinelli.com -autodo.info -autogrep.ru -auto-moto-elektronika.cz -aversis.net -aviav.ru.com -avkzarabotok.com -avon-ukraine.com +arenda-avtoprokat-krasnodar.ru +arenda-yeisk.ru +artparquet.ru +artpicso.com +ascat.porn +asdfg.pro +asociatia-tipografilor-transilvania.ro +asrvrep-a.akamaihd.net +asscenihotosidea.blogspot.co.za +atmovs.com +ausmepa.org.au +autobudpostach.club +autoplate.info +avkzarabotok.info avtocredit-legko.ru +awency.com +ayerbo.xhost.ro azartniy-bonus.com +azazaporn.com +azazu.ru +azbaseimages.net azbukadiets.ru -balitouroffice.com +babyfactory.fr +balans.shahterworld.org balkanfarma.org +balook.com baltgem.com -bamo.xsl.pt -bannerpower.com -bard-real.com.ua -battle.net -bbtec.net -beachdriveblog.com -beachpics.com -becuo.com -begalka.xyz -beget.tech -belstaffstore.online -best-businessman.ru +bankmib.ru +bannerads.de +bannerspace.com +bannerswap.com +bbsoldes.fr +bdsmgalls.net +beatifullandscape.co +beauty-things.com +best-coupon-offer.com best-games.today -bestpornuha.com -best-seo-offer.com -bestsexyblog.com bestssaker.com -bestwebsiteawards.com -betterhdporn.com -betteroffers.review -betterscooter.com -betune.onlinewebshop.net +bet365.com +beta.hotkeys.com +betonka.pro bezcmexa.ru -bimatoprost-careprost.com -binaryoptionscops.info -bizfly.info -bizru.info -bjgugu.net.ua -blackhatworth.com -blockety.co -blog.f00kclan.de -blogos.kz -blogrankers.com -blogs.rediff.com -bloke.com -bluesalt.co -bluesman.nu -bmsco.co -boc.kz -bocoarchives.org +bibys.com +bigpictures.club +biketank.ga +bikini-image.com +bird1.ru +bitcoinremote.com +bitporno.sx +bjanshee.ru +blackcurranthumidifiers.site +blog.koorg.ru +blogsfunky672.weebly.com +blpmovies.com +bluerobot.info +bmw-ark.ru +bnt-team.com +boazpower.com +bobba.dzaba.com +boleznikogi.com +bonky.biz bonzybuddy.com boostmyppc.com -bosefux.esy.es -bpro1.top -bracketsmackdown.com -braindaily.xyz -brandov.ru -bronzeaid-a.akamaihd.net -brothers-smaller.ru -browsepulse-a.akamaihd.net -brus-vsem.ru -bryansk.zrus.org -builtwith.com -bumascloset.com -bumskontakte.org -buttons-for-website.com -buttons-for-your-website.com -buyclomidonlaine.com -buydissertation.net -buy-meds24.com -bwlx.prepedu.cn -cakemediahq.com.au -candelluxsklep.pl -cand.jp -candlehandmade.com -carina-sy.de -cartierbraceletsreplica.pw -cartierlove2u.xyz -cartierring.xyz -cejewelry.xyz -cerev.info -cfjump.com -c-gainsbourg.com -chatseo.com -cheapbarbour.online -cheapjerseysap.com -checkm8.com -chelnytruck.ru +bouda.kvalitne.cz +break-the-chains.com +brg8.com +btc4u.ru +buigas.00it.com +buntube.net +businesn.men +bus-offer.com +buyessayonline19.blogspot.ru +buyhoverboard.com +buynorxx.com +buytizanidineonlinenoprescription.blogspot.com +buzzurl.jp +bvlgariring.xyz +bvlgariwallet.xyz +bytimedance.ru +calc-for-credit.ru +callstevens.com +camdolls.com +candcstuccoandstone.com +canrole.com +cars-modification.net +carstrends2015.com +cartechnic.ru +caveavins.fr +cdncash.net +celebsopics.com +chatroulette.online +cheapcigarettesc.info cherrypointplace.ca -chidporn.com -children-learningreading.info -chuckguilford.com -cialischstgerts.com -ciekawinki.pl -cienum.fr -cinemaenergy-hd.ru +chocolateslim-original.com +chocolateslim-u-srbiji.com +choice-credit.ru +christianlouboutinreplica.pw +ciceros.co +cigarpass.com +cindymatches.com +citizenclsdriveri7.pen.io +citysecurity.nu +classicakuhni.ru cleanersoft.com -cleanmypc.com -clean-start.net +cleanpcnow.com +clevernt.com +click2pawn.com +clickbanksites.info clickhype.com -clickpapa.com clipartnew.com -clodo.ru -clubfashionista.com -cocyq.inwtrade.com -coderstate.com -compliance-brian.xyz -compliance-fred.top -compliance-ivan.top -compliance-jess.top -compliance-julianna.top -compliance-walter.top -connectingsingles.com -construmac.com.mx -cookielawblog.wordpress.com -cookie-law-enforcement-aa.xyz -cookie-law-enforcement-hh.xyz -copro.pw -cosmetic.donna7753191.ru -countercrazy.com -cpajunkies.com -craftinsta.ru -creams.makeforum.eu -creativehutindia.com -creditmoney.com.ua +clippingphotoindia.com +clips.ua.ac.be +club-lukojl.ru +cmhomestayagency.com +collegeessay19.blogspot.ru +compliance-andrew.xyz +compliance-george.top +compliance-olga.top +contentlook.co +copenergo.ru +coprofam.org +cornerstone-countertops.com +cougfan.info +counter.yadro.ru +country-chic.ru +cowlmash.com crynet.cc -culmimg.pw -culturevie.info -custom-product-labels.com -cutalltheshit.com -cxpromote.com -cymbaltaandtizanidine.blogspot.com czat.wp.pl -czeshop.info -d0t.ru -damasarenai.info -date-withme.com -dawlenie.com +dailyrank.net +daooda.com +daretodonate.co +dataloading.net dbmkfhqk.bloger.index.hr -dealighted.com -degerlund.net -deluxedumps.com -deshevo-nedorogo.ru -destinationrealestate.com -detalizaciya-tut.biz +dealitright.click +dear-diary.ru +decenttools.com +decoratinghomes.ga +delta-line.men +detki-opt.ru +deutschehobbyhuren.net +dev78.dioniqlabb.se dev.citetick.com devochki.top -dgfitness.co -digest-project.ru -directrev.com -distver.ru -dividendo.ru -diy-handmade-ideas.com -djekxa.ru +dfiles.me +dichvuvesinhhanoi.com +digilander.libero.it +dipstar.org +discountliv.com +divisioncore.com dlya-android.org -dms-sw.ru -dogclothing.store -dokfilms.net -domain2008.com -domain-tracker.com -donna7753191.ru +dneprsvet.com.ua +doctissimo.fr +dogoimage.com +doktoronline.no +dolohen.com +domik-derevne.ru +domini.cat +domoysshop.ru dostavka-v-ukrainu.ru -dotnetdotcom.org -dprtb.com +dpihatinh.gov.vn dptaughtme.com -droidlook.net -dvervmoskvu.ru -e-biznes.info -econom.co -eldiariodeguadalajara.com -electrik-avenue.com -elektrischezigaretten1.blogse.nl +drugspowerstore.com +drunkenstepfather.com +drunkmoms.net +druzhininevgeniy63.blogspot.com +easytuningshop.ru +e-avon.ru +ebooknovel.club +e-commerce-seo1.com +e-commerce-seo.com +ecommerce-seo.org +ecxtracking.com +editors.choice6912650.hulfingtonpost.com +educatemetv.com +educontest.net +eduinfosite.com +ege-essay.ru +ekto.ee elektrischezigaretten2.beeplog.com -elektrozigaretten1.tumblr.com -el-nation.com +elektrozigarettekaufen2.blogse.nl +elite-sex-finders.com +elmacho.xyz +elvenmachine.com +emailaccountlogin.co empathica.com -englate.com -enhand.se -eorogo.top -eralph.tk -eropornosex.ru -escort-russian.com +empis.magix.net +englishdictionaryfree.com +epngo.bz +ertelecom.ru +escortplius.com +esfchat.tk +eshop.md espaceinventoristes.com -essay-edu.biz -essay-writing.work -este-line.com.ua -etm-consult.de -etur.ru +estibot.com +etenininrade.ga euronis-free.com -europages.com.ru euroskat.ru -evaashop.ru -everflora.ru -extremepornos.net -f00kclan.de -familienzahnaerzte.com -fanpagerobot.com -faracontrol.ir -fastfixing.tech -fast-wordpress-start.com -fatfasts-4tmz.com +evidencecleanergold.com +ewebarticle.info +exchanges-bet.com +executehosting.com +exoneration-project.us +exportshop.us +extrabot.com +extractorandburner.com +extstat.com +ezigarettekaufen2.yolasite.com +ezigarettekaufen.myblog.de +ezigarettenkaufen1.deviantart.com +ezigarettenshop2.postbit.com +ezrvrentals.com +f0815.de +f1nder.org +familyphysician.ru +farm26.ru +fashion.stellaconstance.co +fba-mexico.com fbdownloader.com -fearcrow.com -felizporno.com -femdom.twiclub.in -ferrotodo.com -fertilitetsradgivningen.se +fdzone.org +feeriaclub.ru +fefo.gdn +ferretsoft.com +ff30236ddef1465f88547e760973d70a.com +fickenprivat.info fidelityfunding.com -filesdatabase.net -findsexguide.com -finejewelryshop.ru -finteks.ru -fishtauto.ru -fitnesspiks.com -flavors.me -floridahuntingfishingadventures.com -floridamhca.org -footbalive.org -formseo.com -formularz-konkurs.tk -forsex.info -forum20.smailik.org -fotopop.club -fototravel.eu -frcls.fr -freelinkbuilding.website.tk -freelotto.com -freeuploader.ml -fsalas.com -ftns.ru -fuckingawesome.com -fun-mobi.pl -funnymama.com -funponsel.com -g33.org -gallerylisting.com -game300.ru +fifa-coins.online +filkhbr.com +financetip.eu +fitness-video.net +fivedwld.cf +fiverr.com +forex-indextop20.ru +fortunejack.com +forums.toucharcade.com +foxinsocks.ru +foxydeal.com +frameimage.org +franklinfire.co +fr-bearings.ru +freejabs.com +freenom.link +freeseedsonline.com +free-share-buttons.blogspot.com +freesitetest.com +free-social-buttons.xyz +freeuploader.com +freshsuperbloop.com +freza-sverlo.ru +frighteningremain.cf +fr.netlog.com +fsakhalin.ru +fuckbuddybestgilf.info +funcrushgames.com +fyl.com.ru +galaxyflowers.ru +galeria-zdjec.com +gamedayhouse.com game-mmorpg.net -gdebestkupit.ru -gearsadspromo.club -gemara.com -generic-pills-online.com -gentamicineyedrops.blogspot.com -germetiki.com.ua -ghazel.ru -gheus.altervista.org +gamerscorps.com +gaming-journal.com +gate5.co.za +gateway.zscalerone.net +ge0ip.org +gelezki.com +getaclueamerica.com +getlamborghini.ga +getresponse.com +gettpromos.com +gg.zzyjxs.com +gifspics.com +girlgamerdaily.com +glastecfilms.com.my glavprofit.ru -glavtral.ru -go2mike.ru -gogps.me -gok-kasten.net -golden-praga.ru +globatur.ru +globetrotting-culture.ru +gloverid.site +go2jump.org +gobongo.info +goldadpremium.com +golden-catalog.pro +golmau.host.sk +good-mummy.ru goodprotein.ru -goodwriterssales.com -gopixdatabase.com -gotorussia.com -graphicwe.org -green-tea.tv -grupografico-pilar.com.ar -gsasearchenginerankerdiscount.com -gsasearchenginerankerhelp.com -gsasearchengineranker.space -guildebzh.info +googlemare.com +googlsucks.com +graphid.com +greendream.com.ua +gsasearchengineranker.site +gsbs.com.ua +gtopstats.com guruofcasino.com gwebtools.com.br -hackers-crackers.tk -hacktougroup.ru -handicapbathtubarea.com -handicapvansarea.com -hatdc.org -hawaiielectriclight.com -hawaiisurf.com -hem.passagen.se -hidemyass.com -highland-homes.com -histats.com -hiwibyh.bugs3.com -hmmm.cz -homedecoguide.info -hosting-tracker.com +gyffu.com +habermetre.com +hahashka.ru +handicapvantoday.com +hanink.biz.ly +hard-porn.mobi +hasshe.com +hayate.biz +hcate.com +hd720kino.ru +hdhc.ru +hdpixion.com +headpress.ru +healgastro.com +healmytrauma.info +hebr.myddns-flir.com +herokuapp.com +high-speed1.net +hitsbox.info +homik.pw +horoshieokna.com host-tracker.com +hotblognetwork.com +hotchatdate.com houdom.net -houston-vikings.com -hoverboard360.es -hoverboard360.se howlongdoestizanidinestayinyoursystem.blogspot.com -howmuchdoestizanidinecost.blogspot.com -howopen.ru -hubbble.com -huhn.altervista.org -huimin764128.com -hulfingtonpost.com +hplaserjetpdriver8y.pen.io humanelydrew.com -hut1.ru -hvd-store.com +husky-shop.cz +hyipmanager.in iboss.com +ictizanidinehcl4mg.blogspot.com +idc.com.ua idealtits.net -igru-xbox.net -iideaidekonkatu.info -ikearugs.xyz -ilovevitaly.com -ilovevitaly.ru -imagecoolpub.com -imagerydatabase.com -iminent.com -imitex-plus.ru +ideawheel.com +ideibiznesa2015.ru +ilbe.club +ilikevitaly.com +imgarcade.com +imgata.com +i-midias.net.br +imk.com.ua +importchinacoach-teach.com inbabes.sexushost.com -infokonkurs.ru -infospot.pt -integritylandscapeservices.com -intelhdgraphicsgtdrive6w.metroblog.com +inclk.com +indo-export.ru +innoslicon.com +inspiring-desperate.tk +intellego.info +internetartfair.com +investmac.com +invitefashion.com invivo.hu -iopeninghours.co.uk +inzn.ru ipchicken.com -ip-guide.com -ipornox.xxx -iskalko.ru -istock-mebel.ru -itag.pw -it-max.com.ua -ivanstroi.ru +iqoption-bin.com +irkutsk.online-podarki.com +iscblog.info +isistaylorporn.info +isitpaleo.info +istanbulit.com +ivanovo.zrus.org +iwantmyfreecash.com iweblist.info -izoll.ru -jacago.com -jamiembrown.com -jasonpartington.com +iyasimasennka.com +jackpotchances.com javatex.co.id -jav-fetish.site -javlibrary.cc -jerseychinabizwholesale.com +javitas.info +javrip.net +javstock.com +jennyfire.ru jerseyschinabizwholesale.us -jikoman.info -joingames.org +jetsli.de +jillepille.com +jmat.cn journeydownthescale.info jovencitas.gratis -jrcigars.com -juliadiets.com -jus0wil.pp.ua -kadashihotel.com -kakadu-interior.com.ua -kambasoft.com -kanimage.com -karadene.com -karaganda.xkaz.org -kasino-money.pw -kazan.xrus.org +justbcause.com +jyvopys.com +kaidalibor.de +kandidos.com +karpun-iris.ru +kazinogames.lv kazka.ru -keki.info -kenaba.su -ketrzyn.pl -kicknights.gq -kinofak.net -kino-key.info -kinostorm.net -klumba55.ru -knogg.net -knowyournextmove.com -kollesa.ru -konpax.com +ketoanhanoi.info +keywords-monitoring-your-success.com +keywordsuggests.com +kichenaid.ru +kidskunst.info +kinobest.pl +kinogolos.ru +kinogonew.ru +kmgamex.cf +knigonosha.net +knowsitall.info +koleso24.com.ua +kollekcioner.ru +komp-pomosch.ru +koptims.tiu.ru kostenlos-sexvideos.com -kovrenok.ru -krafte.ru -krasivoe-hd.com -krasivoe-hd.net +kosynka-games.ru kreativperlen.ch -kunstaktien.info -kustanay.kz -kutikomi.net -kvartir-remont.biz +kupit-adenu.ru +kurbappeal.info +kurwa.win +kvartira-sutochno.com kvartiry-remont.ucoz.ru -kw21.org -labafydjxa.su -lacasamorett.com -lacloop.info -lafriore.ru -lanasshop.ru -larutti.ru -laulini.soclog.se -law-check-nine.xyz -lawyers.cafe -lazymanyoga.com -ledis.top -ledpolice.ru +kwzf.net +labvis.host.sk +lanadelreyfans.us +laptop-4-less.com +lasvegaslockandsafe.com +law-check-seven.xyz +layola.biz.tc +leboard.ru +le-clos-des-alouettes.com legionalpha.com -lennatin.info +lego4x4.ru letmacworkfaster.world -letmacwork.world -letolove.ru -letslowbefast.today -lifehacĸer.com -likesuccess.com -linkarena.com -lion.conboy.us +lider-zhaluzi.kiev.ua +lifeinsurancekb.com +linkbolic.com +linkpulse.com lion.snapmap.us -live-sexcam.tk -livingcanarias.com -localflirtbuddies.com -lolnex.us -lomb.co -long-beach-air-conditioning.com -lookingglassemb.com -lovi-moment.com.ua -luckyday.world -lyrster.com -macdamaged.tech -mackeeper-center.club -mactechinfo.info -mail.allnews24.in -makedo.ru -make-money-online.com -makemoneyonline.com -makis.nu -marblestyle.ru -marketingtechniques.info -maslenka.kz -massage-info.nl -mastershef.club -maximilitary.ru -mazda-roadsters.com -m.b00kmarks.com -mbiologi.ru -mcadamssupplyco.com -mebel-alait.ru +liquimondo.com +littleberry.ru +liveinternet.ru +liver-chintai.org +liveshoppersmac.com +lkbennettoutlet.online +locatellicorretor.com.br +locksmith.jp +lombardfinder.ru +lombia.com +lonerangergames.com +louboutinreplica.pw +louboutinshoes.xyz +magnetic-bracelets.ru +malls.com +manuscript.su +manyresultshub-a.akamaihd.net +marketingvici.com +matchpal-a.akamaihd.net +mathgym.com.au +maximpartnerspr.com mebel-vstroika.ru -meccadumps.net -mediawhirl.net -medicineseasybuy.com +medanestesia.ru +medicine-4u.org +medicines-choice.com +medictube.ru medkletki.ru -medmajor.ru -megahdporno.net +megakino.net +mega-polis.biz.ua meltwater.com -members.ghanaweb.com -menstennisforums.com -metaxalonevstizanidine.blogspot.com +mere.host.sk +merryhouse.co.uk +message-warning.net +metalonly.info methodsmarketing.com -mex-annushka.ru -m-google.xyz -micasainvest.com -michaelkorsoutlet.store -mikozstop.com -militarysale.pro -mil-stak.com -minegam.com -minet.club -mir-betting.ru -mir-business-24.ru -mitrasound.ru -mmostrike.ru -mncrftpcs.com +michaelkorssaleoutletonline.net +mielec.pl +minecraft-rus.org +mir-limuzinov.ru +mirtorrent.net +missvietnam.org +mister-shop.com +mjchamonix.org +mmog-play.ru mnogabukaff.net -mobile-appster.ru -mobioffertrck.com -mobstarr.com -mockupui.com -mojpregled.com -monarhs.info +mnogolok.info +mobifunapp.weebly.com +mobilemedia.md +moje-recenze.cz +moncleroutletonline.pw moncleroutletonline.xyz -moneyteam24.com -moneyviking-a.akamaihd.net +monclervests.xyz +monetizer.com-01.site +moneymaster.ru monthlywinners.com -morefastermac.trade -morocco-nomad-excursions.com -mossmesi.com -mozello.ru -mrvideospornogratis.xxx -mulberryoutletonlineeu.com -mundoaberrante.com -musicvidz.ru -mybinaryoptionsrobot.com +moomi-daeri.com +moroccosurfadventures.com +most-kerch.org +mouselink.co +moyakuhnia.ru +moyaterapiya.ru +mp3downloadhq.com +mp3ringtone.info +mrbojikobi4.biz +msfsaar.de +msk-diplomat.com +musezone.su +mustwineblog.com +muz-baza.net +muznachas-service.ru +mwtpludn.review +my-big-family.com mydearest.co -mydownloadengine.com -my-floor.in.ua -mygameplus.com -myhitmp3.club -myhydros.org -myiptest.com -narkologiya-orel.ru +mydirtyhobby.com +mydirtystuff.com +mydoctorok.ru +myftpupload.com +mypets.by +myplaycity.com +mywallpaper.top +na15.ru +nailsimg.com +namenectar.com +narco24.me +nash-krym.info +nastroyke.net nate.com -nationalbreakdown.com -needtosellmyhousefast.com -negral.pluto.ro -net-profits.xyz -newsperuse.com -nextconseil.com -nikhilbahl.com -niroo.info -nonews.co -normalegal.ru -notaria-desalas.com -nottyu.xyz +naturalshair.site +netcheckcdn.xyz +netpics.org +netvouz.com +new-apps.ru +newstraveller.ru +nfljerseysforsalewholesaler.com +nightvision746.weebly.com +nlfjjunb5.ru +notasprensa.info novatech.vn -novodigs.com -nsatc.net +novosibirsk.xrus.org ntic.fr +nufaq.com nyfinance.ml o333o.com -obsessionphrases.com -oddamzadarmo.eu -odesproperty.com -offers.bycontext.com -offf.info -okmedia.sk -okuos.com +odywpjtw.bloger.index.hr +offer.camp +offergroup.info +of-ireland.info +ok-ua.info old-rock.com -omoikiri-japan.ru +onetravelguides.com onlinebay.ru -onlinedomains.ru -onlinemegax.com -online-templatestore.com -o-o-6-o-o.ru -openmediasoft.com -openstat.com -optibuymac.com -optitrade24.com -optom-deshevo.ru -ororodnik.goodbb.ru -osnova3.ru -osoznanie-narkotikam.net -ossmalta.com +online.ktc45.ru +onlinewritingjobs17.blogspot.ru +o-o-8-o-o.com +openlibrary.org +openmultipleurl.com +orgasmatrix.com +orsonet.ru +ostroike.org otbelivanie-zubov.com oynat.info pacificair.com -paleohub.info -panouri-solare-acoperis.com +paidonlinesites.com +palvira.com.ua +panamaforbeginners.com +pandarastore.top +pandroid.co paparazzistudios.com.au -parajumpersjakkesalgnorge.info -park.above.com +partner-cdn.men +partner-host.men +partner-print.men partners-ship.pro -partybunny.ru -pc-services.ru -pc-virus-d0l92j2.pw +partner-stop.men +pastaleads.com +pcads.ru +pcboa.se pdamods.ru -petitions.whitehouse.gov -pewit.pw -phormchina.com -physfunc.ru -picscout.com -piluli.info -pinkduck.ga -pisanieprac.info -piter.xrus.org -piuminiita.com -pix-hd.com -pizza-tycoon.com -pkr1hand.com -pl.id-forex.com -pogruztehnik.ru -polcin.de -pomoc-drogowa.cba.pl -pontiacsolstice.info -popcash.net -popmarker.com -pornhive.org -pornhub-forum.ga -pornoblood.com -porno-chaman.info -pornogig.com -pornophoto.xyz -pornotubexxx.name -pornstartits.xblog.in -portal-eu.ru -porto.abuilder.net -portside.xyz -potolokelekor.ru +pearlisland.ru +pelfind.me +pensplan4u.com +petrovka-online.com +photochki.com +photosaga.info +piccshare.com +picsforkeywordsuggestion.com +pigrafix.at +pirateday.ru +pizda.lol +pk-pomosch.ru +pladform.ru +plaff-go.ru +playmsn.com +playtap.us +pl-top.pl +poffet.net +pokemongooo.ml +popugaychiki.com +porndl.org +pornogratisdiario.com +porno-play.net +pornosee.info +pornpost.in +porn-w.org +portside.cc +postclass.com powc.r.ca.d.sendibm2.com -pozvonim.com -pregnant.guru -preventheadacheguide.info +powenlite24.ru +pozdravleniya-c.ru +praisong.net +pretty-mart.com +priceg.com +printdirectforless.com +privatamateure.com +privatbank46.ru privat-girl.net -privetsochi.ru -prize44.com proekt-gaz.ru -prohoster.info -prombudpostach.com.ua -promgirldresses.xyz -promodj.com -pronekut.com -propranolol40mg.blogspot.com -prostitutki-tyumeni.org -pulseonclick.com -puserving.com -pussyspace.net +profit-opportunity.com +proftests.net +projectforte.ru +prostitutki-almata.org +prostitutki-rostova.org +prostitutki-yaroslavlya.org +prpops.com +psoriasis-file.trade +pufip.com +push-ad.com pxhdwsm.com -pyramidlitho.webs.com -pyrodesigns.com.au -qpypcx.com -quickcashlimited.com -quit-smoking.ga -quizzitch.net +qitt.ru +questionmarque.ch +quickbuck.com +quickchange.cc qwarckoine.com -rada.ru -radiogambling.com +randalljhoward.com rank-checker.online -rankings-analytics.com -razamicroelectronics.com -realresultslist.com -refudiatethissarah.info -registrydomainservices.com -remedyotc.com -remontbiz.ru -remote-dba.de +ranksays.com +ranksonic.com +rcpmda.ikan1080.xyz +real-time-analytics.com +r-e-f-e-r-e-r.com +reimageplus.com +relayblog.com +remont-mobile-phones.ru +rentaremotecomputer.com +rentehno.ru replica-watch.ru +responsinator.com +responsive-test.net +respublica-otel.ru restaurantlescampi.com -restorator-msk.ru -reversing.cc +rethinkwasteni.info +retreatia.com revistaindustria.com -rezeptiblud.ru -rfd-split.hr -rialp.getenjoyment.net -rn-to-bsn.com +richinvestmonitor.com +rome2rio.com romhacking.ru -rosbalt.com.ua -royalcar-ufa.ru -rubanners.com +roofers.org.uk +royalvegascasino.com +rubbed.us +ruclicks.com rucrypt.com -rukino.org ruspdd.com -rybalka-opt.ru +rus-pornuha.com +rusprostitute.com +rvi.biz sady-urala.ru -salut-camp.ru -salutmontreal.com -sammyweaver.com -samolet.fr -sandhillsonline.com -saratov.xrus.org -sa-rewards.co.za -sarm.tk -savememoney.co.za -sbdl.no -scanner-fred.top -scanner-marwin.top -scanner-nelson.top -schoolfiles.net -scripted.com +saitevpatorii.com +sarafangel.ru +sardinie.us +saugatuck.com +sbtdesign.co.uk +scanner-irvin.top +scanner-jack.top +scanner-jane.top +scanner-jess.top +scanner-julianna.top +scanner-margo.top +scanspyware.net +school-diplomat.ru +scrapy.org scrnet.biz.ua -sc-specialhost.com -sdi-pme.com -sdsjweb.com -search.1and1.com -search.alot.com -searchengineranker.email -searchinterneat-a.akamaihd.net -searchlock.com -secretscook.ru -seminarykansas904.ml -semprofile.com -semrush.com -seoanalyses.com -seolab.top -seorank.info -serdcenebolit.com -servisural.ru -sethrollins.net -sevenstars7.com -sexad.net -sexblog.pw +seansonline24.pl +searchmywindow-a.akamaihd.net +seasaltwithfood.com +see-your-website-here.com +seksvideoonlain.com +seo-2-0.com +seocheckupx.com +seorankinglinks.com +serialsway.ucoz.ru +service-core.ru +sexflirtbook.com +sexgalleries.top sexkontakte-seite.com sexpartygirls.net -sexs-foto.com -sexs-foto.top -sexstream.pl +sexpornotales.com +sexreliz.com +sexsaoy.com +sex-tracker.com +sex-watch.com +sexy-pings.com +sexyteens.hol.es sfd-chess.ru -shikiso.info -shlyahten.ru -shodanhq.com -shoesonlinebuy.cn -shopcheermakeup.info +sharebutton.org +shisha-swag.de shopwme.ru -shtaketniki.ru sibdevice.ru -sicfor.bcu.cc +sibvitr.ru siha.de -silvercash.com -similardeals.net -simplepooltips.com -site-speed-check.site -sitopreferito.it -si-unique.com -sixcooler.de -slimcdn.com -slkrm.ru +sildenafilcitratemed.com +site3.free-share-buttons.com +siteaero.com +siteripz.net +site.ru +sitevaluation.com +sizeplus.work +skanninge.se +skincrate.net +skladvaz.ru +skutecznetabletkinaporostwlosow.pl +slomm.ru slowmacfaster.trade -smart-scripts.com -snip.tw -snowplanes.com -social-buttons.com -socialmadesimple.com -soc-proof.su -soda.media -solinf.co -solmarket.by -sonata-arctica.wz.cz -sortthemesitesby.com -souvenir.cc -souvenirua.com +slujbauborki.ru +smadihome.com +smartbalanceworld.com +smartshoppymac.com +smokewithrabbits.com +sneakyboy.com +snomer1.ru +snsdeainavi.info +snts.shell-pmr.ru +soblaznu.net +social-vestnik.ru +sodexo.com +soft-terminal.ru +solicita.info +sonnikforme.ru +sovetogorod.ru +soviet-portal.do.am +spabali.org +spacash.com space2019.top -spitfiremusic.com -sport7777.net +speedup-my.site +spidtest.org +sport-video-obzor.ru sprttrack.com -spywarespy.com -squidoo.com -sstroy44.ru -starpages.net +spywarebegone.com +sr-rekneskap.no +ssconstruction.co +startwp.org stathat.com -staticfs.host -statoutlook.info -stmassage.ru -storehouse.ua -stpolice.com +statustroll.com +steame.ru +stevemonsen.com +stickers-market.ru +stpicks.com strag-invest.ru -strana-solnca.ru -strangeduckfilms.com streetfire.net -stretchmate.net -strigkaomsk.ru -stroicol.net -sugarlyflex.pw -sumo.com -sureone.pro -surffoundation.nl -surflinksmedical.com -surintech.ac.th -svnuppsalaorebro.se +strongsignal-a.akamaihd.net +sudexpert66.ru +sukirgenk.dvrlists.com +superlist.biz +supermesta.ru +superoboi.com.ua +surgut.zrus.org +svbur.ru swsociety.se -taaaak.com -tabakur77.com +syvertsen-da.no +takeflyte.com tamada69.com -tasteidea.com -t-bygg.com +tanieaukcje.com.pl +taqywu51.soup.io +taraz.xkaz.org +tastyfoodideas.com +tattooha.com td-33.ru tdsing.ru techart24.com -teenbbw.yopoint.in -telegraf.by -thaisamkok.com -thebitcoincode.com -thedownloadfreeonlinegames.blogspot.com +tecspb.ru +tedxrj.com +tedy.su +teguh.info +telefonsex-ohne0900.net +telegramdownload10.com +terrafootwear.us +theallgirlarcade.com +thebluffs.com +thecoolimages.net thegalerie.eu -theguardlan.com -themeforest.net -thetardistimes.ovh -thiegs.reco.ws -thruport.com -tiandeural.ru -tizanidinecapsules.blogspot.com -tizanidinehcl2mgsideeffects.blogspot.com -tizanidinehcl4mgisitanarcotic.blogspot.com -tizanidinehclsideeffects.blogspot.com -tizanidineinfo.blogspot.com -tizanidinenarcotic.blogspot.com -tizanidinerecreationaluse.blogspot.com -tizanidinesideeffectsweightloss.blogspot.com +thegameriders.com +thewebsitetemplate.info +thexart.club +thin.me.pn +time-japan.ru +titelhelden.eu +tizanidine4mg.blogspot.com +tizanidine4mgtablets.blogspot.com +tizanidinehcl4mgtabinfo.blogspot.com +tizanidineorflexeril.blogspot.com +tizanidinesideeffects.blogspot.com tizanidinevscyclobenzaprine.blogspot.com -tizanidinezanaflex.blogspot.com +tizanidinevsvalium.blogspot.com +tizanidinewithdrawal.blogspot.com tk-assortiment.ru tksn.ru -tomck.com +tmm-kurs.ru +tnctrx.com +tomatis.gospartner.com tooplay.com -topclickguru.com -topdownloads.ru -top-l2.com -topmira.com -topshef.ru -topsy.com +topsiteminecraft.com +torrents.cd torrents-tracker.com +torture.ml totu.us -tourcroatia.co.uk -track2.shop -trafaret74.ru +tovaroboom.vast.ru +toy-shop.top +tozup.com +track-rankings.online +tracksurf.daooda.com +tradgardspartner.se +traffic2cash.org trafficfactory.biz -trafficmania.com -trahvid.com -trion.od.ua -tri-slona.org -trustl.life -tubeoffline.com -tuckermktg.com +trafficreceiver.club +trkdf.com +trumpetedextremes.com +trustedhealthtips.com +tsan.net turbabitload.weebly.com -twitlinks.com -twittrading.com -twittruth.com -twodollarshows.com -typer.one -uamtrk.com -ukkelberg.no -uk-zheu20.ru -undergroundcityphoto.com -underthesite.com -unimodemhalfduplefw.pen.io -univerfiles.com +turbo-suslik.org +turvgori.ru +tvteleport.ru +tytoona.com +uchetunet.su +udsgame.online +uggbootsoutletsale.us +umnovocaminho.com +unitexindia.com +unitygame3d.com unrealcommander.org -uptime-alpha.net -uptime-as.net +uptimebot.net upupa.net -usiad.net -utiblog.fr -uzungil.com -vabasa.inwtrade.com -vacances-voyages.info +url2image.com +v24s.net vacuumscleaner.com valkiria-tk.ru valuado.com -vapomnoncri.tk -varikoz24.com -varikozdok.ru +vashsvet.com +vasileostrovsky-rayon.ru +vbabule.net +vduplo.ru +vegan-foods.us +vepad.com +vereo.eu verymes.xyz -vesnatehno.com +via-energy-order.com +via-gra.webstarts.com +viagroid.ru viberdownload10.com -video-chat.love -videositename.com -vietimgy.pw -vinsit.ru +video-camer.com +videochaty.ru +videojam.tv +vilingstore.net vinylvault.co.uk -vip2ch.com -vip-dom.in -vipsexfinders.com -visa-pasport.ru -viven.host.sk -vkonche.com -vkontaktemusic.ru -vmnmvzsmn.over-blog.com -volume-pills.biz -vvon.co.uk -vysigy.su -vzglyadriv.kg -vzube.com -wallpapers-best.com -wallpapersdesk.info -wanker.us -we-are-gamers.com -webads.co.nz -web.cvut.cz -webenlace.com.ar -webinstantservice.com -webix.me -webshoppermac.com -website-speed-checker.site +vip.51.la +vita.com.hr +vkmusics.ru +vodaodessa.com +voloslove.ru +vozbujdenie.com +vpnhowto.info +vremya.eu +vrotike.ru +vsesubwaysurfers.com +w7s.ru +waterefficiency.co +webalan.ru +weboptimizes.com webstatsdomain.org -wesharepics.site -westen-z.life +wedding0venues.tk +wemedinc.com westum.se -wheelchairliftsarea.com -whereverdesperate.gq +westvilletowingservices.co.za +whatzmyip.net +whereiskentoday.com +where-toget.com +whiteelephantwellington.com +whiteproduct.com +wholesalejerseysgaa.com +wholesalenfljerseys.us.com whos.amung.us -wineration.com -witherrom55.eklablog.fr -withstandingheartwarming.com -wma-x.com -wnhjavlhezp.gq -wordpresscore.com -workle.website -worldinternetauthority.com -worldis.me -worldmusicfests.com -wosik-dach.service-for-web.de -wurr.voila.net -xbaboon.com -xcombear.ru -xerox-douglas.cf -xfire.com -xjrul.com -x-lime.net -xmlinde.com -xn--24-glceagatoq7c2a6ioc.xn--p1ai -xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai +wildworld.site +williamrobsonproperty.com +windowssearch-exp.com +winterclassichockeyjerseys.com +worldhistory.biz +wowcasinoonline.ooo +woweb.com.ua +www.event-tracking.com +wyniki-lista.pl +x5market.ru +xboxster.ru +xep.info +xitjw.info +xjlottery.com +x-mix.info +xmronta.com xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai -xn--80aaagvmjabrs1aoc9luc.xn--p1ai -xn--80aaaks3bbhabgbigamdr2h.xn--p1ai -xn--80aanaardaperhcem4a6i.com -xn--80aodinpgi.xn--p1ai -xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai -xn----9sbubg3ambdfl1j.xn--p1ai -xn----dtbndd4ae7eub.top -xn--h1aakne2ba.xn--p1ai -xn----itbkqkfiq.xn--p1ai -xn--lifehacer-1rb.com -xolodremont.ru -xrus.org +xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai +xn--80aafb2a.xn--p1ai +xn--80aeahghtf8ac5i.xn--p1ai +xn--80aikhbrhr.net +xn--90acenikpebbdd4f6d.xn--p1ai +xn--e1afanlbnfckd7c3d.xn--p1ai +xn--h1ahbi.com.ua +xn--l1aengat.xn--p1ai +xpresscare.ru x-stars.ru -x-true.info +xuki.us xwatt.ru -xxart.ru -xxxkaz.org -yhit.press -youbrainboost.asia -yougame.biz -yougotanewdomain.com -yourmovies.pl -yurgorod.ru -zagadki.in.ua -zajm-pod-zalog-nedvizhimosti.ru -zakazvzloma.com -za-music.mymobiplanet.com +xxx-cam.webcam +xxxdatinglocal.us +yaaknaa.info +yaminecraft.ru +yebocasino.co.za +yginekologa.com +yopoint.in +youbloodyripper.com +yougetsignal.com +yourporn.com +youtoner.it +ypmuseum.ru +yuarra.pluto.ro +yun56.co +zaidia.xhost.ro zaobao.com.sg -zastroyka.org -zatjmuzu.info -zawyna.ua -zed21.net +zebramart.ru zerocash.msk.ru -zeroredirect12.com -zigarettenonlinekaufen1.blox.pl -zlatnajesen.com -zoodrawings.com -zoogdinsney.com -zoosexart.com -zootoplist.com -zrelaya.pw -zvetki.ru +zeroredirect5.com +zetgie.com.pl +zhorapankratov7.blogspot.com +znaturaloriginal.com +zocaparj.kz +zojirushi-products.ru +zona-aqua.ru +zs2vm.top +zynax.ua diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index b97287088..a9d225d09 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -2020iscoming.info -24videos.tv -256bit.by -7zap.com -adktrailmap.com -ads-cool.pro -adultactioncam.com -advancedsoftwaresupport.com -ad-words.ru -alekseevec.ru -allblogroll.com -aosexkontakte.net -apartmentbay.ru -apibetweenlinesn-a.akamaihd.net -atmagroup.ru -audiofree.ru -auto4style.ru -autochoixspinelli.com -avtovolop.ru -ayakino.net -azadnegar.com -batanga.net -battle.net -benchmarkcommunications.co.uk -berrymall.ru -betonka.pro -bif-ru.info -bigames.online -binaryoptionscops.info -bio-market.kz -blackwitchcraft.ru -bolezniorganov.ru -bongacams.com -bookmaker-bet.com -bosman.pluto.ro +1001watch.com.ua +1688.com +4webmasters.org +8xv8.com +abiente.ru +academiacsmendoza.org +acc.eu.org +actulite.com +adclickthru.net +adsloads.com +adtech.fr +adultfriendfinder.com +adultfullhd.com +aktivator-windows10.blogspot.com +alarmobninsk.ru +alessandraleone.com +all4invest.info +allkrim.com +alpinism.ru +amazingpic.net +ameblo.jp +animal-drawings.com +apartamentwroclaw.eu +api.stathat.com +apparel-offer.com +autobudpostach.club +aviav.ru.com +azazu.ru +babieca.com +babyfactory.fr +bdsmgalls.net +beclean-nn.ru +bedandbreakfast.com +best-deals-products.com +besttorrentknifta.weebly.com +bezlimitko.xyz +binomo.com +bloggers.nl +bobba.dzaba.com +brakehawk.com brimstonehillfortress.org -bristolhotel.com.ua -budilneg.xyz -buigas.00it.com -buzzonclick.com -calcularpagerank.com -canacopegdl.com -candycrushshop.com -chiblackhawks-jerseys.com -cialis-samples.com -ciproandtizanidine.blogspot.com -clickcash.com -clodo.ru -codysbbq.com -compliance-alexa.xyz -custom-product-labels.com -dailyfinancefix.com -dalavia.ru -dandiyabeats.in -dealwifi.com -defenderxtactical.com -dekoration.us +buketeg.xyz +businesn.men +buy-cheap-pills-order-online.com +candypeople.se +cartierlove2u.xyz +cartujano-pre.de +ccbill.com +chatseo.com +cheap-pills-norx.com +chelnytruck.ru +christianlouboutinshoes.xyz +colehaanoutlet.store +community.allhiphop.com +compliance-barak.top +compliance-barak.xyz +compliance-olga.top +comsysnet.com +crynet.cc +d2jsp.org +data1.scopich.com dekorkeramik.ru -destinationrealestate.com -detmebel.su -dlya-android.org -dominateforex.ml -dptaughtme.com -elektrischezigarette1.blog.pl -elkacentr.ru -elmifarhangi.com -elvel.com.ua -enternet.ee -estelight.ru -european-torches.ru +dengi-pod-zalog-nedvizhimosti.ru +den-noch24.ru +diusyawiga.tk +dmmspy.com +dnsrsearch.com +docsportal.net +dojki-hd.com +domaineaneblanc.com +donna7753191.ru +dvrlists.com +editors.choice6912650.hulfingtonpost.com +elektronischezigarettekaufen1.myblog.de +encodable.com +engines-usa.com +enhand.se +eralph.tk evogarage.com -extstat.com -fbdownloader.com -fdzone.org -fickkontaktehobbyhuren.com -fickluder69.com -fix-website-errors.com -formulaantiuban.com -forum-engineering.ru -free-today.com -freza-sverlo.ru -funcrushgames.com -gazoblok.net.ua -geileweiber.tk -gfaq.ru -goroda-vsego-mira.ru -grtyi.com -handsandlegs.ru -havepussy.com -hornymatches.com -hothor.se -houseofgaga.ru -ideoworld.org -igrovyeavtomaty777.ru -ilovevitaly.com -imagerydatabase.com -impotentik.com -instakink.com -int.search.tb.ask.com -it-max.com.ua -jamiembrown.com -jeremyeaton.co -jpcycles.com -juliaworld.net -kanimage.com -karadene.com -keywordsdoctor.com -kidskunst.info -lafriore.ru -landinez.co -larchik.net -laserpen.club -leadwayau.com -lenvred.org -lescinq.com -letslowbefast.today -lflash.ru -lider-zhaluzi.kiev.ua -lignofix.ua -linkpulse.com -linkredirect.biz -linksharingt.com -littleberry.ru -mac-shield.com -master-muznachas.ru +exdocsfiles.com +extremepornos.net +ezigarettekaufen2.yolasite.com +f07.de +filmi-v.online +forum.tvmir.org +freejabs.com +galeon.com +gamblingnerd.com +game-top.su +gamewrath.com +gamezblox.com +gemara.com +generousdeal-a.akamaihd.net +getyourimage.club +glavprofit.ru +gojiberriess.apishops.ru +grizzlysgrill.com +gsasearchengineranker.pw +habermetre.com +hidemyass.com +homemature.net +hoverboard360.es +ilmexico.com +imgarcade.com +inspiring-desperate.tk +instasexyblog.com +investyb.com +invivo.hu +istizanidineanarcoticdrug.blogspot.com +istizanidineanopiate.blogspot.com +ivoiretechnocom.ci +iwantmyfreecash.com +iyasimasennka.com +japfm.com +jav-way.site +jerseychinabizwholesale.us +job.icivil.ir +journalhome.com +kakablog.net +kino-ecran.ru +koleso24.com.ua +komputernaya-pomosh-moscow.ru +krasnodar.zrus.org +krassh.ru +kupiproday.com.ua +kursy-ege.ru +law-check-seven.xyz +letsrepair.in +locksmith.jp +madot.onlinewebshop.net +massage-info.nl +matb3aa.com +matchpal-a.akamaihd.net +m.b00kmarks.com +mbiologi.ru +mesto-x.com metarip.ru -minegam.com -mojpreskumanie.com -moneymaster.ru +mielec.pl +mir-betting.ru +mmoguider.ru +mnogolok.info +moinozhki.com +monclerboots.xyz +monetizer.com-01.site +money-for-placing-articles.com montazhnic.ru -moroccosurfadventures.com -myfreecams.com -mylida.org -myonigroup.com -myplaycity.com -nextlnk12.com -nikhilbahl.com -november-lax.com -oconto.ru -olgacvetmet.com -onstrapon.purplesphere.in -orgasmatrix.com -ororodnik.goodbb.ru -osb.se11.ru -paclitor.com -pandarastore.top -pandroid.co -partner-host.men -picsearch.com -pinkduck.ga -playfortuna-play.ru -pochtovyi-index.ru -podshipniki-nsk.ru -pony-business.com -pornoblood.com +moviemail-online.co.uk +music7s.me +mypets.by +narkologiya-belgorod.ru +narosty.com +newstraveller.ru +nfljerseysforsalewholesaler.com +nfljerseys.online +normalegal.ru +notfastfood.ru +novodigs.com +nvssf.com +obnal.org +onlainbesplatno.ru +onlinewritingjobs17.blogspot.ru +orion-code-access.net +osoznanie-narkotikam.net +paleohub.info +pensplan4u.com +photosaga.info +pic2fly.com +piccdata.com +picquery.com +picsforkeywordsuggestion.com +pictures-and-images.net +pierrehardysale.online +pipki.r.acdnpro.com pornobrazzers.biz -pornofiljmi.com -prchecker.info +potolokelekor.ru +pozdrawleniya.ru predmety.in.ua -primedice.com -privacyassistant.net -prizesbook.online -profitkode.com -propranolol40mg.blogspot.com -ptr.ruvds.com -pukaporn.com -rasteniya-vs-zombi.ru -realitykings.com -remontvsamare.su -responsive-test.net -rockingclicks.com -sad-torg.com.ua -scanmyphones.com -scanner-margo.top -scanner-mary.top -searchengineranker.email -searchimpression.com -searchmywindow-a.akamaihd.net -seccioncontrabajo.com -security60-e.com -seeresultshub-a.akamaihd.net -seorank.info -serpstat.com -sexblog.pw -sexkontakteao.info -sex-watch.com -sfj-ror.no -shiksabd.com -shoesonlinebuy.cn -shoppingmiracles.co.uk -sideeffectsoftizanidine.blogspot.com -simul.co -sisiynas.ru -site-auditor.online -slowmac.tech -smokewithrabbits.com -snworks.com -socialbookmarksubmission.org -softxaker.ru -souvenir.cc -sticken.co -stretchingabuckblog.com -strigkaomsk.ru -stroicol.net -suchenindeutschland.com -superstats.com -susanholtphotography.com -swinger-mobil.net -swiped.su +push-ad.com +pushdata.sendpulse.com +qitt.ru +qld10000.net +qualitymarketzone.com +quelle.ru +razleton.com +refererx.com +rennlist.com +ritlweb.com +roznica.com.ua +runtnc.net +rvtv.ru +sbprabooks.com +scanmarine.info +scanner-jess.top +scanner-willy.top +scrapinghub.com +scripted.com +seeingmeerkat.com +segol.tv +semaltmedia.com +sexs-foto.com +sfd-chess.ru +shopfishing.com.ua +skylta.com +smart-balancewheel.com +social-s-iii.xyz +sonata-arctica.wz.cz +soundfrost.org +sukirgenk.dvrlists.com +sundrugstore.com +sweepstakes.rewardit.com +syvertsen-da.no +tamada69.com tfxiq.com -thefarmergame.com -thegamerznetwork.com -thetardistimes.ovh -thexart.club -tizanidineduringpregnancy.blogspot.com -tizanidineformigraines.blogspot.com -tizanidineingredients.blogspot.com -tjkckpytpnje.com -tmearegion26.com +theguardlan.com +tmtrck.com +tootoo.to topclickguru.com -track-rankings.online -traffixer.com -trailer.cinemaflix.website -tx41tclega.ru -uhdtv.website -ultimatesetnewfreeallsoftupgradesystems.pw -uogonline.com -uprour.com -ussearche.cf -v24s.net -vapomnoncri.tk -videochat.cafe -vintontech.info -vinylvault.co.uk -webenlace.com.ar -webshoppermac.com -website-analytics.online +topshef.ru +tourismvictoria.com +traffmonster.info +tripper.de +turkeyreport.tk +twu.com.ua +uptime.com +urzedowski.eu +veopornogratis.xxx +villakohlanta.nu +vkak.ru +vriel.batcave.net +vsesubwaysurfers.com +wait3sec.org +watchmyfb.pl +waterefficiency.co +weprik.ru +wetgames.ru wjgony.com -w-journal.ru -wladimirpayen.com -www.888.com -wwwadultcheck.com -wygraj-skiny.win -x-diesel.info -xn----itbkqkfiq.xn--p1ai -xtrafficplus.com -your-bearings.com -youtubedownload.org -zoogdisany.com -zrizvtrnpale.tk +word-vorlagen.net +worldhistory.biz +www.kabbalah-red-bracelets.com +xboxster.ru +x-mix.info +xn----7sbho2agebbhlivy.xn--p1ai +xn--e1aggki3c.xn--80adxhks +xn--h1ahbi.com.ua +yeartwit.com +youtoner.it +zdesoboi.com +zeroredirect5.com +zigzog.ru +zoogdinsney.com diff --git a/.dev-tools/test_units/ratelimittest.txt b/.dev-tools/test_units/ratelimittest.txt index e69de29bb..0e3c8bc75 100644 --- a/.dev-tools/test_units/ratelimittest.txt +++ b/.dev-tools/test_units/ratelimittest.txt @@ -0,0 +1,99 @@ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.14.2
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.14.2
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.14.2
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.14.2
+ + + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + \ No newline at end of file diff --git a/.dev-tools/test_units/whitelist-ips.conf b/.dev-tools/test_units/whitelist-ips.conf index f45bae986..95fbc95fc 100644 --- a/.dev-tools/test_units/whitelist-ips.conf +++ b/.dev-tools/test_units/whitelist-ips.conf @@ -1,4 +1,4 @@ -35.192.85.2 0; +104.154.120.187 0; 127.0.0.1 1; 127.0.0.1 1; 127.0.0.1 1; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 29d07923d..ae58d3099 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index e994c3f8b..a804c179b 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 7e57b1bd9..4921fa69d 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2019.06.1644 +#### Version: V4.2019.06.1645 #### Bad Referrer Count: 6713 #### Bad Bot Count: 556 ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 51cb66be1..3935f98c6 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2019.06.1644 +#### Version: V4.2019.06.1645 #### Bad Referrer Count: 6713 #### Bad Bot Count: 556 ____________________ diff --git a/README.md b/README.md index d010f5346..254c9c5f8 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2019.06.1644 +#### Version: V4.2019.06.1645 #### Bad Referrer Count: 6713 #### Bad Bot Count: 556 ____________________ diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 9ffa9d34d..efea89303 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2019.06.1644 +#### Version: V4.2019.06.1645 #### Bad Referrer Count: 6713 #### Bad Bot Count: 556 ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 453934fe3..19a19648d 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1644 -### Updated: Thu Jun 27 13:12:21 SAST 2019 +### Version: V4.2019.06.1645 +### Updated: Thu Jun 27 13:24:55 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 2f7d1d3a5..9c649adb5 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1644 -### Updated: Thu Jun 27 13:12:23 SAST 2019 +### Version: V3.2019.06.1645 +### Updated: Thu Jun 27 13:24:56 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 453934fe3..19a19648d 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1644 -### Updated: Thu Jun 27 13:12:21 SAST 2019 +### Version: V4.2019.06.1645 +### Updated: Thu Jun 27 13:24:55 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 041a57a60..0ccb3f33e 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,8 +6,8 @@ ### Version Information # ################################################### -### Version: V4.2019.06.1644 -### Updated: Thu Jun 27 13:12:24 SAST 2019 +### Version: V4.2019.06.1645 +### Updated: Thu Jun 27 13:24:58 SAST 2019 ### Bad Bot Count: 556 ################################################### ### Version Information ##