vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-02 10:40:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

ADD Nginx Mainstream Test (Step 14)

Browse source
This commit is contained in:
Mitchell Krog 2017-10-03 15:53:03 +02:00
parent b0ee3c8bb4
commit ae106c33ab
No known key found for this signature in database
GPG key ID: 2BB9A077C8DD6994
27 changed files with 7868 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
.dev-tools/_conf_files_test4/.keep Normal file
View file

1
.dev-tools/_conf_files_test4/bots.d/.keep Normal file
View file

@ -0,0 +1 @@
null

66
.dev-tools/_conf_files_test4/bots.d/bad-referrer-words.conf Normal file
View file

@ -0,0 +1,66 @@
# EDIT THIS FILE AS YOU LIKE TO ADD OR REMOVE ANY BAD WORDS YOU WANT TO SCAN FOR ###
### VERSION INFORMATION #
###################################################
### Version: V3.2017.04
### Updated: Mon Oct 02 16:41:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# This is merely an example and gets auto included as since Version 2.2017.07 introduced on 2017-04-20
# This file must exist on your system or Nginx will fail a reload due to a missing file
# For all intensive purpose you can delete everything inside this file and leave it
# completely blank if you do not want your Nginx Blocker to include scanning for bad words within urls or referrer string
# Only add one entry per line
# *******************************
# !!! WARNING WARNING WARNING !!!
# *******************************
# ***************************************
# PLEASE BE VERY CAREFUL HOW YOU USE THIS
# ***************************************
# Here is an example of how one supposed bad word can cause your whole site to go down.
# An issue was logged where the users own domain name was specialisteparquet.com
# Because this list contained the word "cialis" it was detected within his domain name causing
# his entire site to go down and not server any assets.
# That one entry would even cause any site containing a word like "specialist" anywhere in any
# of their sites pages to cause them to be blocked and whitelisting your own domain name in the
# whitelist-domains.conf file will not even bypass this, SO BE CAREFUL PLEASE
# Think very carefully before you add any word here
# *****************************************************************************************
# PLEASE MAKE SURE that you use word regex boundaries to avoid false positive detection !!!
# *****************************************************************************************
# As you can see in the examples below the word "adultgalls" is entered with a preceding \b and an ending \b
# this makes it now "\badultgalls\b". It is crucial to use the word boundaries regex formatting.
"~*\badultgalls\b" 1;
"~*\bamateurxpass\b" 1;
"~*\bbigblackbooty\b" 1;
"~*\bblacktits\b" 1;
"~*\bcookie\-law\-enforcement\b" 1;
"~*\bfree\-share\-buttons\b" 1;
"~*\bfree\-social\-buttons\b" 1;
"~*\bfuck\-paid\-share\-buttons\b" 1;
"~*\bilovevitaly\b" 1;
"~*\blaw\-enforcement\-bot\b" 1;
"~*\blaw\-enforcement\-check\b" 1;
"~*\bshare\-buttons\-for\-free\b" 1;
"~*\bwebfuck\b" 1;
"~*\bxxxrus\b" 1;
"~*\bzeroredirect\b" 1;

39
.dev-tools/_conf_files_test4/bots.d/blacklist-domains.conf Normal file
View file

@ -0,0 +1,39 @@
# EDIT THIS FILE AS YOU LIKE TO BLACKLIST YOUR OWN CUSTOM DOMAIN NAMES ###
### VERSION INFORMATION #
###################################################
### Version: V3.2017.02
### Updated: Mon Oct 02 16:48:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Add One Entry Per Line - List all your own extra domains you want to blacklist.
# This file must exist on your system or Nginx will fail a reload due to a missing file
# Automatic updates will never be able to remove this custom list of yours
# Add One Entry Per Line
# Make sure any domains have dots and special characters escaped as per the Regex examples below.
# For example myblacklisteddomainname.com should be entered as myfirstowndomainname\.com
# and my-second-blacklisted.com should be entered as my\-second\-owndomainname\.com
# *****************************************************************************************
# PLEASE MAKE SURE that you use word regex boundaries to avoid false positive detection !!!
# *****************************************************************************************
# As you can see in the examples below the domain "myblacklisteddomainname\.com" is entered with a preceding \b and an ending \b
# this makes it now "\bmyblacklisteddomainname\.com\b". It is crucial to use the word boundaries regex formatting.
"~*\bmyblacklisteddomainname\.com\b" 1;
"~*\bmy\-second\-blacklisted\.com\b" 1;

218
.dev-tools/_conf_files_test4/bots.d/blacklist-ips.conf Normal file
View file

@ -0,0 +1,218 @@
# EDIT THIS FILE AS YOU LIKE TO ADD OR REMOVE ANY BAD IP ADDRESSES OR IP RANGES YOU WANT TO BLOCK ###
### VERSION INFORMATION #
###################################################
### Version: V3.2017.05
### Updated: Mon Sep 11 10:40:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# This is merely an example and gets auto included as since Version 2.2017.07 introduced on 2017-04-20
# This file must exist on your system or Nginx will fail a reload due to a missing file
# For all intensive purpose you can delete everything inside this file and leave it
# completely blank if you do not want your Nginx Blocker to do any blocking of bad IP's
104.223.37.150 1;
104.5.92.27 1;
107.150.63.170 1;
109.236.83.247 1;
137.74.49.205 1;
137.74.49.208 1;
146.0.74.150 1;
148.251.54.44 1;
149.56.151.180 1;
149.56.232.146 1;
150.70.0.0/16 1;
151.80.27.90 1;
151.80.99.90 1;
151.80.99.91 1;
154.16.199.144 1;
154.16.199.34 1;
154.16.199.48 1;
154.16.199.78 1;
158.69.142.34 1;
166.62.80.172 1;
173.212.192.219 1;
173.234.11.105 1;
173.234.153.106 1;
173.234.153.30 1;
173.234.175.68 1;
173.234.31.9 1;
173.234.38.25 1;
176.126.245.213 1;
178.238.234.1 1;
185.35.63.128 1;
185.100.87.238 1;
185.115.125.99 1;
185.119.81.11 1;
185.119.81.63 1;
185.119.81.77 1;
185.119.81.78 1;
185.130.225.65 1;
185.130.225.66 1;
185.130.225.83 1;
185.130.225.90 1;
185.130.225.94 1;
185.130.225.95 1;
185.130.226.105 1;
185.153.197.103 1;
185.159.36.6 1;
185.47.62.199 1;
185.62.190.38 1;
185.70.105.161 1;
185.70.105.164 1;
185.85.239.156 1;
185.85.239.157 1;
185.86.13.213 1;
185.86.5.199 1;
185.86.5.212 1;
185.92.72.88 1;
185.93.185.11 1;
185.93.185.12 1;
185.183.96.33 1;
188.209.52.101 1;
190.152.223.27 1;
191.96.249.29 1;
192.69.89.173 1;
193.201.224.205 1;
195.154.183.190 1;
195.229.241.174 1;
200.7.105.43 1;
210.212.194.60 1;
216.218.147.194 1;
220.227.234.129 1;
23.253.230.158 1;
23.89.159.176 1;
31.170.160.209 1;
45.32.186.11 1;
45.76.21.179 1;
46.249.38.145 1;
46.249.38.146 1;
46.249.38.148 1;
46.249.38.149 1;
46.249.38.150 1;
46.249.38.151 1;
46.249.38.152 1;
46.249.38.153 1;
46.249.38.154 1;
46.249.38.159 1;
51.255.172.22 1;
5.39.218.232 1;
5.39.219.24 1;
5.39.222.18 1;
5.39.223.134 1;
54.213.16.154 1;
54.213.9.111 1;
62.210.146.49 1;
62.210.88.4 1;
65.98.91.181 1;
69.162.124.237 1;
69.64.147.24 1;
72.8.183.202 1;
77.247.178.191 1;
77.247.178.47 1;
77.247.181.219 1;
78.31.184.0/21 1;
78.31.211.0/24 1;
79.110.128.17 1;
79.110.128.63 1;
79.110.128.252 1;
79.110.128.128 1;
80.87.205.10 1;
80.87.205.11 1;
85.17.230.23 1;
85.17.26.68 1;
91.185.190.172 1;
91.200.12.0/22 1;
91.200.12.15 1;
91.200.12.49 1;
91.200.12.91 1;
92.222.66.137 1;
93.238.198.203 1;
93.238.196.48 1;
93.104.209.11 1;
93.158.200.103 1;
93.158.200.105 1;
93.158.200.115 1;
93.158.200.124 1;
93.158.200.126 1;
93.158.200.66 1;
93.158.200.68 1;
93.238.202.44 1;
# Cyveillance / Qwest Communications / PSINET
# *******************************************
# I am extensively researching this subject - appears to be US government involved
# and also appears to be used by all sorts of law enforcement agencies. For one they
# do not obey robots.txt and continually disguise their User-Agent strings. Time will
# tell if this is all correct or not.
# For now see - https://en.wikipedia.org/wiki/Cyveillance
# IMPORTANT UPDATE ON Cyveillance / Qwest Communications !!!
# **********************************************************
# I have done a lot of research on Cyveillance now and through monitoring my logs I know
# for sure what companies are using them and what they are actually looking for.
# My research has led me to understand that Cyveillance services are used by hundreds
# of companies to help them dicsover theft of copyrighted materials like images, movies
# music and other materials. I personally believe a lot of block lists who originally recommended
# blocking Cyveillance have done so to protect their torrent or p2p sites from being scanned.
# I personally have now unblocked them as image theft is a big problem of mine but if you
# do want to block Cyveillance you can simply modify the entries in the block below from "0" to "1"
# Getty Images is one such company who appears to use Cyveillance to help monitor for copyright theft.
# If you really do want to block them change all the 0's below to 1.
# Use this section at YOUR OWN RISK, you may block some legitimate networks but after many hours of
# Research this is now the completely updated list of all IP ranges IPV4 and IPV6 owned Qwest Communications
# PSINET and Cyveillance
# IMPORTANT NOTE: If you really want to keeps bot and things out of certain parts of your web site
# Rather implement a comlex Google Re-Captcha to reach sections of your sites and for people to be able
# to access download links. Google Re-Captcha with images is too complex for any bot.
206.2.138.0/23 0;
208.71.164.0/22 0;
38.100.19.8/29 0;
38.100.21.0/24 0;
38.100.41.64/26 0;
38.105.71.0/25 0;
38.105.83.0/27 0;
38.112.21.140/30 0;
38.118.42.32/29 0;
4.17.135.32/27 0;
63.144.0.0/13 0;
65.112.0.0/12 0;
65.192.0.0/11 0;
65.213.208.128/27 0;
65.222.176.96/27 0;
65.222.185.72/29 0;
# ****************
# Berkely Scanner
# ****************
# The Berkeley University has a scanner testing all over the web sending a complex
# payload an expecting a reply from servers who are infected or who just respond to such
# a payload. The payload looks similar to this
# "$\xC9\xE1\xDC\x9B+\x8F\x1C\xE71\x99\xA8\xDB6\x1E#\xBB\x19#Hx\xA7\xFD\x0F9-"
# and is sometime VERY long. You may have noticed this in your logs.
# I support research projects and all my servers respond with an error to this type of
# string so I do not block them but if you want to block just uncomment the following line
# or email them asking them not to scan your server. They do respond.
# Visit http://169.229.3.91/ for more info
# If you really do want to block them change all the 0 below to 1.
169.229.3.91 0;

60
.dev-tools/_conf_files_test4/bots.d/blacklist-user-agents.conf Normal file
View file

@ -0,0 +1,60 @@
# EDIT THIS FILE AS YOU LIKE TO BLACKLIST OR WHITELIST ANY BAD USER-AGENT STRINGS YOU WANT TO SCAN FOR
# ****************************************************************************************************
### VERSION INFORMATION #
###################################################
### Version: V3.2017.02
### Updated: Mon Aug 21 11:30:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Add One Entry Per Line - List all the extra bad User-Agents you want to permanently block or whitelist.
# This is for User-Agents that are not included in the main list of the bot blocker
# This file must exist on your system or Nginx will fail a reload due to a missing file
# This allows you finer control of keeping certain bots blocked and automatic updates will
# Never be able to remove this custom list of yours
# Please note this include file loads first before any of the already whitelisted User-Agents
# in the bad bot blocker. By loading first in line it over-rides anything below it so for instance
# if you want to block Baidu, Google or Bing for any reason you add them to this file which loads
# first and takes precedence over anything below it. This now allows even finer control over the
# bad bot blocker. Enjoy !!!
# Even though this file is called blacklist-user-agents, as mentioned it can also be used to whitelist user agents
# By adding them below and setting the 3; to 0; this will permanently whitelist the User-Agent.
# Make sure any words that contain special characters are escaped and include word boundaries as per the Regex examples below.
# Example the User-Agent name "someverybaduseragentname1" is entered as "\bsomeverybaduseragentname1\b"
# Example the User-Agent name "some-very-bad-useragentname2" is entered as "\bsome\-very\-bad\-useragentname1\b"
# the "\b" are word boundaries which prevents partial matching and false positives.
# *********************
# WHITELISTING EXAMPLES
# *********************
"~*\bsomeverygooduseragentname1\b" 0;
"~*\bsomeverygooduseragentname2\b" 0;
"~*\bsome\-very\-good\-useragentname2\b" 0;
# *********************
# BLACKLISTING EXAMPLES
# *********************
"~*\bsomeverybaduseragentname1\b" 3;
"~*\bsomeverybaduseragentname2\b" 3;
"~*\bsome\-very\-bad\-useragentname2\b" 3;

63
.dev-tools/_conf_files_test4/bots.d/blockbots.conf Normal file
View file

@ -0,0 +1,63 @@
# Author: Mitchell Krog <mitchellkrog@gmail.com> - https://github.com/mitchellkrogza/
### VERSION INFORMATION #
###################################################
### Version: V3.2017.02
### Updated: Mon Aug 21 11:29:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Include this in a vhost file within a server {} block using and include statement like below
# server {
# #Config stuff here
# include /etc/nginx/bots.d/blockbots.conf
# include /etc/nginx/bots.d/ddos.conf
# #Other config stuff here
# }
#######################################################################
# BOTS
# ****
#limit_conn bot1_connlimit 100;
limit_conn bot2_connlimit 10;
#limit_req zone=bot1_reqlimitip burst=50;
limit_req zone=bot2_reqlimitip burst=10;
if ($bad_bot = '3') {
return 444;
}
# BAD REFER WORDS
# ***************
if ($bad_words) {
return 444;
}
# REFERERS
# ********
if ($bad_referer) {
return 444;
}
# IP BLOCKS
# *********
if ($validate_client) {
return 444;
}
#######################################################################

44
.dev-tools/_conf_files_test4/bots.d/custom-bad-referrers.conf Normal file
View file

@ -0,0 +1,44 @@
# EDIT THIS FILE AS YOU LIKE TO ADD ANY ADDITIONAL BAD REFERRER DOMAINS YOU WANT TO SCAN FOR ###
### VERSION INFORMATION #
###################################################
### Version: V3.2017.03
### Updated: Mon Oct 02 16:50:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# This is merely an example and gets auto included as since Version 2.2017.07 introduced on 2017-04-20
# This file must exist on your system or Nginx will fail a reload due to a missing file
# Only add one entry per line
# Make sure any domains have dots and special characters escaped as per the Regex examples below.
# For example some-veryvery-randomwebsitename-thatdoesnotexist4.com should be entered as
# some\-veryvery\-randomwebsitename\-thatdoesnotexist4\.com
# *****************************************************************************************
# PLEASE MAKE SURE that you use word regex boundaries to avoid false positive detection !!!
# *****************************************************************************************
# As you can see in the examples below the domain "someveryveryrandomwebsitenamethatdoesnotexist1\.com"
# is entered with a preceding \b and an ending \b
# this makes it now "\bsomeveryveryrandomwebsitenamethatdoesnotexist1\.com\b".
# It is crucial to use the word boundaries regex formatting.
"~*\bsomeveryveryrandomwebsitenamethatdoesnotexist1\.com\b" 1;
"~*\bsomeveryveryrandomwebsitenamethatdoesnotexist2\.com\b" 1;
"~*\bsomeveryveryrandomwebsitenamethatdoesnotexist3\.com\b" 1;
"~*\bsome\-veryvery\-randomweb\-sitenamethatdoesnotexist4\.com\b" 1;

36
.dev-tools/_conf_files_test4/bots.d/ddos.conf Normal file
View file

@ -0,0 +1,36 @@
#######################################################################
### VERSION INFORMATION #
###################################################
### Version: V3.2017.01
### Updated: Sun Jan 29 11:35:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Author: Mitchell Krog <mitchellkrog@gmail.com> - https://github.com/mitchellkrogza/
# Include this in a vhost file within a server {} block using and include statement like below
# server {
# #Config stuff here
# include /etc/nginx/bots.d/blockbots.conf
# include /etc/nginx/bots.d/ddos.conf
# #Other config stuff here
# }
#######################################################################
limit_conn addr 200;
limit_req zone=flood burst=200 nodelay;

39
.dev-tools/_conf_files_test4/bots.d/whitelist-domains.conf Normal file
View file

@ -0,0 +1,39 @@
# EDIT THIS FILE AS YOU LIKE TO WHITELIST YOUR OWN DOMAIN NAMES AND SPARE THEM FROM ANY REFERRER CHECKING ###
### VERSION INFORMATION #
###################################################
### Version: V3.2017.02
### Updated: Mon Oct 02 16:52:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Add One Entry Per Line - List all your own domains of the sites you host on the server
# This file must exist on your system or Nginx will fail a reload due to a missing file
# Automatic updates will never be able to remove this custom list of yours
# Add One Entry Per Line
# Make sure any domains have dots and special characters escaped as per the Regex examples below.
# For example myfirstowndomainname.com should be entered as myfirstowndomainname\.com
# and my-second-owndomainname.com should be entered as my\-second\-owndomainname\.com
# *****************************************************************************************
# PLEASE MAKE SURE that you use word regex boundaries to avoid false positive detection !!!
# *****************************************************************************************
# As you can see in the examples below the domain "myfirstowndomainname\.com" is entered with a preceding \b and an ending \b
# this makes it now "\bmyfirstowndomainname\.com\b". It is crucial to use the word boundaries regex formatting.
"~*\bmyfirstowndomainname\.com\b" 0;
"~*\bmy\-second\-owndomainname\.com\b" 0;

29
.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf Normal file
View file

@ -0,0 +1,29 @@
# EDIT THIS FILE AS YOU LIKE TO WHITELIST ALL YOUR IP ADDRESSES AND IP RANGES ###
### VERSION INFORMATION #
###################################################
### Version: V3.2017.01
### Updated: Mon Aug 21 11:32:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Add One Entry Per Line - List all your IP's and IP Ranges you want to whitelist
# This file must exist on your system or Nginx will fail a reload due to a missing file
# Automatic updates will never be able to remove this custom list of yours
# Add One Entry Per Line - No need to include 127.0.0.1 as it is covered elsewhere
# Only add actual IP addresses and ranges here
111.111.111.111 0;
35.188.66.27 0;

1
.dev-tools/_conf_files_test4/conf.d/.keep Normal file
View file

@ -0,0 +1 @@
null

17
.dev-tools/_conf_files_test4/conf.d/botblocker-nginx-settings.conf Normal file
View file

@ -0,0 +1,17 @@
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
server_names_hash_bucket_size 64;
server_names_hash_max_size 4096;
limit_req_zone $binary_remote_addr zone=flood:50m rate=90r/s;
limit_conn_zone $binary_remote_addr zone=addr:50m;

7008
.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf Normal file
View file

File diff suppressed because it is too large Load diff

0
.dev-tools/_curl_tests_4/.keep Normal file
View file

6
.dev-tools/_curl_tests_4/curltest1.txt Executable file
View file

@ -0,0 +1,6 @@
Last Tested: Tue Oct 3 14:31:56 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

6
.dev-tools/_curl_tests_4/curltest2.txt Executable file
View file

@ -0,0 +1,6 @@
Last Tested: Tue Oct 3 14:31:56 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

6
.dev-tools/_curl_tests_4/curltest3.txt Executable file
View file

@ -0,0 +1,6 @@
Last Tested: Tue Oct 3 14:31:56 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

6
.dev-tools/_curl_tests_4/curltest4.txt Executable file
View file

@ -0,0 +1,6 @@
Last Tested: Tue Oct 3 14:31:56 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

7
.dev-tools/_curl_tests_4/curltest5.txt Executable file
View file

@ -0,0 +1,7 @@
Last Tested: Tue Oct 3 14:31:56 SAST 2017
<?php
echo "Welcome to Nginx\n";
echo "This is the Travis Testing Environment for the Nginx Ultimate Bad Bot Blocker\n";
echo "Visit https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n";

7
.dev-tools/_curl_tests_4/curltest6.txt Executable file
View file

@ -0,0 +1,7 @@
Last Tested: Tue Oct 3 14:31:56 SAST 2017
<?php
echo "Welcome to Nginx\n";
echo "This is the Travis Testing Environment for the Nginx Ultimate Bad Bot Blocker\n";
echo "Visit https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n";

7
.dev-tools/_curl_tests_4/curltest7.txt Normal file
View file

@ -0,0 +1,7 @@
Last Tested: Tue Oct 3 14:31:56 SAST 2017
<?php
echo "Welcome to Nginx\n";
echo "This is the Travis Testing Environment for the Nginx Ultimate Bad Bot Blocker\n";
echo "Visit https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n";

7
.dev-tools/_curl_tests_4/curltest8.txt Normal file
View file

@ -0,0 +1,7 @@
Last Tested: Tue Oct 3 14:31:56 SAST 2017
<?php
echo "Welcome to Nginx\n";
echo "This is the Travis Testing Environment for the Nginx Ultimate Bad Bot Blocker\n";
echo "Visit https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n";

1
.dev-tools/generate-files.sh
View file

@ -55,6 +55,7 @@ sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/modify-files-and-commit.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/run-curl-tests-1.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/run-curl-tests-2.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/run-curl-tests-3.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/run-curl-tests-4.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/prepare-robots-input.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/install-nginx-1.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/install-nginx-2.sh

18
.dev-tools/install-nginx-4.sh
View file

@ -47,10 +47,6 @@
# Lets Uninstall Current Nginx 1.12.1
# ***********************************
#sudo apt-get -y remove nginx
#sudo apt-get -y remove nginx-extras
#sudo apt-get -y install software-properties-common
#sudo apt-get -y autoremove
sudo rm -rfv /etc/nginx/mybots.d/
sudo rm -rfv /etc/nginx/myconf.d/
sudo rm /etc/nginx/conf.d/*.conf
@ -81,9 +77,6 @@ sudo /etc/init.d/nginx reload
ls -la /etc/nginx/
sudo service nginx restart
sudo service nginx reload
#sudo systemctl status nginx.service
#sudo systemctl enable nginx.service
#sudo systemctl status nginx.service
# **************************************
# Make Sure We Cleanup From Nginx Test 3
@ -292,16 +285,17 @@ sudo service nginx reload
# Now Run our Curl Tests
# **********************
#printf '\n%s\n%s\n%s\n\n' "######################" "Now Run our Curl Tests" "######################"
printf '\n%s\n%s\n%s\n\n' "######################" "Now Run our Curl Tests" "######################"
# ************************************************************
# Copy all .conf files used in Test 3 to a folder for checking
# ************************************************************
#sudo chown -R travis:travis $TRAVIS_BUILD_DIR/
#sudo chown -R travis:travis $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test3/
#sudo cp /etc/nginx/mybots.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test3/bots.d/
#sudo cp /etc/nginx/myconf.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test3/conf.d/
sudo chown -R travis:travis $TRAVIS_BUILD_DIR/
sudo chown -R travis:travis $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test4/
sudo cp /etc/nginx/bots.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test4/bots.d/
sudo cp /etc/nginx/conf.d/* $TRAVIS_BUILD_DIR/.dev-tools/_conf_files_test4/conf.d/
sudo chown -R travis:travis $TRAVIS_BUILD_DIR/
# *****************************************************************************************
# Travis now moves into running the rest of the tests in the script: section of .travis.yml

187
.dev-tools/run-curl-tests-4.sh Executable file
View file

@ -0,0 +1,187 @@
#!/bin/bash
# Curl Testing Script for Nginx Ultimate Bad Bot Blocker (using non standard folder locations)
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# *******************************************
# Set Location of our Curl Test Results Files
# *******************************************
_curltest1=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest1.txt
_curltest2=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest2.txt
_curltest3=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest3.txt
_curltest4=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest4.txt
_curltest5=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest5.txt
_curltest6=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest6.txt
_curltest7=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest7.txt
_curltest8=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests_4/curltest8.txt
_now="$(date)"
# *************************************************
# Function Curl Test 1 - Check for Bad Bot "80legs"
# *************************************************
run_curltest1 () {
truncate -s 0 $_curltest1
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest1"
curl -A "80legs" http://localhost:9000/index.php 2>> $_curltest1
if grep -i '(52)' $_curltest1; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# **************************************************
# Function Curl Test 2 - Check for Bad Bot "masscan"
# **************************************************
run_curltest2 () {
truncate -s 0 $_curltest2
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest2"
curl -A "masscan" http://localhost:9000/index.php 2>> $_curltest2
if grep -i '(52)' $_curltest2; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# ******************************************************************
# Function Curl Test 3 - Check for Bad Referrer "100dollars-seo.com"
# ******************************************************************
run_curltest3 () {
truncate -s 0 $_curltest3
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest3"
curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2>> $_curltest3
if grep -i '(52)' $_curltest3; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# ******************************************************
# Function Curl Test 4 - Check for Bad Referrer "zx6.ru"
# ******************************************************
run_curltest4 () {
truncate -s 0 $_curltest4
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest4"
curl -I http://localhost:9000/index.php -e http://zx6.ru 2>> $_curltest4
if grep -i '(52)' $_curltest4; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# *****************************************************
# Function Curl Test 5 - Check for Good Bot "GoogleBot"
# *****************************************************
run_curltest5 () {
truncate -s 0 $_curltest5
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest5"
curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> $_curltest5
if grep -i 'Welcome' $_curltest5; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
#exit 1
fi
}
# ***************************************************
# Function Curl Test 6 - Check for Good Bot "BingBot"
# ***************************************************
run_curltest6 () {
truncate -s 0 $_curltest6
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest6"
curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> $_curltest6
if grep -i 'Welcome' $_curltest6; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
#exit 1
fi
}
# ***********************************************************
# Function Curl Test 7 - Check for Good Referrer "google.com"
# ***********************************************************
run_curltest7 () {
truncate -s 0 $_curltest7
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest7"
curl http://localhost:9000/index.php -e http://google.com 2>&1 >> $_curltest7
if grep -i 'Welcome' $_curltest7; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
else
echo 'GOOD REFERRER NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# ***********************************************************
# Function Curl Test 8 - Check for Good Referrer "bing.com"
# ***********************************************************
run_curltest8 () {
truncate -s 0 $_curltest8
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "$_curltest8"
curl http://localhost:9000/index.php -e http://bing.com 2>&1 >> $_curltest8
if grep -i 'Welcome' $_curltest8; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
else
echo 'GOOD REFERRER NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# *********************************
# Trigger our curl functions to run
# *********************************
run_curltest1
run_curltest2
run_curltest3
run_curltest4
run_curltest5
run_curltest6
run_curltest7
run_curltest8
# ****************************************
# If everything passed then we exit with 0
# ****************************************
exit 0

1
.travis.yml
View file

@ -42,6 +42,7 @@ script:
- bash .dev-tools/install-nginx-3.sh
- bash .dev-tools/run-curl-tests-3.sh
- bash .dev-tools/install-nginx-4.sh
- bash .dev-tools/run-curl-tests-4.sh
- bash .dev-tools/modify-files-and-commit.sh
before_deploy: