vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-01 18:19:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Introduce New Tests to Test New / Potentially Breaking Changes to the Blocker

Browse source
This commit is contained in:
Mitchell Krog 2019-06-20 12:13:22 +02:00
parent f22ce483bc
commit 7a1011aea5
No known key found for this signature in database
GPG key ID: C243C388553EDE5D
30 changed files with 22845 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
.dev-tools/_conf_files_testing_changes/.keep Normal file
View file

1
.dev-tools/_conf_files_testing_changes/bots.d/.keep Normal file
View file

@ -0,0 +1 @@
null

73
.dev-tools/_conf_files_testing_changes/bots.d/bad-referrer-words.conf Normal file
View file

@ -0,0 +1,73 @@
# EDIT THIS FILE AS YOU LIKE TO ADD OR REMOVE ANY BAD WORDS YOU WANT TO SCAN FOR ###
### VERSION INFORMATION #
###################################################
### Version: V3.2018.06
### Updated: 2018-08-21
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# This is merely an example and gets auto included as since Version 2.2017.07 introduced on 2017-04-20
# This file must exist on your system or Nginx will fail a reload due to a missing file
# For all intensive purpose you can delete everything inside this file and leave it
# completely blank if you do not want your Nginx Blocker to include scanning for bad words within urls or referrer string
# Only add one entry per line
# *******************************
# !!! WARNING WARNING WARNING !!!
# *******************************
# ***************************************
# PLEASE BE VERY CAREFUL HOW YOU USE THIS
# ***************************************
# Here is an example of how one supposed bad word can cause your whole site to go down.
# An issue was logged where the users own domain name was specialisteparquet.com
# Because this list contained the word "cialis" it was detected within his domain name causing
# his entire site to go down and not server any assets.
# That one entry would even cause any site containing a word like "specialist" anywhere in any
# of their sites pages to cause them to be blocked and whitelisting your own domain name in the
# whitelist-domains.conf file will not even bypass this, SO BE CAREFUL PLEASE
# Think very carefully before you add any word here
# *****************************************************************************************
# PLEASE MAKE SURE that you use word regex boundaries to avoid false positive detection !!!
# *****************************************************************************************
# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
# As you can see in the examples below the word "adultgalls" is entered with a preceding \b and an ending \b
# this makes it now "\badultgalls\b". It is crucial to use the word boundaries regex formatting.
# ---------
# EXAMPLES:
# ---------
# "~*\badultgalls\b" 1;
# "~*\bamateurxpass\b" 1;
# "~*\bbigblackbooty\b" 1;
# "~*\bblacktits\b" 1;
# "~*\bcookie\-law\-enforcement\b" 1;
# "~*\bfree\-share\-buttons\b" 1;
# "~*\bfree\-social\-buttons\b" 1;
# "~*\bfuck\-paid\-share\-buttons\b" 1;
# "~*\bilovevitaly\b" 1;
# "~*\blaw\-enforcement\-bot\b" 1;
# "~*\blaw\-enforcement\-check\b" 1;
# "~*\bshare\-buttons\-for\-free\b" 1;
# "~*\bwebfuck\b" 1;
# "~*\bxxxrus\b" 1;
# "~*\bzeroredirect\b" 1;

45
.dev-tools/_conf_files_testing_changes/bots.d/blacklist-domains.conf Normal file
View file

@ -0,0 +1,45 @@
# EDIT THIS FILE AS YOU LIKE TO BLACKLIST YOUR OWN CUSTOM DOMAIN NAMES ###
### VERSION INFORMATION #
###################################################
### Version: V3.2018.04
### Updated: 2018-08-21
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Add One Entry Per Line - List all your own extra domains you want to blacklist.
# This file must exist on your system or Nginx will fail a reload due to a missing file
# Automatic updates will never be able to remove this custom list of yours
# Add One Entry Per Line
# Make sure any domains have dots and special characters escaped as per the Regex examples below.
# For example myblacklisteddomainname.com should be entered as myfirstowndomainname\.com
# and my-second-blacklisted.com should be entered as my\-second\-owndomainname\.com
# *****************************************************************************************
# PLEASE MAKE SURE that you use word regex boundaries to avoid false positive detection !!!
# *****************************************************************************************
# As you can see in the examples below the domain "myblacklisteddomainname\.com" is entered with a preceding \b and an ending \b
# this makes it now "\bmyblacklisteddomainname\.com\b". It is crucial to use the word boundaries regex formatting.
# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
# ---------
# EXAMPLES:
# ---------
# "~*\bmyblacklisteddomainname\.com\b" 1;
# "~*\bmy\-second\-blacklisted\.com\b" 1;

91
.dev-tools/_conf_files_testing_changes/bots.d/blacklist-ips.conf Normal file
View file

@ -0,0 +1,91 @@
# EDIT THIS FILE AS YOU LIKE TO ADD OR REMOVE ANY BAD IP ADDRESSES OR IP RANGES YOU WANT TO BLOCK ###
### VERSION INFORMATION #
###################################################
### Version: V3.2018.07
### Updated: 2018-08-06
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# This is merely an example and gets auto included as since Version 2.2017.07 introduced on 2017-04-20
# This file must exist on your system or Nginx will fail a reload due to a missing file
# For all intensive purpose you can delete everything inside this file and leave it
# completely blank if you do not want your Nginx Blocker to do any blocking of bad IP's
# Add IP's you want to blacklist below this line, one per line as per example
# Nginx [warn] notices may be reported when you try reload Nginx if you happen to include an
# IP here that may already be included by the blocker with it's daily updates
# NOTE: It is only an Nginx Warning message and will not cause Nginx to fail a reload.
# 111.111.111.111 1;
# Cyveillance / Qwest Communications / PSINET
# *******************************************
# I am extensively researching this subject - appears to be US government involved
# and also appears to be used by all sorts of law enforcement agencies. For one they
# do not obey robots.txt and continually disguise their User-Agent strings. Time will
# tell if this is all correct or not.
# For now see - https://en.wikipedia.org/wiki/Cyveillance
# IMPORTANT UPDATE ON Cyveillance / Qwest Communications !!!
# **********************************************************
# I have done a lot of research on Cyveillance now and through monitoring my logs I know
# for sure what companies are using them and what they are actually looking for.
# My research has led me to understand that Cyveillance services are used by hundreds
# of companies to help them dicsover theft of copyrighted materials like images, movies
# music and other materials. I personally believe a lot of block lists who originally recommended
# blocking Cyveillance have done so to protect their torrent or p2p sites from being scanned.
# I personally have now unblocked them as image theft is a big problem of mine but if you
# do want to block Cyveillance you can simply modify the entries in the block below from "0" to "1"
# Getty Images is one such company who appears to use Cyveillance to help monitor for copyright theft.
# If you really do want to block them change all the 0's below to 1.
# Use this section at YOUR OWN RISK, you may block some legitimate networks but after many hours of
# Research this is now the completely updated list of all IP ranges IPV4 and IPV6 owned Qwest Communications
# PSINET and Cyveillance
# IMPORTANT NOTE: If you really want to keeps bot and things out of certain parts of your web site
# Rather implement a comlex Google Re-Captcha to reach sections of your sites and for people to be able
# to access download links. Google Re-Captcha with images is too complex for any bot.
38.0.0.0/8 0;
206.2.138.0/23 0;
208.71.164.0/22 0;
4.17.135.32/27 0;
63.144.0.0/13 0;
65.112.0.0/12 0;
65.192.0.0/11 0;
65.213.208.128/27 0;
65.222.176.96/27 0;
65.222.185.72/29 0;
# ****************
# Berkely Scanner
# ****************
# The Berkeley University has a scanner testing all over the web sending a complex
# payload an expecting a reply from servers who are infected or who just respond to such
# a payload. The payload looks similar to this
# "$\xC9\xE1\xDC\x9B+\x8F\x1C\xE71\x99\xA8\xDB6\x1E#\xBB\x19#Hx\xA7\xFD\x0F9-"
# and is sometime VERY long. You may have noticed this in your logs.
# I support research projects and all my servers respond with an error to this type of
# string so I do not block them but if you want to block just uncomment the following line
# or email them asking them not to scan your server. They do respond.
# Visit http://169.229.3.91/ for more info
# If you really do want to block them change all the 0 below to 1.
169.229.3.91 0;

62
.dev-tools/_conf_files_testing_changes/bots.d/blacklist-user-agents.conf Normal file
View file

@ -0,0 +1,62 @@
# EDIT THIS FILE AS YOU LIKE TO BLACKLIST OR WHITELIST ANY BAD USER-AGENT STRINGS YOU WANT TO SCAN FOR
# ****************************************************************************************************
### VERSION INFORMATION #
###################################################
### Version: V3.2018.05
### Updated: 2018-08-21
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Add One Entry Per Line - List all the extra bad User-Agents you want to permanently block or whitelist.
# This is for User-Agents that are not included in the main list of the bot blocker
# This file must exist on your system or Nginx will fail a reload due to a missing file
# This allows you finer control of keeping certain bots blocked and automatic updates will
# Never be able to remove this custom list of yours
# Please note this include file loads first before any of the already whitelisted User-Agents
# in the bad bot blocker. By loading first in line it over-rides anything below it so for instance
# if you want to block Baidu, Google or Bing for any reason you add them to this file which loads
# first and takes precedence over anything below it. This now allows even finer control over the
# bad bot blocker. Enjoy !!!
# Even though this file is called blacklist-user-agents, as mentioned it can also be used to whitelist user agents
# By adding them below and setting the 3; to 0; this will permanently whitelist the User-Agent.
# Make sure any words that contain special characters are escaped and include word boundaries as per the Regex examples below.
# Example the User-Agent name "someverybaduseragentname1" is entered as "\bsomeverybaduseragentname1\b"
# Example the User-Agent name "some-very-bad-useragentname2" is entered as "\bsome\-very\-bad\-useragentname1\b"
# the "\b" are word boundaries which prevents partial matching and false positives.
# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
# ---------------------
# WHITELISTING EXAMPLES
# ---------------------
# "~*\bsomeverygooduseragentname1\b" 0;
# "~*\bsomeverygooduseragentname2\b" 0;
# "~*\bsome\-very\-good\-useragentname2\b" 0;
# ---------------------
# BLACKLISTING EXAMPLES
# ---------------------
# "~*\bsomeverybaduseragentname1\b" 3;
# "~*\bsomeverybaduseragentname2\b" 3;
# "~*\bsome\-very\-bad\-useragentname2\b" 3;

63
.dev-tools/_conf_files_testing_changes/bots.d/blockbots.conf Normal file
View file

@ -0,0 +1,63 @@
# Author: Mitchell Krog <mitchellkrog@gmail.com> - https://github.com/mitchellkrogza/
### VERSION INFORMATION #
###################################################
### Version: V3.2017.02
### Updated: Mon Aug 21 11:29:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Include this in a vhost file within a server {} block using and include statement like below
# server {
# #Config stuff here
# include /etc/nginx/bots.d/blockbots.conf
# include /etc/nginx/bots.d/ddos.conf
# #Other config stuff here
# }
#######################################################################
# BOTS
# ****
#limit_conn bot1_connlimit 100;
limit_conn bot2_connlimit 10;
#limit_req zone=bot1_reqlimitip burst=50;
limit_req zone=bot2_reqlimitip burst=10;
if ($bad_bot = '3') {
return 444;
}
# BAD REFER WORDS
# ***************
if ($bad_words) {
return 444;
}
# REFERERS
# ********
if ($bad_referer) {
return 444;
}
# IP BLOCKS
# *********
if ($validate_client) {
return 444;
}
#######################################################################

48
.dev-tools/_conf_files_testing_changes/bots.d/custom-bad-referrers.conf Normal file
View file

@ -0,0 +1,48 @@
# EDIT THIS FILE AS YOU LIKE TO ADD ANY ADDITIONAL BAD REFERRER DOMAINS YOU WANT TO SCAN FOR ###
### VERSION INFORMATION #
###################################################
### Version: V3.2018.05
### Updated: 2018-08-21
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# This is merely an example and gets auto included as since Version 2.2017.07 introduced on 2017-04-20
# This file must exist on your system or Nginx will fail a reload due to a missing file
# Only add one entry per line
# Make sure any domains have dots and special characters escaped as per the Regex examples below.
# For example some-veryvery-randomwebsitename-thatdoesnotexist4.com should be entered as
# some\-veryvery\-randomwebsitename\-thatdoesnotexist4\.com
# *****************************************************************************************
# PLEASE MAKE SURE that you use word regex boundaries to avoid false positive detection !!!
# *****************************************************************************************
# As you can see in the examples below the domain "someveryveryrandomwebsitenamethatdoesnotexist1\.com"
# is entered with a preceding \b and an ending \b
# this makes it now "\bsomeveryveryrandomwebsitenamethatdoesnotexist1\.com\b".
# It is crucial to use the word boundaries regex formatting.
# ---------
# EXAMPLES:
# ---------
# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
# "~*\bsomeveryveryrandomwebsitenamethatdoesnotexist1\.com\b" 1;
# "~*\bsomeveryveryrandomwebsitenamethatdoesnotexist2\.com\b" 1;
# "~*\bsomeveryveryrandomwebsitenamethatdoesnotexist3\.com\b" 1;
# "~*\bsome\-veryvery\-randomweb\-sitenamethatdoesnotexist4\.com\b" 1;

36
.dev-tools/_conf_files_testing_changes/bots.d/ddos.conf Normal file
View file

@ -0,0 +1,36 @@
#######################################################################
### VERSION INFORMATION #
###################################################
### Version: V3.2017.01
### Updated: Sun Jan 29 11:35:32 SAST 2017
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Author: Mitchell Krog <mitchellkrog@gmail.com> - https://github.com/mitchellkrogza/
# Include this in a vhost file within a server {} block using and include statement like below
# server {
# #Config stuff here
# include /etc/nginx/bots.d/blockbots.conf
# include /etc/nginx/bots.d/ddos.conf
# #Other config stuff here
# }
#######################################################################
limit_conn addr 200;
limit_req zone=flood burst=200 nodelay;

45
.dev-tools/_conf_files_testing_changes/bots.d/whitelist-domains.conf Normal file
View file

@ -0,0 +1,45 @@
# EDIT THIS FILE AS YOU LIKE TO WHITELIST YOUR OWN DOMAIN NAMES AND SPARE THEM FROM ANY REFERRER CHECKING ###
### VERSION INFORMATION #
###################################################
### Version: V3.2018.04
### Updated: 2018-08-21
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Add One Entry Per Line - List all your own domains of the sites you host on the server
# This file must exist on your system or Nginx will fail a reload due to a missing file
# Automatic updates will never be able to remove this custom list of yours
# Add One Entry Per Line
# Make sure any domains have dots and special characters escaped as per the Regex examples below.
# For example myfirstowndomainname.com should be entered as myfirstowndomainname\.com
# and my-second-owndomainname.com should be entered as my\-second\-owndomainname\.com
# *****************************************************************************************
# PLEASE MAKE SURE that you use word regex boundaries to avoid false positive detection !!!
# *****************************************************************************************
# As you can see in the examples below the domain "myfirstowndomainname\.com" is entered with a preceding \b and an ending \b
# this makes it now "\bmyfirstowndomainname\.com\b". It is crucial to use the word boundaries regex formatting.
# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
# ---------
# EXAMPLES:
# ---------
# "~*\bmyfirstowndomainname\.com\b" 0;
# "~*\bmy\-second\-owndomainname\.com\b" 0;

37
.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf Normal file
View file

@ -0,0 +1,37 @@
# EDIT THIS FILE AS YOU LIKE TO WHITELIST ALL YOUR IP ADDRESSES AND IP RANGES ###
### VERSION INFORMATION #
###################################################
### Version: V3.2018.03
### Updated: 2018-08-21
###################################################
### VERSION INFORMATION ##
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Add One Entry Per Line - List all your IP's and IP Ranges you want to whitelist
# This file must exist on your system or Nginx will fail a reload due to a missing file
# Automatic updates will never be able to remove this custom list of yours
# Add One Entry Per Line as per the example
# Only add actual IP addresses and ranges here
# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
# REFRAIN FROM USING 127.0.0.1 AS IT MAY CAUSE UNDESIRABLE RESULTS ON SOME SYSTEMS
# ---------
# EXAMPLES:
# ---------
# 111.111.111.111 0;
35.193.7.13 0;
104.154.120.187 0;

1
.dev-tools/_conf_files_testing_changes/conf.d/.keep Normal file
View file

@ -0,0 +1 @@
null

29
.dev-tools/_conf_files_testing_changes/conf.d/botblocker-nginx-settings.conf Normal file
View file

@ -0,0 +1,29 @@
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# Version 1.1
# ! new directives also to be added to include_filelist.txt ! #
server_names_hash_bucket_size 128;
server_names_hash_max_size 4096;
limit_req_zone $binary_remote_addr zone=flood:50m rate=90r/s;
limit_conn_zone $binary_remote_addr zone=addr:50m;
# ****************************************************************************
# NOTE: IF you are using a system like Nginx-Proxy from @JWilder
# ****************************************************************************
# Repo URL: https://github.com/jwilder/nginx-proxy
# You will need to comment out the first line here as follows.
# #server_names_hash_bucket_size 128;
# You will also need to modify the nginx.tmpl file to add the default include
# include /etc/nginx/conf.d/*
# ****************************************************************************

10618
.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf Normal file
View file

File diff suppressed because it is too large Load diff

13
.dev-tools/_conf_files_testing_changes/default.vhost Normal file
View file

@ -0,0 +1,13 @@
server {
listen *:9000;
root /var/www/html;
server_name localhost;
charset UTF-8;
index index.html;
# Bad Bot Blocker
include /etc/nginx/bots.d/ddos.conf;
include /etc/nginx/bots.d/blockbots.conf;
}

0
.dev-tools/_curl_tests_changetesting/.keep Normal file
View file

6
.dev-tools/_curl_tests_changetesting/curltest1.txt Executable file
View file

@ -0,0 +1,6 @@
Last Tested: Thu Jun 20 10:24:15 SAST 2019
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

6
.dev-tools/_curl_tests_changetesting/curltest2.txt Executable file
View file

@ -0,0 +1,6 @@
Last Tested: Thu Jun 20 10:24:15 SAST 2019
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

6
.dev-tools/_curl_tests_changetesting/curltest3.txt Executable file
View file

@ -0,0 +1,6 @@
Last Tested: Thu Jun 20 10:24:15 SAST 2019
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

6
.dev-tools/_curl_tests_changetesting/curltest4.txt Executable file
View file

@ -0,0 +1,6 @@
Last Tested: Thu Jun 20 10:24:15 SAST 2019
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

13
.dev-tools/_curl_tests_changetesting/curltest5.txt Executable file
View file

@ -0,0 +1,13 @@
Last Tested: Thu Jun 20 10:24:15 SAST 2019
<!DOCTYPE html>
<html>
<head>
<title>Welcome to the Nginx</title>
</head>
<body>
<div align=center>
<h1>Welcome to the Nginx Ultimate Bad Bot Blocker</h1>
<p>This is merely used for testing !!!</p>
</body>
</html>

13
.dev-tools/_curl_tests_changetesting/curltest6.txt Executable file
View file

@ -0,0 +1,13 @@
Last Tested: Thu Jun 20 10:24:15 SAST 2019
<!DOCTYPE html>
<html>
<head>
<title>Welcome to the Nginx</title>
</head>
<body>
<div align=center>
<h1>Welcome to the Nginx Ultimate Bad Bot Blocker</h1>
<p>This is merely used for testing !!!</p>
</body>
</html>

13
.dev-tools/_curl_tests_changetesting/curltest7.txt Normal file
View file

@ -0,0 +1,13 @@
Last Tested: Thu Jun 20 10:24:15 SAST 2019
<!DOCTYPE html>
<html>
<head>
<title>Welcome to the Nginx</title>
</head>
<body>
<div align=center>
<h1>Welcome to the Nginx Ultimate Bad Bot Blocker</h1>
<p>This is merely used for testing !!!</p>
</body>
</html>

13
.dev-tools/_curl_tests_changetesting/curltest8.txt Normal file
View file

@ -0,0 +1,13 @@
Last Tested: Thu Jun 20 10:24:15 SAST 2019
<!DOCTYPE html>
<html>
<head>
<title>Welcome to the Nginx</title>
</head>
<body>
<div align=center>
<h1>Welcome to the Nginx Ultimate Bad Bot Blocker</h1>
<p>This is merely used for testing !!!</p>
</body>
</html>

482
.dev-tools/generate-blacklist-testing-of-changes.sh Normal file
View file

@ -0,0 +1,482 @@
#!/bin/bash
# Generator Script for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ******************
# Set Some Variables
# ******************
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
MY_GIT_TAG=V3.${YEAR}.${MONTH}.${TRAVIS_BUILD_NUMBER}
BAD_REFERRERS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < ${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list)
_now="$(date)"
# *************************************
# Specify input lists for the generator
# *************************************
_input1=${TRAVIS_BUILD_DIR}/_generator_lists/good-user-agents.list
_input2=${TRAVIS_BUILD_DIR}/_generator_lists/allowed-user-agents.list
_input3=${TRAVIS_BUILD_DIR}/_generator_lists/limited-user-agents.list
_input4=${TRAVIS_BUILD_DIR}/_generator_lists/bad-user-agents.list
_input5=${TRAVIS_BUILD_DIR}/.dev-tools/referrers-regex-format.txt
_input6=${TRAVIS_BUILD_DIR}/_generator_lists/google-ip-ranges.list
_input7=${TRAVIS_BUILD_DIR}/_generator_lists/bing-ip-ranges.list
_input8=${TRAVIS_BUILD_DIR}/_generator_lists/wordpress-theme-detectors.list
_input9=${TRAVIS_BUILD_DIR}/_generator_lists/nibbler-seo.list
_input10=${TRAVIS_BUILD_DIR}/_generator_lists/cloudflare-ip-ranges.list
_input11=${TRAVIS_BUILD_DIR}/_generator_lists/bad-ip-addresses.list
# *******************************************************
# Declare temporary database files used during generation
# *******************************************************
_inputdbA=/tmp/version-information.db
_inputdb1=/tmp/good-user-agents.db
_inputdb2=/tmp/allowed-user-agents.db
_inputdb3=/tmp/limited-user-agents.db
_inputdb4=/tmp/bad-user-agents.db
_inputdb5=/tmp/bad-referrers.db
_inputdb6=/tmp/google-ip-ranges.db
_inputdb7=/tmp/bing-ip-ranges.db
_inputdb8=/tmp/wordpress-theme-detectors.db
_inputdb9=/tmp/nibbler-seo.db
_inputdb10=/tmp/cloudflare-ip-ranges.db
_inputdb11=/tmp/bad-ip-addresses.db
# **************************************************
# Declare temporary variables used during generation
# **************************************************
_nginx=${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-testing.template
_tmpnginxA=_tmpnginxA
_tmpnginx1=_tmpnginx1
_tmpnginx2=_tmpnginx2
_tmpnginx3=_tmpnginx3
_tmpnginx4=_tmpnginx4
_tmpnginx5=_tmpnginx5
_tmpnginx6=_tmpnginx6
_tmpnginx7=_tmpnginx7
_tmpnginx8=_tmpnginx8
_tmpnginx9=_tmpnginx9
_tmpnginx10=_tmpnginx10
_tmpnginx11=_tmpnginx11
# *************************************************************
# Sort all input lists alphabetically and remove any duplicates
# *************************************************************
sort -u ${_input1} -o ${_input1}
sort -u ${_input2} -o ${_input2}
sort -u ${_input3} -o ${_input3}
sort -u ${_input4} -o ${_input4}
sort -u ${_input5} -o ${_input5}
sort -u ${_input6} -o ${_input6}
sort -u ${_input7} -o ${_input7}
sort -u ${_input8} -o ${_input8}
sort -u ${_input9} -o ${_input9}
sort -u ${_input10} -o ${_input10}
# ***************************************************************
# Start and End Strings to Search for to do inserts into template
# ***************************************************************
_start1="# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end1="# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start2="# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end2="# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start3="# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end3="# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start4="# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end4="# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start5="# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###"
_end5="# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###"
_start6="# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_end6="# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_start7="# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_end7="# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_start8="# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###"
_end8="# END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###"
_start9="# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###"
_end9="# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###"
_start10="# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_end10="# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_start11="# START KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ###"
_end11="# END KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ###"
_startmarker="### VERSION INFORMATION #"
_endmarker="### VERSION INFORMATION ##"
# **********************************
# SET ALLOW OR DENY ACTION VARIABLES
# **********************************
_action1="0;"
_action2="1;"
_action3="2;"
_action4="3;"
# ************************************
# GOOD USER AGENTS - Create and Insert
# ************************************
printf '%s\n' "${_start1}" >> ${_tmpnginx1}
while IFS= read -r LINE
do
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "${_action1}" >> ${_tmpnginx1}
done < ${_input1}
printf '%s\n' "${_end1}" >> ${_tmpnginx1}
mv ${_tmpnginx1} ${_inputdb1}
ed -s ${_inputdb1}<<\IN
1,/# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb1}
# ********************************
# ALLOWED BOTS - Create and Insert
# ********************************
printf '%s\n' "$_start2" >> ${_tmpnginx2}
while IFS= read -r LINE
do
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action2" >> ${_tmpnginx2}
done < ${_input2}
printf '%s\n' "$_end2" >> ${_tmpnginx2}
mv ${_tmpnginx2} ${_inputdb2}
ed -s ${_inputdb2}<<\IN
1,/# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb2}
# ********************************
# LIMITED BOTS - Create and Insert
# ********************************
printf '%s\n' "$_start3" >> ${_tmpnginx3}
while IFS= read -r LINE
do
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action3" >> ${_tmpnginx3}
done < ${_input3}
printf '%s\n' "$_end3" >> ${_tmpnginx3}
mv ${_tmpnginx3} ${_inputdb3}
ed -s ${_inputdb3}<<\IN
1,/# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb3}
# ******************************************************
# BAD BOTS with and Without Boundary - Create and Insert
# ******************************************************
printf '%s\n' "$_start4" >> ${_tmpnginx4}
while IFS= read -r LINE
do
printf '\t"~*%s%s"\t\t%s\n' "${LINE}" "\b" "$_action4" >> ${_tmpnginx4}
printf '\t"~*%s"\t\t%s\n' "${LINE}" "$_action4" >> ${_tmpnginx4}
done < ${_input4}
printf '%s\n' "$_end4" >> ${_tmpnginx4}
mv ${_tmpnginx4} ${_inputdb4}
ed -s ${_inputdb4}<<\IN
1,/# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb4}
# ********************************
# BAD REFERERS - Create and Insert
# ********************************
printf '%s\n' "$_start5" >> ${_tmpnginx5}
while IFS= read -r LINE
do
printf '\t%s\n' "${LINE}" >> ${_tmpnginx5}
done < ${_input5}
printf '%s\n' "$_end5" >> ${_tmpnginx5}
mv ${_tmpnginx5} ${_inputdb5}
ed -s ${_inputdb5}<<\IN
1,/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb5}
# ************************************
# GOOGLE IP RANGES - Create and Insert
# ************************************
printf '%s\n' "$_start6" >> ${_tmpnginx6}
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> ${_tmpnginx6}
done < ${_input6}
printf '%s\n' "$_end6" >> ${_tmpnginx6}
mv ${_tmpnginx6} ${_inputdb6}
ed -s ${_inputdb6}<<\IN
1,/# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb6}
# **********************************
# BING IP RANGES - Create and Insert
# **********************************
printf '%s\n' "$_start7" >> ${_tmpnginx7}
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> ${_tmpnginx7}
done < ${_input7}
printf '%s\n' "$_end7" >> ${_tmpnginx7}
mv ${_tmpnginx7} ${_inputdb7}
ed -s ${_inputdb7}<<\IN
1,/# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb7}
# *********************************************
# Wordpress Theme Detectors - Create and Insert
# *********************************************
printf '%s\n' "$_start8" >> ${_tmpnginx8}
while IFS= read -r LINE
do
printf '%s\n' "${LINE}" >> ${_tmpnginx8}
done < ${_input8}
printf '%s\n' "$_end8" >> ${_tmpnginx8}
mv ${_tmpnginx8} ${_inputdb8}
ed -s ${_inputdb8}<<\IN
1,/# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb8}
# *******************************
# Nibbler SEO - Create and Insert
# *******************************
printf '%s\n' "$_start9" >> ${_tmpnginx9}
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action2" >> ${_tmpnginx9}
done < ${_input9}
printf '%s\n' "$_end9" >> ${_tmpnginx9}
mv ${_tmpnginx9} ${_inputdb9}
ed -s ${_inputdb9}<<\IN
1,/# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb9}
# ****************************************
# CLOUDFLARE IP RANGES - Create and Insert
# ****************************************
printf '%s\n' "$_start10" >> ${_tmpnginx10}
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> ${_tmpnginx10}
done < ${_input10}
printf '%s\n' "$_end10" >> ${_tmpnginx10}
mv ${_tmpnginx10} ${_inputdb10}
ed -s ${_inputdb10}<<\IN
1,/# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb10}
# ****************************************
# KNOWN BAD IP RANGES - Create and Insert
# ****************************************
printf '%s\n' "$_start11" >> ${_tmpnginx11}
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action2" >> ${_tmpnginx11}
done < ${_input11}
printf '%s\n' "$_end11" >> ${_tmpnginx11}
mv ${_tmpnginx11} ${_inputdb11}
ed -s ${_inputdb11}<<\IN
1,/# START KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/# START KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END KNOWN BAD IP ADDRESSES ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdb11}
# *******************************************************************************
# PRINT VERSION, SCRIPT RUNTIME and UPDATE INFORMATION INTO GLOBALBLACKLIST FILES
# *******************************************************************************
printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Referrer Count: " "$BAD_REFERRERS" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" >> ${_tmpnginxA}
mv ${_tmpnginxA} ${_inputdbA}
ed -s ${_inputdbA}<<\IN
1,/### VERSION INFORMATION #/d
/### VERSION INFORMATION ##/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
/### VERSION INFORMATION #/x
.t.
.,/### VERSION INFORMATION ##/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist-testing.template
q
IN
rm ${_inputdbA}
# **************************************************
# Generate Additional Files and Copy Them to Folders
# **************************************************
sudo cp ${_nginx} ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist-testing-version.conf
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

2
.dev-tools/generate-files.sh
View file

@ -45,6 +45,7 @@
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/deploy-package.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-blacklist.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-blacklist-testing-of-changes.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-files.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-robots.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-google-disavow.sh
@ -71,6 +72,7 @@ cd ${TRAVIS_BUILD_DIR}
php ./.dev-tools/generate-regex-format-referrers.php
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/generate-blacklist.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/generate-blacklist-testing-of-changes.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/modify-config-readme-files.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/prepare-robots-input.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/generate-robots.sh

10624
.dev-tools/globalblacklist-testing.template Normal file
View file

File diff suppressed because it is too large Load diff

270
.dev-tools/install-nginx-testing-of-changes.sh Normal file
View file

@ -0,0 +1,270 @@
#!/bin/bash
# Travis CI Generating and Building for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ***************************************************************
# Start Getting Nginx Ready for Testing the Nginx Bad Bot Blocker
# ***************************************************************
printf '\n%s\n%s\n%s\n\n' "###################################" " STARTING NGINX TEST 1" "###################################"
# ******************************************************
# Make a backup of the clean and default nginx.conf file
# Needed to run Nginx test 3
# ******************************************************
printf '\n%s\n%s\n%s\n\n' "###################################" "Making backup of default nginx.conf" "###################################"
sudo cp /etc/nginx/nginx.conf ${TRAVIS_BUILD_DIR}/.dev-tools/_nginx_conf_backup/nginx.conf
# *************************************************
# Delete default site created by Nginx Installation
# *************************************************
printf '\n%s\n%s\n%s\n\n' "###########################################" "Delete any default files installed by Nginx" "###########################################"
sudo rm /etc/nginx/sites-available/default
sudo rm /etc/nginx/sites-enabled/default
sudo rm /var/www/html/*
printf '\n%s\n%s\n%s\n\n' "#######################################" "Nginx prepared for our Testing Sequence" "#######################################"
# ********************************************************
# Copy our default.vhost file into Nginx /sites-available/
# ********************************************************
printf '\n%s\n%s\n%s\n\n' "################################" "Get Nginx Setup for Nginx Test 1" "################################"
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/default.vhost /etc/nginx/sites-available/default.vhost
# **********************************************
# Link the vhost file into Nginx /sites-enabled/
# **********************************************
sudo ln -s /etc/nginx/sites-available/default.vhost /etc/nginx/sites-enabled/default.vhost
# ***********************************************************
# Copy our index.php file into the default site's root folder
# ***********************************************************
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/index.php /var/www/html/index.php
printf '\n%s\n%s\n%s\n\n' "#####################################" "Finished Nginx Setup for Nginx Test 1" "#####################################"
# ***********************************************
# Fetch our install-ngxblocker file from the repo
# ***********************************************
printf '\n%s\n%s\n%s\n\n' "######################################" "Fetch install-ngxblocker from the repo" "######################################"
sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/install-ngxblocker -O /usr/sbin/install-ngxblocker
sudo chmod +x /usr/sbin/install-ngxblocker
# **********************
# Run Install-NgxBlocker
# **********************
printf '\n%s\n%s\n%s\n\n' "######################" "Run install-ngxblocker" "######################"
cd /usr/sbin
sudo bash ./install-ngxblocker -s /usr/sbin/ -x
# **************************************************
# Set our install and setup scripts to be executable
# **************************************************
printf '\n%s\n%s\n%s\n\n' "##########################################################" "Set our install, setup and update scripts to be executable" "##########################################################"
sudo chmod +x /usr/sbin/install-ngxblocker
sudo chmod +x /usr/sbin/setup-ngxblocker
sudo chmod +x /usr/sbin/update-ngxblocker
# ********************
# Run setup-ngxblocker
# ********************
printf '\n%s\n%s\n%s\n\n' "####################" "Run setup-ngxblocker" "####################"
cd /usr/sbin
sudo bash ./setup-ngxblocker -i /usr/sbin/install-ngxblocker -x
# ************************
# Load our Nginx.conf file
# ************************
sudo nginx -c /etc/nginx/nginx.conf
# ****************************************************************************************
# Copy a dummy version of globalblacklist.conf with an older version number to test update
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################" "Copy older globalblacklist.conf file to force update" "####################################################"
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /etc/nginx/conf.d/globalblacklist.conf
# ****************************************************************************************
# Run update-ngxblocker test which downloads latest globalblacklist.conf and reloads Nginx
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "######################" "Run update-ngxblocker" "######################"
cd /usr/sbin
sudo bash ./update-ngxblocker -n
# *********************
# Force reload of Nginx
# *********************
printf '\n%s\n%s\n%s\n\n' "########################" "Force Reloading of Nginx" "########################"
sudo service nginx reload
# *******************************************************************************************
# Test that update-ngxblocker can install all missing required files by deleting some of them
# *******************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################################" "Delete existing files to force update-ngxblocker to re-download them" "####################################################################"
sudo rm /etc/nginx/conf.d/*.conf
sudo rm /etc/nginx/bots.d/*.conf
# *************************************
# List Directories to Confirm Deletions
# *************************************
printf '\n%s\n%s\n%s\n\n' "################################################" "Confirming /etc/nginx/conf.d/ directory is EMPTY" "################################################"
ls -la /etc/nginx/conf.d/
printf '\n%s\n%s\n%s\n\n' "################################################" "Confirming /etc/nginx/bots.d/ directory is EMPTY" "################################################"
ls -la /etc/nginx/bots.d/
# *********************************************************************************************************
# Run update-ngxblocker to test for missing files and download latest globalblacklist.conf and reload Nginx
# *********************************************************************************************************
printf '\n%s\n%s\n%s\n\n' "######################" "Run update-ngxblocker" "######################"
cd /usr/sbin
sudo bash ./update-ngxblocker -n
# ****************************************************
# List Directories to Confirm Downloaded Missing Files
# ****************************************************
printf '\n%s\n%s\n%s\n\n' "#######################################################################################" "Confirming /etc/nginx/conf.d/ directory has missing files replaced by update-ngxblocker" "#######################################################################################"
ls -la /etc/nginx/conf.d/
printf '\n%s\n%s\n%s\n\n' "#######################################################################################" "Confirming /etc/nginx/bots.d/ directory has missing files replaced by update-ngxblocker" "#######################################################################################"
ls -la /etc/nginx/bots.d/
# ****************************************************************************************
# Copy a dummy version of globalblacklist.conf with an older version number to test update
# ****************************************************************************************
printf '\n%s\n%s\n%s\n\n' "####################################################" "Copy older globalblacklist.conf file to force update" "####################################################"
sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/globalblacklist-dummy.conf /etc/nginx/conf.d/globalblacklist.conf
# *********************************************************************************************************
# Run update-ngxblocker to test for missing files and download latest globalblacklist.conf and reload Nginx
# *********************************************************************************************************
printf '\n%s\n%s\n%s\n\n' "######################" "Run update-ngxblocker" "######################"
cd /usr/sbin
sudo bash ./update-ngxblocker -n
# **************************
# Run setup-ngxblocker again
# **************************
printf '\n%s\n%s\n%s\n\n' "####################" "Run setup-ngxblocker" "####################"
cd /usr/sbin
sudo bash ./setup-ngxblocker -x
# *********************
# Force reload of Nginx
# *********************
printf '\n%s\n%s\n%s\n\n' "########################" "Force Reloading of Nginx" "########################"
sudo service nginx reload
# *******************************************************************
# Make sure we test using out testing version of globalblacklist.conf
# *******************************************************************
printf '\n%s\n%s\n%s\n\n' "##############################################################################" "Copy latest generated globalblacklist.conf file to test for any compile errors" "##############################################################################"
sudo cp ${TRAVIS_BUILD_DIR}/conf.d/globalblacklist-testing-version.conf /etc/nginx/conf.d/globalblacklist.conf
# *********************
# Force reload of Nginx
# *********************
printf '\n%s\n%s\n%s\n\n' "########################" "Force Reloading of Nginx" "########################"
sudo service nginx reload
# **********************
# Now Run our Curl Tests
# **********************
printf '\n%s\n%s\n%s\n\n' "######################" "Now Run our Curl Tests" "######################"
# ************************************************************
# Copy all .conf files used in Test 1 to a folder for checking
# ************************************************************
sudo cp /etc/nginx/bots.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/bots.d/
sudo cp /etc/nginx/conf.d/* ${TRAVIS_BUILD_DIR}/.dev-tools/_conf_files_testing_changes/conf.d/
# *****************************************************************************************
# Travis now moves into running the rest of the tests in the script: section of .travis.yml
# *****************************************************************************************
# **********************
# Exit With Error Number
# **********************
exit ${?}
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

227
.dev-tools/run-curl-tests-testing-of-changes.sh Normal file
View file

@ -0,0 +1,227 @@
#!/bin/bash
# Curl Testing Script for Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
##############################################################################
# _ __ _ #
# / |/ /__ _(_)__ __ __ #
# / / _ `/ / _ \\ \ / #
# /_/|_/\_, /_/_//_/_\_\ #
# __/___/ __ ___ __ ___ __ __ #
# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ #
# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ #
# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ #
# #
##############################################################################
# *******************************************
# Set Location of our Curl Test Results Files
# *******************************************
_curltest1=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_changetesting/curltest1.txt
_curltest2=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_changetesting/curltest2.txt
_curltest3=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_changetesting/curltest3.txt
_curltest4=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_changetesting/curltest4.txt
_curltest5=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_changetesting/curltest5.txt
_curltest6=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_changetesting/curltest6.txt
_curltest7=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_changetesting/curltest7.txt
_curltest8=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_changetesting/curltest8.txt
_curltest9=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_changetesting/curltest9.txt
_curltest10=${TRAVIS_BUILD_DIR}/.dev-tools/_curl_tests_changetesting/curltest10.txt
_now="$(date)"
# *************************************************
# Function Curl Test 1 - Check for Bad Bot "80legs"
# *************************************************
run_curltest1 () {
truncate -s 0 ${_curltest1}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest1}"
curl -A "80legs" http://localhost:9000/index.php 2>> ${_curltest1}
if grep -i '(52)' ${_curltest1}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# **************************************************
# Function Curl Test 2 - Check for Bad Bot "Nutch"
# **************************************************
run_curltest2 () {
truncate -s 0 ${_curltest2}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest2}"
curl -A "Nutch" http://localhost:9000/index.php 2>> ${_curltest2}
if grep -i '(52)' ${_curltest2}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# ******************************************************************
# Function Curl Test 3 - Check for Bad Referrer "100dollars-seo.com"
# ******************************************************************
run_curltest3 () {
truncate -s 0 ${_curltest3}
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest3}"
curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2>> ${_curltest3}
if grep -i '(52)' ${_curltest3}; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# ******************************************************
# Function Curl Test 4 - Check for Bad Referrer "zx6.ru"
# ******************************************************
run_curltest4 () {
truncate -s 0 ${_curltest4}
printf '\n%s\n%s\n%s\n\n' "##############################" "TESTING BAD REFERRER IS DENIED" "##############################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest4}"
curl -I http://localhost:9000/index.php -e http://zx6.ru 2>> ${_curltest4}
if grep -i '(52)' ${_curltest4}; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# *****************************************************
# Function Curl Test 5 - Check for Good Bot "GoogleBot"
# *****************************************************
run_curltest5 () {
truncate -s 0 ${_curltest5}
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest5}"
curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> ${_curltest5}
if grep -i 'Welcome' ${_curltest5}; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
#exit 1
fi
}
# ***************************************************
# Function Curl Test 6 - Check for Good Bot "BingBot"
# ***************************************************
run_curltest6 () {
truncate -s 0 ${_curltest6}
printf '\n%s\n%s\n%s\n\n' "###########################" "TESTING GOOD BOT IS ALLOWED" "###########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest6}"
curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> ${_curltest6}
if grep -i 'Welcome' ${_curltest6}; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
#exit 1
fi
}
# ***********************************************************
# Function Curl Test 7 - Check for Good Referrer "google.com"
# ***********************************************************
run_curltest7 () {
truncate -s 0 ${_curltest7}
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest7}"
curl http://localhost:9000/index.php -e http://google.com 2>&1 >> ${_curltest7}
if grep -i 'Welcome' ${_curltest7}; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
else
echo 'GOOD REFERRER NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# ***********************************************************
# Function Curl Test 8 - Check for Good Referrer "bing.com"
# ***********************************************************
run_curltest8 () {
truncate -s 0 ${_curltest8}
printf '\n%s\n%s\n%s\n\n' "################################" "TESTING GOOD REFERRER IS ALLOWED" "################################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest8}"
curl http://localhost:9000/index.php -e http://bing.com 2>&1 >> ${_curltest8}
if grep -i 'Welcome' ${_curltest8}; then
echo 'GOOD REFERRER DETECTED - TEST PASSED'
exit 0
else
echo 'GOOD REFERRER NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# **************************************************
# Function Curl Test 9 - Check for Bad Bot "Nutch"
# **************************************************
run_curltest9 () {
truncate -s 0 ${_curltest9}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest9}"
curl -A "Googlebot/Nutch-1.7" http://localhost:9000/index.php 2>> ${_curltest9}
if grep -i '(52)' ${_curltest9}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# **************************************************
# Function Curl Test 9 - Check for Bad Bot "Nutch"
# **************************************************
run_curltest10 () {
truncate -s 0 ${_curltest10}
printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING BAD BOT IS DENIED" "#########################"
printf '%s%s\n\n' "Last Tested: " "$_now" >> "${_curltest10}"
curl -A "Mozilla/5.0 (compatible; Googlebot/Nutch2.1; +http://www.google.com/bot.html)" http://localhost:9000/index.php 2>> ${_curltest10}
if grep -i '(52)' ${_curltest10}; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
#exit 1
fi
}
# *********************************
# Trigger our curl functions to run
# *********************************
run_curltest1
run_curltest2
run_curltest3
run_curltest4
run_curltest5
run_curltest6
run_curltest7
run_curltest8
run_curltest9
run_curltest10
# **********************
# Exit With Error Number
# **********************
exit ${?}

2
.travis.yml
View file

@ -46,6 +46,8 @@ script:
- bash .dev-tools/run-curl-tests-3.sh
- bash .dev-tools/install-nginx-4.sh
- bash .dev-tools/run-curl-tests-4.sh
- bash .dev-tools/install-nginx-testing-of-changes.sh
- bash .dev-tools/run-curl-tests-testing-of-changes.sh
- bash .dev-tools/modify-files-and-commit.sh
before_deploy: