TEST Whitelisting IPs Functions REF: #201

.dev-tools/generate-files.sh

@@ -58,6 +58,7 @@ sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-quick.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-badwords.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-whitelist-domains.sh
+sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-whitelist-ips.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-blacklist-domains.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-whitelist.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/prepare-robots-input.sh

.dev-tools/test-blocker-whitelist-ips.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+# Curl Testing Script for Nginx Ultimate Bad Bot Blocker
+# Created by: Mitchell Krog (mitchellkrog@gmail.com)
+# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
+# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+
+##############################################################################                                                                
+#       _  __     _                                                          #
+#      / |/ /__ _(_)__ __ __                                                 #
+#     /    / _ `/ / _ \\ \ /                                                 #
+#    /_/|_/\_, /_/_//_/_\_\                                                  #
+#       __/___/      __   ___       __     ___  __         __                #
+#      / _ )___ ____/ /  / _ )___  / /_   / _ )/ /__  ____/ /_____ ____      #
+#     / _  / _ `/ _  /  / _  / _ \/ __/  / _  / / _ \/ __/  '_/ -_) __/      #
+#    /____/\_,_/\_,_/  /____/\___/\__/  /____/_/\___/\__/_/\_\\__/_/         #
+#                                                                            #
+##############################################################################                                                                
+
+# ************************
+# Set Terminal Font Colors
+# ************************
+
+bold=$(tput bold)
+red=$(tput setaf 1)
+green=$(tput setaf 2)
+yellow=$(tput setaf 3)
+blue=$(tput setaf 4)
+magenta=$(tput setaf 5)
+cyan=$(tput setaf 6)
+white=$(tput setaf 7)
+defaultcolor=$(tput setaf default)
+ip=$(curl -s ifconfig.co)
+
+echo "${bold}${green}--------------------------"
+echo "${bold}${green}Whitelist IP Test Starting"
+echo "${bold}${green}--------------------------"
+printf "\n\n"
+
+echo "${bold}${green}-------------------------"
+echo "${bold}${green}Blacklisting own IP First"
+echo "${bold}${green}-------------------------"
+printf "\n\n"
+
+sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-ips.conf /etc/nginx/bots.d/blacklist-ips.conf
+printf '%s%s\n' "${ip}" "1;" >> /etc/nginx/bots.d/blacklist-ips.conf
+
+echo "${bold}${green}--------------------"
+echo "${bold}${green}Now Whitelist own IP"
+echo "${bold}${green}--------------------"
+printf "\n\n"
+
+sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf /etc/nginx/bots.d/whitelist-ips.conf
+printf '%s%s\n' "${ip}" "0;" >> /etc/nginx/bots.d/whitelist-ips.conf
+
+echo "${bold}${green}---------------"
+echo "${bold}${green}Reloading Nginx"
+echo "${bold}${green}---------------"
+printf "\n\n"
+sudo nginx -t && sudo nginx -s reload
+
+
+echo "${bold}${yellow}-----------------------------------------------------------------------"
+echo "${bold}${yellow}Sleeping for 30 seconds to allow Nginx to Properly Reload inside Travis"
+echo "${bold}${yellow}-----------------------------------------------------------------------"
+printf "\n\n"
+#sleep 30s
+
+# *************************************************
+# Function Curl Test 1 - Test User Domain Whitelist
+# *************************************************
+
+#run_curltest1 () {
+#if curl http://localhost:9000 -e http://www.myowndomain.com 2>&1 | grep -i 'Welcome'; then
+#   echo "${bold}${green}PASSED - User whitelist-domains.conf working"
+#else
+#   echo "${bold}${red}FAILED - User whitelist-domains.conf NOT working"
+#   #exit 1
+#fi
+#}
+#run_curltest1
+
+# *************************************************
+# Function Curl Test 2 - Test User Domain Whitelist
+# *************************************************
+
+#run_curltest2 () {
+#if curl http://localhost:9000 -e http://www.myotherdomain.com 2>&1 | grep -i 'Welcome'; then
+#   echo "${bold}${green}PASSED - User whitelist-domains.conf working"
+#else
+#   echo "${bold}${red}FAILED - User whitelist-domains.conf NOT working"
+#   #exit 1
+#fi
+#}
+#run_curltest2
+
+
+echo "${bold}${green}-----------------------------"
+echo "${bold}${green}Whitelisting IP Test Complete"
+echo "${bold}${green}-----------------------------"
+printf "\n\n"
+
+# **********************
+# Exit With Error Number
+# **********************
+
+exit ${?}
+
+

.dev-tools/test_units/blacklist-ips.conf

@@ -0,0 +1,91 @@
+# EDIT THIS FILE AS YOU LIKE TO ADD OR REMOVE ANY BAD IP ADDRESSES OR IP RANGES YOU WANT TO BLOCK ###
+
+### VERSION INFORMATION #
+###################################################
+### Version: V4.2019.08
+### Updated: 2019-06-24
+###################################################
+### VERSION INFORMATION ##
+
+
+##############################################################################                                                                
+#       _  __     _                                                          #
+#      / |/ /__ _(_)__ __ __                                                 #
+#     /    / _ `/ / _ \\ \ /                                                 #
+#    /_/|_/\_, /_/_//_/_\_\                                                  #
+#       __/___/      __   ___       __     ___  __         __                #
+#      / _ )___ ____/ /  / _ )___  / /_   / _ )/ /__  ____/ /_____ ____      #
+#     / _  / _ `/ _  /  / _  / _ \/ __/  / _  / / _ \/ __/  '_/ -_) __/      #
+#    /____/\_,_/\_,_/  /____/\___/\__/  /____/_/\___/\__/_/\_\\__/_/         #
+#                                                                            #
+##############################################################################                                                                
+
+# This is merely an example and gets auto included as since Version 2.2017.07 introduced on 2017-04-20
+# This file must exist on your system or Nginx will fail a reload due to a missing file
+# For all intensive purpose you can delete everything inside this file and leave it
+# completely blank if you do not want your Nginx Blocker to do any blocking of bad IP's
+
+# Add IP's you want to blacklist below this line, one per line as per example
+# Nginx [warn] notices may be reported when you try reload Nginx if you happen to include an
+# IP here that may already be included by the blocker with it's daily updates
+# NOTE: It is only an Nginx Warning message and will not cause Nginx to fail a reload.
+
+# 111.111.111.111	1;
+
+# Cyveillance / Qwest Communications / PSINET
+# *******************************************
+# I am extensively researching this subject - appears to be US government involved
+# and also appears to be used by all sorts of law enforcement agencies. For one they 
+# do not obey robots.txt and continually disguise their User-Agent strings. Time will
+# tell if this is all correct or not.
+# For now see - https://en.wikipedia.org/wiki/Cyveillance
+
+# IMPORTANT UPDATE ON Cyveillance / Qwest Communications !!!
+# **********************************************************
+# I have done a lot of research on Cyveillance now and through monitoring my logs I know
+# for sure what companies are using them and what they are actually looking for.
+# My research has led me to understand that Cyveillance services are used by hundreds
+# of companies to help them dicsover theft of copyrighted materials like images, movies
+# music and other materials. I personally believe a lot of block lists who originally recommended
+# blocking Cyveillance have done so to protect their torrent or p2p sites from being scanned.
+# I personally have now unblocked them as image theft is a big problem of mine but if you
+# do want to block Cyveillance you can simply modify the entries in the block below from "0" to "1"
+# Getty Images is one such company who appears to use Cyveillance to help monitor for copyright theft.
+  
+# If you really do want to block them change all the 0's below to 1.
+# Use this section at YOUR OWN RISK, you may block some legitimate networks but after many hours of
+# Research this is now the completely updated list of all IP ranges IPV4 and IPV6 owned Qwest Communications
+# PSINET and Cyveillance
+
+# IMPORTANT NOTE: If you really want to keeps bot and things out of certain parts of your web site
+# Rather implement a comlex Google Re-Captcha to reach sections of your sites and for people to be able
+# to access download links. Google Re-Captcha with images is too complex for any bot.
+
+	38.0.0.0/8			0;
+	206.2.138.0/23		0;
+	208.71.164.0/22		0;
+	4.17.135.32/27		0;
+	63.144.0.0/13		0;
+	65.112.0.0/12		0;
+	65.192.0.0/11		0;
+	65.213.208.128/27	0;
+	65.222.176.96/27	0;
+	65.222.185.72/29	0;
+
+# ****************
+# Berkely Scanner
+# ****************
+
+# The Berkeley University has a scanner testing all over the web sending a complex
+# payload an expecting a reply from servers who are infected or who just respond to such
+# a payload. The payload looks similar to this
+# "$\xC9\xE1\xDC\x9B+\x8F\x1C\xE71\x99\xA8\xDB6\x1E#\xBB\x19#Hx\xA7\xFD\x0F9-"
+# and is sometime VERY long. You may have noticed this in your logs.
+# I support research projects and all my servers respond with an error to this type of
+# string so I do not block them but if you want to block just uncomment the following line 
+# or email them asking them not to scan your server. They do respond.
+# Visit http://169.229.3.91/ for more info
+
+# If you really do want to block them change all the 0 below to 1.
+
+	169.229.3.91		0;

.dev-tools/test_units/whitelist-ips.conf

@@ -0,0 +1,35 @@
+# EDIT THIS FILE AS YOU LIKE TO WHITELIST ALL YOUR IP ADDRESSES AND IP RANGES ###
+
+### VERSION INFORMATION #
+###################################################
+### Version: V4.2019.04
+### Updated: 2019-06-24
+###################################################
+### VERSION INFORMATION ##
+
+##############################################################################                                                                
+#       _  __     _                                                          #
+#      / |/ /__ _(_)__ __ __                                                 #
+#     /    / _ `/ / _ \\ \ /                                                 #
+#    /_/|_/\_, /_/_//_/_\_\                                                  #
+#       __/___/      __   ___       __     ___  __         __                #
+#      / _ )___ ____/ /  / _ )___  / /_   / _ )/ /__  ____/ /_____ ____      #
+#     / _  / _ `/ _  /  / _  / _ \/ __/  / _  / / _ \/ __/  '_/ -_) __/      #
+#    /____/\_,_/\_,_/  /____/\___/\__/  /____/_/\___/\__/_/\_\\__/_/         #
+#                                                                            #
+##############################################################################                                                                
+
+# Add One Entry Per Line - List all your IP's and IP Ranges you want to whitelist
+# This file must exist on your system or Nginx will fail a reload due to a missing file
+# Automatic updates will never be able to remove this custom list of yours 
+# Add One Entry Per Line as per the example
+# Only add actual IP addresses and ranges here
+
+# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
+# REFRAIN FROM USING 127.0.0.1 AS IT MAY CAUSE UNDESIRABLE RESULTS ON SOME SYSTEMS
+# ---------
+# EXAMPLES:
+# ---------
+#	111.111.111.111		0;
+
+

.travis.yml

@@ -50,6 +50,7 @@ script:
   - bash .dev-tools/test-blocker.sh
   - bash .dev-tools/test-blocker-badwords.sh
   - bash .dev-tools/test-blocker-whitelist-domains.sh
+  - bash .dev-tools/test-blocker-whitelist-ips.sh
   - bash .dev-tools/test-blocker-whitelist.sh
   - bash .dev-tools/modify-files-and-commit.sh