From bbcd9f22d045638a4bdce181195237832cf94688 Mon Sep 17 00:00:00 2001 From: Mitchell Krog Date: Tue, 25 Jun 2019 16:48:58 +0200 Subject: [PATCH] TEST Whitelisting IPs Functions REF: #201 --- .dev-tools/generate-files.sh | 1 + .dev-tools/test-blocker-whitelist-ips.sh | 108 +++++++++++++++++++++++ .dev-tools/test_units/blacklist-ips.conf | 91 +++++++++++++++++++ .dev-tools/test_units/whitelist-ips.conf | 35 ++++++++ .travis.yml | 1 + 5 files changed, 236 insertions(+) create mode 100644 .dev-tools/test-blocker-whitelist-ips.sh create mode 100644 .dev-tools/test_units/blacklist-ips.conf create mode 100644 .dev-tools/test_units/whitelist-ips.conf diff --git a/.dev-tools/generate-files.sh b/.dev-tools/generate-files.sh index 4f0c156fd..78b07dca1 100755 --- a/.dev-tools/generate-files.sh +++ b/.dev-tools/generate-files.sh @@ -58,6 +58,7 @@ sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-quick.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-badwords.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-whitelist-domains.sh +sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-whitelist-ips.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-blacklist-domains.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-whitelist.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/prepare-robots-input.sh diff --git a/.dev-tools/test-blocker-whitelist-ips.sh b/.dev-tools/test-blocker-whitelist-ips.sh new file mode 100644 index 000000000..dcd7e3bb0 --- /dev/null +++ b/.dev-tools/test-blocker-whitelist-ips.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Curl Testing Script for Nginx Ultimate Bad Bot Blocker +# Created by: Mitchell Krog (mitchellkrog@gmail.com) +# Copyright: Mitchell Krog - https://github.com/mitchellkrogza +# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# ************************ +# Set Terminal Font Colors +# ************************ + +bold=$(tput bold) +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +white=$(tput setaf 7) +defaultcolor=$(tput setaf default) +ip=$(curl -s ifconfig.co) + +echo "${bold}${green}--------------------------" +echo "${bold}${green}Whitelist IP Test Starting" +echo "${bold}${green}--------------------------" +printf "\n\n" + +echo "${bold}${green}-------------------------" +echo "${bold}${green}Blacklisting own IP First" +echo "${bold}${green}-------------------------" +printf "\n\n" + +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/blacklist-ips.conf /etc/nginx/bots.d/blacklist-ips.conf +printf '%s%s\n' "${ip}" "1;" >> /etc/nginx/bots.d/blacklist-ips.conf + +echo "${bold}${green}--------------------" +echo "${bold}${green}Now Whitelist own IP" +echo "${bold}${green}--------------------" +printf "\n\n" + +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/whitelist-ips.conf /etc/nginx/bots.d/whitelist-ips.conf +printf '%s%s\n' "${ip}" "0;" >> /etc/nginx/bots.d/whitelist-ips.conf + +echo "${bold}${green}---------------" +echo "${bold}${green}Reloading Nginx" +echo "${bold}${green}---------------" +printf "\n\n" +sudo nginx -t && sudo nginx -s reload + + +echo "${bold}${yellow}-----------------------------------------------------------------------" +echo "${bold}${yellow}Sleeping for 30 seconds to allow Nginx to Properly Reload inside Travis" +echo "${bold}${yellow}-----------------------------------------------------------------------" +printf "\n\n" +#sleep 30s + +# ************************************************* +# Function Curl Test 1 - Test User Domain Whitelist +# ************************************************* + +#run_curltest1 () { +#if curl http://localhost:9000 -e http://www.myowndomain.com 2>&1 | grep -i 'Welcome'; then +# echo "${bold}${green}PASSED - User whitelist-domains.conf working" +#else +# echo "${bold}${red}FAILED - User whitelist-domains.conf NOT working" +# #exit 1 +#fi +#} +#run_curltest1 + +# ************************************************* +# Function Curl Test 2 - Test User Domain Whitelist +# ************************************************* + +#run_curltest2 () { +#if curl http://localhost:9000 -e http://www.myotherdomain.com 2>&1 | grep -i 'Welcome'; then +# echo "${bold}${green}PASSED - User whitelist-domains.conf working" +#else +# echo "${bold}${red}FAILED - User whitelist-domains.conf NOT working" +# #exit 1 +#fi +#} +#run_curltest2 + + +echo "${bold}${green}-----------------------------" +echo "${bold}${green}Whitelisting IP Test Complete" +echo "${bold}${green}-----------------------------" +printf "\n\n" + +# ********************** +# Exit With Error Number +# ********************** + +exit ${?} + + diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf new file mode 100644 index 000000000..c64c6f943 --- /dev/null +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -0,0 +1,91 @@ +# EDIT THIS FILE AS YOU LIKE TO ADD OR REMOVE ANY BAD IP ADDRESSES OR IP RANGES YOU WANT TO BLOCK ### + +### VERSION INFORMATION # +################################################### +### Version: V4.2019.08 +### Updated: 2019-06-24 +################################################### +### VERSION INFORMATION ## + + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# This is merely an example and gets auto included as since Version 2.2017.07 introduced on 2017-04-20 +# This file must exist on your system or Nginx will fail a reload due to a missing file +# For all intensive purpose you can delete everything inside this file and leave it +# completely blank if you do not want your Nginx Blocker to do any blocking of bad IP's + +# Add IP's you want to blacklist below this line, one per line as per example +# Nginx [warn] notices may be reported when you try reload Nginx if you happen to include an +# IP here that may already be included by the blocker with it's daily updates +# NOTE: It is only an Nginx Warning message and will not cause Nginx to fail a reload. + +# 111.111.111.111 1; + +# Cyveillance / Qwest Communications / PSINET +# ******************************************* +# I am extensively researching this subject - appears to be US government involved +# and also appears to be used by all sorts of law enforcement agencies. For one they +# do not obey robots.txt and continually disguise their User-Agent strings. Time will +# tell if this is all correct or not. +# For now see - https://en.wikipedia.org/wiki/Cyveillance + +# IMPORTANT UPDATE ON Cyveillance / Qwest Communications !!! +# ********************************************************** +# I have done a lot of research on Cyveillance now and through monitoring my logs I know +# for sure what companies are using them and what they are actually looking for. +# My research has led me to understand that Cyveillance services are used by hundreds +# of companies to help them dicsover theft of copyrighted materials like images, movies +# music and other materials. I personally believe a lot of block lists who originally recommended +# blocking Cyveillance have done so to protect their torrent or p2p sites from being scanned. +# I personally have now unblocked them as image theft is a big problem of mine but if you +# do want to block Cyveillance you can simply modify the entries in the block below from "0" to "1" +# Getty Images is one such company who appears to use Cyveillance to help monitor for copyright theft. + +# If you really do want to block them change all the 0's below to 1. +# Use this section at YOUR OWN RISK, you may block some legitimate networks but after many hours of +# Research this is now the completely updated list of all IP ranges IPV4 and IPV6 owned Qwest Communications +# PSINET and Cyveillance + +# IMPORTANT NOTE: If you really want to keeps bot and things out of certain parts of your web site +# Rather implement a comlex Google Re-Captcha to reach sections of your sites and for people to be able +# to access download links. Google Re-Captcha with images is too complex for any bot. + + 38.0.0.0/8 0; + 206.2.138.0/23 0; + 208.71.164.0/22 0; + 4.17.135.32/27 0; + 63.144.0.0/13 0; + 65.112.0.0/12 0; + 65.192.0.0/11 0; + 65.213.208.128/27 0; + 65.222.176.96/27 0; + 65.222.185.72/29 0; + +# **************** +# Berkely Scanner +# **************** + +# The Berkeley University has a scanner testing all over the web sending a complex +# payload an expecting a reply from servers who are infected or who just respond to such +# a payload. The payload looks similar to this +# "$\xC9\xE1\xDC\x9B+\x8F\x1C\xE71\x99\xA8\xDB6\x1E#\xBB\x19#Hx\xA7\xFD\x0F9-" +# and is sometime VERY long. You may have noticed this in your logs. +# I support research projects and all my servers respond with an error to this type of +# string so I do not block them but if you want to block just uncomment the following line +# or email them asking them not to scan your server. They do respond. +# Visit http://169.229.3.91/ for more info + +# If you really do want to block them change all the 0 below to 1. + + 169.229.3.91 0; diff --git a/.dev-tools/test_units/whitelist-ips.conf b/.dev-tools/test_units/whitelist-ips.conf new file mode 100644 index 000000000..f415804df --- /dev/null +++ b/.dev-tools/test_units/whitelist-ips.conf @@ -0,0 +1,35 @@ +# EDIT THIS FILE AS YOU LIKE TO WHITELIST ALL YOUR IP ADDRESSES AND IP RANGES ### + +### VERSION INFORMATION # +################################################### +### Version: V4.2019.04 +### Updated: 2019-06-24 +################################################### +### VERSION INFORMATION ## + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# Add One Entry Per Line - List all your IP's and IP Ranges you want to whitelist +# This file must exist on your system or Nginx will fail a reload due to a missing file +# Automatic updates will never be able to remove this custom list of yours +# Add One Entry Per Line as per the example +# Only add actual IP addresses and ranges here + +# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED +# REFRAIN FROM USING 127.0.0.1 AS IT MAY CAUSE UNDESIRABLE RESULTS ON SOME SYSTEMS +# --------- +# EXAMPLES: +# --------- +# 111.111.111.111 0; + + diff --git a/.travis.yml b/.travis.yml index 2f48fa758..80b39a777 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,6 +50,7 @@ script: - bash .dev-tools/test-blocker.sh - bash .dev-tools/test-blocker-badwords.sh - bash .dev-tools/test-blocker-whitelist-domains.sh + - bash .dev-tools/test-blocker-whitelist-ips.sh - bash .dev-tools/test-blocker-whitelist.sh - bash .dev-tools/modify-files-and-commit.sh