From 7ad2fcf77a19824eb894733a77bc75d6b0764662 Mon Sep 17 00:00:00 2001 From: Mitchell Krog Date: Mon, 24 Jun 2019 16:01:50 +0200 Subject: [PATCH] Introduce Testing of bad-referrer-words.conf --- .dev-tools/generate-files.sh | 1 + .dev-tools/test-blocker-badwords.sh | 96 +++++++++++++++++++ .dev-tools/test_units/bad-referrer-words.conf | 75 +++++++++++++++ .travis.yml | 1 + 4 files changed, 173 insertions(+) create mode 100644 .dev-tools/test-blocker-badwords.sh create mode 100644 .dev-tools/test_units/bad-referrer-words.conf diff --git a/.dev-tools/generate-files.sh b/.dev-tools/generate-files.sh index c8f1ec4f2..88793ecc7 100755 --- a/.dev-tools/generate-files.sh +++ b/.dev-tools/generate-files.sh @@ -55,6 +55,7 @@ sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-config-readme-files.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-files-and-commit.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-quick.sh +sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-badwords.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-whitelist.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/prepare-robots-input.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-1.sh diff --git a/.dev-tools/test-blocker-badwords.sh b/.dev-tools/test-blocker-badwords.sh new file mode 100644 index 000000000..094d847d0 --- /dev/null +++ b/.dev-tools/test-blocker-badwords.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Curl Testing Script for Nginx Ultimate Bad Bot Blocker +# Created by: Mitchell Krog (mitchellkrog@gmail.com) +# Copyright: Mitchell Krog - https://github.com/mitchellkrogza +# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# ************************ +# Set Terminal Font Colors +# ************************ + +bold=$(tput bold) +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +white=$(tput setaf 7) +defaultcolor=$(tput setaf default) + +echo "${bold}${green}--------------------------------" +echo "${bold}${green}Bad Referrer Words Test Starting" +echo "${bold}${green}--------------------------------" +printf "\n\n" + +echo "${bold}${green}----------------------------------------" +echo "${bold}${green}Activating Users bad-referrer-words.conf" +echo "${bold}${green}----------------------------------------" +printf "\n\n" +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/test_units/bad-referrer-words.conf /etc/nginx/bots.d/bad-referrer-words.conf +echo "${bold}${green}---------------" +echo "${bold}${green}Reloading Nginx" +echo "${bold}${green}---------------" +printf "\n\n" +sudo nginx -t && sudo nginx -s reload + + +echo "${bold}${yellow}-----------------------------------------------------------------------" +echo "${bold}${yellow}Sleeping for 30 seconds to allow Nginx to Properly Reload inside Travis" +echo "${bold}${yellow}-----------------------------------------------------------------------" +printf "\n\n" +sleep 30s + +# ******************************************************* +# Function Curl Test 1 - Test User Whitelist for "Nutch" +# ******************************************************* + +run_curltest1 () { +if curl -v -A "thisisabadword" http://localhost:9000 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - User bad-referrer-words.conf working" +else + echo "${bold}${red}FAILED - User bad-referrer-words.conf NOT working" + #exit 1 +fi +} +run_curltest1 + +# ************************************************************** +# Function Curl Test 2 - Check for Whitelisted Referrer "zx6.ru" +# ************************************************************** + +run_curltest2 () { +if curl -v -A "thisisanotherbadword" http://localhost:9000 2>&1 | grep -i '(52)'; then + echo "${bold}${green}PASSED - User bad-referrer-words.conf working" +else + echo "${bold}${red}FAILED - User bad-referrer-words.conf NOT working" + #exit 1 +fi +} +run_curltest2 + + +echo "${bold}${green}--------------------------------" +echo "${bold}${green}Bad Referrer Words Test Complete" +echo "${bold}${green}--------------------------------" +printf "\n\n" + +# ********************** +# Exit With Error Number +# ********************** + +exit ${?} + + diff --git a/.dev-tools/test_units/bad-referrer-words.conf b/.dev-tools/test_units/bad-referrer-words.conf new file mode 100644 index 000000000..5885d44af --- /dev/null +++ b/.dev-tools/test_units/bad-referrer-words.conf @@ -0,0 +1,75 @@ +# EDIT THIS FILE AS YOU LIKE TO ADD OR REMOVE ANY BAD WORDS YOU WANT TO SCAN FOR ### + +### VERSION INFORMATION # +################################################### +### Version: V4.2019.07 +### Updated: 2019-06-24 +################################################### +### VERSION INFORMATION ## + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# This is merely an example and gets auto included as since Version 2.2017.07 introduced on 2017-04-20 +# This file must exist on your system or Nginx will fail a reload due to a missing file +# For all intensive purpose you can delete everything inside this file and leave it +# completely blank if you do not want your Nginx Blocker to include scanning for bad words within urls or referrer string +# Only add one entry per line + +# ******************************* +# !!! WARNING WARNING WARNING !!! +# ******************************* + +# *************************************** +# PLEASE BE VERY CAREFUL HOW YOU USE THIS +# *************************************** +# Here is an example of how one supposed bad word can cause your whole site to go down. +# An issue was logged where the users own domain name was specialisteparquet.com +# Because this list contained the word "cialis" it was detected within his domain name causing +# his entire site to go down and not server any assets. +# That one entry would even cause any site containing a word like "specialist" anywhere in any +# of their sites pages to cause them to be blocked and whitelisting your own domain name in the +# whitelist-domains.conf file will not even bypass this, SO BE CAREFUL PLEASE + +# Think very carefully before you add any word here + +# ***************************************************************************************** +# PLEASE MAKE SURE that you use word regex boundaries to avoid false positive detection !!! +# ***************************************************************************************** + +# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED + +# As you can see in the examples below the word "adultgalls" is entered with a preceding \b and an ending \b +# this makes it now "\badultgalls\b". It is crucial to use the word boundaries regex formatting. + +# --------- +# EXAMPLES: +# --------- +# "~*\badultgalls\b" 1; +# "~*\bamateurxpass\b" 1; +# "~*\bbigblackbooty\b" 1; +# "~*\bblacktits\b" 1; +# "~*\bcookie\-law\-enforcement\b" 1; +# "~*\bfree\-share\-buttons\b" 1; +# "~*\bfree\-social\-buttons\b" 1; +# "~*\bfuck\-paid\-share\-buttons\b" 1; +# "~*\bilovevitaly\b" 1; +# "~*\blaw\-enforcement\-bot\b" 1; +# "~*\blaw\-enforcement\-check\b" 1; +# "~*\bshare\-buttons\-for\-free\b" 1; +# "~*\bwebfuck\b" 1; +# "~*\bxxxrus\b" 1; +# "~*\bzeroredirect\b" 1; + + "~*\bthisisabadword\b" 1; + "~*\bthisisanotherbadword\b" 1; + diff --git a/.travis.yml b/.travis.yml index 4ace5659e..aece026cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,7 @@ script: #- bash .dev-tools/test-blocker-quick.sh - bash .dev-tools/install-nginx-testing-of-changes.sh - bash .dev-tools/test-blocker.sh + - bash .dev-tools/test-blocker-badwords.sh - bash .dev-tools/test-blocker-whitelist.sh - bash .dev-tools/modify-files-and-commit.sh