diff --git a/.dev-tools/blacklist-user-agents.conf b/.dev-tools/blacklist-user-agents.conf new file mode 100644 index 000000000..aa4a49b56 --- /dev/null +++ b/.dev-tools/blacklist-user-agents.conf @@ -0,0 +1,63 @@ +# EDIT THIS FILE AS YOU LIKE TO BLACKLIST OR WHITELIST ANY BAD USER-AGENT STRINGS YOU WANT TO SCAN FOR +# **************************************************************************************************** + +### VERSION INFORMATION # +################################################### +### Version: V3.2018.05 +### Updated: 2018-08-21 +################################################### +### VERSION INFORMATION ## + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +# Add One Entry Per Line - List all the extra bad User-Agents you want to permanently block or whitelist. + +# This is for User-Agents that are not included in the main list of the bot blocker +# This file must exist on your system or Nginx will fail a reload due to a missing file +# This allows you finer control of keeping certain bots blocked and automatic updates will +# Never be able to remove this custom list of yours + +# Please note this include file loads first before any of the already whitelisted User-Agents +# in the bad bot blocker. By loading first in line it over-rides anything below it so for instance +# if you want to block Baidu, Google or Bing for any reason you add them to this file which loads +# first and takes precedence over anything below it. This now allows even finer control over the +# bad bot blocker. Enjoy !!! + +# Even though this file is called blacklist-user-agents, as mentioned it can also be used to whitelist user agents +# By adding them below and setting the 3; to 0; this will permanently whitelist the User-Agent. + +# Make sure any words that contain special characters are escaped and include word boundaries as per the Regex examples below. + +# Example the User-Agent name "someverybaduseragentname1" is entered as "\bsomeverybaduseragentname1\b" +# Example the User-Agent name "some-very-bad-useragentname2" is entered as "\bsome\-very\-bad\-useragentname1\b" +# the "\b" are word boundaries which prevents partial matching and false positives. + +# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED + +# --------------------- +# WHITELISTING EXAMPLES +# --------------------- +# "~*\bsomeverygooduseragentname1\b" 0; +# "~*\bsomeverygooduseragentname2\b" 0; +# "~*\bsome\-very\-good\-useragentname2\b" 0; + +# --------------------- +# BLACKLISTING EXAMPLES +# --------------------- +# "~*\bsomeverybaduseragentname1\b" 3; +# "~*\bsomeverybaduseragentname2\b" 3; +# "~*\bsome\-very\-bad\-useragentname2\b" 3; + + "~*(?:\b)Nutch(?:\b|)" 0; + + diff --git a/.dev-tools/generate-files.sh b/.dev-tools/generate-files.sh index 6f5b18d82..adc15af28 100755 --- a/.dev-tools/generate-files.sh +++ b/.dev-tools/generate-files.sh @@ -54,6 +54,7 @@ sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-regex-format-referrers.php sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-config-readme-files.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-files-and-commit.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker.sh +sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker-whitelist.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/prepare-robots-input.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-1.sh sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/install-nginx-2.sh diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 3c841c5ad..d8668f84b 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -131,8 +131,6 @@ map $http_user_agent $bad_bot { # END CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # ********************************************************************************* -# Test User Whitelisting - "~*(?:\b)Nutch(?:\b|)" 0; # ********************************************* # Bad User-Agent Strings That We Block Outright diff --git a/.dev-tools/test-blocker-whitelist.sh b/.dev-tools/test-blocker-whitelist.sh new file mode 100644 index 000000000..ef9864e80 --- /dev/null +++ b/.dev-tools/test-blocker-whitelist.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Curl Testing Script for Nginx Ultimate Bad Bot Blocker +# Created by: Mitchell Krog (mitchellkrog@gmail.com) +# Copyright: Mitchell Krog - https://github.com/mitchellkrogza +# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker + +############################################################################## +# _ __ _ # +# / |/ /__ _(_)__ __ __ # +# / / _ `/ / _ \\ \ / # +# /_/|_/\_, /_/_//_/_\_\ # +# __/___/ __ ___ __ ___ __ __ # +# / _ )___ ____/ / / _ )___ / /_ / _ )/ /__ ____/ /_____ ____ # +# / _ / _ `/ _ / / _ / _ \/ __/ / _ / / _ \/ __/ '_/ -_) __/ # +# /____/\_,_/\_,_/ /____/\___/\__/ /____/_/\___/\__/_/\_\\__/_/ # +# # +############################################################################## + +echo "Whitelist Tests Starting" + +echo "Activating User Whitelist/Blacklist" +sudo cp ${TRAVIS_BUILD_DIR}/.dev-tools/blacklist-user-agents.conf /etc/nginx/bots.d/blacklist-user-agents.conf +echo "Reloading Nginx" +sudo nginx -t && sudo nginx -s reload + + +# ******************************************************* +# Function Curl Test 16 - Test User Whitelist for "Nutch" +# ******************************************************* + +run_curltest1 () { +printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING USER WHITELIST" "#########################" +if curl -v -A "Nutch" http://localhost:9000/index.php 2>&1 | grep -i 'Welcome'; then + echo "$(tput setaf 2)BAD BOT ALLOWED - TEST PASSED" +else + echo "$(tput setaf 1)WHITELISTING FAILED - TEST FAILED" + #exit 1 + curl -v -A "Nutch" http://localhost:9000/index.php 2>&1 +fi +} +run_curltest1 + + +echo "Whitelist Tests Completed" + +# ********************** +# Exit With Error Number +# ********************** + +exit ${?} + + diff --git a/.dev-tools/test-blocker.sh b/.dev-tools/test-blocker.sh index dfa9696b2..2a5960f3d 100755 --- a/.dev-tools/test-blocker.sh +++ b/.dev-tools/test-blocker.sh @@ -251,22 +251,6 @@ fi } run_curltest15 -# ******************************************************* -# Function Curl Test 16 - Test User Whitelist for "Nutch" -# ******************************************************* - -run_curltest16 () { -printf '\n%s\n%s\n%s\n\n' "#########################" "TESTING USER WHITELIST" "#########################" -if curl -v -A "Nutch" http://localhost:9000/index.php 2>&1 | grep -i 'Welcome'; then - echo "$(tput setaf 2)BAD BOT ALLOWED - TEST PASSED" -else - echo "$(tput setaf 1)WHITELISTING FAILED - TEST FAILED" - #exit 1 - curl -v -A "Nutch" http://localhost:9000/index.php 2>&1 -fi -} -run_curltest16 - echo "Tests Completed" diff --git a/.travis.yml b/.travis.yml index d2da4a9ba..eb6ae0898 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,8 +40,10 @@ script: - bash .dev-tools/generate-files.sh - bash .dev-tools/install-nginx-1.sh - bash .dev-tools/test-blocker.sh + - bash .dev-tools/test-blocker-whitelist.sh - bash .dev-tools/install-nginx-testing-of-changes.sh - bash .dev-tools/test-blocker.sh + - bash .dev-tools/test-blocker-whitelist.sh #- bash .dev-tools/install-nginx-2.sh #- bash .dev-tools/test-blocker.sh #- bash .dev-tools/install-nginx-3.sh