From 751f2198cff29ef7be6871cbe99bc77ea79630bf Mon Sep 17 00:00:00 2001 From: Mitchell Krog Date: Thu, 6 Jul 2017 15:31:09 +0200 Subject: [PATCH] Bring New Travis Generator and Testing Scripts Online --- .travis.yml | 9 +- AUTO-CONFIGURATION.md | 9 +- MANUAL-CONFIGURATION.md | 9 +- README.md | 9 +- travisCI/_curl_tests/curltest1.txt | 4 + travisCI/_curl_tests/curltest2.txt | 4 + travisCI/_curl_tests/curltest3.txt | 4 + travisCI/_curl_tests/curltest4.txt | 4 + travisCI/_curl_tests/curltest5.txt | 5 + travisCI/_curl_tests/curltest6.txt | 5 + travisCI/generate-blacklist.sh | 221 ++-- travisCI/generate-google-disavow.sh | 7 +- travisCI/generate-robots.sh | 23 +- travisCI/globalblacklist.template | 1360 ++++++++++++------------ travisCI/install-nginx.sh | 1 + travisCI/modify-config-readme-files.sh | 58 +- travisCI/run-curl-tests.sh | 97 ++ 17 files changed, 933 insertions(+), 896 deletions(-) create mode 100755 travisCI/_curl_tests/curltest1.txt create mode 100755 travisCI/_curl_tests/curltest2.txt create mode 100755 travisCI/_curl_tests/curltest3.txt create mode 100755 travisCI/_curl_tests/curltest4.txt create mode 100755 travisCI/_curl_tests/curltest5.txt create mode 100755 travisCI/_curl_tests/curltest6.txt create mode 100755 travisCI/run-curl-tests.sh diff --git a/.travis.yml b/.travis.yml index 00b73d9a8..a6f93a92d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,14 +33,7 @@ install: script: - travisCI/install-nginx.sh - - curl -vsf 'http://localhost:9000/index.php' &> /dev/stdout - - curl -A "googlebot" http://localhost:9000/index.php &> /dev/stdout - - curl -A "bingbot" http://localhost:9000/index.php &> /dev/stdout - - curl -I http://localhost:9000/index.php -e http://google.com - - STATUSCODE=$(curl -A "80legs" http://localhost:9000/index.php &> /dev/stderr --write-out "%{http_code}") | if test $STATUSCODE 52; then exit 0; fi - - STATUSCODE=$(curl -A "masscan" http://localhost:9000/index.php &> /dev/stderr --write-out "%{http_code}") | if test $STATUSCODE 52; then exit 0; fi - - STATUSCODE=$(curl -I http://localhost:9000/index.php -e http://100dollars-seo.com &> /dev/stderr --write-out "%{http_code}") | if test $STATUSCODE 52; then exit 0; fi - - STATUSCODE=$(curl -I http://localhost:9000/index.php -e http://zx6.ru &> /dev/stderr --write-out "%{http_code}") | if test $STATUSCODE 52; then exit 0; fi + - sudo travisCI/run-curl-tests.sh - travisCI/modify-files-and-commit.sh before_deploy: diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 4f5149b17..8fc24a9eb 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,13 +4,8 @@ ##### Created by: https://github.com/mitchellkrogza ##### Copyright Mitchell Krog -##### Version Information # -******************************************** -#### Version: V3.2017.06.631 -#### Bad Referrer Count: 4948 -#### Bad Bot Count: 480 -******************************************** -##### Version Information ## +### Version Information # +### Version Information ## ## Update Notification System diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index c6070e242..1483f4080 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,13 +4,8 @@ ##### Created by: https://github.com/mitchellkrogza ##### Copyright Mitchell Krog -##### Version Information # -******************************************** -#### Version: V3.2017.06.631 -#### Bad Referrer Count: 4948 -#### Bad Bot Count: 480 -******************************************** -##### Version Information ## +### Version Information # +### Version Information ## ## Update Notification System diff --git a/README.md b/README.md index 00ddabc2b..022c1ad96 100755 --- a/README.md +++ b/README.md @@ -8,13 +8,8 @@ # Nginx Bad Bot and User-Agent Blocker, Spam Referrer Blocker, Anti DDOS, Bad IP Blocker and Wordpress Theme Detector Blocker ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking -##### Version Information # -******************************************** -#### Version: V3.2017.06.631 -#### Bad Referrer Count: 4948 -#### Bad Bot Count: 480 -******************************************** -##### Version Information ## +### Version Information # +### Version Information ## ## If this helps you [why not buy me a beer](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BKF9XT6WHATLG):beer: diff --git a/travisCI/_curl_tests/curltest1.txt b/travisCI/_curl_tests/curltest1.txt new file mode 100755 index 000000000..f425d04ee --- /dev/null +++ b/travisCI/_curl_tests/curltest1.txt @@ -0,0 +1,4 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +curl: (52) Empty reply from server diff --git a/travisCI/_curl_tests/curltest2.txt b/travisCI/_curl_tests/curltest2.txt new file mode 100755 index 000000000..f425d04ee --- /dev/null +++ b/travisCI/_curl_tests/curltest2.txt @@ -0,0 +1,4 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +curl: (52) Empty reply from server diff --git a/travisCI/_curl_tests/curltest3.txt b/travisCI/_curl_tests/curltest3.txt new file mode 100755 index 000000000..f425d04ee --- /dev/null +++ b/travisCI/_curl_tests/curltest3.txt @@ -0,0 +1,4 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +curl: (52) Empty reply from server diff --git a/travisCI/_curl_tests/curltest4.txt b/travisCI/_curl_tests/curltest4.txt new file mode 100755 index 000000000..f425d04ee --- /dev/null +++ b/travisCI/_curl_tests/curltest4.txt @@ -0,0 +1,4 @@ + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +curl: (52) Empty reply from server diff --git a/travisCI/_curl_tests/curltest5.txt b/travisCI/_curl_tests/curltest5.txt new file mode 100755 index 000000000..1081dac02 --- /dev/null +++ b/travisCI/_curl_tests/curltest5.txt @@ -0,0 +1,5 @@ +> $_tmpnginx1 -for line in $(cat $_input1); do -printf "\t\"~${line}\"\t\t$ACTION1\n" >> $_tmpnginx1 -done -echo $_end1 >> $_tmpnginx1 -IFS=$GOODBOTSIFS +printf '%s\n' "$_start1" >> $_tmpnginx1 +while IFS= read -r LINE +do +printf '\t"~%s"\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx1" +done < $_input1 +printf '%s\n' "$_end1" >> $_tmpnginx1 mv $_tmpnginx1 $_inputdb1 ed -s $_inputdb1<<\IN 1,/# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d @@ -169,14 +171,12 @@ rm $_inputdb1 # ALLOWED BOTS - Create and Insert # ******************************** -ALLOWEDBOTSIFS=$IFS -IFS=$'\n' -echo $_start2 >> $_tmpnginx2 -for line in $(cat $_input2); do -printf "\t\"~${line}\"\t\t$ACTION2\n" >> $_tmpnginx2 -done -echo $_end2 >> $_tmpnginx2 -IFS=$ALLOWEDBOTSIFS +printf '%s\n' "$_start2" >> $_tmpnginx2 +while IFS= read -r LINE +do +printf '\t"~%s"\t\t%s\n' "${LINE}" "$_action2" >> "$_tmpnginx2" +done < $_input2 +printf '%s\n' "$_end2" >> $_tmpnginx2 mv $_tmpnginx2 $_inputdb2 ed -s $_inputdb2<<\IN 1,/# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d @@ -195,14 +195,12 @@ rm $_inputdb2 # LIMITED BOTS - Create and Insert # ******************************** -LIMITEDBOTSIFS=$IFS -IFS=$'\n' -echo $_start3 >> $_tmpnginx3 -for line in $(cat $_input3); do -printf "\t\"~${line}\"\t\t$ACTION3\n" >> $_tmpnginx3 -done -echo $_end3 >> $_tmpnginx3 -IFS=$LIMITEDBOTSIFS +printf '%s\n' "$_start3" >> $_tmpnginx3 +while IFS= read -r LINE +do +printf '\t"~%s"\t\t%s\n' "${LINE}" "$_action3" >> "$_tmpnginx3" +done < $_input3 +printf '%s\n' "$_end3" >> $_tmpnginx3 mv $_tmpnginx3 $_inputdb3 ed -s $_inputdb3<<\IN 1,/# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d @@ -221,14 +219,12 @@ rm $_inputdb3 # BAD BOTS - Create and Insert # **************************** -BADBOTSIFS=$IFS -IFS=$'\n' -echo $_start4 >> $_tmpnginx4 -for line in $(cat $_input4); do -printf "\t\"~*${line}\"\t\t$ACTION4\n" >> $_tmpnginx4 -done -echo $_end4 >> $_tmpnginx4 -IFS=$BADBOTSIFS +printf '%s\n' "$_start4" >> $_tmpnginx4 +while IFS= read -r LINE +do +printf '\t"~%s"\t\t%s\n' "${LINE}" "$_action4" >> "$_tmpnginx4" +done < $_input4 +printf '%s\n' "$_end4" >> $_tmpnginx4 mv $_tmpnginx4 $_inputdb4 ed -s $_inputdb4<<\IN 1,/# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d @@ -247,23 +243,21 @@ rm $_inputdb4 # BAD REFERERS - Create and Insert # ******************************** -BADREFERER=$IFS -IFS=$'\n' -echo $_start5 >> $_tmpnginx5 -for line in $(cat $_input5); do -printf "\t\"~*${line}\"\t\t$ACTION2\n" >> $_tmpnginx5 -done -echo $_end5 >> $_tmpnginx5 -IFS=$BADREFERER +printf '%s\n' "$_start5" >> $_tmpnginx5 +while IFS= read -r LINE +do +printf '\t"~*%s"\t\t%s\n' "${LINE}" "$_action2" >> "$_tmpnginx5" +done < $_input5 +printf '%s\n' "$_end5" >> $_tmpnginx5 mv $_tmpnginx5 $_inputdb5 ed -s $_inputdb5<<\IN -1,/# START BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/d -/# END BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/,$d +1,/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/d +/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/,$d ,d .r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template -/# START BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/x +/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/x .t. -.,/# END BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/-d +.,/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/-d w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template q IN @@ -273,14 +267,12 @@ rm $_inputdb5 # GOOGLE IP RANGES - Create and Insert # ************************************ -GOOGLE=$IFS -IFS=$'\n' -echo $_start6 >> $_tmpnginx6 -for line in $(cat $_input6); do -printf "\t${line}\t\t$ACTION1\n" >> $_tmpnginx6 -done -echo $_end6 >> $_tmpnginx6 -IFS=$GOOGLE +printf '%s\n' "$_start6" >> $_tmpnginx6 +while IFS= read -r LINE +do +printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx6" +done < $_input6 +printf '%s\n' "$_end6" >> $_tmpnginx6 mv $_tmpnginx6 $_inputdb6 ed -s $_inputdb6<<\IN 1,/# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d @@ -299,14 +291,12 @@ rm $_inputdb6 # BING IP RANGES - Create and Insert # ********************************** -BING=$IFS -IFS=$'\n' -echo $_start7 >> $_tmpnginx7 -for line in $(cat $_input7); do -printf "\t${line}\t\t$ACTION1\n" >> $_tmpnginx7 -done -echo $_end7 >> $_tmpnginx7 -IFS=$BING +printf '%s\n' "$_start7" >> $_tmpnginx7 +while IFS= read -r LINE +do +printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx7" +done < $_input7 +printf '%s\n' "$_end7" >> $_tmpnginx7 mv $_tmpnginx7 $_inputdb7 ed -s $_inputdb7<<\IN 1,/# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d @@ -325,14 +315,12 @@ rm $_inputdb7 # Wordpress Theme Detectors - Create and Insert # ********************************************* -WPTHEME=$IFS -IFS=$'\n' -echo $_start8 >> $_tmpnginx8 -for line in $(cat $_input8); do -printf "\t${line}\n" >> $_tmpnginx8 -done -echo $_end8 >> $_tmpnginx8 -IFS=$WPTHEME +printf '%s\n' "$_start8" >> $_tmpnginx8 +while IFS= read -r LINE +do +printf '%s\n' "${LINE}" >> "$_tmpnginx8" +done < $_input8 +printf '%s\n' "$_end8" >> $_tmpnginx8 mv $_tmpnginx8 $_inputdb8 ed -s $_inputdb8<<\IN 1,/# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/d @@ -351,14 +339,12 @@ rm $_inputdb8 # Nibbler SEO - Create and Insert # ******************************* -NIBBLER=$IFS -IFS=$'\n' -echo $_start9 >> $_tmpnginx9 -for line in $(cat $_input9); do -printf "\t${line}\t\t$ACTION2\n" >> $_tmpnginx9 -done -echo $_end9 >> $_tmpnginx9 -IFS=$NIBBLER +printf '%s\n' "$_start9" >> $_tmpnginx9 +while IFS= read -r LINE +do +printf '\t%s\t\t%s\n' "${LINE}" "$_action2" >> "$_tmpnginx9" +done < $_input9 +printf '%s\n' "$_end9" >> $_tmpnginx9 mv $_tmpnginx9 $_inputdb9 ed -s $_inputdb9<<\IN 1,/# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/d @@ -377,14 +363,12 @@ rm $_inputdb9 # CLOUDFLARE IP RANGES - Create and Insert # **************************************** -CLOUDFLARE=$IFS -IFS=$'\n' -echo $_start10 >> $_tmpnginx10 -for line in $(cat $_input10); do -printf "\t${line}\t\t$ACTION1\n" >> $_tmpnginx10 -done -echo $_end10 >> $_tmpnginx10 -IFS=$CLOUDFLARE +printf '%s\n' "$_start10" >> $_tmpnginx10 +while IFS= read -r LINE +do +printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx10" +done < $_input10 +printf '%s\n' "$_end10" >> $_tmpnginx10 mv $_tmpnginx10 $_inputdb10 ed -s $_inputdb10<<\IN 1,/# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d @@ -404,23 +388,16 @@ rm $_inputdb10 # PRINT VERSION, SCRIPT RUNTIME and UPDATE INFORMATION INTO GLOBALBLACKLIST FILES # ******************************************************************************* -LASTUPDATEIFS=$IFS -IFS=$'\n' -now="$(date)" -end=$(date +%s.%N) -echo $_startmarker >> $_tmpnginxA -printf "###################################################\n### Version: "$MY_GIT_TAG"\n### Updated: "$now"\n### Bad Referrer Count: "$BAD_REFERRERS"\n### Bad Bot Count: "$BAD_BOTS"\n###################################################\n" >> $_tmpnginxA -echo $_endmarker >> $_tmpnginxA -IFS=$LASTUPDATEIFS +printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Referrer Count: " "$BAD_REFERRERS" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" >> $_tmpnginxA mv $_tmpnginxA $_inputdbA ed -s $_inputdbA<<\IN -1,/### Version Information #/d -/### Version Information ##/,$d +1,/### VERSION INFORMATION #/d +/### VERSION INFORMATION ##/,$d ,d .r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template -/### Version Information #/x +/### VERSION INFORMATION #/x .t. -.,/### Version Information ##/-d +.,/### VERSION INFORMATION ##/-d #,p #,p used to print output replaced with w below to write w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template @@ -432,18 +409,8 @@ rm $_inputdbA # Generate Additional Files and Copy Them to Folders # ************************************************** -sudo cp $_input1 $TRAVIS_BUILD_DIR/_generator_lists/good-user-agents.list -sudo cp $_input2 $TRAVIS_BUILD_DIR/_generator_lists/allowed-user-agents.list -sudo cp $_input3 $TRAVIS_BUILD_DIR/_generator_lists/limited-user-agents.list -sudo cp $_input4 $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list -sudo cp $_input5 $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list -sudo cp $_input6 $TRAVIS_BUILD_DIR/_generator_lists/google-ip-ranges.list -sudo cp $_input7 $TRAVIS_BUILD_DIR/_generator_lists/bing-ip-ranges.list -sudo cp $_input8 $TRAVIS_BUILD_DIR/_generator_lists/wordpress-theme-detectors.list -sudo cp $_input9 $TRAVIS_BUILD_DIR/_generator_lists/nibbler-seo.list -sudo cp $_input10 $TRAVIS_BUILD_DIR/_generator_lists/cloudflare-ip-ranges.list sudo cp $_nginx $TRAVIS_BUILD_DIR/conf.d/globalblacklist.conf -sudo cp $_nginx $TRAVIS_BUILD_DIR/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +sudo cp $_nginx $TRAVIS_BUILD_DIR/_sample_config_files/Engintron_for_CPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf exit 0 diff --git a/travisCI/generate-google-disavow.sh b/travisCI/generate-google-disavow.sh index aa2dbea72..7306e97b0 100755 --- a/travisCI/generate-google-disavow.sh +++ b/travisCI/generate-google-disavow.sh @@ -38,9 +38,10 @@ _input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list # ************************** sudo truncate -s 0 $TRAVIS_BUILD_DIR/google-disavow.txt -for line in $(cat $_input1); do -printf "domain:${line}\n" >> $TRAVIS_BUILD_DIR/google-disavow.txt -done +while IFS= read -r LINE +do +printf '%s%s\n' "domain:" "${LINE}" >> $TRAVIS_BUILD_DIR/google-disavow.txt +done < $_input1 exit 0 diff --git a/travisCI/generate-robots.sh b/travisCI/generate-robots.sh index 19c278eeb..584fda0d4 100755 --- a/travisCI/generate-robots.sh +++ b/travisCI/generate-robots.sh @@ -33,9 +33,7 @@ # Set Input Files # *************** -_input4a=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list -_robotsinput1=/tmp/robotsinput1.txt -_robotsinput1b=/tmp/robotsinput1b.txt +_input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list _tmprobots=/tmp/robots.txt # ****************** @@ -47,7 +45,7 @@ MONTH=$(date +"%m") MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list) BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list) -now="$(date)" +_now="$(date)" # ************************* # Set Start and End Markers @@ -61,17 +59,12 @@ _endmarker="### Version Information ##" # Create the robots.txt file # ************************** -cp $_input4a $_robotsinput1 -sed 's/[\]//g' $_robotsinput1 > $_robotsinput1b -IFS='' -echo $_startmarker >> $_tmprobots -printf "###################################################\n### Version: "$MY_GIT_TAG"\n### Updated: "$now"\n### Bad Referrer Count: "$BAD_REFERRERS"\n### Bad Bot Count: "$BAD_BOTS"\n###################################################\n" >> $_tmprobots -echo $_endmarker >> $_tmprobots -printf "\n\n" >> $_tmprobots -cat $_robotsinput1b | -while read line; do -printf 'User-agent: '${line}'\n Disallow:/ \n' >> $_tmprobots -done +printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n\n%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" "User-agent: *" "Disallow: /wp-admin/" "Allow: /wp-admin/admin-ajax.php" >> "$_tmprobots" +while IFS= read -r LINE +do +printf 'User-agent: %s\n%s\n' "${LINE}" "Disallow:/" >> $_tmprobots +done < $_input1 +printf '\n' >> $_tmprobots sudo cp $_tmprobots $TRAVIS_BUILD_DIR/robots.txt/robots.txt exit 0 diff --git a/travisCI/globalblacklist.template b/travisCI/globalblacklist.template index 615eda5f0..72fc7be80 100755 --- a/travisCI/globalblacklist.template +++ b/travisCI/globalblacklist.template @@ -2,30 +2,31 @@ ### THE ULTIMATE NGINX BAD BOT BLOCKER ### ********************************** -### Version Information # +### VERSION INFORMATION # ################################################### ### Version: V3.2017.06.631 ### Updated: Wed Jun 28 16:42:57 SAST 2017 ### Bad Referrer Count: 4947 ### Bad Bot Count: 480 ################################################### -### Version Information ## +### VERSION INFORMATION # ### This file implements a checklist / blacklist for good user agents, bad user agents and -### bad referrers. It also has whitelisting for your own IP's and known good IP Ranges +### bad referrers on Nginx Web Server. It also has whitelisting for your own IP's and known good IP Ranges ### and also has rate limiting functionality for bad bots who you only want to rate limit -### and not actually block out entirely. It is powerful and also flexible. +### and not actually block out entirely. It is very powerful and also very flexible. ### Created By: https://github.com/mitchellkrogza/ ### Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker ### Copyright Mitchell Krog - +### Contributors: Stuart Cardall - https://github.com/itoffshore ### Tested on: nginx/1.10.3 (Ubuntu 16.04) ### This list was developed and is in use on a live Nginx server running some very busy web sites. ### It was built from the ground up using real data from daily logs and is updated almost daily. ### It has been extensively tested for false positives and all additions to the lists of bad user agents, -### spam referers, rogue IP address, scanners, scrapers and domain hijacking sites are extensively checked +### spam referrers, rogue IP address, scanners, scrapers and domain hijacking sites are extensively checked ### before they are added. It is monitored extensively for any false positives. ### ********* @@ -35,39 +36,24 @@ ### Alphabetically ordered lists for Ease of Maintenance. ### Extensive Commenting for Ease of Reference. ### Extensive bad_bot list -### Extensive bad_referer list (please excuse the nasty words and domains) +### Extensive bad_referrer list (please excuse the nasty words and domains) ### Simple regex patterns versus complicated messy regex patterns. ### Checks regardless of http / https urls or the lack of any protocol sent. ### IP range blocking / whitelisting. ### Rate Limiting Functions. -### *** PLEASE READ ALL INLINE NOTES ON TESTING !!!! +### ************ +### INSTALLATION +### ************ -### I have this set up as an include in nginx.conf as -### Include /etc/nginx/conf.d/globalblacklist.conf -### This is loaded and available for any vhost to use in its config -### Each vhost then just needs the include file mentioned below for it to take effect. +### PLEASE use the install, setup and update scripts provided for you to ease your installation. +### This Auto Installation procedure is documented in the README.md and AUTO-CONFIGURATION.md files. +### Installation, Setup and Update Scripts Contributed by Stuart Cardall - https://github.com/itoffshore +### There are also manual configuration instructions provided for those not wishing to do an auto install. -### In Most cases your nginx.conf should already have an include statement as follows -### Include /etc/nginx/conf.d/* -### If that is the case then you can ignore the above include statement as Nginx will -### load anything in the conf.d folder and make it available to all sites. - -### All you then need to do is use the include statements below in the server {} block of a vhost file for it to take effect. -# server { -# #Config stuff here -# include /etc/nginx/bots.d/blockbots.conf -# include /etc/nginx/bots.d/ddos.conf -# #Other config stuff here -# } - -### Need I say, please don't just copy and paste this without reviewing what bots and -### referers are being blocked, you may want to exclude certain of them -### Also make SURE to whitelist your own IP's in the geo $bad_referer section. -### Know why you are using this or why you want to use it before you do, the implications -### are quite severe. - -### *** PLEASE READ INLINE NOTES ON TESTING !!!! +### *********************************************** +### !!!!! PLEASE READ INLINE NOTES ON TESTING !!!!! +### *********************************************** ### Note that: ### 0 = allowed - no limits @@ -75,45 +61,36 @@ ### 2 = rate limited more ### 3 = block completely -### NEED I say do a "sudo nginx -t" to test the config is okay after adding these -### and if so then "sudo service nginx reload" for it to take effect. - -### *** MAKE SURE TO ADD to your nginx.conf *** -### server_names_hash_bucket_size 64; -### server_names_hash_max_size 4096; -### limit_req_zone $binary_remote_addr zone=flood:50m rate=90r/s; -### limit_conn_zone $binary_remote_addr zone=addr:50m; -### to allow it to load this large set of domains into memory and to set the rate limiting zones for the DDOS filter. - -### ADDING YOUR OWN BAD REFERERS -### Fork your own local copy and then -### Send a Pull Request by following the instructions in the Pull_Requests_Here_Please folder. +### ************************************************************ +### CONTRIBUTING / PULL REQUESTS / ADDING YOUR OWN BAD REFERRERS +### ************************************************************ +### For contributing, corrections or adding bots or referrers to this repo, +### Send a Pull Request (PR) on any of the .list files in the _generator_lists folder +### All Pull Requests will be checked for accuracy before being merged. # ********************************* # FIRST BLOCK BY USER-AGENT STRINGS # ********************************* -# *************** -# PLEASE TEST !!! -# *************** +# *********************** +# !!!!! PLEASE TEST !!!!! +# *********************** # ALWAYS test any User-Agent Strings you add here to make sure you have it right # Use a Chrome Extension called "User-Agent Switcher for Chrome" where you can create your # own custom lists of User-Agents and test them easily against your rules below. -# You can also use Curl to test user-agents as per example below -# curl -I http://www.yourdomain.com -A "GoogleBot" << 200 OK -# curl -I http://www.yourdomain.com -A "80legs" <<< 444 Dropped Connection +# You can also use curl from the command line to test user-agents as per the examples below: -# Here we also allow specific User Agents to come through that we want to allow +# curl -I http://www.yourdomain.com -A "GoogleBot" ---- GIVES YOU: HTTP/1.1 200 OK (Meaning web page was served to Client) +# curl -I http://www.yourdomain.com -A "80legs" ---- GIVES YOU: curl: (52) Empty reply from server (Meaning Nginx gave a 444 Dropped Connection) -# PLEASE NOTE: In all lists below I use Nginx case-insensitive matching ~* -# This means regardless of how you type the word, upper or lowercase or mixed it will -# be detected by Nginx Regex. Some Names are Capitalised simply for Ease of Reading. -# Especially important for both Googlebot and googlebot to be allowed through no? +# In this section we allow/disallow specific User Agents / Bots. +# ********************************************************************* # Now we map all good and bad user agents to a variable called $bad_bot +# ********************************************************************* map $http_user_agent $bad_bot { @@ -122,12 +99,18 @@ map $http_user_agent $bad_bot { # *********************************************** # Include your Own Custom List of Bad User Agents # *********************************************** -# use the include file below to further customize your own list of additional -# user-agents you wish to permanently block -# START BLACKLISTED USER AGENTS ### DO NOT EDIT THIS LINE AT ALL ### +# Use the include file below to further customize your own list of additional user-agents you wish to permanently block + +# *********************************************************************************** +# START CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# *********************************************************************************** + include /etc/nginx/bots.d/blacklist-user-agents.conf; -# END BLACKLISTED USER AGENTS ### DO NOT EDIT THIS LINE AT ALL ### + +# ********************************************************************************* +# END CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ********************************************************************************* # *********************************************** @@ -156,9 +139,11 @@ map $http_user_agent $bad_bot { "~Kraken/0.1" 0; "~LinkedInBot" 0; "~Mediapartners-Google" 0; + "~Mozilla/5.0" 0; "~msnbot" 0; "~msnbot-media" 0; "~SAMSUNG" 0; + "~SAMSUNG-SGH-E250" 0; "~slurp" 0; "~teoma" 0; "~TwitterBot" 0; @@ -166,9 +151,10 @@ map $http_user_agent $bad_bot { "~yahoo" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### -# ************************************************** -# User-Agent Strings Allowed Throug but Rate Limited -# ************************************************** +# *************************************************** +# User-Agent Strings Allowed Through but Rate Limited +# *************************************************** + # Some people block libwww-perl, it us widely used in many valid (non rogue) agents # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited @@ -208,543 +194,527 @@ map $http_user_agent $bad_bot { # ********************************************* # Bad User-Agent Strings That We Block Outright # ********************************************* + # This includes: # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*360Spider" 3; - "~*80legs" 3; - "~*Abonti" 3; - "~*Aboundex" 3; - "~*Acunetix" 3; - "~*ADmantX" 3; - "~*AhrefsBot" 3; - "~*AIBOT" 3; - "~*AiHitBot" 3; - "~*Aipbot" 3; - "~*Alexibot" 3; - "~*Alligator" 3; - "~*AllSubmitter" 3; - "~*Anarchie" 3; - "~*Apexoo" 3; - "~*ASPSeek" 3; - "~*Asterias" 3; - "~*Attach" 3; - "~*autoemailspider" 3; - "~*BackDoorBot" 3; - "~*BackStreet" 3; - "~*BackWeb" 3; - "~*Badass" 3; - "~*Bandit" 3; - "~*BatchFTP" 3; - "~*Battleztar\ Bazinga" 3; - "~*BBBike" 3; - "~*BDFetch" 3; - "~*BetaBot" 3; - "~*Bigfoot" 3; - "~*Bitacle" 3; - "~*Blackboard" 3; - "~*Black\ Hole" 3; - "~*BlackWidow" 3; - "~*BLEXBot" 3; - "~*Blow" 3; - "~*BlowFish" 3; - "~*Boardreader" 3; - "~*Bolt" 3; - "~*BotALot" 3; - "~*Brandprotect" 3; - "~*Brandwatch" 3; - "~*Bubing" 3; - "~*Buddy" 3; - "~*BuiltBotTough" 3; - "~*BuiltWith" 3; - "~*Bullseye" 3; - "~*BunnySlippers" 3; - "~*BuzzSumo" 3; - "~*Calculon" 3; - "~*CATExplorador" 3; - "~*CazoodleBot" 3; - "~*CCBot" 3; - "~*Cegbfeieh" 3; - "~*CheeseBot" 3; - "~*CherryPicker" 3; - "~*ChinaClaw" 3; - "~*Chlooe" 3; - "~*Claritybot" 3; - "~*Cliqzbot" 3; - "~*Cogentbot" 3; - "~*Collector" 3; - "~*Copier" 3; - "~*CopyRightCheck" 3; - "~*Copyscape" 3; - "~*Cosmos" 3; - "~*Craftbot" 3; - "~*CrazyWebCrawler" 3; - "~*Crescent" 3; - "~*CSHttp" 3; - "~*Curious" 3; - "~*Custo" 3; - "~*DatabaseDriverMysqli" 3; - "~*DBLBot" 3; - "~*Demon" 3; - "~*Deusu" 3; - "~*Devil" 3; - "~*DIIbot" 3; - "~*Dirbuster" 3; - "~*Disco" 3; - "~*Discobot" 3; - "~*Discoverybot" 3; - "~*DittoSpyder" 3; - "~*DomainAppender" 3; - "~*DomainCrawler" 3; - "~*DomainSigmaCrawler" 3; - "~*Dotbot" 3; - "~*Download\ Demon" 3; - "~*Download\ Devil" 3; - "~*Download\ Wonder" 3; - "~*Dragonfly" 3; - "~*Drip" 3; - "~*DTS\ Agent" 3; - "~*EasyDL" 3; - "~*Ebingbong" 3; - "~*eCatch" 3; - "~*ECCP/1.0" 3; - "~*Ecxi" 3; - "~*EirGrabber" 3; - "~*EMail\ Collector" 3; - "~*EMail\ Extractor" 3; - "~*EMail\ Siphon" 3; - "~*EMail\ Wolf" 3; - "~*EroCrawler" 3; - "~*Evil" 3; - "~*Exabot" 3; - "~*Express\ WebPictures" 3; - "~*Extractor" 3; - "~*ExtractorPro" 3; - "~*EyeNetIE" 3; - "~*Ezooms" 3; - "~*FHscan" 3; - "~*Fimap" 3; - "~*Findxbot" 3; - "~*Firefox/7.0" 3; - "~*FlashGet" 3; - "~*Flunky" 3; - "~*Foobot" 3; - "~*Freeuploader" 3; - "~*FrontPage" 3; - "~*Fyrebot" 3; - "~*GalaxyBot" 3; - "~*Genieo" 3; - "~*Getintent" 3; - "~*GetRight" 3; - "~*GetWeb" 3; - "~*Gigablast" 3; - "~*Gigabot" 3; - "~*Go-Ahead-Got-It" 3; - "~*Gotit" 3; - "~*GoZilla" 3; - "~*Go!Zilla" 3; - "~*Grabber" 3; - "~*GrabNet" 3; - "~*Grafula" 3; - "~*GrapeFX" 3; - "~*GrapeshotCrawler" 3; - "~*GridBot" 3; - "~*GT::WWW" 3; - "~*GuzzleHttp" 3; - "~*HaosouSpider" 3; - "~*Harvest" 3; - "~*Havij" 3; - "~*HEADMasterSEO" 3; - "~*Heritrix" 3; - "~*Hloader" 3; - "~*HMView" 3; - "~*HTMLparser" 3; - "~*HTTP::Lite" 3; - "~*HTTrack" 3; - "~*Humanlinks" 3; - "~*HybridBot" 3; - "~*Iblog" 3; - "~*IDBot" 3; - "~*Id-search" 3; - "~*IlseBot" 3; - "~*Image\ Fetch" 3; - "~*Image\ Stripper" 3; - "~*Image\ Sucker" 3; - "~*Indy\ Library" 3; - "~*InfoNaviRobot" 3; - "~*InfoTekies" 3; - "~*Intelliseek" 3; - "~*InterGET" 3; - "~*Internet\ Ninja" 3; - "~*InternetSeer" 3; - "~*internetVista\ monitor" 3; - "~*Iria" 3; - "~*IRLbot" 3; - "~*Iskanie" 3; - "~*JamesBOT" 3; - "~*Jbrofuzz" 3; - "~*JennyBot" 3; - "~*JetCar" 3; - "~*JikeSpider" 3; - "~*JOC\ Web\ Spider" 3; - "~*Joomla" 3; - "~*JustView" 3; - "~*Jyxobot" 3; - "~*Kenjin\ Spider" 3; - "~*Keyword\ Density" 3; - "~*Lanshanbot" 3; - "~*Larbin" 3; - "~*LeechFTP" 3; - "~*LeechGet" 3; - "~*LexiBot" 3; - "~*Lftp" 3; - "~*LibWeb" 3; - "~*Libwhisker" 3; - "~*Lightspeedsystems" 3; - "~*Likse" 3; - "~*Linkdexbot" 3; - "~*LinkextractorPro" 3; - "~*LinkpadBot" 3; - "~*LinkScan" 3; - "~*LinksManager" 3; - "~*LinkWalker" 3; - "~*LinqiaMetadataDownloaderBot" 3; - "~*LinqiaRSSBot" 3; - "~*LinqiaScrapeBot" 3; - "~*Lipperhey" 3; - "~*Litemage_walker" 3; - "~*Lmspider" 3; - "~*LNSpiderguy" 3; - "~*Ltx71" 3; - "~*lwp-request" 3; - "~*LWP::Simple" 3; - "~*lwp-trivial" 3; - "~*Magnet" 3; - "~*Mag-Net" 3; - "~*magpie-crawler" 3; - "~*Mail.ru" 3; - "~*Majestic12" 3; - "~*MarkMonitor" 3; - "~*MarkWatch" 3; - "~*Masscan" 3; - "~*Mass\ Downloader" 3; - "~*Mata\ Hari" 3; - "~*Meanpathbot" 3; - "~*MegaIndex.ru" 3; - "~*Metauri" 3; - "~*MFC_Tear_Sample" 3; - "~*Microsoft\ Data\ Access" 3; - "~*Microsoft\ URL\ Control" 3; - "~*MIDown\ tool" 3; - "~*MIIxpc" 3; - "~*Mister\ PiX" 3; - "~*MJ12bot" 3; - "~*Mojeek" 3; - "~*MSFrontPage" 3; - "~*MSIE\ 6.0" 3; - "~*MSIECrawler" 3; - "~*Msrabot" 3; - "~*MS\ Web\ Services\ Client\ Protocol" 3; - "~*Musobot" 3; - "~*Name\ Intelligence" 3; - "~*Nameprotect" 3; - "~*Navroad" 3; - "~*NearSite" 3; - "~*Needle" 3; - "~*Nessus" 3; - "~*NetAnts" 3; - "~*Netcraft" 3; - "~*netEstate\ NE\ Crawler" 3; - "~*NetLyzer" 3; - "~*NetMechanic" 3; - "~*NetSpider" 3; - "~*Nettrack" 3; - "~*Net\ Vampire" 3; - "~*NetZIP" 3; - "~*NextGenSearchBot" 3; - "~*Nibbler" 3; - "~*NICErsPRO" 3; - "~*Niki-bot" 3; - "~*Nikto" 3; - "~*NimbleCrawler" 3; - "~*Ninja" 3; - "~*Nmap" 3; - "~*NPbot" 3; - "~*Nutch" 3; - "~*Octopus" 3; - "~*Offline\ Explorer" 3; - "~*Offline\ Navigator" 3; - "~*Openfind" 3; - "~*OpenLinkProfiler" 3; - "~*Openvas" 3; - "~*OrangeBot" 3; - "~*OrangeSpider" 3; - "~*OutfoxBot" 3; - "~*PageAnalyzer" 3; - "~*Page\ Analyzer" 3; - "~*PageGrabber" 3; - "~*Page\ Grabber" 3; - "~*page\ scorer" 3; - "~*PageScorer" 3; - "~*Panscient" 3; - "~*Papa\ Foto" 3; - "~*Pavuk" 3; - "~*pcBrowser" 3; - "~*PECL::HTTP" 3; - "~*PeoplePal" 3; - "~*PHPCrawl" 3; - "~*Picscout" 3; - "~*Picsearch" 3; - "~*Pimonster" 3; - "~*Pi-Monster" 3; - "~*Pixray" 3; - "~*PleaseCrawl" 3; - "~*Pockey" 3; - "~*POE-Component-Client-HTTP" 3; - "~*Probethenet" 3; - "~*ProPowerBot" 3; - "~*ProWebWalker" 3; - "~*Proximic" 3; - "~*Psbot" 3; - "~*Pump" 3; - "~*PyCurl" 3; - "~*QueryN\ Metasearch" 3; - "~*Qwantify" 3; - "~*RankActiveLinkBot" 3; - "~*Rankivabot" 3; - "~*RealDownload" 3; - "~*Reaper" 3; - "~*Recorder" 3; - "~*RedesScrapy" 3; - "~*ReGet" 3; - "~*RepoMonkey" 3; - "~*Ripper" 3; - "~*RocketCrawler" 3; - "~*Rogerbot" 3; - "~*SalesIntelligent" 3; - "~*SBIder" 3; - "~*ScanAlert" 3; - "~*Scanbot" 3; - "~*Scrapy" 3; - "~*Screaming" 3; - "~*Screaming\ Frog\ SEO\ Spider" 3; - "~*ScreenerBot" 3; - "~*Searchestate" 3; - "~*SearchmetricsBot" 3; - "~*Semrush" 3; - "~*SemrushBot" 3; - "~*SEOkicks" 3; - "~*SEOkicks-Robot" 3; - "~*SEOlyticsCrawler" 3; - "~*Seomoz" 3; - "~*SEOprofiler" 3; - "~*SEOstats" 3; - "~*Siphon" 3; - "~*SISTRIX" 3; - "~*SISTRIX\ Crawler" 3; - "~*Sitebeam" 3; - "~*SiteExplorer" 3; - "~*Siteimprove" 3; - "~*SiteLockSpider" 3; - "~*SiteSnagger" 3; - "~*SiteSucker" 3; - "~*Site\ Sucker" 3; - "~*Sitevigil" 3; - "~*Slackbot-LinkExpanding" 3; - "~*SlySearch" 3; - "~*SmartDownload" 3; - "~*Snake" 3; - "~*Snapbot" 3; - "~*Snoopy" 3; - "~*SocialRankIOBot" 3; - "~*Sogou\ web\ spider" 3; - "~*Sosospider" 3; - "~*SpaceBison" 3; - "~*Spammen" 3; - "~*SpankBot" 3; - "~*Spanner" 3; - "~*Spbot" 3; - "~*Spinn3r" 3; - "~*SputnikBot" 3; - "~*Sqlmap" 3; - "~*Sqlworm" 3; - "~*Sqworm" 3; - "~*Steeler" 3; - "~*Stripper" 3; - "~*Sucker" 3; - "~*Sucuri" 3; - "~*SuperBot" 3; - "~*SuperHTTP" 3; - "~*Surfbot" 3; - "~*SurveyBot" 3; - "~*Suzuran" 3; - "~*Swiftbot" 3; - "~*Szukacz" 3; - "~*T0PHackTeam" 3; - "~*T8Abot" 3; - "~*tAkeOut" 3; - "~*Teleport" 3; - "~*TeleportPro" 3; - "~*Telesoft" 3; - "~*Telesphoreo" 3; - "~*Telesphorep" 3; - "~*The\ Intraformant" 3; - "~*TheNomad" 3; - "~*TightTwatBot" 3; - "~*Titan" 3; - "~*Toata" 3; - "~*Toweyabot" 3; - "~*Trendictionbot" 3; - "~*True_Robot" 3; - "~*Turingos" 3; - "~*TurnitinBot" 3; - "~*Turnitin\ Bot" 3; - "~*Turnitin\ Robot" 3; - "~*TwengaBot" 3; - "~*Twice" 3; - "~*Typhoeus" 3; - "~*UnisterBot" 3; - "~*URLy.Warning" 3; - "~*URLy\ Warning" 3; - "~*Vacuum" 3; - "~*Vagabondo" 3; - "~*VB\ Project" 3; - "~*VCI" 3; - "~*VeriCiteCrawler" 3; - "~*VidibleScraper" 3; - "~*VoidEYE" 3; - "~*Voil" 3; - "~*Voltron" 3; - "~*Wallpapers/3.0" 3; - "~*WallpapersHD" 3; - "~*WASALive-Bot" 3; - "~*WBSearchBot" 3; - "~*Webalta" 3; - "~*WebAuto" 3; - "~*Web\ Auto" 3; - "~*WebBandit" 3; - "~*Web\ Bandit" 3; - "~*WebCollage" 3; - "~*Web\ Collage" 3; - "~*WebCopier" 3; - "~*Web\ Copier" 3; - "~*WEBDAV" 3; - "~*WEBDAV\ Client" 3; - "~*WebEnhancer" 3; - "~*Web\ Enhancer" 3; - "~*WebFetch" 3; - "~*Web\ Fetch" 3; - "~*WebFuck" 3; - "~*Web\ Fuck" 3; - "~*WebGo\ IS" 3; - "~*WebImageCollector" 3; - "~*Web\ Image\ Collector" 3; - "~*WebLeacher" 3; - "~*WebmasterWorldForumBot" 3; - "~*webmeup-crawler" 3; - "~*WebPix" 3; - "~*Web\ Pix" 3; - "~*WebReaper" 3; - "~*Web\ Reaper" 3; - "~*WebSauger" 3; - "~*Web\ Sauger" 3; - "~*Webshag" 3; - "~*WebsiteExtractor" 3; - "~*Website\ Extractor" 3; - "~*WebsiteQuester" 3; - "~*Website\ Quester" 3; - "~*Webster" 3; - "~*WebStripper" 3; - "~*Web\ Stripper" 3; - "~*WebSucker" 3; - "~*Web\ Sucker" 3; - "~*WebWhacker" 3; - "~*Web\ Whacker" 3; - "~*WebZIP" 3; - "~*WeSEE" 3; - "~*Whack" 3; - "~*Whacker" 3; - "~*Whatweb" 3; - "~*Widow" 3; - "~*WinHTTrack" 3; - "~*WiseGuys\ Robot" 3; - "~*WISENutbot" 3; - "~*Wonderbot" 3; - "~*Woobot" 3; - "~*Wotbox" 3; - "~*Wprecon" 3; - "~*WPScan" 3; - "~*WWW-Collector-E" 3; - "~*WWW-Mechanize" 3; - "~*WWW::Mechanize" 3; - "~*WWWOFFLE" 3; - "~*x22Mozilla" 3; - "~*Xaldon_WebSpider" 3; - "~*Xaldon\ WebSpider" 3; - "~*Xenu" 3; - "~*YoudaoBot" 3; - "~*Zade" 3; - "~*Zermelo" 3; - "~*Zeus" 3; - "~*Zgrab" 3; - "~*Zitebot" 3; - "~*ZmEu" 3; - "~*ZumBot" 3; - "~*ZyBorg" 3; + "~360Spider" 3; + "~80legs" 3; + "~Abonti" 3; + "~Aboundex" 3; + "~Acunetix" 3; + "~ADmantX" 3; + "~AhrefsBot" 3; + "~AIBOT" 3; + "~AiHitBot" 3; + "~Aipbot" 3; + "~Alexibot" 3; + "~Alligator" 3; + "~AllSubmitter" 3; + "~Anarchie" 3; + "~Apexoo" 3; + "~ASPSeek" 3; + "~Asterias" 3; + "~Attach" 3; + "~autoemailspider" 3; + "~BackDoorBot" 3; + "~BackStreet" 3; + "~BackWeb" 3; + "~Badass" 3; + "~Bandit" 3; + "~BatchFTP" 3; + "~Battleztar\ Bazinga" 3; + "~BBBike" 3; + "~BDFetch" 3; + "~BetaBot" 3; + "~Bigfoot" 3; + "~Bitacle" 3; + "~Blackboard" 3; + "~Black\ Hole" 3; + "~BlackWidow" 3; + "~BLEXBot" 3; + "~Blow" 3; + "~BlowFish" 3; + "~Boardreader" 3; + "~Bolt" 3; + "~BotALot" 3; + "~Brandprotect" 3; + "~Brandwatch" 3; + "~Bubing" 3; + "~Buddy" 3; + "~BuiltBotTough" 3; + "~BuiltWith" 3; + "~Bullseye" 3; + "~BunnySlippers" 3; + "~BuzzSumo" 3; + "~Calculon" 3; + "~CATExplorador" 3; + "~CazoodleBot" 3; + "~CCBot" 3; + "~Cegbfeieh" 3; + "~CheeseBot" 3; + "~CherryPicker" 3; + "~ChinaClaw" 3; + "~Chlooe" 3; + "~Claritybot" 3; + "~Cliqzbot" 3; + "~Cogentbot" 3; + "~cognitiveseo" 3; + "~Collector" 3; + "~Copier" 3; + "~CopyRightCheck" 3; + "~Copyscape" 3; + "~Cosmos" 3; + "~Craftbot" 3; + "~CrazyWebCrawler" 3; + "~Crescent" 3; + "~CSHttp" 3; + "~Curious" 3; + "~Custo" 3; + "~DatabaseDriverMysqli" 3; + "~DBLBot" 3; + "~Demon" 3; + "~Deusu" 3; + "~Devil" 3; + "~DIIbot" 3; + "~Dirbuster" 3; + "~Disco" 3; + "~Discobot" 3; + "~Discoverybot" 3; + "~DittoSpyder" 3; + "~DomainAppender" 3; + "~DomainCrawler" 3; + "~DomainSigmaCrawler" 3; + "~Dotbot" 3; + "~Download\ Demon" 3; + "~Download\ Devil" 3; + "~Download\ Wonder" 3; + "~Dragonfly" 3; + "~Drip" 3; + "~DTS\ Agent" 3; + "~EasyDL" 3; + "~Ebingbong" 3; + "~eCatch" 3; + "~ECCP/1.0" 3; + "~Ecxi" 3; + "~EirGrabber" 3; + "~EMail\ Collector" 3; + "~EMail\ Extractor" 3; + "~EMail\ Siphon" 3; + "~EMail\ Wolf" 3; + "~EroCrawler" 3; + "~Evil" 3; + "~Exabot" 3; + "~Express\ WebPictures" 3; + "~Extractor" 3; + "~ExtractorPro" 3; + "~EyeNetIE" 3; + "~Ezooms" 3; + "~FHscan" 3; + "~Fimap" 3; + "~Findxbot" 3; + "~Firefox/7.0" 3; + "~FlashGet" 3; + "~Flunky" 3; + "~Foobot" 3; + "~Freeuploader" 3; + "~FrontPage" 3; + "~Fyrebot" 3; + "~GalaxyBot" 3; + "~Genieo" 3; + "~Getintent" 3; + "~GetRight" 3; + "~GetWeb" 3; + "~Gigablast" 3; + "~Gigabot" 3; + "~Go-Ahead-Got-It" 3; + "~Gotit" 3; + "~GoZilla" 3; + "~Go!Zilla" 3; + "~Grabber" 3; + "~GrabNet" 3; + "~Grafula" 3; + "~GrapeFX" 3; + "~GrapeshotCrawler" 3; + "~GridBot" 3; + "~GT::WWW" 3; + "~HaosouSpider" 3; + "~Harvest" 3; + "~Havij" 3; + "~HEADMasterSEO" 3; + "~Heritrix" 3; + "~Hloader" 3; + "~HMView" 3; + "~HTMLparser" 3; + "~HTTP::Lite" 3; + "~HTTrack" 3; + "~Humanlinks" 3; + "~HybridBot" 3; + "~Iblog" 3; + "~IDBot" 3; + "~Id-search" 3; + "~IlseBot" 3; + "~Image\ Fetch" 3; + "~Image\ Stripper" 3; + "~Image\ Sucker" 3; + "~Indy\ Library" 3; + "~InfoNaviRobot" 3; + "~InfoTekies" 3; + "~Intelliseek" 3; + "~InterGET" 3; + "~Internet\ Ninja" 3; + "~InternetSeer" 3; + "~internetVista\ monitor" 3; + "~Iria" 3; + "~IRLbot" 3; + "~Iskanie" 3; + "~JamesBOT" 3; + "~Jbrofuzz" 3; + "~JennyBot" 3; + "~JetCar" 3; + "~JikeSpider" 3; + "~JOC\ Web\ Spider" 3; + "~Joomla" 3; + "~JustView" 3; + "~Jyxobot" 3; + "~Kenjin\ Spider" 3; + "~Keyword\ Density" 3; + "~Lanshanbot" 3; + "~Larbin" 3; + "~LeechFTP" 3; + "~LeechGet" 3; + "~LexiBot" 3; + "~Lftp" 3; + "~LibWeb" 3; + "~Libwhisker" 3; + "~Lightspeedsystems" 3; + "~Likse" 3; + "~Linkdexbot" 3; + "~LinkextractorPro" 3; + "~LinkpadBot" 3; + "~LinkScan" 3; + "~LinksManager" 3; + "~LinkWalker" 3; + "~LinqiaMetadataDownloaderBot" 3; + "~LinqiaRSSBot" 3; + "~LinqiaScrapeBot" 3; + "~Lipperhey" 3; + "~Litemage_walker" 3; + "~Lmspider" 3; + "~LNSpiderguy" 3; + "~Ltx71" 3; + "~lwp-request" 3; + "~LWP::Simple" 3; + "~lwp-trivial" 3; + "~Magnet" 3; + "~Mag-Net" 3; + "~magpie-crawler" 3; + "~Mail.ru" 3; + "~Majestic12" 3; + "~MarkMonitor" 3; + "~MarkWatch" 3; + "~Masscan" 3; + "~Mass\ Downloader" 3; + "~Mata\ Hari" 3; + "~Meanpathbot" 3; + "~MegaIndex.ru" 3; + "~Metauri" 3; + "~MFC_Tear_Sample" 3; + "~Microsoft\ Data\ Access" 3; + "~Microsoft\ URL\ Control" 3; + "~MIDown\ tool" 3; + "~MIIxpc" 3; + "~Mister\ PiX" 3; + "~MJ12bot" 3; + "~Mojeek" 3; + "~MSFrontPage" 3; + "~MSIE\ 6.0" 3; + "~MSIECrawler" 3; + "~Msrabot" 3; + "~MS\ Web\ Services\ Client\ Protocol" 3; + "~Musobot" 3; + "~Name\ Intelligence" 3; + "~Nameprotect" 3; + "~Navroad" 3; + "~NearSite" 3; + "~Needle" 3; + "~Nessus" 3; + "~NetAnts" 3; + "~Netcraft" 3; + "~netEstate\ NE\ Crawler" 3; + "~NetLyzer" 3; + "~NetMechanic" 3; + "~NetSpider" 3; + "~Nettrack" 3; + "~Net\ Vampire" 3; + "~NetZIP" 3; + "~NextGenSearchBot" 3; + "~Nibbler" 3; + "~NICErsPRO" 3; + "~Niki-bot" 3; + "~Nikto" 3; + "~NimbleCrawler" 3; + "~Ninja" 3; + "~Nmap" 3; + "~NPbot" 3; + "~Nutch" 3; + "~Octopus" 3; + "~Offline\ Explorer" 3; + "~Offline\ Navigator" 3; + "~Openfind" 3; + "~OpenLinkProfiler" 3; + "~Openvas" 3; + "~OrangeBot" 3; + "~OrangeSpider" 3; + "~OutfoxBot" 3; + "~PageAnalyzer" 3; + "~Page\ Analyzer" 3; + "~PageGrabber" 3; + "~Page\ Grabber" 3; + "~page\ scorer" 3; + "~PageScorer" 3; + "~Panscient" 3; + "~Papa\ Foto" 3; + "~Pavuk" 3; + "~pcBrowser" 3; + "~PECL::HTTP" 3; + "~PeoplePal" 3; + "~PHPCrawl" 3; + "~Picscout" 3; + "~Picsearch" 3; + "~Pimonster" 3; + "~Pi-Monster" 3; + "~Pixray" 3; + "~PleaseCrawl" 3; + "~Pockey" 3; + "~POE-Component-Client-HTTP" 3; + "~Probethenet" 3; + "~ProPowerBot" 3; + "~ProWebWalker" 3; + "~Proximic" 3; + "~Psbot" 3; + "~Pump" 3; + "~PyCurl" 3; + "~QueryN\ Metasearch" 3; + "~Qwantify" 3; + "~RankActiveLinkBot" 3; + "~Rankivabot" 3; + "~RealDownload" 3; + "~Reaper" 3; + "~Recorder" 3; + "~RedesScrapy" 3; + "~ReGet" 3; + "~RepoMonkey" 3; + "~Ripper" 3; + "~RocketCrawler" 3; + "~Rogerbot" 3; + "~SalesIntelligent" 3; + "~SBIder" 3; + "~ScanAlert" 3; + "~Scanbot" 3; + "~Scrapy" 3; + "~Screaming" 3; + "~Screaming\ Frog\ SEO\ Spider" 3; + "~ScreenerBot" 3; + "~Searchestate" 3; + "~SearchmetricsBot" 3; + "~Semrush" 3; + "~SemrushBot" 3; + "~SEOkicks" 3; + "~SEOkicks-Robot" 3; + "~SEOlyticsCrawler" 3; + "~Seomoz" 3; + "~SEOprofiler" 3; + "~seoscanners" 3; + "~SEOstats" 3; + "~Siphon" 3; + "~SISTRIX" 3; + "~SISTRIX\ Crawler" 3; + "~Sitebeam" 3; + "~SiteExplorer" 3; + "~Siteimprove" 3; + "~SiteLockSpider" 3; + "~SiteSnagger" 3; + "~SiteSucker" 3; + "~Site\ Sucker" 3; + "~Sitevigil" 3; + "~Slackbot-LinkExpanding" 3; + "~SlySearch" 3; + "~SmartDownload" 3; + "~Snake" 3; + "~Snapbot" 3; + "~Snoopy" 3; + "~SocialRankIOBot" 3; + "~Sogou\ web\ spider" 3; + "~Sosospider" 3; + "~SpaceBison" 3; + "~Spammen" 3; + "~SpankBot" 3; + "~Spanner" 3; + "~Spbot" 3; + "~Spinn3r" 3; + "~SputnikBot" 3; + "~Sqlmap" 3; + "~Sqlworm" 3; + "~Sqworm" 3; + "~Steeler" 3; + "~Stripper" 3; + "~Sucker" 3; + "~Sucuri" 3; + "~SuperBot" 3; + "~SuperHTTP" 3; + "~Surfbot" 3; + "~SurveyBot" 3; + "~Suzuran" 3; + "~Swiftbot" 3; + "~Szukacz" 3; + "~T0PHackTeam" 3; + "~T8Abot" 3; + "~tAkeOut" 3; + "~Teleport" 3; + "~TeleportPro" 3; + "~Telesoft" 3; + "~Telesphoreo" 3; + "~Telesphorep" 3; + "~The\ Intraformant" 3; + "~TheNomad" 3; + "~TightTwatBot" 3; + "~Titan" 3; + "~Toata" 3; + "~Toweyabot" 3; + "~Trendictionbot" 3; + "~True_Robot" 3; + "~Turingos" 3; + "~TurnitinBot" 3; + "~Turnitin\ Bot" 3; + "~Turnitin\ Robot" 3; + "~TwengaBot" 3; + "~Twice" 3; + "~Typhoeus" 3; + "~UnisterBot" 3; + "~URLy.Warning" 3; + "~URLy\ Warning" 3; + "~Vacuum" 3; + "~Vagabondo" 3; + "~VB\ Project" 3; + "~VCI" 3; + "~VeriCiteCrawler" 3; + "~VidibleScraper" 3; + "~VoidEYE" 3; + "~Voil" 3; + "~Voltron" 3; + "~Wallpapers/3.0" 3; + "~WallpapersHD" 3; + "~WASALive-Bot" 3; + "~WBSearchBot" 3; + "~Webalta" 3; + "~WebAuto" 3; + "~Web\ Auto" 3; + "~WebBandit" 3; + "~Web\ Bandit" 3; + "~WebCollage" 3; + "~Web\ Collage" 3; + "~WebCopier" 3; + "~Web\ Copier" 3; + "~WEBDAV" 3; + "~WEBDAV\ Client" 3; + "~WebEnhancer" 3; + "~Web\ Enhancer" 3; + "~WebFetch" 3; + "~Web\ Fetch" 3; + "~WebFuck" 3; + "~Web\ Fuck" 3; + "~WebGo\ IS" 3; + "~WebImageCollector" 3; + "~Web\ Image\ Collector" 3; + "~WebLeacher" 3; + "~WebmasterWorldForumBot" 3; + "~webmeup-crawler" 3; + "~WebPix" 3; + "~Web\ Pix" 3; + "~WebReaper" 3; + "~Web\ Reaper" 3; + "~WebSauger" 3; + "~Web\ Sauger" 3; + "~Webshag" 3; + "~WebsiteExtractor" 3; + "~Website\ Extractor" 3; + "~WebsiteQuester" 3; + "~Website\ Quester" 3; + "~Webster" 3; + "~WebStripper" 3; + "~Web\ Stripper" 3; + "~WebSucker" 3; + "~Web\ Sucker" 3; + "~WebWhacker" 3; + "~Web\ Whacker" 3; + "~WebZIP" 3; + "~WeSEE" 3; + "~Whack" 3; + "~Whacker" 3; + "~Whatweb" 3; + "~Widow" 3; + "~WinHTTrack" 3; + "~WiseGuys\ Robot" 3; + "~WISENutbot" 3; + "~Wonderbot" 3; + "~Woobot" 3; + "~Wotbox" 3; + "~Wprecon" 3; + "~WPScan" 3; + "~WWW-Collector-E" 3; + "~WWW-Mechanize" 3; + "~WWW::Mechanize" 3; + "~WWWOFFLE" 3; + "~x22Mozilla" 3; + "~Xaldon_WebSpider" 3; + "~Xaldon\ WebSpider" 3; + "~Xenu" 3; + "~YoudaoBot" 3; + "~Zade" 3; + "~Zermelo" 3; + "~Zeus" 3; + "~Zgrab" 3; + "~Zitebot" 3; + "~ZmEu" 3; + "~ZumBot" 3; + "~ZyBorg" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### } -# **************************************** -# SECOND BLOCK BY REFERER STRINGS AND URLS -# **************************************** - -# Add here all referrer words and URL's that are to blocked. +# ***************************************** +# SECOND BLOCK BY REFERRER STRINGS AND URLS +# ***************************************** # ***************** # PLEASE TEST !!!! # ***************** -# ALWAYS test referers that you add. This is done manually as follows +# ALWAYS test referrers that you add. This is done manually as follows -# curl -I http://www.yourdomain.com -e http://anything.adcash.com -# curl -I http://www.yourdomain.com -e http://www.goodwebsite.com/not-adcash -# curl -I http://www.yourdomain.com -e http://www.betterwebsite.com/not/adcash +# curl -I http://www.yourdomain.com -e http://anything.adcash.com --- GIVES YOU: curl: (52) Empty reply from server +# curl -I http://www.yourdomain.com -e http://www.goodwebsite.com/not-adcash --- GIVES YOU: curl: (52) Empty reply from server +# curl -I http://www.yourdomain.com -e http://www.betterwebsite.com/not/adcash --- GIVES YOU: curl: (52) Empty reply from server -# This uses curl to send the referer string to your site and you should see an immediate -# 403 Forbidden Error or No Response at all if you use the 444 error like I do. +# curl -I http://www.yourdomain.com -e http://www.google.com --- GIVES YOU: full html output of the web page +# curl -I http://www.yourdomain.com -e http://www.microsoft.com --- GIVES YOU: full html output of the web page -# Because of case-insensitive matching any combination of capitilization in the names -# will all produce a positive hit - make sure you always test thoroughly and monitor logs -# This also does NOT check for a preceding www. nor does it check for it ending in .com -# .net .org or any long string attached at the end. It also does not care if the referer -# was sent with http https or even ftp. +# Because of case-insensitive matching any combination of capitilization in the names will all produce a positive hit +# make sure you always test thoroughly and monitor logs. This section below also does NOT check for a preceding www. +# and it also does not care if the referrer request was sent with http https or even ftp. -# REAL WORLD EXAMPLE -# ******************* -# If you were a photographer like me and say took a photo of a "girl" and you then posted -# a blog showing everyone your new photo and your blog slug / permalink was -# http://www.mysite.com/blog/photo-of-girl/ -# You can go and monitor your logs and you will see lots of 444 from other pages on your -# site that have been clicked on sending that page as a referer so in the example below -# you will generate a 403 error. - -# curl --referer http://www.mysite.com/blog/photo-of-girl/ http://www.mysite.com/ - -# So please be careful with these and think carefully before you add new words. -# Remember we are trying to keep out the general riff-raff not kill your web sites. - -# ********************************************************************** -# Now we map all bad referer words below to a variable called $bad_words -# ********************************************************************** +# *********************************************************************** +# Now we map all bad referrer words below to a variable called $bad_words +# *********************************************************************** map $http_referer $bad_words { default 0; -# ************************* -# Bad Referer Word Scanning -# ************************* +# ************************** +# Bad Referrer Word Scanning +# ************************** # These are Words and Terms often found tagged onto domains or within url query strings. # Create and Customize Your Own Bad Referrer Words Here using the new Include File Method @@ -752,18 +722,27 @@ map $http_referer $bad_words { # customized list of bad referrer words are automatically now included for you # Read Comments inside bad-referrer-words.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files + # BE VERY CAREFUL using the bad-referrer-words.conf file - please read the comment and + # examples inside the include file for detailed explanations into how seriously this can + # affect your own site from serving assets or other innocent sites from accessing your site + +# ****************************************************************************** +# START CUSTOM BAD REFERRER WORDS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ****************************************************************************** -# START CUSTOM BAD REFERRER WORDS ### DO NOT EDIT THIS LINE AT ALL ### include /etc/nginx/bots.d/bad-referrer-words.conf; -# END CUSTOM BAD REFERRER WORDS ### DO NOT EDIT THIS LINE AT ALL ### + +# **************************************************************************** +# END CUSTOM BAD REFERRER WORDS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# **************************************************************************** } -# ************************ -# Bad Referer Domain Names -# ************************ +# ************************* +# Bad Referrer Domain Names +# ************************* -# Now a list of bad referer urls these domains or any combination of them ie .com .net +# Now a list of bad referrer urls these domains or any combination of them ie .com .net # will be blocked out. Doesn't matter if the protocol is http, https or even ftp # This section includes: @@ -777,72 +756,79 @@ map $http_referer $bad_words { # PLEASE TEST !!!! # ***************** -# ALWAYS test referers that you add. This is done manually as follows +# ALWAYS test referrers that you add. This is done manually as follows -# curl -I http://www.yourdomain.com -e http://8gold.com +# curl -I http://www.yourdomain.com -e http://8gold.com --- GIVES YOU: curl: (52) Empty reply from server -# This uses curl to send the referer string to your site and you should see an immediate -# 403 Forbidden Error or No Response at all if you use the 444 error like I do. +# Because of case-insensitive matching any combination of capitilization will all produce a positive hit - make sure you always test. -# Because of case-insensitive matching any combination of capitilization -# will all produce a positive hit - make sure you always test. +# For Example any of the following variations below of 8gold.com will be detected and blocked -# curl -I http://www.yourdomain.com -e http://NOT-8gold.com -# curl -I http://www.yourdomain.com -e http://this.is.not8gOlD.net -# curl -I http://www.yourdomain.com -e ftp://8gold.com -# curl -I http://www.yourdomain.com -e ftp://www.weare8gold.NET -# curl -I http://www.yourdomain.com -e https://subdomain.8gold.com -# curl -I http://www.yourdomain.com -e https://NOT8GolD.org +# curl -I http://www.yourdomain.com -e http://NOT-8gold.com --- GIVES YOU: curl: (52) Empty reply from server +# curl -I http://www.yourdomain.com -e http://this.is.not8gOlD.net --- GIVES YOU: curl: (52) Empty reply from server +# curl -I http://www.yourdomain.com -e ftp://8gold.com --- GIVES YOU: curl: (52) Empty reply from server +# curl -I http://www.yourdomain.com -e ftp://www.weare8gold.NET --- GIVES YOU: curl: (52) Empty reply from server +# curl -I http://www.yourdomain.com -e https://subdomain.8gold.com --- GIVES YOU: curl: (52) Empty reply from server +# curl -I http://www.yourdomain.com -e https://NOT8GolD.org --- GIVES YOU: curl: (52) Empty reply from server -# This works exactly like the bad referer word lists above and is very strict !!! -# I have gone for the simple stricter approach which blocks all variants for those -# who just hop out and but another domain name. - -# So if you see a bad referer from wearegoogle.com and you want to block them just add +# So if you see a bad referrer from wearegoogle.com and you want to block them just add # them as "~*wearegoogle.com" don't ever go and do something like "~*google(-|.)" you will -# kill all your SEO in a week. Rather also send a Pull Request by following the instructions -# in the Pull_Requests_Here_Please folder. +# kill all your SEO in a week. -# I also include any sites that hotlink images from my sites into the list below. -# There are hundreds of image stealing sites out there so this list WILL grow now doubt. +# To add your own custom bad referrers use the custom include file +# /etc/nginx/bots.d/custom-bad-referrers.conf +# Or send a Pull Request to add it to the global blacklist for other users. +# In the bad referrers section I also include sites that hotlink images without permission. # *********************************************************************** -# Now we map all good & bad referer urls to variable called #bad_referer +# Now we map all good & bad referrer urls to variable called #bad_referer # *********************************************************************** map $http_referer $bad_referer { hostnames; default 0; -# ************************************ -# GOOD REFERERS - Spared from Checking -# ************************************ +# ************************************* +# GOOD REFERRERS - Spared from Checking +# ************************************* # Add all your own web site domain names and server names in this section -# WHITELIST Your Own Domain Names Here using the Include File Method +# WHITELIST Your Own Domain Names Here using the new Include File Method # New Method Uses the include file below so that when pulling future updates your # whitelisted domain names are automatically now included for you. # Read Comments inside whitelist-domains.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files -# START WHITELISTED DOMAINS ### DO NOT EDIT THIS LINE AT ALL ### +# ************************************************************************ +# START WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ************************************************************************ + include /etc/nginx/bots.d/whitelist-domains.conf; -# END WHITELISTED DOMAINS ### DO NOT EDIT THIS LINE AT ALL ### -# ******************************************* -# CUSTOM BAD REFERERS - Add your Own -# ******************************************* +# ********************************************************************** +# END WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ********************************************************************** -# Add any extra bad referers in the following include file to have them +# *********************************** +# CUSTOM BAD REFERRERS - Add your Own +# *********************************** + +# Add any extra bad referrers in the following include file to have them # permanently included and blocked - avoid duplicates in your custom file -# START CUSTOM BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ### - include /etc/nginx/bots.d/custom-bad-referrers.conf; -# END CUSTOM BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ### +# ************************************************************************* +# START CUSTOM BAD REFERRERS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ************************************************************************* -# START BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ### + include /etc/nginx/bots.d/custom-bad-referrers.conf; + +# *********************************************************************** +# END CUSTOM BAD REFERRERS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# *********************************************************************** + +# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ### "~*000free.us" 1; "~*007angels.com" 1; "~*00author.com" 1; @@ -5791,8 +5777,7 @@ map $http_referer $bad_referer { "~*zx6.ru" 1; "~*zytpirwai.net" 1; "~*zzbroya.com.ua" 1; -# END BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ### - +# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ### } @@ -5822,29 +5807,21 @@ geo $validate_client { # Read Comments inside whitelist-ips.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files -# START WHITELISTED IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### +# ************************************************************************** +# START WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ************************************************************************** + include /etc/nginx/bots.d/whitelist-ips.conf; -# END WHITELISTED IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### -# *********** -# Google Bots -# *********** +# ************************************************************************ +# END WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ************************************************************************ -# For Safety Sake Google's Known BOT IP Ranges are all white listed in case you add -# anything lower down that you mistakenly picked up as a bad bot. +# **************** +# Google IP Ranges +# **************** -# UA "AdsBot-Google (+http://www.google.com/adsbot.html)" -# UA "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.3; +http://www.google.com/bot.html)" -# UA "Googlebot-Image/1.0" -# UA "Googlebot/2.1 (+http://www.google.com/bot.html)" -# UA "Googlebot/2.1 (+http://www.googlebot.com/bot.html)" -# UA "Googlebot/Test (+http://www.googlebot.com/bot.html)" -# UA "Googlebot/Test" -# UA "Mediapartners-Google/2.1 (+http://www.googlebot.com/bot.html)" -# UA "Mediapartners-Google/2.1" -# UA "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" -# UA "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)" -# UA "gsa-crawler (Enterprise; S4-E9LJ2B82FJJAA; me@mycompany.com)" +# For Safety Sake all Google's Known IP Ranges are all white listed # START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### 108.177.8.0/21 0; @@ -5871,9 +5848,11 @@ geo $validate_client { 74.125.0.0/16 0; # END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### -# ********* -# Bing Bots -# ********* +# ************** +# Bing IP Ranges +# ************** + +# For Safety Sake all Bing's Known IP Ranges are all white listed # START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### 131.253.24.0/22 0; @@ -5901,6 +5880,12 @@ geo $validate_client { 65.55.55.0/24 0; # END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### +# ******************** +# Cloudflare IP Ranges +# ******************** + +# For Safety Sake all Cloudflare's Known IP Ranges are all white listed + # START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ### 103.21.244.0/22 0; 103.22.200.0/22 0; @@ -5931,34 +5916,34 @@ geo $validate_client { # ************************* # START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ### - 104.197.51.76 1; #makeawebsitehub.com/theme-detector/ - 108.167.189.81 1; #whatpress.net - 109.73.225.87 1; #theseotools.net/wp-theme-detector - 13.68.211.181 1; #hackertarget.com - 142.4.218.201 1; #builtwith.com (bilby.builtwith.com / monty.builtwith.com / trends.builtwith.com) - 149.56.33.22 1; #freeonlineseo.org/wordpress-theme-detector - 158.69.187.171 1; #builtwith.com - 158.69.26.58 1; #www.mythemedetector.com/detector - 162.13.185.20 1; #makeawebsitehub.com/theme-detector/ - 173.237.189.235 1; #seo-tools.freedirectorywebsites.com/wordpress-theme-detector - 173.255.210.133 1; #www.scanwp.com - 185.45.14.186 1; #whoiswp.com - 192.163.217.239 1; #scanwp.net - 192.185.4.40 1; #whatwpthemeisthat.com - 192.95.29.139 1; #seotoolstation.com/wordpress-theme-detector - 192.99.17.79 1; #builtwith.com (oscar.builtwith.com) - 198.27.69.229 1; #builtwith.com (fluffy.builtwith.com) - 198.58.124.46 1; #makeawebsitehub.com/theme-detector/ - 199.241.28.124 1; #wordpressthemedetector.org - 212.71.238.108 1; #pentest-tools.com (Vulnerability Scanner) - 37.247.121.179 1; #wpthemedetector.com - 37.60.253.215 1; #wploop.com/wordpress-theme-detector-tool/ - 45.63.68.250 1; #www.cuteseotools.net/wordpress-theme-detector - 45.79.139.191 1; #wprecon.com - 50.116.84.148 1; #detectwptheme.com - 52.87.112.125 1; #whattheme.com - 66.96.183.60 1; #www.callseotools.com/wordpress-theme-detector - 89.36.223.188 1; #www.cuteseotools.net/wordpress-theme-detector + 104.197.51.76 1; #makeawebsitehub.com/theme-detector/ + 108.167.189.81 1; #whatpress.net + 109.73.225.87 1; #theseotools.net/wp-theme-detector + 13.68.211.181 1; #hackertarget.com + 142.4.218.201 1; #builtwith.com (bilby.builtwith.com / monty.builtwith.com / trends.builtwith.com) + 149.56.33.22 1; #freeonlineseo.org/wordpress-theme-detector + 158.69.187.171 1; #builtwith.com + 158.69.26.58 1; #www.mythemedetector.com/detector + 162.13.185.20 1; #makeawebsitehub.com/theme-detector/ + 173.237.189.235 1; #seo-tools.freedirectorywebsites.com/wordpress-theme-detector + 173.255.210.133 1; #www.scanwp.com + 185.45.14.186 1; #whoiswp.com + 192.163.217.239 1; #scanwp.net + 192.185.4.40 1; #whatwpthemeisthat.com + 192.95.29.139 1; #seotoolstation.com/wordpress-theme-detector + 192.99.17.79 1; #builtwith.com (oscar.builtwith.com) + 198.27.69.229 1; #builtwith.com (fluffy.builtwith.com) + 198.58.124.46 1; #makeawebsitehub.com/theme-detector/ + 199.241.28.124 1; #wordpressthemedetector.org + 212.71.238.108 1; #pentest-tools.com (Vulnerability Scanner) + 37.247.121.179 1; #wpthemedetector.com + 37.60.253.215 1; #wploop.com/wordpress-theme-detector-tool/ + 45.63.68.250 1; #www.cuteseotools.net/wordpress-theme-detector + 45.79.139.191 1; #wprecon.com + 50.116.84.148 1; #detectwptheme.com + 52.87.112.125 1; #whattheme.com + 66.96.183.60 1; #www.callseotools.com/wordpress-theme-detector + 89.36.223.188 1; #www.cuteseotools.net/wordpress-theme-detector # END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ### # **************************************** @@ -5989,24 +5974,36 @@ geo $validate_client { # Read Comments inside blacklist-ips.conf for customization tips. # Updating the main globalblacklist.conf file will not touch your custom include files -# START BLACKLISTED IPS ### DO NOT EDIT THIS LINE AT ALL ### +# ******************************************************************** +# START BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ******************************************************************** + include /etc/nginx/bots.d/blacklist-ips.conf; -# END BLACKLISTED IPS ### DO NOT EDIT THIS LINE AT ALL ### + +# ****************************************************************** +# END BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ****************************************************************** } -# Keep own IPs out of DDOS Filter -# Add your own IP addresses and ranges below to spare them from the rate -# limiting DDOS filter (one per line) -# This now automatically includes the whitelist-ips.conf file so you only -# need to edit that include file and have it include here too for you +# WHITELIST your own IPs from the DDOS Filter + # Add your own IP addresses and ranges into the custom include file whitelist-ips.conf + # to spare them from the rate limiting DDOS filter. + # This section includes the same / single whitelist-ips.conf file so you only + # need to edit that include file and have it include here for you too. geo $ratelimited { default 1; -# START WHITELISTED IP RANGES2 ### DO NOT EDIT THIS LINE AT ALL ### +# *************************************************************************** +# START WHITELISTED IP RANGES2 ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# *************************************************************************** + include /etc/nginx/bots.d/whitelist-ips.conf; -# END WHITELISTED IP RANGES2 ### DO NOT EDIT THIS LINE AT ALL ### + +# ************************************************************************* +# END WHITELISTED IP RANGES2 ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### +# ************************************************************************* } @@ -6046,3 +6043,4 @@ geo $ratelimited { ### Also check out my Ultimate Apache Bad Bot Blocker on Github ### https://github.com/mitchellkrogza/apache-ultimate-bad-bot-blocker + diff --git a/travisCI/install-nginx.sh b/travisCI/install-nginx.sh index 6be6b1727..1cd69141f 100755 --- a/travisCI/install-nginx.sh +++ b/travisCI/install-nginx.sh @@ -115,6 +115,7 @@ sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-google-disavow.sh sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-google-exclude.php sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-config-readme-files.sh sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-files-and-commit.sh +sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/run-curl-tests.sh # ***************************************************************************************** # Travis now moves into running the rest of the tests in the script: section of .travis.yml diff --git a/travisCI/modify-config-readme-files.sh b/travisCI/modify-config-readme-files.sh index 4fa52b7cd..1f13a29f8 100755 --- a/travisCI/modify-config-readme-files.sh +++ b/travisCI/modify-config-readme-files.sh @@ -48,31 +48,23 @@ _tmpnginxA=tmpnginxA # Start and End Strings to Search for to do inserts into template # *************************************************************** -_startmarker="##### Version Information #" -_endmarker="##### Version Information ##" +_startmarker="### Version Information #" +_endmarker="### Version Information ##" # **************************************** # PRINT VERSION INFORMATION INTO README.md # **************************************** -LASTUPDATEIFS=$IFS -IFS=$'\n' -now="$(date)" -end=$(date +%s.%N) -echo $_startmarker >> $_tmpnginxA -runtime=$(python -c "print(${end} - ${start})") -printf "********************************************\n#### Version: "$MY_GIT_TAG"\n#### Bad Referrer Count: "$BAD_REFERRERS"\n#### Bad Bot Count: "$BAD_BOTS"\n********************************************\n" >> $_tmpnginxA -echo $_endmarker >> $_tmpnginxA -IFS=$LASTUPDATEIFS +printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s' "$_startmarker" "********************************************" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "********************************************" "$_endmarker" >> $_tmpnginxA mv $_tmpnginxA $_inputdbA ed -s $_inputdbA<<\IN -1,/##### Version Information #/d -/##### Version Information ##/,$d +1,/### Version Information #/d +/### Version Information ##/,$d ,d .r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/README.md -/##### Version Information #/x +/### Version Information #/x .t. -.,/##### Version Information ##/-d +.,/### Version Information ##/-d #,p #,p used to print output replaced with w below to write w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/README.md @@ -84,24 +76,16 @@ rm $_inputdbA # PRINT VERSION INFORMATION INTO AUTO-CONFIGURATION.md # **************************************************** -LASTUPDATEIFS2=$IFS -IFS=$'\n' -now="$(date)" -end=$(date +%s.%N) -echo $_startmarker >> $_tmpnginxA -runtime=$(python -c "print(${end} - ${start})") -printf "********************************************\n#### Version: "$MY_GIT_TAG"\n#### Bad Referrer Count: "$BAD_REFERRERS"\n#### Bad Bot Count: "$BAD_BOTS"\n********************************************\n" >> $_tmpnginxA -echo $_endmarker >> $_tmpnginxA -IFS=$LASTUPDATEIFS2 +printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s' "$_startmarker" "********************************************" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "********************************************" "$_endmarker" >> $_tmpnginxA mv $_tmpnginxA $_inputdbA ed -s $_inputdbA<<\IN -1,/##### Version Information #/d -/##### Version Information ##/,$d +1,/### Version Information #/d +/### Version Information ##/,$d ,d .r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/AUTO-CONFIGURATION.md -/##### Version Information #/x +/### Version Information #/x .t. -.,/##### Version Information ##/-d +.,/### Version Information ##/-d #,p #,p used to print output replaced with w below to write w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/AUTO-CONFIGURATION.md @@ -113,24 +97,16 @@ rm $_inputdbA # PRINT VERSION INFORMATION INTO MANUAL-CONFIGURATION.md # ****************************************************** -LASTUPDATEIFS3=$IFS -IFS=$'\n' -now="$(date)" -end=$(date +%s.%N) -echo $_startmarker >> $_tmpnginxA -runtime=$(python -c "print(${end} - ${start})") -printf "********************************************\n#### Version: "$MY_GIT_TAG"\n#### Bad Referrer Count: "$BAD_REFERRERS"\n#### Bad Bot Count: "$BAD_BOTS"\n********************************************\n" >> $_tmpnginxA -echo $_endmarker >> $_tmpnginxA -IFS=$LASTUPDATEIFS3 +printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s' "$_startmarker" "********************************************" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "********************************************" "$_endmarker" >> $_tmpnginxA mv $_tmpnginxA $_inputdbA ed -s $_inputdbA<<\IN -1,/##### Version Information #/d -/##### Version Information ##/,$d +1,/### Version Information #/d +/### Version Information ##/,$d ,d .r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/MANUAL-CONFIGURATION.md -/##### Version Information #/x +/### Version Information #/x .t. -.,/##### Version Information ##/-d +.,/### Version Information ##/-d #,p #,p used to print output replaced with w below to write w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/MANUAL-CONFIGURATION.md diff --git a/travisCI/run-curl-tests.sh b/travisCI/run-curl-tests.sh new file mode 100755 index 000000000..c377cfb7d --- /dev/null +++ b/travisCI/run-curl-tests.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# Curl Testing Script for Nginx Ultimate Bad Bot Blocker + +# Set Location of our Curl Test Results Files + +_curltest1=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest1.txt +_curltest2=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest2.txt +_curltest3=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest3.txt +_curltest4=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest4.txt +_curltest5=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest5.txt +_curltest6=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest6.txt + + +# Function Curl Test 1 - Check for Bad Bot "80legs" +run_curltest1 () { +truncate -s 0 $_curltest1 +curl -A "80legs" http://localhost:9000/index.php 2> $_curltest1 +if grep -i '(52)' $_curltest1; then + echo 'BAD BOT DETECTED - TEST PASSED' +else + echo 'BAD BOT NOT DETECTED - TEST FAILED' + exit 1 +fi +} + +# Function Curl Test 2 - Check for Bad Bot "masscan" +run_curltest2 () { +truncate -s 0 $_curltest2 +curl -A "masscan" http://localhost:9000/index.php 2> $_curltest2 +if grep -i '(52)' $_curltest2; then + echo 'BAD BOT DETECTED - TEST PASSED' +else + echo 'BAD BOT NOT DETECTED - TEST FAILED' + exit 1 +fi +} + +# Function Curl Test 3 - Check for Bad Referrer "100dollars-seo.com" +run_curltest3 () { +truncate -s 0 $_curltest3 +curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2> $_curltest3 +if grep -i '(52)' $_curltest3; then + echo 'BAD REFERRER DETECTED - TEST PASSED' +else + echo 'BAD REFERRER NOT DETECTED - TEST FAILED' + exit 1 +fi +} + +# Function Curl Test 4 - Check for Bad Referrer "zx6.ru" +run_curltest4 () { +truncate -s 0 $_curltest4 +curl -I http://localhost:9000/index.php -e http://zx6.ru 2> $_curltest4 +if grep -i '(52)' $_curltest4; then + echo 'BAD REFERRER DETECTED - TEST PASSED' +else + echo 'BAD REFERRER NOT DETECTED - TEST FAILED' + exit 1 +fi +} + +# Function Curl Test 5 - Check for Good Bot "GoogleBot" +run_curltest5 () { +truncate -s 0 $_curltest5 +curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> $_curltest5 +if grep -i 'Welcome' $_curltest5; then + echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED' +else + echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED' + exit 1 +fi +} + +# Function Curl Test 6 - Check for Good Bot "BingBot" +run_curltest6 () { +truncate -s 0 $_curltest6 +curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> $_curltest6 +if grep -i 'Welcome' $_curltest6; then + echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED' +else + echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED' + exit 1 +fi +} + + +# Trigger our functions to run +run_curltest1 +run_curltest2 +run_curltest3 +run_curltest4 +run_curltest5 +run_curltest6 + +# If everything passed then we exit with 0 +exit 0 +