Bring New Travis Generator and Testing Scripts Online

.travis.yml

@@ -33,14 +33,7 @@ install:
 
 script:
   - travisCI/install-nginx.sh
-  - curl -vsf 'http://localhost:9000/index.php' &> /dev/stdout
+  - sudo travisCI/run-curl-tests.sh
-  - curl -A "googlebot" http://localhost:9000/index.php &> /dev/stdout
-  - curl -A "bingbot" http://localhost:9000/index.php &> /dev/stdout
-  - curl -I http://localhost:9000/index.php -e http://google.com
-  - STATUSCODE=$(curl -A "80legs" http://localhost:9000/index.php &> /dev/stderr --write-out "%{http_code}") | if test $STATUSCODE 52; then exit 0; fi
-  - STATUSCODE=$(curl -A "masscan" http://localhost:9000/index.php &> /dev/stderr --write-out "%{http_code}") | if test $STATUSCODE 52; then exit 0; fi
-  - STATUSCODE=$(curl -I http://localhost:9000/index.php -e http://100dollars-seo.com &> /dev/stderr --write-out "%{http_code}") | if test $STATUSCODE 52; then exit 0; fi
-  - STATUSCODE=$(curl -I http://localhost:9000/index.php -e http://zx6.ru &> /dev/stderr --write-out "%{http_code}") | if test $STATUSCODE 52; then exit 0; fi
   - travisCI/modify-files-and-commit.sh
 
 before_deploy:

AUTO-CONFIGURATION.md

@@ -4,13 +4,8 @@
 ##### Created by: https://github.com/mitchellkrogza
 ##### Copyright Mitchell Krog <mitchellkrog@gmail.com>
 
-##### Version Information #
+### Version Information #
-********************************************
+### Version Information ##
-#### Version: V3.2017.06.631
-#### Bad Referrer Count: 4948
-#### Bad Bot Count: 480
-********************************************
-##### Version Information ##
 
 ## Update Notification System
 

MANUAL-CONFIGURATION.md

@@ -4,13 +4,8 @@
 ##### Created by: https://github.com/mitchellkrogza
 ##### Copyright Mitchell Krog <mitchellkrog@gmail.com>
 
-##### Version Information #
+### Version Information #
-********************************************
+### Version Information ##
-#### Version: V3.2017.06.631
-#### Bad Referrer Count: 4948
-#### Bad Bot Count: 480
-********************************************
-##### Version Information ##
 
 ## Update Notification System
 

README.md

@@ -8,13 +8,8 @@
 # Nginx Bad Bot and User-Agent Blocker, Spam Referrer Blocker, Anti DDOS, Bad IP Blocker and Wordpress Theme Detector Blocker
 ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking
 
-##### Version Information #
+### Version Information #
-********************************************
+### Version Information ##
-#### Version: V3.2017.06.631
-#### Bad Referrer Count: 4948
-#### Bad Bot Count: 480
-********************************************
-##### Version Information ##
 
 ## If this helps you [why not buy me a beer](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=BKF9XT6WHATLG):beer:
 

travisCI/_curl_tests/curltest1.txt

@@ -0,0 +1,4 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+curl: (52) Empty reply from server

travisCI/_curl_tests/curltest2.txt

@@ -0,0 +1,4 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+curl: (52) Empty reply from server

travisCI/_curl_tests/curltest3.txt

@@ -0,0 +1,4 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+curl: (52) Empty reply from server

travisCI/_curl_tests/curltest4.txt

@@ -0,0 +1,4 @@
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
+curl: (52) Empty reply from server

travisCI/_curl_tests/curltest5.txt

@@ -0,0 +1,5 @@
+<?php
+
+echo "Welcome to Nginx\n";
+echo "This is the Travis Testing Environment for the Nginx Ultimate Bad Bot Blocker\n";
+echo "Visit https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n";

travisCI/_curl_tests/curltest6.txt

@@ -0,0 +1,5 @@
+<?php
+
+echo "Welcome to Nginx\n";
+echo "This is the Travis Testing Environment for the Nginx Ultimate Bad Bot Blocker\n";
+echo "Visit https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n";

travisCI/generate-blacklist.sh

@@ -36,6 +36,10 @@ MONTH=$(date +"%m")
 MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
 BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
 BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
+_now="$(date)"
+# To get DATE output into uppercase format (if needed)
+#_now=$(date | tr -s '[:lower:]'  '[:upper:]')
+
 
 # *************************************
 # Specify input lists for the generator
@@ -61,7 +65,7 @@ _inputdb1=/tmp/good-user-agents.db
 _inputdb2=/tmp/allowed-user-agents.db
 _inputdb3=/tmp/limited-user-agents.db
 _inputdb4=/tmp/bad-user-agents.db
-_inputdb5=/tmp/bad-referers.db
+_inputdb5=/tmp/bad-referrers.db
 _inputdb6=/tmp/google-ip-ranges.db
 _inputdb7=/tmp/bing-ip-ranges.db
 _inputdb8=/tmp/wordpress-theme-detectors.db
@@ -73,17 +77,17 @@ _inputdb10=/tmp/cloudflare-ip-ranges.db
 # **************************************************
 
 _nginx=$TRAVIS_BUILD_DIR/travisCI/globalblacklist.template
-_tmpnginxA=tmpnginxA
+_tmpnginxA=_tmpnginxA
-_tmpnginx1=tmpnginx1
+_tmpnginx1=_tmpnginx1
-_tmpnginx2=tmpnginx2
+_tmpnginx2=_tmpnginx2
-_tmpnginx3=tmpnginx3
+_tmpnginx3=_tmpnginx3
-_tmpnginx4=tmpnginx4
+_tmpnginx4=_tmpnginx4
-_tmpnginx5=tmpnginx5
+_tmpnginx5=_tmpnginx5
-_tmpnginx6=tmpnginx6
+_tmpnginx6=_tmpnginx6
-_tmpnginx7=tmpnginx7
+_tmpnginx7=_tmpnginx7
-_tmpnginx8=tmpnginx8
+_tmpnginx8=_tmpnginx8
-_tmpnginx9=tmpnginx9
+_tmpnginx9=_tmpnginx9
-_tmpnginx10=tmpnginx10
+_tmpnginx10=_tmpnginx10
 
 # *************************************************************
 # Sort all input lists alphabetically and remove any duplicates
@@ -112,8 +116,8 @@ _start3="# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
 _end3="# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
 _start4="# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
 _end4="# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
-_start5="# START BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###"
+_start5="# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###"
-_end5="# END BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###"
+_end5="# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###"
 _start6="# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
 _end6="# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
 _start7="# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
@@ -124,31 +128,29 @@ _start9="# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###"
 _end9="# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###"
 _start10="# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
 _end10="# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
-_startmarker="### Version Information #"
+_startmarker="### VERSION INFORMATION #"
-_endmarker="### Version Information ##"
+_endmarker="### VERSION INFORMATION ##"
 
 # **********************************
 # SET ALLOW OR DENY ACTION VARIABLES
 # **********************************
 
-ACTION1="0;"
+_action1="0;"
-ACTION2="1;"
+_action2="1;"
-ACTION3="2;"
+_action3="2;"
-ACTION4="3;"
+_action4="3;"
 
 
 # ************************************
 # GOOD USER AGENTS - Create and Insert
 # ************************************
 
-GOODBOTSIFS=$IFS
+printf '%s\n' "$_start1" >> $_tmpnginx1
-IFS=$'\n'
+while IFS= read -r LINE
-echo $_start1 >> $_tmpnginx1
+do
-for line in $(cat $_input1); do
+printf '\t"~%s"\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx1"
-printf "\t\"~${line}\"\t\t$ACTION1\n" >> $_tmpnginx1
+done < $_input1
-done
+printf '%s\n' "$_end1"  >> $_tmpnginx1
-echo $_end1  >> $_tmpnginx1
-IFS=$GOODBOTSIFS
 mv $_tmpnginx1 $_inputdb1
 ed -s $_inputdb1<<\IN
 1,/# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
@@ -169,14 +171,12 @@ rm $_inputdb1
 # ALLOWED BOTS - Create and Insert
 # ********************************
 
-ALLOWEDBOTSIFS=$IFS
+printf '%s\n' "$_start2" >> $_tmpnginx2
-IFS=$'\n'
+while IFS= read -r LINE
-echo $_start2 >> $_tmpnginx2
+do
-for line in $(cat $_input2); do
+printf '\t"~%s"\t\t%s\n' "${LINE}" "$_action2" >> "$_tmpnginx2"
-printf "\t\"~${line}\"\t\t$ACTION2\n" >> $_tmpnginx2
+done < $_input2
-done
+printf '%s\n' "$_end2"  >> $_tmpnginx2
-echo $_end2  >> $_tmpnginx2
-IFS=$ALLOWEDBOTSIFS
 mv $_tmpnginx2 $_inputdb2
 ed -s $_inputdb2<<\IN
 1,/# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
@@ -195,14 +195,12 @@ rm $_inputdb2
 # LIMITED BOTS - Create and Insert
 # ********************************
 
-LIMITEDBOTSIFS=$IFS
+printf '%s\n' "$_start3" >> $_tmpnginx3
-IFS=$'\n'
+while IFS= read -r LINE
-echo $_start3 >> $_tmpnginx3
+do
-for line in $(cat $_input3); do
+printf '\t"~%s"\t\t%s\n' "${LINE}" "$_action3" >> "$_tmpnginx3"
-printf "\t\"~${line}\"\t\t$ACTION3\n" >> $_tmpnginx3
+done < $_input3
-done
+printf '%s\n' "$_end3"  >> $_tmpnginx3
-echo $_end3  >> $_tmpnginx3
-IFS=$LIMITEDBOTSIFS
 mv $_tmpnginx3 $_inputdb3
 ed -s $_inputdb3<<\IN
 1,/# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
@@ -221,14 +219,12 @@ rm $_inputdb3
 # BAD BOTS - Create and Insert
 # ****************************
 
-BADBOTSIFS=$IFS
+printf '%s\n' "$_start4" >> $_tmpnginx4
-IFS=$'\n'
+while IFS= read -r LINE
-echo $_start4 >> $_tmpnginx4
+do
-for line in $(cat $_input4); do
+printf '\t"~%s"\t\t%s\n' "${LINE}" "$_action4" >> "$_tmpnginx4"
-printf "\t\"~*${line}\"\t\t$ACTION4\n" >> $_tmpnginx4
+done < $_input4
-done
+printf '%s\n' "$_end4"  >> $_tmpnginx4
-echo $_end4  >> $_tmpnginx4
-IFS=$BADBOTSIFS
 mv $_tmpnginx4 $_inputdb4
 ed -s $_inputdb4<<\IN
 1,/# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
@@ -247,23 +243,21 @@ rm $_inputdb4
 # BAD REFERERS - Create and Insert
 # ********************************
 
-BADREFERER=$IFS
+printf '%s\n' "$_start5" >> $_tmpnginx5
-IFS=$'\n'
+while IFS= read -r LINE
-echo $_start5 >> $_tmpnginx5
+do
-for line in $(cat $_input5); do
+printf '\t"~*%s"\t\t%s\n' "${LINE}" "$_action2" >> "$_tmpnginx5"
-printf "\t\"~*${line}\"\t\t$ACTION2\n" >> $_tmpnginx5
+done < $_input5
-done
+printf '%s\n' "$_end5"  >> $_tmpnginx5
-echo $_end5  >> $_tmpnginx5
-IFS=$BADREFERER
 mv $_tmpnginx5 $_inputdb5
 ed -s $_inputdb5<<\IN
-1,/# START BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/d
+1,/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/d
-/# END BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
+/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
 ,d
 .r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
-/# START BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/x
+/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/x
 .t.
-.,/# END BAD REFERERS ### DO NOT EDIT THIS LINE AT ALL ###/-d
+.,/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/-d
 w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
 q
 IN
@@ -273,14 +267,12 @@ rm $_inputdb5
 # GOOGLE IP RANGES - Create and Insert
 # ************************************
 
-GOOGLE=$IFS
+printf '%s\n' "$_start6" >> $_tmpnginx6
-IFS=$'\n'
+while IFS= read -r LINE
-echo $_start6 >> $_tmpnginx6
+do
-for line in $(cat $_input6); do
+printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx6"
-printf "\t${line}\t\t$ACTION1\n" >> $_tmpnginx6
+done < $_input6
-done
+printf '%s\n' "$_end6"  >> $_tmpnginx6
-echo $_end6  >> $_tmpnginx6
-IFS=$GOOGLE
 mv $_tmpnginx6 $_inputdb6
 ed -s $_inputdb6<<\IN
 1,/# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
@@ -299,14 +291,12 @@ rm $_inputdb6
 # BING IP RANGES - Create and Insert
 # **********************************
 
-BING=$IFS
+printf '%s\n' "$_start7" >> $_tmpnginx7
-IFS=$'\n'
+while IFS= read -r LINE
-echo $_start7 >> $_tmpnginx7
+do
-for line in $(cat $_input7); do
+printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx7"
-printf "\t${line}\t\t$ACTION1\n" >> $_tmpnginx7
+done < $_input7
-done
+printf '%s\n' "$_end7"  >> $_tmpnginx7
-echo $_end7  >> $_tmpnginx7
-IFS=$BING
 mv $_tmpnginx7 $_inputdb7
 ed -s $_inputdb7<<\IN
 1,/# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
@@ -325,14 +315,12 @@ rm $_inputdb7
 # Wordpress Theme Detectors - Create and Insert
 # *********************************************
 
-WPTHEME=$IFS
+printf '%s\n' "$_start8" >> $_tmpnginx8
-IFS=$'\n'
+while IFS= read -r LINE
-echo $_start8 >> $_tmpnginx8
+do
-for line in $(cat $_input8); do
+printf '%s\n' "${LINE}" >> "$_tmpnginx8"
-printf "\t${line}\n" >> $_tmpnginx8
+done < $_input8
-done
+printf '%s\n' "$_end8"  >> $_tmpnginx8
-echo $_end8  >> $_tmpnginx8
-IFS=$WPTHEME
 mv $_tmpnginx8 $_inputdb8
 ed -s $_inputdb8<<\IN
 1,/# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/d
@@ -351,14 +339,12 @@ rm $_inputdb8
 # Nibbler SEO - Create and Insert
 # *******************************
 
-NIBBLER=$IFS
+printf '%s\n' "$_start9" >> $_tmpnginx9
-IFS=$'\n'
+while IFS= read -r LINE
-echo $_start9 >> $_tmpnginx9
+do
-for line in $(cat $_input9); do
+printf '\t%s\t\t%s\n' "${LINE}" "$_action2" >> "$_tmpnginx9"
-printf "\t${line}\t\t$ACTION2\n" >> $_tmpnginx9
+done < $_input9
-done
+printf '%s\n' "$_end9"  >> $_tmpnginx9
-echo $_end9  >> $_tmpnginx9
-IFS=$NIBBLER
 mv $_tmpnginx9 $_inputdb9
 ed -s $_inputdb9<<\IN
 1,/# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/d
@@ -377,14 +363,12 @@ rm $_inputdb9
 # CLOUDFLARE IP RANGES - Create and Insert
 # ****************************************
 
-CLOUDFLARE=$IFS
+printf '%s\n' "$_start10" >> $_tmpnginx10
-IFS=$'\n'
+while IFS= read -r LINE
-echo $_start10 >> $_tmpnginx10
+do
-for line in $(cat $_input10); do
+printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx10"
-printf "\t${line}\t\t$ACTION1\n" >> $_tmpnginx10
+done < $_input10
-done
+printf '%s\n' "$_end10"  >> $_tmpnginx10
-echo $_end10  >> $_tmpnginx10
-IFS=$CLOUDFLARE
 mv $_tmpnginx10 $_inputdb10
 ed -s $_inputdb10<<\IN
 1,/# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
@@ -404,23 +388,16 @@ rm $_inputdb10
 # PRINT VERSION, SCRIPT RUNTIME and UPDATE INFORMATION INTO GLOBALBLACKLIST FILES
 # *******************************************************************************
 
-LASTUPDATEIFS=$IFS
+printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Referrer Count: " "$BAD_REFERRERS" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" >> $_tmpnginxA
-IFS=$'\n'
-now="$(date)"
-end=$(date +%s.%N)    
-echo $_startmarker >> $_tmpnginxA
-printf "###################################################\n### Version: "$MY_GIT_TAG"\n### Updated: "$now"\n### Bad Referrer Count: "$BAD_REFERRERS"\n### Bad Bot Count: "$BAD_BOTS"\n###################################################\n" >> $_tmpnginxA
-echo $_endmarker  >> $_tmpnginxA
-IFS=$LASTUPDATEIFS
 mv $_tmpnginxA $_inputdbA
 ed -s $_inputdbA<<\IN
-1,/### Version Information #/d
+1,/### VERSION INFORMATION #/d
-/### Version Information ##/,$d
+/### VERSION INFORMATION ##/,$d
 ,d
 .r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
-/### Version Information #/x
+/### VERSION INFORMATION #/x
 .t.
-.,/### Version Information ##/-d
+.,/### VERSION INFORMATION ##/-d
 #,p
 #,p used to print output replaced with w below to write
 w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
@@ -432,18 +409,8 @@ rm $_inputdbA
 # Generate Additional Files and Copy Them to Folders
 # **************************************************
 
-sudo cp $_input1 $TRAVIS_BUILD_DIR/_generator_lists/good-user-agents.list
-sudo cp $_input2 $TRAVIS_BUILD_DIR/_generator_lists/allowed-user-agents.list
-sudo cp $_input3 $TRAVIS_BUILD_DIR/_generator_lists/limited-user-agents.list
-sudo cp $_input4 $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
-sudo cp $_input5 $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list
-sudo cp $_input6 $TRAVIS_BUILD_DIR/_generator_lists/google-ip-ranges.list
-sudo cp $_input7 $TRAVIS_BUILD_DIR/_generator_lists/bing-ip-ranges.list
-sudo cp $_input8 $TRAVIS_BUILD_DIR/_generator_lists/wordpress-theme-detectors.list
-sudo cp $_input9 $TRAVIS_BUILD_DIR/_generator_lists/nibbler-seo.list
-sudo cp $_input10 $TRAVIS_BUILD_DIR/_generator_lists/cloudflare-ip-ranges.list
 sudo cp $_nginx $TRAVIS_BUILD_DIR/conf.d/globalblacklist.conf
-sudo cp $_nginx $TRAVIS_BUILD_DIR/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
+sudo cp $_nginx $TRAVIS_BUILD_DIR/_sample_config_files/Engintron_for_CPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
 
 exit 0
 

travisCI/generate-google-disavow.sh

@@ -38,9 +38,10 @@ _input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list
 # **************************
 
 sudo truncate -s 0 $TRAVIS_BUILD_DIR/google-disavow.txt
-for line in $(cat $_input1); do
+while IFS= read -r LINE
-printf "domain:${line}\n" >> $TRAVIS_BUILD_DIR/google-disavow.txt
+do
-done
+printf '%s%s\n' "domain:" "${LINE}" >> $TRAVIS_BUILD_DIR/google-disavow.txt
+done < $_input1
 
 exit 0
 

travisCI/generate-robots.sh

@@ -33,9 +33,7 @@
 # Set Input Files
 # ***************
 
-_input4a=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
+_input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
-_robotsinput1=/tmp/robotsinput1.txt
-_robotsinput1b=/tmp/robotsinput1b.txt
 _tmprobots=/tmp/robots.txt
 
 # ******************
@@ -47,7 +45,7 @@ MONTH=$(date +"%m")
 MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
 BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
 BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
-now="$(date)"
+_now="$(date)"
 
 # *************************
 # Set Start and End Markers
@@ -61,17 +59,12 @@ _endmarker="### Version Information ##"
 # Create the robots.txt file
 # **************************
 
-cp $_input4a $_robotsinput1
+printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n\n%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" "User-agent: *" "Disallow: /wp-admin/" "Allow: /wp-admin/admin-ajax.php" >> "$_tmprobots"
-sed 's/[\]//g' $_robotsinput1 > $_robotsinput1b
+while IFS= read -r LINE
-IFS=''
+do
-echo $_startmarker  >> $_tmprobots
+printf 'User-agent: %s\n%s\n' "${LINE}" "Disallow:/" >> $_tmprobots
-printf "###################################################\n### Version: "$MY_GIT_TAG"\n### Updated: "$now"\n### Bad Referrer Count: "$BAD_REFERRERS"\n### Bad Bot Count: "$BAD_BOTS"\n###################################################\n" >> $_tmprobots
+done < $_input1
-echo $_endmarker  >> $_tmprobots
+printf '\n' >> $_tmprobots
-printf "\n\n" >> $_tmprobots
-cat $_robotsinput1b |
-while read line; do
-printf 'User-agent: '${line}'\n Disallow:/ \n' >> $_tmprobots
-done
 sudo cp $_tmprobots $TRAVIS_BUILD_DIR/robots.txt/robots.txt
 exit 0
 

travisCI/install-nginx.sh

@@ -115,6 +115,7 @@ sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-google-disavow.sh
 sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/generate-google-exclude.php
 sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-config-readme-files.sh
 sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/modify-files-and-commit.sh
+sudo chmod +x $TRAVIS_BUILD_DIR/travisCI/run-curl-tests.sh
 
 # *****************************************************************************************
 # Travis now moves into running the rest of the tests in the script: section of .travis.yml

travisCI/modify-config-readme-files.sh

@@ -48,31 +48,23 @@ _tmpnginxA=tmpnginxA
 # Start and End Strings to Search for to do inserts into template
 # ***************************************************************
 
-_startmarker="##### Version Information #"
+_startmarker="### Version Information #"
-_endmarker="##### Version Information ##"
+_endmarker="### Version Information ##"
 
 # ****************************************
 # PRINT VERSION INFORMATION INTO README.md
 # ****************************************
 
-LASTUPDATEIFS=$IFS
+printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s' "$_startmarker" "********************************************" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "********************************************" "$_endmarker" >> $_tmpnginxA
-IFS=$'\n'
-now="$(date)"
-end=$(date +%s.%N)    
-echo $_startmarker >> $_tmpnginxA
-runtime=$(python -c "print(${end} - ${start})")
-printf "********************************************\n#### Version: "$MY_GIT_TAG"\n#### Bad Referrer Count: "$BAD_REFERRERS"\n#### Bad Bot Count: "$BAD_BOTS"\n********************************************\n" >> $_tmpnginxA
-echo $_endmarker  >> $_tmpnginxA
-IFS=$LASTUPDATEIFS
 mv $_tmpnginxA $_inputdbA
 ed -s $_inputdbA<<\IN
-1,/##### Version Information #/d
+1,/### Version Information #/d
-/##### Version Information ##/,$d
+/### Version Information ##/,$d
 ,d
 .r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/README.md
-/##### Version Information #/x
+/### Version Information #/x
 .t.
-.,/##### Version Information ##/-d
+.,/### Version Information ##/-d
 #,p
 #,p used to print output replaced with w below to write
 w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/README.md
@@ -84,24 +76,16 @@ rm $_inputdbA
 # PRINT VERSION INFORMATION INTO AUTO-CONFIGURATION.md
 # ****************************************************
 
-LASTUPDATEIFS2=$IFS
+printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s' "$_startmarker" "********************************************" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "********************************************" "$_endmarker" >> $_tmpnginxA
-IFS=$'\n'
-now="$(date)"
-end=$(date +%s.%N)    
-echo $_startmarker >> $_tmpnginxA
-runtime=$(python -c "print(${end} - ${start})")
-printf "********************************************\n#### Version: "$MY_GIT_TAG"\n#### Bad Referrer Count: "$BAD_REFERRERS"\n#### Bad Bot Count: "$BAD_BOTS"\n********************************************\n" >> $_tmpnginxA
-echo $_endmarker  >> $_tmpnginxA
-IFS=$LASTUPDATEIFS2
 mv $_tmpnginxA $_inputdbA
 ed -s $_inputdbA<<\IN
-1,/##### Version Information #/d
+1,/### Version Information #/d
-/##### Version Information ##/,$d
+/### Version Information ##/,$d
 ,d
 .r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/AUTO-CONFIGURATION.md
-/##### Version Information #/x
+/### Version Information #/x
 .t.
-.,/##### Version Information ##/-d
+.,/### Version Information ##/-d
 #,p
 #,p used to print output replaced with w below to write
 w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/AUTO-CONFIGURATION.md
@@ -113,24 +97,16 @@ rm $_inputdbA
 # PRINT VERSION INFORMATION INTO MANUAL-CONFIGURATION.md
 # ******************************************************
 
-LASTUPDATEIFS3=$IFS
+printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s' "$_startmarker" "********************************************" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "********************************************" "$_endmarker" >> $_tmpnginxA
-IFS=$'\n'
-now="$(date)"
-end=$(date +%s.%N)    
-echo $_startmarker >> $_tmpnginxA
-runtime=$(python -c "print(${end} - ${start})")
-printf "********************************************\n#### Version: "$MY_GIT_TAG"\n#### Bad Referrer Count: "$BAD_REFERRERS"\n#### Bad Bot Count: "$BAD_BOTS"\n********************************************\n" >> $_tmpnginxA
-echo $_endmarker  >> $_tmpnginxA
-IFS=$LASTUPDATEIFS3
 mv $_tmpnginxA $_inputdbA
 ed -s $_inputdbA<<\IN
-1,/##### Version Information #/d
+1,/### Version Information #/d
-/##### Version Information ##/,$d
+/### Version Information ##/,$d
 ,d
 .r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/MANUAL-CONFIGURATION.md
-/##### Version Information #/x
+/### Version Information #/x
 .t.
-.,/##### Version Information ##/-d
+.,/### Version Information ##/-d
 #,p
 #,p used to print output replaced with w below to write
 w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/MANUAL-CONFIGURATION.md

travisCI/run-curl-tests.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+# Curl Testing Script for Nginx Ultimate Bad Bot Blocker
+
+# Set Location of our Curl Test Results Files
+
+_curltest1=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest1.txt
+_curltest2=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest2.txt
+_curltest3=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest3.txt
+_curltest4=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest4.txt
+_curltest5=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest5.txt
+_curltest6=$TRAVIS_BUILD_DIR/travisCI/_curl_tests/curltest6.txt
+
+
+# Function Curl Test 1 - Check for Bad Bot "80legs"
+run_curltest1 () {
+truncate -s 0 $_curltest1
+curl -A "80legs" http://localhost:9000/index.php 2> $_curltest1
+if grep -i '(52)' $_curltest1; then
+   echo 'BAD BOT DETECTED - TEST PASSED'
+else
+   echo 'BAD BOT NOT DETECTED - TEST FAILED'
+   exit 1
+fi
+}
+
+# Function Curl Test 2 - Check for Bad Bot "masscan"
+run_curltest2 () {
+truncate -s 0 $_curltest2
+curl -A "masscan" http://localhost:9000/index.php 2> $_curltest2
+if grep -i '(52)' $_curltest2; then
+   echo 'BAD BOT DETECTED - TEST PASSED'
+else
+   echo 'BAD BOT NOT DETECTED - TEST FAILED'
+   exit 1
+fi
+}
+
+# Function Curl Test 3 - Check for Bad Referrer "100dollars-seo.com"
+run_curltest3 () {
+truncate -s 0 $_curltest3
+curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2> $_curltest3
+if grep -i '(52)' $_curltest3; then
+   echo 'BAD REFERRER DETECTED - TEST PASSED'
+else
+   echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
+   exit 1
+fi
+}
+
+# Function Curl Test 4 - Check for Bad Referrer "zx6.ru"
+run_curltest4 () {
+truncate -s 0 $_curltest4
+curl -I http://localhost:9000/index.php -e http://zx6.ru 2> $_curltest4
+if grep -i '(52)' $_curltest4; then
+   echo 'BAD REFERRER DETECTED - TEST PASSED'
+else
+   echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
+   exit 1
+fi
+}
+
+# Function Curl Test 5 - Check for Good Bot "GoogleBot"
+run_curltest5 () {
+truncate -s 0 $_curltest5
+curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> $_curltest5
+if grep -i 'Welcome' $_curltest5; then
+   echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
+else
+   echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
+   exit 1
+fi
+}
+
+# Function Curl Test 6 - Check for Good Bot "BingBot"
+run_curltest6 () {
+truncate -s 0 $_curltest6
+curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> $_curltest6
+if grep -i 'Welcome' $_curltest6; then
+   echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
+else
+   echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
+   exit 1
+fi
+}
+
+
+# Trigger our functions to run
+run_curltest1
+run_curltest2
+run_curltest3
+run_curltest4
+run_curltest5
+run_curltest6
+
+# If everything passed then we exit with 0
+exit 0
+