vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-10 06:15:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Referrers (+) / Changes to Directory Naming

Browse source
This commit is contained in:
Mitchell Krog 2017-07-19 11:10:00 +02:00
parent 97accd5152
commit f8d0b5cab8
36 changed files with 86 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.dev-tools/_curl_tests/curltest1.txt Executable file
View file

@ -0,0 +1,4 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

4
.dev-tools/_curl_tests/curltest2.txt Executable file
View file

@ -0,0 +1,4 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

4
.dev-tools/_curl_tests/curltest3.txt Executable file
View file

@ -0,0 +1,4 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

4
.dev-tools/_curl_tests/curltest4.txt Executable file
View file

@ -0,0 +1,4 @@
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
curl: (52) Empty reply from server

5
.dev-tools/_curl_tests/curltest5.txt Executable file
View file

@ -0,0 +1,5 @@
<?php
echo "Welcome to Nginx\n";
echo "This is the Travis Testing Environment for the Nginx Ultimate Bad Bot Blocker\n";
echo "Visit https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n";

5
.dev-tools/_curl_tests/curltest6.txt Executable file
View file

@ -0,0 +1,5 @@
<?php
echo "Welcome to Nginx\n";
echo "This is the Travis Testing Environment for the Nginx Ultimate Bad Bot Blocker\n";
echo "Visit https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n";

11
.dev-tools/default.vhost Normal file
View file

@ -0,0 +1,11 @@
server {
listen *:9000;
root /var/www/html;
server_name localhost;
charset UTF-8;
location / {
root /var/www/html/;
}
}

86
.dev-tools/deploy-package.sh Executable file
View file

@ -0,0 +1,86 @@
#!/bin/bash
# TravisCI Package Deploy Script for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ******************
# Set Some Variables
# ******************
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
# ***************************************
# Make Sure we are in the Build Directory
# ***************************************
cd $TRAVIS_BUILD_DIR
# *************************
# Create our Version Number
# *************************
export GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
# ***************
# Tag our release
# ***************
git tag $GIT_TAG -a -m "V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER"
# *****************************************
# Push our commit and tags back to the repo
# *****************************************
sudo git push origin master && git push origin master --tags
# *************************************************************************
# Now TravisCI moves into the deploy: section of TravisCI - see .travis.yml
# *************************************************************************
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

439
.dev-tools/generate-blacklist.sh Executable file
View file

@ -0,0 +1,439 @@
#!/bin/bash
# Generator Script for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ******************
# Set Some Variables
# ******************
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
_now="$(date)"
# To get DATE output into uppercase format (if needed)
#_now=$(date | tr -s '[:lower:]' '[:upper:]')
# *************************************
# Specify input lists for the generator
# *************************************
_input1=$TRAVIS_BUILD_DIR/_generator_lists/good-user-agents.list
_input2=$TRAVIS_BUILD_DIR/_generator_lists/allowed-user-agents.list
_input3=$TRAVIS_BUILD_DIR/_generator_lists/limited-user-agents.list
_input4=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
_input5=$TRAVIS_BUILD_DIR/.dev-tools/referrers-regex-format.txt
_input6=$TRAVIS_BUILD_DIR/_generator_lists/google-ip-ranges.list
_input7=$TRAVIS_BUILD_DIR/_generator_lists/bing-ip-ranges.list
_input8=$TRAVIS_BUILD_DIR/_generator_lists/wordpress-theme-detectors.list
_input9=$TRAVIS_BUILD_DIR/_generator_lists/nibbler-seo.list
_input10=$TRAVIS_BUILD_DIR/_generator_lists/cloudflare-ip-ranges.list
# *******************************************************
# Declare temporary database files used during generation
# *******************************************************
_inputdbA=/tmp/version-information.db
_inputdb1=/tmp/good-user-agents.db
_inputdb2=/tmp/allowed-user-agents.db
_inputdb3=/tmp/limited-user-agents.db
_inputdb4=/tmp/bad-user-agents.db
_inputdb5=/tmp/bad-referrers.db
_inputdb6=/tmp/google-ip-ranges.db
_inputdb7=/tmp/bing-ip-ranges.db
_inputdb8=/tmp/wordpress-theme-detectors.db
_inputdb9=/tmp/nibbler-seo.db
_inputdb10=/tmp/cloudflare-ip-ranges.db
# **************************************************
# Declare temporary variables used during generation
# **************************************************
_nginx=$TRAVIS_BUILD_DIR/.dev-tools/globalblacklist.template
_tmpnginxA=_tmpnginxA
_tmpnginx1=_tmpnginx1
_tmpnginx2=_tmpnginx2
_tmpnginx3=_tmpnginx3
_tmpnginx4=_tmpnginx4
_tmpnginx5=_tmpnginx5
_tmpnginx6=_tmpnginx6
_tmpnginx7=_tmpnginx7
_tmpnginx8=_tmpnginx8
_tmpnginx9=_tmpnginx9
_tmpnginx10=_tmpnginx10
# *************************************************************
# Sort all input lists alphabetically and remove any duplicates
# *************************************************************
sort -u $_input1 -o $_input1
sort -u $_input2 -o $_input2
sort -u $_input3 -o $_input3
sort -u $_input4 -o $_input4
sort -u $_input5 -o $_input5
sort -u $_input6 -o $_input6
sort -u $_input7 -o $_input7
sort -u $_input8 -o $_input8
sort -u $_input9 -o $_input9
sort -u $_input10 -o $_input10
# ***************************************************************
# Start and End Strings to Search for to do inserts into template
# ***************************************************************
_start1="# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end1="# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start2="# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end2="# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start3="# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end3="# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start4="# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_end4="# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###"
_start5="# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###"
_end5="# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###"
_start6="# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_end6="# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_start7="# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_end7="# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_start8="# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###"
_end8="# END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###"
_start9="# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###"
_end9="# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###"
_start10="# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_end10="# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###"
_startmarker="### VERSION INFORMATION #"
_endmarker="### VERSION INFORMATION ##"
# **********************************
# SET ALLOW OR DENY ACTION VARIABLES
# **********************************
_action1="0;"
_action2="1;"
_action3="2;"
_action4="3;"
# ************************************
# GOOD USER AGENTS - Create and Insert
# ************************************
printf '%s\n' "$_start1" >> "$_tmpnginx1"
while IFS= read -r LINE
do
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action1" >> "$_tmpnginx1"
done < $_input1
printf '%s\n' "$_end1" >> "$_tmpnginx1"
mv $_tmpnginx1 $_inputdb1
ed -s $_inputdb1<<\IN
1,/# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb1
# ********************************
# ALLOWED BOTS - Create and Insert
# ********************************
printf '%s\n' "$_start2" >> "$_tmpnginx2"
while IFS= read -r LINE
do
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action2" >> "$_tmpnginx2"
done < $_input2
printf '%s\n' "$_end2" >> "$_tmpnginx2"
mv $_tmpnginx2 $_inputdb2
ed -s $_inputdb2<<\IN
1,/# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb2
# ********************************
# LIMITED BOTS - Create and Insert
# ********************************
printf '%s\n' "$_start3" >> "$_tmpnginx3"
while IFS= read -r LINE
do
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action3" >> "$_tmpnginx3"
done < $_input3
printf '%s\n' "$_end3" >> "$_tmpnginx3"
mv $_tmpnginx3 $_inputdb3
ed -s $_inputdb3<<\IN
1,/# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb3
# ****************************
# BAD BOTS - Create and Insert
# ****************************
printf '%s\n' "$_start4" >> "$_tmpnginx4"
while IFS= read -r LINE
do
printf '\t"~*%s%s%s"\t\t%s\n' "\b" "${LINE}" "\b" "$_action4" >> "$_tmpnginx4"
done < $_input4
printf '%s\n' "$_end4" >> "$_tmpnginx4"
mv $_tmpnginx4 $_inputdb4
ed -s $_inputdb4<<\IN
1,/# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb4
# ********************************
# BAD REFERERS - Create and Insert
# ********************************
printf '%s\n' "$_start5" >> "$_tmpnginx5"
while IFS= read -r LINE
do
printf '\t%s\n' "${LINE}" >> "$_tmpnginx5"
done < $_input5
printf '%s\n' "$_end5" >> "$_tmpnginx5"
mv $_tmpnginx5 $_inputdb5
ed -s $_inputdb5<<\IN
1,/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb5
# ************************************
# GOOGLE IP RANGES - Create and Insert
# ************************************
printf '%s\n' "$_start6" >> "$_tmpnginx6"
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx6"
done < $_input6
printf '%s\n' "$_end6" >> "$_tmpnginx6"
mv $_tmpnginx6 $_inputdb6
ed -s $_inputdb6<<\IN
1,/# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/# START GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END GOOGLE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb6
# **********************************
# BING IP RANGES - Create and Insert
# **********************************
printf '%s\n' "$_start7" >> "$_tmpnginx7"
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx7"
done < $_input7
printf '%s\n' "$_end7" >> "$_tmpnginx7"
mv $_tmpnginx7 $_inputdb7
ed -s $_inputdb7<<\IN
1,/# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/# START BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END BING IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb7
# *********************************************
# Wordpress Theme Detectors - Create and Insert
# *********************************************
printf '%s\n' "$_start8" >> "$_tmpnginx8"
while IFS= read -r LINE
do
printf '%s\n' "${LINE}" >> "$_tmpnginx8"
done < $_input8
printf '%s\n' "$_end8" >> "$_tmpnginx8"
mv $_tmpnginx8 $_inputdb8
ed -s $_inputdb8<<\IN
1,/# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/# START WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END WP THEME DETECTORS ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb8
# *******************************
# Nibbler SEO - Create and Insert
# *******************************
printf '%s\n' "$_start9" >> "$_tmpnginx9"
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action2" >> "$_tmpnginx9"
done < $_input9
printf '%s\n' "$_end9" >> "$_tmpnginx9"
mv $_tmpnginx9 $_inputdb9
ed -s $_inputdb9<<\IN
1,/# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/# START NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END NIBBLER ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb9
# ****************************************
# CLOUDFLARE IP RANGES - Create and Insert
# ****************************************
printf '%s\n' "$_start10" >> "$_tmpnginx10"
while IFS= read -r LINE
do
printf '\t%s\t\t%s\n' "${LINE}" "$_action1" >> "$_tmpnginx10"
done < $_input10
printf '%s\n' "$_end10" >> "$_tmpnginx10"
mv $_tmpnginx10 $_inputdb10
ed -s $_inputdb10<<\IN
1,/# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/d
/# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/# START CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/x
.t.
.,/# END CLOUDFLARE IP RANGES ### DO NOT EDIT THIS LINE AT ALL ###/-d
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdb10
# *******************************************************************************
# PRINT VERSION, SCRIPT RUNTIME and UPDATE INFORMATION INTO GLOBALBLACKLIST FILES
# *******************************************************************************
printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Referrer Count: " "$BAD_REFERRERS" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" >> $_tmpnginxA
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
1,/### VERSION INFORMATION #/d
/### VERSION INFORMATION ##/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
/### VERSION INFORMATION #/x
.t.
.,/### VERSION INFORMATION ##/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/globalblacklist.template
q
IN
rm $_inputdbA
# **************************************************
# Generate Additional Files and Copy Them to Folders
# **************************************************
sudo cp $_nginx $TRAVIS_BUILD_DIR/conf.d/globalblacklist.conf
sudo cp $_nginx $TRAVIS_BUILD_DIR/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
exit 0
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

69
.dev-tools/generate-google-disavow.sh Executable file
View file

@ -0,0 +1,69 @@
#!/bin/bash
# Google Disavow file generator for Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ************************************
# Specify input list for the generator
# ************************************
_input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list
# **************************
# Create Google Disavow File
# **************************
sudo truncate -s 0 $TRAVIS_BUILD_DIR/_google_webmaster_disavow_links/google-disavow.txt
while IFS= read -r LINE
do
printf '%s%s\n' "domain:" "${LINE}" >> $TRAVIS_BUILD_DIR/_google_webmaster_disavow_links/google-disavow.txt
done < $_input1
exit 0
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

97
.dev-tools/generate-google-exclude.php Executable file
View file

@ -0,0 +1,97 @@
<?php
/* Google Exclude File Generator Script for the Nginx Ultimate Bad Bot Blocker
* Adapted from Script Copyright (c) 2017 Stevie-Ray - https://github.com/Stevie-Ray
* Adapted by: Mitchell Krog (mitchellkrog@gmail.com) - https://github.com/mitchellkrogza
* Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
*/
namespace mitchellkrogza;
use Mso\IdnaConvert\IdnaConvert;
class Generator
{
private $projectUrl = "https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker";
public function generateFiles()
{
date_default_timezone_set('Africa/Johannesburg');
$date = date('Y-m-d H:i:s');
$lines = $this->domainWorker();
$this->createGoogleExclude($lines);
}
/**
* Open our input domain list and create our array
* @return array
*/
public function domainWorker()
{
$domainsFile = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/_generator_lists/bad-referrers.list";
$handle = fopen($domainsFile, "r");
if (!$handle) {
throw new \RuntimeException('Error opening file ' . $domainsFile);
}
$lines = array();
while (($line = fgets($handle)) !== false) {
$line = trim(preg_replace('/\s\s+/', ' ', $line));
// convert russian domains
if (preg_match('/[А-Яа-яЁё]/u', $line)) {
$IDN = new IdnaConvert();
$line = $IDN->encode($line);
}
if (empty($line)) {
continue;
}
$lines[] = $line;
}
fclose($handle);
$uniqueLines = array_unique($lines, SORT_STRING);
sort($uniqueLines, SORT_STRING);
if (is_writable($domainsFile)) {
file_put_contents($domainsFile, implode("\n", $uniqueLines));
} else {
trigger_error("Permission denied");
}
return $lines;
}
/**
* Write to File Function
* @param $filename
* @param $data
*/
protected function writeToFile($filename, $data)
{
$file = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/_google_analytics_ghost_spam/$filename";
$handle = fopen($file, 'w') or die('Cannot open file: '.$file);
fwrite($handle, $data);
}
/**
* Create Google Exclude Files Splitting them at Google's 30,000 Character Limit
* @param $lines
*/
public function createGoogleExclude(array $lines)
{
$regexLines = [];
foreach ($lines as $line) {
$regexLines[] = preg_quote($line);
}
$data = implode('|', $regexLines);
$googleLimit = 30000;
$dataLength = strlen($data);
$lastPosition = 0;
for ($x = 1; $lastPosition < $dataLength; $x++) {
if( ($dataLength-$lastPosition) >= $googleLimit){
$pipePosition = strrpos(substr($data, $lastPosition, $googleLimit), '|');
$dataSplit = substr($data, $lastPosition, $pipePosition);
$lastPosition = $lastPosition + $pipePosition+1;
}else{
$dataSplit = substr($data, $lastPosition);
$lastPosition = $dataLength; // Break
}
$this->writeToFile('google-exclude-0' . $x . '.txt', $dataSplit);
}
}
}
$generator = new Generator();
$generator->generateFiles();

94
.dev-tools/generate-regex-format-referrers.php Executable file
View file

@ -0,0 +1,94 @@
<?php
/* Regex Formatting Generator Script for the Nginx Ultimate Bad Bot Blocker
* Adapted from Script Copyright (c) 2017 Stevie-Ray - https://github.com/Stevie-Ray
* Adapted by: Mitchell Krog (mitchellkrog@gmail.com) - https://github.com/mitchellkrogza
* Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
*/
namespace mitchellkrogza;
use Mso\IdnaConvert\IdnaConvert;
class Generator
{
private $projectUrl = "https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker";
public function generateFiles()
{
date_default_timezone_set('Africa/Johannesburg');
$date = date('Y-m-d H:i:s');
$lines = $this->domainWorker();
$this->createNginx($lines);
}
/**
* @return array
*/
public function domainWorker()
{
$domainsFile = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/_generator_lists/bad-referrers.list";
$handle = fopen($domainsFile, "r");
if (!$handle) {
throw new \RuntimeException('Error opening file ' . $domainsFile);
}
$lines = array();
while (($line = fgets($handle)) !== false) {
$line = trim(preg_replace('/\s\s+/', ' ', $line));
// convert internationalized domain names
if (preg_match('/[А-Яа-яЁёɢ]/u', $line)) {
$IDN = new IdnaConvert();
$line = $IDN->encode($line);
}
if (empty($line)) {
continue;
}
$lines[] = $line;
}
fclose($handle);
$uniqueLines = array_unique($lines, SORT_STRING);
sort($uniqueLines, SORT_STRING);
if (is_writable($domainsFile)) {
file_put_contents($domainsFile, implode("\n", $uniqueLines));
} else {
trigger_error("Permission denied");
}
return $lines;
}
/**
* Write to File Function
* @param $filename
* @param $data
*/
protected function writeToFile($filename, $data)
{
$file = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/referrers-regex-format.txt";
$handle = fopen($file, 'w') or die('Cannot open file: '.$file);
fwrite($handle, $data);
}
/**
* @param string $date
* @param array $lines
*/
public function createNginx(array $lines)
{
$file = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/referrers-regex-format.txt";
$data = "";
{
foreach ($lines as $line) {
$data .= "\"~*" . preg_quote($line) . "\" \t1;\n";
}
$this->writeToFile($file, $data);
}
}
}
$generator = new Generator();
$generator->generateFiles();

92
.dev-tools/generate-robots.sh Executable file
View file

@ -0,0 +1,92 @@
#!/bin/bash
# Generator Script for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# Generate a robots.txt file for those unable to use the full Nginx Ultimate Bad Bot Blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ***************
# Set Input Files
# ***************
_input1=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
_tmprobots=/tmp/robots.txt
# ******************
# Set Some Variables
# ******************
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
_now="$(date)"
# *************************
# Set Start and End Markers
# *************************
_startmarker="### Version Information #"
_endmarker="### Version Information ##"
# **************************
# Create the robots.txt file
# **************************
printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s\n\n%s\n%s\n%s\n' "$_startmarker" "###################################################" "### Version: " "$MY_GIT_TAG" "### Updated: " "$_now" "### Bad Bot Count: " "$BAD_BOTS" "###################################################" "$_endmarker" "User-agent: *" "Disallow: /wp-admin/" "Allow: /wp-admin/admin-ajax.php" >> "$_tmprobots"
while IFS= read -r LINE
do
printf 'User-agent: %s\n%s\n' "${LINE}" "Disallow:/" >> "$_tmprobots"
done < $_input1
printf '\n' >> "$_tmprobots"
sudo cp $_tmprobots $TRAVIS_BUILD_DIR/robots.txt/robots.txt
exit 0
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

6437
.dev-tools/globalblacklist.template Executable file
View file

File diff suppressed because it is too large Load diff

5
.dev-tools/index.php Normal file
View file

@ -0,0 +1,5 @@
<?php
echo "Welcome to Nginx\n";
echo "This is the Travis Testing Environment for the Nginx Ultimate Bad Bot Blocker\n";
echo "Visit https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker\n";

154
.dev-tools/install-nginx.sh Executable file
View file

@ -0,0 +1,154 @@
#!/bin/bash
# Travis CI Generating and Building for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ***************************************************************
# Start Getting Nginx Ready for Testing the Nginx Bad Bot Blocker
# ***************************************************************
# *************************************************
# Delete default site created by Nginx Installation
# *************************************************
sudo rm /etc/nginx/sites-available/default
sudo rm /etc/nginx/sites-enabled/default
sudo rm /var/www/html/*
# ********************************************************
# Copy our default.vhost file into Nginx /sites-available/
# ********************************************************
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/default.vhost /etc/nginx/sites-available/default.vhost
# **********************************************
# Link the vhost file into Nginx /sites-enabled/
# **********************************************
sudo ln -s /etc/nginx/sites-available/default.vhost /etc/nginx/sites-enabled/default.vhost
# ***********************************************************
# Copy our index.php file into the default site's root folder
# ***********************************************************
sudo cp $TRAVIS_BUILD_DIR/.dev-tools/index.php /var/www/html/index.php
# ***********************************************************************
# Download the Nginx Bad Bot Blocker setup files from the Live Repository
# ***********************************************************************
sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/install-ngxblocker -O /usr/sbin/install-ngxblocker
sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/setup-ngxblocker -O /usr/sbin/setup-ngxblocker
sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/update-ngxblocker -O /usr/sbin/update-ngxblocker
# **************************************************
# Set our install and setup scripts to be executable
# **************************************************
sudo chmod +x /usr/sbin/install-ngxblocker
sudo chmod +x /usr/sbin/setup-ngxblocker
sudo chmod +x /usr/sbin/update-ngxblocker
# **********************
# Run Install-NgxBlocker
# **********************
cd /usr/sbin
sudo ./install-ngxblocker -x
# ********************
# Run setup-ngxblocker
# ********************
cd /usr/sbin
sudo ./setup-ngxblocker -x
# ******************************************************************************************************
# NOTE: for Verbose Testing of any shell scripts use below format adding sh -x before running the script
# this helps a lot inside the TravisCI environment to see where a shell script may be failing
# sudo sh -x ./setup-ngxblocker -x
# *******************************************************************************************************
# ************************
# Load our Nginx.conf file
# ************************
sudo nginx -c /etc/nginx/nginx.conf
# ****************************************************************************************
# Run update-ngxblocker test which downloads latest globalblacklist.conf and reloads Nginx
# ****************************************************************************************
cd /usr/sbin
sudo ./update-ngxblocker -e mitchellkrog@gmail.com
# *********************
# Force reload of Nginx
# *********************
sudo service nginx reload
# ***********************************************************
# Set all our other setup and deploy scripts to be executable
# ***********************************************************
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/deploy-package.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-blacklist.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-robots.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-google-disavow.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-google-exclude.php
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/generate-regex-format-referrers.php
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/modify-config-readme-files.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/modify-files-and-commit.sh
sudo chmod +x $TRAVIS_BUILD_DIR/.dev-tools/run-curl-tests.sh
# *****************************************************************************************
# Travis now moves into running the rest of the tests in the script: section of .travis.yml
# *****************************************************************************************
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

141
.dev-tools/modify-config-readme-files.sh Executable file
View file

@ -0,0 +1,141 @@
#!/bin/bash
# Write Build / Version Number into README.md, AUTO-CONFIGURATION.md and MANUAL-CONFIGURATION.md files
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ******************
# Set Some Variables
# ******************
YEAR=$(date +%Y)
MONTH=$(date +%m)
MY_GIT_TAG=V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER
BAD_REFERRERS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list)
BAD_BOTS=$(wc -l < $TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list)
# **********************************
# Temporary database files we create
# **********************************
_inputdbA=/tmp/lastupdated.db
_tmpnginxA=tmpnginxA
# ***************************************************************
# Start and End Strings to Search for to do inserts into template
# ***************************************************************
_startmarker="### Version Information #"
_endmarker="### Version Information ##"
# ****************************************
# PRINT VERSION INFORMATION INTO README.md
# ****************************************
printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s' "$_startmarker" "********************************************" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "********************************************" "$_endmarker" >> "$_tmpnginxA"
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
1,/### Version Information #/d
/### Version Information ##/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/README.md
/### Version Information #/x
.t.
.,/### Version Information ##/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/README.md
q
IN
rm $_inputdbA
# ****************************************************
# PRINT VERSION INFORMATION INTO AUTO-CONFIGURATION.md
# ****************************************************
printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s' "$_startmarker" "********************************************" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "********************************************" "$_endmarker" >> "$_tmpnginxA"
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
1,/### Version Information #/d
/### Version Information ##/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/AUTO-CONFIGURATION.md
/### Version Information #/x
.t.
.,/### Version Information ##/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/AUTO-CONFIGURATION.md
q
IN
rm $_inputdbA
# ******************************************************
# PRINT VERSION INFORMATION INTO MANUAL-CONFIGURATION.md
# ******************************************************
printf '%s\n%s\n%s%s\n%s%s\n%s%s\n%s\n%s' "$_startmarker" "********************************************" "#### Version: " "$MY_GIT_TAG" "#### Bad Referrer Count: " "$BAD_REFERRERS" "#### Bad Bot Count: " "$BAD_BOTS" "********************************************" "$_endmarker" >> "$_tmpnginxA"
mv $_tmpnginxA $_inputdbA
ed -s $_inputdbA<<\IN
1,/### Version Information #/d
/### Version Information ##/,$d
,d
.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/MANUAL-CONFIGURATION.md
/### Version Information #/x
.t.
.,/### Version Information ##/-d
#,p
#,p used to print output replaced with w below to write
w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/MANUAL-CONFIGURATION.md
q
IN
rm $_inputdbA
exit 0
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

115
.dev-tools/modify-files-and-commit.sh Executable file
View file

@ -0,0 +1,115 @@
#!/bin/bash
# Travis CI Generating and Building for the Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ******************
# Set Some Variables
# ******************
#set -v
YEAR=$(date +"%Y")
MONTH=$(date +"%m")
cd $TRAVIS_BUILD_DIR
# *******************************
# Remove Remote Added by TravisCI
# *******************************
git remote rm origin
# **************************
# Add Remote with Secure Key
# **************************
git remote add origin https://${GH_TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git
# **********************************************************************************
# List Remotes ONLY DURING testing - do not do this on live repo / possible key leak
# git remote -v
# ***********************************************************************************
# *********************
# Set Our Git Variables
# *********************
git config --global user.email "${GIT_EMAIL}"
git config --global user.name "${GIT_NAME}"
git config --global push.default simple
# *******************************************
# Make sure we have checked out master branch
# *******************************************
git checkout master
# ***************************************************
# Modify our files with build and version information
# ***************************************************
php ./.dev-tools/generate-regex-format-referrers.php
sudo $TRAVIS_BUILD_DIR/.dev-tools/generate-blacklist.sh
sudo $TRAVIS_BUILD_DIR/.dev-tools/modify-config-readme-files.sh
sudo $TRAVIS_BUILD_DIR/.dev-tools/generate-robots.sh
sudo $TRAVIS_BUILD_DIR/.dev-tools/generate-google-disavow.sh
php ./.dev-tools/generate-google-exclude.php
# *************************************
# Add all the modified files and commit
# *************************************
git add -A
git commit -am "V3.$YEAR.$MONTH.$TRAVIS_BUILD_NUMBER [ci skip]"
# *************************************************************
# Travis now moves to the before_deploy: section of .travis.yml
# *************************************************************
# MIT License
# Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
# https://github.com/mitchellkrogza
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

5317
.dev-tools/referrers-regex-format.txt Normal file
View file

File diff suppressed because it is too large Load diff

126
.dev-tools/run-curl-tests.sh Executable file
View file

@ -0,0 +1,126 @@
#!/bin/bash
# Curl Testing Script for Nginx Ultimate Bad Bot Blocker
# Created by: Mitchell Krog (mitchellkrog@gmail.com)
# Copyright: Mitchell Krog - https://github.com/mitchellkrogza
# Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# *******************************************
# Set Location of our Curl Test Results Files
# *******************************************
_curltest1=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest1.txt
_curltest2=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest2.txt
_curltest3=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest3.txt
_curltest4=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest4.txt
_curltest5=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest5.txt
_curltest6=$TRAVIS_BUILD_DIR/.dev-tools/_curl_tests/curltest6.txt
# *************************************************
# Function Curl Test 1 - Check for Bad Bot "80legs"
# *************************************************
run_curltest1 () {
truncate -s 0 $_curltest1
curl -A "80legs" http://localhost:9000/index.php 2> $_curltest1
if grep -i '(52)' $_curltest1; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
exit 1
fi
}
# **************************************************
# Function Curl Test 2 - Check for Bad Bot "masscan"
# **************************************************
run_curltest2 () {
truncate -s 0 $_curltest2
curl -A "masscan" http://localhost:9000/index.php 2> $_curltest2
if grep -i '(52)' $_curltest2; then
echo 'BAD BOT DETECTED - TEST PASSED'
else
echo 'BAD BOT NOT DETECTED - TEST FAILED'
exit 1
fi
}
# ******************************************************************
# Function Curl Test 3 - Check for Bad Referrer "100dollars-seo.com"
# ******************************************************************
run_curltest3 () {
truncate -s 0 $_curltest3
curl -I http://localhost:9000/index.php -e http://100dollars-seo.com 2> $_curltest3
if grep -i '(52)' $_curltest3; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
exit 1
fi
}
# ******************************************************
# Function Curl Test 4 - Check for Bad Referrer "zx6.ru"
# ******************************************************
run_curltest4 () {
truncate -s 0 $_curltest4
curl -I http://localhost:9000/index.php -e http://zx6.ru 2> $_curltest4
if grep -i '(52)' $_curltest4; then
echo 'BAD REFERRER DETECTED - TEST PASSED'
else
echo 'BAD REFERRER NOT DETECTED - TEST FAILED'
exit 1
fi
}
# *****************************************************
# Function Curl Test 5 - Check for Good Bot "GoogleBot"
# *****************************************************
run_curltest5 () {
truncate -s 0 $_curltest5
curl -v -A "GoogleBot" http://localhost:9000/index.php 2>&1 >> $_curltest5
if grep -i 'Welcome' $_curltest5; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
exit 1
fi
}
# ***************************************************
# Function Curl Test 6 - Check for Good Bot "BingBot"
# ***************************************************
run_curltest6 () {
truncate -s 0 $_curltest6
curl -v -A "BingBot" http://localhost:9000/index.php 2>&1 >> $_curltest6
if grep -i 'Welcome' $_curltest6; then
echo 'GOOD BOT ALLOWED THROUGH - TEST PASSED'
else
echo 'GOOD BOT NOT ALLOWED THROUGH - TEST FAILED'
exit 1
fi
}
# *********************************
# Trigger our curl functions to run
# *********************************
run_curltest1
run_curltest2
run_curltest3
run_curltest4
run_curltest5
run_curltest6
# ****************************************
# If everything passed then we exit with 0
# ****************************************
exit 0