nginx-ultimate-bad-bot-blocker/update-ngxblocker

#!/bin/sh

# Shell Script for Auto Updating the Nginx Bad Bot Blocker
# Copyright: https://github.com/mitchellkrogza
# Project Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
# Update script & Alpine Linux package by Stuart Cardall: https://github.com/itoffshore

# MAKE SURE you have all the following files in /etc/nginx/bots.d/ folder
# ***********************************************************************
# whitelist-ips.conf
# whitelist-domains.conf
# blacklist-user-agents.conf
# bad-referrer-words.conf
# custom-bad-referrers.conf
# blacklist-ips.conf
# A major change to using include files was introduced in
# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/commit/7e3ab02172dafdd524de5dd450a9732328622779
# **************************************************************************
# Nginx will fail a reload with [EMERG] without the presence of these files.

# PLEASE READ UPDATED CONFIGURATION INSTRUCTIONS BEFORE USING THIS

# Save this file as /usr/local/sbin/update-ngxblocker
# cd /usr/local/sbin
# sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/update-ngxblocker -O update-ngxblocker
# Make it Executable chmod 700 /usr/local/sbin/update-ngxblocker

# RUN THE UPDATE
# Here our script runs, pulls the latest update, reloads nginx and emails you a notification

EMAIL="you@example.com"
SEND_EMAIL="N"
SEND_MG_EMAIL="N"
SEND_EMAIL_UPDATE="N"
#Mailgun
MG_API_KEY="key-yadayadayada"
MG_DOMAIN="mg.example.com"
MG_FROM=""
CONF_DIR=/etc/nginx/conf.d
BOTS_DIR=/etc/nginx/bots.d
INSTALLER=/usr/local/sbin/install-ngxblocker
LOGGING="N"

##### end user configuration ##############################################################

BOLDGREEN="\033[1m\033[32m"
BOLDMAGENTA="\033[1m\033[35m"
BOLDRED="\033[1m\033[31m"
BOLDYELLOW="\033[1m\033[33m"
BOLDWHITE="\033[1m\033[37m"
RESET="\033[0m"
OS=$(uname -s)
CURL_PATH=""

usage() {
        local script=$(basename $0)
        cat <<EOF
$script: UPDATE Nginx Bad Bot Blocker blacklist in: [ $CONF_DIR ]

Usage: $script [OPTIONS]
        [ -c ] : NGINX conf directory          (default: $CONF_DIR)
        [ -b ] : NGINX bots directory          (default: $BOTS_DIR)
        [ -i ] : Change installer path         (default: $INSTALLER)
        [ -r ] : Change repo url               (default: $REPO)
        [ -e ] : Change @email address         (default: $EMAIL)
        [ -g ] : Change @email address Mailgun (default: $EMAIL)
        [ -d ] : Mailgun Domain
        [ -a ] : Mailgun API Key
        [ -f ] : Mailgun / Mail From Address
        [ -m ] : Change mail (system alias)    (default: $EMAIL)
        [ -n ] : Do not send email report      (default: $SEND_EMAIL)
        [ -o ] : Only send email on update     (default: $SEND_EMAIL_UPDATE)
        [ -q ] : Suppress non error messages
        [ -v ] : Print blacklist version
        [ -h ] : this help message

Examples:
 $script                         (Download globalblacklist.conf to: $CONF_DIR)
 $script -c /my/custom/conf.d    (Download globalblacklist.conf to a custom location)
 $script -b /my/custom/bots.d    (Download globalblacklist.conf & update with your custom bots.d location)
 $script -e you@example.com    (Download globalblacklist.conf specifying your email address for the notification)
 $script -g you@example.com -d domain -a mailgunapikey -f fromaddress   (Download globalblacklist.conf specifying your email address for the notification sent via mailgun)
 $script -q -m webmaster         (Send mail to a system alias address & give less verbose messages for cron)
 $script -o -e you@example.com (Send mail notification only on updates)
 $script -i /path/to/install-ngxblocker (Use custom path to install-ngxblocker to update bots.d / conf.d include files)
EOF
        exit 0
}

check_version() {
	local remote_ver= remote_date= version= date= file=$CONF_DIR/globalblacklist.conf
	local tmp=$(mktemp) url=$REPO/conf.d/globalblacklist.conf range="145-345"

	if [ -f $file ]; then
		# local version
		version=$(grep "Version:" $file | ${SED_CMD} 's|^.*: V||g')
		date=$(grep "Updated:" $file | ${SED_CMD} 's|^.*: ||g')
		print_message "\nLOCAL Version: $BOLDWHITE$version$RESET\n"
		print_message "Updated: $date\n\n"
		# remote version
		$CURL_PATH -s --limit-rate 5k -r $range --location $url -o $tmp
		remote_ver=$(grep "Version:" $tmp | ${SED_CMD} 's|^.*: V||g')
		remote_date=$(grep "Updated:" $tmp | ${SED_CMD} 's|^.*: ||g')
		print_message "REMOTE Version: $BOLDWHITE$remote_ver$RESET\n"
		print_message "Updated: $remote_date\n"
		rm -f $tmp

		if [ "$version" != "$remote_ver" ]; then
			print_message "\nUpdate Available => $BOLDMAGENTA$remote_ver$RESET\n\n"
			return 1
		else
			print_message "\nLatest Blacklist Already Installed: $BOLDGREEN$version$RESET\n\n"
		fi
	else
		printf "${BOLDRED}ERROR${RESET}: Missing '$file' => ${BOLDWHITE}running $INSTALLER:${RESET}\n"
		$INSTALL_INC
		if [ -f $file ]; then
			check_version
		fi
	fi
}

check_dirs() {
	local x= dirs="$*"

	for x in $dirs; do
		if [ ! -d $x ]; then
			printf "${BOLDRED}ERROR${RESET}: Missing directory: $x => ${BOLDWHITE}running $INSTALLER:${RESET}\n"
			$INSTALL_INC
		fi
	done
}

find_binary() {
	local x= path= binary=$1 bin_paths='/bin /usr/bin /usr/local/bin /sbin /usr/sbin /usr/local/sbin /root/bin /root/.bin'

	for x in $bin_paths; do
		path="$x/$binary"

		if [ -x $path ]; then
			echo $path
			return
		fi
	done
}

update_paths() {
	# variables in nginx include files not currently possible
	# updates hard coded bots.d path in globalblacklist.conf
	local blacklist=$1 include_paths= dir= x=

	if ! grep "$BOTS_DIR" $blacklist 1>/dev/null; then
		if [ -d $BOTS_DIR ]; then
			printf "${BOLDGREEN}Updating bots.d path${RESET}: ${BOLDWHITE}$BOTS_DIR => $blacklist${RESET}\n"
			include_paths=$(grep -E "include /.*.conf;$" $blacklist | awk '{print $2}' | tr -d ';')

			for x in $include_paths; do
				dir=$(dirname $x)
				${SED_CMD} -i "s|$dir|$BOTS_DIR|" $blacklist
			done
		else
			printf "${BOLDRED}ERROR${RESET}: '$BOTS_DIR' does not exist => ${BOLDWHITE}running $INSTALLER${RESET}.\n"
			$INSTALL_INC
			update_paths $blacklist
		fi
	fi
}

sanitize_path() {
	echo $1 |tr -cd '[:alnum:] [=@=] [=.=] [=-=] [=/=] [=_=]' \
		|tr -s '@.-/_' |awk '{print tolower($0)}'
}

sanitize_url() {
	echo $1 |tr -cd '[:alnum:] [=:=] [=.=] [=-=] [=/=]' \
		|tr -s ':.-' |awk '{print tolower($0)}'
}

sanitize_email() {
	echo $1 |tr -cd '[:alnum:] [=@=] [=.=] [=-=] [=_=] [=+=]' \
		|tr -s '@-_.+' |awk '{print tolower($0)}'
}

check_args() {
	local option=$1 type=$2 arg=$3
	local msg="ERROR: option '-$option' argument '$arg' requires:"

	case "$type" in
	        path)   if ! echo $arg | grep ^/ 1>/dev/null; then
				printf "$msg absolute path.\n"
				exit 1
			fi
			;;
	       email)   if ! echo $arg | grep -E ^[-+_\.[:alnum:]]+@[-_\.[:alnum:]]+ 1>/dev/null; then
				printf "$msg email@domain.com\n"
				exit 1
			fi
			;;
	         url)   if ! echo $arg | grep -E ^http[s]?://[0-9a-zA-Z-]+[.]+[/0-9a-zA-Z.]+ 1>/dev/null; then
				printf "$msg url => http[s]://the.url\n"
				exit 1
			fi
			;;
	      script)	if [ ! -x $arg ]; then
				printf "$msg '$arg' is not executable / does not exist.\n"
				exit 1
			fi
			;;
	        none)   printf "$msg argument.\n"; exit 1;;
        esac
}

check_depends() {
	# global var is needed here, it is used in other places
	CURL_PATH=$(find_binary curl)

	case $OS in
		Linux)
			SED_CMD=$(find_binary sed)
			;;
		*BSD)
			SED_CMD=$(find_binary gsed)
			;;
	esac

	# centos does not have which by default
	if [ -z $CURL_PATH ]; then
		printf "${BOLDRED}ERROR${RESET}: $0 requires: 'curl' => ${BOLDWHITE}cannot check remote version.${RESET}\n"
		exit 1
	fi

	# install-ngxblocker downloads missing scripts / includes as part of the update process
	if [ ! -x $INSTALLER ]; then
		printf "${BOLDRED}ERROR${RESET}: $0 requires: '$INSTALLER' => ${BOLDWHITE}cannot update includes.${RESET}\n"
		exit 1
	fi
}

print_message() {
	local msg="$@"

	if [ "$VERBOSE" != "N" ]; then
		printf "$msg"
	fi
}

log_output() {
	local logger=$(find_binary logger)
	local script=$(basename $0)

	if [ -n "$logger" ]; then
		# remove ansi color codes
		${SED_CMD} -i 's/\x1b\[[0-9;]*m//g' $EMAIL_REPORT
		# remove blank lines
		${SED_CMD} -i '/^\s*$/d' $EMAIL_REPORT
		# log output
		$logger -t $script -f $EMAIL_REPORT 2>&1
		print_message "Output logged to syslog\n";
	else
		print_message "${BOLDRED}ERROR: cannot find logger${RESET}\n\n";
	fi
}

send_email() {
	# email report (mailx + ssmtp are enough to send emails)
	local mail_path=$(find_binary mail)

	if [ -n "$mail_path" ]; then
		print_message "Emailing report to: ${BOLDWHITE}$EMAIL${RESET}\n\n";

		# remove ansi colour codes
		${SED_CMD} -i 's/\x1b\[[0-9;]*m//g' $EMAIL_REPORT

		if [ -n "$MG_FROM" ]; then
			cat $EMAIL_REPORT | $mail_path -f "$MG_FROM" -s "Nginx Bad Bot Blocker Updated" $EMAIL
		else
			cat $EMAIL_REPORT | $mail_path -s "Nginx Bad Bot Blocker Updated" $EMAIL
		fi
	else
		print_message "${BOLDYELLOW}WARN${RESET}: missing mail command => ${BOLDWHITE}disabling emails${RESET}.\n\n"
	fi
}

send_email_via_mailgun() {
	local report= subject= endpoint="https://api.mailgun.net/v3/$MG_DOMAIN/messages"

	echo "Mailgunning report to: ${BOLDWHITE}$EMAIL${RESET}\n\n";
	${SED_CMD} -i 's/\x1b\[[0-9;]*m//g' $EMAIL_REPORT
	report="$(cat $EMAIL_REPORT)"
	subject='Nginx Bad Bot Blocker Updated'

	$CURL_PATH -s --user api:$MG_API_KEY $endpoint -F from='botblocker<'$MG_FROM'>' -F to=$EMAIL -F subject="$subject" -F text="$report"
}

get_options() {
	local arg= opts=

	while getopts :c:b:i:r:e:g:a:d:f:m:lnovqh opts "$@"
	do
		if [ -n "${OPTARG}" ]; then
			case "$opts" in
				r) arg=$(sanitize_url ${OPTARG});;
				e) arg=$(sanitize_email ${OPTARG});;
				g) arg=$(sanitize_email ${OPTARG});;
				*) arg=$(sanitize_path ${OPTARG});;
			esac
		fi

		case "$opts" in
			c) CONF_DIR=$arg; check_args $opts path $arg ;;
			b) BOTS_DIR=$arg; check_args $opts path $arg ;;
			i) INSTALLER=$arg; check_args $opts script $arg ;;
			r) REPO=$arg; check_args $opts url $arg ;;
			e) EMAIL=$arg; SEND_EMAIL=Y; check_args $opts email $arg ;;
			g) EMAIL=$arg; SEND_MG_EMAIL=Y; check_args $opts email $arg ;;
			a) MG_API_KEY=$arg;;
			d) MG_DOMAIN=$arg;;
			f) MG_FROM=$arg;;
			m) EMAIL=$arg; SEND_EMAIL=Y ;; # /etc/aliases no sanity checks
			l) LOGGING=Y ;;
			n) SEND_EMAIL=N ;;
			o) SEND_EMAIL_UPDATE=Y ;;
			v) check_version; exit 0 ;;
			q) export VERBOSE=N ;;
			h) usage ;;
			\?) usage ;;
			:) check_args $OPTARG none none ;;
		esac
	done

	INSTALL_INC="$INSTALLER -b $BOTS_DIR -c $CONF_DIR -x"
}

main() {
	local REPO=https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master
	local file=globalblacklist.conf remote_dir=conf.d url= output= update= status= tmp= retval=
	local nginx_path=$(find_binary nginx)
	local pidof_path=$(find_binary pidof)

	# require root
	if [ "$(id -u)" != "0" ]; then
		echo "This script must be run as root" 1>&2
		exit 1
	fi

	# parse command line
	get_options $@
	check_depends
	check_dirs $BOTS_DIR $CONF_DIR
	url=$REPO/$remote_dir/$file
	output=$CONF_DIR/$file

	# check for updated blacklist
	check_version
	update=$?

	if [ $update = 1 ]; then

		# download globalblacklist update
		tmp=$(mktemp)
		mkdir -p $CONF_DIR
		local dl_msg="${BOLDWHITE}Downloading: $file "
		$CURL_PATH --fail --connect-timeout 60 --retry 10 --retry-delay 5 -so $tmp $url
		retval=$?

		case "$retval" in
			 0) print_message "$dl_msg...${BOLDGREEN}[OK]${RESET}\n\n"
			    mv $tmp $output
			    ;;
			22) printf "$dl_msg...${BOLDRED}ERROR 404: $url${RESET}\n\n";;
			28) printf "$dl_msg...${BOLDRED}ERROR TIMEOUT: $url${RESET}\n\n";;
			 *) printf "$dl_msg...${BOLDRED}ERROR CURL: ($retval){RESET}\n\n";;
		esac

		# download new bots.d / conf.d files
		$INSTALL_INC

		# set custom bots.d path
		update_paths $output

		# re-read nginx configuration
		if [ $retval = 0 ]; then

			# use full paths to workaround crontabs without $PATH configured
			if $pidof_path nginx 1>/dev/null; then

				$nginx_path -s reload 2>&1 >/dev/null

				if [ $? = 0 ]; then
					status="${BOLDGREEN}[OK]${RESET}"
					print_message "\nReloading NGINX configuration...$status\n"
				else
					status="${BOLDRED}[FAILED]${RESET}"
					printf "\nReloading NGINX configuration...$status\n"
				fi
			else
				printf "\n${BOLDRED}NGINX is not running${RESET}: not reloading NGINX config\n"
			fi
		else
			printf "\n${BOLDRED}Download failed${RESET}: not reloading NGINX config\n"
		fi

		# in silent mode print a single message after an update
		if [ "$VERBOSE" = "N" ]; then
			printf "NGINX Blacklist updated =>$(grep "Version:" $CONF_DIR/globalblacklist.conf | tr -d '#')\n"
		fi

		# enable update only email
		if [ "$SEND_EMAIL_UPDATE" = "Y" ] ; then
			SEND_EMAIL=Y
		fi

	else
		# set custom bots.d path
		update_paths $output

		# disable update only email
		if [ "$SEND_EMAIL_UPDATE" = "Y" ] ; then
			SEND_EMAIL=N
		fi
	fi

	# email report
	case "$SEND_EMAIL" in
		y*|Y*) send_email;;
	esac
	# email report via mailgun
	case "$SEND_MG_EMAIL" in
		y*|Y*) send_email_via_mailgun;;
	esac

	# log report
	case "$LOGGING" in
		y*|Y*) log_output;;
	esac
}

## start ##
EMAIL_REPORT=$(mktemp)
main $@ | tee $EMAIL_REPORT
rm -f $EMAIL_REPORT

exit $?

# Add this as a cron to run daily / weekly as you like
# Here's a sample CRON entry to update every day at 10pm
# 00 22 * * * sudo /usr/local/sbin/update-ngxblocker -q

# Here's another example to run it daily at midday using a command line switch to set the email address for the notification
# 00 12 * * * sudo /usr/local/sbin/update-ngxblocker -e yourname@youremailprovider.com

# Less verbose logging to a system alias mail address (root crontab)
# 00 12 * * * /usr/local/sbin/update-ngxblocker -q -m webmaster

# better logging for cron jobs:
# https://serverfault.com/questions/137468/better-logging-for-cronjobs-send-cron-output-to-syslog