diff --git a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf index 64898e68c..d45bdddbd 100644 --- a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf @@ -26,4 +26,4 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.184.226.236 0; +104.154.113.151 0; diff --git a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf index c38f89124..7a75303e4 100644 --- a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2018.04.1081 -### Updated: Tue Apr 3 09:34:44 SAST 2018 +### Version: V3.2018.04.1082 +### Updated: Tue Apr 3 09:40:52 SAST 2018 ### Bad Referrer Count: 6248 -### Bad Bot Count: 519 +### Bad Bot Count: 520 ################################################### ### VERSION INFORMATION ## @@ -292,6 +292,7 @@ map $http_user_agent $bad_bot { "~*\bCopyscape\b" 3; "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; + "~*\bcrawler4j\b" 3; "~*\bcrawler.feedback\b" 3; "~*\bCrazyWebCrawler\b" 3; "~*\bCrescent\b" 3; diff --git a/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf index 90c689df8..326edf9a5 100644 --- a/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf @@ -26,5 +26,5 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.224.112.202 0; -104.154.120.187 0; +35.192.187.174 0; +35.188.73.34 0; diff --git a/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf index 35ed6dfe2..50e70cd32 100644 --- a/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2018.04.1081 -### Updated: Tue Apr 3 09:34:44 SAST 2018 +### Version: V3.2018.04.1082 +### Updated: Tue Apr 3 09:40:52 SAST 2018 ### Bad Referrer Count: 6248 -### Bad Bot Count: 519 +### Bad Bot Count: 520 ################################################### ### VERSION INFORMATION ## @@ -292,6 +292,7 @@ map $http_user_agent $bad_bot { "~*\bCopyscape\b" 3; "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; + "~*\bcrawler4j\b" 3; "~*\bcrawler.feedback\b" 3; "~*\bCrazyWebCrawler\b" 3; "~*\bCrescent\b" 3; diff --git a/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf index 2cd0c1658..3d6d50c44 100644 --- a/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf @@ -26,5 +26,5 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.193.7.13 0; -104.154.120.187 0; +35.188.73.34 0; +35.192.187.174 0; diff --git a/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf index 7f3fdb774..462ea2d89 100644 --- a/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2018.04.1081 -### Updated: Tue Apr 3 09:34:44 SAST 2018 +### Version: V3.2018.04.1082 +### Updated: Tue Apr 3 09:40:52 SAST 2018 ### Bad Referrer Count: 6248 -### Bad Bot Count: 519 +### Bad Bot Count: 520 ################################################### ### VERSION INFORMATION ## @@ -292,6 +292,7 @@ map $http_user_agent $bad_bot { "~*\bCopyscape\b" 3; "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; + "~*\bcrawler4j\b" 3; "~*\bcrawler.feedback\b" 3; "~*\bCrazyWebCrawler\b" 3; "~*\bCrescent\b" 3; diff --git a/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf index 7694806d5..d068d5b65 100644 --- a/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf @@ -26,5 +26,5 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.188.1.99 0; -35.184.226.236 0; +104.154.113.151 0; +35.188.73.34 0; diff --git a/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf index c38f89124..7a75303e4 100644 --- a/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2018.04.1081 -### Updated: Tue Apr 3 09:34:44 SAST 2018 +### Version: V3.2018.04.1082 +### Updated: Tue Apr 3 09:40:52 SAST 2018 ### Bad Referrer Count: 6248 -### Bad Bot Count: 519 +### Bad Bot Count: 520 ################################################### ### VERSION INFORMATION ## @@ -292,6 +292,7 @@ map $http_user_agent $bad_bot { "~*\bCopyscape\b" 3; "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; + "~*\bcrawler4j\b" 3; "~*\bcrawler.feedback\b" 3; "~*\bCrazyWebCrawler\b" 3; "~*\bCrescent\b" 3; diff --git a/.dev-tools/_curl_tests/curltest1.txt b/.dev-tools/_curl_tests/curltest1.txt index 691f73292..26f5e1cc9 100755 --- a/.dev-tools/_curl_tests/curltest1.txt +++ b/.dev-tools/_curl_tests/curltest1.txt @@ -1,4 +1,4 @@ -Last Tested: Tue Apr 3 09:34:51 SAST 2018 +Last Tested: Tue Apr 3 09:40:58 SAST 2018 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest2.txt b/.dev-tools/_curl_tests/curltest2.txt index 691f73292..26f5e1cc9 100755 --- a/.dev-tools/_curl_tests/curltest2.txt +++ b/.dev-tools/_curl_tests/curltest2.txt @@ -1,4 +1,4 @@ -Last Tested: Tue Apr 3 09:34:51 SAST 2018 +Last Tested: Tue Apr 3 09:40:58 SAST 2018 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest3.txt b/.dev-tools/_curl_tests/curltest3.txt index 691f73292..26f5e1cc9 100755 --- a/.dev-tools/_curl_tests/curltest3.txt +++ b/.dev-tools/_curl_tests/curltest3.txt @@ -1,4 +1,4 @@ -Last Tested: Tue Apr 3 09:34:51 SAST 2018 +Last Tested: Tue Apr 3 09:40:58 SAST 2018 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest4.txt b/.dev-tools/_curl_tests/curltest4.txt index 691f73292..26f5e1cc9 100755 --- a/.dev-tools/_curl_tests/curltest4.txt +++ b/.dev-tools/_curl_tests/curltest4.txt @@ -1,4 +1,4 @@ -Last Tested: Tue Apr 3 09:34:51 SAST 2018 +Last Tested: Tue Apr 3 09:40:58 SAST 2018 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest5.txt b/.dev-tools/_curl_tests/curltest5.txt index e22fef318..40b184e59 100755 --- a/.dev-tools/_curl_tests/curltest5.txt +++ b/.dev-tools/_curl_tests/curltest5.txt @@ -1,4 +1,4 @@ -Last Tested: Tue Apr 3 09:34:51 SAST 2018 +Last Tested: Tue Apr 3 09:40:58 SAST 2018 diff --git a/.dev-tools/_curl_tests_4/curltest6.txt b/.dev-tools/_curl_tests_4/curltest6.txt index 1a8492dc2..7e9169597 100755 --- a/.dev-tools/_curl_tests_4/curltest6.txt +++ b/.dev-tools/_curl_tests_4/curltest6.txt @@ -1,4 +1,4 @@ -Last Tested: Tue Apr 3 09:35:30 SAST 2018 +Last Tested: Tue Apr 3 09:41:39 SAST 2018 diff --git a/.dev-tools/_curl_tests_4/curltest7.txt b/.dev-tools/_curl_tests_4/curltest7.txt index 1a8492dc2..7e9169597 100644 --- a/.dev-tools/_curl_tests_4/curltest7.txt +++ b/.dev-tools/_curl_tests_4/curltest7.txt @@ -1,4 +1,4 @@ -Last Tested: Tue Apr 3 09:35:30 SAST 2018 +Last Tested: Tue Apr 3 09:41:39 SAST 2018 diff --git a/.dev-tools/_curl_tests_4/curltest8.txt b/.dev-tools/_curl_tests_4/curltest8.txt index 1a8492dc2..7e9169597 100644 --- a/.dev-tools/_curl_tests_4/curltest8.txt +++ b/.dev-tools/_curl_tests_4/curltest8.txt @@ -1,4 +1,4 @@ -Last Tested: Tue Apr 3 09:35:30 SAST 2018 +Last Tested: Tue Apr 3 09:41:39 SAST 2018 diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index a6b264199..135f599c0 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -78,6 +78,7 @@ CopyRightCheck Copyscape Cosmos Craftbot +crawler4j crawler.feedback CrazyWebCrawler Crescent diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index c38f89124..7a75303e4 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2018.04.1081 -### Updated: Tue Apr 3 09:34:44 SAST 2018 +### Version: V3.2018.04.1082 +### Updated: Tue Apr 3 09:40:52 SAST 2018 ### Bad Referrer Count: 6248 -### Bad Bot Count: 519 +### Bad Bot Count: 520 ################################################### ### VERSION INFORMATION ## @@ -292,6 +292,7 @@ map $http_user_agent $bad_bot { "~*\bCopyscape\b" 3; "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; + "~*\bcrawler4j\b" 3; "~*\bcrawler.feedback\b" 3; "~*\bCrazyWebCrawler\b" 3; "~*\bCrescent\b" 3; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 8fd989281..e29a2710a 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index e6078a4ef..04d2ecae2 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 56f7842cd..163ed7e55 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V3.2018.04.1081 +#### Version: V3.2018.04.1082 #### Bad Referrer Count: 6248 -#### Bad Bot Count: 519 +#### Bad Bot Count: 520 ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 86c598558..08fc8ec20 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V3.2018.04.1081 +#### Version: V3.2018.04.1082 #### Bad Referrer Count: 6248 -#### Bad Bot Count: 519 +#### Bad Bot Count: 520 ____________________ ## Help Support This Project diff --git a/README.md b/README.md index fc078fa95..0b8e1110f 100755 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites. _______________ -#### Version: V3.2018.04.1081 +#### Version: V3.2018.04.1082 #### Bad Referrer Count: 6248 -#### Bad Bot Count: 519 +#### Bad Bot Count: 520 ____________________ ## Help Support This Project diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index f56bdd11f..8bb2cfb36 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,4 +1,3 @@ -crawler4j 360Spider 404checker 404enemy @@ -79,6 +78,7 @@ CopyRightCheck Copyscape Cosmos Craftbot +crawler4j crawler.feedback CrazyWebCrawler Crescent diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 3b1739f78..bd1088441 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V3.2018.04.1081 +#### Version: V3.2018.04.1082 #### Bad Referrer Count: 6248 -#### Bad Bot Count: 519 +#### Bad Bot Count: 520 ____________________ ## Help Support This Project diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index c38f89124..7a75303e4 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2018.04.1081 -### Updated: Tue Apr 3 09:34:44 SAST 2018 +### Version: V3.2018.04.1082 +### Updated: Tue Apr 3 09:40:52 SAST 2018 ### Bad Referrer Count: 6248 -### Bad Bot Count: 519 +### Bad Bot Count: 520 ################################################### ### VERSION INFORMATION ## @@ -292,6 +292,7 @@ map $http_user_agent $bad_bot { "~*\bCopyscape\b" 3; "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; + "~*\bcrawler4j\b" 3; "~*\bcrawler.feedback\b" 3; "~*\bCrazyWebCrawler\b" 3; "~*\bCrescent\b" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index c38f89124..7a75303e4 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2018.04.1081 -### Updated: Tue Apr 3 09:34:44 SAST 2018 +### Version: V3.2018.04.1082 +### Updated: Tue Apr 3 09:40:52 SAST 2018 ### Bad Referrer Count: 6248 -### Bad Bot Count: 519 +### Bad Bot Count: 520 ################################################### ### VERSION INFORMATION ## @@ -292,6 +292,7 @@ map $http_user_agent $bad_bot { "~*\bCopyscape\b" 3; "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; + "~*\bcrawler4j\b" 3; "~*\bcrawler.feedback\b" 3; "~*\bCrazyWebCrawler\b" 3; "~*\bCrescent\b" 3; diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index b79ab7be8..2419c3d5d 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -1,8 +1,8 @@ ### Version Information # ################################################### -### Version: V3.2018.04.1081 -### Updated: Tue Apr 3 09:34:44 SAST 2018 -### Bad Bot Count: 519 +### Version: V3.2018.04.1082 +### Updated: Tue Apr 3 09:40:53 SAST 2018 +### Bad Bot Count: 520 ################################################### ### Version Information ## @@ -169,6 +169,8 @@ User-agent: Cosmos Disallow:/ User-agent: Craftbot Disallow:/ +User-agent: crawler4j +Disallow:/ User-agent: crawler.feedback Disallow:/ User-agent: CrazyWebCrawler