vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-02 18:50:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Introduce new test files & generator of bad-referrers for testing REF: #270

Browse source
This commit is contained in:
Mitchell Krog 2019-06-25 13:32:19 +02:00
parent b9e0a22dc1
commit 3dc64b2c77
No known key found for this signature in database
GPG key ID: C243C388553EDE5D
4 changed files with 103 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.dev-tools/generate-files.sh
View file

@ -51,6 +51,7 @@ sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-robots.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-google-disavow.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-google-exclude.php
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-regex-format-referrers.php
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-regex-format-referrers-whitelist-test.php
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-config-readme-files.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-files-and-commit.sh
sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker.sh
@ -73,6 +74,7 @@ cd ${TRAVIS_BUILD_DIR}
# ***************************************************
php ./.dev-tools/generate-regex-format-referrers.php
php ./.dev-tools/generate-regex-format-referrers-whitelist-test.php
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/generate-blacklist.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/generate-blacklist-testing-of-changes.sh
sudo ${TRAVIS_BUILD_DIR}/.dev-tools/modify-config-readme-files.sh

95
.dev-tools/generate-regex-format-referrers-whitelist-test.php Normal file
View file

@ -0,0 +1,95 @@
<?php
/* Regex Formatting Generator Script for the Nginx Ultimate Bad Bot Blocker
* Adapted from Script Copyright (c) 2017 Stevie-Ray - https://github.com/Stevie-Ray
* Adapted by: Mitchell Krog (mitchellkrog@gmail.com) - https://github.com/mitchellkrogza
* Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
*/
namespace mitchellkrogza;
use Mso\IdnaConvert\IdnaConvert;
class Generator
{
private $projectUrl = "https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker";
public function generateFiles()
{
date_default_timezone_set('Africa/Johannesburg');
$date = date('Y-m-d H:i:s');
$lines = $this->domainWorker();
$this->createNginx($lines);
}
/**
* @return array
*/
public function domainWorker()
{
$domainsFile = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/_generator_lists/bad-referrers.list";
$handle = fopen($domainsFile, "r");
if (!$handle) {
throw new \RuntimeException('Error opening file ' . $domainsFile);
}
$lines = array();
while (($line = fgets($handle)) !== false) {
$line = trim(preg_replace('/\s\s+/', ' ', $line));
// convert internationalized domain names
if (preg_match('/[А-Яа-яЁёɢ]/u', $line)) {
$IDN = new IdnaConvert();
$line = $IDN->encode($line);
}
if (empty($line)) {
continue;
}
$lines[] = $line;
}
fclose($handle);
$uniqueLines = array_unique($lines, SORT_STRING);
sort($uniqueLines, SORT_STRING);
if (is_writable($domainsFile)) {
file_put_contents($domainsFile, implode("\n", $uniqueLines));
} else {
trigger_error("Permission denied");
}
return $lines;
}
/**
* Write to File Function
* @param $filename
* @param $data
*/
protected function writeToFile($filename, $data)
{
$file = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/referrers-regex-format-whitelist-test.txt";
$handle = fopen($file, 'w') or die('Cannot open file: '.$file);
fwrite($handle, $data);
}
/**
* @param string $date
* @param array $lines
*/
public function createNginx(array $lines)
{
$file = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/referrers-regex-format-whitelist-test.txt";
$data = "";
{
foreach ($lines as $line) {
//$data .= "\"~*\b" . preg_quote($line) . "\"\b" "\t1;\n";
$data .= '"' . '~*(?:\\' . 'b)' . preg_quote($line) . '(?:\\' . "b|)\" \t0;\n";
}
$this->writeToFile($file, $data);
}
}
}
$generator = new Generator();
$generator->generateFiles();

3
.dev-tools/test_units/blacklist-user-agents.conf
View file

@ -60,3 +60,6 @@
"~*(?:\b)Nutch(?:\b|)" 0;
# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###
# END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###

4
.dev-tools/test_units/custom-bad-referrers.conf
View file

@ -46,4 +46,6 @@
"~*(?:\b)zx6\.ru(?:\b|)" 0;
# START MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###
# END MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###