Introduce new test files & generator of bad-referrers for testing REF: #270

.dev-tools/generate-files.sh

@@ -51,6 +51,7 @@ sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-robots.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-google-disavow.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-google-exclude.php
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-regex-format-referrers.php
+sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/generate-regex-format-referrers-whitelist-test.php
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-config-readme-files.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/modify-files-and-commit.sh
 sudo chmod +x ${TRAVIS_BUILD_DIR}/.dev-tools/test-blocker.sh
@@ -73,6 +74,7 @@ cd ${TRAVIS_BUILD_DIR}
 # ***************************************************
 
 php ./.dev-tools/generate-regex-format-referrers.php
+php ./.dev-tools/generate-regex-format-referrers-whitelist-test.php
 sudo ${TRAVIS_BUILD_DIR}/.dev-tools/generate-blacklist.sh
 sudo ${TRAVIS_BUILD_DIR}/.dev-tools/generate-blacklist-testing-of-changes.sh
 sudo ${TRAVIS_BUILD_DIR}/.dev-tools/modify-config-readme-files.sh

.dev-tools/generate-regex-format-referrers-whitelist-test.php

@@ -0,0 +1,95 @@
+<?php
+    /* Regex Formatting Generator Script for the Nginx Ultimate Bad Bot Blocker
+     * Adapted from Script Copyright (c) 2017 Stevie-Ray - https://github.com/Stevie-Ray
+     * Adapted by: Mitchell Krog (mitchellkrog@gmail.com) - https://github.com/mitchellkrogza
+     * Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+     */
+namespace mitchellkrogza;
+
+use Mso\IdnaConvert\IdnaConvert;
+
+class Generator
+{
+    
+    private $projectUrl = "https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker";
+    public function generateFiles()
+    {
+        date_default_timezone_set('Africa/Johannesburg');
+        $date = date('Y-m-d H:i:s');
+        $lines = $this->domainWorker();
+        $this->createNginx($lines);
+    }
+
+
+    /**
+     * @return array
+     */
+    public function domainWorker()
+    {
+        $domainsFile = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/_generator_lists/bad-referrers.list";
+
+        $handle = fopen($domainsFile, "r");
+        if (!$handle) {
+            throw new \RuntimeException('Error opening file ' . $domainsFile);
+        }
+        $lines = array();
+        while (($line = fgets($handle)) !== false) {
+            $line = trim(preg_replace('/\s\s+/', ' ', $line));
+
+            // convert internationalized domain names
+            if (preg_match('/[А-Яа-яЁёɢ]/u', $line)) {
+
+                $IDN = new IdnaConvert();
+
+                $line = $IDN->encode($line);
+
+            }
+
+            if (empty($line)) {
+                continue;
+            }
+            $lines[] = $line;
+        }
+        fclose($handle);
+        $uniqueLines = array_unique($lines, SORT_STRING);
+        sort($uniqueLines, SORT_STRING);
+        if (is_writable($domainsFile)) {
+            file_put_contents($domainsFile, implode("\n", $uniqueLines));
+        } else {
+            trigger_error("Permission denied");
+        }
+
+        return $lines;
+    }
+
+	/**
+     * Write to File Function
+     * @param $filename
+     * @param $data
+     */
+    protected function writeToFile($filename, $data)
+    {
+    $file = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/referrers-regex-format-whitelist-test.txt";
+	$handle = fopen($file, 'w') or die('Cannot open file:  '.$file);
+	fwrite($handle, $data);
+    }
+
+    /**
+     * @param string $date
+     * @param array  $lines
+     */
+    public function createNginx(array $lines) 
+    {
+    $file = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/referrers-regex-format-whitelist-test.txt";
+        $data = "";
+    {
+        foreach ($lines as $line) {
+            //$data .= "\"~*\b" . preg_quote($line) . "\"\b" "\t1;\n";
+            $data .= '"' . '~*(?:\\' . 'b)' . preg_quote($line) . '(?:\\' . "b|)\" \t0;\n";
+        }
+        $this->writeToFile($file, $data);
+    	}
+	}
+}
+$generator = new Generator();
+$generator->generateFiles();

.dev-tools/test_units/blacklist-user-agents.conf

@@ -60,3 +60,6 @@
 
 	"~*(?:\b)Nutch(?:\b|)"		0;
 
+# START MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###
+
+# END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ###

.dev-tools/test_units/custom-bad-referrers.conf

@@ -46,4 +46,6 @@
 
 	"~*(?:\b)zx6\.ru(?:\b|)"			0;
 
-	
+# START MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###
+
+# END MAKE BAD REFERRERS GOOD ### DO NOT EDIT THIS LINE AT ALL ###