From 84be6acbe4ddd4ccd37d1206d0356196abe24934 Mon Sep 17 00:00:00 2001
From: Mitchell Krog <mitchellkrog@gmail.com>
Date: Mon, 10 Jul 2017 18:06:21 +0200
Subject: [PATCH] Clean up generate-blacklist script.

---
 travisCI/_backup/generate-google-exclude.php | 98 --------------------
 travisCI/generate-blacklist.sh               | 25 -----
 2 files changed, 123 deletions(-)
 delete mode 100755 travisCI/_backup/generate-google-exclude.php

diff --git a/travisCI/_backup/generate-google-exclude.php b/travisCI/_backup/generate-google-exclude.php
deleted file mode 100755
index 0cd6f63a7..000000000
--- a/travisCI/_backup/generate-google-exclude.php
+++ /dev/null
@@ -1,98 +0,0 @@
-<?php
-    /* Google Exclude File Generator Script for the Nginx Ultimate Bad Bot Blocker
-     * Created by: Mitchell Krog (mitchellkrog@gmail.com)
-     * Copyright: Mitchell Krog - https://github.com/mitchellkrogza
-     * Repo Url: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
-     * MIT License
-     * Copyright (c) 2017 Mitchell Krog - mitchellkrog@gmail.com
-     */
-class Generate
-{
-    
-    private $projectUrl = "https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker";
-    public function generateFiles()
-    {
-        date_default_timezone_set('Africa/Johannesburg');
-        $date = date('Y-m-d H:i:s');
-        $lines = $this->domainWorker();
-        $this->createGoogleExclude($lines);
-    }
-    /**
-     * @return array
-     */
-    public function domainWorker()
-    {
-        $domainsFile = "/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/_generator_lists/bad-referrers.list";
-        $handle = fopen($domainsFile, "r");
-        if (!$handle) {
-            throw new \RuntimeException('Error opening file ' . $domainsFile);
-        }
-        $lines = array();
-        while (($line = fgets($handle)) !== false) {
-            $line = trim(preg_replace('/\s\s+/', ' ', $line));
-            // convert russian domains
-            if (preg_match('/[А-Яа-яЁё]/u', $line)) {
-                $IDN = new IdnaConvert();
-                $line = $IDN->encode($line);
-            }
-            if (empty($line)) {
-                continue;
-            }
-            $lines[] = $line;
-        }
-        fclose($handle);
-        $uniqueLines = array_unique($lines, SORT_STRING);
-        sort($uniqueLines, SORT_STRING);
-        if (is_writable($domainsFile)) {
-            file_put_contents($domainsFile, implode("\n", $uniqueLines));
-        } else {
-            trigger_error("Permission denied");
-        }
-        return $lines;
-    }
-    /**
-     * @param $file
-     * @param $data
-     */
-    protected function writeToFile($file, $data)
-    {
-        if (is_writable($file)) {
-            file_put_contents($file, $data);
-            if (!chmod($file, 0755)) {
-                trigger_error("Couldn't not set " . basename($file) . " permissions to 755");
-            }
-        } else {
-            trigger_error("Permission denied");
-        }
-    }
-    public function createGoogleExclude(array $lines)
-    {
-        $regexLines = [];
-        foreach ($lines as $line) {
-            $regexLines[] = preg_quote($line);
-        }
-        $data = implode('|', $regexLines);
-        $googleLimit = 30000;
-        $dataLength = strlen($data);
-        // keep track of the last split
-        $lastPosition = 0;
-        for ($x = 1; $lastPosition < $dataLength; $x++) {
-            // already in the boundary limits?
-            if( ($dataLength-$lastPosition) >= $googleLimit){
-                // search for the last occurence of | in the boundary limits
-                $pipePosition = strrpos(substr($data, $lastPosition, $googleLimit), '|');
-                $dataSplit = substr($data, $lastPosition, $pipePosition);
-                // without trailing pipe at the beginning of next round
-                $lastPosition = $lastPosition + $pipePosition+1;
-            }else{
-                // Rest of the regex (no pipe at the end)
-                $dataSplit = substr($data, $lastPosition);
-                $lastPosition = $dataLength; // Break
-            }
-            $file = '/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/google-exclude-0' . $x . '.txt';
-            $this->writeToFile($file, $dataSplit);
-        }
-    }
-}
-$generator = new Generate();
-$generator->generateFiles();
\ No newline at end of file
diff --git a/travisCI/generate-blacklist.sh b/travisCI/generate-blacklist.sh
index 547b04031..67042aef0 100755
--- a/travisCI/generate-blacklist.sh
+++ b/travisCI/generate-blacklist.sh
@@ -49,7 +49,6 @@ _input1=$TRAVIS_BUILD_DIR/_generator_lists/good-user-agents.list
 _input2=$TRAVIS_BUILD_DIR/_generator_lists/allowed-user-agents.list
 _input3=$TRAVIS_BUILD_DIR/_generator_lists/limited-user-agents.list
 _input4=$TRAVIS_BUILD_DIR/_generator_lists/bad-user-agents.list
-#_input5=$TRAVIS_BUILD_DIR/_generator_lists/bad-referrers.list
 _input5=$TRAVIS_BUILD_DIR/travisCI/referrers-regex-format.txt
 _input6=$TRAVIS_BUILD_DIR/_generator_lists/google-ip-ranges.list
 _input7=$TRAVIS_BUILD_DIR/_generator_lists/bing-ip-ranges.list
@@ -244,30 +243,6 @@ rm $_inputdb4
 # BAD REFERERS - Create and Insert
 # ********************************
 
-#printf '%s\n' "$_start5" >> "$_tmpnginx5"
-#while IFS= read -r LINE
-#do
-#printf '\t"~*%s"\t\t%s\n' "${LINE}" "$_action2" >> "$_tmpnginx5"
-#done < $_input5
-#printf '%s\n' "$_end5"  >> "$_tmpnginx5"
-#mv $_tmpnginx5 $_inputdb5
-#ed -s $_inputdb5<<\IN
-#1,/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/d
-#/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/,$d
-#,d
-#.r /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
-#/# START BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/x
-#.t.
-#.,/# END BAD REFERRERS ### DO NOT EDIT THIS LINE AT ALL ###/-d
-#w /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisCI/globalblacklist.template
-#q
-#IN
-#rm $_inputdb5
-
-# ********************************
-# BAD REFERERS - Create and Insert
-# ********************************
-
 printf '%s\n' "$_start5" >> "$_tmpnginx5"
 while IFS= read -r LINE
 do