Merge srs and convert scripts

.github/workflows/create-lists.yml

@@ -2,7 +2,7 @@ name: Create lists
 
 on:
   push:
-    branches: [ "main", "categorize" ]
+    branches: [ "main", "srs" ]
     paths:
       - .github/workflows/create-lists.yml
       - Categories/**
@@ -19,14 +19,18 @@ jobs:
   generate-lists:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v4.1.4
-    - name: Generate lists
-      uses: actions/setup-python@v5.1.0
-      with:
-        python-version: '3.10' 
-    - run: |
-        pip install tldextract
-        python  convert.py
+    - uses: actions/checkout@4.1.7
+    - name: Compile ruleset srs
+      run: |
+        docker run --rm \
+        -v ${{ github.workspace }}/src:/app/src \
+        -v ${{ github.workspace }}/Russia:/app/Russia \
+        -v ${{ github.workspace }}/Ukraine:/app/Ukraine \
+        -v ${{ github.workspace }}/Categories:/app/Categories \
+        -v ${{ github.workspace }}/Services:/app/Services \
+        -v ${{ github.workspace }}/SRS:/app/SRS \
+        itdoginfo/compilesrs:0.1.1
+
     - name: Check Russia/inside-dnsmasq-ipset
       uses: itdoginfo/dnsmasq-action@0.1
       with:
@@ -57,6 +61,7 @@ jobs:
       with:
         file: "Ukraine/inside-dnsmasq-nfset.lst"
         version: 2.89
+
     - name: Push lists
       uses: EndBug/add-and-commit@v9
       with:
@@ -65,3 +70,12 @@ jobs:
         author_email: githubaction@githubaction.com
         message: 'Update lists'
         push: true
+
+    - name: Set release tag
+      run: echo "TAG_NAME=$(date +'%Y-%m-%d_%H-%M')" >> $GITHUB_ENV
+
+    - name: Release
+      uses: softprops/action-gh-release@v2.1.0
+      with:
+        files: "${{ github.workspace }}/SRS/*.srs"
+        tag_name: ${{ env.TAG_NAME }}

.github/workflows/create-srs.yml

@@ -1,35 +0,0 @@
-name: Create srs
-
-on:
-  push:
-    branches: [ "main", "srs" ]
-    paths:
-      - .github/workflows/create-srs.yml
-      - Categories/**
-      - Services/**
-      - src/**
-      - Sing-box-compile-srs/**
-
-jobs:
-  compile-and-release:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4.1.7
-
-      - name: Compile ruleset srs
-        run: |
-          docker run --rm \
-          -v ${{ github.workspace }}/Categories:/app/Categories \
-          -v ${{ github.workspace }}/Services:/app/Services \
-          -v ${{ github.workspace }}/SRS:/app/SRS \
-          itdoginfo/compilesrs:0.1
-
-      - name: Set release tag
-        run: echo "TAG_NAME=$(date +'%Y-%m-%d_%H-%M')" >> $GITHUB_ENV
-
-      - name: Release
-        uses: softprops/action-gh-release@v2.1.0
-        with:
-          files: "${{ github.workspace }}/SRS/*.srs"
-          tag_name: ${{ env.TAG_NAME }}

.gitignore

@@ -1,4 +1,5 @@
 antifilter-domains.lst
 uablacklist-domains.lst
 zaboronahelp-domains.lst
-SRS
+SRS
+JSON

Dockerfile

@@ -4,10 +4,10 @@ FROM python:3.10.16-alpine3.21
 
 COPY --from=sing-box /usr/local/bin/sing-box /bin/sing-box
 
+RUN pip install --no-cache-dir tldextract
+
 WORKDIR /app
 
-COPY compile-srs.py /app/compile-srs.py
+COPY convert.py /app/convert.py
 
-VOLUME ["/app/Categories", "/app/Services"]
-
-CMD ["python3", "compile-srs.py"]
+CMD ["python3", "convert.py"]

Russia/inside-clashx.lst

@@ -69,6 +69,7 @@ DOMAIN-SUFFIX,brawlstarsgame.com
 DOMAIN-SUFFIX,broadcom.com
 DOMAIN-SUFFIX,broncosportforum.com
 DOMAIN-SUFFIX,btdig.com
+DOMAIN-SUFFIX,btod.com
 DOMAIN-SUFFIX,buanzo.org
 DOMAIN-SUFFIX,buf.build
 DOMAIN-SUFFIX,builds.parsec.app
@@ -153,6 +154,7 @@ DOMAIN-SUFFIX,download3.omnissa.com
 DOMAIN-SUFFIX,doxa.team
 DOMAIN-SUFFIX,dpidetector.org
 DOMAIN-SUFFIX,dreamhost.com
+DOMAIN-SUFFIX,ducati.com
 DOMAIN-SUFFIX,dw.com
 DOMAIN-SUFFIX,e621.net
 DOMAIN-SUFFIX,echofm.online
@@ -688,6 +690,7 @@ DOMAIN-SUFFIX,ytimg.com
 DOMAIN-SUFFIX,ytimg.l.google.com
 DOMAIN-SUFFIX,yummyani.me
 DOMAIN-SUFFIX,zahav.ru
+DOMAIN-SUFFIX,zapier.com
 DOMAIN-SUFFIX,zbigz.com
 DOMAIN-SUFFIX,zedge.net
 DOMAIN-SUFFIX,zendesk.com

Russia/inside-dnsmasq-ipset.lst

@@ -69,6 +69,7 @@ ipset=/brawlstarsgame.com/vpn_domains
 ipset=/broadcom.com/vpn_domains
 ipset=/broncosportforum.com/vpn_domains
 ipset=/btdig.com/vpn_domains
+ipset=/btod.com/vpn_domains
 ipset=/buanzo.org/vpn_domains
 ipset=/buf.build/vpn_domains
 ipset=/builds.parsec.app/vpn_domains
@@ -153,6 +154,7 @@ ipset=/download3.omnissa.com/vpn_domains
 ipset=/doxa.team/vpn_domains
 ipset=/dpidetector.org/vpn_domains
 ipset=/dreamhost.com/vpn_domains
+ipset=/ducati.com/vpn_domains
 ipset=/dw.com/vpn_domains
 ipset=/e621.net/vpn_domains
 ipset=/echofm.online/vpn_domains
@@ -688,6 +690,7 @@ ipset=/ytimg.com/vpn_domains
 ipset=/ytimg.l.google.com/vpn_domains
 ipset=/yummyani.me/vpn_domains
 ipset=/zahav.ru/vpn_domains
+ipset=/zapier.com/vpn_domains
 ipset=/zbigz.com/vpn_domains
 ipset=/zedge.net/vpn_domains
 ipset=/zendesk.com/vpn_domains

Russia/inside-dnsmasq-nfset.lst

@@ -69,6 +69,7 @@ nftset=/brawlstarsgame.com/4#inet#fw4#vpn_domains
 nftset=/broadcom.com/4#inet#fw4#vpn_domains
 nftset=/broncosportforum.com/4#inet#fw4#vpn_domains
 nftset=/btdig.com/4#inet#fw4#vpn_domains
+nftset=/btod.com/4#inet#fw4#vpn_domains
 nftset=/buanzo.org/4#inet#fw4#vpn_domains
 nftset=/buf.build/4#inet#fw4#vpn_domains
 nftset=/builds.parsec.app/4#inet#fw4#vpn_domains
@@ -153,6 +154,7 @@ nftset=/download3.omnissa.com/4#inet#fw4#vpn_domains
 nftset=/doxa.team/4#inet#fw4#vpn_domains
 nftset=/dpidetector.org/4#inet#fw4#vpn_domains
 nftset=/dreamhost.com/4#inet#fw4#vpn_domains
+nftset=/ducati.com/4#inet#fw4#vpn_domains
 nftset=/dw.com/4#inet#fw4#vpn_domains
 nftset=/e621.net/4#inet#fw4#vpn_domains
 nftset=/echofm.online/4#inet#fw4#vpn_domains
@@ -688,6 +690,7 @@ nftset=/ytimg.com/4#inet#fw4#vpn_domains
 nftset=/ytimg.l.google.com/4#inet#fw4#vpn_domains
 nftset=/yummyani.me/4#inet#fw4#vpn_domains
 nftset=/zahav.ru/4#inet#fw4#vpn_domains
+nftset=/zapier.com/4#inet#fw4#vpn_domains
 nftset=/zbigz.com/4#inet#fw4#vpn_domains
 nftset=/zedge.net/4#inet#fw4#vpn_domains
 nftset=/zendesk.com/4#inet#fw4#vpn_domains

Russia/inside-kvas.lst

@@ -68,6 +68,7 @@ brawlstarsgame.com
 broadcom.com
 broncosportforum.com
 btdig.com
+btod.com
 buanzo.org
 buf.build
 builds.parsec.app
@@ -152,6 +153,7 @@ download3.omnissa.com
 doxa.team
 dpidetector.org
 dreamhost.com
+ducati.com
 dw.com
 e621.net
 echofm.online
@@ -688,6 +690,7 @@ ytimg.com
 ytimg.l.google.com
 yummyani.me
 zahav.ru
+zapier.com
 zbigz.com
 zedge.net
 zendesk.com

Russia/inside-mikrotik-fwd.lst

@@ -69,6 +69,7 @@
 /ip dns static add name=broadcom.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=broncosportforum.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=btdig.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
+/ip dns static add name=btod.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=buanzo.org type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=buf.build type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=builds.parsec.app type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
@@ -153,6 +154,7 @@
 /ip dns static add name=doxa.team type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=dpidetector.org type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=dreamhost.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
+/ip dns static add name=ducati.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=dw.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=e621.net type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=echofm.online type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
@@ -688,6 +690,7 @@
 /ip dns static add name=ytimg.l.google.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=yummyani.me type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=zahav.ru type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
+/ip dns static add name=zapier.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=zbigz.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=zedge.net type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost
 /ip dns static add name=zendesk.com type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost

Russia/inside-raw.lst

@@ -69,6 +69,7 @@ brawlstarsgame.com
 broadcom.com
 broncosportforum.com
 btdig.com
+btod.com
 buanzo.org
 buf.build
 builds.parsec.app
@@ -153,6 +154,7 @@ download3.omnissa.com
 doxa.team
 dpidetector.org
 dreamhost.com
+ducati.com
 dw.com
 e621.net
 echofm.online
@@ -688,6 +690,7 @@ ytimg.com
 ytimg.l.google.com
 yummyani.me
 zahav.ru
+zapier.com
 zbigz.com
 zedge.net
 zendesk.com

Sing-box-compile-srs/compile-srs.py

@@ -1,53 +0,0 @@
-#!/usr/bin/python3.10
-
-import json
-import os
-import subprocess
-
-directories = ['Categories', 'Services']
-
-output_directory = 'JSON'
-os.makedirs(output_directory, exist_ok=True)
-compiled_output_directory = 'SRS'
-os.makedirs(compiled_output_directory, exist_ok=True)
-
-for directory in directories:
-    for filename in os.listdir(directory):
-        file_path = os.path.join(directory, filename)
-        
-        if os.path.isfile(file_path):
-            domains = []
-            with open(file_path, 'r', encoding='utf-8') as file:
-                for line in file:
-                    domain = line.strip()
-                    if domain:
-                        domains.append(domain)
-
-        data = {
-            "version": 2,
-            "rules": [
-                {
-                    "domain_suffix": domains
-                }
-            ]
-        }
-
-        output_file_path = os.path.join(output_directory, f"{os.path.splitext(filename)[0]}.json")
-
-        with open(output_file_path, 'w', encoding='utf-8') as output_file:
-            json.dump(data, output_file, indent=4)
-
-        print(f"JSON file generated: {output_file_path}")
-
-print("\nCompile JSON files to .srs files...")
-for filename in os.listdir(output_directory):
-    if filename.endswith('.json'):
-        json_file_path = os.path.join(output_directory, filename)
-        srs_file_path = os.path.join(compiled_output_directory, f"{os.path.splitext(filename)[0]}.srs")
-        try:
-            subprocess.run(
-                ["sing-box", "rule-set", "compile", json_file_path, "-o", srs_file_path], check=True
-            )
-            print(f"Compiled .srs file: {srs_file_path}")
-        except subprocess.CalledProcessError as e:
-            print(f"Compile error {json_file_path}: {e}")

convert.py

@@ -4,6 +4,9 @@ import tldextract
 import urllib.request
 import re
 from pathlib import Path
+import json
+import os
+import subprocess
 
 rusDomainsInsideOut='Russia/inside'
 rusDomainsInsideSrcSingle='src/Russia-domains-inside-single.lst'
@@ -15,7 +18,7 @@ uaDomainsSrc='src/Ukraine-domains-inside.lst'
 uaDomainsOut='Ukraine/inside'
 
 def raw(src, out):
-    domains_raw = set()
+    domains = set()
     files = []
 
     if isinstance(src, list):
@@ -27,15 +30,19 @@ def raw(src, out):
                 files.append(path)
 
     for f in files:
-        with open(f) as infile:
-                for line in infile:
-                    if tldextract.extract(line).suffix:
-                        domains_raw.add(line.rstrip())
+        if f.is_file():
+            with open(f) as infile:
+                    for line in infile:
+                        if tldextract.extract(line).suffix:
+                            if re.search(r'[^а-я\-]', tldextract.extract(line).domain):
+                                domains.add(tldextract.extract(line.rstrip()).fqdn)
+                            if not tldextract.extract(line).domain and tldextract.extract(line).suffix:
+                                domains.add("." + tldextract.extract(line.rstrip()).suffix)
 
-    domains_raw = sorted(domains_raw)
+    domains = sorted(domains)
 
     with open(f'{out}-raw.lst', 'w') as file:
-        for name in domains_raw:
+        for name in domains:
             file.write(f'{name}\n')
 
 def dnsmasq(src, out, remove={'google.com'}):
@@ -158,6 +165,94 @@ def mikrotik_fwd(src, out, remove={'google.com'}):
         for name in domains:
             file.write(f'/ip dns static add name={name} type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost\n')
 
+def domains_from_file(filepath):
+    domains = []
+    try:
+        with open(filepath, 'r', encoding='utf-8') as file:
+            for line in file:
+                domain = line.strip()
+                if domain:
+                    domains.append(domain)
+    except FileNotFoundError:
+        print(f"File not found: {filepath}")
+    return domains
+
+def generate_srs(domains, output_name):
+    output_directory = 'JSON'
+    compiled_output_directory = 'SRS'
+
+    os.makedirs(output_directory, exist_ok=True)
+    os.makedirs(compiled_output_directory, exist_ok=True)
+
+    data = {
+        "version": 2,
+        "rules": [
+            {"domain_suffix": domains}
+        ]
+    }
+
+    json_file_path = os.path.join(output_directory, f"{output_name}.json")
+    srs_file_path = os.path.join(compiled_output_directory, f"{output_name}.srs")
+
+    try:
+        with open(json_file_path, 'w', encoding='utf-8') as json_file:
+            json.dump(data, json_file, indent=4)
+        print(f"JSON file generated: {json_file_path}")
+
+        subprocess.run(
+            ["sing-box", "rule-set", "compile", json_file_path, "-o", srs_file_path], check=True
+        )
+        print(f"Compiled .srs file: {srs_file_path}")
+    except subprocess.CalledProcessError as e:
+        print(f"Compile error {json_file_path}: {e}")
+    except Exception as e:
+        print(f"Error while processing {output_name}: {e}")
+
+def generate_srs_for_categories(directories, output_json_directory='JSON', compiled_output_directory='SRS'):
+    os.makedirs(output_json_directory, exist_ok=True)
+    os.makedirs(compiled_output_directory, exist_ok=True)
+
+    for directory in directories:
+        for filename in os.listdir(directory):
+            file_path = os.path.join(directory, filename)
+            
+            if os.path.isfile(file_path):
+                domains = []
+                with open(file_path, 'r', encoding='utf-8') as file:
+                    for line in file:
+                        domain = line.strip()
+                        if domain:
+                            domains.append(domain)
+
+            data = {
+                "version": 2,
+                "rules": [
+                    {
+                        "domain_suffix": domains
+                    }
+                ]
+            }
+
+            output_file_path = os.path.join(output_json_directory, f"{os.path.splitext(filename)[0]}.json")
+
+            with open(output_file_path, 'w', encoding='utf-8') as output_file:
+                json.dump(data, output_file, indent=4)
+
+            print(f"JSON file generated: {output_file_path}")
+
+    print("\nCompile JSON files to .srs files...")
+    for filename in os.listdir(output_json_directory):
+        if filename.endswith('.json'):
+            json_file_path = os.path.join(output_json_directory, filename)
+            srs_file_path = os.path.join(compiled_output_directory, f"{os.path.splitext(filename)[0]}.srs")
+            try:
+                subprocess.run(
+                    ["sing-box", "rule-set", "compile", json_file_path, "-o", srs_file_path], check=True
+                )
+                print(f"Compiled .srs file: {srs_file_path}")
+            except subprocess.CalledProcessError as e:
+                print(f"Compile error {json_file_path}: {e}")
+
 if __name__ == '__main__':
     # Russia inside
     Path("Russia").mkdir(parents=True, exist_ok=True)
@@ -189,11 +284,24 @@ if __name__ == '__main__':
     urllib.request.urlretrieve("https://raw.githubusercontent.com/zhovner/zaborona_help/master/config/domainsdb.txt", "zaboronahelp-domains.lst")
 
     ua_lists = ['uablacklist-domains.lst', 'zaboronahelp-domains.lst', uaDomainsSrc]
-
+    
+    raw(ua_lists, uaDomainsOut)
     dnsmasq(ua_lists, uaDomainsOut)
     clashx(ua_lists, uaDomainsOut)
     kvas(ua_lists, uaDomainsOut)
     mikrotik_fwd(ua_lists, uaDomainsOut)
 
     for temp_file in ['uablacklist-domains.lst', 'zaboronahelp-domains.lst']:
-        Path(temp_file).unlink()
+        Path(temp_file).unlink()
+
+    # Sing-box ruleset main
+    russia_inside = domains_from_file('Russia/inside-raw.lst')
+    russia_outside = domains_from_file('Russia/outside-raw.lst')
+    ukraine_inside = domains_from_file('Ukraine/inside-raw.lst')
+    generate_srs(russia_inside, 'russia-inside')
+    generate_srs(russia_outside, 'russia-outside')
+    generate_srs(ukraine_inside, 'ukraine-inside')
+
+    # Sing-box categories
+    directories = ['Categories', 'Services']
+    generate_srs_for_categories(directories)