Merge pull request #71 from itdoginfo/srs

Added sing-box rulesets files

.github/workflows/create-lists.yml

@@ -2,7 +2,7 @@ name: Create lists
 
 on:
   push:
-    branches: [ "main", "categorize" ]
+    branches: [ "main", "srs" ]
     paths:
       - .github/workflows/create-lists.yml
       - Categories/**
@@ -17,16 +17,20 @@ permissions:
 
 jobs:
   generate-lists:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v4.1.4
-    - name: Generate lists
-      uses: actions/setup-python@v5.1.0
-      with:
-        python-version: '3.10' 
-    - run: |
-        pip install tldextract
-        python  convert.py
+    - uses: actions/checkout@v4.2.2
+    - name: Compile ruleset srs
+      run: |
+        docker run --rm \
+        -v ${{ github.workspace }}/src:/app/src \
+        -v ${{ github.workspace }}/Russia:/app/Russia \
+        -v ${{ github.workspace }}/Ukraine:/app/Ukraine \
+        -v ${{ github.workspace }}/Categories:/app/Categories \
+        -v ${{ github.workspace }}/Services:/app/Services \
+        -v ${{ github.workspace }}/SRS:/app/SRS \
+        itdoginfo/compilesrs:0.1.1
+
     - name: Check Russia/inside-dnsmasq-ipset
       uses: itdoginfo/dnsmasq-action@0.1
       with:
@@ -57,6 +61,7 @@ jobs:
       with:
         file: "Ukraine/inside-dnsmasq-nfset.lst"
         version: 2.89
+
     - name: Push lists
       uses: EndBug/add-and-commit@v9
       with:
@@ -65,3 +70,12 @@ jobs:
         author_email: githubaction@githubaction.com
         message: 'Update lists'
         push: true
+
+    - name: Set release tag
+      run: echo "TAG_NAME=$(date +'%Y-%m-%d_%H-%M')" >> $GITHUB_ENV
+
+    - name: Release
+      uses: softprops/action-gh-release@v2.1.0
+      with:
+        files: "${{ github.workspace }}/SRS/*.srs"
+        tag_name: ${{ env.TAG_NAME }}

.github/workflows/create-srs.yml

@@ -1,34 +0,0 @@
-name: Convert to srs
-
-on:
-  push:
-    branches: [ "main" ]
-    paths:
-      - .github/workflows/create-srs.yml
-      # - src/**
-      # - convert.py
-  # schedule:
-  #   - cron: '33 */8 * * *'
-
-jobs:
-  convert-and-release:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4.1.7
-
-      - name: Run Plain2srs Docker Container
-        run: |
-          docker run --name plain2srs-container \
-            -v "${{ github.workspace }}:/workspace" \
-            itdoginfo/plain2srs-action:20241216 \
-            -i /workspace/Russia/inside-raw.lst \
-            -o russia-inside \
-            -c
-          docker cp plain2srs-container:/app/plain2srs/output/russia-inside.srs ${{ github.workspace }}/russia-inside.srs
-
-      - name: Release
-        uses: softprops/action-gh-release@v2.1.0
-        with:
-          files: "${{ github.workspace }}/russia-inside.srs"
-          tag_name: latest

.gitignore

@@ -1,3 +1,5 @@
 antifilter-domains.lst
 uablacklist-domains.lst
-zaboronahelp-domains.lst
+zaboronahelp-domains.lst
+SRS
+JSON

Dockerfile

@@ -4,12 +4,10 @@ FROM python:3.10.16-alpine3.21
 
 COPY --from=sing-box /usr/local/bin/sing-box /bin/sing-box
 
+RUN pip install --no-cache-dir tldextract
+
 WORKDIR /app
 
-COPY compile-srs.py /app/compile-srs.py
+COPY convert.py /app/convert.py
 
-VOLUME ["/app/Categories", "/app/Services"]
-
-CMD ["python3", "compile-srs.py"]
-
-# docker run --rm -v ./Categories:/app/Categories -v ./Services:/app/Services -v ./json:/app/json py-sg:1
+CMD ["python3", "convert.py"]

compile-srs.py

@@ -1,53 +0,0 @@
-#!/usr/bin/python3.10
-
-import json
-import os
-import subprocess
-
-directories = ['Categories', 'Services']
-
-output_directory = 'JSON'
-os.makedirs(output_directory, exist_ok=True)
-compiled_output_directory = 'SRS'
-os.makedirs(compiled_output_directory, exist_ok=True)
-
-for directory in directories:
-    for filename in os.listdir(directory):
-        file_path = os.path.join(directory, filename)
-        
-        if os.path.isfile(file_path):
-            domains = []
-            with open(file_path, 'r', encoding='utf-8') as file:
-                for line in file:
-                    domain = line.strip()
-                    if domain:
-                        domains.append(domain)
-
-        data = {
-            "version": 2,
-            "rules": [
-                {
-                    "domain_suffix": domains
-                }
-            ]
-        }
-
-        output_file_path = os.path.join(output_directory, f"{os.path.splitext(filename)[0]}.json")
-
-        with open(output_file_path, 'w', encoding='utf-8') as output_file:
-            json.dump(data, output_file, indent=4)
-
-        print(f"JSON file generated: {output_file_path}")
-
-print("\nCompile JSON files to .srs files...")
-for filename in os.listdir(output_directory):
-    if filename.endswith('.json'):
-        json_file_path = os.path.join(output_directory, filename)
-        srs_file_path = os.path.join(compiled_output_directory, f"{os.path.splitext(filename)[0]}.srs")
-        try:
-            subprocess.run(
-                ["sing-box", "rule-set", "compile", json_file_path, "-o", srs_file_path], check=True
-            )
-            print(f"Compiled .srs file: {srs_file_path}")
-        except subprocess.CalledProcessError as e:
-            print(f"Compile error {json_file_path}: {e}")

convert.py

@@ -4,6 +4,9 @@ import tldextract
 import urllib.request
 import re
 from pathlib import Path
+import json
+import os
+import subprocess
 
 rusDomainsInsideOut='Russia/inside'
 rusDomainsInsideSrcSingle='src/Russia-domains-inside-single.lst'
@@ -15,7 +18,7 @@ uaDomainsSrc='src/Ukraine-domains-inside.lst'
 uaDomainsOut='Ukraine/inside'
 
 def raw(src, out):
-    domains_raw = set()
+    domains = set()
     files = []
 
     if isinstance(src, list):
@@ -27,15 +30,19 @@ def raw(src, out):
                 files.append(path)
 
     for f in files:
-        with open(f) as infile:
-                for line in infile:
-                    if tldextract.extract(line).suffix:
-                        domains_raw.add(line.rstrip())
+        if f.is_file():
+            with open(f) as infile:
+                    for line in infile:
+                        if tldextract.extract(line).suffix:
+                            if re.search(r'[^а-я\-]', tldextract.extract(line).domain):
+                                domains.add(tldextract.extract(line.rstrip()).fqdn)
+                            if not tldextract.extract(line).domain and tldextract.extract(line).suffix:
+                                domains.add("." + tldextract.extract(line.rstrip()).suffix)
 
-    domains_raw = sorted(domains_raw)
+    domains = sorted(domains)
 
     with open(f'{out}-raw.lst', 'w') as file:
-        for name in domains_raw:
+        for name in domains:
             file.write(f'{name}\n')
 
 def dnsmasq(src, out, remove={'google.com'}):
@@ -158,6 +165,94 @@ def mikrotik_fwd(src, out, remove={'google.com'}):
         for name in domains:
             file.write(f'/ip dns static add name={name} type=FWD address-list=allow-domains match-subdomain=yes forward-to=localhost\n')
 
+def domains_from_file(filepath):
+    domains = []
+    try:
+        with open(filepath, 'r', encoding='utf-8') as file:
+            for line in file:
+                domain = line.strip()
+                if domain:
+                    domains.append(domain)
+    except FileNotFoundError:
+        print(f"File not found: {filepath}")
+    return domains
+
+def generate_srs(domains, output_name):
+    output_directory = 'JSON'
+    compiled_output_directory = 'SRS'
+
+    os.makedirs(output_directory, exist_ok=True)
+    os.makedirs(compiled_output_directory, exist_ok=True)
+
+    data = {
+        "version": 2,
+        "rules": [
+            {"domain_suffix": domains}
+        ]
+    }
+
+    json_file_path = os.path.join(output_directory, f"{output_name}.json")
+    srs_file_path = os.path.join(compiled_output_directory, f"{output_name}.srs")
+
+    try:
+        with open(json_file_path, 'w', encoding='utf-8') as json_file:
+            json.dump(data, json_file, indent=4)
+        print(f"JSON file generated: {json_file_path}")
+
+        subprocess.run(
+            ["sing-box", "rule-set", "compile", json_file_path, "-o", srs_file_path], check=True
+        )
+        print(f"Compiled .srs file: {srs_file_path}")
+    except subprocess.CalledProcessError as e:
+        print(f"Compile error {json_file_path}: {e}")
+    except Exception as e:
+        print(f"Error while processing {output_name}: {e}")
+
+def generate_srs_for_categories(directories, output_json_directory='JSON', compiled_output_directory='SRS'):
+    os.makedirs(output_json_directory, exist_ok=True)
+    os.makedirs(compiled_output_directory, exist_ok=True)
+
+    for directory in directories:
+        for filename in os.listdir(directory):
+            file_path = os.path.join(directory, filename)
+            
+            if os.path.isfile(file_path):
+                domains = []
+                with open(file_path, 'r', encoding='utf-8') as file:
+                    for line in file:
+                        domain = line.strip()
+                        if domain:
+                            domains.append(domain)
+
+            data = {
+                "version": 2,
+                "rules": [
+                    {
+                        "domain_suffix": domains
+                    }
+                ]
+            }
+
+            output_file_path = os.path.join(output_json_directory, f"{os.path.splitext(filename)[0]}.json")
+
+            with open(output_file_path, 'w', encoding='utf-8') as output_file:
+                json.dump(data, output_file, indent=4)
+
+            print(f"JSON file generated: {output_file_path}")
+
+    print("\nCompile JSON files to .srs files...")
+    for filename in os.listdir(output_json_directory):
+        if filename.endswith('.json'):
+            json_file_path = os.path.join(output_json_directory, filename)
+            srs_file_path = os.path.join(compiled_output_directory, f"{os.path.splitext(filename)[0]}.srs")
+            try:
+                subprocess.run(
+                    ["sing-box", "rule-set", "compile", json_file_path, "-o", srs_file_path], check=True
+                )
+                print(f"Compiled .srs file: {srs_file_path}")
+            except subprocess.CalledProcessError as e:
+                print(f"Compile error {json_file_path}: {e}")
+
 if __name__ == '__main__':
     # Russia inside
     Path("Russia").mkdir(parents=True, exist_ok=True)
@@ -189,11 +284,24 @@ if __name__ == '__main__':
     urllib.request.urlretrieve("https://raw.githubusercontent.com/zhovner/zaborona_help/master/config/domainsdb.txt", "zaboronahelp-domains.lst")
 
     ua_lists = ['uablacklist-domains.lst', 'zaboronahelp-domains.lst', uaDomainsSrc]
-
+    
+    raw(ua_lists, uaDomainsOut)
     dnsmasq(ua_lists, uaDomainsOut)
     clashx(ua_lists, uaDomainsOut)
     kvas(ua_lists, uaDomainsOut)
     mikrotik_fwd(ua_lists, uaDomainsOut)
 
     for temp_file in ['uablacklist-domains.lst', 'zaboronahelp-domains.lst']:
-        Path(temp_file).unlink()
+        Path(temp_file).unlink()
+
+    # Sing-box ruleset main
+    russia_inside = domains_from_file('Russia/inside-raw.lst')
+    russia_outside = domains_from_file('Russia/outside-raw.lst')
+    ukraine_inside = domains_from_file('Ukraine/inside-raw.lst')
+    generate_srs(russia_inside, 'russia-inside')
+    generate_srs(russia_outside, 'russia-outside')
+    generate_srs(ukraine_inside, 'ukraine-inside')
+
+    # Sing-box categories
+    directories = ['Categories', 'Services']
+    generate_srs_for_categories(directories)