init

.gitignore

@@ -0,0 +1,11 @@
+.idea
+*.iws
+*.iml
+*.ipr
+
+cache.db
+
+/.idea/
+/options/
+/models/
+venv

app.py

@@ -0,0 +1,73 @@
+from contextlib import asynccontextmanager
+from fastapi import FastAPI
+import uvicorn
+import logging
+
+from starlette.staticfiles import StaticFiles
+
+from app.app_core import AppCore
+from app.cuda import cuda_info
+from app.dto import TranslateReq
+from app.struct import Request
+from app.properties import Properties
+
+
+core: AppCore
+logger = logging.getLogger('uvicorn')
+
+
+@asynccontextmanager
+async def lifespan(fast_api: FastAPI):
+    cuda_info()
+    logger.info("Starting llm-translate")
+    global core
+    core = AppCore()
+    core.init_with_plugins()
+
+    yield
+    logger.info("Stopping llm-translate")
+
+
+app = FastAPI(lifespan=lifespan)
+properties = Properties()
+
+
+@app.get("/translate")
+async def translate_get(text: str, from_lang: str = "", to_lang: str = "", translator_plugin: str = ""):
+    """
+       Return translation
+
+       :param str text: text to translate
+
+       :param str from_lang: from language (2 symbols, like "en").
+       May be empty (will be replaced to "default_from_lang" from options)
+
+       :param str to_lang: to language (2 symbols, like "en").
+       May be empty (will be replaced to "default_to_lang" from options)
+
+       :param str translator_plugin: to use. If blank, default will be used.
+        If not initialized (not in "default_translate_plugin" and not in "init_on_start" from options - throw error)
+
+       :param str api_key: api key for access (if service setup in security mode with api keys)
+
+       :return: dict (result: text)
+       """
+
+    request = Request(text, from_lang, to_lang, translator_plugin)
+
+    return core.translate(request)
+
+
+@app.post("/translate")
+async def translate_post(req: TranslateReq):
+    request = Request(req.text, req.from_lang, req.to_lang, req.translator_plugin)
+    return core.translate(request)
+
+
+if __name__ == "__main__":
+    log_config = uvicorn.config.LOGGING_CONFIG
+    log_config["formatters"]["access"]["fmt"] = "%(asctime)s %(levelname)s %(message)s"
+    log_config["formatters"]["default"]["fmt"] = "%(asctime)s %(levelname)s %(message)s"
+
+    app.mount('/', StaticFiles(directory='static', html=True), name='static')
+    uvicorn.run(app, host="127.0.0.1", port=properties.port, log_level="info", log_config=log_config, use_colors=False)

app/app_core.py

@@ -0,0 +1,154 @@
+import logging
+import traceback
+
+from app import text_splitter
+from app.cache import Cache
+from app.dto import TranslateResp
+from app.struct import TranslateStruct, TranslationParams, TextSplitParams, TextProcessParams, Request, Part, \
+    CacheParams
+from app.text_processor import pre_process
+from jaa import JaaCore
+
+logger = logging.getLogger('uvicorn')
+version = "0.1.0"
+
+
+class AppCore(JaaCore):
+    def __init__(self):
+        JaaCore.__init__(self)
+
+        self.default_translate_plugin = ""
+        self.init_on_start = ""
+
+        self.translation_params = TranslationParams("", "")
+        self.text_split_params: TextSplitParams = None
+        self.text_process_params: TextProcessParams = None
+        self.cache_params: CacheParams = None
+
+        self.translators: dict = {}
+        self.initialized_translator_engines = dict()
+        self.cache: Cache = None
+
+    def process_plugin_manifest(self, modname, manifest):
+        if "translate" in manifest:  # process commands
+            for cmd in manifest["translate"].keys():
+                self.translators[cmd] = manifest["translate"][cmd]
+
+        return manifest
+
+    def init_with_plugins(self):
+        self.init_plugins(["core"])
+        self.cache = Cache(self.cache_params)
+
+        logger.info("Default translator: %s", self.default_translate_plugin)
+
+        self.init_translator_engine(self.default_translate_plugin)
+
+        init_on_start_list = self.init_on_start.replace(" ", "").split(",")
+        for translator in init_on_start_list:
+            if translator != "":
+                self.init_translator_engine(translator)
+
+        logger.info("Found translation engines: %s", ", ".join(str(key) for key in self.translators.keys()))
+
+    def init_translator_engine(self, translator_engine: str):
+        if translator_engine in self.initialized_translator_engines:
+            # already inited
+            return
+
+        try:
+            logger.info("Try to init translation plugin '%s'...", translator_engine)
+            modname = self.translators[translator_engine][0](self)
+            self.initialized_translator_engines[translator_engine] = modname
+            logger.info("Success init translation plugin: '%s'.", translator_engine)
+
+        except Exception as e:
+            logger.error("Error init translation plugin '%s'...", translator_engine, e)
+
+    def get_plugin_options(self, translator_engine: str):
+        modname = self.initialized_translator_engines[translator_engine]
+        return self.plugin_options(modname)
+
+    def get_translation_params(self, translator_engine: str):
+        options = self.get_plugin_options(translator_engine)
+        if options['translation_params_struct']:
+            return options['translation_params_struct']
+        else:
+            return self.translation_params
+
+    def get_text_split_params(self, translator_engine: str):
+        options = self.get_plugin_options(translator_engine)
+        if options['text_split_params_struct']:
+            return options['text_split_params_struct']
+        else:
+            return self.text_split_params
+
+    def get_text_process_params(self, translator_engine: str):
+        options = self.get_plugin_options(translator_engine)
+        if options['text_process_params_struct']:
+            return options['text_process_params_struct']
+        else:
+            return self.text_process_params
+
+    def translate(self, req: Request):
+        if req.text == '':
+            return TranslateResp(result='', parts=[], error=None)
+
+        try:
+            if not req.translator_plugin or req.translator_plugin == "":
+                req.translator_plugin = self.default_translate_plugin
+
+            if req.translator_plugin not in self.initialized_translator_engines:
+                raise ValueError("This translate_plugin not in initialized: " + req.translator_plugin)
+
+            if req.from_lang == "":
+                req.from_lang = self.get_translation_params(req.translator_plugin).default_from_lang
+
+            if req.to_lang == "":
+                req.to_lang = self.get_translation_params(req.translator_plugin).default_to_lang
+
+            processed_text: str
+            if self.get_text_process_params(req.translator_plugin).apply_for_request:
+                processed_text: str = pre_process(self.get_text_process_params(req.translator_plugin), req.text)
+            else:
+                processed_text = req.text
+
+            text_parts: list[Part] = text_splitter.split_text(processed_text,
+                                                              self.get_text_split_params(req.translator_plugin),
+                                                              req.from_lang)
+            self.cache_read(req, text_parts)
+
+            translate_struct = TranslateStruct(req=req, processed_text=processed_text, parts=text_parts)
+
+            translate_struct: TranslateStruct = self.translators[req.translator_plugin][1](self, translate_struct)
+            self.cache_write(req, translate_struct.parts)
+
+            (translate_text, translate_parts) = text_splitter.join_text(translate_struct.parts)
+
+            if self.text_process_params.apply_for_response:
+                translate_text: str = pre_process(self.text_process_params, translate_text)
+            else:
+                translate_text = req.text
+
+            return TranslateResp(result=translate_text, parts=translate_parts, error=None)
+        except ValueError as ve:
+            return TranslateResp(result=None, parts=None, error=ve.args[0])
+        except Exception as e:
+            traceback.print_tb(e.__traceback__, limit=10)
+            return TranslateResp(result=None, parts=None, error=getattr(e, 'message', repr(e)))
+
+    def cache_read(self, req: Request, parts: list[Part]):
+        if self.cache_params.enabled and req.translator_plugin not in self.cache_params.disable_for_plugins:
+            for part in parts:
+                cached_translate = self.cache.get(req, part.text)
+                if cached_translate:
+                    part.cache_found = True
+                    part.translate = cached_translate
+                else:
+                    part.cache_found = False
+
+    def cache_write(self, req: Request, parts: list[Part]):
+        if self.cache_params.enabled and req.translator_plugin not in self.cache_params.disable_for_plugins:
+            for part in parts:
+                if not part.cache_found:
+                    self.cache.put(req, part.text, part.translate)

app/cache.py

@@ -0,0 +1,75 @@
+import logging
+import sqlite3
+
+from app.struct import CacheParams, Request
+
+logger = logging.getLogger('uvicorn')
+
+
+class Cache:
+    cache_table_name = "cache_translate"
+    params: CacheParams
+    connection: sqlite3.Connection
+
+    def __init__(self, params: CacheParams):
+        self.params = params
+        self.connection = self.get_connection()
+        self.init()
+
+    def get_connection(self):
+        return sqlite3.connect(self.params.file)
+
+    def init(self):
+        if not self.params.enabled:
+            return None
+
+        cursor = self.connection.cursor()
+        cursor.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name='{0}'".format(self.cache_table_name))
+        table_exists = cursor.fetchall()
+
+        if len(table_exists) == 0:
+            logger.info("Init cache table: %s, file db: %s", self.cache_table_name, self.params.file)
+            create_table = """
+                CREATE TABLE IF NOT EXISTS {0} 
+                (key TEXT NOT NULL, created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
+                from_lang TEXT NOT NULL, to_lang TEXT NOT NULL, plugin TEXT NOT NULL, value TEXT NOT NULL) 
+            """.format(self.cache_table_name)
+            create_idx_translate_cols = ('CREATE UNIQUE INDEX IF NOT EXISTS idx_translate_cols '
+                                         'ON {0} (key, from_lang, to_lang, plugin)').format(self.cache_table_name)
+            create_idx_created = ('CREATE INDEX IF NOT EXISTS idx_created '
+                                  'ON {0} (created)').format(self.cache_table_name)
+
+            with cursor:
+                cursor.execute(create_table)
+                cursor.execute(create_idx_translate_cols)
+                cursor.execute(create_idx_created)
+        else:
+            if (self.params.expire_days > 0):
+                delete_expired_values = "DELETE FROM {0} WHERE created < date('now', '-{1} day')".format(
+                    self.cache_table_name, self.params.expire_days)
+                cursor.execute(delete_expired_values)
+
+        self.connection.commit()
+
+    def get(self, req: Request, text: str):
+        select = "SELECT value FROM {0} WHERE key = ? AND from_lang = ? AND to_lang = ? AND plugin = ?".format(
+            self.cache_table_name)
+        cursor = self.connection.cursor()
+        cursor.execute(select, (text, req.from_lang, req.to_lang, req.translator_plugin))
+        value = cursor.fetchone()
+        if value:
+            return value[0]
+        else:
+            return None
+
+    def put(self, req: Request, text: str, value: str):
+        try:
+            insert_connection = self.get_connection()
+            cursor = insert_connection.cursor()
+            cursor.execute('INSERT INTO {0} (KEY, from_lang, to_lang, plugin, VALUE) VALUES (?, ?, ?, ?, ?)'.format(
+                self.cache_table_name),(text, req.from_lang, req.to_lang, req.translator_plugin, value))
+            insert_connection.commit()
+            insert_connection.close()
+        except Exception as e:
+            logger.error("Error save cache entry, text = %s, req = %s, error=%s", text, req, e)

app/cuda.py

@@ -0,0 +1,32 @@
+import logging
+
+import torch
+
+logger = logging.getLogger('uvicorn')
+
+
+def cuda_info():
+    cuda_is_available = torch.cuda.is_available()
+    device_count = torch.cuda.device_count()
+    current_device = torch.cuda.current_device()
+
+    logger.info("CUDA info: is available={0}, device count={1}, current_device={2}. Devices:"
+                .format(cuda_is_available, device_count, current_device))
+
+    for i in range(device_count):
+        logger.info("GPU #%d: %s", i, torch.cuda.get_device_name(i))
+
+
+def get_device(options: dict):
+    cuda_opt = options["cuda"]
+    if cuda_opt:
+        return "cuda"
+    else:
+        return "cpu"
+
+def get_device_with_gpu_num(options: dict):
+    cuda_opt = options["cuda"]
+    if cuda_opt:
+        return "cuda:{0}".format(options["cuda_device_index"])
+    else:
+        return "cpu"

app/dto.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+
+from pydantic import BaseModel
+
+
+class TranslateReq(BaseModel):
+    text: str
+    from_lang: str | None = ""
+    to_lang: str | None = ""
+    translator_plugin: str | None = ""
+
+
+@dataclass
+class TranslatePart:
+    text: str
+    translate: str
+    paragraph_end: bool
+
+
+@dataclass
+class TranslateResp:
+    result: str | None
+    parts: list[TranslatePart] | None
+    error: str | None
+

app/lang_dict.py

@@ -0,0 +1,368 @@
+lang_2_chars_to_name: dict = {
+    'ab': 'abkhazian',
+    'aa': 'afar',
+    'af': 'afrikaans',
+    'sq': 'albanian',
+    'am': 'amharic',
+    'ar': 'arabic',
+    'hy': 'armenian',
+    'as': 'assamese',
+    'ay': 'aymara',
+    'az': 'azerbaijani',
+    'ba': 'bashkir',
+    'eu': 'basque',
+    'bn': 'bengali',
+    'dz': 'bhutani',
+    'bh': 'bihari',
+    'bi': 'bislama',
+    'br': 'breton',
+    'bg': 'bulgarian',
+    'my': 'burmese',
+    'be': 'byelorussian',
+    'km': 'cambodian',
+    'ca': 'catalan',
+    'zh': 'chinese',
+    'co': 'corsican',
+    'hr': 'croatian',
+    'cs': 'czech',
+    'da': 'danish',
+    'nl': 'dutch',
+    'en': 'english',
+    'eo': 'esperanto',
+    'et': 'estonian',
+    'fo': 'faeroese',
+    'fj': 'fiji',
+    'fi': 'finnish',
+    'fr': 'french',
+    'fy': 'frisian',
+    'gd': 'gaelic',
+    'gl': 'galician',
+    'ka': 'georgian',
+    'de': 'german',
+    'el': 'greek',
+    'kl': 'greenlandic',
+    'gn': 'guarani',
+    'gu': 'gujarati',
+    'ha': 'hausa',
+    'iw': 'hebrew',
+    'hi': 'hindi',
+    'hu': 'hungarian',
+    'is': 'icelandic',
+    'in': 'indonesian',
+    'ia': 'interlingua',
+    'ie': 'interlingue',
+    'ik': 'inupiak',
+    'ga': 'irish',
+    'it': 'italian',
+    'ja': 'japanese',
+    'jw': 'javanese',
+    'kn': 'kannada',
+    'ks': 'kashmiri',
+    'kk': 'kazakh',
+    'rw': 'kinyarwanda',
+    'ky': 'kirghiz',
+    'rn': 'kirundi',
+    'ko': 'korean',
+    'ku': 'kurdish',
+    'lo': 'laothian',
+    'la': 'latin',
+    'lv': 'latvian',
+    'ln': 'lingala',
+    'lt': 'lithuanian',
+    'mk': 'macedonian',
+    'mg': 'malagasy',
+    'ms': 'malay',
+    'ml': 'malayalam',
+    'mt': 'maltese',
+    'mi': 'maori',
+    'mr': 'marathi',
+    'mo': 'moldavian',
+    'mn': 'mongolian',
+    'na': 'nauru',
+    'ne': 'nepali',
+    'no': 'norwegian',
+    'oc': 'occitan',
+    'or': 'oriya',
+    'om': 'oromo',
+    'ps': 'pashto',
+    'fa': 'persian',
+    'pl': 'polish',
+    'pt': 'portuguese',
+    'pa': 'punjabi',
+    'qu': 'quechua',
+    'rm': 'rhaeto-romance',
+    'ro': 'romanian',
+    'ru': 'russian',
+    'sm': 'samoan',
+    'sg': 'sangro',
+    'sa': 'sanskrit',
+    'sr': 'serbian',
+    'sh': 'serbo-croatian',
+    'st': 'sesotho',
+    'tn': 'setswana',
+    'sn': 'shona',
+    'sd': 'sindhi',
+    'si': 'singhalese',
+    'ss': 'siswati',
+    'sk': 'slovak',
+    'sl': 'slovenian',
+    'so': 'somali',
+    'es': 'spanish',
+    'su': 'sudanese',
+    'sw': 'swahili',
+    'sv': 'swedish',
+    'tl': 'tagalog',
+    'tg': 'tajik',
+    'ta': 'tamil',
+    'tt': 'tatar',
+    'te': 'tegulu',
+    'th': 'thai',
+    'bo': 'tibetan',
+    'ti': 'tigrinya',
+    'to': 'tonga',
+    'ts': 'tsonga',
+    'tr': 'turkish',
+    'tk': 'turkmen',
+    'tw': 'twi',
+    'uk': 'ukrainian',
+    'ur': 'urdu',
+    'uz': 'uzbek',
+    'vi': 'vietnamese',
+    'vo': 'volapuk',
+    'cy': 'welsh',
+    'wo': 'wolof',
+    'xh': 'xhosa',
+    'ji': 'yiddish',
+    'yo': 'yoruba',
+    'zu': 'zulu',
+}
+
+
+def get_lang_by_2_chars_code(code: str):
+    result = lang_2_chars_to_name.get(code)
+    if result:
+        return result
+    else:
+        return code
+
+
+nllb_200_langlist_str = """
+ace_Arab    | Acehnese (Arabic script)
+ace_Latn    | Acehnese (Latin script)
+acm_Arab    | Mesopotamian Arabic
+acq_Arab    | Ta’izzi-Adeni Arabic
+aeb_Arab    | Tunisian Arabic
+afr_Latn    | Afrikaans
+ajp_Arab    | South Levantine Arabic
+aka_Latn    | Akan
+als_Latn    | Tosk Albanian
+amh_Ethi    | Amharic
+apc_Arab    | North Levantine Arabic
+arb_Arab    | Modern Standard Arabic
+arb_Latn    | Modern Standard Arabic (Romanized)
+ars_Arab    | Najdi Arabic
+ary_Arab    | Moroccan Arabic
+arz_Arab    | Egyptian Arabic
+asm_Beng    | Assamese
+ast_Latn    | Asturian
+awa_Deva    | Awadhi
+ayr_Latn    | Central Aymara
+azb_Arab    | South Azerbaijani
+azj_Latn    | North Azerbaijani
+bak_Cyrl    | Bashkir
+bam_Latn    | Bambara
+ban_Latn    | Balinese
+bel_Cyrl    | Belarusian
+bem_Latn    | Bemba
+ben_Beng    | Bengali
+bho_Deva    | Bhojpuri
+bjn_Arab    | Banjar (Arabic script)
+bjn_Latn    | Banjar (Latin script)
+bod_Tibt    | Standard Tibetan
+bos_Latn    | Bosnian
+bug_Latn    | Buginese
+bul_Cyrl    | Bulgarian
+cat_Latn    | Catalan
+ceb_Latn    | Cebuano
+ces_Latn    | Czech
+cjk_Latn    | Chokwe
+ckb_Arab    | Central Kurdish
+crh_Latn    | Crimean Tatar
+cym_Latn    | Welsh
+dan_Latn    | Danish
+deu_Latn    | German
+dik_Latn    | Southwestern Dinka
+dyu_Latn    | Dyula
+dzo_Tibt    | Dzongkha
+ell_Grek    | Greek
+eng_Latn    | English
+epo_Latn    | Esperanto
+est_Latn    | Estonian
+eus_Latn    | Basque
+ewe_Latn    | Ewe
+fao_Latn    | Faroese
+fij_Latn    | Fijian
+fin_Latn    | Finnish
+fon_Latn    | Fon
+fra_Latn    | French
+fur_Latn    | Friulian
+fuv_Latn    | Nigerian Fulfulde
+gaz_Latn    | West Central Oromo
+gla_Latn    | Scottish Gaelic
+gle_Latn    | Irish
+glg_Latn    | Galician
+grn_Latn    | Guarani
+guj_Gujr    | Gujarati
+hat_Latn    | Haitian Creole
+hau_Latn    | Hausa
+heb_Hebr    | Hebrew
+hin_Deva    | Hindi
+hne_Deva    | Chhattisgarhi
+hrv_Latn    | Croatian
+hun_Latn    | Hungarian
+hye_Armn    | Armenian
+ibo_Latn    | Igbo
+ilo_Latn    | Ilocano
+ind_Latn    | Indonesian
+isl_Latn    | Icelandic
+ita_Latn    | Italian
+jav_Latn    | Javanese
+jpn_Jpan    | Japanese
+kab_Latn    | Kabyle
+kac_Latn    | Jingpho
+kam_Latn    | Kamba
+kan_Knda    | Kannada
+kas_Arab    | Kashmiri (Arabic script)
+kas_Deva    | Kashmiri (Devanagari script)
+kat_Geor    | Georgian
+kaz_Cyrl    | Kazakh
+kbp_Latn    | Kabiyè
+kea_Latn    | Kabuverdianu
+khk_Cyrl    | Halh Mongolian
+khm_Khmr    | Khmer
+kik_Latn    | Kikuyu
+kin_Latn    | Kinyarwanda
+kir_Cyrl    | Kyrgyz
+kmb_Latn    | Kimbundu
+kmr_Latn    | Northern Kurdish
+knc_Arab    | Central Kanuri (Arabic script)
+knc_Latn    | Central Kanuri (Latin script)
+kon_Latn    | Kikongo
+kor_Hang    | Korean
+lao_Laoo    | Lao
+lij_Latn    | Ligurian
+lim_Latn    | Limburgish
+lin_Latn    | Lingala
+lit_Latn    | Lithuanian
+lmo_Latn    | Lombard
+ltg_Latn    | Latgalian
+ltz_Latn    | Luxembourgish
+lua_Latn    | Luba-Kasai
+lug_Latn    | Ganda
+luo_Latn    | Luo
+lus_Latn    | Mizo
+lvs_Latn    | Standard Latvian
+mag_Deva    | Magahi
+mai_Deva    | Maithili
+mal_Mlym    | Malayalam
+mar_Deva    | Marathi
+min_Arab    | Minangkabau (Arabic script)
+min_Latn    | Minangkabau (Latin script)
+mkd_Cyrl    | Macedonian
+mlt_Latn    | Maltese
+mni_Beng    | Meitei (Bengali script)
+mos_Latn    | Mossi
+mri_Latn    | Maori
+mya_Mymr    | Burmese
+nld_Latn    | Dutch
+nno_Latn    | Norwegian Nynorsk
+nob_Latn    | Norwegian Bokmål
+npi_Deva    | Nepali
+nso_Latn    | Northern Sotho
+nus_Latn    | Nuer
+nya_Latn    | Nyanja
+oci_Latn    | Occitan
+ory_Orya    | Odia
+pag_Latn    | Pangasinan
+pan_Guru    | Eastern Panjabi
+pap_Latn    | Papiamento
+pbt_Arab    | Southern Pashto
+pes_Arab    | Western Persian
+plt_Latn    | Plateau Malagasy
+pol_Latn    | Polish
+por_Latn    | Portuguese
+prs_Arab    | Dari
+quy_Latn    | Ayacucho Quechua
+ron_Latn    | Romanian
+run_Latn    | Rundi
+rus_Cyrl    | Russian
+sag_Latn    | Sango
+san_Deva    | Sanskrit
+sat_Olck    | Santali
+scn_Latn    | Sicilian
+shn_Mymr    | Shan
+sin_Sinh    | Sinhala
+slk_Latn    | Slovak
+slv_Latn    | Slovenian
+smo_Latn    | Samoan
+sna_Latn    | Shona
+snd_Arab    | Sindhi
+som_Latn    | Somali
+sot_Latn    | Southern Sotho
+spa_Latn    | Spanish
+srd_Latn    | Sardinian
+srp_Cyrl    | Serbian
+ssw_Latn    | Swati
+sun_Latn    | Sundanese
+swe_Latn    | Swedish
+swh_Latn    | Swahili
+szl_Latn    | Silesian
+tam_Taml    | Tamil
+taq_Latn    | Tamasheq (Latin script)
+taq_Tfng    | Tamasheq (Tifinagh script)
+tat_Cyrl    | Tatar
+tel_Telu    | Telugu
+tgk_Cyrl    | Tajik
+tgl_Latn    | Tagalog
+tha_Thai    | Thai
+tir_Ethi    | Tigrinya
+tpi_Latn    | Tok Pisin
+tsn_Latn    | Tswana
+tso_Latn    | Tsonga
+tuk_Latn    | Turkmen
+tum_Latn    | Tumbuka
+tur_Latn    | Turkish
+twi_Latn    | Twi
+tzm_Tfng    | Central Atlas Tamazight
+uig_Arab    | Uyghur
+ukr_Cyrl    | Ukrainian
+umb_Latn    | Umbundu
+urd_Arab    | Urdu
+uzn_Latn    | Northern Uzbek
+vec_Latn    | Venetian
+vie_Latn    | Vietnamese
+war_Latn    | Waray
+wol_Latn    | Wolof
+xho_Latn    | Xhosa
+ydd_Hebr    | Eastern Yiddish
+yor_Latn    | Yoruba
+yue_Hant    | Yue Chinese
+zho_Hans    | Chinese (Simplified)
+zho_Hant    | Chinese (Traditional)
+zsm_Latn    | Standard Malay
+zul_Latn    | Zulu
+"""
+
+
+#  SEE LANGUAGE CODES in transformers_fb_nllb200distilled600M_text.md
+# need to extend this dict from nllb_200_langlist_str
+lang_2_chars_to_nllb_lang: dict = {
+    'zh': 'zho_Hant',
+    'en': 'eng_Latn',
+    'fr': 'fra_Latn',
+    'de': 'deu_Latn',
+    'it': 'ita_Latn',
+    'ja': 'jpn_Jpan',
+    'ru': 'rus_Cyrl',
+    'es': 'spa_Latn',
+}

app/properties.py

@@ -0,0 +1,11 @@
+import json
+
+
+class Properties:
+    def __init__(self, file="properties.json"):
+        with open(file) as file:
+            data = json.load(file)
+
+            self.port = data['port']
+
+

app/struct.py

@@ -0,0 +1,168 @@
+from dataclasses import dataclass, field
+
+
+# dict_field: dict = field(default_factory=lambda: {})
+@dataclass
+class Request:
+    text: str
+    from_lang: str | None
+    to_lang: str | None
+    translator_plugin: str | None
+
+
+@dataclass
+class Sentence:
+    text: str
+
+
+@dataclass
+class Part:
+    text: str
+    translate: str
+    paragraph_end: bool
+    cache_found: bool
+
+    def need_to_translate(self):
+        return not self.cache_found and self.text and self.text != ""
+
+    def __init__(self, text: str, paragraph_end: bool):
+        self.text = text
+        self.translate = ""
+        self.paragraph_end = paragraph_end
+        self.cache_found = False
+
+
+@dataclass
+class TranslateStruct:
+    req: Request
+    processed_text: str
+    parts: list[Part]
+
+
+@dataclass
+class TranslationParams:
+    default_from_lang: str
+    default_to_lang: str
+
+
+@dataclass
+class TextSplitParams:
+    split_by_paragraphs_and_length: bool
+    split_by_sentences_and_length: bool
+    split_expected_length: int
+
+    split_by_paragraphs_only: bool
+    split_by_sentences_only: bool
+
+    # pysbd (default) / blingfire
+    sentence_splitter: str
+
+    def split_enabled(self):
+        return (self.split_by_paragraphs_only or self.split_by_paragraphs_and_length
+                or self.split_by_sentences_and_length or self.split_by_sentences_only)
+
+
+@dataclass
+class TextProcessParams:
+    apply_for_request: bool
+    apply_for_response: bool
+
+    replace_non_standard_new_lines_chars: bool
+    replace_not_text_chars: bool
+    allowed_chars_ignoring_replace: set
+    replace_not_text_target_char: str
+
+    remove_identical_characters: bool
+    remove_identical_characters_extra_chars: str
+    remove_identical_characters_max_repeats: int
+
+    remove_multiple_spaces: bool
+    replace_text_from_to: dict
+
+
+@dataclass
+class CacheParams:
+    enabled: bool
+    file: str
+    disable_for_plugins: list[str]
+    expire_days: int
+
+
+@dataclass
+class TranslateProgress:
+    unit: str
+    ascii: bool
+    desc: str
+
+
+tp: TranslateProgress = TranslateProgress(unit="part", ascii=True, desc="translate parts: ")
+
+
+def read_plugin_params(manifest: dict):
+    manifest["options"]["translation_params_struct"] = read_translation_params(manifest)
+    manifest["options"]["text_split_params_struct"] = read_text_split_params(manifest)
+    manifest["options"]["text_process_params_struct"] = read_text_process_params(manifest)
+
+
+def read_translation_params(manifest: dict):
+    options = manifest["options"]
+    if "translation_params" not in options:
+        return None
+
+    return TranslationParams(
+        default_from_lang=options["translation_params"]["default_from_lang"],
+        default_to_lang=options["translation_params"]["default_to_lang"]
+    )
+
+
+def read_text_split_params(manifest: dict):
+    options = manifest["options"]
+
+    if "text_split_params" not in options:
+        return None
+
+    return TextSplitParams(
+        split_by_paragraphs_and_length=options["text_split_params"].get("split_by_paragraphs_and_length", False),
+        split_by_sentences_and_length=options["text_split_params"].get("split_by_sentences_and_length", False),
+        split_expected_length=options["text_split_params"].get("split_expected_length", 1000),
+
+        split_by_paragraphs_only=options["text_split_params"].get("split_by_paragraphs_only", False),
+        split_by_sentences_only=options["text_split_params"].get("split_by_sentences_only", False),
+
+        sentence_splitter=options["text_split_params"].get("sentence_splitter", "default"),
+    )
+
+
+def read_text_process_params(manifest: dict):
+    options = manifest["options"]
+
+    if "text_processing_params" not in options:
+        return None
+
+    return TextProcessParams(
+        apply_for_request=options["text_processing_params"].get("apply_for_request", True),
+        apply_for_response=options["text_processing_params"].get("apply_for_response", True),
+
+        replace_non_standard_new_lines_chars=options["text_processing_params"].get("replace_non_standard_new_lines_chars", True),
+        replace_not_text_chars=options["text_processing_params"].get("replace_not_text_chars", False),
+        allowed_chars_ignoring_replace=options["text_processing_params"].get("allowed_chars_ignoring_replace", " .,<>:;\"'-–…?!#@№$%+/\\^&[]=*()«»—\r\t\n"),
+        replace_not_text_target_char=options["text_processing_params"].get("replace_not_text_target_char", " "),
+
+        remove_identical_characters=options["text_processing_params"].get("remove_identical_characters", False),
+        remove_identical_characters_extra_chars=options["text_processing_params"].get("remove_identical_characters_extra_chars", ""),
+        remove_identical_characters_max_repeats=options["text_processing_params"].get("remove_identical_characters_max_repeats", 3),
+
+        remove_multiple_spaces=options["text_processing_params"].get("remove_multiple_spaces", True),
+        replace_text_from_to=options["text_processing_params"].get("replace_text_from_to", {}),
+    )
+
+
+def read_cache_params(manifest: dict):
+    options = manifest["options"]
+
+    return CacheParams(
+        enabled=options["cache_params"]["enabled"],
+        file=options["cache_params"]["file"],
+        disable_for_plugins=options["cache_params"]["disable_for_plugins"],
+        expire_days=options["cache_params"]["expire_days"],
+    )

app/text_processor.py

@@ -0,0 +1,73 @@
+import logging
+import re
+
+from app.struct import TextProcessParams
+
+
+logger = logging.getLogger('uvicorn')
+
+
+def pre_process(params: TextProcessParams, original_text: str):
+    processed_text = replace_text_from_to(original_text, params.replace_text_from_to)
+
+    if params.replace_non_standard_new_lines_chars:
+        processed_text = replace_non_standard_new_lines_chars(processed_text)
+
+    if params.remove_multiple_spaces:
+        processed_text = remove_multiple_spaces(processed_text)
+
+    if params.replace_not_text_chars:
+        processed_text = replace_not_text_chars(
+            original_text, params.allowed_chars_ignoring_replace, params.replace_not_text_target_char)
+
+    if params.remove_identical_characters:
+        processed_text = remove_identical_characters(
+            processed_text, params.remove_identical_characters_max_repeats,
+            params.remove_identical_characters_extra_chars)
+
+    return processed_text
+
+
+def replace_not_text_chars(text: str, allowed_chars_ignoring_replace: set, replace_not_text_target_char: str):
+    result = ""
+    replaced_chars = []
+    for char in text:
+        if char.isalpha() or char.isdigit() or char in allowed_chars_ignoring_replace:
+            result = result + char
+        else:
+            result = result + replace_not_text_target_char
+            replaced_chars.append(char)
+
+    if len(replaced_chars) > 0:
+        replaced_chars_set = set(replaced_chars)
+        logger.info("Replaced chars in text {0}: {1}".format(text, replaced_chars_set))
+
+    return result
+
+
+def replace_non_standard_new_lines_chars(text: str):
+    return text.replace("\r\n", "\n").replace("\n\r", "\n").replace("\r", "\n")
+
+
+def remove_identical_characters(text: str,
+                                remove_identical_characters_max_repeats: int,
+                                remove_identical_characters_extra_chars: str):
+    chars = 'A-zА-яЁё' + remove_identical_characters_extra_chars
+    regexp = re.compile(r'([%s])\1{%d,}' % (chars, remove_identical_characters_max_repeats))
+    return re.sub(regexp, r'\1' * remove_identical_characters_max_repeats, text)
+
+
+def remove_multiple_spaces(text: str):
+    while '  ' in text:
+        text = text.replace('  ', ' ')
+
+    return text
+
+
+def replace_text_from_to(text: str, from_to: dict | None):
+    if from_to and len(from_to) > 0:
+        for key, value in from_to.items():
+            text = text.replace(key, value)
+
+    return text
+

app/text_splitter.py

@@ -0,0 +1,128 @@
+import pysbd
+from blingfire import text_to_sentences
+
+from app.dto import TranslatePart
+from app.struct import TextSplitParams, Part
+
+
+def is_arr_fin(arr: list, i):
+    return len(arr) - 1 == i
+
+
+def split_by_sentences(text: str, split_params: TextSplitParams, language="en"):
+    if split_params.sentence_splitter == 'blingfire':
+        return text_to_sentences(text).splitlines(False)
+    else:
+        seg = pysbd.Segmenter(language=language, clean=False)
+        result = []
+        for s in seg.segment(text):
+            result.append(s.strip())
+
+        return result
+
+
+def split_text(processed_text: str, split_params: TextSplitParams, language="en"):
+    result: list[Part] = []
+
+    # no split
+    if not split_params.split_enabled():
+        part = Part(processed_text, False)
+        result.append(part)
+
+        return result
+    else:
+        # split by paragraphs
+        paragraphs = processed_text.splitlines(False)
+
+        if split_params.split_by_paragraphs_and_length:
+            parts_in_paragraph: list[list[str]] = []
+            part_length = 0
+            # current_part = 0
+            part_in_paragraph: list[str] = []
+            for i, paragraph in enumerate(paragraphs):
+                if part_length == 0 or part_length + len(paragraph) <= split_params.split_expected_length:
+                    part_in_paragraph.append(paragraph)
+                    part_length += len(paragraph)
+                else:
+                    parts_in_paragraph.append(part_in_paragraph)
+                    part_in_paragraph = [paragraph]
+                    part_length = len(paragraph)
+
+            parts_in_paragraph.append(part_in_paragraph)
+
+            for part_in_paragraph in parts_in_paragraph:
+                result.append(Part("\n".join(part_in_paragraph), True))
+
+            return result
+
+        if split_params.split_by_paragraphs_only:
+            for i, paragraph in enumerate(paragraphs):
+                part = Part(paragraph, not is_arr_fin(paragraphs, i))
+                result.append(part)
+
+            return result
+
+        # split by sentences
+        sentences_in_paragraph: list[list[str]] = []
+        for paragraph in paragraphs:
+            sentences = split_by_sentences(paragraph, split_params, language)
+            sentences_in_paragraph.append(sentences)
+
+        if split_params.split_by_sentences_and_length:
+            sentences_part = ""
+            arr_fin = False
+            for i, sentences_list in enumerate(sentences_in_paragraph):
+                if len(sentences_list) == 0:
+                    sentences_part = sentences_part + "\n"
+                for j, sentence in enumerate(sentences_list):
+                    if not sentence:
+                        sentences_part = sentences_part + "\n"
+                        pass
+                    if sentence == "" or len(sentences_part) + len(sentence) <= split_params.split_expected_length:
+                        if is_arr_fin(sentences_in_paragraph, i) and is_arr_fin(sentences_list, j):
+                            sentences_part = sentences_part + sentence
+                        elif j == len(sentences_list) - 1:
+                            sentences_part = sentences_part + sentence + "\n"
+                        else:
+                            sentences_part = sentences_part + sentence + " "
+                    else:
+                        part = Part(sentences_part.strip(), arr_fin)
+                        result.append(part)
+                        if is_arr_fin(sentences_in_paragraph, i) and is_arr_fin(sentences_list, j):
+                            sentences_part = sentence
+                        elif is_arr_fin(sentences_list, j):
+                            sentences_part = sentence + "\n"
+                        else:
+                            sentences_part = sentence + " "
+                    arr_fin = is_arr_fin(sentences_list, j)
+
+            result.append(Part(sentences_part, False))
+
+            return result
+
+        if split_params.split_by_sentences_only:
+            for i, sentences_list in enumerate(sentences_in_paragraph):
+                if len(sentences_list) == 0:
+                    result.append(Part("", True))
+                for j, sentence in enumerate(sentences_list):
+                    result.append(Part(sentence, is_arr_fin(sentences_list, j) and not is_arr_fin(sentences_in_paragraph, i)))
+
+            return result
+
+        raise ValueError("Incorrect split params - can't split: " + str(split_params))
+
+
+def join_text(parts: list[Part]):
+    result = ""
+    translate_parts: list[TranslatePart] = []
+    for i, part in enumerate(parts):
+        translate_part = TranslatePart(text=part.text, translate=part.translate, paragraph_end=part.paragraph_end)
+        translate_parts.append(translate_part)
+        result = result + part.translate
+        if not is_arr_fin(parts, i):
+            if part.paragraph_end:
+                result = result + "\n"
+            else:
+                result = result + " "
+
+    return result, translate_parts

jaa.py

@@ -0,0 +1,347 @@
+"""
+Jaa.py Plugin Framework
+Author: Janvarev Vladislav
+
+Jaa.py - minimalistic one-file plugin framework with no dependencies.
+Main functions:
+- run all plugins files from "plugins" folder, base on filename
+- save each plugin options in "options" folder in JSON text files for further editing
+
+- Plugins
+must located in plugins/ folder
+must have "start(core)" function, that returns manifest dict
+manifest must contain keys "name" and "version"
+can contain "default_options"
+- if contain - options will be saved in "options" folder and reload instead next time
+- if contain - "start_with_options(core, manifest)" function will run with manifest with "options" key
+manifest will be processed in "process_plugin_manifest" function if you override it
+
+- Options (for plugins)
+are saved under "options" folder in JSON format
+created at first run plugin with "default_options"
+updated when plugin change "version"
+
+- Example usage:
+class VoiceAssCore(JaaCore): # class must override JaaCore
+    def __init__(self):
+        JaaCore.__init__(self, __file__)
+  ...
+
+main = VoiceAssCore()
+main.init_plugins(["core"]) # 1 param - first plugins to be initialized
+                            # Good if you need some "core" options/plugin to be loaded before others
+                            # not necessary starts with "plugin_" prefix
+
+also can be run like
+
+main.init_plugins()
+
+- Requirements
+Python 3.5+ (due to dict mix in final_options calc), can be relaxed
+"""
+
+import os
+import json
+
+# here we trying to use termcolor to highlight plugin info and errors during load
+try:
+    from termcolor import cprint
+except Exception as e:
+    # not found? making a stub!
+    def cprint(p,color=None):
+        if color == None:
+            print(p)
+        else:
+            print(str(color).upper(),p)
+
+version = "2.2.1"
+
+class JaaCore:
+    def __init__(self, root_file = __file__):
+        self.jaaPluginPrefix = "plugin_"
+        self.jaaVersion = version
+        self.jaaRootFolder = os.path.dirname(root_file)
+        self.jaaOptionsPath = os.path.join(self.jaaRootFolder, "options")
+        self.jaaShowTracebackOnPluginErrors = False
+        cprint("JAA.PY v{0} class created!".format(version), "blue")
+
+    # ------------- plugins -----------------
+    def init_plugins(self, list_first_plugins = []):
+        self.plugin_manifests = {}
+
+        # 1. run first plugins first!
+        for modname in list_first_plugins:
+            self.init_plugin(modname)
+
+        # 2. run all plugins from plugins folder
+        pluginpath = os.path.join(self.jaaRootFolder, "plugins")
+        files = [f for f in os.listdir(pluginpath) if os.path.isfile(os.path.join(pluginpath, f))]
+
+        for fil in files:
+            # print fil[:-3]
+            if fil.startswith(self.jaaPluginPrefix) and fil.endswith(".py"):
+                modfile = fil[:-3]
+                self.init_plugin(modfile)
+
+
+
+    def init_plugin(self, modname):
+        # import
+        try:
+            mod = self.import_plugin("plugins."+modname)
+        except Exception as e:
+            self.print_error("JAA PLUGIN ERROR: {0} error on load: {1}".format(modname, str(e)))
+            return False
+
+        # run start function
+        try:
+            res = mod.start(self)
+        except Exception as e:
+            self.print_error("JAA PLUGIN ERROR: {0} error on start: {1}".format(modname, str(e)))
+            return False
+
+        # if plugin has an options
+        if "default_options" in res:
+            try:
+                # saved options try to read
+                saved_options = {}
+                try:
+                    with open(os.path.join(self.jaaOptionsPath, modname+'.json'), 'r', encoding="utf-8") as f:
+                        s = f.read()
+                    saved_options = json.loads(s)
+                    #print("Saved options", saved_options)
+                except Exception as e:
+                    pass
+
+                res["default_options"]["v"] = res["version"]
+
+
+                # only string needs Python 3.5
+                final_options = {**res["default_options"], **saved_options}
+
+                # if no option found or version is differ from mod version
+                if len(saved_options) == 0 or saved_options["v"] != res["version"]:
+                    final_options["v"] = res["version"]
+                    self.save_plugin_options(modname, final_options)
+
+                res["options"] = final_options
+
+                try:
+                    res2 = mod.start_with_options(self, res)
+                    if res2 != None:
+                        res = res2
+                except Exception as e:
+                    self.print_error("JAA PLUGIN ERROR: {0} error on start_with_options processing: {1}".format(modname, str(e)))
+                    return False
+
+            except Exception as e:
+                self.print_error("JAA PLUGIN ERROR: {0} error on options processing: {1}".format(modname, str(e)))
+                return False
+
+
+        # processing plugin manifest
+        try:
+            # set up name and version
+            plugin_name = res["name"]
+            plugin_version = res["version"]
+
+
+            self.process_plugin_manifest(modname, res)
+
+        except Exception as e:
+            print("JAA PLUGIN ERROR: {0} error on process startup options: {1}".format(modname, str(e)))
+            return False
+
+        self.plugin_manifests[modname] = res
+
+        self.on_succ_plugin_start(modname, plugin_name, plugin_version)
+        return True
+
+    def on_succ_plugin_start(self, modname, plugin_name, plugin_version):
+        cprint("JAA PLUGIN: {1} {2} ({0}) started!".format(modname, plugin_name, plugin_version))
+
+    def print_error(self, p):
+        import traceback
+        cprint(p, "red")
+        if self.jaaShowTracebackOnPluginErrors:
+            traceback.print_exc()
+
+    def import_plugin(self, module_name):
+        import sys
+
+        __import__(module_name)
+
+        if module_name in sys.modules:
+            return sys.modules[module_name]
+        return None
+
+    def save_plugin_options(self, modname, options):
+        # check folder exists
+        if not os.path.exists(self.jaaOptionsPath):
+            os.makedirs(self.jaaOptionsPath)
+
+        str_options = json.dumps(options, sort_keys=True, indent=4, ensure_ascii=False)
+        with open(os.path.join(self.jaaOptionsPath, modname+'.json'), 'w', encoding="utf-8") as f:
+            f.write(str_options)
+            f.close()
+
+    # process manifest must be overrided in inherit class
+    def process_plugin_manifest(self, modname, manifest):
+        print("JAA PLUGIN: {0} manifest dummy procession (override 'process_plugin_manifest' function)".format(modname))
+        return
+
+    def plugin_manifest(self, pluginname):
+        if pluginname in self.plugin_manifests:
+            return self.plugin_manifests[pluginname]
+        return {}
+
+    def plugin_options(self, pluginname):
+        manifest = self.plugin_manifest(pluginname)
+        if "options" in manifest:
+            return manifest["options"]
+        return None
+
+    # ------------ gradio stuff --------------
+    def gradio_save(self, pluginname):
+        print("Saving options for {0}!".format(pluginname))
+        self.save_plugin_options(pluginname, self.plugin_options(pluginname))
+
+    def gradio_upd(self, pluginname, option, val):
+        options = self.plugin_options(pluginname)
+
+        # special case
+        if isinstance(options[option], (list, dict)) and isinstance(val, str):
+            try:
+                options[option] = json.loads(val)
+            except Exception as e:
+                print(e)
+                pass
+        else:
+            options[option] = val
+        print(option, val, options)
+
+    def gradio_render_settings_interface(self, title: str="Settings manager", required_fields_to_show_plugin: list=["default_options"]):
+        import gradio as gr
+
+        with gr.Blocks() as gr_interface:
+            gr.Markdown("# {0}".format(title))
+            for pluginname in self.plugin_manifests:
+                manifest = self.plugin_manifests[pluginname]
+
+                # calculate if we show plugin
+                is_show_plugin = False
+                if len(required_fields_to_show_plugin) == 0:
+                    is_show_plugin = True
+                else:
+                    for k in required_fields_to_show_plugin:
+                        if manifest.get(k) is not None:
+                            is_show_plugin = True
+
+                if is_show_plugin:
+                    with gr.Tab(pluginname):
+                        gr.Markdown("## {0} v{1}".format(manifest["name"], manifest["version"]))
+                        if manifest.get("description") is not None:
+                            gr.Markdown(manifest.get("description"))
+
+                        if manifest.get("url") is not None:
+                            gr.Markdown("**URL:** [{0}]({0})".format(manifest.get("url")))
+
+
+                        if "options" in manifest:
+                            options = manifest["options"]
+                            if len(options) > 1: # not only v
+                                text_button = gr.Button("Save options".format(pluginname))
+                                #options_int_list = []
+                                for option in options:
+
+                                    #gr.Label(label=option)
+                                    if option != "v":
+                                        val = options[option]
+                                        label = option
+
+                                        if manifest.get("options_label") is not None:
+                                            if manifest.get("options_label").get(option) is not None:
+                                                label = option+": "+manifest.get("options_label").get(option)
+
+
+                                        if isinstance(val, (bool, )):
+                                            gr_elem = gr.Checkbox(value=val, label=label)
+                                        elif isinstance(val, (dict, list)):
+                                            gr_elem = gr.Textbox(value=json.dumps(val, ensure_ascii=False), label=label)
+                                        else:
+                                            gr_elem = gr.Textbox(value=val, label=label)
+
+                                        def handler(x, pluginname=pluginname, option=option):
+                                            self.gradio_upd(pluginname, option, x)
+
+                                        gr_elem.change(handler, gr_elem, None)
+
+                                def handler_save(pluginname=pluginname):
+                                    self.gradio_save(pluginname)
+
+                                text_button.click(handler_save, inputs=None, outputs=None)
+                        else:
+                            gr.Markdown("_No options for this plugin_")
+
+        return gr_interface
+
+
+def load_options(options_file=None, py_file=None, default_options={}):
+    # 1. calculating options filename
+    if options_file == None:
+        if py_file == None:
+            raise Exception('JAA: Options or PY file is not defined, cant calc options filename')
+        else:
+            options_file = py_file[:-3]+'.json'
+
+    # 2. try to read saved options
+    saved_options = {}
+    try:
+        with open(options_file, 'r', encoding="utf-8") as f:
+            s = f.read()
+        saved_options = json.loads(s)
+        #print("Saved options", saved_options)
+    except Exception as e:
+        pass
+
+    # 3. calculating final options
+
+    # only string needs Python 3.5
+    final_options = {**default_options, **saved_options}
+
+    # 4. calculating hash from def options to check - is file rewrite needed?
+    import hashlib
+    hash = hashlib.md5((json.dumps(default_options, sort_keys=True)).encode('utf-8')).hexdigest()
+
+    # 5. if no option file found or hash was from other default options
+    if len(saved_options) == 0 or not ("hash" in saved_options.keys()) or saved_options["hash"] != hash:
+        final_options["hash"] = hash
+        #self.save_plugin_options(modname, final_options)
+
+        # saving in file
+        str_options = json.dumps(final_options, sort_keys=True, indent=4, ensure_ascii=False)
+        with open(options_file, 'w', encoding="utf-8") as f:
+            f.write(str_options)
+            f.close()
+
+    return final_options
+
+"""
+The MIT License (MIT)
+Copyright (c) 2021 Janvarev Vladislav
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software without restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or
+substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""

plugins/core.py

@@ -0,0 +1,77 @@
+from app.app_core import AppCore
+from app.struct import TranslationParams, read_text_split_params, \
+    read_text_process_params, read_translation_params, read_cache_params
+
+
+def start(core: AppCore):
+    manifest = {
+        "name": "Core plugin",
+        "version": "1.0",
+
+        # this is DEFAULT options
+        # ACTUAL options is in options/<plugin_name>.json after first run
+        "default_options": {
+            "default_translate_plugin": "lm_studio",  # default translation engine. Will be auto inited on start
+            "init_on_start": "",  # additional list of engines, that must be init on start, separated by ","
+
+            "translation_params": {
+                "default_from_lang": "en",  # default from language
+                "default_to_lang": "ru",  # default to language
+            },
+
+            "text_split_params": {
+                "split_by_paragraphs_and_length": True,
+                "split_by_sentences_and_length": False,
+                "split_expected_length": 1000,
+
+                "split_by_paragraphs_only": False,
+                "split_by_sentences_only": False,
+
+                "sentence_splitter": "default"
+            },
+
+            "text_processing_params": {
+                "apply_for_request": True,  # apply processing params for text to translate
+                "apply_for_response": True,  # apply processing params for result text
+
+                "replace_non_standard_new_lines_chars": True,
+                "replace_not_text_chars": False,
+                # some models has issues with special chars (for example { or }) in text. this option replace all non-digit / non text / non-allowed (allowed_chars_for_replace) chars
+                "allowed_chars_ignoring_replace": " .,<>:;\"'-–…?!#@№$%+/\\^&[]=*()«»—\r\t\n",
+                # allowed chars for replace with replace_not_text_chars
+                "replace_not_text_target_char": " ",  # replace not allowed char to this char
+
+                # replace more than N char consecutive, for example: aaaa -> aaa, bbbbbbb -> bbb
+                "remove_identical_characters": True,
+                "remove_identical_characters_extra_chars": "",
+                "remove_identical_characters_max_repeats": 3,
+
+                "remove_multiple_spaces": True, # replace two or more space to one
+                "replace_text_from_to": {  # additional replace variants, from : to
+                },
+            },
+
+            "cache_params": {
+                "enabled": True,  # enable/disable translate cache
+                "file": "cache.db",  # path to cache file
+                "disable_for_plugins": ["no_translate"], # list of plugin names without cache
+                "expire_days": 0,  # 0 - without expire
+            }
+        },
+    }
+
+    return manifest
+
+
+def start_with_options(core: AppCore, manifest: dict):
+    options = manifest["options"]
+
+    core.default_translate_plugin = options["default_translate_plugin"]
+    core.init_on_start = options["init_on_start"]
+
+    core.translation_params = read_translation_params(manifest)
+    core.text_split_params = read_text_split_params(manifest)
+    core.text_process_params = read_text_process_params(manifest)
+    core.cache_params = read_cache_params(manifest)
+
+    return manifest

plugins/plugin_kobold_cpp.py

@@ -0,0 +1,99 @@
+import os
+
+from app import struct
+from app.app_core import AppCore
+from app.lang_dict import get_lang_by_2_chars_code
+from app.struct import TranslateStruct
+import requests
+
+modname = os.path.basename(__file__)[:-3]  # calculating modname
+
+
+# start function
+def start(core: AppCore):
+    manifest = {  # plugin settings
+        "name": "KoboldCpp Translator",  # name
+        "version": "1.0",  # version
+
+        "default_options": {
+            "custom_url": "http://127.0.0.1:5001",  #
+            "prompt": "\n### Instruction:\nYou are professional translator. Translate this text from {0} to {1}. Don't add any notes or any additional info in your answer, write only translate. Text: {2}\n### Response:\n"
+        },
+        "translate": {
+            "kobold_cpp": (init, translate)  # 1 function - init, 2 - translate
+        }
+    }
+
+    return manifest
+
+
+def start_with_options(core: AppCore, manifest: dict):
+    struct.read_plugin_params(manifest)
+    pass
+
+
+def init(core: AppCore):
+    return modname
+
+
+def translate(core: AppCore, ts: TranslateStruct):
+    options = core.plugin_options(modname)
+
+    from_lang_name = get_lang_by_2_chars_code(ts.req.from_lang)
+    to_lang_name = get_lang_by_2_chars_code(ts.req.to_lang)
+
+    # prompt = options["prompt"].format(from_lang_name, to_lang_name)
+    url = options['custom_url'] + "/api/v1/generate"
+
+    for part in ts.parts:
+        if part.need_to_translate():
+            prompt = options["prompt"].format(from_lang_name, to_lang_name, part.text)
+            length: int
+            min_length = 512
+            if len(part.text) * 2 < min_length:
+                length = min_length
+            else:
+                length = len(part.text) * 2
+            req = {
+                "n": 1,
+                "max_context_length": 4096,
+                "max_length": length,
+                "rep_pen": 1.07,
+                "temperature": 0.01,
+                "top_p": 0.92,
+                "top_k": 100,
+                "top_a": 0,
+                "typical": 1,
+                "tfs": 1,
+                "rep_pen_range": 360,
+                "rep_pen_slope": 0.7,
+                "sampler_order": [6,0,1,3,4,2,5],
+                "memory": "",
+                "trim_stop": True,
+                "genkey": "KCPP9747",
+                "min_p": 0,
+                "dynatemp_range": 0,
+                "dynatemp_exponent": 1,
+                "smoothing_factor": 0,
+                "nsigma": 0,
+                "banned_tokens": [],
+                "render_special": False,
+                "logprobs": False,
+                "presence_penalty": 0,
+                "logit_bias": {},
+                "prompt": prompt,
+                "quiet": True,
+                "stop_sequence": ["### Instruction:", "### Response:"],
+                "use_default_badwordsids": False,
+                "bypass_eos": False
+            }
+
+            response = requests.post(url, json=req)
+
+            if response.status_code != 200:
+                raise ValueError("Response status {0} for request by url {1}".format(response.status_code, url))
+
+            content: str = response.json()["results"][0]['text']
+            part.translate = content.strip()
+
+    return ts

plugins/plugin_lm_studio.py

@@ -0,0 +1,70 @@
+import os
+
+import requests
+from tqdm import tqdm
+
+from app import struct
+from app.app_core import AppCore
+from app.lang_dict import get_lang_by_2_chars_code
+from app.struct import TranslateStruct, tp
+
+modname = os.path.basename(__file__)[:-3]  # calculating modname
+
+
+# start function
+def start(core: AppCore):
+    manifest = {
+        "name": "LM-Studio Translator",  # name
+        "version": "1.0",  # version
+
+        "default_options": {
+            "custom_url": "http://localhost:1234",  #
+            "prompt": "You are professional translator. Translate text from {0} to {1}. Don't add any notes or any additional info in your answer, write only translate. Text: ",
+            "prompt_postfix": ""
+        },
+
+        "translate": {
+            "lm_studio": (init, translate)  # 1 function - init, 2 - translate
+        }
+    }
+
+    return manifest
+
+
+def start_with_options(core: AppCore, manifest: dict):
+    struct.read_plugin_params(manifest)
+    pass
+
+
+def init(core: AppCore):
+    return modname
+
+
+def translate(core: AppCore, ts: TranslateStruct):
+    options = core.plugin_options(modname)
+
+    from_lang_name = get_lang_by_2_chars_code(ts.req.from_lang)
+    to_lang_name = get_lang_by_2_chars_code(ts.req.to_lang)
+
+    prompt = options["prompt"].format(from_lang_name, to_lang_name)
+    url = options['custom_url'] + "/v1/chat/completions"
+
+    for part in tqdm(ts.parts, unit=tp.unit, ascii=tp.ascii, desc=tp.desc):
+        if part.need_to_translate():
+            req = {
+                "messages": [
+                    {"role": "system", "content": prompt + options["prompt_postfix"]},
+                    {"role": "user", "content": part.text}
+                ],
+                "temperature": 0.0
+            }
+
+            response = requests.post(url, json=req)
+
+            if response.status_code != 200:
+                raise ValueError("Response status {0} for request by url {1}".format(response.status_code, url))
+
+            content: str = response.json()["choices"][0]['message']['content']
+            part.translate = content.replace("<think>\n\n</think>\n\n", "").strip()
+
+    return ts

plugins/plugin_madlab_ctranslate2.py

@@ -0,0 +1,100 @@
+import os
+
+import ctranslate2
+import transformers
+from ctranslate2 import Translator
+from tqdm import tqdm
+from transformers import PreTrainedTokenizerBase
+
+from app import cuda, struct
+from app.app_core import AppCore
+from app.struct import TranslateStruct, tp
+
+modname = os.path.basename(__file__)[:-3]
+
+model: Translator
+tokenizer: PreTrainedTokenizerBase
+
+
+def start(core: AppCore):
+    manifest = {  # plugin settings
+        "name": "Madlab CTranslate2",  # name
+        "version": "1.0",  # version
+
+        "translate": {
+            "madlab_ctranslate2": (init, translate)  # 1 function - init, 2 - translate
+        },
+
+        "default_options": {
+            "model": "models/madlad400-10b-mt-bfloat16",  # key model
+            "tokenizer": "jbochi/madlad400-10b-mt",  # transformers.AutoTokenizer
+            "compute_type": "bfloat16",
+            "cuda": True,  # false if you want to run on CPU, true - if on CUDA
+            "cuda_device_index": 0,  # GPU index (if you have more than one GPU)
+            "max_batch_size": 16,  # batch processing requests, increase need to more memory
+
+            "text_split_params": {
+                "split_by_sentences_only": True,
+            }
+        },
+    }
+
+    return manifest
+
+
+def start_with_options(core: AppCore, manifest:dict):
+    struct.read_plugin_params(manifest)
+
+    return manifest
+
+
+def init(core:AppCore):
+    options = core.plugin_options(modname)
+
+    global model
+    global tokenizer
+
+    model = ctranslate2.Translator(options["model"],
+                                   device=cuda.get_device(options), device_index=options["cuda_device_index"])
+    tokenizer = transformers.AutoTokenizer.from_pretrained(options["tokenizer"])
+
+    return modname
+
+
+def translate(core: AppCore, ts: TranslateStruct):
+    options = core.plugin_options(modname)
+
+    # # implementation 1: one part - one batch
+    # for part in ts.parts:
+    #     if not part.text or part.text == "":
+    #         part.translate = ""
+    #     else:
+    #         input_text = "<2" + ts.req.to_lang + ">" + part.text
+    #         tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(input_text))
+    #         translate_results = model.translate_batch([tokens])
+    #         output_tokens = translate_results[0].hypotheses[0]
+    #         decoded_text = tokenizer.decode(tokenizer.convert_tokens_to_ids(output_tokens))
+    #         part.translate = decoded_text
+
+    # implementation 2: all parts - one batch. It's faster, but depends on amount of batches.
+    tokens_list = []
+    for part in tqdm(ts.parts, unit=tp.unit, ascii=tp.ascii, desc=tp.desc):
+        if part.need_to_translate():
+            input_text = "<2" + ts.req.to_lang + ">" + part.text
+            tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(input_text))
+            tokens_list.append(tokens)
+
+    translate_results = model.translate_batch(
+        tokens_list, max_batch_size=options["max_batch_size"], beam_size=1, return_scores=False, disable_unk=False)
+
+    i = 0
+    for part in ts.parts:
+        if part.text and part.text != "":
+            output_tokens = translate_results[i].hypotheses[0]
+            decoded_text = tokenizer.decode(tokenizer.convert_tokens_to_ids(output_tokens))
+            part.translate = decoded_text
+            i += 1
+        else:
+            part.translate = ""
+
+    return ts

plugins/plugin_nllb_200.py

@@ -0,0 +1,81 @@
+# nllb plugin
+# author: Vladislav Janvarev
+
+# from https://github.com/facebookresearch/fairseq/tree/nllb
+import os
+
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+
+from app import struct, cuda
+from app.app_core import AppCore
+from app.lang_dict import lang_2_chars_to_nllb_lang
+from app.struct import TranslateStruct, tp
+
+modname = os.path.basename(__file__)[:-3]  # calculating modname
+
+model = None
+tokenizers:dict = {}
+
+
+def start(core: AppCore):
+    manifest = {  # plugin settings
+        "name": "NLLB 200 Translate",  # name
+        "version": "1.0",  # version
+
+        "translate": {
+            "nllb_200": (init, translate)  # 1 function - init, 2 - translate
+        },
+
+        "default_options": {
+            "model": "facebook/nllb-200-distilled-600M",  # key model
+            "cuda": True,
+            "cuda_device_index": 0,
+
+            "text_split_params": {
+                "split_by_sentences_only": True,
+            }
+        },
+    }
+    return manifest
+
+
+def start_with_options(core: AppCore, manifest: dict):
+    struct.read_plugin_params(manifest)
+
+    return manifest
+
+
+def init(core: AppCore):
+    options = core.plugin_options(modname)
+
+    global model
+    model = AutoModelForSeq2SeqLM.from_pretrained(options["model"]).to(cuda.get_device_with_gpu_num(options))
+
+    return modname
+
+
+def translate(core: AppCore, ts: TranslateStruct):
+    options = core.plugin_options(modname)
+
+    from_lang = lang_2_chars_to_nllb_lang[ts.req.from_lang]
+    to_lang = lang_2_chars_to_nllb_lang[ts.req.to_lang]
+    cuda_device = cuda.get_device_with_gpu_num(options)
+
+    if tokenizers.get(from_lang) is None:
+        tokenizers[from_lang] = AutoTokenizer.from_pretrained(options["model"], src_lang=from_lang)
+    tokenizer = tokenizers[from_lang]
+
+    for part in tqdm(ts.parts, unit=tp.unit, ascii=tp.ascii, desc=tp.desc):
+        if part.need_to_translate():
+            inputs = tokenizer(part.text, return_tensors="pt").to(cuda_device)
+
+            translated_tokens = model.generate(
+                **inputs,
+                forced_bos_token_id=tokenizer.convert_tokens_to_ids(to_lang),
+                max_length=int(len(part.text) * 5)
+            )
+            decoded = tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)[0]
+            part.translate = decoded
+
+    return ts

plugins/plugin_nllb_200_ctranslate2.py

@@ -0,0 +1,86 @@
+import os
+
+import ctranslate2
+from ctranslate2 import Translator
+from tqdm import tqdm
+from transformers import AutoTokenizer
+
+from app import cuda, struct
+from app.app_core import AppCore
+from app.lang_dict import lang_2_chars_to_nllb_lang
+from app.struct import TranslateStruct, tp
+
+modname = os.path.basename(__file__)[:-3]
+
+model: Translator
+tokenizers:dict = {}
+
+
+def start(core: AppCore):
+    manifest = {  # plugin settings
+        "name": "NLLB 200 CTranslate2",  # name
+        "version": "1.0",  # version
+
+        "translate": {
+            "nllb_200_ctranslate2": (init, translate)  # 1 function - init, 2 - translate
+        },
+
+        "default_options": {
+            "model": "models/nllb-200-3.3B-ct2-float16",  # model
+            "compute_type": "bfloat16",
+            "cuda": True,  # false if you want to run on CPU, true - if on CUDA
+            "cuda_device_index": 0,  # GPU index (if you have more than one GPU)
+            "max_batch_size": 16,
+
+            "text_split_params": {
+                "split_by_sentences_only": True,
+            }
+        },
+    }
+
+    return manifest
+
+
+def start_with_options(core: AppCore, manifest:dict):
+    struct.read_plugin_params(manifest)
+
+    return manifest
+
+
+def init(core:AppCore):
+    options = core.plugin_options(modname)
+
+    global model
+
+    model = ctranslate2.Translator(options["model"],
+                                   device=cuda.get_device(options), device_index=options["cuda_device_index"])
+
+    return modname
+
+
+def translate(core: AppCore, ts: TranslateStruct):
+    options = core.plugin_options(modname)
+
+    from_lang = lang_2_chars_to_nllb_lang[ts.req.from_lang]
+    to_lang = lang_2_chars_to_nllb_lang[ts.req.to_lang]
+    if tokenizers.get(from_lang) is None:
+        tokenizers[from_lang] = AutoTokenizer.from_pretrained(options["model"], src_lang=from_lang)
+    tokenizer = tokenizers[from_lang]
+
+    # translate_batch not optimal, but there are problems with try to implement batch processing like madlab_ctranslate2
+    for part in tqdm(ts.parts, unit=tp.unit, ascii=tp.ascii, desc=tp.desc):
+        if part.need_to_translate():
+            input_text = part.text
+            tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(input_text))
+
+            translate_results = model.translate_batch(
+                [tokens], max_batch_size=options["max_batch_size"], beam_size=1, return_scores=False, disable_unk=False,
+                target_prefix=[[to_lang]], batch_type="tokens"
+            )
+            output_tokens = translate_results[0].hypotheses[0]
+            decoded_text = tokenizer.decode(tokenizer.convert_tokens_to_ids(output_tokens))
+            if to_lang in decoded_text:
+                decoded_text = decoded_text.replace(to_lang, "").lstrip()
+            part.translate = decoded_text
+
+    return ts

plugins/plugin_no_translate.py

@@ -0,0 +1,34 @@
+# No Translate dummy plugin
+# author: Vladislav Janvarev
+
+import os
+
+from app.app_core import AppCore
+from app.struct import TranslateStruct
+
+modname = os.path.basename(__file__)[:-3]  # calculating modname
+
+
+# start function
+def start(core: AppCore):
+    manifest = {  # plugin settings
+        "name": "No Translate dummy plugin",  # name
+        "version": "1.0",  # version
+
+        "translate": {
+            "no_translate": (init, translate)  # 1 function - init, 2 - translate
+        }
+    }
+
+    return manifest
+
+
+def init(core: AppCore):
+    return modname
+
+
+def translate(core: AppCore, ts: TranslateStruct):
+    for part in ts.parts:
+        part.translate = part.text
+
+    return ts

properties.json

@@ -0,0 +1,3 @@
+{
+  "port": 4990
+}

requirements.txt

@@ -0,0 +1,11 @@
+uvicorn
+uvicorn[standard]
+fastapi
+termcolor
+
+transformers
+ctranslate2
+
+blingfire
+pysbd
+

requirements_pytorch.txt

@@ -0,0 +1,3 @@
+Open https://pytorch.org/get-started/locally/ and select your variant, for example:
+
+pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128

static/index.html

@@ -0,0 +1,70 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+    <title>LLM translate</title>
+    <script type="application/javascript" src="index.js"></script>
+    <link rel="stylesheet" href="chota.min.css">
+    <style>
+        .loader {
+            border: 4px solid #f3f3f3; /* Light grey */
+            border-top: 4px solid #2a82b6; /* Blue */
+            border-radius: 50%;
+            width: 16px;
+            height: 16px;
+            animation: spin 2s linear infinite;
+        }
+
+        @keyframes spin {
+            0% { transform: rotate(0deg); }
+            100% { transform: rotate(360deg); }
+        }
+    </style>
+</head>
+<body>
+<div id="top" class="container" role="document">
+    <div class="row">
+        <div class="col">
+            <h5>LLM Translate</h5>
+        </div>
+        <div class="col">
+            <span id="errorText" class="text-error"></span>
+        </div>
+    </div>
+    <div class="row">
+        <div class="col">
+            <div class="row">
+                <div class="col">
+                    <select name="from_lang" id="from_lang_select"></select>
+                </div>
+                <div class="col">
+                    <button id="trigger" class="button primary icon" type="submit">
+                        &nbspTranslate&nbsp<div id="progress" class="loader" style="display: none;"></div>
+                    </button>
+                </div>
+            </div>
+        </div>
+        <div class="col">
+            <div class="row">
+                <div class="col">
+                    <select name="to_lang" id="to_lang_select"></select>
+                </div>
+                <div class="col">
+                    <input id="plugin" value="" placeholder="Use translator plugin (optional)"/>
+                </div>
+            </div>
+        </div>
+    </div>
+    <div class="row">
+        <div class="col">
+            <textarea id="text" rows="20"></textarea>
+        </div>
+        <div class="col" aria-busy="true">
+            <textarea id="text_result" rows="20" ></textarea>
+        </div>
+    </div>
+</div>
+
+</body>
+</html>

static/index.js

@@ -0,0 +1,203 @@
+async function translateText() {
+    const elProgress = document.getElementById('progress');
+    const trigger = document.getElementById('trigger');
+    const errorText = document.getElementById('errorText');
+
+    const text = document.getElementById('text').value;
+    const fromLang = document.getElementById('from_lang_select').value;
+    const toLang = document.getElementById('to_lang_select').value;
+    const plugin = document.getElementById('plugin').value;
+
+    trigger.disabled = true;
+    elProgress.style.display = 'inline';
+
+    const reqBody = JSON.stringify({
+        text: text, from_lang: fromLang, to_lang: toLang,
+        translator_plugin: plugin
+    });
+    const reqParam = {
+        method: 'POST',
+        body: reqBody,
+        headers: {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+        }
+    }
+    try {
+        const response = await fetch(`/translate`, reqParam);
+        const data = await response.json();
+        if (data.error) {
+            errorText.innerHTML = data.error;
+            return "";
+        } else {
+            const translation = data.result;
+            document.getElementById('text_result').value = translation;
+
+            errorText.innerHTML = ""
+            return translation;
+        }
+    } catch (error) {
+        errorText.innerHTML = error.message;
+        console.error(error.message);
+    } finally {
+        elProgress.style.display = 'none';
+        trigger.disabled = false;
+    }
+}
+
+
+window.onload = () => {
+    const trigger = document.getElementById('trigger');
+    trigger.onmouseup = () => {
+        translateText();
+    };
+
+    const langDict = {
+        'en': 'english',
+        'ru': 'russian',
+        'ab': 'abkhazian',
+        'aa': 'afar',
+        'af': 'afrikaans',
+        'sq': 'albanian',
+        'am': 'amharic',
+        'ar': 'arabic',
+        'hy': 'armenian',
+        'as': 'assamese',
+        'ay': 'aymara',
+        'az': 'azerbaijani',
+        'ba': 'bashkir',
+        'eu': 'basque',
+        'bn': 'bengali',
+        'dz': 'bhutani',
+        'bh': 'bihari',
+        'bi': 'bislama',
+        'br': 'breton',
+        'bg': 'bulgarian',
+        'my': 'burmese',
+        'be': 'byelorussian',
+        'km': 'cambodian',
+        'ca': 'catalan',
+        'zh': 'chinese',
+        'co': 'corsican',
+        'hr': 'croatian',
+        'cs': 'czech',
+        'da': 'danish',
+        'nl': 'dutch',
+        'eo': 'esperanto',
+        'et': 'estonian',
+        'fo': 'faeroese',
+        'fj': 'fiji',
+        'fi': 'finnish',
+        'fr': 'french',
+        'fy': 'frisian',
+        'gd': 'gaelic',
+        'gl': 'galician',
+        'ka': 'georgian',
+        'de': 'german',
+        'el': 'greek',
+        'kl': 'greenlandic',
+        'gn': 'guarani',
+        'gu': 'gujarati',
+        'ha': 'hausa',
+        'iw': 'hebrew',
+        'hi': 'hindi',
+        'hu': 'hungarian',
+        'is': 'icelandic',
+        'in': 'indonesian',
+        'ia': 'interlingua',
+        'ie': 'interlingue',
+        'ik': 'inupiak',
+        'ga': 'irish',
+        'it': 'italian',
+        'ja': 'japanese',
+        'jw': 'javanese',
+        'kn': 'kannada',
+        'ks': 'kashmiri',
+        'kk': 'kazakh',
+        'rw': 'kinyarwanda',
+        'ky': 'kirghiz',
+        'rn': 'kirundi',
+        'ko': 'korean',
+        'ku': 'kurdish',
+        'lo': 'laothian',
+        'la': 'latin',
+        'lv': 'latvian',
+        'ln': 'lingala',
+        'lt': 'lithuanian',
+        'mk': 'macedonian',
+        'mg': 'malagasy',
+        'ms': 'malay',
+        'ml': 'malayalam',
+        'mt': 'maltese',
+        'mi': 'maori',
+        'mr': 'marathi',
+        'mo': 'moldavian',
+        'mn': 'mongolian',
+        'na': 'nauru',
+        'ne': 'nepali',
+        'no': 'norwegian',
+        'oc': 'occitan',
+        'or': 'oriya',
+        'om': 'oromo',
+        'ps': 'pashto',
+        'fa': 'persian',
+        'pl': 'polish',
+        'pt': 'portuguese',
+        'pa': 'punjabi',
+        'qu': 'quechua',
+        'rm': 'rhaeto-romance',
+        'ro': 'romanian',
+        'sm': 'samoan',
+        'sg': 'sangro',
+        'sa': 'sanskrit',
+        'sr': 'serbian',
+        'sh': 'serbo-croatian',
+        'st': 'sesotho',
+        'tn': 'setswana',
+        'sn': 'shona',
+        'sd': 'sindhi',
+        'si': 'singhalese',
+        'ss': 'siswati',
+        'sk': 'slovak',
+        'sl': 'slovenian',
+        'so': 'somali',
+        'es': 'spanish',
+        'su': 'sudanese',
+        'sw': 'swahili',
+        'sv': 'swedish',
+        'tl': 'tagalog',
+        'tg': 'tajik',
+        'ta': 'tamil',
+        'tt': 'tatar',
+        'te': 'tegulu',
+        'th': 'thai',
+        'bo': 'tibetan',
+        'ti': 'tigrinya',
+        'to': 'tonga',
+        'ts': 'tsonga',
+        'tr': 'turkish',
+        'tk': 'turkmen',
+        'tw': 'twi',
+        'uk': 'ukrainian',
+        'ur': 'urdu',
+        'uz': 'uzbek',
+        'vi': 'vietnamese',
+        'vo': 'volapuk',
+        'cy': 'welsh',
+        'wo': 'wolof',
+        'xh': 'xhosa',
+        'ji': 'yiddish',
+        'yo': 'yoruba',
+        'zu': 'zulu',
+    };
+
+    const fromLangSelect = document.getElementById('from_lang_select');
+    const toLangSelect = document.getElementById('to_lang_select');
+
+    for (const [key, value] of Object.entries(langDict)) {
+        fromLangSelect.innerHTML += "<option value='" + key + "'>" + value + "</option>";
+        toLangSelect.innerHTML += "<option value='" + key + "'>" + value + "</option>";
+    }
+    fromLangSelect.value = 'en';
+    toLangSelect.value = 'ru';
+}

test/test_text_processor.py

@@ -0,0 +1,44 @@
+from unittest import TestCase
+
+from app import text_processor
+
+allowed_chars_ignoring_replace = set(" .,<>:;\"'-…?!#@№$%+/\\^&[]=*()—\r\t\n")
+
+
+class TextProcessorTest(TestCase):
+    def test_replace_not_text_chars_eng_rus_digits(self):
+        text = "Южно-эфиопский грач увёл мышь за хобот на съезд ящериц. Quick wafting zephyrs vex bold Jim. 1234567890"
+        processed = text_processor.replace_not_text_chars(text, allowed_chars_ignoring_replace, "|")
+        self.assertEqual(text, processed)
+
+    def test_replace_not_text_chars_with_japan_german_chars(self):
+        text = "こんにちわ / Ü ü, Ö ö, Ä ä, ẞ ß"
+        processed = text_processor.replace_not_text_chars(text, allowed_chars_ignoring_replace, "|")
+        self.assertEqual(text, processed)
+
+    def test_replace_not_text_chars_with_ignored_chars(self):
+        text = "{ }"
+        processed = text_processor.replace_not_text_chars(text, allowed_chars_ignoring_replace, "|")
+        self.assertEqual("| |", processed)
+
+    def test_trim_duplicate_characters(self):
+        text = "soooooome textttttt оёёёёёёё 12222223 qÖÖÖÖÖÖöööööwe"
+        processed = text_processor.remove_identical_characters(text, 3, "Ö")
+        self.assertEqual("sooome texttt оёёё 12222223 qÖÖÖöööööwe", processed)
+
+        text = "soooooome textttttt оёёёёёёё 12222223 qÖÖÖÖÖÖöööööwe"
+        processed = text_processor.remove_identical_characters(text, 4, "Ö")
+        self.assertEqual("soooome textttt оёёёё 12222223 qÖÖÖÖöööööwe", processed)
+
+    def test_remove_multiple_spaces(self):
+        text = "q  w     e       r     t y"
+        processed = text_processor.remove_multiple_spaces(text)
+        self.assertEqual("q w e r t y", processed)
+
+    def test_replace_text_from_to(self):
+        text = "qwe 123 asd zxc"
+        processed = text_processor.replace_text_from_to(text, None)
+        self.assertEqual(text, processed)
+
+        processed = text_processor.replace_text_from_to(text, {"123": "456", "zxc": "789"})
+        self.assertEqual("qwe 456 asd 789", processed)

test/test_text_splitter.py

@@ -0,0 +1,213 @@
+import unittest
+
+from app import text_splitter
+from app.struct import TextSplitParams, Part
+
+s1 = "Text one."
+s2 = "Text two."
+s3 = "Text three."
+s4 = "Text four."
+s5 = "Text five."
+s6 = "Text six."
+s7 = "Text seven."
+s8 = "Text eight."
+s9 = "Text nine."
+
+split_txt1 = "Some text. Mr. John Johnson Jr. was born in the U.S.A but earned his Ph.D. in Israel before joining Nike Inc. as an engineer. He also worked at craigslist.org as a business analyst. Some text one. Some text two."
+split_txt2 = "Some sentence. Mr. Holmes... This is a new sentence! And This is another one.. Hi "
+
+
+def get_params():
+    return TextSplitParams(
+        split_expected_length=20, sentence_splitter="def",
+        split_by_paragraphs_only=False, split_by_paragraphs_and_length=False,
+        split_by_sentences_only=False, split_by_sentences_and_length=False)
+
+
+def get_text(*sentences):
+    return " ".join(sentences).replace("\n ", "\n")
+
+
+class ReqProcessorTest(unittest.TestCase):
+    def test_split_by_sentences_blingfire(self):
+        params = get_params()
+        params.sentence_splitter = "blingfire"
+
+        split = text_splitter.split_by_sentences(split_txt1, params)
+        exp = ['Some text. Mr. John Johnson Jr. was born in the U.S.A but earned his Ph.D. in Israel before joining Nike Inc. as an engineer.',
+               'He also worked at craigslist.org as a business analyst.',
+               'Some text one.',
+               'Some text two.']
+        self.assertEqual(exp, split)
+
+        split = text_splitter.split_by_sentences(split_txt2, params)
+        exp = ['Some sentence.',
+               'Mr.',
+               'Holmes...',
+               'This is a new sentence!',
+               'And This is another one..',
+               'Hi ']
+        self.assertEqual(exp, split)
+
+    def test_split_by_sentences_pysbd(self):
+        params = get_params()
+
+        split = text_splitter.split_by_sentences(split_txt1, params)
+        exp = ['Some text.',
+               'Mr. John Johnson Jr. was born in the U.S.A but earned his Ph.D. in Israel before joining Nike Inc. as an engineer.',
+               'He also worked at craigslist.org as a business analyst.',
+               'Some text one.',
+               'Some text two.']
+        self.assertEqual(exp, split)
+
+        split = text_splitter.split_by_sentences(split_txt2, params)
+        exp = ['Some sentence.',
+               'Mr. Holmes...',
+               'This is a new sentence!',
+               'And This is another one.',
+               '.',
+               'Hi']
+        self.assertEqual(exp, split)
+
+    def test_no_split(self):
+        params = get_params()
+
+        text = get_text(s1, s2, s3, s4, s5)
+        split = text_splitter.split_text(text, params)
+        exp = [Part(get_text(s1, s2, s3, s4, s5), False)]
+
+        self.assertEqual(exp, split)
+
+    def test_split_by_few_paragraphs_and_length__long_text_with_one_paragraph(self):
+        params = get_params()
+        params.split_by_paragraphs_and_length = True
+
+        text = get_text(s1, s2, s3, s4, s5)
+        split = text_splitter.split_text(text, params)
+        exp = [Part(get_text(s1, s2, s3, s4, s5), True)]
+
+        self.assertEqual(exp, split)
+
+    def test_split_by_few_paragraphs_and_length__short_text_with_one_paragraph(self):
+        params = get_params()
+        params.split_by_paragraphs_and_length = True
+
+        text = get_text(s1, s2)
+        split = text_splitter.split_text(text, params)
+        exp = [Part(get_text(s1, s2), True)]
+
+        self.assertEqual(exp, split)
+
+    def test_split_by_few_paragraphs_and_length__text_with_few_paragraphs_01(self):
+        params = get_params()
+        params.split_by_paragraphs_and_length = True
+        # text with few paragraphs
+        text = get_text(s1 + "\n\n", s2 + "\n", s3, s4 + "\n", s5, s6, s7, s8 + "\n", s9 + "\n", s1)
+        split = text_splitter.split_text(text, params)
+        exp = [Part(get_text(s1 + "\n\n", s2), True),
+               Part(get_text(s3, s4), True),
+               Part(get_text(s5, s6, s7, s8), True),
+               Part(get_text(s9 + "\n", s1), True),]
+
+        self.assertEqual(exp, split)
+
+    def test_split_by_few_paragraphs_and_length__text_with_few_paragraphs_02(self):
+        params = get_params()
+        params.split_by_paragraphs_and_length = True
+
+        text = get_text(s1 + "\n", s2 + "\n", s3 + "\n", s4 + "\n", s5)
+        split = text_splitter.split_text(text, params)
+        exp = [Part(get_text(s1 + "\n", s2), True),
+               Part(get_text(s3), True),
+               Part(get_text(s4 + "\n", s5), True),]
+
+        self.assertEqual(exp, split)
+
+    def test_split_by_paragraphs_only_01(self):
+        params = get_params()
+        params.split_by_paragraphs_only = True
+        # text with few paragraphs
+        text = get_text(s1 + "\n\n", s2 + "\n", s3, s4 + "\n", s5, s6)
+        split = text_splitter.split_text(text, params)
+        exp = [Part(get_text(s1), True),
+               Part(get_text(""), True),
+               Part(get_text(s2), True),
+               Part(get_text(s3, s4), True),
+               Part(get_text(s5, s6), False)]
+
+        self.assertEqual(exp, split)
+
+    def test_split_by_paragraphs_only_02(self):
+        params = get_params()
+        params.split_by_paragraphs_only = True
+        # text with few paragraphs
+        split = text_splitter.split_text(s1, params)
+        exp = [Part(get_text(s1), True),
+               Part(get_text(""), True),
+               Part(get_text(s2), True),
+               Part(get_text(s3, s4), True),
+               Part(get_text(s5, s6), True)]
+
+        self.assertEqual([Part(s1, False)], split)
+
+    def test_split_by_few_sentences_and_length_01(self):
+        params = get_params()
+        params.split_by_sentences_and_length = True
+        params.split_expected_length = 25
+        # text with few paragraphs
+        text = get_text(s1 + "\n\n", s2 + "\n", s3, s4 + "\n", s5, s6, s7, s8 + "\n", s9 + "\n", s1)
+        split = text_splitter.split_text(text, params)
+        exp = [Part(get_text(s1 + "\n\n", s2), True),
+               Part(get_text(s3, s4), True),
+               Part(get_text(s5, s6), False),
+               Part(get_text(s7, s8), True),
+               Part(get_text(s9 + "\n", s1), False),]
+
+        self.assertEqual(exp, split)
+
+    def test_split_by_few_sentences_and_length_02(self):
+        params = get_params()
+        params.split_by_sentences_and_length = True
+        params.split_expected_length = 40
+        # text with few paragraphs
+        text = get_text(s1 + "\n\n", s2 + "\n", s3, s4 + "\n", s5, s6, s7, s8 + "\n", s9 + "\n", s1)
+        split = text_splitter.split_text(text, params)
+        exp = [Part(get_text(s1 + "\n\n", s2 + "\n", s3), False),
+               Part(get_text(s4 + "\n", s5, s6), False),
+               Part(get_text(s7, s8 + "\n", s9), True),
+               Part(get_text(s1), False),]
+
+        self.assertEqual(exp, split)
+
+    def test_split_by_few_sentences_and_length_03(self):
+        params = get_params()
+        params.split_by_sentences_and_length = True
+        split = text_splitter.split_text(s1, params)
+
+        self.assertEqual([Part(s1, False)], split)
+
+    def test_split_by_sentences_only_01(self):
+        params = get_params()
+        params.split_by_sentences_only = True
+        split = text_splitter.split_text(s1, params)
+        self.assertEqual([Part(s1, False)], split)
+
+    def test_split_by_sentences_only_02(self):
+        params = get_params()
+        params.split_by_sentences_only = True
+        # text with few paragraphs
+        text = get_text(s1 + "\n\n", s2 + "\n", s3, s4 + "\n", s5, s6, s7, s8 + "\n", s9 + "\n", s1)
+        split = text_splitter.split_text(text, params)
+        exp = [Part(s1, True),
+               Part("", True),
+               Part(s2, True),
+               Part(s3, False),
+               Part(s4, True),
+               Part(s5, False),
+               Part(s6, False),
+               Part(s7, False),
+               Part(s8, True),
+               Part(s9, True),
+               Part(s1, False),]
+
+        self.assertEqual(exp, split)