every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into translate_api

Browse Source 
# Conflicts:
#	requirements.txt
#	requirements_cuda.txt
#	src-python/config.py
#	src-python/mainloop.py
#	src-python/model.py
#	src-python/models/osc/osc.py
#	src-python/models/translation/translation_languages.py
#	src-python/models/translation/translation_translator.py
#	src-python/models/translation/translation_utils.py
This commit is contained in:
misyaguziya
2025-10-14 12:47:47 +09:00
parent 224eaf3cef b0403cfa70
commit 6fc89fbb3e
86 changed files with 20001 additions and 1584 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
src-python/models/translation/translation_languages.py
View File

@@ -1,4 +1,13 @@
translation_lang = {}
"""Language code mappings for supported translation backends.
Provides `translation_lang` mapping keyed by backend name with `source` and
`target` maps used by `Translator.getLanguageCode`.
"""
from typing import Dict
translation_lang: Dict[str, Dict[str, Dict[str, str]]] = {}
dict_deepl_languages = {
"Arabic":"ar",
"Bulgarian":"bg",
@@ -37,10 +46,7 @@ dict_deepl_languages = {
"Chinese Simplified":"zh",
"Chinese Traditional":"zh"
}
translation_lang["DeepL"] = {
"source":dict_deepl_languages,
"target":dict_deepl_languages,
}
translation_lang["DeepL"] = {"source": dict_deepl_languages, "target": dict_deepl_languages}
dict_deepl_api_source_languages = {
"Japanese":"ja",
@@ -109,10 +115,7 @@ dict_deepl_api_target_languages = {
"Chinese Simplified":"zh",
"Chinese Traditional":"zh"
}
translation_lang["DeepL_API"] = {
"source": dict_deepl_api_source_languages,
"target": dict_deepl_api_target_languages,
}
translation_lang["DeepL_API"] = {"source": dict_deepl_api_source_languages, "target": dict_deepl_api_target_languages}
dict_google_languages = {
"Japanese":"ja",
@@ -176,13 +179,10 @@ dict_google_languages = {
"Belarusian":"be",
"Cebuano":"ceb",
"Esperanto":"eo",
"Basque":"eu",
# "Basque":"eu",
"Irish":"ga"
}
translation_lang["Google"] = {
"source":dict_google_languages,
"target":dict_google_languages,
}
translation_lang["Google"] = {"source": dict_google_languages, "target": dict_google_languages}
dict_bing_languages = {
"Japanese":"ja",
@@ -247,10 +247,7 @@ dict_bing_languages = {
"Punjabi":"pa",
"Irish":"ga"
}
translation_lang["Bing"] = {
"source":dict_bing_languages,
"target":dict_bing_languages,
}
translation_lang["Bing"] = {"source": dict_bing_languages, "target": dict_bing_languages}
dict_papago_languages = {
"German": "de",
@@ -270,10 +267,7 @@ dict_papago_languages = {
"Chinese Traditional":"zh-TW",
}
translation_lang["Papago"] = {
"source":dict_papago_languages,
"target":dict_papago_languages,
}
translation_lang["Papago"] = {"source": dict_papago_languages, "target": dict_papago_languages}
dict_m2m100_languages = {
"English": "en",
@@ -317,7 +311,7 @@ dict_m2m100_languages = {
"Malayalam": "ml",
"Welsh": "cy",
"Slovak": "sk",
"Telugu": "te",
# "Telugu": "te",
"Persian": "fa",
"Latvian": "lv",
"Bengali": "bn",
@@ -328,7 +322,7 @@ dict_m2m100_languages = {
"Estonian": "et",
"Macedonian": "mk",
"Breton": "br",
"Basque": "eu",
# "Basque": "eu",
"Icelandic": "is",
"Armenian": "hy",
"Nepali": "ne",
@@ -378,15 +372,8 @@ dict_m2m100_languages = {
"Sundanese": "su"
}
translation_lang["m2m100_418M-ct2-int8"] = {
"source":dict_m2m100_languages,
"target":dict_m2m100_languages,
}
translation_lang["m2m100_1.2B-ct2-int8"] = {
"source":dict_m2m100_languages,
"target":dict_m2m100_languages,
}
translation_lang["m2m100_418M-ct2-int8"] = {"source":dict_m2m100_languages, "target":dict_m2m100_languages}
translation_lang["m2m100_1.2B-ct2-int8"] = {"source":dict_m2m100_languages, "target":dict_m2m100_languages}
dict_nllb_languages = {
"Acehnese (Arabic script)": "ace_Arab",
@@ -595,15 +582,8 @@ dict_nllb_languages = {
"Zulu": "zul_Latn"
}
translation_lang["nllb-200-distilled-1.3B-ct2-int8"] = {
"source":dict_nllb_languages,
"target":dict_nllb_languages,
}
translation_lang["nllb-200-3.3B-ct2-int8"] = {
"source":dict_nllb_languages,
"target":dict_nllb_languages,
}
translation_lang["nllb-200-distilled-1.3B-ct2-int8"] = {"source":dict_nllb_languages, "target":dict_nllb_languages}
translation_lang["nllb-200-3.3B-ct2-int8"] = {"source":dict_nllb_languages, "target":dict_nllb_languages}
dict_plamo_languages = {
"English": "English",
@@ -638,10 +618,7 @@ dict_plamo_languages = {
"Traditional Chinese":"Traditional Chinese"
}
translation_lang["Plamo_API"] = {
"source":dict_plamo_languages,
"target":dict_plamo_languages,
}
translation_lang["Plamo_API"] = {"source":dict_plamo_languages, "target":dict_plamo_languages}
dict_gemini_languages = {
"Arabic": "Arabic",
@@ -685,7 +662,4 @@ dict_gemini_languages = {
"Vietnamese": "Vietnamese",
}
translation_lang["Gemini_API"] = {
"source":dict_gemini_languages,
"target":dict_gemini_languages,
}
translation_lang["Gemini_API"] = {"source":dict_gemini_languages, "target":dict_gemini_languages}

115
src-python/models/translation/translation_translator.py
View File

@@ -4,6 +4,7 @@ try:
from translators import translate_text as other_web_Translator
ENABLE_TRANSLATORS = True
except Exception:
other_web_Translator = None # type: ignore
ENABLE_TRANSLATORS = False
try:
@@ -25,24 +26,40 @@ import transformers
from utils import errorLogging, getBestComputeType
import warnings
from typing import Any, Optional, Tuple
warnings.filterwarnings("ignore")
# Translator
class Translator():
def __init__(self):
self.deepl_client = None
self.plamo_client = None
self.gemini_client = None
self.ctranslate2_translator = None
self.ctranslate2_tokenizer = None
self.is_loaded_ctranslate2_model = False
self.is_enable_translators = ENABLE_TRANSLATORS
class Translator:
"""High-level translator facade.
This class wraps multiple backends (DeepL, DeepL API, Google, Bing, Papago,
and CTranslate2 local models). Optional dependencies may be unavailable at
runtime; methods degrade gracefully and return False or an empty string on
failure (kept compatible with existing behavior).
"""
def __init__(self) -> None:
self.deepl_client: Optional[DeepLClient] = None
self.plamo_client: Optional[PlamoClient] = None
self.gemini_client: Optional[GeminiClient] = None
self.ctranslate2_translator: Any = None
self.ctranslate2_tokenizer: Any = None
self.is_loaded_ctranslate2_model: bool = False
self.is_changed_translator_parameters: bool = False
self.is_enable_translators: bool = ENABLE_TRANSLATORS
def authenticationDeepLAuthKey(self, auth_key: str) -> bool:
"""Authenticate DeepL API with the provided key.
Returns True on success, False on failure.
"""
result = True
try:
self.deepl_client = DeepLClient(auth_key)
self.deepl_client.translate_text("Hello World", target_lang="EN-US")
# quick smoke test
self.deepl_client.translate_text(" ", target_lang="EN-US")
except Exception:
errorLogging()
self.deepl_client = None
@@ -50,6 +67,10 @@ class Translator():
return result
def authenticationPlamoAuthKey(self, auth_key: str, model: str) -> bool:
"""Authenticate Plamo API with the provided key.
Returns True on success, False on failure.
"""
result = True
try:
self.plamo_client = PlamoClient(auth_key, model=model)
@@ -61,6 +82,10 @@ class Translator():
return result
def authenticationGeminiAuthKey(self, auth_key: str, model: str) -> bool:
"""Authenticate Gemini API with the provided key.
Returns True on success, False on failure.
"""
result = True
try:
self.gemini_client = GeminiClient(auth_key, model=model)
@@ -71,21 +96,27 @@ class Translator():
result = False
return result
def changeCTranslate2Model(self, path, model_type, device="cpu", device_index=0):
def changeCTranslate2Model(self, path: str, model_type: str, device: str = "cpu", device_index: int = 0, compute_type: str = "auto") -> None:
"""Load a CTranslate2 model from weights.
This sets internal translator/tokenizer objects and flips
``is_loaded_ctranslate2_model`` on success.
"""
self.is_loaded_ctranslate2_model = False
directory_name = ctranslate2_weights[model_type]["directory_name"]
tokenizer = ctranslate2_weights[model_type]["tokenizer"]
weight_path = os_path.join(path, "weights", "ctranslate2", directory_name)
tokenizer_path = os_path.join(path, "weights", "ctranslate2", directory_name, "tokenizer")
compute_type = getBestComputeType(device, device_index)
if compute_type == "auto":
compute_type = getBestComputeType(device, device_index)
self.ctranslate2_translator = ctranslate2.Translator(
weight_path,
device=device,
device_index=device_index,
compute_type=compute_type,
inter_threads=1,
intra_threads=4
intra_threads=4,
)
try:
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer, cache_dir=tokenizer_path)
@@ -95,11 +126,21 @@ class Translator():
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer, cache_dir=tokenizer_path)
self.is_loaded_ctranslate2_model = True
def isLoadedCTranslate2Model(self):
def isLoadedCTranslate2Model(self) -> bool:
return self.is_loaded_ctranslate2_model
def translateCTranslate2(self, message, source_language, target_language, weight_type):
result = False
def isChangedTranslatorParameters(self) -> bool:
return self.is_changed_translator_parameters
def setChangedTranslatorParameters(self, is_changed: bool) -> None:
self.is_changed_translator_parameters = is_changed
def translateCTranslate2(self, message: str, source_language: str, target_language, weight_type: str) -> Any:
"""Translate using a loaded CTranslate2 model.
Returns a string on success or False on failure (keeps legacy behavior).
"""
result: Any = False
if self.is_loaded_ctranslate2_model is True:
try:
self.ctranslate2_tokenizer.src_lang = source_language
@@ -119,7 +160,11 @@ class Translator():
return result
@staticmethod
def getLanguageCode(translator_name, weight_type, target_country, source_language, target_language):
def getLanguageCode(translator_name: str, weight_type: str, target_country: str, source_language: str, target_language: str) -> Tuple[str, str]:
"""Resolve a friendly language name to translator-specific codes.
Returns (source_code, target_code).
"""
match translator_name:
case "DeepL_API":
if target_language == "English":
@@ -136,23 +181,31 @@ class Translator():
translator_name = weight_type
case _:
pass
source_language=translation_lang[translator_name]["source"][source_language]
target_language=translation_lang[translator_name]["target"][target_language]
source_language = translation_lang[translator_name]["source"][source_language]
target_language = translation_lang[translator_name]["target"][target_language]
return source_language, target_language
def translate(self, translator_name, weight_type, source_language, target_language, target_country, message):
def translate(self, translator_name: str, weight_type: str, source_language: str, target_language: str, target_country: str, message: str) -> Any:
"""Translate `message` using the named translator backend.
Returns translated string on success, or False on failure. When
source_language == target_language the original message is returned.
"""
try:
result = ""
if source_language == target_language:
return message
result: Any = ""
source_language, target_language = self.getLanguageCode(translator_name, weight_type, target_country, source_language, target_language)
match translator_name:
case "DeepL":
if self.is_enable_translators is True:
if self.is_enable_translators is True and other_web_Translator is not None:
result = other_web_Translator(
query_text=message,
translator="deepl",
from_language=source_language,
to_language=target_language,
)
)
case "DeepL_API":
if self.is_enable_translators is True:
if self.deepl_client is None:
@@ -161,7 +214,7 @@ class Translator():
result = self.deepl_client.translate_text(
message,
source_lang=source_language,
target_lang=target_language,
target_lang=target_language
).text
case "Plamo_API":
if self.plamo_client is None:
@@ -182,29 +235,29 @@ class Translator():
output_lang=target_language,
)
case "Google":
if self.is_enable_translators is True:
if self.is_enable_translators is True and other_web_Translator is not None:
result = other_web_Translator(
query_text=message,
translator="google",
from_language=source_language,
to_language=target_language,
)
)
case "Bing":
if self.is_enable_translators is True:
if self.is_enable_translators is True and other_web_Translator is not None:
result = other_web_Translator(
query_text=message,
translator="bing",
from_language=source_language,
to_language=target_language,
)
)
case "Papago":
if self.is_enable_translators is True:
if self.is_enable_translators is True and other_web_Translator is not None:
result = other_web_Translator(
query_text=message,
translator="papago",
from_language=source_language,
to_language=target_language,
)
)
case "CTranslate2":
result = self.translateCTranslate2(
message=message,

15
src-python/models/translation/translation_utils.py
View File

@@ -1,6 +1,8 @@
from os import path as os_path
from os import makedirs as os_makedirs
from requests import get as requests_get
from typing import Callable
import hashlib
import transformers
import ctranslate2
from huggingface_hub import hf_hub_url, list_repo_files
@@ -14,6 +16,15 @@ except Exception:
sys.path.append(os_path.dirname(os_path.dirname(os_path.dirname(os_path.abspath(__file__)))))
from utils import errorLogging, getBestComputeType
"""Utilities for downloading and verifying CTranslate2 weights and tokenizers.
This module provides a small, dependency-light set of helpers used by the
translation layer. It purposely keeps behavior resilient: network errors are
logged (via utils.errorLogging) and the functions return/complete without
raising, which matches the repository's defensive style.
"""
ctranslate2_weights = {
"m2m100_418M-ct2-int8": {
"hf_repo": "jncraton/m2m100_418M-ct2-int8",
@@ -84,8 +95,8 @@ def downloadCTranslate2Tokenizer(path: str, weight_type: str = "m2m100_418M-ct2-
tokenizer = ctranslate2_weights[weight_type]["tokenizer"]
tokenizer_path = os_path.join(path, "weights", "ctranslate2", directory_name, "tokenizer")
try:
os_makedirs(tokenizer_path, exist_ok=True)
transformers.AutoTokenizer.from_pretrained(tokenizer, cache_dir=tokenizer_path)
os_makedirs(tokenizer_cache, exist_ok=True)
transformers.AutoTokenizer.from_pretrained(tokenizer_name, cache_dir=tokenizer_cache)
except Exception:
errorLogging()
tokenizer_path = os_path.join("./weights", "ctranslate2", directory_name, "tokenizer")