every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

👍️ [Update] pythonのメイン処理部分を移動/webui_mainloop.pyをビルドできるように修正

Browse Source
This commit is contained in:
misyaguziya
2024-07-27 01:30:36 +09:00
parent 7ce3bc9be9
commit 1be04cb571
21 changed files with 46 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

384
src-python/models/translation/translation_languages.py Normal file
View File

@@ -0,0 +1,384 @@
translation_lang = {}
dict_deepl_languages = {
"Arabic":"ar",
"Bulgarian":"bg",
"Czech":"cs",
"Danish":"da",
"German":"de",
"Greek":"el",
"English":"en",
"Spanish":"es",
"Estonian":"et",
"Finnish":"fi",
"French":"fr",
"Irish":"ga",
"Croatian":"hr",
"Hungarian":"hu",
"Indonesian":"id",
"Icelandic":"is",
"Italian":"it",
"Japanese":"ja",
"Korean":"ko",
"Lithuanian":"lt",
"Latvian":"lv",
"Maltese":"mt",
"Bokmal":"nb",
"Dutch":"nl",
"Norwegian":"no",
"Polish":"pl",
"Portuguese":"pt",
"Romanian":"ro",
"Russian":"ru",
"Slovak":"sk",
"Slovenian":"sl",
"Swedish":"sv",
"Turkish":"tr",
"Ukrainian":"uk",
"Chinese Simplified":"zh",
"Chinese Traditional":"zh"
}
translation_lang["DeepL"] = {
"source":dict_deepl_languages,
"target":dict_deepl_languages,
}
dict_deepl_api_source_languages = {
"Japanese":"ja",
"English":"en",
"Bulgarian":"bg",
"Czech":"cs",
"Danish":"da",
"German":"de",
"Greek":"el",
"Spanish":"es",
"Estonian":"et",
"Finnish":"fi",
"French":"fr",
"Hungarian":"hu",
"Indonesian":"id",
"Italian":"it",
"Korean":"ko",
"Lithuanian":"lt",
"Latvian":"lv",
"Norwegian":"nb",
"Dutch":"nl",
"Polish":"pl",
"Portuguese":"pt",
"Romanian":"ro",
"Russian":"ru",
"Slovak":"sk",
"Slovenian":"sl",
"Swedish":"sv",
"Turkish":"tr",
"Ukrainian":"uk",
"Chinese Simplified":"zh",
"Chinese Traditional":"zh"
}
dict_deepl_api_target_languages = {
"Japanese":"ja",
"English American":"en-US",
"English British":"en-GB",
"Bulgarian":"bg",
"Czech":"cs",
"Danish":"da",
"German":"de",
"Greek":"el",
"English":"en",
"Spanish":"es",
"Estonian":"et",
"Finnish":"fi",
"French":"fr",
"Hungarian":"hu",
"Indonesian":"id",
"Italian":"it",
"Korean":"ko",
"Lithuanian":"lt",
"Latvian":"lv",
"Norwegian":"nb",
"Dutch":"nl",
"Polish":"pl",
"Portuguese Brazilian":"pt-BR",
"Portuguese European":"pt-PT",
"Romanian":"ro",
"Russian":"ru",
"Slovak":"sk",
"Slovenian":"sl",
"Swedish":"sv",
"Turkish":"tr",
"Ukrainian":"uk",
"Chinese Simplified":"zh",
"Chinese Traditional":"zh"
}
translation_lang["DeepL_API"] = {
"source": dict_deepl_api_source_languages,
"target": dict_deepl_api_target_languages,
}
dict_google_languages = {
"Japanese":"ja",
"English":"en",
"Chinese Simplified":"zh",
"Chinese Traditional":"zh-TW",
"Arabic":"ar",
"Russian":"ru",
"French":"fr",
"German":"de",
"Spanish":"es",
"Portuguese":"pt",
"Italian":"it",
"Korean":"ko",
"Greek":"el",
"Dutch":"nl",
"Hindi":"hi",
"Turkish":"tr",
"Malay":"ms",
"Thai":"th",
"Vietnamese":"vi",
"Indonesian":"id",
"Hebrew":"he",
"Polish":"pl",
"Mongolian":"mn",
"Czech":"cs",
"Hungarian":"hu",
"Estonian":"et",
"Bulgarian":"bg",
"Danish":"da",
"Finnish":"fi",
"Romanian":"ro",
"Swedish":"sv",
"Slovenian":"sl",
"Persian/Farsi":"fa",
"Bosnian":"bs",
"Serbian":"sr",
"Filipino":"tl",
"Haitiancreole":"ht",
"Catalan":"ca",
"Croatian":"hr",
"Latvian":"lv",
"Lithuanian":"lt",
"Urdu":"ur",
"Ukrainian":"uk",
"Welsh":"cy",
"Swahili":"sw",
"Samoan":"sm",
"Slovak":"sk",
"Afrikaans":"af",
"Norwegian":"no",
"Bengali":"bn",
"Malagasy":"mg",
"Maltese":"mt",
"Gujarati":"gu",
"Tamil":"ta",
"Telugu":"te",
"Punjabi":"pa",
"Amharic":"am",
"Azerbaijani":"az",
"Belarusian":"be",
"Cebuano":"ceb",
"Esperanto":"eo",
"Basque":"eu",
"Irish":"ga"
}
translation_lang["Google"] = {
"source":dict_google_languages,
"target":dict_google_languages,
}
dict_bing_languages = {
"Japanese":"ja",
"English":"en",
"Chinese Simplified":"zh",
"Chinese Traditional":"zh-Hant",
"Arabic":"ar",
"Russian":"ru",
"French":"fr",
"German":"de",
"Spanish":"es",
"Portuguese":"pt",
"Italian":"it",
"Korean":"ko",
"Greek":"el",
"Dutch":"nl",
"Hindi":"hi",
"Turkish":"tr",
"Malay":"ms",
"Thai":"th",
"Vietnamese":"vi",
"Indonesian":"id",
"Hebrew":"he",
"Polish":"pl",
"Czech":"cs",
"Hungarian":"hu",
"Estonian":"et",
"Bulgarian":"bg",
"Danish":"da",
"Finnish":"fi",
"Romanian":"ro",
"Swedish":"sv",
"Slovenian":"sl",
"Persian/Farsi":"fa",
"Bosnian":"bs",
"Serbian":"sr",
"Fijian":"fj",
"Filipino":"tl",
"Haitiancreole":"ht",
"Catalan":"ca",
"Croatian":"hr",
"Latvian":"lv",
"Lithuanian":"lt",
"Urdu":"ur",
"Ukrainian":"uk",
"Welsh":"cy",
"Tahiti":"ty",
"Tongan":"to",
"Swahili":"sw",
"Samoan":"sm",
"Slovak":"sk",
"Afrikaans":"af",
"Norwegian":"no",
"Bengali":"bn",
"Malagasy":"mg",
"Maltese":"mt",
"Queretaro otomi":"otq",
"Klingon/tlhingan Hol":"tlh",
"Gujarati":"gu",
"Tamil":"ta",
"Telugu":"te",
"Punjabi":"pa",
"Irish":"ga"
}
translation_lang["Bing"] = {
"source":dict_bing_languages,
"target":dict_bing_languages,
}
dict_papago_languages = {
"German": "de",
"English": "en",
"Spanish":"es",
"French": "fr",
"Hindi": "hi",
"Indonesian": "id",
"Italian": "it",
"Japanese": "ja",
"Korean": "ko",
"Portuguese": "pt",
"Russian": "ru",
"Thai": "th",
"Vietnamese": "vi",
"Chinese Simplified":"zh-CN",
"Chinese Traditional":"zh-TW",
}
translation_lang["Papago"] = {
"source":dict_papago_languages,
"target":dict_papago_languages,
}
dict_ctranslate2_languages = {
"English": "en",
"Chinese Simplified": "zh",
"Chinese Traditional":"zh",
"German": "de",
"Spanish": "es",
"Russian": "ru",
"Korean": "ko",
"French": "fr",
"Japanese": "ja",
"Portuguese": "pt",
"Turkish": "tr",
"Polish": "pl",
"Catalan": "ca",
"Dutch": "nl",
"Arabic": "ar",
"Swedish": "sv",
"Italian": "it",
"Indonesian": "id",
"Hindi": "hi",
"Finnish": "fi",
"Vietnamese": "vi",
"Hebrew": "he",
"Ukrainian": "uk",
"Greek": "el",
"Malay": "ms",
"Czech": "cs",
"Romanian": "ro",
"Danish": "da",
"Hungarian": "hu",
"Tamil": "ta",
"Norwegian": "no",
"Thai": "th",
"Urdu": "ur",
"Croatian": "hr",
"Bulgarian": "bg",
"Lithuanian": "lt",
"Latin": "la",
"Maori": "mi",
"Malayalam": "ml",
"Welsh": "cy",
"Slovak": "sk",
"Telugu": "te",
"Persian": "fa",
"Latvian": "lv",
"Bengali": "bn",
"Serbian": "sr",
"Azerbaijani": "az",
"Slovenian": "sl",
"Kannada": "kn",
"Estonian": "et",
"Macedonian": "mk",
"Breton": "br",
"Basque": "eu",
"Icelandic": "is",
"Armenian": "hy",
"Nepali": "ne",
"Mongolian": "mn",
"Bosnian": "bs",
"Kazakh": "kk",
"Albanian": "sq",
"Swahili": "sw",
"Galician": "gl",
"Marathi": "mr",
"Punjabi": "pa",
"Sinhala": "si",
"Khmer": "km",
"Shona": "sn",
"Yoruba": "yo",
"Somali": "so",
"Afrikaans": "af",
"Occitan": "oc",
"Georgian": "ka",
"Belarusian": "be",
"Tajik": "tg",
"Sindhi": "sd",
"Gujarati": "gu",
"Amharic": "am",
"Yiddish": "yi",
"Lao": "lo",
"Uzbek": "uz",
"Faroese": "fo",
"Haitian creole": "ht",
"Pashto": "ps",
"Turkmen": "tk",
"Nynorsk": "nn",
"Maltese": "mt",
"Sanskrit": "sa",
"Luxembourgish": "lb",
"Myanmar": "my",
"Tibetan": "bo",
"Filipino": "tl",
"Malagasy": "mg",
"Assamese": "as",
"Tatar": "tt",
"Hawaiian": "haw",
"Lingala": "ln",
"Hausa": "ha",
"Bashkir": "ba",
"Javanese": "jw",
"Sundanese": "su"
}
translation_lang["CTranslate2"] = {
"source":dict_ctranslate2_languages,
"target":dict_ctranslate2_languages,
}

140
src-python/models/translation/translation_translator.py Normal file
View File

@@ -0,0 +1,140 @@
import os
from deepl import Translator as deepl_Translator
from translators import translate_text as other_web_Translator
from .translation_languages import translation_lang
from .translation_utils import ctranslate2_weights
import ctranslate2
import transformers
# Translator
class Translator():
def __init__(self):
self.deepl_client = None
self.ctranslate2_translator = None
self.ctranslate2_tokenizer = None
self.is_loaded_ctranslate2_model = False
def authenticationDeepLAuthKey(self, authkey):
result = True
try:
self.deepl_client = deepl_Translator(authkey)
self.deepl_client.translate_text(" ", target_lang="EN-US")
except Exception:
self.deepl_client = None
result = False
return result
def changeCTranslate2Model(self, path, model_type):
self.is_loaded_ctranslate2_model = False
directory_name = ctranslate2_weights[model_type]["directory_name"]
tokenizer = ctranslate2_weights[model_type]["tokenizer"]
weight_path = os.path.join(path, "weights", "ctranslate2", directory_name)
tokenizer_path = os.path.join(path, "weights", "ctranslate2", directory_name, "tokenizer")
self.ctranslate2_translator = ctranslate2.Translator(
weight_path,
device="cpu",
device_index=0,
compute_type="int8",
inter_threads=1,
intra_threads=4
)
try:
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer, cache_dir=tokenizer_path)
except Exception as e:
print("Error: changeCTranslate2Model()", e)
tokenizer_path = os.path.join("./weights", "ctranslate2", directory_name, "tokenizer")
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer, cache_dir=tokenizer_path)
self.is_loaded_ctranslate2_model = True
def isLoadedCTranslate2Model(self):
return self.is_loaded_ctranslate2_model
def translateCTranslate2(self, message, source_language, target_language):
result = False
if self.is_loaded_ctranslate2_model is True:
try:
self.ctranslate2_tokenizer.src_lang = source_language
source = self.ctranslate2_tokenizer.convert_ids_to_tokens(self.ctranslate2_tokenizer.encode(message))
target_prefix = [self.ctranslate2_tokenizer.lang_code_to_token[target_language]]
results = self.ctranslate2_translator.translate_batch([source], target_prefix=[target_prefix])
target = results[0].hypotheses[0][1:]
result = self.ctranslate2_tokenizer.decode(self.ctranslate2_tokenizer.convert_tokens_to_ids(target))
except Exception:
pass
return result
@staticmethod
def getLanguageCode(translator_name, target_country, source_language, target_language):
match translator_name:
case "DeepL_API":
if target_language == "English":
if target_country in ["United States", "Canada", "Philippines"]:
target_language = "English American"
else:
target_language = "English British"
elif target_language == "Portuguese":
if target_country in ["Portugal"]:
target_language = "Portuguese European"
else:
target_language = "Portuguese Brazilian"
case _:
pass
source_language=translation_lang[translator_name]["source"][source_language]
target_language=translation_lang[translator_name]["target"][target_language]
return source_language, target_language
def translate(self, translator_name, source_language, target_language, target_country, message):
try:
result = ""
source_language, target_language = self.getLanguageCode(translator_name, target_country, source_language, target_language)
match translator_name:
case "DeepL":
result = other_web_Translator(
query_text=message,
translator="deepl",
from_language=source_language,
to_language=target_language,
)
case "DeepL_API":
if self.deepl_client is None:
result = False
else:
result = self.deepl_client.translate_text(
message,
source_lang=source_language,
target_lang=target_language,
).text
case "Google":
result = other_web_Translator(
query_text=message,
translator="google",
from_language=source_language,
to_language=target_language,
)
case "Bing":
result = other_web_Translator(
query_text=message,
translator="bing",
from_language=source_language,
to_language=target_language,
)
case "Papago":
result = other_web_Translator(
query_text=message,
translator="papago",
from_language=source_language,
to_language=target_language,
)
case "CTranslate2":
result = self.translateCTranslate2(
message=message,
source_language=source_language,
target_language=target_language,
)
except Exception:
import traceback
with open('error.log', 'a') as f:
traceback.print_exc(file=f)
result = False
return result

86
src-python/models/translation/translation_utils.py Normal file
View File

@@ -0,0 +1,86 @@
import tempfile
from zipfile import ZipFile
from os import path as os_path
from os import makedirs as os_makedirs
from requests import get as requests_get
from typing import Callable
import hashlib
ctranslate2_weights = {
"Small": { # M2M-100 418M-parameter model
"url": "https://github.com/misyaguziya/VRCT-weights/releases/download/v1.0/m2m100_418m.zip",
"directory_name": "m2m100_418m",
"tokenizer": "facebook/m2m100_418M",
"hash": {
"model.bin": "e7c26a9abb5260abd0268fbe3040714070dec254a990b4d7fd3f74c5230e3acb",
"sentencepiece.model": "d8f7c76ed2a5e0822be39f0a4f95a55eb19c78f4593ce609e2edbc2aea4d380a",
"shared_vocabulary.txt": "bd440aa21b8ca3453fc792a0018a1f3fe68b3464aadddd4d16a4b72f73c86d8c",
}
},
"Large": { # M2M-100 1.2B-parameter model
"url": "https://github.com/misyaguziya/VRCT-weights/releases/download/v1.0/m2m100_12b.zip",
"directory_name": "m2m100_12b",
"tokenizer": "facebook/m2m100_1.2b",
"hash": {
"model.bin": "abb7bf4ba7e5e016b6e3ed480c752459b2f783ac8fca372e7587675e5bf3a919",
"sentencepiece.model": "d8f7c76ed2a5e0822be39f0a4f95a55eb19c78f4593ce609e2edbc2aea4d380a",
"shared_vocabulary.txt": "bd440aa21b8ca3453fc792a0018a1f3fe68b3464aadddd4d16a4b72f73c86d8c",
}
},
}
def calculate_file_hash(file_path, block_size=65536):
hash_object = hashlib.sha256()
with open(file_path, 'rb') as file:
for block in iter(lambda: file.read(block_size), b''):
hash_object.update(block)
return hash_object.hexdigest()
def checkCTranslate2Weight(path, weight_type="Small"):
weight_directory_name = ctranslate2_weights[weight_type]["directory_name"]
hash_data = ctranslate2_weights[weight_type]["hash"]
files = [
"model.bin",
"sentencepiece.model",
"shared_vocabulary.txt"
]
# check already downloaded
already_downloaded = False
if all(os_path.exists(os_path.join(path, weight_directory_name, file)) for file in files):
# check hash
for file in files:
original_hash = hash_data[file]
current_hash = calculate_file_hash(os_path.join(path, weight_directory_name, file))
if original_hash != current_hash:
break
already_downloaded = True
return already_downloaded
def downloadCTranslate2Weight(root, weight_type="Small", func=None):
url = ctranslate2_weights[weight_type]["url"]
filename = "weight.zip"
path = os_path.join(root, "weights", "ctranslate2")
os_makedirs(path, exist_ok=True)
if checkCTranslate2Weight(path, weight_type):
return
try:
with tempfile.TemporaryDirectory() as tmp_path:
res = requests_get(url, stream=True)
file_size = int(res.headers.get('content-length', 0))
total_chunk = 0
with open(os_path.join(tmp_path, filename), 'wb') as file:
for chunk in res.iter_content(chunk_size=1024*5):
file.write(chunk)
if isinstance(func, Callable):
total_chunk += len(chunk)
func(total_chunk/file_size)
with ZipFile(os_path.join(tmp_path, filename)) as zf:
zf.extractall(path)
except Exception as e:
print("error:downloadCTranslate2Weight()", e)