From 1f5a2e92cc7577d78f367fc8f0dc3c4392e62ad5 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Wed, 15 Nov 2023 14:38:37 +0900 Subject: [PATCH 01/20] =?UTF-8?q?[Add]=20Model:=20ctranslate2=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- model.py | 4 +- models/translation/translation_languages.py | 107 +++++++++++++++++++ models/translation/translation_translator.py | 27 +++++ requirements.txt | 5 +- 4 files changed, 140 insertions(+), 3 deletions(-) diff --git a/model.py b/model.py index bff713f4..431898ad 100644 --- a/model.py +++ b/model.py @@ -161,7 +161,7 @@ class Model: elif target_language in ["Portuguese European", "Portuguese Brazilian"]: target_language = "Portuguese" - translation = self.translator.translate( + translation = self.translator.translate_ctranslate2( translator_name=translator_name, source_language=source_language, target_language=target_language, @@ -192,7 +192,7 @@ class Model: elif target_language in ["Portuguese European", "Portuguese Brazilian"]: target_language = "Portuguese" - translation = self.translator.translate( + translation = self.translator.translate_ctranslate2( translator_name=translator_name, source_language=source_language, target_language=target_language, diff --git a/models/translation/translation_languages.py b/models/translation/translation_languages.py index ae57d4cc..d5645cd2 100644 --- a/models/translation/translation_languages.py +++ b/models/translation/translation_languages.py @@ -240,4 +240,111 @@ dict_bing_languages = { translation_lang["Bing"] = { "source":dict_bing_languages, "target":dict_bing_languages, +} + +dict_ctranslate2_lang = { + 'English': 'en', + 'Chinese': 'zh', + 'German': 'de', + 'Spanish': 'es', + 'Russian': 'ru', + 'Korean': 'ko', + 'French': 'fr', + 'Japanese': 'ja', + 'Portuguese': 'pt', + 'Turkish': 'tr', + 'Polish': 'pl', + 'Catalan': 'ca', + 'Dutch': 'nl', + 'Arabic': 'ar', + 'Swedish': 'sv', + 'Italian': 'it', + 'Indonesian': 'id', + 'Hindi': 'hi', + 'Finnish': 'fi', + 'Vietnamese': 'vi', + 'Hebrew': 'he', + 'Ukrainian': 'uk', + 'Greek': 'el', + 'Malay': 'ms', + 'Czech': 'cs', + 'Romanian': 'ro', + 'Danish': 'da', + 'Hungarian': 'hu', + 'Tamil': 'ta', + 'Norwegian': 'no', + 'Thai': 'th', + 'Urdu': 'ur', + 'Croatian': 'hr', + 'Bulgarian': 'bg', + 'Lithuanian': 'lt', + 'Latin': 'la', + 'Maori': 'mi', + 'Malayalam': 'ml', + 'Welsh': 'cy', + 'Slovak': 'sk', + 'Telugu': 'te', + 'Persian': 'fa', + 'Latvian': 'lv', + 'Bengali': 'bn', + 'Serbian': 'sr', + 'Azerbaijani': 'az', + 'Slovenian': 'sl', + 'Kannada': 'kn', + 'Estonian': 'et', + 'Macedonian': 'mk', + 'Breton': 'br', + 'Basque': 'eu', + 'Icelandic': 'is', + 'Armenian': 'hy', + 'Nepali': 'ne', + 'Mongolian': 'mn', + 'Bosnian': 'bs', + 'Kazakh': 'kk', + 'Albanian': 'sq', + 'Swahili': 'sw', + 'Galician': 'gl', + 'Marathi': 'mr', + 'Punjabi': 'pa', + 'Sinhala': 'si', + 'Khmer': 'km', + 'Shona': 'sn', + 'Yoruba': 'yo', + 'Somali': 'so', + 'Afrikaans': 'af', + 'Occitan': 'oc', + 'Georgian': 'ka', + 'Belarusian': 'be', + 'Tajik': 'tg', + 'Sindhi': 'sd', + 'Gujarati': 'gu', + 'Amharic': 'am', + 'Yiddish': 'yi', + 'Lao': 'lo', + 'Uzbek': 'uz', + 'Faroese': 'fo', + 'Haitian creole': 'ht', + 'Pashto': 'ps', + 'Turkmen': 'tk', + 'Nynorsk': 'nn', + 'Maltese': 'mt', + 'Sanskrit': 'sa', + 'Luxembourgish': 'lb', + 'Myanmar': 'my', + 'Tibetan': 'bo', + 'Tagalog': 'tl', + 'Malagasy': 'mg', + 'Assamese': 'as', + 'Tatar': 'tt', + 'Hawaiian': 'haw', + 'Lingala': 'ln', + 'Hausa': 'ha', + 'Bashkir': 'ba', + 'Javanese': 'jw', + 'Sundanese': 'su' +} + +translation_lang["ctranslate2"] = { + "source":dict_ctranslate2_lang, + "target":dict_ctranslate2_lang, } \ No newline at end of file diff --git a/models/translation/translation_translator.py b/models/translation/translation_translator.py index c3a5682b..d15a05c4 100644 --- a/models/translation/translation_translator.py +++ b/models/translation/translation_translator.py @@ -3,12 +3,24 @@ from deepl_translate import translate as deepl_web_Translator from translators import translate_text as other_web_Translator from .translation_languages import translation_lang +from ctranslate2.converters import TransformersConverter +import ctranslate2 +import transformers + +TRANSLATE_MODELS = { + "small": "facebook/m2m100_418M", + "large": "facebook/m2m100_1.2B" +} + # Translator class Translator(): def __init__(self): pass self.translator_status = {} + self.translator = ctranslate2.Translator("D:\\WORKSPACE\\WORK\\VRChatProject\\VRCT\\weight", device="cpu", device_index=0, compute_type="int8", inter_threads=1, intra_threads=4) + self.tokenizer = transformers.AutoTokenizer.from_pretrained("facebook/m2m100_418M") + def authentication(self, translator_name, authkey=None): result = True match translator_name: @@ -57,4 +69,19 @@ class Translator(): with open('error.log', 'a') as f: traceback.print_exc(file=f) result = False + return result + + def translate_ctranslate2(self, translator_name, source_language, target_language, message): + + source_language=translation_lang["ctranslate2"]["source"][source_language] + target_language=translation_lang["ctranslate2"]["target"][target_language] + + self.tokenizer.src_lang = source_language + source = self.tokenizer.convert_ids_to_tokens(self.tokenizer.encode(message)) + target_prefix = [self.tokenizer.lang_code_to_token[target_language]] + results = self.translator.translate_batch([source], target_prefix=[target_prefix]) + target = results[0].hypotheses[0][1:] + + result = self.tokenizer.decode(self.tokenizer.convert_tokens_to_ids(target)) + print(result) return result \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 42f4be2c..ca48a2b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,7 @@ deepl == 1.15.0 flashtext == 2.7 pyyaml == 6.0.1 python-i18n == 0.3.9 -CTkToolTip == 0.8 \ No newline at end of file +CTkToolTip == 0.8 +transformers[torch] +sentencepiece==0.1.99 +ctranslate2==3.21.0 \ No newline at end of file From 1a766cfde3449ceb2312f8e7819957ed258ab3a0 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Wed, 15 Nov 2023 14:39:27 +0900 Subject: [PATCH 02/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20setup=20:=20pip?= =?UTF-8?q?=E3=81=AE=E3=82=A2=E3=83=83=E3=83=97=E3=83=87=E3=83=BC=E3=83=88?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- install.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/install.bat b/install.bat index d54ef800..f312bf4d 100644 --- a/install.bat +++ b/install.bat @@ -1,3 +1,4 @@ +python.exe -m pip install --upgrade pip pip install -r requirements.txt pip install git+https://github.com/misyaguziya/translators pip install git+https://github.com/misyaguziya/deepl-translate From dcb6c07eee40eecf3c4a1d8d58101cb6a108b4c0 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Wed, 15 Nov 2023 16:06:45 +0900 Subject: [PATCH 03/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20weight?= =?UTF-8?q?=E3=81=AE=E3=83=91=E3=82=B9=E3=82=92PATH=5FLOACL=E3=81=8B?= =?UTF-8?q?=E3=82=89=E6=8E=A5=E7=B6=9A=E3=81=99=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 10 +++++----- controller.py | 4 ++-- model.py | 8 ++++---- models/translation/translation_translator.py | 9 ++++----- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/config.py b/config.py index b594a1b0..fb0406d2 100644 --- a/config.py +++ b/config.py @@ -40,8 +40,8 @@ class Config: return self._VERSION @property - def LOCAL_PATH(self): - return self._LOCAL_PATH + def PATH_LOCAL(self): + return self._PATH_LOCAL @property def PATH_CONFIG(self): @@ -530,9 +530,9 @@ class Config: def init_config(self): # Read Only self._VERSION = "2.0.1" - self._LOCAL_PATH = os_path.dirname(sys.argv[0]) - self._PATH_CONFIG = os_path.join(self._LOCAL_PATH, "config.json") - self._PATH_LOGS = os_path.join(self._LOCAL_PATH, "logs") + self._PATH_LOCAL = os_path.dirname(sys.argv[0]) + self._PATH_CONFIG = os_path.join(self._PATH_LOCAL, "config.json") + self._PATH_LOGS = os_path.join(self._PATH_LOCAL, "logs") os_makedirs(self._PATH_LOGS, exist_ok=True) self._GITHUB_URL = "https://api.github.com/repos/misyaguziya/VRCT/releases/latest" self._BOOTH_URL = "https://misyaguziya.booth.pm/" diff --git a/controller.py b/controller.py index 2e254997..e6352411 100644 --- a/controller.py +++ b/controller.py @@ -20,8 +20,8 @@ def callbackFilepathLogs(): Popen(['explorer', config.PATH_LOGS.replace('/', '\\')], shell=True) def callbackFilepathConfigFile(): - print("callbackFilepathConfigFile", config.LOCAL_PATH.replace('/', '\\')) - Popen(['explorer', config.LOCAL_PATH.replace('/', '\\')], shell=True) + print("callbackFilepathConfigFile", config.PATH_LOCAL.replace('/', '\\')) + Popen(['explorer', config.PATH_LOCAL.replace('/', '\\')], shell=True) # func transcription send message def sendMicMessage(message): diff --git a/model.py b/model.py index 431898ad..071a5ff0 100644 --- a/model.py +++ b/model.py @@ -70,12 +70,12 @@ class Model: self.speaker_audio_recorder = None self.speaker_energy_recorder = None self.speaker_energy_plot_progressbar = None - self.translator = Translator() + self.translator = Translator(config.PATH_LOCAL) self.keyword_processor = KeywordProcessor() def resetTranslator(self): del self.translator - self.translator = Translator() + self.translator = Translator(config.PATH_LOCAL) def resetKeywordProcessor(self): del self.keyword_processor @@ -271,7 +271,7 @@ class Model: folder_name = '_internal' tmp_directory_name = 'tmp' batch_name = 'update.bat' - current_directory = config.LOCAL_PATH + current_directory = config.PATH_LOCAL try: res = requests_get(config.GITHUB_URL) @@ -296,7 +296,7 @@ class Model: program_name = 'VRCT.exe' folder_name = '_internal' batch_name = 'restart.bat' - current_directory = config.LOCAL_PATH + current_directory = config.PATH_LOCAL copyfile(os_path.join(current_directory, folder_name, "batch", batch_name), os_path.join(current_directory, batch_name)) command = [os_path.join(current_directory, batch_name), program_name] Popen(command, cwd=current_directory) diff --git a/models/translation/translation_translator.py b/models/translation/translation_translator.py index d15a05c4..e78c803d 100644 --- a/models/translation/translation_translator.py +++ b/models/translation/translation_translator.py @@ -1,3 +1,4 @@ +import os from deepl import Translator as deepl_Translator from deepl_translate import translate as deepl_web_Translator from translators import translate_text as other_web_Translator @@ -14,11 +15,10 @@ TRANSLATE_MODELS = { # Translator class Translator(): - def __init__(self): - pass + def __init__(self, path): self.translator_status = {} - - self.translator = ctranslate2.Translator("D:\\WORKSPACE\\WORK\\VRChatProject\\VRCT\\weight", device="cpu", device_index=0, compute_type="int8", inter_threads=1, intra_threads=4) + self.weight_path = os.path.join(path, "weight") + self.translator = ctranslate2.Translator(self.weight_path, device="cpu", device_index=0, compute_type="int8", inter_threads=1, intra_threads=4) self.tokenizer = transformers.AutoTokenizer.from_pretrained("facebook/m2m100_418M") def authentication(self, translator_name, authkey=None): @@ -83,5 +83,4 @@ class Translator(): target = results[0].hypotheses[0][1:] result = self.tokenizer.decode(self.tokenizer.convert_tokens_to_ids(target)) - print(result) return result \ No newline at end of file From e6e62cf35034f5787d77042ede1da69e39af0168 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Sun, 19 Nov 2023 00:03:57 +0900 Subject: [PATCH 04/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20CTrans?= =?UTF-8?q?late2=E3=81=AEweight=E3=83=87=E3=83=BC=E3=82=BF=E3=82=92?= =?UTF-8?q?=E8=B5=B7=E5=8B=95=E6=99=82=E3=81=AB=E5=8F=96=E5=BE=97=E3=81=99?= =?UTF-8?q?=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- config.py | 28 ++++++++++++++ model.py | 39 +++++++++++++++++++- models/translation/translation_translator.py | 9 +++-- 4 files changed, 72 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 0d95c681..2223cde5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ memo.txt VRCT.spec *.pyc logs/ -.venv/ \ No newline at end of file +.venv/ +weight/ \ No newline at end of file diff --git a/config.py b/config.py index fb0406d2..e9afe1d9 100644 --- a/config.py +++ b/config.py @@ -63,6 +63,10 @@ class Config: def DOCUMENTS_URL(self): return self._DOCUMENTS_URL + @property + def CTRANSLATE2_WIGHTS(self): + return self._CTRANSLATE2_WIGHTS + @property def MAX_MIC_ENERGY_THRESHOLD(self): return self._MAX_MIC_ENERGY_THRESHOLD @@ -447,6 +451,17 @@ class Config: self._AUTH_KEYS[key] = value saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, self.AUTH_KEYS) + @property + @json_serializable('WEIGHT_TYPE') + def WEIGHT_TYPE(self): + return self._WEIGHT_TYPE + + @WEIGHT_TYPE.setter + def WEIGHT_TYPE(self, value): + if isinstance(value, str): + self._WEIGHT_TYPE = value + saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value) + @property @json_serializable('MESSAGE_FORMAT') def MESSAGE_FORMAT(self): @@ -537,6 +552,18 @@ class Config: self._GITHUB_URL = "https://api.github.com/repos/misyaguziya/VRCT/releases/latest" self._BOOTH_URL = "https://misyaguziya.booth.pm/" self._DOCUMENTS_URL = "https://mzsoftware.notion.site/VRCT-Documents-be79b7a165f64442ad8f326d86c22246" + self._CTRANSLATE2_WIGHTS = { + "small": { # M2M-100 418M-parameter model + "url": "https://bit.ly/33fM1AO", + "directory_name": "m2m100_418m", + "tokenizer": "facebook/m2m100_418M" + }, + "large": { # M2M-100 1.2B-parameter model + "url": "https://bit.ly/3GYiaed", + "directory_name": "m2m100_12b", + "tokenizer": "facebook/m2m100_12b" + }, + } self._MAX_MIC_ENERGY_THRESHOLD = 2000 self._MAX_SPEAKER_ENERGY_THRESHOLD = 4000 @@ -594,6 +621,7 @@ class Config: "Bing": None, "Google": None, } + self.WEIGHT_TYPE = "small" self._MESSAGE_FORMAT = "[message]([translation])" self._ENABLE_AUTO_CLEAR_MESSAGE_BOX = True self._ENABLE_NOTICE_XSOVERLAY = False diff --git a/model.py b/model.py index 071a5ff0..21ea5fd7 100644 --- a/model.py +++ b/model.py @@ -1,3 +1,4 @@ +import tempfile from zipfile import ZipFile from subprocess import Popen from os import makedirs as os_makedirs @@ -9,9 +10,10 @@ from logging import getLogger, FileHandler, Formatter, INFO from time import sleep from queue import Queue from threading import Thread, Event -from requests import get as requests_get +from requests import get as requests_get, head as requests_head import webbrowser +from tqdm import tqdm from flashtext import KeywordProcessor from models.translation.translation_translator import Translator from models.transcription.transcription_utils import getInputDevices, getDefaultOutputDevice @@ -70,7 +72,8 @@ class Model: self.speaker_audio_recorder = None self.speaker_energy_recorder = None self.speaker_energy_plot_progressbar = None - self.translator = Translator(config.PATH_LOCAL) + self.downloadCTranslate2Weight() + self.translator = Translator(config.PATH_LOCAL, config.CTRANSLATE2_WIGHTS[config.WEIGHT_TYPE]) self.keyword_processor = KeywordProcessor() def resetTranslator(self): @@ -106,6 +109,38 @@ class Model: self.logger.disabled = True self.logger = None + @staticmethod + def downloadCTranslate2Weight(): + weight_type = config.WEIGHT_TYPE + url = config.CTRANSLATE2_WIGHTS[weight_type]["url"] + filename = 'weight.zip' + directory_name = 'weight' + current_directory = config.PATH_LOCAL + weight_directory_name = config.CTRANSLATE2_WIGHTS[weight_type]["directory_name"] + files = ["model.bin", "sentencepiece.model", "shared_vocabulary.txt"] + + # check already downloaded + if all(os_path.exists(os_path.join(current_directory, directory_name, weight_directory_name, file)) for file in files): + return + + try: + os_makedirs(os_path.join(current_directory, directory_name), exist_ok=True) + print(os_path.join(current_directory, directory_name)) + with tempfile.TemporaryDirectory() as tmp_path: + file_size = int(requests_head(url).headers["content-length"]) + res = requests_get(url, stream=True) + pbar = tqdm(total=file_size, unit="B", unit_scale=True) + with open(os_path.join(tmp_path, filename), 'wb') as file: + for chunk in res.iter_content(chunk_size=1024): + file.write(chunk) + pbar.update(len(chunk)) + pbar.close() + + with ZipFile(os_path.join(tmp_path, filename)) as zf: + zf.extractall(os_path.join(current_directory, directory_name)) + except Exception as e: + print("error:downloadCTranslate2Weight()", e) + @staticmethod def getListLanguageAndCountry(): langs = [] diff --git a/models/translation/translation_translator.py b/models/translation/translation_translator.py index e78c803d..aa9c6d5c 100644 --- a/models/translation/translation_translator.py +++ b/models/translation/translation_translator.py @@ -4,7 +4,6 @@ from deepl_translate import translate as deepl_web_Translator from translators import translate_text as other_web_Translator from .translation_languages import translation_lang -from ctranslate2.converters import TransformersConverter import ctranslate2 import transformers @@ -15,11 +14,13 @@ TRANSLATE_MODELS = { # Translator class Translator(): - def __init__(self, path): + def __init__(self, path, weight_config): self.translator_status = {} - self.weight_path = os.path.join(path, "weight") + directory_name = weight_config["directory_name"] + tokenizer = weight_config["tokenizer"] + self.weight_path = os.path.join(path, "weight", directory_name) self.translator = ctranslate2.Translator(self.weight_path, device="cpu", device_index=0, compute_type="int8", inter_threads=1, intra_threads=4) - self.tokenizer = transformers.AutoTokenizer.from_pretrained("facebook/m2m100_418M") + self.tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer) def authentication(self, translator_name, authkey=None): result = True From 92ea7914b8f4c74862b8c10dfc7711c01282074e Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Sun, 26 Nov 2023 00:30:59 +0900 Subject: [PATCH 05/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20model=20:=20?= =?UTF-8?q?=E3=83=A2=E3=83=87=E3=83=AB=E3=81=AE=E3=83=AD=E3=83=BC=E3=83=89?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=E3=82=92=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 4 ++++ model.py | 33 --------------------------------- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/main.py b/main.py index def1cb0b..14c97ac4 100644 --- a/main.py +++ b/main.py @@ -7,6 +7,10 @@ if __name__ == "__main__": splash = SplashWindow() splash.showSplash() + from config import config + from models.translation.utils import downloadCTranslate2Weight + downloadCTranslate2Weight(config.PATH_LOCAL, config.WEIGHT_TYPE, config.CTRANSLATE2_WIGHTS) + import controller controller.createMainWindow() splash.destroySplash() diff --git a/model.py b/model.py index 21ea5fd7..ef035613 100644 --- a/model.py +++ b/model.py @@ -72,7 +72,6 @@ class Model: self.speaker_audio_recorder = None self.speaker_energy_recorder = None self.speaker_energy_plot_progressbar = None - self.downloadCTranslate2Weight() self.translator = Translator(config.PATH_LOCAL, config.CTRANSLATE2_WIGHTS[config.WEIGHT_TYPE]) self.keyword_processor = KeywordProcessor() @@ -109,38 +108,6 @@ class Model: self.logger.disabled = True self.logger = None - @staticmethod - def downloadCTranslate2Weight(): - weight_type = config.WEIGHT_TYPE - url = config.CTRANSLATE2_WIGHTS[weight_type]["url"] - filename = 'weight.zip' - directory_name = 'weight' - current_directory = config.PATH_LOCAL - weight_directory_name = config.CTRANSLATE2_WIGHTS[weight_type]["directory_name"] - files = ["model.bin", "sentencepiece.model", "shared_vocabulary.txt"] - - # check already downloaded - if all(os_path.exists(os_path.join(current_directory, directory_name, weight_directory_name, file)) for file in files): - return - - try: - os_makedirs(os_path.join(current_directory, directory_name), exist_ok=True) - print(os_path.join(current_directory, directory_name)) - with tempfile.TemporaryDirectory() as tmp_path: - file_size = int(requests_head(url).headers["content-length"]) - res = requests_get(url, stream=True) - pbar = tqdm(total=file_size, unit="B", unit_scale=True) - with open(os_path.join(tmp_path, filename), 'wb') as file: - for chunk in res.iter_content(chunk_size=1024): - file.write(chunk) - pbar.update(len(chunk)) - pbar.close() - - with ZipFile(os_path.join(tmp_path, filename)) as zf: - zf.extractall(os_path.join(current_directory, directory_name)) - except Exception as e: - print("error:downloadCTranslate2Weight()", e) - @staticmethod def getListLanguageAndCountry(): langs = [] From 0311d4cdfc29c69b58c525fe7b9108cab3d95f9e Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Tue, 28 Nov 2023 10:17:50 +0900 Subject: [PATCH 06/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20model=20:=20transl?= =?UTF-8?q?ation/utils.py=E3=82=92=E8=BF=BD=E5=8A=A0=E3=81=AB=E4=BC=B4?= =?UTF-8?q?=E3=81=84=E4=B8=80=E9=83=A8=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 14 ++-------- main.py | 2 +- models/translation/utils.py | 54 +++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 13 deletions(-) create mode 100644 models/translation/utils.py diff --git a/config.py b/config.py index e9afe1d9..c1cc4759 100644 --- a/config.py +++ b/config.py @@ -8,6 +8,7 @@ from tkinter import font from languages import selectable_languages from models.translation.translation_languages import translatorEngine from models.transcription.transcription_utils import getInputDevices, getDefaultInputDevice +from models.translation.utils import ctranslate2_weights from utils import generatePercentageStringsList, isUniqueStrings json_serializable_vars = {} @@ -552,18 +553,7 @@ class Config: self._GITHUB_URL = "https://api.github.com/repos/misyaguziya/VRCT/releases/latest" self._BOOTH_URL = "https://misyaguziya.booth.pm/" self._DOCUMENTS_URL = "https://mzsoftware.notion.site/VRCT-Documents-be79b7a165f64442ad8f326d86c22246" - self._CTRANSLATE2_WIGHTS = { - "small": { # M2M-100 418M-parameter model - "url": "https://bit.ly/33fM1AO", - "directory_name": "m2m100_418m", - "tokenizer": "facebook/m2m100_418M" - }, - "large": { # M2M-100 1.2B-parameter model - "url": "https://bit.ly/3GYiaed", - "directory_name": "m2m100_12b", - "tokenizer": "facebook/m2m100_12b" - }, - } + self._CTRANSLATE2_WEIGHTS = ctranslate2_weights self._MAX_MIC_ENERGY_THRESHOLD = 2000 self._MAX_SPEAKER_ENERGY_THRESHOLD = 4000 diff --git a/main.py b/main.py index 14c97ac4..ff0aa6a6 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ if __name__ == "__main__": from config import config from models.translation.utils import downloadCTranslate2Weight - downloadCTranslate2Weight(config.PATH_LOCAL, config.WEIGHT_TYPE, config.CTRANSLATE2_WIGHTS) + downloadCTranslate2Weight(config.PATH_LOCAL, config.WEIGHT_TYPE, config.CTRANSLATE2_WEIGHTS, print) import controller controller.createMainWindow() diff --git a/models/translation/utils.py b/models/translation/utils.py new file mode 100644 index 00000000..5b15f16a --- /dev/null +++ b/models/translation/utils.py @@ -0,0 +1,54 @@ +import tempfile +from zipfile import ZipFile +from os import path as os_path +from os import makedirs as os_makedirs +from requests import get as requests_get, head as requests_head +from tqdm import tqdm +from typing import Callable + +ctranslate2_weights = { + "small": { # M2M-100 418M-parameter model + "url": "https://bit.ly/33fM1AO", + "directory_name": "m2m100_418m", + "tokenizer": "facebook/m2m100_418M" + }, + "large": { # M2M-100 1.2B-parameter model + "url": "https://bit.ly/3GYiaed", + "directory_name": "m2m100_12b", + "tokenizer": "facebook/m2m100_12b" + }, +} + +def downloadCTranslate2Weight(path, weight_type="small", ctranslate2_weights=ctranslate2_weights, func=None): + url = ctranslate2_weights[weight_type]["url"] + filename = 'weight.zip' + directory_name = 'weight' + current_directory = path + weight_directory_name = ctranslate2_weights[weight_type]["directory_name"] + files = ["model.bin", "sentencepiece.model", "shared_vocabulary.txt"] + + # check already downloaded + if all(os_path.exists(os_path.join(current_directory, directory_name, weight_directory_name, file)) for file in files): + return + + try: + os_makedirs(os_path.join(current_directory, directory_name), exist_ok=True) + print(os_path.join(current_directory, directory_name)) + with tempfile.TemporaryDirectory() as tmp_path: + res = requests_get(url, stream=True) + file_size = int(res.headers.get('content-length', 0)) + pbar = tqdm(total=file_size, unit="B", unit_scale=True) + total_chunk = 0 + with open(os_path.join(tmp_path, filename), 'wb') as file: + for chunk in res.iter_content(chunk_size=1024): + file.write(chunk) + pbar.update(len(chunk)) + if isinstance(func, Callable): + total_chunk += len(chunk) + func(total_chunk/file_size) + pbar.close() + + with ZipFile(os_path.join(tmp_path, filename)) as zf: + zf.extractall(os_path.join(current_directory, directory_name)) + except Exception as e: + print("error:downloadCTranslate2Weight()", e) \ No newline at end of file From 52abfb8a97e1ccea2816e9ce96edc3ebbdc6d6ed Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Tue, 28 Nov 2023 10:18:51 +0900 Subject: [PATCH 07/20] =?UTF-8?q?=F0=9F=90=9B[bugfix]=20typo=20WIGHTS=20->?= =?UTF-8?q?=20WEIGHTS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 4 ++-- model.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index c1cc4759..34a0d341 100644 --- a/config.py +++ b/config.py @@ -65,8 +65,8 @@ class Config: return self._DOCUMENTS_URL @property - def CTRANSLATE2_WIGHTS(self): - return self._CTRANSLATE2_WIGHTS + def CTRANSLATE2_WEIGHTS(self): + return self._CTRANSLATE2_WEIGHTS @property def MAX_MIC_ENERGY_THRESHOLD(self): diff --git a/model.py b/model.py index ef035613..36de1262 100644 --- a/model.py +++ b/model.py @@ -72,7 +72,7 @@ class Model: self.speaker_audio_recorder = None self.speaker_energy_recorder = None self.speaker_energy_plot_progressbar = None - self.translator = Translator(config.PATH_LOCAL, config.CTRANSLATE2_WIGHTS[config.WEIGHT_TYPE]) + self.translator = Translator(config.PATH_LOCAL, config.CTRANSLATE2_WEIGHTS[config.WEIGHT_TYPE]) self.keyword_processor = KeywordProcessor() def resetTranslator(self): From 69f6e4472017a3a82c3a2ca441fee188aaee4aef Mon Sep 17 00:00:00 2001 From: Sakamoto Shiina <68018796+ShiinaSakamoto@users.noreply.github.com> Date: Wed, 29 Nov 2023 17:11:47 +0900 Subject: [PATCH 08/20] =?UTF-8?q?[Update]=20UI=E8=BF=BD=E5=8A=A0=20?= =?UTF-8?q?=E5=88=9D=E6=9C=9F=E8=B5=B7=E5=8B=95=E6=99=82=E3=81=AE=E8=A8=80?= =?UTF-8?q?=E8=AA=9E=E3=83=A2=E3=83=87=E3=83=AB=E3=83=80=E3=82=A6=E3=83=B3?= =?UTF-8?q?=E3=83=AD=E3=83=BC=E3=83=89=E4=B8=AD=E3=80=81splash=E7=94=BB?= =?UTF-8?q?=E9=9D=A2=E3=81=AB=E3=83=97=E3=83=AD=E3=82=B0=E3=83=AC=E3=82=B9?= =?UTF-8?q?=E3=83=90=E3=83=BC=E8=A1=A8=E7=A4=BA=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- img/VRCT_now_downloading.png | Bin 0 -> 31294 bytes main.py | 2 +- vrct_gui/splash_window/SplashWindow.py | 34 ++++++++++++++++++++++++- 3 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 img/VRCT_now_downloading.png diff --git a/img/VRCT_now_downloading.png b/img/VRCT_now_downloading.png new file mode 100644 index 0000000000000000000000000000000000000000..5e5b9daa328daa530d270f8b35fd3ee6eb02da54 GIT binary patch literal 31294 zcmbrm2{@Gf+Xie2MM&AQJ}G2>>|}c?l08(WjCCq8_HBrpzmh(EV^SXX_^wz02BW zR8&X%!S@M=>B0Y-&OS=u%Mtg>H@v8*pqD8BsM!-Qo(DhV_11xSo4eidM%a1Yrn-3B z*~{PSH*;^FTaIqOU3GJF{*X5oNJaGXZdv=LM{K8+d)PQmi|5WBI_)ak^CS*puibpPmJYnD1k55Q0u7yXa=Mu8+5#@DxByqpsL2>=WAoVcKFdbBW zNh9OX=<$U|;NrI=V#C0fCIiaby>4S+6MJ^tQmFSOI{eW)(@Ys5%F9f7DtPYYJjrjA zCGfaP@Fba!Zm;Bv?rHEVBL&Xm;1kJk^n>e&sA*?k)pw3GeDL3{-iSF2Y2GE1=Ff_w zz@J3wz)pcL%Vu`5j~?B9dr@0Irzz>9!d=uUA(y_H=z|a5XS9`h?)$bHYJk>+YUWU2 zcUFGsC2j39od5gJ>^>U9@imQY6B+-yfrpLl3FQC72mXHZsuztPj|vqZbsVjz@*ajM z)GO2=6kfXSCGztg^nTvMc47U_l*~Ts(jSc#m=N7w0plm5EU0r8{5pEfYz3Kr+2vvv zy5%eOJug{{L4{tOPW_O&pg&%yCw)8cz^hMc9E%HBYW4(3`lu@{7FSpKD(GkQ2+eV% z0Hsp5349^FVEW~6i6te;br_;|aeGpLWHvtHGLI8nNbY$_=^f@OMn92Rwr9G)mvTNS z3&)TdO2It5~=FgJof*~C1;!ozn2cJrkK!d^uB+Y>nIq^vx$#N zG~C+0H;b7!o_Q0f+InFd`6&BUg@{nrCL|A+#Z|~@A@jV0@D$n7&ONmTfzhe3qz#Rgn^3#OxE~ejiT`&a z67319grlwNe*rN|Y4 z;L0^l;1@CzH(H4_yf}w$DGa^HZxg_{DaZzHWBc!f_<0+RNVck8v`ctb=A-D9wY46^ zKzi3T4bPacALL_t7hR0B?|;7u@nXc$stfKuYR>`z_~M+A@1x5F3T>it%muAH1NI)T zC1MfUEB|hw#s!)y^eiEZ|NDs`G#wb3zly}u*tPge)Ars2nY3b9(+=jhpHjsJoig@LOKR>7>Rw1j)D z74xBjBI=w^c*Oql!Ks12d~oYs9q@rNEk4RiMTY_hhUo_xlQMbYMK~WU7Gb9xzNAXG zK%Eiq4IKUN*+O(?|{ zI7m?LnsBon#d1{P?EPs?G#zfQ`Aq%)J-h}W6J=P9IG4N-Bvx67X!D;DylchG=k6Rp zQzbQqe$+=vlV|HARxHCc%WfMj_&Bqc?e{%h&zzIJC@E^F)8@eqqU~1pDR8F@p>Rl? zni~bm{uylm8&X5b=+ui;C~ExB=8%jK$@0Hk_dSCj4!tNqSUnQ2lkH2n(?n92;;4$g z>imYjszs0F2F0)iR?I)I`0y3DV&)-DzIFGg?7suhQ?x?+Ow92X9tYYPs@6~~(ZRe& z-g%lf2jwW+Rd_NuJH5D`Or~+Kgbi5}f^*f_pR`@6tFx}I$Lx1b_p()-)Ayg>@XVBO zyxx9r!D~i8FBl?AaV8a>74a|5%sEq`eTy$vQ9C>KuS6BwHwPiIvEf3v)LO9`s&z)4 zlkLfEi=q3jV(~2(geK0iM1Pzt=jc~wf-=8P*Rt6d2*U>L$23eLA)+bZAXay|v{#S4M&hLtLHeXbm zT++XMI=bQe8r9RsyUD|>f$9sdI4=I3Bw}b`Ic6UCqszR1WuEz79)NfvsS+E;9IQoC ztGU$Czl%|uq9uRGjlA_W(;mkbNtbyIDn)8s?3$4`gMeLyDJSphC74axh zKT_(6<7dc4?ZNzRu7lE1kh-Gf^LY&se9l2gXzDZNc6Vz&szfyk$@yC`!KvyPhsXR& zU|0b{{418?WB=E;LNda;=reM*WGlS@?jH4V54-YrzW$GupWLf@f=w)f)4KyQH=xsE z%E$SnZQdM0>(0OA-4QTK8yNKC$+Q2J)l6m#fd_7WmVy=7aQ(8Cn;kP&G=!{9s%C-G z?RYPI;JEO=aln6hE4Rybos8~O5*Z$W|JU2&H3PTD=FY9sx;kn?RT-;V zb>ScRE?l9VdmhHj=9u+Y#at+0p-^+#qQ3r6n%=p;xM3XANw5tU&$9nD_x%LiRaz(l`%O%R=4s>?H0U;XoF%WnBs!cp%Y~$OQMLN~X4;O14iQ-QwNv`nFm5^~gei zdc`v7u)E)cjSIf;vY0Q8h28S98R0nL!h)FXj7;r1cRzi+{US+~K(+MgBkcL$tqA*6 z$%G==Nx6lMy41z`HATW^OMMoum!6fHAyg~x9F98oInFGGapD=yf4K1uUR5jJ!PS4{ z2;odBa;EIztQ!0^x<$cAW3TmdrBvGRW_`CEdg53IhniWcS!Ri*6L)j@9{WJ{nJDB) z!D9Ftqf+9J05#sX9{Bd@nR@h; zUWQ0;@ud2~_J+99(B>9|HxnKe#t@n*qTXe8)`T-|Ma#~pI`~D;8bZ9ovx_}JC*0gx zf?w*pvBR6{jN7vV!tVk}YD0iF~4>#=C_=L=Vy z6K=m{lDJcM($e;Kk#Lk<{i0~dZlW?FQ{Ub|)~VLQc6Yj#1T{SR!-t04Swf>Fuw^aW z1A(O^iS0$mxiKl*N4B6{#t!w)>jB z7NyFHV;GLCUlL1Hlw8uhlrg#AjGK(?D8de$;5V?@KRjH&R0%o|dA{iGuaI)@guKuJ zVfCb&DBB}jn3KxY{nIK-hdJT{G{Y2BODrer)F zNrorN;Yw7|<%Y82NgZfBxu{7aA_6`C%1vijmyrjq(7~b0zQ6h{v^f@o4l6|S(F(lA z`xp8Qr(bYE?#Zp2M!~nv7Q3uWp*Lvu+nt z?FAbr%pOadGQG?P@g>$H>sikO&sn5Ga`rboR5{IEbM|qz%+VzL>=yp(rLAq>oayol z!HBwYKVisiKf~gJ5~~)d36Uh)yjL>fS6?v15GcH?nb&tGmoMT}u%)0rY^~SvsOXJ{ z(kN;_)+P52xNCL>Q$q?9T)Kr*9-Fg($wP^T@)4gqrbl_BPNkk9KZ#@qK-Tx;dHm@8Vba0@FS+}?eK!qTucvn@OJmwX^(@ZP)-U3!NrDG`(4-G)^vNZ8NB ziYTV=F<7U3SJ`&gcrwoJo1FbrWI2chI*s!^<@0Wly|_t)*M9Wb+fF;@kbS0ZxYLUT z$oemHD&2Wpf8S_F>v9HDo^B}~(S@<^_eNtDWSvmY5&nxXW;KU$_9JlAm*SSlbItpE z#m#Z#o|snmV+Ou^EvqRvTSin9hi3}o<)Yw@V#SJuIovtXJzCNB_}`r{W1q-z3?Dam z)$d!)_qiLTcYI#{V7ws4jR@ELExr}~x{~WsUH26BM7GP!JZ?P7M0j`x(w-2Ovtyz% zXVYG{7w2+=V!quL&i+~k%Mfm%+yy{*gozgZQdtctMEynhvw7L-dO?ociKI;S1Q+v6|QgeavQ&WbPeLAlw>=!=-){#S$%z*HLs!}r4n}H_{^l{WBPf> z_=e`VOl}k5dG!sj<*OYN$efK~6D3|80J|YO3Q47 zLF)YjVZ0FDa}k~RJw=@V!kfP403P@c^Iq-r@n9`3fFPv%wG(MUqjC0&gr=5tFQq-J zM@X)K($&w3K`7b{Gi%GUR*sZXxyU{+ZVrW8ILWpj6At2RuCwd&ZM;Xje(Q$N`BiRza{6RvgqTI?MW7uzmdO=MCa1UGAIZ4SR zXcRKp;tLYp;e{jBn|O3GuJeAsslLJIyR(X)JfajFk3mo1WBKO5^GG8*=3@#9Hk>;`@;b*Ga}Ol{TR}cFdcktE}r2m{3n51fLz=kkWykoFm4S z*tFISiK2g7No6CCM*6H&+J2~me^#kp-hQ6YWYdg{c-)hL@}y2ZanUG)Zcph8`rVdv zziTR#aYuX%oj9$JlJZRtQF$o5l4-w5HTeu;SwpL~?~)>-uAdGl&5*H&RNQk9seYOV z4gS7@>@%8)`m1Le$!iw29+mwSS8B|*1J_Jlqu?H5#Xjct zvh7!dk-&dr6#ofU_`T{lL^5@zxu>-hO-(+>i|MLPATkMEgO*ukE}u_Ee+XKbRyOG^ zDH)AKW+HkQ_1GiOr1u>U*BSd=-5mW#^cHY}S^H}nbiAZsu8~0>J!Q!h^9)JJso(~) zea{-R;(9MgQUg!(WY9KezqiyI)wjzv4{-MHPgqu@M@u;mM_Wa7cK2#tg_ySoJsTJ_ zreyGEkiip4WU{GUrwZtwbo}tQoqy#o<}PY9zQz5i&O!rEh#(<2yT=11;uL)1-rPj-2V#mh2ht*6c zIX_+;j1)$uoO^g0+X19o)8ZET->nA6gPd39MMLAq&MnOH1Xx;E#`h+83LAsuR5XV& z$@oH}(OANHLf+|kI8oi%4Be46CrCZYf4AQ1**$b2x1w?8nNA+xVnNy= zIze~*MMspZA@c!h)}A!*_R%w2Y}Iu5xJf-S)WDA&sMy&Tj>8z3>H{v{ZsZ{)&|FT` z7r%6}F#AvNwBo7}81);eq#=@TqDuZ}y4vh^KN43@4OqHjE z?Gb(D79x)^+!c@zJiTC^-O@C9oRW9Lp21pT>-7(UOtK}LnKuWx`9Jh?xBAUB%2oVI z!BpVQOo}(78d6qEZzYl#$QlHaE>Se;DHD7&hHj`rZ6U4Ucp$l-|JBH(k}p$O;;TQt?(9VHM8I)KZq*wu@MX69NW$HOAh24a5|#drx6s=Fwe# zl`!%-rL!rl$MsV|>UsTi;(QsF@Eg&W%*Ux^uQ5ep#`)zfHmI&;BO?#c)JM-Sy@|6y z-Ef#@C$+TMy-suIkgu^9r?eAF7CL+2#^HxjA@JxA5v7=!aO>JZZ*|rQ>mL>2!cPqD zGUZ6*fGRiz*j~i=O)X)Lte9&(4yFt9CV&RB+6{DTww6YYmguU-Rl;V_i#79{?KdB0 znkO%RV66E5+VSZgyOQ+b^ls>f71@n^s-Bv_t{k##}ZqE)8-+)? zT}LtEO7v_r?!uji1An7Ib!I_uW$5(d5G!}k;` z?tVZ#A#*QvDMXaTw(v|Os+;C*;!Tq2E~ySSe8VB8YA(*2F0kATevR@#IO@-?e$}lO zD-O%6>bKErJ_~#H;|4o<+Z3>7e%!|F>X{^r9$T4B=txMj@{>Tu`HMjb1V8m3z+#I*X->&%OSttWwJ+{ zKwSCq^((})hfBej+Tz$=mo@1Fy|R(9J?-PGZ4Svp=vxM<=D3LhF9DL9W8Lc55rS}J ze9j>*TLk^;Zbdxo#KUU)Axx>#XLcC@`cVVb1pX+Ok+a2N9BRcrCnxt$`+B^+%b$uf z*kXSzO}lGCBWUcYaNBA@{8HEik4<-}_W8L9YBd`ZT8DeqWf6Xe4<272u-d1hbL{&V z%wbH2p`~Tjv%u&#NJN8Gv&gu?NknpfqNWWp5wxX{D0pMy)flN}5%^6Tvd~SjfyRwT<&oOOHTh5 zaf~AUi_Glpn3-w}R4m%ps;2gWJW2K1F7PPxVOxkxE{13P_r%_Qh3kU46fSYw&M0}V zC(C(Mv4}l~qaPY0Y|*`3JFT)__5EpvfG2go-saP=Zu|H}CV$cR=u%SKZJ906k^0C0 z`&(bESNKVVXuNeo8Y1%3Z3bV96bbbbf0lr^m*e!ui=fZC=OL*9k9H2+0D?9_sJNVY z#R^f?zI?b-6>n9E6)3}`DntwFSL3e5Fv2|ok&FiH*|jfS#wDN+f0%shft-#pJ23xH zsFdxzM=U|w`-7I=S!mvJZq+@>;ny^P~PHf`$V+< zwj|J%@)yH>s4G;yT&sFqwjbgeok==WmpFO1eTr9bIAn@X`UcMTP^zfmxBcx5$jH^S zl@dr?UYiDXrjh?x;f6N(rpb}RA}qq37*jPO3)bU+OGn#;%js{I48gRUwr%61a3_Ng$9gi{2` z*7XyRFPHU{qYOHgHJ_t3x~A?4UCtb5$G^^i`hQY}Mc+xHd#N1QO}@#Fm00#0yHN%Wxy5P<@sP zuNgb)#S7i=_2a%PxV0*o(ze8U=Bko9=;9dLg6aKePXSVWZ(u9R9QWHswp)8 zPxi7|xNm>;=<4#?MI0>M#_%0pTH3+u)s2L!h?l*P(`muv zSLjtrm}gMJdO$MLwlo*qAo!*?A17-WmoCUJsxC6X?W)1RM>ynU^$K zdp_CC)swk{uK0`kDc0-Gs~a9HMG2W>)-g}sb^^6D38;$p6E*ZCEqgX-Yn%ny1xdAFsS&p@dvq#-Oz1R2}p1cbqS` z-?hnIIEwAM;|U*K(;d>AA}N&JM0BQzO42EPI8#o?$V_)Zyr(&o7pPOUPoqp*#AnOS zu5SSw`^frgS`FkD;%Hi)2A+^O?C5!%>>@dD4hM!7DV*GQMK+5es2o143;G@Hcrm$? zfs9^ar7AIUKl&Gf)JRNiV?=~AfY#~`Mteb>vrOa0_7jmkQFCxGH4o02{4XG47Dyvu z>){XPw_ng9gNA~NO?DrL0EV+3V7LVU4P7h#Z@9e+4|g|AF97=6f5dv)zf+SSaU@PS zalng1q`VO?cHgq%NLseFBi zpgCi;QG*M+)>Jdt`1&~o!Y9ra$0mpuuidb9=a!I^&JlWFbYD})?sgZ&##3we7L4CX zVfs%U*X0a%nI{s|U2 zX@yUpJ)g;V)sS*^ux{V1|7h|VM38qOD2$VEI8R|jj%?bkCxxZ-3th4i-|9Af3$Lx< zHF6&|q#(ZR;NDdCU8{Mu!>eREeC9{GU)>{N)f9EQTG`J*+f{Lkk=!_EM6uHZAHH(F zSZ^LOeF{4%WtyZSY!^NwQad%MFB>%fGQ<=4LX1BqfIOQX3ogiIxTL(H8rh4 z#299sIB6&;c1fy)!#Ry~FWco)m=_QHp~W_(XD+K~7OrkNhkVw@sh`)D?dnMpuYqU6 zWOM8j4-DF(3CPDFCpDCIM`e7kbyHGFZ+P2Id|O;Ti|Wv)*Z*qBvQqjT9SJY;N6S~qn9@6~EkT@umWmeb z>2=|LbV(|jjcAQqiW5eVXEq7Y^X=tl_cRISRitpg<7%T^6a0gYzP9ycxk6PZJ7h~U z$#R?tDSVA@V&RU_T8;aCrPYvw&=f0XPL?b7;h)qxTB}S5y7(IN-ex%gZP%)+i5R>BvE&%`}hveiXrKV2dA+)*&XEaDp9nDq0;n%*OtrmW*uU-<&qJo7f23~^VNfS8c?TRj8;%(K| zz*Ay2T86}0&cqW3;M!g-Zsq7tJSmQfbfF`s)$LprQ=UF+TC7x-575=0(%+0xWdBJZ z`vwqQYv&E7qg0WA4skEY`2!isWd(8Fh+TK3X4#hdRr~zPR4ZH z4@T6z9z;Y=BEn*Yzg14E?mai=VoqTehIaF2P2?^%C%xEyURch3csIj8&Sg*aSZ1#c2b4cfRJ-xHm< zM;9P|IiWQKA)=mS=6zByQ7Pi{`mN}q>tgWpd;YjvfeDS+LLnfUy)E^Zn+on(wAnR~ z2fG;-Jrt&(h4|uw9z9zA1aJgF>6t*O|Fr4i37nC?l@s!*u6Xs}&wzCVnLl%~5Pnqj z!?P=pvJ(cI)?f~3w_s|dDiDx}DV-ju>NfhM?JquK7y9+tA9Zw_mtG=~T$Xo7&-i>D z|B4L_uNS1Xu2`-|*oW574rd>IV?#ysb;dmOB=&Qrk~|j5E1EN52+h?<#c!{0oFYKN zW=;SxdvY=u2(VaW`1+{@?o@T=TIumRFX1HitcKWnCm$uw5@^?Nfo6ZeVpa4r^38YL zrr}IrTyWJTUV9IL`iukHbDeh#gXIcJ_>k-Xp0cGe}ZxMC=i6sej? zJ1Xc{e`+f#!I(-$4{^5nF0Wu(gKNX;h5NAC_T)^c_sON`s)*hpw@@bDy}G!O5V01O zE5ED@WU-!1uw5fDnPatGqh_sYVtq*9)_8X91rTnUs zo7bCm^H#1cuoxO;(2R25)gmlCz{K!Uk7!gOQ zS)~&tDO<+{laH@QrtrX<%;ydOmLFoN(u&?SCK#}-^QOv_{GQj4az{Q_)>l_OL8*2KWl{EeM{4AqrXA2CTd^=B2 zJQ9~v;%2XUUqqHM6wF>zMa>VStj6!#n0m9W7xtcM)zQ+?Yi8Zm#xGXZ8mH1q!aRR= za;g@JXT`cTfK-fzhQtna`#9S)w$yf?QG0!F>aKjW)1$`6bw!S-3r5L(kKIqVmTNz9nV0jr6B7ekd41foVCvn5p!QUh-AiqwcKU#neHyQ~Mj&Br{BQ zjK`kgCTLP6d$T@{hHALJ*Xwlg;8Kd5z^K{8P{6<)fI(4xpGT3)<9g7r5Rl;&_ebpd zu2J!?26Q}ufK33ky*Dr1(E8soy^bh0lLCt7)alyFiC0vO6Lx}Q_4yJ)3P{5MG6O<)z84GjiDkPgfylmk zt9}1r&faZq3J$ab9Eg)ge90-D4kAoYZ&ZK!cPMysUhgNo+R>4ni}MWc$374=@>E#s z_;QC6C>E!Xmc0;-Zip33CGs`cL|bAAE@dpzz`FJDu{;LLs|-qQQ#J4}BUmk+K?F-( z)F2XTL9;mHfOE@ldyjRXS!`|fVaPcFc{#idsk5+6hbJH(s>w(Te&9?|uBO>^oZL+w zXTYw-x#Sp^`mCMa;rjg*g%&6+o;KvY4xJbEfx>ww@r;2QOPa8ykv1>1pLlj3rAZ*j?7PRY*Z{b-^Jol@T=?BMnrIPIRT(~&;6;fq8+p^JS>!w~nm;m%Z@h3gF zs(&)GaB;otzEp+p$g>}u1m4h%lf6c`hb{>u>flUZV*;-r$_`Ivij2#Nmo^!wI<{#A z4BfGDX!Wzd#8eyQ18QX*%n8GLDYYyO)G{OTVYPLV8N}JtkLa5Mgm7`Pr3A@=Y0CwF5bqCQkE$+v490BWqF?k^Zti28g z>}B!pVfZ| z5SY^+pR+XL2xoNpG9dEX4%lHC%Dv(O@hHj!aobNvA6*w?MWy&nx0dNgU!-tzzLCED z7T4gz+KcHeJRZ+fD8@uBvQ(Cv*j7r7np1k*-ZVoViED!8&&F>CN5RL80C&~T6n=KR zW#(nY(0~sl=uMj#YCnH=R?7jV90K<|zBvzOXf*gu+e_6GRm!#|W$t2I#;;3SIln0% z*;Q}_Rt)=?@N+0M=Dj;&bA+Pin18R1u4um3O%u%P9^7HPwe8{AIdI`5`q&)kwc4xy zjMpQb%0bjB_eZp=!Z$6W_+0Log4M~M3ZN(`=kQUg*O$cWV3}*JB?uWGZR~GR1r1ZQ zIYjh!HYmSxP8bhsxFB-WCYHlN@diWJsi0CH4Xg2cgb3zfgvK8EOuuTAxZJAnmLUBo zuVa1Aa&w5-yXFeLrq2|Tm3pCB2?+e2P31SaxFuIKW<{sz?#(qlweQQl4L$bmdl4=y zIDsJ~o^B9?CB>3l0r*onG5eLY@oE+PRMx!iHmcBAB+Ef~nwkXYiT$TducCq2ENQ`V zctO2hYH81GQ`o%#X1N$W!;FMSkaE1Y0u_Yn_vyi%eB*b(i}Ch&xn*4boSTu2glI1c)a`f7Q8DrGM%3o`&mOMRSnCW;3LDO$cxAX(M-TItv?^@Uggg#r%g14mOc8 zOsAf*1hKE83NVW>_)bCB-moZ_DEP?aBY55UpzoN=Et9;gNEW}~?A=^rK(x}OCpQZ( z0VY*0a!RyUq5n}~6J;^rPF=my6?!w0Q8=i7wkYmu7t zwep{XQmUd_0ud&Ik-MYa&N$Ap2kydKCU*QVl&F5<^ak$qR?5O19+zb;$35hE9L1cA z2l49a*`$d7tvGJ2l0fDptIfSH(H{i&Tl(QAm}^kk9&7Qzkz00|Hh>4l(&<$DW~R5)L*B z))x73>B!Vihn0P=XEm%y( zI8OQHCvxa)?sBV-%NDUvlgaz;T$Maw@qORAb20^O>2H5SY=X^9d!mG;O>2{`2{u~@gjE89YS`!rqd&KOXF z8ep>$&A1!>Zx>r8+YkoycVU}VYSZGTa-wDe8DnBc&4b?w; zRi`Uv2fgYAu$M~u7IX;1rrWl4|LLE^sTdHax<7-m4Fu(YU9-F_uOU-&Jh_#+3#vq@A6haYV?fiqq%c6U$Ebd(sX(P z#&uC}5VP{2m(q`-aMFs!AfBoLbkX$uf&_YgO2mY=#K2Foy1Ioyk^S{lEp-z_tfi|x z!9`ibBzGr%J6YOnO$O|F%WMGRG%7tg943QDM-c`1Q&!d{CjV<}qNN+6%(^|CZD@;M zFXRm+N&iAkZwhKIn#F!q3{SYLF9X`sC2B~kNT|I=l7_Bbr!rb?SDB(kJwzs9x&F9d z(zMyNyo+t6CBlhT*wf6xq@Kua@i-Dyozi$!O4ZXIIPU(&?E=0H;Tei11Q1hS@kKER z%a0%|r;{a$tLeAUnP=Dsr3nvVLotr*{bw(^STtUEe$3Y%QC8(Wv2R^5weNPXo#UZ_ zp(6?nbPYgB16@w_tB5cF{5JBJbBswt_6aqY2eA);{{IfB?eEN5DS(OOs`@}NAwS;d zhZhzPc99l<&X<)~{`_?${M1JeLXIPAP9;0cVXCcyHvzDD-CYB&f^9CqK$FR!S#sm@ zBsNq+N~WKiiT^4;Td*-GWi+E|AzX0kfY8HGw$ZaCOSRL|gf9va=I@{WB1RM?$}QRc zfY4vNXxG`^;!#C(r|rM8er%7p{y496IvJvio1`a@+177F9b}3c!>0Mm-CEiJXWs-` zESz@Hp-_-&KzZl=Jk5#?>_N7U>>Eof^34i0B9gc23wjYKX-DFPbFWgWd*=aK+L5F9 z2cQpj975}3tMJ^OXZ+2G=CW^YfE0J-Dti%_BAOvh+aic zcR~dmiBeulJxrDigOABWmapcwQfEYr_aGAlL=!(r*_;&6biz5sK~rBn6$3Zt;Wl|+dg-U(7%{k@fuyZ{AFDoA zwEpv^#1@_o3&@Fpdf7?qcRpfEg@2ISjKUOM&7I_>%hyb!bARvyc^~ZAje#aU&?y=T z_W1^hjHSeHL8q25e~{ZhD0|czPSm8dGL%ama(p^oD`C>_d^f{zOK?rbD(Ab6Q`|`P zBn1~cFLYXgvv1%`7Mrc$TRmX8q+P!L-{WMapv`8M#SW%mt#o4#_6rv0D>0~3DX#~A z3V7L+_*0J7xSnZh^C_*z;YICDM)=G3C&E?YyY;Ogy%oRTn@fPT~63 z9GR>};xt0-z624B`0*2K*bTm+Vwxa1LRCs2mET8n(fUJgn-012DfR#ZrHq`xc2<&n zv5}*#`V`o=$@iWpxgw2~ZjotxYP=d;cVXx8^&<(-dmy0cRlK}>BhY2+D53dpTba*h zaT?94nL(L=1kz|UTZZ+v2QgMv5x}Mt+;fY6(+vbGdVDk3d6s z4jTf+8>DIlJ0(>eDSOamX&x^6v=d@n0Pk{mgPGU#3HMM^HV-sTO5|N@5zSTT|6Lp< zX0y;e4rgi&O{T}`id}_FT=z1I>*jbO8)9uC{s`{)^h0MlF$2b zL!<<3hw?)rYFk%RvVd%!ucn(VjjHBUBN?$!2Xpxa=WNMwLQ)?F34NJ@eDzL9z=eo5 zSsg~$GqrLFk}Fu!Sa;B^0RY#~W29G{5u_%~fZp^nB7>%Ytzh}mu4ZmOr#WSDG$#^p z?4hZ+gB+~}IVwJ~Sr|VbVu{AwfIgq)hF(9J34*Q>Y-k3ewW3R;_~@-UYz1?`(Hct^ zmpgOFs|ka3rv-s_c@-iLjm^A02U?bahfbKfJa&^2SMbBEo#pdVOy^UYG)Z9S(+L_q zSYLfiB`-(+HD4Q|9|(pwL#z8|GiAIJGh?{bzY`%Cl(rGN!Mr%Cca zlk(;Z*`kW)2l;6Q@>9)-2!`_0Hc69IO+rrY}J55nwHm}2#$I>4~y8EK%DQxnRmM{SJr0(bp7f>_|(tzZNJ zO%faEJ##RBd{xzdw3GjXYntF(%ciPSz`t889e-dkG_aU2VL55w;k`uYTmuh&FBj5= zIlI=-B}0W!tI#+csDRak#%o|sY?^g%)Ep*{C{_bKynDlP@?1ZlsJe9sm%!-d%!31{ z_8l4&9ajlJe!4UQn%aZ)YX{uIbPH_WH)II;FbrjebF`n!6V04Hl7AX2$k*!Kw`^S) zV*=`?9oQbM!(}!PaIVw3F_y-g<(IY;O)8befarQ!pH)ZMVzBFU&k0Vt9RuM15>saD zP%I<-PNJ4(emYBn_@BD~w@+P8z|C0w5^7_%lb3yg6QG`8xOhvdKb6w+{uN&3cTfHm zUJW`pqvfC6jV43SSKA}MQ-LxlPXUASnwB{v7~M<=2+wW`PUzYiA022#0Ktj5Y7x`mcC<2>n;_Q-8ml6-tZbsS(yFmt2qcN>kl94 zfVQ1)c~tRcZpHI}x3Fi9>D?k?u+;Iv+~ebeiS#WUHz8$F_E$eOq@+VG%n39IXUBQo zqjB1+&m?VWqEDE2{J?;2p=P(`vDcS=aVMsZ*G)=gpMar`*Jr)@(@;T4x8wfZP{24R zVsDQ#ri)tUt`_VG7lUvUNj77GUw?7^sN+*F^r)O<_^0##tcsmdUd4HO2UW54CN&EU z=wDCF-Px?p@7$zlg28s3;yi_grPJQ#+mZV4Z4O zl@IH5<6c}e^}Ne_#dJiN0RN67VH6Aq7_84NX;2rp#1*!Iq|~iN#3mI;O=m1XZr_|K z(`yHmwxZx6CFRhJ@LUNCuo;Eh8qrv)5B8nF+Zmf^@2Xs#n=dJF zqmUE{o*~g#5F9CS^`M~Xgeh(TIJ)=GDh5{CIuGzztHlT#)V+QhD=@zAY7bT%a={Qz z9Hn+^8 znnAwHE^4 z4?)b?EKqAU!TmpP{~fsfAmKpWSAY6{3+z6n0{?r4_ULv_JfINlFAx!Fj{c*0dpRtM ztwgX*k^l}It*kLxenNiN5{yWKHc(;UH-7_Dozj*;SytDQzPB@pdud# zk437mw;$cxiC`-T8b{O9H3fDnyzNQqFRItw0tI1#Cvh`4Uzw#Bm&v|7zK{J(oR08#z=~JPXA4o6=dgnc}X*G>A{&lnnbjW#L1tzxW&%;d*NYn41VKZd}Cumf!Xm&x|bzHQ?C`I?|Yt z@$M@fR<^YQ$S50u^&}c)g&m~UQf>xGkwe)ov%W3no;{4KnldvPZvwKk+(9idmqwY! zLt{ogrJrY3Mk9u_Ij5z;{f|U~e-A+vMQ?a1?c?p*&1NbDTr| zS(oyrKg(vScf@h4D2f33gCa7;{7vA-Tk@?S!yv17sjeTkxY@9%zku#H`Ul@t<#J&@ zS(XTnsJK=S#%$*k*#~L>7=e80*aESWQm~l9UX9I4RQw{{_*Gpq|3B!*KjWt4&A-G= zDf<(`9h>^ETSUmGgQ>Kkqvp1t&`JOB>)3DjVv> zbz~Km+IOwwDq>RkL3gnK%IJQ{*|9TA1Alc={-Bc@oCOmN0JeEhNESDT@}(^S>VQL4 zG@HF2kxc~4&C>QwO2-a@W-5C3hIQn~T0!Js!^+360C;@B8g}}4ik=n3i(#q!FgZpT zh@bX0^5s?uLR?k!I~#wHpVa;dJqMj9j?#Hv+BHx;=+S@jTZ?OugrvWE&ddq0-QN2% z!4T!N5(FGrd$q!5IkJeV{JWV@&%ZS%YIaN-p@9gDy15<*FvvFYy>zIA4|vJ|$wM%B zVf<+t5aPR~9oQ<5Zyr1(1VOvsH}O8`(kel2a$_c7cnefywYqMuP&M8?&8)h0!5(SU zv2NjYl5Ea?*oe+cy^mqP19`}PFNuA?Sot*aPp12ldrD5}l2we9fJ!5JSDYYvdQ>a- zpaUtA(*(L@IM@%}vIh*-iHQQ#Ux2;Vvi`JJS$s`K4d6u9trhQ^Bv7UcUgkrJr-q2Z zR5Rx)(mPg0KRfFGX}Bm#%U*U0`9W??rhuXiwG^oO&OD_EY|2a=TK{UjZHo0w4&Vu~ z)(shp={SLPiHs9gD1YHW39&!bAmpVRvSrZ+01hVXSRd?$J0gS%oUB`1vIeqH&>`pq z&X;K^7RW-Oq~Z`{=Jns3e6YQ7ET-_}CAu{8)u{i>X zT2pJ`v&e*2s67AB`jwdp$<$5pM;9pvaa_364tfRM0}IYkiri4xBU2kJ<5mjFY`v+E z9MY{?QFMG7r`Qj+D45LxIm_t)N4pDf97yK#U>ON%*O`t@7Fv+!#9Og#94{%ogzg3d zGv}c-sZweGC-=QAe$gAVG|){QXp;M{_j1lq>b)oBbob(gu7AtRHA-G)9+W%McoD^5 zYzyad`*{!LcgC25A=ocDH|DED;t6?7$^ou}hxi22RCSjj`Kzcm?K;R{#dG=EuCLVL zH&065tOsE6x}*Dy>$tTto{AzngzB8!*_~rcZQ=E;o5Vf3s`4?Cd`nov-L$&dW1i7(;DO^*aqM(r&S3jvphv+Xg_;EDzN#f zLIN1($=;N#t8kZCS_1;n;(m@llpfGC3ON9?oBsImn`u?+0tCC(3NWA4!8u53Q57Sc zhYZqy?o{XNgs~eUT7;^BJp&p6<|>)CtY65VtpYmuza;Xf2SxGM^rrsJzhHkN1^f4Y zVt;2d)^6B8OA+<_eSaz~FeBwe%0JA=CQp8Tz@NDKQMRq5H-`@u#tqT#d{|8Zlbbrd zaZMBM{LZ}40RABX8qMbf?!Y!2Zz##zkJUohDF$O)9h5nbJ0Up8`3W<)59$Go#aXEC z$nXMA>Dh;FI-Je}gO3N(FEdaU6%+oSzP>yj>izqlgi;7)ml7s>$i7vm>=!ArOqq~u zAzO?kC9-9m>^G4$`@Re^h_RGy>|>XGi?RE@X1cn!`@5gtKC44t2Ou>9pEVV<#4McciwQ;#X z0hE(v)?2i;zN1vHfrQj6y|x3X&z_F!auvHEoSqCBa~`#CDgg|#l094wY88~v0GE17 zHv3w(3Zl|zU*@h&^<&nPcg2Vyn`7H(d6gKCj;ywgxyY z>0Uf4ZTT{YAv{T-^6h#SP;9kdpk2!ifc9A-oShWz0PdkR;$5+pEBHio0-vrL0}2bg z@+>GBZNto=_>@%wD1vvK_da17DiUTG0{(mGTY2F-pa6tz#-TMwIhRgOIr=DYONJEY z%WR6V-9ji&;uD)K_tr}D3hB%e(sg9zt*C>?^>;j0Cw|7#=+FnGm2&0F4koAda2dZ5px#Kqr@wn;hZv*~)E&t?>QH*Yl}kisgq&A;n6 z!+m0%{|MjDxU?YZch6M9*P;KPhTGrgq!E#$#T{L61WE4TDDhWzS zX{V(Ax7lK+{&IdQ;>DqbJ?4%0ddL8NY-v^pI6uH-rx8#DbGKOY_TKI{_KXg&S) z+`feEIFwOqe4tQ9{EoK!q2K=bSR=B%&+FI&qWLon#)|99e8=yx*cx6rq_Pfsjy<%E z*-DHj?z((!O^%!Pv(QUz>%mA75ErHb@fArSMZCIwJR{tKxK?)D)pK&UY4Q!k@pqAp z>*u`~OgcVrPY{a=UlSxK99;O*PP+LUtNqJ44FOij{4UU(qS<6pG?W?nC9JW*B_SU* zwn0^B-i5^@P0Iw)wf`fbL)~2ynKIp%p!OFXWm^xV`l)|c+zQ?qmM5B*d!fs5*h$lD zFF73qF( z`oMBXX%E5Wov5jRaUecpoL8wV#Vvx zybYioL^k@X^Mi9JFr-EI9N$dU#g|Q@g$SwRX?#?TYXm^IudBdF6?$!?q|Z9)X#fxC z^55T7`FKVLPJfSsz7t9Zm}|wPn3Us@>3 zIvufF8t;A>SdpDkO{j(|rHa$t{F}hToYnD_QIq37m_L4;n?stF;@TwR`_+E$=Fw9beKmmOFXX)(^d|0)#4q9tSpsQ88S><9alMnkLK&H=S z1X8BpvS@fX&&xYW4W(bUPOPkX@2W0JkIqJ8A`|mM1lIte-p(m>D_bz&cUX{={?Q#W7d#c<#W@lQ}Ey z4)AecyhjOr2jmN+pACm8_qw7F7489KN1fMDOW=~)V|ljQ?Q@!2ps>&_!=ey@AGQ^{ zsaYbD4KHl(aeWV(7TJFBiy&4|YZJyRZPRjPNHF`#U*T1nY_vWc!;|<1!5?PQ^ z350@d0RcFyXFps}52~QwO^jtt%}#0Iz?R1t{w}zww%Q0FjdYywI#LGL3hE>MK`S;l z;oVn&rGga}V=PdxMnS&QQ_)TAY5J7dKjdaZcUVeEk5L2;pKfM}1^(II~12k>8gYc=S9q8?)vHhiNC4k+CwWoSO z$#S+e0)bST(H?g`VMFpR7P4Bht>5qc#tzKRvgJD#4SqTqNt_U(4Bk)f13ofL#nSHB7`?XcNBJs{=Id9QVzh z_;+gwapiiSxj>`J-wI#0xV>R8K;IyX=kFir^uH=5#TutvdnLx3JUdL!Fm8h5%84oK`_!Z)gs)cVP@*abQC@#2@QpEkDo409q-b9l9T@( z5Pd!(R!xy7q7S=|IJArUa6drp{3Zgj2BQ$lGafD$?nGOR z+gW;p+*P1GOuNl!x+7s-P}a=59e$3l);<X<$hSsQuW}?D6?n zu$Ywk!ACM=KMSXnT%QADnq91_XJ#I*W-U@how38K+VeM7Zt9?j$p1?0PsVc`AVHOz z>q+eVx0SGYtCynW+QSw)G&=LhI0?-@K#T=r(|-8;REIE% zp!K&egOVU5OQOODZ#x0R#>7oPY{UmleDU@fV;Q50>1{8yL7pIs@A9pv`p3K{l!yv| z(5gh&Wb5k(cy#V9BAphj)yQ1d!zO7b4UzrtIHi# z3pPW2a4c$rlc>8~Tyc)Kw`FwkVPODxIQuf&&qOOKqxCd6xo(OT)+LpuPn)zzkhJEq z<4v|auH4F-{nkJ!N_Tw=nEEi!uPMY_JPuI-h*wJX?>A6R&AEZGK2m$t%?pmA_`}@`YsnHfUcG z_dbf+3!s3$Ub(RHE%*@5R^saZZK8INgB5`W>vThG49UH68puI2*itBqAOl-^j}Kw=W%3;h-O*Uh{Oa1L zwoxmqMqrSG5LUaBrr@5tNT+Rt`32vY+HR!?STKfCQk=wm^wef+%|s5 z=;2gI|3yHqP#-B^=#3AiWON49YtGzX@^xKzxm_V9O*ZBLusi=KABO9NqAscx>_?a`YaiJiIWx+*i`0-O1HAcz)5D zm1S@5V%u0yr?bRIfN|5LBJbu9_b}fj-s4!YDuIMz@|xEaZpl?F8u=o&M>{r(-hg~) zg01&a(Y|w1J=o6!=QQuSHUqX34li!mnF_7w5_IwtUMB%h02uU|u|@o6`t+!;79$}I zk)O=<7o_I4?Cv%@VswJVb`0^@*uf*Zq;iM7etenz(xO+k7qPAJ)}TtZKpFi^CLXy+ z)`)Is1K1M$0nlM-~`?Yf!Pzci07phn@r+R^kMVy_v0$VC=wAyrUlTI=Uto`(|d*ZhpM;td39NV)yZC~uO2b^OFG~znnj8G(jPM)zhKBw`hWif0h!Zz_dr>#;> zh}*I(sEM^)Z5AFKdgsMXb@}q#{1x(Y$ZQhb+g*cEesQ7Yo;R z^z{l(+^?L@fK~a^S=BP?xkTxZ#t;!PrQ+WaP>Fz4B`KOtoq6iB7^;p;y8Sqt-oq|t znpSsHtn>blICY+9`9f++))nrbEd;_)Gj;O`32r4pZ5er#wm!3l1(6XRFI=(a~^JL!5EL{4CGZ4{vlW zptr8Bh^!!UPIbPQ$5DfiV_APR`|eN$y_s4i-?Wr7`Mfo#*-|q9qw)0oyx|X+{#9q9 zwHU^))A=G8U1JQ!fnhL7{K^5d@ZHNZ@`}8@F*`e8Z^&+iEmSTzrvs+CUL@q$5g-Uewbu>7RY??-YL26%k1j3%O67*rKqY9i=r>zSH|IF z{I9PXvJ^Ru;(U@sJjGh~Ydhiu28PPW%RW!08C9RyGm~lHG3!LVboU8-Ci71zr_)&l zX>XjRyhH5j>nh!ANNM99Tj_6QIj`D=R_e6Wsbcl~c=DHYi-mABt4=z`lBYa!>O}5r z9M@JTt+;_2$k`p1nje=g49;vUyPIS$@QDr8b>q!lZZF?0$RuvQ?H% z$wjSfVi=Y`yK+{8Mg2fp8)^i?}X3B`VsJ90Wc)DqZu8NXoVx-%s;b72@d?9c zaqkLp!`hRvk5u2I;oF9$dK)SZ8K;y<^f3VyE23-PU1l-^=Uc@ynJ4ZuHx?IhPZhr( zgan-3Xn+#QcT5(rzb_9EYee?zyL46fmNAIVmOjbxO$w!Ha{2Lbivu^YGoMf%b91M+vLpJ&eQ+_!by zmGM^*i%3LyFmI`jFHW~h>q3|887p$UY^KY(kEcRen zTmGnE)y#G)CHMdH3eUUX`>An2woL|#vO9~CEA`WXqI%i|PTlBQivb=@GcQV6(P=D{ z!<@$s-uaDm6sE!O=L{9s-f@m-YR6DO7MwZ%Y61gx8+)~y6`Ox%A9w_k^? zR9ElE+-{0f-*74iuA%dAgeS@UoTRdg(-_zIbiL#U2`-0Omznf`%G;UN_Yy=Pvb*6y zcPMREx!osp7E0r5>8!13aK4>*@9>W|iO;XhN!yFczt+=T<`{48jc4L_R>!6CH%E8Lj9AlF1oa7Or5&gK*&&2nJ7B@6DV!)3)(2g!W%%H+DniR@0}7uBu2 zZY(I(TOA}dHdRW4iyX{;6T0%4sktySR*v)4B|RF+;Wluc8yKxL$!pnPih5X3bdVBh z_4AaD*adFe^yP%L^1xb^fdm5&@^262byg0tYLkNZ{d!#(O$(b~dGUdz(LX3$HNb|O8P4djq!Hm%6wY{5PMK3rC# zTG6cF_;VrVN0nmv?{sTqB#ezMO0=h(*iPxpJ%WK+-Rs&{8q_=q(egHN<$ZA&B%?E>_PmrqZS%Z65{w`R9hf7nz|$1?ceTq3EBYooWwJRBbm zJtee8w(^=1VU~DK&d)we!k?HG`=m^^^o;Nr(;N0Cmzc7A;eop5DS~FRH5NM)G&@cB zre3MiX+asuDMt(cN7v~TJ}W5g5mnTT?%VUNO}+&#L_0|$W)A%jx^yw;lS*j3-#gAd zQjS&3qZ@Wj4`j3o;WyU4Ds%Ks>o7QAdYsHz^h~JoFGR6!Hk3v8D$qzca+j;0HnM(F zP|t>%e3g7|hJ9w3M4~cYAGQ#e;3QFPnUzLm%sr=VsKp?8jTNIplK2|8T*pGEWd{ZNL6#0a~IYi3&_ws&@9{l*w-6G)eBh(d|5F{_8j*I#0)L?&>RlEBTh z%mX(A+a>pJ-c^QgJB`autsHO4(YKgGOX2-BoDZBs^P+6;w+7@M-_-PVYoc8rXuBm7 zFQPs`Y+{%y`-+@?b|?Hrwe{CmeQ6B-w;o6c7urL@Qar+E`S`9IL8-%Vwd?OyLi1Rh zXtpB!&^nD+y5Nzk#nhw>SDv}i$!CSIFMYkT z6Zd5nCUJIgW{e*)c7(Mtx?F315?NXrTdh)~#|P<5%Q-(TSanZmeB5#9M<}9r zCC%nO>m>`uLu?A%8E8J7PxErR=wfVwSM^ESQ#xPFfU3_faO@0s(ZkAAezZZ4kEj+c z4x>{YpWl#EKOXT(S zn!yeSLfna7Gh3xe)_{51tFr*k6K!~U)>ocQ6z1Uc>kJ+abV-W)?{zpvqKO4)0#oC; z2NuoDLy3*q_#EY?p1}nqs`uGYQb>T65-&CO{OHl>GH9O5msQn?6Iqz#at2c7$oZq+&xI&Nb(kWd?C@EOdlk$pXE;EAc?y#RKe z8EA$(k*=2W^fh+YwE-Dfp-0v2VfsEgrI`fU))vLJ!3YnXiGS_ll=9s(I|IR%uU820_ z)sPcBq5q}^WUr9YkfKK1m}| zI=j+RfQIi`w80-{k6%e9&Uhzb`ZndqaU0s1`$)esGELE4yBxQ-x^efT=&U+Lw4(jC zlZvdNjc8%N$sD_;)^|q1zZ{Z22U#UnX39|_-~i2)yj&!{He!RKqoJY`wko17Q9tDF z^)62o0d6X8#IqkFvDq&&n?pI@y5JCk4+=u2(CS#DjtTyO5ukT+g$2(!?~ zz<<%~6txZl(MNAu}Xb@NW++*qgae6M=H1v%84(0YYs&hwSe-j3g@IA;L4 zYrofddTODQ%CvW&h|G-IZRj?i?ns;xd1`>cr@-%29Q|^Zy@!&ybUwShG`hBDoSwo~ z4ov1S)ieH*(C)hCMf0Z1Fhj>6tHrzsY>my`he4T3Cr!=53v6j;b}8($d;RsFI)W=q zS0-1shB|)S5~r@d#)?SZ6i!2(?Zdo_!n-dU=n}`=jUN57Mswq`o87)Fz6?t2<9Cw< z7cS1#4d2Of6r%HmI%JZnaY}o{&9>siRv*+Qk#xl>STMJhI{QXO|lhpI8E4Yn~y4`xLO#OuP zbNGn3q7=rzXB{bD?=qudYxCqBFs_cD##IP+D>Z`MZr^bQ*nUdtC$?W_wB_9`3NYaB z0&x97jzw%vhgQU3(&q6nw|w;ht9ohCN~4J@&!8Acey)j0`%Hdj32sV0*)w`WWFyVmWOWH zNTM||BHM{NG+h-C;4v4n>MP<#NKo5DUG`Fq1`x9QveZ>Q_681Nu(9W*Kw5!c?Audn&y*K! zQty47D467)5qDyS^j<|DIq;Kk|BL0by8xkUO&=ljvf+9q%}Y}+UzqL4<&(8;oSog- zyIQ=@PNQ|x;5hoKqm90WIl>T+U7j0k_CV&*Zu}nt;V~tnJ73AdAp0Bf(eE9k;`Mdg z(*ol14L^n3y?_VkCoI`loT`%WKj&=nNdE5A%7nHi_s~LwLqdGX0!hxfs?zJvj$^HJ zB~MvERwb(-KDd&y;dPWoNRr-`=8}ZWO{T$4ksPrOP0zy;gV(~DvBoIsH97%ntKi3o zv`<5_ycmM;9Pq+h0>}5oGXCRmMV?6CQYoKDTDo9%xSj2LfpRWqNrLbxHS9Bq^Q2&91r6|8_EMv05zo3UG8}F1V#xxxw0xSS?S&={(`lT6x%+V_ z)Ydsjq$Cvv2Er2d4s1k8VRyt{`?EvL?vkfJiP~EAqTjMhCViW9^koR9*pDFK;%N8y+CjR>RNZ#)xv>Mviu8O z>vvdOV<%9t7hzC2kgZVBY;4iQ9>U=fjOqs(qsQE`Ti+ETi!vrar9y7Hx%0|l#7J_9 z5Ipdr)VnkQpGVpQ#1=)M(4?lP1+N1(_rILR8eiPGf$lbJ6>Wl<1a$?K_7MS&-fw*p3Y z50nl;+7LH?>?zfjS@Pk%ScqD!V(n^OlG8kva-3{uyN<7^UBfWQCfvb1h<90Vlr#UB zdP0<>Em3Y*Q}CR4Kj!Uc0i{+AJqB@qSD7F_r^M-d&tMZGD`U)uxv>^3se(+HPR2z( zxf}N5RoQ$>3LY`r@bFjpx##C?pVwnCO#yN1Jc&bj#7116yG9p6uJ>H03~v$Dx|_6} z^urojlpfR^KHa~l1f4Y*wKdI;s=l_r%g%=;Kd(Ud@(FaA4({yG_P9o z@!#|nN_|HC%}w_9GbD~S;m=Q5X(o`l5%7|}IJOppyQd7#Hj1lAiQbHIBwu|u>h7~2 zWCxM(pD4J=ia5t{^69H>d2utmAmo%}`#UBp@@{5da8K~(sV{SQ^^7F&0I^C^4keKf zd(84A(I&FO+$hT~;QOUvE!L)e>&<4I_4Yzu?lYT}9pSJ?NtouV%0ciA15i+?8^EokIA#8x47wTKN@%hyc1N`T|5$r2kV6-CVnKI)U-}msG+z zRl>Wo*-#hy?M=w!yG#u@c}OQ>yR~|a3BLlrO%eY4??_N@iYMZU6K!%Yhey7&*O-IM z=YT&U{Oe@Y%nl>`^rW}R2SYI;Lt52oqRq*3@_&C`nl}T*NG1E0N}yR-%Gn?*9R&jy zuA?t*d}9>RU!Q) zn0PxP2zcOQdur~=VJV#ddJqc9{F{V}GMHq%#YTIDmp_p#lY0(63E<2)|1aU3{PmeZ zw|u5v-~@@-t^P~c$A8^pV+}Emz6!^<((_-!(HK1v&JX!xM!c?eBwIu)=2od;DLenV z%a6ar{ak@4n?i2v)E!;hXhvR%TPj&4K&D30cTD9+!1*GN|zNHGu z)FC~RPUbS>ipHoMoxJ1OEfblBu#|y+JvFbxk&?-ok>w?l$oFXe#~m^qo#50Y(NoCE zV$La@PP*Dt>gQ`K$c8AZ`c(rsry9$+HzJ(w9chQ<^}nyVRLk{%R8kS?Yv6Ls!iP>* zzLc1i6p3{OSGD}ls~$=EJ3S_KrIMts>QeS;+`5|2v6o1jBuD9y+CP4~CI0tE?tF1{ zd@aqr*RvHdJ_v8KQ3-))i|%^Vn+yglg6oQfw-y@elJ0rCJh45b zA2k`RtDAq?u#Dv-tMbe;m*Q-r{@1CK9_dkJ1h21t>3#AXDwA9i40*281Di=lcW)d~ zDW!?Kk4!)h7Dqi2*Ff=FShim{G(| za|Pd#TPeD5q;3vH)ULFWq(cKX+7vcKxyLNLp-zTpRpcK0e!y#N3J literal 0 HcmV?d00001 diff --git a/main.py b/main.py index ff0aa6a6..00bd351a 100644 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ if __name__ == "__main__": from config import config from models.translation.utils import downloadCTranslate2Weight - downloadCTranslate2Weight(config.PATH_LOCAL, config.WEIGHT_TYPE, config.CTRANSLATE2_WEIGHTS, print) + downloadCTranslate2Weight(config.PATH_LOCAL, config.WEIGHT_TYPE, config.CTRANSLATE2_WEIGHTS, splash.updateDownloadProgress) import controller controller.createMainWindow() diff --git a/vrct_gui/splash_window/SplashWindow.py b/vrct_gui/splash_window/SplashWindow.py index 2a2db684..c7af9802 100644 --- a/vrct_gui/splash_window/SplashWindow.py +++ b/vrct_gui/splash_window/SplashWindow.py @@ -1,4 +1,4 @@ -from customtkinter import CTkImage, CTkLabel, CTkToplevel +from customtkinter import CTkImage, CTkLabel, CTkToplevel, CTkProgressBar from ..ui_utils import openImageKeepAspectRatio, getImageFileFromUiUtils, setGeometryToCenterOfScreen, fadeInAnimation class SplashWindow(CTkToplevel): @@ -10,6 +10,7 @@ class SplashWindow(CTkToplevel): self.title("SplashWindow") self.wm_attributes("-toolwindow", True) + self.is_showed_progressbar = False sw=self.winfo_screenwidth() @@ -28,6 +29,37 @@ class SplashWindow(CTkToplevel): label.grid(row=1, column=1, padx=int(desired_width/7), pady=int(height/3)) + self.progressbar_widget = CTkProgressBar( + self, + height=10, + corner_radius=0, + fg_color="#4b4c4f", + progress_color="#48a495", + ) + self.progressbar_widget.set(0) + + + (img, desired_width, height) = openImageKeepAspectRatio(getImageFileFromUiUtils("VRCT_now_downloading.png"), 320) + self.text_label = CTkLabel( + self, + text=None, + height=0, + fg_color="#292a2d", + image=CTkImage(img, size=(desired_width, height)) + ) + + + + def updateDownloadProgress(self, progress:float): + if self.is_showed_progressbar is False: + self.progressbar_widget.place(relwidth=0.9, relx=0.5, rely=0.9, anchor="s") + self.text_label.place(relx=0.98, rely=0.98, anchor="se") + self.is_showed_progressbar = True + + self.progressbar_widget.set(progress) + self.update_idletasks() + + def showSplash(self): self.attributes("-alpha", 0) self.deiconify() From 8cc95078e9378b3d624cdf2705014642d9ee7e5b Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Wed, 29 Nov 2023 23:36:55 +0900 Subject: [PATCH 09/20] =?UTF-8?q?=F0=9F=90=9B[bugfix]=20typo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 34a0d341..44529b5a 100644 --- a/config.py +++ b/config.py @@ -611,7 +611,7 @@ class Config: "Bing": None, "Google": None, } - self.WEIGHT_TYPE = "small" + self._WEIGHT_TYPE = "small" self._MESSAGE_FORMAT = "[message]([translation])" self._ENABLE_AUTO_CLEAR_MESSAGE_BOX = True self._ENABLE_NOTICE_XSOVERLAY = False From 8146b8b98870f5cdef17df530a600257b02b3f4a Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Wed, 29 Nov 2023 23:37:44 +0900 Subject: [PATCH 10/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20hash?= =?UTF-8?q?=E3=81=AB=E3=82=88=E3=82=8B=E6=A4=9C=E8=A8=BC=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/translation/utils.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/models/translation/utils.py b/models/translation/utils.py index 5b15f16a..c41fbb88 100644 --- a/models/translation/utils.py +++ b/models/translation/utils.py @@ -5,30 +5,57 @@ from os import makedirs as os_makedirs from requests import get as requests_get, head as requests_head from tqdm import tqdm from typing import Callable +import hashlib ctranslate2_weights = { "small": { # M2M-100 418M-parameter model "url": "https://bit.ly/33fM1AO", "directory_name": "m2m100_418m", - "tokenizer": "facebook/m2m100_418M" + "tokenizer": "facebook/m2m100_418M", + "hash": { + "model.bin": "e7c26a9abb5260abd0268fbe3040714070dec254a990b4d7fd3f74c5230e3acb", + "sentencepiece.model": "d8f7c76ed2a5e0822be39f0a4f95a55eb19c78f4593ce609e2edbc2aea4d380a", + "shared_vocabulary.txt": "bd440aa21b8ca3453fc792a0018a1f3fe68b3464aadddd4d16a4b72f73c86d8c", + } }, "large": { # M2M-100 1.2B-parameter model "url": "https://bit.ly/3GYiaed", "directory_name": "m2m100_12b", - "tokenizer": "facebook/m2m100_12b" + "tokenizer": "facebook/m2m100_1.2b", + "hash": { + "model.bin": "abb7bf4ba7e5e016b6e3ed480c752459b2f783ac8fca372e7587675e5bf3a919", + "sentencepiece.model": "d8f7c76ed2a5e0822be39f0a4f95a55eb19c78f4593ce609e2edbc2aea4d380a", + "shared_vocabulary.txt": "bd440aa21b8ca3453fc792a0018a1f3fe68b3464aadddd4d16a4b72f73c86d8c", + } }, } +def calculate_file_hash(file_path, block_size=65536): + hash_object = hashlib.sha256() + + with open(file_path, 'rb') as file: + for block in iter(lambda: file.read(block_size), b''): + hash_object.update(block) + + return hash_object.hexdigest() + def downloadCTranslate2Weight(path, weight_type="small", ctranslate2_weights=ctranslate2_weights, func=None): url = ctranslate2_weights[weight_type]["url"] filename = 'weight.zip' directory_name = 'weight' current_directory = path weight_directory_name = ctranslate2_weights[weight_type]["directory_name"] + hash_data = ctranslate2_weights[weight_type]["hash"] files = ["model.bin", "sentencepiece.model", "shared_vocabulary.txt"] # check already downloaded if all(os_path.exists(os_path.join(current_directory, directory_name, weight_directory_name, file)) for file in files): + # check hash + for file in files: + original_hash = hash_data[file] + current_hash = calculate_file_hash(os_path.join(current_directory, directory_name, weight_directory_name, file)) + if original_hash != current_hash: + break return try: @@ -40,7 +67,7 @@ def downloadCTranslate2Weight(path, weight_type="small", ctranslate2_weights=ctr pbar = tqdm(total=file_size, unit="B", unit_scale=True) total_chunk = 0 with open(os_path.join(tmp_path, filename), 'wb') as file: - for chunk in res.iter_content(chunk_size=1024): + for chunk in res.iter_content(chunk_size=1024*5): file.write(chunk) pbar.update(len(chunk)) if isinstance(func, Callable): From 8463ccda694c029770d1e8f5da62fe5037cc0940 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Wed, 27 Dec 2023 22:14:10 +0900 Subject: [PATCH 11/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20transl?= =?UTF-8?q?ators=E6=9B=B4=E6=96=B0=E3=81=AB=E4=BC=B4=E3=81=84=E8=A8=80?= =?UTF-8?q?=E8=AA=9E=E3=82=BB=E3=83=83=E3=83=88=E3=82=92=E5=A4=89=E6=9B=B4?= =?UTF-8?q?/Papago=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/translation/translation_languages.py | 94 +++++++++++++-------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/models/translation/translation_languages.py b/models/translation/translation_languages.py index d5645cd2..959b9077 100644 --- a/models/translation/translation_languages.py +++ b/models/translation/translation_languages.py @@ -1,35 +1,41 @@ -translatorEngine = ["DeepL", "DeepL_API", "Google", "Bing"] +translatorEngine = ["DeepL", "DeepL_API", "Google", "Bing", "Papago"] translation_lang = {} dict_deepl_languages = { - "Japanese":"JA", - "English":"EN", - "Korean":"KO", - "Bulgarian":"BG", - "Chinese":"ZH", - "Czech":"CS", - "Danish":"DA", - "Dutch":"NL", - "Estonian":"ET", - "Finnish":"FI", - "French":"FR", - "German":"DE", - "Greek":"EL", - "Hungarian":"HU", - "Italian":"IT", - "Latvian":"LV", - "Lithuanian":"LT", - "Polish":"PL", - "Portuguese":"PT", - "Romanian":"RO", - "Russian":"RU", - "Slovak":"SK", - "Slovenian":"SL", - "Spanish":"ES", - "Swedish":"SV", - "Indonesian":"ID", - "Ukrainian":"UK", - "Turkish":"TR", - "Norwegian":"NB", + 'Arabic':'ar', + 'Bulgarian':'bg', + 'Czech':'cs', + 'Danish':'da', + 'German':'de', + 'Greek':'el', + 'English':'en', + 'Spanish':'es', + 'Estonian':'et', + 'Finnish':'fi', + 'French':'fr', + 'Irish':'ga', + 'Croatian':'hr', + 'Hungarian':'hu', + 'Indonesian':'id', + 'Icelandic':'is', + 'Italian':'it', + 'Japanese':'ja', + 'Korean':'ko', + 'Lithuanian':'lt', + 'Latvian':'lv', + 'Maltese':'mt', + 'Bokmal':'nb', + 'Dutch':'nl', + 'Norwegian':'no', + 'Polish':'pl', + 'Portuguese':'pt', + 'Romanian':'ro', + 'Russian':'ru', + 'Slovak':'sk', + 'Slovenian':'sl', + 'Swedish':'sv', + 'Turkish':'tr', + 'Ukrainian':'uk', + 'Chinese':'zh', } translation_lang["DeepL"] = { "source":dict_deepl_languages, @@ -242,7 +248,29 @@ translation_lang["Bing"] = { "target":dict_bing_languages, } -dict_ctranslate2_lang = { +dict_papago_languages = { + 'German': 'de', + 'English': 'en', + 'Spanish':'es', + 'French': 'fr', + 'Hindi': 'hi', + 'Indonesian': 'id', + 'Italian': 'it', + 'Japanese': 'ja', + 'Korean': 'ko', + 'Portuguese': 'pt', + 'Russian': 'ru', + 'Thai': 'th', + 'Vietnamese': 'vi', + 'Chinese':'zh-CN', +} + +translation_lang["Papago"] = { + "source":dict_papago_languages, + "target":dict_papago_languages, +} + +dict_ctranslate2_languages = { 'English': 'en', 'Chinese': 'zh', 'German': 'de', @@ -345,6 +373,6 @@ dict_ctranslate2_lang = { } translation_lang["ctranslate2"] = { - "source":dict_ctranslate2_lang, - "target":dict_ctranslate2_lang, + "source":dict_ctranslate2_languages, + "target":dict_ctranslate2_languages, } \ No newline at end of file From a82435dc29a9ad4d1707524996207f588d408b7a Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Wed, 27 Dec 2023 22:37:38 +0900 Subject: [PATCH 12/20] =?UTF-8?q?[Update]=20Model=20:=20deepl=5Ftranslate?= =?UTF-8?q?=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- install.bat | 1 - model.py | 17 +++++++- models/translation/translation_translator.py | 44 +++++++++++++------- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/install.bat b/install.bat index f312bf4d..036f6a51 100644 --- a/install.bat +++ b/install.bat @@ -1,5 +1,4 @@ python.exe -m pip install --upgrade pip pip install -r requirements.txt pip install git+https://github.com/misyaguziya/translators -pip install git+https://github.com/misyaguziya/deepl-translate pip install git+https://github.com/misyaguziya/custom_speech_recognition \ No newline at end of file diff --git a/model.py b/model.py index 36de1262..f7fa4642 100644 --- a/model.py +++ b/model.py @@ -163,7 +163,14 @@ class Model: elif target_language in ["Portuguese European", "Portuguese Brazilian"]: target_language = "Portuguese" - translation = self.translator.translate_ctranslate2( + # translation = self.translator.translate_ctranslate2( + # translator_name=translator_name, + # source_language=source_language, + # target_language=target_language, + # message=message + # ) + + translation = self.translator.translate( translator_name=translator_name, source_language=source_language, target_language=target_language, @@ -194,7 +201,13 @@ class Model: elif target_language in ["Portuguese European", "Portuguese Brazilian"]: target_language = "Portuguese" - translation = self.translator.translate_ctranslate2( + # translation = self.translator.translate_ctranslate2( + # translator_name=translator_name, + # source_language=source_language, + # target_language=target_language, + # message=message + # ) + translation = self.translator.translate( translator_name=translator_name, source_language=source_language, target_language=target_language, diff --git a/models/translation/translation_translator.py b/models/translation/translation_translator.py index aa9c6d5c..496a9b44 100644 --- a/models/translation/translation_translator.py +++ b/models/translation/translation_translator.py @@ -1,6 +1,5 @@ import os from deepl import Translator as deepl_Translator -from deepl_translate import translate as deepl_web_Translator from translators import translate_text as other_web_Translator from .translation_languages import translation_lang @@ -40,10 +39,11 @@ class Translator(): target_language=translation_lang[translator_name]["target"][target_language] match translator_name: case "DeepL": - result = deepl_web_Translator( - source_language=source_language, - target_language=target_language, - text=message + result = other_web_Translator( + query_text=message, + translator="deepl", + from_language=source_language, + to_language=target_language, ) case "DeepL_API": result = self.deepl_client.translate_text( @@ -65,6 +65,20 @@ class Translator(): from_language=source_language, to_language=target_language, ) + case "Papago": + result = other_web_Translator( + query_text=message, + translator="papago", + from_language=source_language, + to_language=target_language, + ) + case "CTranslate2": + self.tokenizer.src_lang = source_language + source = self.tokenizer.convert_ids_to_tokens(self.tokenizer.encode(message)) + target_prefix = [self.tokenizer.lang_code_to_token[target_language]] + results = self.translator.translate_batch([source], target_prefix=[target_prefix]) + target = results[0].hypotheses[0][1:] + result = self.tokenizer.decode(self.tokenizer.convert_tokens_to_ids(target)) except Exception: import traceback with open('error.log', 'a') as f: @@ -72,16 +86,16 @@ class Translator(): result = False return result - def translate_ctranslate2(self, translator_name, source_language, target_language, message): + # def translate_ctranslate2(self, translator_name, source_language, target_language, message): - source_language=translation_lang["ctranslate2"]["source"][source_language] - target_language=translation_lang["ctranslate2"]["target"][target_language] + # source_language=translation_lang["ctranslate2"]["source"][source_language] + # target_language=translation_lang["ctranslate2"]["target"][target_language] - self.tokenizer.src_lang = source_language - source = self.tokenizer.convert_ids_to_tokens(self.tokenizer.encode(message)) - target_prefix = [self.tokenizer.lang_code_to_token[target_language]] - results = self.translator.translate_batch([source], target_prefix=[target_prefix]) - target = results[0].hypotheses[0][1:] + # self.tokenizer.src_lang = source_language + # source = self.tokenizer.convert_ids_to_tokens(self.tokenizer.encode(message)) + # target_prefix = [self.tokenizer.lang_code_to_token[target_language]] + # results = self.translator.translate_batch([source], target_prefix=[target_prefix]) + # target = results[0].hypotheses[0][1:] - result = self.tokenizer.decode(self.tokenizer.convert_tokens_to_ids(target)) - return result \ No newline at end of file + # result = self.tokenizer.decode(self.tokenizer.convert_tokens_to_ids(target)) + # return result \ No newline at end of file From 1777988294f01cac4974c7c1fa8a9a7932e6346e Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Sat, 30 Dec 2023 23:38:09 +0900 Subject: [PATCH 13/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Main=20:=20?= =?UTF-8?q?=E7=BF=BB=E8=A8=B3=E3=82=A8=E3=83=B3=E3=82=B8=E3=83=B3=E3=81=AE?= =?UTF-8?q?=E6=89=8B=E5=8B=95=E8=A8=AD=E5=AE=9A=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 翻訳エンジンの自動設定機能を削除 input/outputの翻訳エンジンを別に設定できるように機能を追加 Aurhkeyの認証をDeepLのみ削減 --- config.py | 54 ++++++++++++++++-- controller.py | 52 +++++++++++++----- model.py | 58 +++++++------------- models/translation/translation_translator.py | 14 ++--- 4 files changed, 112 insertions(+), 66 deletions(-) diff --git a/config.py b/config.py index 44529b5a..93e70c2d 100644 --- a/config.py +++ b/config.py @@ -150,13 +150,22 @@ class Config: self._TARGET_LANGUAGE = value @property - def CHOICE_TRANSLATOR(self): - return self._CHOICE_TRANSLATOR + def CHOICE_INPUT_TRANSLATOR(self): + return self._CHOICE_INPUT_TRANSLATOR - @CHOICE_TRANSLATOR.setter - def CHOICE_TRANSLATOR(self, value): + @CHOICE_INPUT_TRANSLATOR.setter + def CHOICE_INPUT_TRANSLATOR(self, value): if value in translatorEngine: - self._CHOICE_TRANSLATOR = value + self._CHOICE_INPUT_TRANSLATOR= value + + @property + def CHOICE_OUTPUT_TRANSLATOR(self): + return self._CHOICE_OUTPUT_TRANSLATOR + + @CHOICE_OUTPUT_TRANSLATOR.setter + def CHOICE_OUTPUT_TRANSLATOR(self, value): + if value in translatorEngine: + self._CHOICE_OUTPUT_TRANSLATOR = value # Save Json Data ## Main Window @@ -171,6 +180,28 @@ class Config: self._SELECTED_TAB_NO = value saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value) + @property + @json_serializable('SELECTED_TAB_YOUR_TRANSLATOR_ENGINES') + def SELECTED_TAB_YOUR_TRANSLATOR_ENGINES(self): + return self._SELECTED_TAB_YOUR_TRANSLATOR_ENGINES + + @SELECTED_TAB_YOUR_TRANSLATOR_ENGINES.setter + def SELECTED_TAB_YOUR_TRANSLATOR_ENGINES(self, value): + if isinstance(value, dict): + self._SELECTED_TAB_YOUR_TRANSLATOR_ENGINES = value + saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value) + + @property + @json_serializable('SELECTED_TAB_TARGET_TRANSLATOR_ENGINES') + def SELECTED_TAB_TARGET_TRANSLATOR_ENGINES(self): + return self._SELECTED_TAB_TARGET_TRANSLATOR_ENGINES + + @SELECTED_TAB_TARGET_TRANSLATOR_ENGINES.setter + def SELECTED_TAB_TARGET_TRANSLATOR_ENGINES(self, value): + if isinstance(value, dict): + self._SELECTED_TAB_TARGET_TRANSLATOR_ENGINES = value + saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value) + @property @json_serializable('SELECTED_TAB_YOUR_LANGUAGES') def SELECTED_TAB_YOUR_LANGUAGES(self): @@ -562,7 +593,8 @@ class Config: self._ENABLE_TRANSCRIPTION_SEND = False self._ENABLE_TRANSCRIPTION_RECEIVE = False self._ENABLE_FOREGROUND = False - self._CHOICE_TRANSLATOR = translatorEngine[0] + self._CHOICE_INPUT_TRANSLATOR = translatorEngine[0] + self._CHOICE_OUTPUT_TRANSLATOR = translatorEngine[0] self._SOURCE_LANGUAGE = "Japanese" self._SOURCE_COUNTRY = "Japan" self._TARGET_LANGUAGE = "English" @@ -571,6 +603,16 @@ class Config: # Save Json Data ## Main Window self._SELECTED_TAB_NO = "1" + self._SELECTED_TAB_YOUR_TRANSLATOR_ENGINES = { + "1":translatorEngine[0], + "2":translatorEngine[0], + "3":translatorEngine[0], + } + self._SELECTED_TAB_TARGET_TRANSLATOR_ENGINES = { + "1":translatorEngine[0], + "2":translatorEngine[0], + "3":translatorEngine[0], + } self._SELECTED_TAB_YOUR_LANGUAGES = { "1":"Japanese\n(Japan)", "2":"Japanese\n(Japan)", diff --git a/controller.py b/controller.py index e6352411..fceebc2c 100644 --- a/controller.py +++ b/controller.py @@ -209,6 +209,12 @@ def messageBoxFocusOut(e): model.oscStopSendTyping() # func select languages +def initSetTranslateEngine(): + engine = config.SELECTED_TAB_YOUR_TRANSLATOR_ENGINES[config.SELECTED_TAB_NO] + config.CHOICE_INPUT_TRANSLATOR = engine + engine = config.SELECTED_TAB_TARGET_TRANSLATOR_ENGINES[config.SELECTED_TAB_NO] + config.CHOICE_OUTPUT_TRANSLATOR = engine + def initSetLanguageAndCountry(): select = config.SELECTED_TAB_YOUR_LANGUAGES[config.SELECTED_TAB_NO] language, country = model.getLanguageAndCountry(select) @@ -218,7 +224,18 @@ def initSetLanguageAndCountry(): language, country = model.getLanguageAndCountry(select) config.TARGET_LANGUAGE = language config.TARGET_COUNTRY = country - config.CHOICE_TRANSLATOR = model.findTranslationEngine(config.SOURCE_LANGUAGE, config.TARGET_LANGUAGE) + +def setYourTranslateEngine(select): + engines = config.SELECTED_TAB_YOUR_TRANSLATOR_ENGINES + engines[config.SELECTED_TAB_NO] = select + config.SELECTED_TAB_YOUR_TRANSLATOR_ENGINES = engines + config.CHOICE_INPUT_TRANSLATOR = select + +def setTargetTranslateEngine(select): + engines = config.SELECTED_TAB_TARGET_TRANSLATOR_ENGINES + engines[config.SELECTED_TAB_NO] = select + config.SELECTED_TAB_TARGET_TRANSLATOR_ENGINES = engines + config.CHOICE_OUTPUT_TRANSLATOR = select def setYourLanguageAndCountry(select): languages = config.SELECTED_TAB_YOUR_LANGUAGES @@ -227,7 +244,6 @@ def setYourLanguageAndCountry(select): language, country = model.getLanguageAndCountry(select) config.SOURCE_LANGUAGE = language config.SOURCE_COUNTRY = country - config.CHOICE_TRANSLATOR = model.findTranslationEngine(config.SOURCE_LANGUAGE, config.TARGET_LANGUAGE) view.printToTextbox_selectedYourLanguages(select) def setTargetLanguageAndCountry(select): @@ -237,7 +253,6 @@ def setTargetLanguageAndCountry(select): language, country = model.getLanguageAndCountry(select) config.TARGET_LANGUAGE = language config.TARGET_COUNTRY = country - config.CHOICE_TRANSLATOR = model.findTranslationEngine(config.SOURCE_LANGUAGE, config.TARGET_LANGUAGE) view.printToTextbox_selectedTargetLanguages(select) def swapYourLanguageAndTargetLanguage(): @@ -252,17 +267,26 @@ def swapYourLanguageAndTargetLanguage(): def callbackSelectedLanguagePresetTab(selected_tab_no): config.SELECTED_TAB_NO = selected_tab_no view.updateGuiVariableByPresetTabNo(config.SELECTED_TAB_NO) + + engines = config.SELECTED_TAB_YOUR_TRANSLATOR_ENGINES + engine = engines[config.SELECTED_TAB_NO] + config.CHOICE_INPUT_TRANSLATOR = engine + + engines = config.SELECTED_TAB_TARGET_TRANSLATOR_ENGINES + engine = engines[config.SELECTED_TAB_NO] + config.CHOICE_OUTPUT_TRANSLATOR = engine + languages = config.SELECTED_TAB_YOUR_LANGUAGES select = languages[config.SELECTED_TAB_NO] language, country = model.getLanguageAndCountry(select) config.SOURCE_LANGUAGE = language config.SOURCE_COUNTRY = country + languages = config.SELECTED_TAB_TARGET_LANGUAGES select = languages[config.SELECTED_TAB_NO] language, country = model.getLanguageAndCountry(select) config.TARGET_LANGUAGE = language config.TARGET_COUNTRY = country - config.CHOICE_TRANSLATOR = model.findTranslationEngine(config.SOURCE_LANGUAGE, config.TARGET_LANGUAGE) view.printToTextbox_changedLanguagePresetTab(config.SELECTED_TAB_NO) # command func @@ -392,7 +416,7 @@ def callbackSetUiLanguage(value): def callbackSetDeeplAuthkey(value): print("callbackSetDeeplAuthkey", str(value)) if len(value) == 39: - result = model.authenticationTranslator(choice_translator="DeepL_API", auth_key=value) + result = model.authenticationTranslatorDeepLAuthKey(auth_key=value) if result is True: key = value view.printToTextbox_AuthenticationSuccess() @@ -402,12 +426,10 @@ def callbackSetDeeplAuthkey(value): auth_keys = config.AUTH_KEYS auth_keys["DeepL_API"] = key config.AUTH_KEYS = auth_keys - config.CHOICE_TRANSLATOR = model.findTranslationEngine(config.SOURCE_LANGUAGE, config.TARGET_LANGUAGE) elif len(value) == 0: auth_keys = config.AUTH_KEYS auth_keys["DeepL_API"] = None config.AUTH_KEYS = auth_keys - config.CHOICE_TRANSLATOR = model.findTranslationEngine(config.SOURCE_LANGUAGE, config.TARGET_LANGUAGE) # Transcription Tab (Mic) def callbackSetMicHost(value): @@ -703,15 +725,17 @@ def createMainWindow(): # init config initSetConfigByExeArguments() + initSetTranslateEngine() initSetLanguageAndCountry() - if model.authenticationTranslator(config.CHOICE_TRANSLATOR, config.AUTH_KEYS[config.CHOICE_TRANSLATOR]) is False: - # error update Auth key - auth_keys = config.AUTH_KEYS - auth_keys[config.CHOICE_TRANSLATOR] = None - config.AUTH_KEYS = auth_keys - view.printToTextbox_AuthenticationError() - config.CHOICE_TRANSLATOR = model.findTranslationEngine(config.SOURCE_LANGUAGE, config.TARGET_LANGUAGE) + # if (config.SELECTED_TAB_YOUR_TRANSLATOR_ENGINES[config.SELECTED_TAB_NO] == "DeepL_API" or + # config.SELECTED_TAB_TARGET_TRANSLATOR_ENGINES[config.SELECTED_TAB_NO] == "DeepL_API"): + # if model.authenticationTranslator("DeepL_API", config.AUTH_KEYS["DeepL_API"]) is False: + # # error update Auth key + # auth_keys = config.AUTH_KEYS + # auth_keys["DeepL_API"] = None + # config.AUTH_KEYS = auth_keys + # view.printToTextbox_AuthenticationError() # set word filter model.addKeywords() diff --git a/model.py b/model.py index f7fa4642..30e6a852 100644 --- a/model.py +++ b/model.py @@ -83,13 +83,8 @@ class Model: del self.keyword_processor self.keyword_processor = KeywordProcessor() - def authenticationTranslator(self, choice_translator=None, auth_key=None): - if choice_translator is None: - choice_translator = config.CHOICE_TRANSLATOR - if auth_key is None: - auth_key = config.AUTH_KEYS[choice_translator] - - result = self.translator.authentication(choice_translator, auth_key) + def authenticationTranslatorDeepLAuthKey(self, auth_key): + result = self.translator.authenticationDeepLAuthKey(auth_key) return result def startLogger(self): @@ -123,25 +118,25 @@ class Model: country = parts[1][1:-1] return language, country - def findTranslationEngine(self, source_lang, target_lang): - compatible_engines = [] - for engine in translatorEngine: - source_languages = translation_lang.get(engine, {}).get("source", {}) - target_languages = translation_lang.get(engine, {}).get("target", {}) - if source_lang in source_languages and target_lang in target_languages: - compatible_engines.append(engine) - engine_name = compatible_engines[0] + # def findTranslationEngine(self, source_lang, target_lang): + # compatible_engines = [] + # for engine in translatorEngine: + # source_languages = translation_lang.get(engine, {}).get("source", {}) + # target_languages = translation_lang.get(engine, {}).get("target", {}) + # if source_lang in source_languages and target_lang in target_languages: + # compatible_engines.append(engine) + # engine_name = compatible_engines[0] - if engine_name == "DeepL" and config.AUTH_KEYS["DeepL_API"] is not None: - if self.authenticationTranslator(engine_name, config.AUTH_KEYS["DeepL_API"]) is True: - engine_name = "DeepL_API" - elif engine_name == "DeepL_API" and config.AUTH_KEYS["DeepL_API"] is None: - engine_name = "DeepL" + # if engine_name == "DeepL" and config.AUTH_KEYS["DeepL_API"] is not None: + # if self.authenticationTranslator(engine_name, config.AUTH_KEYS["DeepL_API"]) is True: + # engine_name = "DeepL_API" + # elif engine_name == "DeepL_API" and config.AUTH_KEYS["DeepL_API"] is None: + # engine_name = "DeepL" - return engine_name + # return engine_name def getInputTranslate(self, message): - translator_name=config.CHOICE_TRANSLATOR + translator_name=config.CHOICE_INPUT_TRANSLATOR source_language=config.SOURCE_LANGUAGE target_language=config.TARGET_LANGUAGE target_country = config.TARGET_COUNTRY @@ -157,19 +152,12 @@ class Model: target_language = "Portuguese European" else: target_language = "Portuguese Brazilian" - elif translator_name == "DeepL": + else: if target_language in ["English American", "English British"]: target_language = "English" elif target_language in ["Portuguese European", "Portuguese Brazilian"]: target_language = "Portuguese" - # translation = self.translator.translate_ctranslate2( - # translator_name=translator_name, - # source_language=source_language, - # target_language=target_language, - # message=message - # ) - translation = self.translator.translate( translator_name=translator_name, source_language=source_language, @@ -179,7 +167,7 @@ class Model: return translation def getOutputTranslate(self, message): - translator_name=config.CHOICE_TRANSLATOR + translator_name=config.CHOICE_OUTPUT_TRANSLATOR source_language=config.TARGET_LANGUAGE target_language=config.SOURCE_LANGUAGE target_country = config.SOURCE_COUNTRY @@ -195,18 +183,12 @@ class Model: target_language = "Portuguese European" else: target_language = "Portuguese Brazilian" - elif translator_name == "DeepL": + else: if target_language in ["English American", "English British"]: target_language = "English" elif target_language in ["Portuguese European", "Portuguese Brazilian"]: target_language = "Portuguese" - # translation = self.translator.translate_ctranslate2( - # translator_name=translator_name, - # source_language=source_language, - # target_language=target_language, - # message=message - # ) translation = self.translator.translate( translator_name=translator_name, source_language=source_language, diff --git a/models/translation/translation_translator.py b/models/translation/translation_translator.py index 496a9b44..371c907e 100644 --- a/models/translation/translation_translator.py +++ b/models/translation/translation_translator.py @@ -21,15 +21,13 @@ class Translator(): self.translator = ctranslate2.Translator(self.weight_path, device="cpu", device_index=0, compute_type="int8", inter_threads=1, intra_threads=4) self.tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer) - def authentication(self, translator_name, authkey=None): + def authenticationDeepLAuthKey(self, authkey): result = True - match translator_name: - case "DeepL_API": - try: - self.deepl_client = deepl_Translator(authkey) - self.deepl_client.translate_text(" ", target_lang="EN-US") - except Exception: - result = False + try: + self.deepl_client = deepl_Translator(authkey) + self.deepl_client.translate_text(" ", target_lang="EN-US") + except Exception: + result = False return result def translate(self, translator_name, source_language, target_language, message): From fa32d0ff87de3b6955d86d807923f57cea323aae Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Sun, 31 Dec 2023 00:36:50 +0900 Subject: [PATCH 14/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20?= =?UTF-8?q?=E6=96=87=E5=AD=97=E8=B5=B7=E3=81=93=E3=81=97=E3=81=A8=E7=BF=BB?= =?UTF-8?q?=E8=A8=B3=E3=82=A8=E3=83=B3=E3=82=B8=E3=83=B3=E3=81=AE=E7=B5=84?= =?UTF-8?q?=E3=82=8F=E3=81=9B=E3=81=AE=E6=9C=89=E5=8A=B9=E6=80=A7=E8=BE=9E?= =?UTF-8?q?=E6=9B=B8=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getDictTranslationEngineValidity --- model.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/model.py b/model.py index 30e6a852..f5d4c0d7 100644 --- a/model.py +++ b/model.py @@ -126,15 +126,32 @@ class Model: # if source_lang in source_languages and target_lang in target_languages: # compatible_engines.append(engine) # engine_name = compatible_engines[0] - # if engine_name == "DeepL" and config.AUTH_KEYS["DeepL_API"] is not None: # if self.authenticationTranslator(engine_name, config.AUTH_KEYS["DeepL_API"]) is True: # engine_name = "DeepL_API" # elif engine_name == "DeepL_API" and config.AUTH_KEYS["DeepL_API"] is None: # engine_name = "DeepL" - # return engine_name + def getDictTranslationEngineValidity(self): + ts_keys = transcription_lang.keys() + tl_keys = translation_lang.keys() + te_dict = {} + for ts_key in ts_keys: + te_dict[ts_key] = {} + for tl_key in tl_keys: + te_dict[ts_key][tl_key] = False + if ts_key in translation_lang[tl_key]["source"]: + te_dict[ts_key][tl_key] = True + + keys_to_remove = [] + for language, translation_dict in te_dict.items(): + if all(value is False for value in translation_dict.values()): + keys_to_remove.append(language) + for key in keys_to_remove: + del te_dict[key] + return te_dict + def getInputTranslate(self, message): translator_name=config.CHOICE_INPUT_TRANSLATOR source_language=config.SOURCE_LANGUAGE From aac8f08da96bfa6ad2a277a68fe33b5c9d4ddfba Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Thu, 4 Jan 2024 00:46:20 +0900 Subject: [PATCH 15/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20?= =?UTF-8?q?=E7=BF=BB=E8=A8=B3=E5=A4=B1=E6=95=97=E6=99=82=E3=81=AE=E3=83=95?= =?UTF-8?q?=E3=82=A7=E3=83=BC=E3=83=AB=E3=82=BB=E3=83=BC=E3=83=95=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- model.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/model.py b/model.py index f5d4c0d7..bf048737 100644 --- a/model.py +++ b/model.py @@ -187,7 +187,7 @@ class Model: translator_name=config.CHOICE_OUTPUT_TRANSLATOR source_language=config.TARGET_LANGUAGE target_language=config.SOURCE_LANGUAGE - target_country = config.SOURCE_COUNTRY + target_country=config.SOURCE_COUNTRY if translator_name == "DeepL_API": if target_language == "English": @@ -212,6 +212,15 @@ class Model: target_language=target_language, message=message ) + + # 翻訳失敗時のフェールセーフ処理 + if translation is False and "Filipino": + translation = self.translator.translate( + translator_name="CTranslate2", + source_language=config.TARGET_LANGUAGE, + target_language=config.SOURCE_LANGUAGE, + message=message + ) return translation def addKeywords(self): @@ -496,4 +505,4 @@ class Model: def notificationXSOverlay(self, message): xsoverlayForVRCT(content=f"{message}") -model = Model() +model = Model() \ No newline at end of file From 9e91be145e5792a6350f1aed1a0d3facfcc17ac8 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Thu, 4 Jan 2024 03:01:29 +0900 Subject: [PATCH 16/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20getLis?= =?UTF-8?q?tLanguageAndCountry=E3=82=92=E5=8B=95=E7=9A=84=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E5=8C=96=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit findTranslationEnginesで翻訳エンジンのリストを取得するように変更 --- model.py | 68 ++++++++++++++++++-------------------------------------- 1 file changed, 22 insertions(+), 46 deletions(-) diff --git a/model.py b/model.py index bf048737..72d7a0cb 100644 --- a/model.py +++ b/model.py @@ -45,15 +45,6 @@ class threadFnc(Thread): self.fnc(*self._args, **self._kwargs) class Model: - # Languages available for both transcription and translation - SUPPORTED_LANGUAGES = [ - 'Afrikaans', 'Arabic', 'Basque', 'Bulgarian', 'Catalan', 'Chinese', 'Croatian', - 'Czech', 'Danish', 'Dutch', 'English', 'Filipino', 'Finnish', 'French', 'German', - 'Greek', 'Hebrew', 'Hindi', 'Hungarian', 'Indonesian', 'Italian', 'Japanese', - 'Korean', 'Lithuanian', 'Malay', 'Norwegian', 'Polish', 'Portuguese', 'Romanian', - 'Russian', 'Serbian', 'Slovak', 'Slovenian', 'Spanish', 'Swedish', 'Thai', 'Turkish', - 'Ukrainian', 'Vietnamese' - ] _instance = None def __new__(cls): @@ -103,13 +94,23 @@ class Model: self.logger.disabled = True self.logger = None - @staticmethod - def getListLanguageAndCountry(): + def getListLanguageAndCountry(self): + transcription_langs = list(transcription_lang.keys()) + tl_keys = translation_lang.keys() + translation_langs = [] + for tl_key in tl_keys: + for lang in translation_lang[tl_key]["source"]: + translation_langs.append(lang) + for lang in translation_lang[tl_key]["target"]: + translation_langs.append(lang) + translation_langs = list(set(translation_langs)) + supported_langs = list(filter(lambda x: x in transcription_langs, translation_langs)) + langs = [] - for lang in model.SUPPORTED_LANGUAGES: + for lang in supported_langs: for country in transcription_lang[lang]: langs.append(f"{lang}\n({country})") - return langs + return sorted(langs) @staticmethod def getLanguageAndCountry(select): @@ -118,39 +119,14 @@ class Model: country = parts[1][1:-1] return language, country - # def findTranslationEngine(self, source_lang, target_lang): - # compatible_engines = [] - # for engine in translatorEngine: - # source_languages = translation_lang.get(engine, {}).get("source", {}) - # target_languages = translation_lang.get(engine, {}).get("target", {}) - # if source_lang in source_languages and target_lang in target_languages: - # compatible_engines.append(engine) - # engine_name = compatible_engines[0] - # if engine_name == "DeepL" and config.AUTH_KEYS["DeepL_API"] is not None: - # if self.authenticationTranslator(engine_name, config.AUTH_KEYS["DeepL_API"]) is True: - # engine_name = "DeepL_API" - # elif engine_name == "DeepL_API" and config.AUTH_KEYS["DeepL_API"] is None: - # engine_name = "DeepL" - # return engine_name - - def getDictTranslationEngineValidity(self): - ts_keys = transcription_lang.keys() - tl_keys = translation_lang.keys() - te_dict = {} - for ts_key in ts_keys: - te_dict[ts_key] = {} - for tl_key in tl_keys: - te_dict[ts_key][tl_key] = False - if ts_key in translation_lang[tl_key]["source"]: - te_dict[ts_key][tl_key] = True - - keys_to_remove = [] - for language, translation_dict in te_dict.items(): - if all(value is False for value in translation_dict.values()): - keys_to_remove.append(language) - for key in keys_to_remove: - del te_dict[key] - return te_dict + def findTranslationEngines(self, source_lang, target_lang): + compatible_engines = [] + for engine in translatorEngine: + source_languages = translation_lang.get(engine, {}).get("source", {}) + target_languages = translation_lang.get(engine, {}).get("target", {}) + if source_lang in source_languages and target_lang in target_languages: + compatible_engines.append(engine) + return compatible_engines def getInputTranslate(self, message): translator_name=config.CHOICE_INPUT_TRANSLATOR From 0742d635af70e13f6d6998d6184c2d6757eaf5f1 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Sun, 7 Jan 2024 01:43:50 +0900 Subject: [PATCH 17/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20DeepL?= =?UTF-8?q?=5FAPI=E3=81=AE=E8=AA=8D=E8=A8=BC=E5=87=A6=E7=90=86=E3=81=AE?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 3 --- controller.py | 15 +++++++-------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/config.py b/config.py index 93e70c2d..e206305d 100644 --- a/config.py +++ b/config.py @@ -649,9 +649,6 @@ class Config: self._OSC_PORT = 9000 self._AUTH_KEYS = { "DeepL_API": None, - "DeepL": None, - "Bing": None, - "Google": None, } self._WEIGHT_TYPE = "small" self._MESSAGE_FORMAT = "[message]([translation])" diff --git a/controller.py b/controller.py index fceebc2c..05c3cb91 100644 --- a/controller.py +++ b/controller.py @@ -728,14 +728,13 @@ def createMainWindow(): initSetTranslateEngine() initSetLanguageAndCountry() - # if (config.SELECTED_TAB_YOUR_TRANSLATOR_ENGINES[config.SELECTED_TAB_NO] == "DeepL_API" or - # config.SELECTED_TAB_TARGET_TRANSLATOR_ENGINES[config.SELECTED_TAB_NO] == "DeepL_API"): - # if model.authenticationTranslator("DeepL_API", config.AUTH_KEYS["DeepL_API"]) is False: - # # error update Auth key - # auth_keys = config.AUTH_KEYS - # auth_keys["DeepL_API"] = None - # config.AUTH_KEYS = auth_keys - # view.printToTextbox_AuthenticationError() + if config.AUTH_KEYS["DeepL_API"] is not None: + if model.authenticationTranslatorDeepLAuthKey("DeepL_API", config.AUTH_KEYS["DeepL_API"]) is False: + # error update Auth key + auth_keys = config.AUTH_KEYS + auth_keys["DeepL_API"] = None + config.AUTH_KEYS = auth_keys + view.printToTextbox_AuthenticationError() # set word filter model.addKeywords() From 82dde149ee70398b9abf53a1729c42b3f4c3b91d Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Sun, 7 Jan 2024 01:46:29 +0900 Subject: [PATCH 18/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20?= =?UTF-8?q?=E7=BF=BB=E8=A8=B3=E5=A4=B1=E6=95=97=E6=99=82=E3=81=ABCTranslat?= =?UTF-8?q?e2=E3=81=A7=E3=81=AE=E7=BF=BB=E8=A8=B3=E5=87=A6=E7=90=86?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 不要なコードを削除 --- model.py | 83 ++++++++------------ models/translation/translation_languages.py | 4 +- models/translation/translation_translator.py | 43 +++++----- 3 files changed, 58 insertions(+), 72 deletions(-) diff --git a/model.py b/model.py index 72d7a0cb..16fc086b 100644 --- a/model.py +++ b/model.py @@ -1,4 +1,3 @@ -import tempfile from zipfile import ZipFile from subprocess import Popen from os import makedirs as os_makedirs @@ -10,10 +9,9 @@ from logging import getLogger, FileHandler, Formatter, INFO from time import sleep from queue import Queue from threading import Thread, Event -from requests import get as requests_get, head as requests_head +from requests import get as requests_get import webbrowser -from tqdm import tqdm from flashtext import KeywordProcessor from models.translation.translation_translator import Translator from models.transcription.transcription_utils import getInputDevices, getDefaultOutputDevice @@ -68,7 +66,7 @@ class Model: def resetTranslator(self): del self.translator - self.translator = Translator(config.PATH_LOCAL) + self.translator = Translator(config.PATH_LOCAL, config.CTRANSLATE2_WEIGHTS[config.WEIGHT_TYPE]) def resetKeywordProcessor(self): del self.keyword_processor @@ -101,8 +99,6 @@ class Model: for tl_key in tl_keys: for lang in translation_lang[tl_key]["source"]: translation_langs.append(lang) - for lang in translation_lang[tl_key]["target"]: - translation_langs.append(lang) translation_langs = list(set(translation_langs)) supported_langs = list(filter(lambda x: x in transcription_langs, translation_langs)) @@ -122,81 +118,70 @@ class Model: def findTranslationEngines(self, source_lang, target_lang): compatible_engines = [] for engine in translatorEngine: - source_languages = translation_lang.get(engine, {}).get("source", {}) - target_languages = translation_lang.get(engine, {}).get("target", {}) - if source_lang in source_languages and target_lang in target_languages: + languages = translation_lang.get(engine, {}).get("source", {}) + if source_lang in languages and target_lang in languages: compatible_engines.append(engine) + if "DeepL_API" in compatible_engines: + if self.translator.deepl_client is None: + compatible_engines.remove('DeepL_API') return compatible_engines - def getInputTranslate(self, message): + def getInputTranslate(self, message, fnc=None): translator_name=config.CHOICE_INPUT_TRANSLATOR source_language=config.SOURCE_LANGUAGE target_language=config.TARGET_LANGUAGE target_country = config.TARGET_COUNTRY - if translator_name == "DeepL_API": - if target_language == "English": - if target_country in ["United States", "Canada", "Philippines"]: - target_language = "English American" - else: - target_language = "English British" - elif target_language == "Portuguese": - if target_country in ["Portugal"]: - target_language = "Portuguese European" - else: - target_language = "Portuguese Brazilian" - else: - if target_language in ["English American", "English British"]: - target_language = "English" - elif target_language in ["Portuguese European", "Portuguese Brazilian"]: - target_language = "Portuguese" - translation = self.translator.translate( translator_name=translator_name, source_language=source_language, target_language=target_language, + target_country=target_country, message=message ) + + # 翻訳失敗時のフェールセーフ処理 + if translation is False: + translation = self.translator.translate( + translator_name="CTranslate2", + source_language=source_language, + target_language=target_language, + target_country=target_country, + message=message + ) + try: + fnc() + except Exception: + pass return translation - def getOutputTranslate(self, message): + def getOutputTranslate(self, message, fnc=None): translator_name=config.CHOICE_OUTPUT_TRANSLATOR source_language=config.TARGET_LANGUAGE target_language=config.SOURCE_LANGUAGE target_country=config.SOURCE_COUNTRY - if translator_name == "DeepL_API": - if target_language == "English": - if target_country in ["United States", "Canada", "Philippines"]: - target_language = "English American" - else: - target_language = "English British" - elif target_language == "Portuguese": - if target_country in ["Portugal"]: - target_language = "Portuguese European" - else: - target_language = "Portuguese Brazilian" - else: - if target_language in ["English American", "English British"]: - target_language = "English" - elif target_language in ["Portuguese European", "Portuguese Brazilian"]: - target_language = "Portuguese" - translation = self.translator.translate( translator_name=translator_name, source_language=source_language, target_language=target_language, + target_country=target_country, message=message ) - + # 翻訳失敗時のフェールセーフ処理 - if translation is False and "Filipino": + if translation is False: translation = self.translator.translate( translator_name="CTranslate2", - source_language=config.TARGET_LANGUAGE, - target_language=config.SOURCE_LANGUAGE, + source_language=source_language, + target_language=target_language, + target_country=target_country, message=message ) + try: + fnc() + except Exception: + pass return translation def addKeywords(self): diff --git a/models/translation/translation_languages.py b/models/translation/translation_languages.py index 959b9077..1103a5c0 100644 --- a/models/translation/translation_languages.py +++ b/models/translation/translation_languages.py @@ -360,7 +360,7 @@ dict_ctranslate2_languages = { 'Luxembourgish': 'lb', 'Myanmar': 'my', 'Tibetan': 'bo', - 'Tagalog': 'tl', + 'Filipino': 'tl', 'Malagasy': 'mg', 'Assamese': 'as', 'Tatar': 'tt', @@ -372,7 +372,7 @@ dict_ctranslate2_languages = { 'Sundanese': 'su' } -translation_lang["ctranslate2"] = { +translation_lang["CTranslate2"] = { "source":dict_ctranslate2_languages, "target":dict_ctranslate2_languages, } \ No newline at end of file diff --git a/models/translation/translation_translator.py b/models/translation/translation_translator.py index 371c907e..21710e2d 100644 --- a/models/translation/translation_translator.py +++ b/models/translation/translation_translator.py @@ -20,6 +20,7 @@ class Translator(): self.weight_path = os.path.join(path, "weight", directory_name) self.translator = ctranslate2.Translator(self.weight_path, device="cpu", device_index=0, compute_type="int8", inter_threads=1, intra_threads=4) self.tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer) + self.deepl_client = None def authenticationDeepLAuthKey(self, authkey): result = True @@ -27,10 +28,11 @@ class Translator(): self.deepl_client = deepl_Translator(authkey) self.deepl_client.translate_text(" ", target_lang="EN-US") except Exception: + self.deepl_client = None result = False return result - def translate(self, translator_name, source_language, target_language, message): + def translate(self, translator_name, source_language, target_language, target_country, message): try: result = "" source_language=translation_lang[translator_name]["source"][source_language] @@ -44,11 +46,24 @@ class Translator(): to_language=target_language, ) case "DeepL_API": - result = self.deepl_client.translate_text( - message, - source_lang=source_language, - target_lang=target_language, - ).text + if self.deepl_client is None: + result = False + else: + if target_language == "English": + if target_country in ["United States", "Canada", "Philippines"]: + target_language = "English American" + else: + target_language = "English British" + elif target_language == "Portuguese": + if target_country in ["Portugal"]: + target_language = "Portuguese European" + else: + target_language = "Portuguese Brazilian" + result = self.deepl_client.translate_text( + message, + source_lang=source_language, + target_lang=target_language, + ).text case "Google": result = other_web_Translator( query_text=message, @@ -82,18 +97,4 @@ class Translator(): with open('error.log', 'a') as f: traceback.print_exc(file=f) result = False - return result - - # def translate_ctranslate2(self, translator_name, source_language, target_language, message): - - # source_language=translation_lang["ctranslate2"]["source"][source_language] - # target_language=translation_lang["ctranslate2"]["target"][target_language] - - # self.tokenizer.src_lang = source_language - # source = self.tokenizer.convert_ids_to_tokens(self.tokenizer.encode(message)) - # target_prefix = [self.tokenizer.lang_code_to_token[target_language]] - # results = self.translator.translate_batch([source], target_prefix=[target_prefix]) - # target = results[0].hypotheses[0][1:] - - # result = self.tokenizer.decode(self.tokenizer.convert_tokens_to_ids(target)) - # return result \ No newline at end of file + return result \ No newline at end of file From fe3b38c5fddb3f67b611711ed495462f41ee1274 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Sun, 7 Jan 2024 01:50:50 +0900 Subject: [PATCH 19/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20?= =?UTF-8?q?=E7=BF=BB=E8=A8=B3=E8=87=AA=E4=BD=93=E3=81=8C=E5=A4=B1=E6=95=97?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=81=93=E3=81=A8=E3=81=8C=E3=81=AA=E3=81=8F?= =?UTF-8?q?=E3=81=AA=E3=81=A3=E3=81=9F=E3=81=AE=E3=81=A7=E3=83=95=E3=82=A7?= =?UTF-8?q?=E3=83=BC=E3=83=AB=E3=82=BB=E3=83=BC=E3=83=95=E3=81=AE=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ※別途、ポップアップにて翻訳エンジンが使用できなかった旨のメッセージを出す必要がある --- controller.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/controller.py b/controller.py index 05c3cb91..9c3bd9e0 100644 --- a/controller.py +++ b/controller.py @@ -34,10 +34,10 @@ def sendMicMessage(message): pass else: translation = model.getInputTranslate(message) - if translation is False: - config.ENABLE_TRANSLATION = False - translation = "" - view.translationEngineLimitErrorProcess() + # if translation is False: + # config.ENABLE_TRANSLATION = False + # translation = "" + # view.translationEngineLimitErrorProcess() if config.ENABLE_TRANSCRIPTION_SEND is True: if config.ENABLE_SEND_MESSAGE_TO_VRC is True: @@ -98,10 +98,10 @@ def receiveSpeakerMessage(message): pass else: translation = model.getOutputTranslate(message) - if translation is False: - config.ENABLE_TRANSLATION = False - translation = "" - view.translationEngineLimitErrorProcess() + # if translation is False: + # config.ENABLE_TRANSLATION = False + # translation = "" + # view.translationEngineLimitErrorProcess() if config.ENABLE_TRANSCRIPTION_RECEIVE is True: if config.ENABLE_NOTICE_XSOVERLAY is True: @@ -164,10 +164,10 @@ def sendChatMessage(message): pass else: translation = model.getInputTranslate(message) - if translation is False: - config.ENABLE_TRANSLATION = False - translation = "" - view.translationEngineLimitErrorProcess() + # if translation is False: + # config.ENABLE_TRANSLATION = False + # translation = "" + # view.translationEngineLimitErrorProcess() # send OSC message if config.ENABLE_SEND_MESSAGE_TO_VRC is True: From de66233b90dcbfe290e2449820fc219d4281ddd0 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Sun, 7 Jan 2024 02:05:59 +0900 Subject: [PATCH 20/20] =?UTF-8?q?=F0=9F=91=8D[Update]=20Model=20:=20?= =?UTF-8?q?=E4=B8=8D=E8=A6=81=E3=81=AA=E5=A4=89=E6=95=B0=E2=80=9Dtranslato?= =?UTF-8?q?rEngine=E2=80=9D=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 22 ++++++++++----------- model.py | 4 ++-- models/translation/translation_languages.py | 1 - 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/config.py b/config.py index e206305d..4dfae139 100644 --- a/config.py +++ b/config.py @@ -6,7 +6,7 @@ from json import dump as json_dump import tkinter as tk from tkinter import font from languages import selectable_languages -from models.translation.translation_languages import translatorEngine +from models.translation.translation_languages import translation_lang from models.transcription.transcription_utils import getInputDevices, getDefaultInputDevice from models.translation.utils import ctranslate2_weights from utils import generatePercentageStringsList, isUniqueStrings @@ -155,7 +155,7 @@ class Config: @CHOICE_INPUT_TRANSLATOR.setter def CHOICE_INPUT_TRANSLATOR(self, value): - if value in translatorEngine: + if value in list(translation_lang.keys()): self._CHOICE_INPUT_TRANSLATOR= value @property @@ -164,7 +164,7 @@ class Config: @CHOICE_OUTPUT_TRANSLATOR.setter def CHOICE_OUTPUT_TRANSLATOR(self, value): - if value in translatorEngine: + if value in list(translation_lang.keys()): self._CHOICE_OUTPUT_TRANSLATOR = value # Save Json Data @@ -593,8 +593,8 @@ class Config: self._ENABLE_TRANSCRIPTION_SEND = False self._ENABLE_TRANSCRIPTION_RECEIVE = False self._ENABLE_FOREGROUND = False - self._CHOICE_INPUT_TRANSLATOR = translatorEngine[0] - self._CHOICE_OUTPUT_TRANSLATOR = translatorEngine[0] + self._CHOICE_INPUT_TRANSLATOR = "CTranslate2" + self._CHOICE_OUTPUT_TRANSLATOR = "CTranslate2" self._SOURCE_LANGUAGE = "Japanese" self._SOURCE_COUNTRY = "Japan" self._TARGET_LANGUAGE = "English" @@ -604,14 +604,14 @@ class Config: ## Main Window self._SELECTED_TAB_NO = "1" self._SELECTED_TAB_YOUR_TRANSLATOR_ENGINES = { - "1":translatorEngine[0], - "2":translatorEngine[0], - "3":translatorEngine[0], + "1":"CTranslate2", + "2":"CTranslate2", + "3":"CTranslate2", } self._SELECTED_TAB_TARGET_TRANSLATOR_ENGINES = { - "1":translatorEngine[0], - "2":translatorEngine[0], - "3":translatorEngine[0], + "1":"CTranslate2", + "2":"CTranslate2", + "3":"CTranslate2", } self._SELECTED_TAB_YOUR_LANGUAGES = { "1":"Japanese\n(Japan)", diff --git a/model.py b/model.py index 16fc086b..ee72b973 100644 --- a/model.py +++ b/model.py @@ -20,7 +20,7 @@ from models.transcription.transcription_recorder import SelectedMicRecorder, Sel from models.transcription.transcription_recorder import SelectedMicEnergyRecorder, SelectedSpeakeEnergyRecorder from models.transcription.transcription_transcriber import AudioTranscriber from models.xsoverlay.notification import xsoverlayForVRCT -from models.translation.translation_languages import translatorEngine, translation_lang +from models.translation.translation_languages import translation_lang from models.transcription.transcription_languages import transcription_lang from config import config @@ -117,7 +117,7 @@ class Model: def findTranslationEngines(self, source_lang, target_lang): compatible_engines = [] - for engine in translatorEngine: + for engine in list(translation_lang.keys()): languages = translation_lang.get(engine, {}).get("source", {}) if source_lang in languages and target_lang in languages: compatible_engines.append(engine) diff --git a/models/translation/translation_languages.py b/models/translation/translation_languages.py index 1103a5c0..44051f8a 100644 --- a/models/translation/translation_languages.py +++ b/models/translation/translation_languages.py @@ -1,4 +1,3 @@ -translatorEngine = ["DeepL", "DeepL_API", "Google", "Bing", "Papago"] translation_lang = {} dict_deepl_languages = { 'Arabic':'ar',