every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'bugfix_tokenizer' into develop

Browse Source
This commit is contained in:
misyaguziya
2024-01-29 23:28:14 +09:00
parent f8fe2053aa 82eab0db3c
commit 5e79cb5539
1 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
models/translation/translation_translator.py
View File

@@ -28,6 +28,7 @@ class Translator():
directory_name = ctranslate2_weights[model_type]["directory_name"] directory_name = ctranslate2_weights[model_type]["directory_name"]
tokenizer = ctranslate2_weights[model_type]["tokenizer"] tokenizer = ctranslate2_weights[model_type]["tokenizer"]
weight_path = os.path.join(path, "weight", directory_name) weight_path = os.path.join(path, "weight", directory_name)
tokenizer_path = os.path.join(path, "weight", directory_name, "tokenizer")
self.ctranslate2_translator = ctranslate2.Translator( self.ctranslate2_translator = ctranslate2.Translator(
weight_path, weight_path,
device="cpu", device="cpu",
@@ -36,7 +37,12 @@ class Translator():
inter_threads=1, inter_threads=1,
intra_threads=4 intra_threads=4
) )
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer) try:
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer, cache_dir=tokenizer_path)
except Exception as e:
print("Error: changeCTranslate2Model()", e)
tokenizer_path = os.path.join("./weight", directory_name, "tokenizer")
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer, cache_dir=tokenizer_path)
@staticmethod @staticmethod
def getLanguageCode(translator_name, target_country, source_language, target_language): def getLanguageCode(translator_name, target_country, source_language, target_language):