🐛[bugfix] Model : tokenizerをローカルの指定パスに保存するように変更

This commit is contained in:
misyaguziya
2024-01-29 16:38:13 +09:00
parent f8fe2053aa
commit 86e43371e6

View File

@@ -28,6 +28,7 @@ class Translator():
directory_name = ctranslate2_weights[model_type]["directory_name"]
tokenizer = ctranslate2_weights[model_type]["tokenizer"]
weight_path = os.path.join(path, "weight", directory_name)
tokenizer_path = os.path.join(path, "weight", directory_name, "tokenizer")
self.ctranslate2_translator = ctranslate2.Translator(
weight_path,
device="cpu",
@@ -36,7 +37,7 @@ class Translator():
inter_threads=1,
intra_threads=4
)
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer)
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer, cache_dir=tokenizer_path)
@staticmethod
def getLanguageCode(translator_name, target_country, source_language, target_language):