every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

👍️[Update] Model : status 348のlog を printLog関数に集約

Browse Source
This commit is contained in:
misyaguziya
2024-09-01 17:33:22 +09:00
parent 8f2c81a7d7
commit 4c2e4a6e04
7 changed files with 159 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src-python/models/translation/translation_translator.py
View File

@@ -1,4 +1,7 @@
import os
from os import path as os_path
import sys
sys.path.append(os_path.join(os_path.dirname(__file__), "..", ".."))
from utils import printLog
from deepl import Translator as deepl_Translator
from translators import translate_text as other_web_Translator
from .translation_languages import translation_lang
@@ -32,8 +35,8 @@ class Translator():
self.is_loaded_ctranslate2_model = False
directory_name = ctranslate2_weights[model_type]["directory_name"]
tokenizer = ctranslate2_weights[model_type]["tokenizer"]
weight_path = os.path.join(path, "weights", "ctranslate2", directory_name)
tokenizer_path = os.path.join(path, "weights", "ctranslate2", directory_name, "tokenizer")
weight_path = os_path.join(path, "weights", "ctranslate2", directory_name)
tokenizer_path = os_path.join(path, "weights", "ctranslate2", directory_name, "tokenizer")
self.ctranslate2_translator = ctranslate2.Translator(
weight_path,
device="cpu",
@@ -45,9 +48,8 @@ class Translator():
try:
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer, cache_dir=tokenizer_path)
except Exception as e:
import json
print(json.dumps({"status":348, "log": f"error:changeCTranslate2Model() {e}"}), flush=True)
tokenizer_path = os.path.join("./weights", "ctranslate2", directory_name, "tokenizer")
printLog("error:changeCTranslate2Model()", e)
tokenizer_path = os_path.join("./weights", "ctranslate2", directory_name, "tokenizer")
self.ctranslate2_tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer, cache_dir=tokenizer_path)
self.is_loaded_ctranslate2_model = True

9
src-python/models/translation/translation_utils.py
View File

@@ -2,6 +2,9 @@ import tempfile
from zipfile import ZipFile
from os import path as os_path
from os import makedirs as os_makedirs
import sys
sys.path.append(os_path.join(os_path.dirname(__file__), "..", ".."))
from utils import printLog
from requests import get as requests_get
from typing import Callable
import hashlib
@@ -75,14 +78,14 @@ def downloadCTranslate2Weight(root, weight_type="Small", callbackFunc=None):
file_size = int(res.headers.get('content-length', 0))
total_chunk = 0
with open(os_path.join(tmp_path, filename), 'wb') as file:
for chunk in res.iter_content(chunk_size=1024*5):
for chunk in res.iter_content(chunk_size=1024*2000):
file.write(chunk)
if isinstance(callbackFunc, Callable):
total_chunk += len(chunk)
callbackFunc(total_chunk/file_size)
printLog(f"Downloading {filename}: {total_chunk/file_size:.0%}")
with ZipFile(os_path.join(tmp_path, filename)) as zf:
zf.extractall(path)
except Exception as e:
import json
print(json.dumps({"status":348, "log": f"error:downloadCTranslate2Weight() {e}"}), flush=True)
printLog("error:downloadCTranslate2Weight()", e)