diff --git a/src-python/controller.py b/src-python/controller.py index 315113f5..969fff98 100644 --- a/src-python/controller.py +++ b/src-python/controller.py @@ -259,17 +259,44 @@ class Controller: elif config.ENABLE_TRANSLATION is False: pass else: - translation, success = model.getInputTranslate(message, source_language=language) - if all(success) is not True: - self.changeToCTranslate2Process() - self.run( - 400, - self.run_mapping["error_translation_engine"], - { - "message":"Translation engine limit error", - "data": None - }, - ) + try: + translation, success = model.getInputTranslate(message, source_language=language) + if all(success) is not True: + self.changeToCTranslate2Process() + self.run( + 400, + self.run_mapping["error_translation_engine"], + { + "message":"Translation engine limit error", + "data": None + }, + ) + except Exception as e: + # VRAM不足エラーの検出 + is_vram_error, error_message = model.detectVRAMError(e) + if is_vram_error: + self.run( + 400, + self.run_mapping["error_translation_mic_vram_overflow"], + { + "message":"VRAM out of memory during translation of mic", + "data": error_message + }, + ) + # 翻訳機能をOFFにする + self.setDisableTranslation() + self.run( + 400, + self.run_mapping["enable_translation"], + { + "message":"Translation disabled due to VRAM overflow", + "data": False + }, + ) + return + else: + # その他のエラーは通常通り処理 + raise if config.CONVERT_MESSAGE_TO_ROMAJI is True or config.CONVERT_MESSAGE_TO_HIRAGANA is True: if config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] == "Japanese": @@ -346,17 +373,44 @@ class Controller: elif config.ENABLE_TRANSLATION is False: pass else: - translation, success = model.getOutputTranslate(message, source_language=language) - if all(success) is not True: - self.changeToCTranslate2Process() - self.run( - 400, - self.run_mapping["error_translation_engine"], - { - "message":"Translation engine limit error", - "data": None - }, - ) + try: + translation, success = model.getOutputTranslate(message, source_language=language) + if all(success) is not True: + self.changeToCTranslate2Process() + self.run( + 400, + self.run_mapping["error_translation_engine"], + { + "message":"Translation engine limit error", + "data": None + }, + ) + except Exception as e: + # VRAM不足エラーの検出 + is_vram_error, error_message = model.detectVRAMError(e) + if is_vram_error: + self.run( + 400, + self.run_mapping["error_translation_speaker_vram_overflow"], + { + "message":"VRAM out of memory during translation of speaker", + "data": error_message + }, + ) + # 翻訳機能をOFFにする + self.setDisableTranslation() + self.run( + 400, + self.run_mapping["enable_translation"], + { + "message":"Translation disabled due to VRAM overflow", + "data": False + }, + ) + return + else: + # その他のエラーは通常通り処理 + raise if config.CONVERT_MESSAGE_TO_ROMAJI is True or config.CONVERT_MESSAGE_TO_HIRAGANA is True: if config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] == "Japanese": @@ -417,26 +471,62 @@ class Controller: if config.ENABLE_TRANSLATION is False: pass else: - if config.USE_EXCLUDE_WORDS is True: - replacement_message, replacement_dict = self.replaceExclamationsWithRandom(message) - translation, success = model.getInputTranslate(replacement_message) + try: + if config.USE_EXCLUDE_WORDS is True: + replacement_message, replacement_dict = self.replaceExclamationsWithRandom(message) + translation, success = model.getInputTranslate(replacement_message) - message = self.removeExclamations(message) - for i in range(len(translation)): - translation[i] = self.restoreText(translation[i], replacement_dict) - else: - translation, success = model.getInputTranslate(message) + message = self.removeExclamations(message) + for i in range(len(translation)): + translation[i] = self.restoreText(translation[i], replacement_dict) + else: + translation, success = model.getInputTranslate(message) - if all(success) is not True: - self.changeToCTranslate2Process() - self.run( - 400, - self.run_mapping["error_translation_engine"], - { - "message":"Translation engine limit error", - "data": None - }, - ) + if all(success) is not True: + self.changeToCTranslate2Process() + self.run( + 400, + self.run_mapping["error_translation_engine"], + { + "message":"Translation engine limit error", + "data": None + }, + ) + except Exception as e: + # VRAM不足エラーの検出 + is_vram_error, error_message = model.detectVRAMError(e) + if is_vram_error: + self.run( + 400, + self.run_mapping["error_translation_chat_vram_overflow"], + { + "message":"VRAM out of memory during translation of chat", + "data": error_message + }, + ) + # 翻訳機能をOFFにする + self.setDisableTranslation() + self.run( + 400, + self.run_mapping["enable_translation"], + { + "message":"Translation disabled due to VRAM overflow", + "data": False + }, + ) + # エラー時は翻訳なしで返す + return {"status":200, + "result": + { + "id":id, + "message":message, + "translation":[], + "transliteration":[], + }, + } + else: + # その他のエラーは通常通り処理 + raise if config.CONVERT_MESSAGE_TO_ROMAJI is True or config.CONVERT_MESSAGE_TO_HIRAGANA is True: if config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] == "Japanese": @@ -514,8 +604,21 @@ class Controller: @staticmethod def setSelectedTranslationComputeDevice(device:str, *args, **kwargs) -> dict: printLog("setSelectedTranslationComputeDevice", device) + pre_device = config.SELECTED_TRANSLATION_COMPUTE_DEVICE config.SELECTED_TRANSLATION_COMPUTE_DEVICE = device - model.changeTranslatorCTranslate2Model() + try: + model.changeTranslatorCTranslate2Model() + except Exception as e: + # VRAM不足エラーの検出（デバイス切り替え時） + is_vram_error, error_message = model.detectVRAMError(e) + if is_vram_error: + # 前のデバイス設定に戻す + printLog("VRAM error detected, reverting device setting") + config.SELECTED_TRANSLATION_COMPUTE_DEVICE = pre_device + model.changeTranslatorCTranslate2Model() + else: + # その他のエラーは通常通り処理 + errorLogging() return {"status":200,"result":config.SELECTED_TRANSLATION_COMPUTE_DEVICE} @staticmethod @@ -1632,8 +1735,35 @@ class Controller: while self.device_access_status is False: sleep(1) self.device_access_status = False - model.startMicTranscript(self.micMessage) - self.device_access_status = True + try: + model.startMicTranscript(self.micMessage) + except Exception as e: + # VRAM不足エラーの検出 + is_vram_error, error_message = model.detectVRAMError(e) + if is_vram_error: + self.run( + 400, + self.run_mapping["error_transcription_mic_vram_overflow"], + { + "message":"VRAM out of memory during mic transcription", + "data": error_message + }, + ) + # ここでマイクの音声認識を停止 + self.stopTranscriptionSendMessage() + self.run( + 400, + self.run_mapping["enable_transcription_send"], + { + "message":"Transcription send disabled due to VRAM overflow", + "data": False + }, + ) + else: + # その他のエラーは通常通り処理 + errorLogging() + finally: + self.device_access_status = True @staticmethod def stopTranscriptionSendMessage() -> None: @@ -1654,8 +1784,35 @@ class Controller: while self.device_access_status is False: sleep(1) self.device_access_status = False - model.startSpeakerTranscript(self.speakerMessage) - self.device_access_status = True + try: + model.startSpeakerTranscript(self.speakerMessage) + except Exception as e: + # VRAM不足エラーの検出 + is_vram_error, error_message = model.detectVRAMError(e) + if is_vram_error: + self.run( + 400, + self.run_mapping["error_transcription_speaker_vram_overflow"], + { + "message":"VRAM out of memory during speaker transcription", + "data": error_message + }, + ) + # ここでスピーカーの音声認識を停止 + self.stopTranscriptionReceiveMessage() + self.run( + 400, + self.run_mapping["enable_transcription_receive"], + { + "message":"Transcription receive disabled due to VRAM overflow", + "data": False + }, + ) + else: + # その他のエラーは通常通り処理 + errorLogging() + finally: + self.device_access_status = True @staticmethod def stopTranscriptionReceiveMessage() -> None: diff --git a/src-python/mainloop.py b/src-python/mainloop.py index 00bc8cb5..28207de5 100644 --- a/src-python/mainloop.py +++ b/src-python/mainloop.py @@ -11,6 +11,10 @@ from utils import printLog, printResponse, errorLogging, encodeBase64 # noqa: E4 logging.getLogger("huggingface_hub").setLevel(logging.ERROR) run_mapping = { + "enable_translation":"/run/enable_translation", + "enable_transcription_send":"/run/enable_transcription_send", + "enable_transcription_receive":"/run/enable_transcription_receive", + "connected_network":"/run/connected_network", "enable_ai_models":"/run/enable_ai_models", @@ -22,6 +26,13 @@ run_mapping = { "error_device":"/run/error_device", "error_translation_engine":"/run/error_translation_engine", + + "error_translation_chat_vram_overflow":"/run/error_translation_chat_vram_overflow", + "error_translation_mic_vram_overflow":"/run/error_translation_mic_vram_overflow", + "error_translation_speaker_vram_overflow":"/run/error_translation_speaker_vram_overflow", + "error_transcription_mic_vram_overflow":"/run/error_transcription_mic_vram_overflow", + "error_transcription_speaker_vram_overflow":"/run/error_transcription_speaker_vram_overflow", + "word_filter":"/run/word_filter", "download_progress_ctranslate2_weight":"/run/download_progress_ctranslate2_weight", diff --git a/src-python/model.py b/src-python/model.py index 3b4e51b6..b40936b5 100644 --- a/src-python/model.py +++ b/src-python/model.py @@ -509,6 +509,15 @@ class Model: if isinstance(self.mic_audio_recorder, SelectedMicEnergyAndAudioRecorder): self.mic_audio_recorder.pause() + # VRAM 不足エラーを検出するメソッドを追加 + def detectVRAMError(self, error): + error_str = str(error) + if isinstance(error, ValueError) and len(error.args) > 0 and error.args[0] == "VRAM_OUT_OF_MEMORY": + return True, error.args[1] if len(error.args) > 1 else "VRAM out of memory" + if "CUDA out of memory" in error_str or "CUBLAS_STATUS_ALLOC_FAILED" in error_str: + return True, error_str + return False, None + def changeMicTranscriptStatus(self): if config.VRC_MIC_MUTE_SYNC is True: match self.mic_mute_status: diff --git a/src-python/models/transcription/transcription_whisper.py b/src-python/models/transcription/transcription_whisper.py index 69499260..04f89626 100644 --- a/src-python/models/transcription/transcription_whisper.py +++ b/src-python/models/transcription/transcription_whisper.py @@ -77,15 +77,24 @@ def downloadWhisperWeight(root, weight_type, callback=None, end_callback=None): def getWhisperModel(root, weight_type, device="cpu", device_index=0): path = os_path.join(root, "weights", "whisper", weight_type) compute_type = getBestComputeType(device, device_index) - return WhisperModel( - path, - device=device, - device_index=device_index, - compute_type=compute_type, - cpu_threads=4, - num_workers=1, - local_files_only=True, - ) + try: + model = WhisperModel( + path, + device=device, + device_index=device_index, + compute_type=compute_type, + cpu_threads=4, + num_workers=1, + local_files_only=True, + ) + return model + except RuntimeError as e: + # VRAM不足エラーの検出 + error_message = str(e) + if "CUDA out of memory" in error_message or "CUBLAS_STATUS_ALLOC_FAILED" in error_message: + raise ValueError("VRAM_OUT_OF_MEMORY", error_message) + # その他のエラーは通常通り再送出 + raise if __name__ == "__main__": def callback(value): diff --git a/src-ui/logics/_useBackendErrorHandling.js b/src-ui/logics/_useBackendErrorHandling.js index c1cdd15f..e44b293c 100644 --- a/src-ui/logics/_useBackendErrorHandling.js +++ b/src-ui/logics/_useBackendErrorHandling.js @@ -4,6 +4,10 @@ import { useNotificationStatus, } from "@logics_common"; +import { + useMainFunction, +} from "@logics_main"; + import { useMicRecordTimeout, useMicPhraseTimeout, @@ -26,6 +30,8 @@ export const _useBackendErrorHandling = () => { const { t } = useTranslation(); const { showNotification_Error } = useNotificationStatus(); + const { updateTranslationStatus, updateTranscriptionSendStatus, updateTranscriptionReceiveStatus } = useMainFunction(); + const { updateMicRecordTimeout } = useMicRecordTimeout(); const { updateMicPhraseTimeout } = useMicPhraseTimeout(); const { updateMicMaxWords } = useMicMaxWords(); @@ -74,6 +80,43 @@ export const _useBackendErrorHandling = () => { if (message === "Translation engine limit error") showNotification_Error(t("common_error.translation_limit")); return; + case "/run/enable_translation": + if (message === "Translation disabled due to VRAM overflow") { + updateTranslationStatus(data); + showNotification_Error("Translation disabled due to VRAM overflow"); + } + return; + + case "/run/enable_transcription_send": + if (message === "Transcription send disabled due to VRAM overflow") { + updateTranscriptionSendStatus(data); + showNotification_Error("Transcription send disabled due to VRAM overflow"); + } + return; + + case "/run/enable_transcription_send": + if (message === "Transcription receive disabled due to VRAM overflow") { + updateTranscriptionReceiveStatus(data); + showNotification_Error("Transcription receive disabled due to VRAM overflow"); + } + return; + + case "/run/error_translation_chat_vram_overflow": + if (message === "VRAM out of memory during translation of chat") showNotification_Error("VRAM out of memory during translation of chat"); + return; + case "/run/error_translation_mic_vram_overflow": + if (message === "VRAM out of memory during translation of mic") showNotification_Error("VRAM out of memory during translation of mic"); + return; + case "/run/error_translation_speaker_vram_overflow": + if (message === "VRAM out of memory during translation of speaker") showNotification_Error("VRAM out of memory during translation of speaker"); + return; + case "/run/error_transcription_mic_vram_overflow": + if (message === "VRAM out of memory during mic transcription") showNotification_Error("VRAM out of memory during mic transcription"); + return; + case "/run/error_transcription_speaker_vram_overflow": + if (message === "VRAM out of memory during speaker transcription") showNotification_Error("VRAM out of memory during speaker transcription"); + return; + case "/set/data/deepl_auth_key": if (message === "DeepL auth key length is not correct") { updateDeepLAuthKey(data);