Merge branch 'gpu_error' into develop

src-python/controller.py

@@ -259,17 +259,44 @@ class Controller:
             elif config.ENABLE_TRANSLATION is False:
                 pass
             else:
-                translation, success = model.getInputTranslate(message, source_language=language)
+                try:
-                if all(success) is not True:
+                    translation, success = model.getInputTranslate(message, source_language=language)
-                    self.changeToCTranslate2Process()
+                    if all(success) is not True:
-                    self.run(
+                        self.changeToCTranslate2Process()
-                        400,
+                        self.run(
-                        self.run_mapping["error_translation_engine"],
+                            400,
-                        {
+                            self.run_mapping["error_translation_engine"],
-                            "message":"Translation engine limit error",
+                            {
-                            "data": None
+                                "message":"Translation engine limit error",
-                        },
+                                "data": None
-                    )
+                            },
+                        )
+                except Exception as e:
+                    # VRAM不足エラーの検出
+                    is_vram_error, error_message = model.detectVRAMError(e)
+                    if is_vram_error:
+                        self.run(
+                            400,
+                            self.run_mapping["error_translation_mic_vram_overflow"],
+                            {
+                                "message":"VRAM out of memory during translation of mic",
+                                "data": error_message
+                            },
+                        )
+                        # 翻訳機能をOFFにする
+                        self.setDisableTranslation()
+                        self.run(
+                            400,
+                            self.run_mapping["enable_translation"],
+                            {
+                                "message":"Translation disabled due to VRAM overflow",
+                                "data": False
+                            },
+                        )
+                        return
+                    else:
+                        # その他のエラーは通常通り処理
+                        raise
 
                 if config.CONVERT_MESSAGE_TO_ROMAJI is True or config.CONVERT_MESSAGE_TO_HIRAGANA is True:
                     if config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] == "Japanese":
@@ -346,17 +373,44 @@ class Controller:
             elif config.ENABLE_TRANSLATION is False:
                 pass
             else:
-                translation, success = model.getOutputTranslate(message, source_language=language)
+                try:
-                if all(success) is not True:
+                    translation, success = model.getOutputTranslate(message, source_language=language)
-                    self.changeToCTranslate2Process()
+                    if all(success) is not True:
-                    self.run(
+                        self.changeToCTranslate2Process()
-                        400,
+                        self.run(
-                        self.run_mapping["error_translation_engine"],
+                            400,
-                        {
+                            self.run_mapping["error_translation_engine"],
-                            "message":"Translation engine limit error",
+                            {
-                            "data": None
+                                "message":"Translation engine limit error",
-                        },
+                                "data": None
-                    )
+                            },
+                        )
+                except Exception as e:
+                    # VRAM不足エラーの検出
+                    is_vram_error, error_message = model.detectVRAMError(e)
+                    if is_vram_error:
+                        self.run(
+                            400,
+                            self.run_mapping["error_translation_speaker_vram_overflow"],
+                            {
+                                "message":"VRAM out of memory during translation of speaker",
+                                "data": error_message
+                            },
+                        )
+                        # 翻訳機能をOFFにする
+                        self.setDisableTranslation()
+                        self.run(
+                            400,
+                            self.run_mapping["enable_translation"],
+                            {
+                                "message":"Translation disabled due to VRAM overflow",
+                                "data": False
+                            },
+                        )
+                        return
+                    else:
+                        # その他のエラーは通常通り処理
+                        raise
 
                 if config.CONVERT_MESSAGE_TO_ROMAJI is True or config.CONVERT_MESSAGE_TO_HIRAGANA is True:
                     if config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] == "Japanese":
@@ -417,26 +471,62 @@ class Controller:
             if config.ENABLE_TRANSLATION is False:
                 pass
             else:
-                if config.USE_EXCLUDE_WORDS is True:
+                try:
-                    replacement_message, replacement_dict = self.replaceExclamationsWithRandom(message)
+                    if config.USE_EXCLUDE_WORDS is True:
-                    translation, success = model.getInputTranslate(replacement_message)
+                        replacement_message, replacement_dict = self.replaceExclamationsWithRandom(message)
+                        translation, success = model.getInputTranslate(replacement_message)
 
-                    message = self.removeExclamations(message)
+                        message = self.removeExclamations(message)
-                    for i in range(len(translation)):
+                        for i in range(len(translation)):
-                        translation[i] = self.restoreText(translation[i], replacement_dict)
+                            translation[i] = self.restoreText(translation[i], replacement_dict)
-                else:
+                    else:
-                    translation, success = model.getInputTranslate(message)
+                        translation, success = model.getInputTranslate(message)
 
-                if all(success) is not True:
+                    if all(success) is not True:
-                    self.changeToCTranslate2Process()
+                        self.changeToCTranslate2Process()
-                    self.run(
+                        self.run(
-                        400,
+                            400,
-                        self.run_mapping["error_translation_engine"],
+                            self.run_mapping["error_translation_engine"],
-                        {
+                            {
-                            "message":"Translation engine limit error",
+                                "message":"Translation engine limit error",
-                            "data": None
+                                "data": None
-                        },
+                            },
-                    )
+                        )
+                except Exception as e:
+                    # VRAM不足エラーの検出
+                    is_vram_error, error_message = model.detectVRAMError(e)
+                    if is_vram_error:
+                        self.run(
+                            400,
+                            self.run_mapping["error_translation_chat_vram_overflow"],
+                            {
+                                "message":"VRAM out of memory during translation of chat",
+                                "data": error_message
+                            },
+                        )
+                        # 翻訳機能をOFFにする
+                        self.setDisableTranslation()
+                        self.run(
+                            400,
+                            self.run_mapping["enable_translation"],
+                            {
+                                "message":"Translation disabled due to VRAM overflow",
+                                "data": False
+                            },
+                        )
+                        # エラー時は翻訳なしで返す
+                        return {"status":200,
+                                "result":
+                                {
+                                    "id":id,
+                                    "message":message,
+                                    "translation":[],
+                                    "transliteration":[],
+                                    },
+                                }
+                    else:
+                        # その他のエラーは通常通り処理
+                        raise
 
                 if config.CONVERT_MESSAGE_TO_ROMAJI is True or config.CONVERT_MESSAGE_TO_HIRAGANA is True:
                     if config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] == "Japanese":
@@ -514,8 +604,21 @@ class Controller:
     @staticmethod
     def setSelectedTranslationComputeDevice(device:str, *args, **kwargs) -> dict:
         printLog("setSelectedTranslationComputeDevice", device)
+        pre_device = config.SELECTED_TRANSLATION_COMPUTE_DEVICE
         config.SELECTED_TRANSLATION_COMPUTE_DEVICE = device
-        model.changeTranslatorCTranslate2Model()
+        try:
+            model.changeTranslatorCTranslate2Model()
+        except Exception as e:
+            # VRAM不足エラーの検出（デバイス切り替え時）
+            is_vram_error, error_message = model.detectVRAMError(e)
+            if is_vram_error:
+                # 前のデバイス設定に戻す
+                printLog("VRAM error detected, reverting device setting")
+                config.SELECTED_TRANSLATION_COMPUTE_DEVICE = pre_device
+                model.changeTranslatorCTranslate2Model()
+            else:
+                # その他のエラーは通常通り処理
+                errorLogging()
         return {"status":200,"result":config.SELECTED_TRANSLATION_COMPUTE_DEVICE}
 
     @staticmethod
@@ -1632,8 +1735,35 @@ class Controller:
         while self.device_access_status is False:
             sleep(1)
         self.device_access_status = False
-        model.startMicTranscript(self.micMessage)
+        try:
-        self.device_access_status = True
+            model.startMicTranscript(self.micMessage)
+        except Exception as e:
+            # VRAM不足エラーの検出
+            is_vram_error, error_message = model.detectVRAMError(e)
+            if is_vram_error:
+                self.run(
+                    400,
+                    self.run_mapping["error_transcription_mic_vram_overflow"],
+                    {
+                        "message":"VRAM out of memory during mic transcription",
+                        "data": error_message
+                    },
+                )
+                # ここでマイクの音声認識を停止
+                self.stopTranscriptionSendMessage()
+                self.run(
+                    400,
+                    self.run_mapping["enable_transcription_send"],
+                    {
+                        "message":"Transcription send disabled due to VRAM overflow",
+                        "data": False
+                    },
+                )
+            else:
+                # その他のエラーは通常通り処理
+                errorLogging()
+        finally:
+            self.device_access_status = True
 
     @staticmethod
     def stopTranscriptionSendMessage() -> None:
@@ -1654,8 +1784,35 @@ class Controller:
         while self.device_access_status is False:
             sleep(1)
         self.device_access_status = False
-        model.startSpeakerTranscript(self.speakerMessage)
+        try:
-        self.device_access_status = True
+            model.startSpeakerTranscript(self.speakerMessage)
+        except Exception as e:
+            # VRAM不足エラーの検出
+            is_vram_error, error_message = model.detectVRAMError(e)
+            if is_vram_error:
+                self.run(
+                    400,
+                    self.run_mapping["error_transcription_speaker_vram_overflow"],
+                    {
+                        "message":"VRAM out of memory during speaker transcription",
+                        "data": error_message
+                    },
+                )
+                # ここでスピーカーの音声認識を停止
+                self.stopTranscriptionReceiveMessage()
+                self.run(
+                    400,
+                    self.run_mapping["enable_transcription_receive"],
+                    {
+                        "message":"Transcription receive disabled due to VRAM overflow",
+                        "data": False
+                    },
+                )
+            else:
+                # その他のエラーは通常通り処理
+                errorLogging()
+        finally:
+            self.device_access_status = True
 
     @staticmethod
     def stopTranscriptionReceiveMessage() -> None:

src-python/mainloop.py

@@ -11,6 +11,10 @@ from utils import printLog, printResponse, errorLogging, encodeBase64 # noqa: E4
 logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
 
 run_mapping = {
+    "enable_translation":"/run/enable_translation",
+    "enable_transcription_send":"/run/enable_transcription_send",
+    "enable_transcription_receive":"/run/enable_transcription_receive",
+
     "connected_network":"/run/connected_network",
     "enable_ai_models":"/run/enable_ai_models",
 
@@ -22,6 +26,13 @@ run_mapping = {
 
     "error_device":"/run/error_device",
     "error_translation_engine":"/run/error_translation_engine",
+
+    "error_translation_chat_vram_overflow":"/run/error_translation_chat_vram_overflow",
+    "error_translation_mic_vram_overflow":"/run/error_translation_mic_vram_overflow",
+    "error_translation_speaker_vram_overflow":"/run/error_translation_speaker_vram_overflow",
+    "error_transcription_mic_vram_overflow":"/run/error_transcription_mic_vram_overflow",
+    "error_transcription_speaker_vram_overflow":"/run/error_transcription_speaker_vram_overflow",
+
     "word_filter":"/run/word_filter",
 
     "download_progress_ctranslate2_weight":"/run/download_progress_ctranslate2_weight",

src-python/model.py

@@ -509,6 +509,15 @@ class Model:
         if isinstance(self.mic_audio_recorder, SelectedMicEnergyAndAudioRecorder):
             self.mic_audio_recorder.pause()
 
+    # VRAM 不足エラーを検出するメソッドを追加
+    def detectVRAMError(self, error):
+        error_str = str(error)
+        if isinstance(error, ValueError) and len(error.args) > 0 and error.args[0] == "VRAM_OUT_OF_MEMORY":
+            return True, error.args[1] if len(error.args) > 1 else "VRAM out of memory"
+        if "CUDA out of memory" in error_str or "CUBLAS_STATUS_ALLOC_FAILED" in error_str:
+            return True, error_str
+        return False, None
+
     def changeMicTranscriptStatus(self):
         if config.VRC_MIC_MUTE_SYNC is True:
             match self.mic_mute_status:

src-python/models/transcription/transcription_whisper.py

@@ -77,15 +77,24 @@ def downloadWhisperWeight(root, weight_type, callback=None, end_callback=None):
 def getWhisperModel(root, weight_type, device="cpu", device_index=0):
     path = os_path.join(root, "weights", "whisper", weight_type)
     compute_type = getBestComputeType(device, device_index)
-    return WhisperModel(
+    try:
-        path,
+        model = WhisperModel(
-        device=device,
+            path,
-        device_index=device_index,
+            device=device,
-        compute_type=compute_type,
+            device_index=device_index,
-        cpu_threads=4,
+            compute_type=compute_type,
-        num_workers=1,
+            cpu_threads=4,
-        local_files_only=True,
+            num_workers=1,
-    )
+            local_files_only=True,
+        )
+        return model
+    except RuntimeError as e:
+        # VRAM不足エラーの検出
+        error_message = str(e)
+        if "CUDA out of memory" in error_message or "CUBLAS_STATUS_ALLOC_FAILED" in error_message:
+            raise ValueError("VRAM_OUT_OF_MEMORY", error_message)
+        # その他のエラーは通常通り再送出
+        raise
 
 if __name__ == "__main__":
     def callback(value):

src-ui/logics/_useBackendErrorHandling.js

@@ -4,6 +4,10 @@ import {
     useNotificationStatus,
 } from "@logics_common";
 
+import {
+    useMainFunction,
+} from "@logics_main";
+
 import {
     useMicRecordTimeout,
     useMicPhraseTimeout,
@@ -26,6 +30,8 @@ export const _useBackendErrorHandling = () => {
     const { t } = useTranslation();
     const { showNotification_Error } = useNotificationStatus();
 
+    const { updateTranslationStatus, updateTranscriptionSendStatus, updateTranscriptionReceiveStatus } = useMainFunction();
+
     const { updateMicRecordTimeout } = useMicRecordTimeout();
     const { updateMicPhraseTimeout } = useMicPhraseTimeout();
     const { updateMicMaxWords } = useMicMaxWords();
@@ -74,6 +80,43 @@ export const _useBackendErrorHandling = () => {
                 if (message === "Translation engine limit error") showNotification_Error(t("common_error.translation_limit"));
                 return;
 
+            case "/run/enable_translation":
+                if (message === "Translation disabled due to VRAM overflow") {
+                    updateTranslationStatus(data);
+                    showNotification_Error("Translation disabled due to VRAM overflow");
+                }
+                return;
+
+            case "/run/enable_transcription_send":
+                if (message === "Transcription send disabled due to VRAM overflow") {
+                    updateTranscriptionSendStatus(data);
+                    showNotification_Error("Transcription send disabled due to VRAM overflow");
+                }
+                return;
+
+            case "/run/enable_transcription_send":
+                if (message === "Transcription receive disabled due to VRAM overflow") {
+                    updateTranscriptionReceiveStatus(data);
+                    showNotification_Error("Transcription receive disabled due to VRAM overflow");
+                }
+                return;
+
+            case "/run/error_translation_chat_vram_overflow":
+                if (message === "VRAM out of memory during translation of chat") showNotification_Error("VRAM out of memory during translation of chat");
+                return;
+            case "/run/error_translation_mic_vram_overflow":
+                if (message === "VRAM out of memory during translation of mic") showNotification_Error("VRAM out of memory during translation of mic");
+                return;
+            case "/run/error_translation_speaker_vram_overflow":
+                if (message === "VRAM out of memory during translation of speaker") showNotification_Error("VRAM out of memory during translation of speaker");
+                return;
+            case "/run/error_transcription_mic_vram_overflow":
+                if (message === "VRAM out of memory during mic transcription") showNotification_Error("VRAM out of memory during mic transcription");
+                return;
+            case "/run/error_transcription_speaker_vram_overflow":
+                if (message === "VRAM out of memory during speaker transcription") showNotification_Error("VRAM out of memory during speaker transcription");
+                return;
+
             case "/set/data/deepl_auth_key":
                 if (message === "DeepL auth key length is not correct") {
                     updateDeepLAuthKey(data);