diff --git a/src-python/model.py b/src-python/model.py index 060bcc38..fa19b06c 100644 --- a/src-python/model.py +++ b/src-python/model.py @@ -207,9 +207,10 @@ class Model: sleep(0.1) return translation, success_flag - def getInputTranslate(self, message): + def getInputTranslate(self, message, source_language=None): translator_name=config.SELECTED_TRANSLATION_ENGINES[config.SELECTED_TAB_NO] - source_language=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] + if source_language is None: + source_language=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] target_languages=config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO] translations = [] @@ -231,9 +232,10 @@ class Model: return translations, success_flags - def getOutputTranslate(self, message): + def getOutputTranslate(self, message, source_language=None): translator_name=config.SELECTED_TRANSLATION_ENGINES[config.SELECTED_TAB_NO] - source_language=config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] + if source_language is None: + source_language=config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] target_language=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] target_country=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["country"] @@ -428,16 +430,19 @@ class Model: ) def sendMicTranscript(): try: + selected_your_languages = config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO] + languages = [data["language"] for data in selected_your_languages.values() if data["enable"] is True] + countries = [data["country"] for data in selected_your_languages.values() if data["enable"] is True] res = self.mic_transcriber.transcribeAudioQueue( self.mic_audio_queue, - config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"], - config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["country"], + languages, + countries, config.MIC_AVG_LOGPROB, config.MIC_NO_SPEECH_PROB ) if res: - message = self.mic_transcriber.getTranscript() - fnc(message) + result = self.mic_transcriber.getTranscript() + fnc(result) except Exception: errorLogging() @@ -592,16 +597,19 @@ class Model: ) def sendSpeakerTranscript(): try: + selected_target_languages = config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO] + languages = [data["language"] for data in selected_target_languages.values() if data["enable"] is True] + countries = [data["country"] for data in selected_target_languages.values() if data["enable"] is True] res = self.speaker_transcriber.transcribeAudioQueue( speaker_audio_queue, - config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"], - config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["country"], + languages, + countries, config.SPEAKER_AVG_LOGPROB, config.SPEAKER_NO_SPEECH_PROB ) if res: - message = self.speaker_transcriber.getTranscript() - fnc(message) + result = self.speaker_transcriber.getTranscript() + fnc(result) except Exception: errorLogging() diff --git a/src-python/models/transcription/transcription_transcriber.py b/src-python/models/transcription/transcription_transcriber.py index cc231006..e2ab95b7 100644 --- a/src-python/models/transcription/transcription_transcriber.py +++ b/src-python/models/transcription/transcription_transcriber.py @@ -44,39 +44,59 @@ class AudioTranscriber: self.whisper_model = getWhisperModel(root, whisper_weight_type, device=device, device_index=device_index) self.transcription_engine = "Whisper" - def transcribeAudioQueue(self, audio_queue, language, country, avg_logprob=-0.8, no_speech_prob=0.6): + def transcribeAudioQueue(self, audio_queue, languages, countries, avg_logprob=-0.8, no_speech_prob=0.6): if audio_queue.empty(): time.sleep(0.01) return False audio, time_spoken = audio_queue.get() self.updateLastSampleAndPhraseStatus(audio, time_spoken) - text = '' + result = {"confidence": 0, "text": "", "language": None} try: audio_data = self.audio_sources["process_data_func"]() match self.transcription_engine: case "Google": - text = self.audio_recognizer.recognize_google(audio_data, language=transcription_lang[language][country][self.transcription_engine]) + confidences = [] + for language, country in zip(languages, countries): + text, confidence = self.audio_recognizer.recognize_google( + audio_data, + language=transcription_lang[language][country][self.transcription_engine], + with_confidence=True + ) + confidences.append({"confidence": confidence, "text": text, "language": language}) + result = max(confidences, key=lambda x: x["confidence"]) + case "Whisper": + confidences = [] + audio_data = np.frombuffer(audio_data.get_raw_data(convert_rate=16000, convert_width=2), np.int16).flatten().astype(np.float32) / 32768.0 if isinstance(audio_data, torch.Tensor): audio_data = audio_data.detach().numpy() - segments, _ = self.whisper_model.transcribe( - audio_data, - beam_size=5, - temperature=0.0, - log_prob_threshold=-0.8, - no_speech_threshold=0.6, - language=transcription_lang[language][country][self.transcription_engine], - word_timestamps=False, - without_timestamps=True, - task="transcribe", - vad_filter=False, - ) - for s in segments: - if s.avg_logprob < avg_logprob or s.no_speech_prob > no_speech_prob: - continue - text += s.text + + for language, country in zip(languages, countries): + text = "" + source_language = transcription_lang[language][country][self.transcription_engine] if len(languages) == 1 else None + segments, info = self.whisper_model.transcribe( + audio_data, + beam_size=5, + temperature=0.0, + log_prob_threshold=-0.8, + no_speech_threshold=0.6, + language=source_language, + word_timestamps=False, + without_timestamps=True, + task="transcribe", + vad_filter=False, + ) + for s in segments: + if s.avg_logprob < avg_logprob or s.no_speech_prob > no_speech_prob: + continue + text += s.text + confidences.append({"confidence": info.language_probability, "text": text, "language": language}) + if (len(languages) == 1) or (transcription_lang[language][country][self.transcription_engine] == info.language): + break + result = max(confidences, key=lambda x: x["confidence"]) + except UnknownValueError: pass except Exception: @@ -84,8 +104,8 @@ class AudioTranscriber: finally: pass - if text != '': - self.updateTranscript(text) + if result["text"] != "": + self.updateTranscript(result) return True def updateLastSampleAndPhraseStatus(self, data, time_spoken): @@ -123,23 +143,23 @@ class AudioTranscriber: audio = self.audio_recognizer.record(source) return audio - def updateTranscript(self, text): + def updateTranscript(self, result): source_info = self.audio_sources transcript = self.transcript_data if source_info["new_phrase"] or len(transcript) == 0: if len(transcript) > self.max_phrases: transcript.pop(-1) - transcript.insert(0, text) + transcript.insert(0, result) else: - transcript[0] = text + transcript[0] = result def getTranscript(self): if len(self.transcript_data) > 0: - text = self.transcript_data.pop(-1) + result = self.transcript_data.pop(-1) else: - text = "" - return text + result = {"confidence": 0, "text": "", "language": None} + return result def clearTranscriptData(self): self.transcript_data.clear() diff --git a/src-python/webui_controller.py b/src-python/webui_controller.py index e3634183..6c86c651 100644 --- a/src-python/webui_controller.py +++ b/src-python/webui_controller.py @@ -183,7 +183,9 @@ class Controller: self.weight_type, ) - def micMessage(self, message: Union[str, bool]) -> None: + def micMessage(self, result: dict) -> None: + message = result["text"] + language = result["language"] if isinstance(message, bool) and message is False: self.run( 400, @@ -209,7 +211,7 @@ class Controller: elif config.ENABLE_TRANSLATION is False: pass else: - translation, success = model.getInputTranslate(message) + translation, success = model.getInputTranslate(message, source_language=language) if all(success) is not True: self.changeToCTranslate2Process() self.run( @@ -256,7 +258,9 @@ class Controller: overlay_image = model.createOverlayImageLargeLog("send", message, translation[0] if len(translation) > 0 else "") model.updateOverlayLargeLog(overlay_image) - def speakerMessage(self, message) -> None: + def speakerMessage(self, result:dict) -> None: + message = result["text"] + language = result["language"] if isinstance(message, bool) and message is False: self.run( 400, @@ -274,7 +278,7 @@ class Controller: elif config.ENABLE_TRANSLATION is False: pass else: - translation, success = model.getOutputTranslate(message) + translation, success = model.getOutputTranslate(message, source_language=language) if all(success) is not True: self.changeToCTranslate2Process() self.run(