From 5a0c6392e02a03269f72102d27a6c926b1b75578 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Tue, 13 Feb 2024 22:23:13 +0900 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=90=9B[bugfix]=20Controller=20:=20whi?= =?UTF-8?q?sper=E4=B8=8D=E4=BD=BF=E7=94=A8=E6=99=82=E3=81=AE=E3=83=95?= =?UTF-8?q?=E3=83=A9=E3=82=B0=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- controller.py | 1 + 1 file changed, 1 insertion(+) diff --git a/controller.py b/controller.py index e977a5ef..a40282df 100644 --- a/controller.py +++ b/controller.py @@ -806,6 +806,7 @@ def callbackSetUserWhisperFeature(value): config.SELECTED_TRANSCRIPTION_ENGINE = "Google" else: view.closeWhisperWeightTypeWidget() + config.SELECTED_TRANSCRIPTION_ENGINE = "Google" view.showRestartButtonIfRequired() def callbackSetWhisperWeightType(value): From 07b3c92f1b74dc1805af3f7cc8a64212a41b0346 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Tue, 13 Feb 2024 23:20:14 +0900 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=9A=A7[WIP/TEST]=20Model=20:=20?= =?UTF-8?q?=E6=96=87=E5=AD=97=E8=B5=B7=E3=81=93=E3=81=97=E8=B5=B7=E5=8B=95?= =?UTF-8?q?=E6=99=82=E3=81=AB=E3=82=A8=E3=83=B3=E3=82=B8=E3=83=B3=E3=82=92?= =?UTF-8?q?=E9=81=B8=E6=8A=9E=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- model.py | 2 ++ models/transcription/transcription_transcriber.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/model.py b/model.py index 6bc62b0c..e68c01bc 100644 --- a/model.py +++ b/model.py @@ -341,6 +341,7 @@ class Model: source=self.mic_audio_recorder.source, phrase_timeout=phase_timeout, max_phrases=config.INPUT_MIC_MAX_PHRASES, + transcription_engine=config.SELECTED_TRANSCRIPTION_ENGINE, root=config.PATH_LOCAL, whisper_weight_type=config.WHISPER_WEIGHT_TYPE, ) @@ -443,6 +444,7 @@ class Model: source=self.speaker_audio_recorder.source, phrase_timeout=phase_timeout, max_phrases=config.INPUT_SPEAKER_MAX_PHRASES, + transcription_engine=config.SELECTED_TRANSCRIPTION_ENGINE, root=config.PATH_LOCAL, whisper_weight_type=config.WHISPER_WEIGHT_TYPE, ) diff --git a/models/transcription/transcription_transcriber.py b/models/transcription/transcription_transcriber.py index 08cc6a1a..35f79c43 100644 --- a/models/transcription/transcription_transcriber.py +++ b/models/transcription/transcription_transcriber.py @@ -14,13 +14,15 @@ PHRASE_TIMEOUT = 3 MAX_PHRASES = 10 class AudioTranscriber: - def __init__(self, speaker, source, phrase_timeout, max_phrases, root=None, whisper_weight_type=None, ): + def __init__(self, speaker, source, phrase_timeout, max_phrases, transcription_engine, root=None, whisper_weight_type=None): self.speaker = speaker self.phrase_timeout = phrase_timeout self.max_phrases = max_phrases self.transcript_data = [] self.transcript_changed_event = Event() self.audio_recognizer = Recognizer() + self.transcription_engine = "Google" + self.whisper_model = None self.audio_sources = { "sample_rate": source.SAMPLE_RATE, "sample_width": source.SAMPLE_WIDTH, @@ -30,10 +32,10 @@ class AudioTranscriber: "new_phrase": True, "process_data_func": self.processSpeakerData if speaker else self.processSpeakerData } - if whisper_weight_type is not None and root is not None and checkWhisperWeight(root, whisper_weight_type) is True: + + if transcription_engine == "Whisper" and checkWhisperWeight(root, whisper_weight_type) is True: self.whisper_model = getWhisperModel(root, whisper_weight_type) - else: - self.whisper_model = None + self.transcription_engine = "Whisper" def transcribeAudioQueue(self, audio_queue, language, country, transcription_engine): audio, time_spoken = audio_queue.get() From 7cdd9d19d7561966b0dd7f2f46c598880a7cf7fc Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Tue, 13 Feb 2024 23:25:31 +0900 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=9A=A7[WIP/TEST]=20Model=20:=20?= =?UTF-8?q?=E6=96=87=E5=AD=97=E8=B5=B7=E3=81=93=E3=81=97=E8=B5=B7=E5=8B=95?= =?UTF-8?q?=E6=99=82=E3=81=AB=E3=82=A8=E3=83=B3=E3=82=B8=E3=83=B3=E3=82=92?= =?UTF-8?q?=E9=81=B8=E6=8A=9E=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4#2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- model.py | 4 ++-- models/transcription/transcription_transcriber.py | 13 ++++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/model.py b/model.py index e68c01bc..62d727cd 100644 --- a/model.py +++ b/model.py @@ -346,7 +346,7 @@ class Model: whisper_weight_type=config.WHISPER_WEIGHT_TYPE, ) def sendMicTranscript(): - mic_transcriber.transcribeAudioQueue(mic_audio_queue, config.SOURCE_LANGUAGE, config.SOURCE_COUNTRY, config.SELECTED_TRANSCRIPTION_ENGINE) + mic_transcriber.transcribeAudioQueue(mic_audio_queue, config.SOURCE_LANGUAGE, config.SOURCE_COUNTRY) message = mic_transcriber.getTranscript() try: fnc(message) @@ -449,7 +449,7 @@ class Model: whisper_weight_type=config.WHISPER_WEIGHT_TYPE, ) def sendSpeakerTranscript(): - speaker_transcriber.transcribeAudioQueue(speaker_audio_queue, config.TARGET_LANGUAGE, config.TARGET_COUNTRY, config.SELECTED_TRANSCRIPTION_ENGINE) + speaker_transcriber.transcribeAudioQueue(speaker_audio_queue, config.TARGET_LANGUAGE, config.TARGET_COUNTRY) message = speaker_transcriber.getTranscript() try: fnc(message) diff --git a/models/transcription/transcription_transcriber.py b/models/transcription/transcription_transcriber.py index 35f79c43..c5a6cbff 100644 --- a/models/transcription/transcription_transcriber.py +++ b/models/transcription/transcription_transcriber.py @@ -37,21 +37,16 @@ class AudioTranscriber: self.whisper_model = getWhisperModel(root, whisper_weight_type) self.transcription_engine = "Whisper" - def transcribeAudioQueue(self, audio_queue, language, country, transcription_engine): + def transcribeAudioQueue(self, audio_queue, language, country): audio, time_spoken = audio_queue.get() self.updateLastSampleAndPhraseStatus(audio, time_spoken) text = '' try: - # Whisperが使用できない場合はGoogle Speech-to-Textを使用する - if transcription_engine == "Whisper": - if self.whisper_model is None: - transcription_engine = "Google" - audio_data = self.audio_sources["process_data_func"]() - match transcription_engine: + match self.transcription_engine: case "Google": - text = self.audio_recognizer.recognize_google(audio_data, language=transcription_lang[language][country][transcription_engine]) + text = self.audio_recognizer.recognize_google(audio_data, language=transcription_lang[language][country][self.transcription_engine]) case "Whisper": audio_data = np.frombuffer(audio_data.get_raw_data(convert_rate=16000, convert_width=2), np.int16).flatten().astype(np.float32) / 32768.0 if isinstance(audio_data, torch.Tensor): @@ -62,7 +57,7 @@ class AudioTranscriber: temperature=0.0, log_prob_threshold=-0.8, no_speech_threshold=0.6, - language=transcription_lang[language][country][transcription_engine], + language=transcription_lang[language][country][self.transcription_engine], word_timestamps=False, without_timestamps=True, task="transcribe", From bc0d2f246b31173b09a9a10e11511efaa31f7643 Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Wed, 14 Feb 2024 00:00:14 +0900 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=9A=A7[WIP/TEST]=20Model=20:=20?= =?UTF-8?q?=E6=96=87=E5=AD=97=E8=B5=B7=E3=81=93=E3=81=97=E6=A9=9F=E8=83=BD?= =?UTF-8?q?=E3=81=AE=E7=B5=82=E4=BA=86=E6=99=82=E3=81=AB=E3=82=AD=E3=83=A3?= =?UTF-8?q?=E3=83=83=E3=82=B7=E3=83=A5=E3=82=AF=E3=83=AA=E3=82=A2=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- model.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/model.py b/model.py index 62d727cd..f5913d17 100644 --- a/model.py +++ b/model.py @@ -1,3 +1,4 @@ +import gc import tempfile from zipfile import ZipFile from subprocess import Popen @@ -336,7 +337,7 @@ class Model: ) # self.mic_audio_recorder.recordIntoQueue(mic_audio_queue, mic_energy_queue) self.mic_audio_recorder.recordIntoQueue(mic_audio_queue, None) - mic_transcriber = AudioTranscriber( + self.mic_transcriber = AudioTranscriber( speaker=False, source=self.mic_audio_recorder.source, phrase_timeout=phase_timeout, @@ -346,13 +347,19 @@ class Model: whisper_weight_type=config.WHISPER_WEIGHT_TYPE, ) def sendMicTranscript(): - mic_transcriber.transcribeAudioQueue(mic_audio_queue, config.SOURCE_LANGUAGE, config.SOURCE_COUNTRY) - message = mic_transcriber.getTranscript() + self.mic_transcriber.transcribeAudioQueue(mic_audio_queue, config.SOURCE_LANGUAGE, config.SOURCE_COUNTRY) + message = self.mic_transcriber.getTranscript() try: fnc(message) except Exception: pass + def endMicTranscript(): + mic_audio_queue.queue.clear() + # mic_energy_queue.queue.clear() + del self.mic_transcriber + gc.collect() + # def sendMicEnergy(): # if mic_energy_queue.empty() is False: # energy = mic_energy_queue.get() @@ -363,7 +370,7 @@ class Model: # pass # sleep(0.01) - self.mic_print_transcript = threadFnc(sendMicTranscript) + self.mic_print_transcript = threadFnc(sendMicTranscript, end_fnc=endMicTranscript) self.mic_print_transcript.daemon = True self.mic_print_transcript.start() @@ -439,7 +446,7 @@ class Model: ) # self.speaker_audio_recorder.recordIntoQueue(speaker_audio_queue, speaker_energy_queue) self.speaker_audio_recorder.recordIntoQueue(speaker_audio_queue ,None) - speaker_transcriber = AudioTranscriber( + self.speaker_transcriber = AudioTranscriber( speaker=True, source=self.speaker_audio_recorder.source, phrase_timeout=phase_timeout, @@ -449,13 +456,19 @@ class Model: whisper_weight_type=config.WHISPER_WEIGHT_TYPE, ) def sendSpeakerTranscript(): - speaker_transcriber.transcribeAudioQueue(speaker_audio_queue, config.TARGET_LANGUAGE, config.TARGET_COUNTRY) - message = speaker_transcriber.getTranscript() + self.speaker_transcriber.transcribeAudioQueue(speaker_audio_queue, config.TARGET_LANGUAGE, config.TARGET_COUNTRY) + message = self.speaker_transcriber.getTranscript() try: fnc(message) except Exception: pass + def endSpeakerTranscript(): + speaker_audio_queue.queue.clear() + # speaker_energy_queue.queue.clear() + del self.speaker_transcriber + gc.collect() + # def sendSpeakerEnergy(): # if speaker_energy_queue.empty() is False: # energy = speaker_energy_queue.get() @@ -466,7 +479,7 @@ class Model: # pass # sleep(0.01) - self.speaker_print_transcript = threadFnc(sendSpeakerTranscript) + self.speaker_print_transcript = threadFnc(sendSpeakerTranscript, end_fnc=endSpeakerTranscript) self.speaker_print_transcript.daemon = True self.speaker_print_transcript.start()