every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'bugfix_whisper' into develop

Browse Source
This commit is contained in:
misyaguziya
2024-02-14 00:22:13 +09:00
parent badffc234e bc0d2f246b
commit d3ea3c2c54
3 changed files with 35 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
controller.py
View File

@@ -806,6 +806,7 @@ def callbackSetUserWhisperFeature(value):
config.SELECTED_TRANSCRIPTION_ENGINE = "Google" config.SELECTED_TRANSCRIPTION_ENGINE = "Google"
else: else:
view.closeWhisperWeightTypeWidget() view.closeWhisperWeightTypeWidget()
config.SELECTED_TRANSCRIPTION_ENGINE = "Google"
view.showRestartButtonIfRequired() view.showRestartButtonIfRequired()
def callbackSetWhisperWeightType(value): def callbackSetWhisperWeightType(value):

31
model.py
View File

@@ -1,3 +1,4 @@
import gc
import tempfile import tempfile
from zipfile import ZipFile from zipfile import ZipFile
from subprocess import Popen from subprocess import Popen
@@ -336,22 +337,29 @@ class Model:
) )
# self.mic_audio_recorder.recordIntoQueue(mic_audio_queue, mic_energy_queue) # self.mic_audio_recorder.recordIntoQueue(mic_audio_queue, mic_energy_queue)
self.mic_audio_recorder.recordIntoQueue(mic_audio_queue, None) self.mic_audio_recorder.recordIntoQueue(mic_audio_queue, None)
mic_transcriber = AudioTranscriber( self.mic_transcriber = AudioTranscriber(
speaker=False, speaker=False,
source=self.mic_audio_recorder.source, source=self.mic_audio_recorder.source,
phrase_timeout=phase_timeout, phrase_timeout=phase_timeout,
max_phrases=config.INPUT_MIC_MAX_PHRASES, max_phrases=config.INPUT_MIC_MAX_PHRASES,
transcription_engine=config.SELECTED_TRANSCRIPTION_ENGINE,
root=config.PATH_LOCAL, root=config.PATH_LOCAL,
whisper_weight_type=config.WHISPER_WEIGHT_TYPE, whisper_weight_type=config.WHISPER_WEIGHT_TYPE,
) )
def sendMicTranscript(): def sendMicTranscript():
mic_transcriber.transcribeAudioQueue(mic_audio_queue, config.SOURCE_LANGUAGE, config.SOURCE_COUNTRY, config.SELECTED_TRANSCRIPTION_ENGINE) self.mic_transcriber.transcribeAudioQueue(mic_audio_queue, config.SOURCE_LANGUAGE, config.SOURCE_COUNTRY)
message = mic_transcriber.getTranscript() message = self.mic_transcriber.getTranscript()
try: try:
fnc(message) fnc(message)
except Exception: except Exception:
pass pass
def endMicTranscript():
mic_audio_queue.queue.clear()
# mic_energy_queue.queue.clear()
del self.mic_transcriber
gc.collect()
# def sendMicEnergy(): # def sendMicEnergy():
# if mic_energy_queue.empty() is False: # if mic_energy_queue.empty() is False:
# energy = mic_energy_queue.get() # energy = mic_energy_queue.get()
@@ -362,7 +370,7 @@ class Model:
# pass # pass
# sleep(0.01) # sleep(0.01)
self.mic_print_transcript = threadFnc(sendMicTranscript) self.mic_print_transcript = threadFnc(sendMicTranscript, end_fnc=endMicTranscript)
self.mic_print_transcript.daemon = True self.mic_print_transcript.daemon = True
self.mic_print_transcript.start() self.mic_print_transcript.start()
@@ -438,22 +446,29 @@ class Model:
) )
# self.speaker_audio_recorder.recordIntoQueue(speaker_audio_queue, speaker_energy_queue) # self.speaker_audio_recorder.recordIntoQueue(speaker_audio_queue, speaker_energy_queue)
self.speaker_audio_recorder.recordIntoQueue(speaker_audio_queue ,None) self.speaker_audio_recorder.recordIntoQueue(speaker_audio_queue ,None)
speaker_transcriber = AudioTranscriber( self.speaker_transcriber = AudioTranscriber(
speaker=True, speaker=True,
source=self.speaker_audio_recorder.source, source=self.speaker_audio_recorder.source,
phrase_timeout=phase_timeout, phrase_timeout=phase_timeout,
max_phrases=config.INPUT_SPEAKER_MAX_PHRASES, max_phrases=config.INPUT_SPEAKER_MAX_PHRASES,
transcription_engine=config.SELECTED_TRANSCRIPTION_ENGINE,
root=config.PATH_LOCAL, root=config.PATH_LOCAL,
whisper_weight_type=config.WHISPER_WEIGHT_TYPE, whisper_weight_type=config.WHISPER_WEIGHT_TYPE,
) )
def sendSpeakerTranscript(): def sendSpeakerTranscript():
speaker_transcriber.transcribeAudioQueue(speaker_audio_queue, config.TARGET_LANGUAGE, config.TARGET_COUNTRY, config.SELECTED_TRANSCRIPTION_ENGINE) self.speaker_transcriber.transcribeAudioQueue(speaker_audio_queue, config.TARGET_LANGUAGE, config.TARGET_COUNTRY)
message = speaker_transcriber.getTranscript() message = self.speaker_transcriber.getTranscript()
try: try:
fnc(message) fnc(message)
except Exception: except Exception:
pass pass
def endSpeakerTranscript():
speaker_audio_queue.queue.clear()
# speaker_energy_queue.queue.clear()
del self.speaker_transcriber
gc.collect()
# def sendSpeakerEnergy(): # def sendSpeakerEnergy():
# if speaker_energy_queue.empty() is False: # if speaker_energy_queue.empty() is False:
# energy = speaker_energy_queue.get() # energy = speaker_energy_queue.get()
@@ -464,7 +479,7 @@ class Model:
# pass # pass
# sleep(0.01) # sleep(0.01)
self.speaker_print_transcript = threadFnc(sendSpeakerTranscript) self.speaker_print_transcript = threadFnc(sendSpeakerTranscript, end_fnc=endSpeakerTranscript)
self.speaker_print_transcript.daemon = True self.speaker_print_transcript.daemon = True
self.speaker_print_transcript.start() self.speaker_print_transcript.start()

25
models/transcription/transcription_transcriber.py
View File

@@ -14,13 +14,15 @@ PHRASE_TIMEOUT = 3
MAX_PHRASES = 10 MAX_PHRASES = 10
class AudioTranscriber: class AudioTranscriber:
def __init__(self, speaker, source, phrase_timeout, max_phrases, root=None, whisper_weight_type=None, ): def __init__(self, speaker, source, phrase_timeout, max_phrases, transcription_engine, root=None, whisper_weight_type=None):
self.speaker = speaker self.speaker = speaker
self.phrase_timeout = phrase_timeout self.phrase_timeout = phrase_timeout
self.max_phrases = max_phrases self.max_phrases = max_phrases
self.transcript_data = [] self.transcript_data = []
self.transcript_changed_event = Event() self.transcript_changed_event = Event()
self.audio_recognizer = Recognizer() self.audio_recognizer = Recognizer()
self.transcription_engine = "Google"
self.whisper_model = None
self.audio_sources = { self.audio_sources = {
"sample_rate": source.SAMPLE_RATE, "sample_rate": source.SAMPLE_RATE,
"sample_width": source.SAMPLE_WIDTH, "sample_width": source.SAMPLE_WIDTH,
@@ -30,26 +32,21 @@ class AudioTranscriber:
"new_phrase": True, "new_phrase": True,
"process_data_func": self.processSpeakerData if speaker else self.processSpeakerData "process_data_func": self.processSpeakerData if speaker else self.processSpeakerData
} }
if whisper_weight_type is not None and root is not None and checkWhisperWeight(root, whisper_weight_type) is True:
self.whisper_model = getWhisperModel(root, whisper_weight_type)
else:
self.whisper_model = None
def transcribeAudioQueue(self, audio_queue, language, country, transcription_engine): if transcription_engine == "Whisper" and checkWhisperWeight(root, whisper_weight_type) is True:
self.whisper_model = getWhisperModel(root, whisper_weight_type)
self.transcription_engine = "Whisper"
def transcribeAudioQueue(self, audio_queue, language, country):
audio, time_spoken = audio_queue.get() audio, time_spoken = audio_queue.get()
self.updateLastSampleAndPhraseStatus(audio, time_spoken) self.updateLastSampleAndPhraseStatus(audio, time_spoken)
text = '' text = ''
try: try:
# Whisperが使用できない場合はGoogle Speech-to-Textを使用する
if transcription_engine == "Whisper":
if self.whisper_model is None:
transcription_engine = "Google"
audio_data = self.audio_sources["process_data_func"]() audio_data = self.audio_sources["process_data_func"]()
match transcription_engine: match self.transcription_engine:
case "Google": case "Google":
text = self.audio_recognizer.recognize_google(audio_data, language=transcription_lang[language][country][transcription_engine]) text = self.audio_recognizer.recognize_google(audio_data, language=transcription_lang[language][country][self.transcription_engine])
case "Whisper": case "Whisper":
audio_data = np.frombuffer(audio_data.get_raw_data(convert_rate=16000, convert_width=2), np.int16).flatten().astype(np.float32) / 32768.0 audio_data = np.frombuffer(audio_data.get_raw_data(convert_rate=16000, convert_width=2), np.int16).flatten().astype(np.float32) / 32768.0
if isinstance(audio_data, torch.Tensor): if isinstance(audio_data, torch.Tensor):
@@ -60,7 +57,7 @@ class AudioTranscriber:
temperature=0.0, temperature=0.0,
log_prob_threshold=-0.8, log_prob_threshold=-0.8,
no_speech_threshold=0.6, no_speech_threshold=0.6,
language=transcription_lang[language][country][transcription_engine], language=transcription_lang[language][country][self.transcription_engine],
word_timestamps=False, word_timestamps=False,
without_timestamps=True, without_timestamps=True,
task="transcribe", task="transcribe",