every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

👍️[Update] Model : 複数言語選択時に複数の音声に対して文字起こしを行う機能を追加

Browse Source
This commit is contained in:
misyaguziya
2024-12-19 13:44:52 +09:00
parent 01d9c27728
commit 4d2fd379a8
3 changed files with 75 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src-python/model.py
View File

@@ -207,9 +207,10 @@ class Model:
sleep(0.1) sleep(0.1)
return translation, success_flag return translation, success_flag
def getInputTranslate(self, message): def getInputTranslate(self, message, source_language=None):
translator_name=config.SELECTED_TRANSLATION_ENGINES[config.SELECTED_TAB_NO] translator_name=config.SELECTED_TRANSLATION_ENGINES[config.SELECTED_TAB_NO]
source_language=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] if source_language is None:
source_language=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"]
target_languages=config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO] target_languages=config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]
translations = [] translations = []
@@ -231,9 +232,10 @@ class Model:
return translations, success_flags return translations, success_flags
def getOutputTranslate(self, message): def getOutputTranslate(self, message, source_language=None):
translator_name=config.SELECTED_TRANSLATION_ENGINES[config.SELECTED_TAB_NO] translator_name=config.SELECTED_TRANSLATION_ENGINES[config.SELECTED_TAB_NO]
source_language=config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] if source_language is None:
source_language=config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"]
target_language=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"] target_language=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"]
target_country=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["country"] target_country=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["country"]
@@ -428,16 +430,19 @@ class Model:
) )
def sendMicTranscript(): def sendMicTranscript():
try: try:
selected_your_languages = config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]
languages = [data["language"] for data in selected_your_languages.values() if data["enable"] is True]
countries = [data["country"] for data in selected_your_languages.values() if data["enable"] is True]
res = self.mic_transcriber.transcribeAudioQueue( res = self.mic_transcriber.transcribeAudioQueue(
self.mic_audio_queue, self.mic_audio_queue,
config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"], languages,
config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["country"], countries,
config.MIC_AVG_LOGPROB, config.MIC_AVG_LOGPROB,
config.MIC_NO_SPEECH_PROB config.MIC_NO_SPEECH_PROB
) )
if res: if res:
message = self.mic_transcriber.getTranscript() result = self.mic_transcriber.getTranscript()
fnc(message) fnc(result)
except Exception: except Exception:
errorLogging() errorLogging()
@@ -592,16 +597,19 @@ class Model:
) )
def sendSpeakerTranscript(): def sendSpeakerTranscript():
try: try:
selected_target_languages = config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]
languages = [data["language"] for data in selected_target_languages.values() if data["enable"] is True]
countries = [data["country"] for data in selected_target_languages.values() if data["enable"] is True]
res = self.speaker_transcriber.transcribeAudioQueue( res = self.speaker_transcriber.transcribeAudioQueue(
speaker_audio_queue, speaker_audio_queue,
config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"], languages,
config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["country"], countries,
config.SPEAKER_AVG_LOGPROB, config.SPEAKER_AVG_LOGPROB,
config.SPEAKER_NO_SPEECH_PROB config.SPEAKER_NO_SPEECH_PROB
) )
if res: if res:
message = self.speaker_transcriber.getTranscript() result = self.speaker_transcriber.getTranscript()
fnc(message) fnc(result)
except Exception: except Exception:
errorLogging() errorLogging()

74
src-python/models/transcription/transcription_transcriber.py
View File

@@ -44,39 +44,59 @@ class AudioTranscriber:
self.whisper_model = getWhisperModel(root, whisper_weight_type, device=device, device_index=device_index) self.whisper_model = getWhisperModel(root, whisper_weight_type, device=device, device_index=device_index)
self.transcription_engine = "Whisper" self.transcription_engine = "Whisper"
def transcribeAudioQueue(self, audio_queue, language, country, avg_logprob=-0.8, no_speech_prob=0.6): def transcribeAudioQueue(self, audio_queue, languages, countries, avg_logprob=-0.8, no_speech_prob=0.6):
if audio_queue.empty(): if audio_queue.empty():
time.sleep(0.01) time.sleep(0.01)
return False return False
audio, time_spoken = audio_queue.get() audio, time_spoken = audio_queue.get()
self.updateLastSampleAndPhraseStatus(audio, time_spoken) self.updateLastSampleAndPhraseStatus(audio, time_spoken)
text = '' result = {"confidence": 0, "text": "", "language": None}
try: try:
audio_data = self.audio_sources["process_data_func"]() audio_data = self.audio_sources["process_data_func"]()
match self.transcription_engine: match self.transcription_engine:
case "Google": case "Google":
text = self.audio_recognizer.recognize_google(audio_data, language=transcription_lang[language][country][self.transcription_engine]) confidences = []
for language, country in zip(languages, countries):
text, confidence = self.audio_recognizer.recognize_google(
audio_data,
language=transcription_lang[language][country][self.transcription_engine],
with_confidence=True
)
confidences.append({"confidence": confidence, "text": text, "language": language})
result = max(confidences, key=lambda x: x["confidence"])
case "Whisper": case "Whisper":
confidences = []
audio_data = np.frombuffer(audio_data.get_raw_data(convert_rate=16000, convert_width=2), np.int16).flatten().astype(np.float32) / 32768.0 audio_data = np.frombuffer(audio_data.get_raw_data(convert_rate=16000, convert_width=2), np.int16).flatten().astype(np.float32) / 32768.0
if isinstance(audio_data, torch.Tensor): if isinstance(audio_data, torch.Tensor):
audio_data = audio_data.detach().numpy() audio_data = audio_data.detach().numpy()
segments, _ = self.whisper_model.transcribe(
audio_data, for language, country in zip(languages, countries):
beam_size=5, text = ""
temperature=0.0, source_language = transcription_lang[language][country][self.transcription_engine] if len(languages) == 1 else None
log_prob_threshold=-0.8, segments, info = self.whisper_model.transcribe(
no_speech_threshold=0.6, audio_data,
language=transcription_lang[language][country][self.transcription_engine], beam_size=5,
word_timestamps=False, temperature=0.0,
without_timestamps=True, log_prob_threshold=-0.8,
task="transcribe", no_speech_threshold=0.6,
vad_filter=False, language=source_language,
) word_timestamps=False,
for s in segments: without_timestamps=True,
if s.avg_logprob < avg_logprob or s.no_speech_prob > no_speech_prob: task="transcribe",
continue vad_filter=False,
text += s.text )
for s in segments:
if s.avg_logprob < avg_logprob or s.no_speech_prob > no_speech_prob:
continue
text += s.text
confidences.append({"confidence": info.language_probability, "text": text, "language": language})
if (len(languages) == 1) or (transcription_lang[language][country][self.transcription_engine] == info.language):
break
result = max(confidences, key=lambda x: x["confidence"])
except UnknownValueError: except UnknownValueError:
pass pass
except Exception: except Exception:
@@ -84,8 +104,8 @@ class AudioTranscriber:
finally: finally:
pass pass
if text != '': if result["text"] != "":
self.updateTranscript(text) self.updateTranscript(result)
return True return True
def updateLastSampleAndPhraseStatus(self, data, time_spoken): def updateLastSampleAndPhraseStatus(self, data, time_spoken):
@@ -123,23 +143,23 @@ class AudioTranscriber:
audio = self.audio_recognizer.record(source) audio = self.audio_recognizer.record(source)
return audio return audio
def updateTranscript(self, text): def updateTranscript(self, result):
source_info = self.audio_sources source_info = self.audio_sources
transcript = self.transcript_data transcript = self.transcript_data
if source_info["new_phrase"] or len(transcript) == 0: if source_info["new_phrase"] or len(transcript) == 0:
if len(transcript) > self.max_phrases: if len(transcript) > self.max_phrases:
transcript.pop(-1) transcript.pop(-1)
transcript.insert(0, text) transcript.insert(0, result)
else: else:
transcript[0] = text transcript[0] = result
def getTranscript(self): def getTranscript(self):
if len(self.transcript_data) > 0: if len(self.transcript_data) > 0:
text = self.transcript_data.pop(-1) result = self.transcript_data.pop(-1)
else: else:
text = "" result = {"confidence": 0, "text": "", "language": None}
return text return result
def clearTranscriptData(self): def clearTranscriptData(self):
self.transcript_data.clear() self.transcript_data.clear()

12
src-python/webui_controller.py
View File

@@ -183,7 +183,9 @@ class Controller:
self.weight_type, self.weight_type,
) )
def micMessage(self, message: Union[str, bool]) -> None: def micMessage(self, result: dict) -> None:
message = result["text"]
language = result["language"]
if isinstance(message, bool) and message is False: if isinstance(message, bool) and message is False:
self.run( self.run(
400, 400,
@@ -209,7 +211,7 @@ class Controller:
elif config.ENABLE_TRANSLATION is False: elif config.ENABLE_TRANSLATION is False:
pass pass
else: else:
translation, success = model.getInputTranslate(message) translation, success = model.getInputTranslate(message, source_language=language)
if all(success) is not True: if all(success) is not True:
self.changeToCTranslate2Process() self.changeToCTranslate2Process()
self.run( self.run(
@@ -256,7 +258,9 @@ class Controller:
overlay_image = model.createOverlayImageLargeLog("send", message, translation[0] if len(translation) > 0 else "") overlay_image = model.createOverlayImageLargeLog("send", message, translation[0] if len(translation) > 0 else "")
model.updateOverlayLargeLog(overlay_image) model.updateOverlayLargeLog(overlay_image)
def speakerMessage(self, message) -> None: def speakerMessage(self, result:dict) -> None:
message = result["text"]
language = result["language"]
if isinstance(message, bool) and message is False: if isinstance(message, bool) and message is False:
self.run( self.run(
400, 400,
@@ -274,7 +278,7 @@ class Controller:
elif config.ENABLE_TRANSLATION is False: elif config.ENABLE_TRANSLATION is False:
pass pass
else: else:
translation, success = model.getOutputTranslate(message) translation, success = model.getOutputTranslate(message, source_language=language)
if all(success) is not True: if all(success) is not True:
self.changeToCTranslate2Process() self.changeToCTranslate2Process()
self.run( self.run(