every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

👍️[Update] Model : 複数言語選択時に複数の音声に対して文字起こしを行う機能を追加

Browse Source
This commit is contained in:
misyaguziya
2024-12-19 13:44:52 +09:00
parent 01d9c27728
commit 4d2fd379a8
3 changed files with 75 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src-python/model.py
View File

@@ -207,8 +207,9 @@ class Model:
sleep(0.1)
return translation, success_flag
def getInputTranslate(self, message):
def getInputTranslate(self, message, source_language=None):
translator_name=config.SELECTED_TRANSLATION_ENGINES[config.SELECTED_TAB_NO]
if source_language is None:
source_language=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"]
target_languages=config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]
@@ -231,8 +232,9 @@ class Model:
return translations, success_flags
def getOutputTranslate(self, message):
def getOutputTranslate(self, message, source_language=None):
translator_name=config.SELECTED_TRANSLATION_ENGINES[config.SELECTED_TAB_NO]
if source_language is None:
source_language=config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"]
target_language=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"]
target_country=config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["country"]
@@ -428,16 +430,19 @@ class Model:
)
def sendMicTranscript():
try:
selected_your_languages = config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]
languages = [data["language"] for data in selected_your_languages.values() if data["enable"] is True]
countries = [data["country"] for data in selected_your_languages.values() if data["enable"] is True]
res = self.mic_transcriber.transcribeAudioQueue(
self.mic_audio_queue,
config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"],
config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]["1"]["country"],
languages,
countries,
config.MIC_AVG_LOGPROB,
config.MIC_NO_SPEECH_PROB
)
if res:
message = self.mic_transcriber.getTranscript()
fnc(message)
result = self.mic_transcriber.getTranscript()
fnc(result)
except Exception:
errorLogging()
@@ -592,16 +597,19 @@ class Model:
)
def sendSpeakerTranscript():
try:
selected_target_languages = config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]
languages = [data["language"] for data in selected_target_languages.values() if data["enable"] is True]
countries = [data["country"] for data in selected_target_languages.values() if data["enable"] is True]
res = self.speaker_transcriber.transcribeAudioQueue(
speaker_audio_queue,
config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["language"],
config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]["1"]["country"],
languages,
countries,
config.SPEAKER_AVG_LOGPROB,
config.SPEAKER_NO_SPEECH_PROB
)
if res:
message = self.speaker_transcriber.getTranscript()
fnc(message)
result = self.speaker_transcriber.getTranscript()
fnc(result)
except Exception:
errorLogging()

46
src-python/models/transcription/transcription_transcriber.py
View File

@@ -44,30 +44,45 @@ class AudioTranscriber:
self.whisper_model = getWhisperModel(root, whisper_weight_type, device=device, device_index=device_index)
self.transcription_engine = "Whisper"
def transcribeAudioQueue(self, audio_queue, language, country, avg_logprob=-0.8, no_speech_prob=0.6):
def transcribeAudioQueue(self, audio_queue, languages, countries, avg_logprob=-0.8, no_speech_prob=0.6):
if audio_queue.empty():
time.sleep(0.01)
return False
audio, time_spoken = audio_queue.get()
self.updateLastSampleAndPhraseStatus(audio, time_spoken)
text = ''
result = {"confidence": 0, "text": "", "language": None}
try:
audio_data = self.audio_sources["process_data_func"]()
match self.transcription_engine:
case "Google":
text = self.audio_recognizer.recognize_google(audio_data, language=transcription_lang[language][country][self.transcription_engine])
confidences = []
for language, country in zip(languages, countries):
text, confidence = self.audio_recognizer.recognize_google(
audio_data,
language=transcription_lang[language][country][self.transcription_engine],
with_confidence=True
)
confidences.append({"confidence": confidence, "text": text, "language": language})
result = max(confidences, key=lambda x: x["confidence"])
case "Whisper":
confidences = []
audio_data = np.frombuffer(audio_data.get_raw_data(convert_rate=16000, convert_width=2), np.int16).flatten().astype(np.float32) / 32768.0
if isinstance(audio_data, torch.Tensor):
audio_data = audio_data.detach().numpy()
segments, _ = self.whisper_model.transcribe(
for language, country in zip(languages, countries):
text = ""
source_language = transcription_lang[language][country][self.transcription_engine] if len(languages) == 1 else None
segments, info = self.whisper_model.transcribe(
audio_data,
beam_size=5,
temperature=0.0,
log_prob_threshold=-0.8,
no_speech_threshold=0.6,
language=transcription_lang[language][country][self.transcription_engine],
language=source_language,
word_timestamps=False,
without_timestamps=True,
task="transcribe",
@@ -77,6 +92,11 @@ class AudioTranscriber:
if s.avg_logprob < avg_logprob or s.no_speech_prob > no_speech_prob:
continue
text += s.text
confidences.append({"confidence": info.language_probability, "text": text, "language": language})
if (len(languages) == 1) or (transcription_lang[language][country][self.transcription_engine] == info.language):
break
result = max(confidences, key=lambda x: x["confidence"])
except UnknownValueError:
pass
except Exception:
@@ -84,8 +104,8 @@ class AudioTranscriber:
finally:
pass
if text != '':
self.updateTranscript(text)
if result["text"] != "":
self.updateTranscript(result)
return True
def updateLastSampleAndPhraseStatus(self, data, time_spoken):
@@ -123,23 +143,23 @@ class AudioTranscriber:
audio = self.audio_recognizer.record(source)
return audio
def updateTranscript(self, text):
def updateTranscript(self, result):
source_info = self.audio_sources
transcript = self.transcript_data
if source_info["new_phrase"] or len(transcript) == 0:
if len(transcript) > self.max_phrases:
transcript.pop(-1)
transcript.insert(0, text)
transcript.insert(0, result)
else:
transcript[0] = text
transcript[0] = result
def getTranscript(self):
if len(self.transcript_data) > 0:
text = self.transcript_data.pop(-1)
result = self.transcript_data.pop(-1)
else:
text = ""
return text
result = {"confidence": 0, "text": "", "language": None}
return result
def clearTranscriptData(self):
self.transcript_data.clear()

12
src-python/webui_controller.py
View File

@@ -183,7 +183,9 @@ class Controller:
self.weight_type,
)
def micMessage(self, message: Union[str, bool]) -> None:
def micMessage(self, result: dict) -> None:
message = result["text"]
language = result["language"]
if isinstance(message, bool) and message is False:
self.run(
400,
@@ -209,7 +211,7 @@ class Controller:
elif config.ENABLE_TRANSLATION is False:
pass
else:
translation, success = model.getInputTranslate(message)
translation, success = model.getInputTranslate(message, source_language=language)
if all(success) is not True:
self.changeToCTranslate2Process()
self.run(
@@ -256,7 +258,9 @@ class Controller:
overlay_image = model.createOverlayImageLargeLog("send", message, translation[0] if len(translation) > 0 else "")
model.updateOverlayLargeLog(overlay_image)
def speakerMessage(self, message) -> None:
def speakerMessage(self, result:dict) -> None:
message = result["text"]
language = result["language"]
if isinstance(message, bool) and message is False:
self.run(
400,
@@ -274,7 +278,7 @@ class Controller:
elif config.ENABLE_TRANSLATION is False:
pass
else:
translation, success = model.getOutputTranslate(message)
translation, success = model.getOutputTranslate(message, source_language=language)
if all(success) is not True:
self.changeToCTranslate2Process()
self.run(