every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[WIP/TEST] Wisperの処理についてUIと内部の処理を接続

Browse Source
This commit is contained in:
misyaguziya
2024-02-03 02:35:40 +09:00
parent 11c8f6eb4d
commit 801d948513
6 changed files with 57 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
config.py
View File

@@ -210,6 +210,15 @@ class Config:
if isinstance(value, bool): if isinstance(value, bool):
self._IS_RESET_BUTTON_DISPLAYED_FOR_TRANSLATION = value self._IS_RESET_BUTTON_DISPLAYED_FOR_TRANSLATION = value
@property
def IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER(self):
return self._IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER
@IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER.setter
def IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER(self, value):
if isinstance(value, bool):
self._IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER = value
# Save Json Data # Save Json Data
## Main Window ## Main Window
@property @property
@@ -268,14 +277,14 @@ class Config:
saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value) saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value)
@property @property
@json_serializable('SELECTED_RECOGNIZER') @json_serializable('SELECTED_TRANSCRIPTION_ENGINE')
def SELECTED_RECOGNIZER(self): def SELECTED_TRANSCRIPTION_ENGINE(self):
return self._SELECTED_RECOGNIZER return self._SELECTED_TRANSCRIPTION_ENGINE
@SELECTED_RECOGNIZER.setter @SELECTED_TRANSCRIPTION_ENGINE.setter
def SELECTED_RECOGNIZER(self, value): def SELECTED_TRANSCRIPTION_ENGINE(self, value):
if isinstance(value, str): if isinstance(value, str):
self._SELECTED_RECOGNIZER = value self._SELECTED_TRANSCRIPTION_ENGINE = value
saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value) saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value)
@property @property
@@ -820,6 +829,7 @@ class Config:
self._TARGET_LANGUAGE = "English" self._TARGET_LANGUAGE = "English"
self._TARGET_COUNTRY = "United States" self._TARGET_COUNTRY = "United States"
self._IS_RESET_BUTTON_DISPLAYED_FOR_TRANSLATION = False self._IS_RESET_BUTTON_DISPLAYED_FOR_TRANSLATION = False
self._IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER = False
# Save Json Data # Save Json Data
## Main Window ## Main Window
@@ -844,7 +854,7 @@ class Config:
"2":"English\n(United States)", "2":"English\n(United States)",
"3":"English\n(United States)", "3":"English\n(United States)",
} }
self._SELECTED_RECOGNIZER = "Google" self._SELECTED_TRANSCRIPTION_ENGINE = "Google"
self._IS_MAIN_WINDOW_SIDEBAR_COMPACT_MODE = False self._IS_MAIN_WINDOW_SIDEBAR_COMPACT_MODE = False
## Config Window ## Config Window

30
controller.py
View File

@@ -773,29 +773,27 @@ def callbackSetUserWhisperFeature(value):
config.USE_WHISPER_FEATURE = value config.USE_WHISPER_FEATURE = value
if config.USE_WHISPER_FEATURE is True: if config.USE_WHISPER_FEATURE is True:
view.openWhisperWeightTypeWidget() view.openWhisperWeightTypeWidget()
if model.checkTranscriptionWhisperModelWeight() is True:
config.IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER = False
config.SELECTED_TRANSCRIPTION_ENGINE = "Whisper"
else:
config.IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER = True
config.SELECTED_TRANSCRIPTION_ENGINE = "Google"
else: else:
view.closeWhisperWeightTypeWidget() view.closeWhisperWeightTypeWidget()
view.showRestartButtonIfRequired()
def callbackSetWhisperWeightType(value): def callbackSetWhisperWeightType(value):
print("callbackSetWhisperWeightType", value) print("callbackSetWhisperWeightType", value)
config.WHISPER_WEIGHT_TYPE = str(value) config.WHISPER_WEIGHT_TYPE = str(value)
view.updateSelectedWhisperWeightType(config.WHISPER_WEIGHT_TYPE) view.updateSelectedWhisperWeightType(config.WHISPER_WEIGHT_TYPE)
# view.setWidgetsStatus_changeWeightType_Pending() if model.checkTranscriptionWhisperModelWeight() is True:
# if model.checkCTranslatorCTranslate2ModelWeight(): config.IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER = False
# config.IS_RESET_BUTTON_DISPLAYED_FOR_TRANSLATION = False config.SELECTED_TRANSCRIPTION_ENGINE = "Whisper"
# def callback(): else:
# model.changeTranslatorCTranslate2Model() config.IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER = True
# view.useTranslationFeatureProcess("Normal") config.SELECTED_TRANSCRIPTION_ENGINE = "Google"
# view.setWidgetsStatus_changeWeightType_Done() view.showRestartButtonIfRequired()
# th_callback = Thread(target=callback)
# th_callback.daemon = True
# th_callback.start()
# else:
# config.IS_RESET_BUTTON_DISPLAYED_FOR_TRANSLATION = True
# view.useTranslationFeatureProcess("Restart")
# view.setWidgetsStatus_changeWeightType_Done()
# view.showRestartButtonIfRequired()
# Others Tab # Others Tab
def callbackSetEnableAutoClearMessageBox(value): def callbackSetEnableAutoClearMessageBox(value):

12
model.py
View File

@@ -24,6 +24,7 @@ from models.xsoverlay.notification import xsoverlayForVRCT
from models.translation.translation_languages import translation_lang from models.translation.translation_languages import translation_lang
from models.transcription.transcription_languages import transcription_lang from models.transcription.transcription_languages import transcription_lang
from models.translation.translation_utils import checkCTranslate2Weight from models.translation.translation_utils import checkCTranslate2Weight
from models.transcription.transcription_whisper import checkWhisperWeight
from config import config from config import config
class threadFnc(Thread): class threadFnc(Thread):
@@ -74,6 +75,9 @@ class Model:
def changeTranslatorCTranslate2Model(self): def changeTranslatorCTranslate2Model(self):
self.translator.changeCTranslate2Model(config.PATH_LOCAL, config.CTRANSLATE2_WEIGHT_TYPE) self.translator.changeCTranslate2Model(config.PATH_LOCAL, config.CTRANSLATE2_WEIGHT_TYPE)
def checkTranscriptionWhisperModelWeight(self):
return checkWhisperWeight(config.PATH_LOCAL, config.WHISPER_WEIGHT_TYPE)
def resetKeywordProcessor(self): def resetKeywordProcessor(self):
del self.keyword_processor del self.keyword_processor
self.keyword_processor = KeywordProcessor() self.keyword_processor = KeywordProcessor()
@@ -335,12 +339,12 @@ class Model:
source=self.mic_audio_recorder.source, source=self.mic_audio_recorder.source,
phrase_timeout=phase_timeout, phrase_timeout=phase_timeout,
max_phrases=config.INPUT_MIC_MAX_PHRASES, max_phrases=config.INPUT_MIC_MAX_PHRASES,
whisper_enabled=config.USE_WHISPER_FEATURE, transcription_engine=config.SELECTED_TRANSCRIPTION_ENGINE,
whisper_weight_type=config.WHISPER_WEIGHT_TYPE, whisper_weight_type=config.WHISPER_WEIGHT_TYPE,
root=config.PATH_LOCAL, root=config.PATH_LOCAL,
) )
def sendMicTranscript(): def sendMicTranscript():
mic_transcriber.transcribeAudioQueue(config.SELECTED_RECOGNIZER, mic_audio_queue, config.SOURCE_LANGUAGE, config.SOURCE_COUNTRY) mic_transcriber.transcribeAudioQueue(mic_audio_queue, config.SOURCE_LANGUAGE, config.SOURCE_COUNTRY)
message = mic_transcriber.getTranscript() message = mic_transcriber.getTranscript()
try: try:
fnc(message) fnc(message)
@@ -419,12 +423,12 @@ class Model:
source=self.speaker_audio_recorder.source, source=self.speaker_audio_recorder.source,
phrase_timeout=phase_timeout, phrase_timeout=phase_timeout,
max_phrases=config.INPUT_SPEAKER_MAX_PHRASES, max_phrases=config.INPUT_SPEAKER_MAX_PHRASES,
whisper_enabled=config.USE_WHISPER_FEATURE, transcription_engine=config.SELECTED_TRANSCRIPTION_ENGINE,
whisper_weight_type=config.WHISPER_WEIGHT_TYPE, whisper_weight_type=config.WHISPER_WEIGHT_TYPE,
root=config.PATH_LOCAL, root=config.PATH_LOCAL,
) )
def sendSpeakerTranscript(): def sendSpeakerTranscript():
speaker_transcriber.transcribeAudioQueue(config.SELECTED_RECOGNIZER, speaker_audio_queue, config.TARGET_LANGUAGE, config.TARGET_COUNTRY) speaker_transcriber.transcribeAudioQueue(speaker_audio_queue, config.TARGET_LANGUAGE, config.TARGET_COUNTRY)
message = speaker_transcriber.getTranscript() message = speaker_transcriber.getTranscript()
try: try:
fnc(message) fnc(message)

28
models/transcription/transcription_transcriber.py
View File

@@ -14,7 +14,7 @@ PHRASE_TIMEOUT = 3
MAX_PHRASES = 10 MAX_PHRASES = 10
class AudioTranscriber: class AudioTranscriber:
def __init__(self, speaker, source, phrase_timeout, max_phrases, whisper_enabled, whisper_weight_type, root): def __init__(self, speaker, source, phrase_timeout, max_phrases, transcription_engine, whisper_weight_type=None, root=None):
self.speaker = speaker self.speaker = speaker
self.phrase_timeout = phrase_timeout self.phrase_timeout = phrase_timeout
self.max_phrases = max_phrases self.max_phrases = max_phrases
@@ -30,38 +30,34 @@ class AudioTranscriber:
"new_phrase": True, "new_phrase": True,
"process_data_func": self.processSpeakerData if speaker else self.processSpeakerData "process_data_func": self.processSpeakerData if speaker else self.processSpeakerData
} }
if whisper_enabled is True: self.transcription_engine = transcription_engine
self.whisper_model = getWhisperModel(root, whisper_weight_type) match self.transcription_engine:
else: case "Google":
self.whisper_model = None self.audio_recognizer = Recognizer()
case "Whisper":
self.audio_recognizer = getWhisperModel(root, whisper_weight_type)
def transcribeAudioQueue(self, recognizer, audio_queue, language, country): def transcribeAudioQueue(self, audio_queue, language, country):
# while True:
audio, time_spoken = audio_queue.get() audio, time_spoken = audio_queue.get()
self.updateLastSampleAndPhraseStatus(audio, time_spoken) self.updateLastSampleAndPhraseStatus(audio, time_spoken)
text = '' text = ''
try: try:
# Whisperが使用できない場合はGoogle Speech-to-Textを使用する
if recognizer == "Whisper":
if self.whisper_model is None:
recognizer = "Google"
audio_data = self.audio_sources["process_data_func"]() audio_data = self.audio_sources["process_data_func"]()
match recognizer: match self.transcription_engine:
case "Google": case "Google":
text = self.audio_recognizer.recognize_google(audio_data, language=transcription_lang[language][country][recognizer]) text = self.audio_recognizer.recognize_google(audio_data, language=transcription_lang[language][country][self.transcription_engine])
case "Whisper": case "Whisper":
audio_data = np.frombuffer(audio_data.get_raw_data(convert_rate=16000, convert_width=2), np.int16).flatten().astype(np.float32) / 32768.0 audio_data = np.frombuffer(audio_data.get_raw_data(convert_rate=16000, convert_width=2), np.int16).flatten().astype(np.float32) / 32768.0
if isinstance(audio_data, torch.Tensor): if isinstance(audio_data, torch.Tensor):
audio_data = audio_data.detach().numpy() audio_data = audio_data.detach().numpy()
segments, _ = self.whisper_model.transcribe( segments, _ = self.audio_recognizer.transcribe(
audio_data, audio_data,
beam_size=5, beam_size=5,
temperature=0.0, temperature=0.0,
log_prob_threshold=-0.8, log_prob_threshold=-0.8,
no_speech_threshold=0.6, no_speech_threshold=0.6,
language=transcription_lang[language][country][recognizer], language=transcription_lang[language][country][self.transcription_engine],
word_timestamps=False, word_timestamps=False,
without_timestamps=True, without_timestamps=True,
task="transcribe", task="transcribe",

5
models/transcription/transcription_whisper.py
View File

@@ -42,7 +42,8 @@ def downloadFile(url, path, func=None):
except Exception as e: except Exception as e:
print("error:downloadFile()", e) print("error:downloadFile()", e)
def checkWhisperWeight(path): def checkWhisperWeight(root, weight_type):
path = os_path.join(root, "weights", "whisper", weight_type)
result = False result = False
try: try:
WhisperModel( WhisperModel(
@@ -62,7 +63,7 @@ def checkWhisperWeight(path):
def downloadWhisperWeight(root, weight_type, callbackFunc): def downloadWhisperWeight(root, weight_type, callbackFunc):
path = os_path.join(root, "weights", "whisper", weight_type) path = os_path.join(root, "weights", "whisper", weight_type)
os_makedirs(path, exist_ok=True) os_makedirs(path, exist_ok=True)
if checkWhisperWeight(path) is True: if checkWhisperWeight(root, weight_type) is True:
return return
for filename in _FILENAMES: for filename in _FILENAMES:

4
view.py
View File

@@ -29,6 +29,7 @@ class View():
font_family=config.FONT_FAMILY, font_family=config.FONT_FAMILY,
ui_language=config.UI_LANGUAGE, ui_language=config.UI_LANGUAGE,
is_reset_button_displayed_for_translation=config.IS_RESET_BUTTON_DISPLAYED_FOR_TRANSLATION, is_reset_button_displayed_for_translation=config.IS_RESET_BUTTON_DISPLAYED_FOR_TRANSLATION,
is_reset_button_displayed_for_whisper=config.IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER,
) )
if config.ENABLE_SPEAKER2CHATBOX is False: if config.ENABLE_SPEAKER2CHATBOX is False:
@@ -1049,7 +1050,8 @@ class View():
self.restart_required_configs_pre_data.ui_scaling == config.UI_SCALING and self.restart_required_configs_pre_data.ui_scaling == config.UI_SCALING and
self.restart_required_configs_pre_data.font_family == config.FONT_FAMILY and self.restart_required_configs_pre_data.font_family == config.FONT_FAMILY and
self.restart_required_configs_pre_data.ui_language == config.UI_LANGUAGE and self.restart_required_configs_pre_data.ui_language == config.UI_LANGUAGE and
self.restart_required_configs_pre_data.is_reset_button_displayed_for_translation == config.IS_RESET_BUTTON_DISPLAYED_FOR_TRANSLATION self.restart_required_configs_pre_data.is_reset_button_displayed_for_translation == config.IS_RESET_BUTTON_DISPLAYED_FOR_TRANSLATION and
self.restart_required_configs_pre_data.is_reset_button_displayed_for_whisper == config.IS_RESET_BUTTON_DISPLAYED_FOR_WHISPER
) )
if locale is None: if locale is None: