every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

👍️[Update] Model : VRChatのmute/voiceを検出し文字起こしを停止する機能を追加

Browse Source
This commit is contained in:
misyaguziya
2024-04-16 21:44:26 +09:00
parent 2e2c237a26
commit 1c19236d62
4 changed files with 102 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
config.py
View File

@@ -830,6 +830,17 @@ class Config:
self._ENABLE_LOGGER = value
saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value)
@property
@json_serializable('ENABLE_MUTE_DETECT')
def ENABLE_MUTE_DETECT(self):
return self._ENABLE_MUTE_DETECT
@ENABLE_MUTE_DETECT.setter
def ENABLE_MUTE_DETECT(self, value):
if isinstance(value, bool):
self._ENABLE_MUTE_DETECT = value
saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value)
@property
@json_serializable('IS_CONFIG_WINDOW_COMPACT_MODE')
def IS_CONFIG_WINDOW_COMPACT_MODE(self):
@@ -995,6 +1006,7 @@ class Config:
self._ENABLE_SEND_RECEIVED_MESSAGE_TO_VRC = False # Speaker2Chatbox
self._ENABLE_SPEAKER2CHATBOX_PASS = "000000000"
self._ENABLE_LOGGER = False
self._ENABLE_MUTE_DETECT = False
self._IS_CONFIG_WINDOW_COMPACT_MODE = False
def load_config(self):

24
controller.py
View File

@@ -909,20 +909,30 @@ def callbackSetEnableSendReceivedMessageToVrc(value):
config.ENABLE_SEND_RECEIVED_MESSAGE_TO_VRC = value
# ---------------------Speaker2Chatbox---------------------
def createDictOSCReceiveParameters():
osc_parameter_prefix = "/avatar/parameters/"
param_MuteSelf = "MuteSelf"
param_Voice = "Voice"
def change_handler_muteself(address, osc_arguments):
config.VRCHAT_MUTESELF = osc_arguments
def change_handler_mute(address, osc_arguments):
if config.ENABLE_MUTE_DETECT is True:
if osc_arguments is True and change_handler_mute.status_mute is False:
model.stopPutQueueMicAudio()
change_handler_mute.status_mute = True
elif osc_arguments is False and change_handler_mute.status_mute is True:
model.startPutQueueMicAudio()
change_handler_mute.status_mute = False
def change_handler_voice(address, osc_arguments):
config.VRCHAT_MUTESELF = False
if config.ENABLE_MUTE_DETECT is True:
if change_handler_mute.status_mute is True:
model.startPutQueueMicAudio()
change_handler_mute.status_mute = False
change_handler_mute.status_mute = False
dict_filter_and_target = {
osc_parameter_prefix + param_MuteSelf: change_handler_muteself,
osc_parameter_prefix + param_MuteSelf: change_handler_mute,
osc_parameter_prefix + param_Voice: change_handler_voice,
}
return dict_filter_and_target
@@ -993,8 +1003,8 @@ def createMainWindow(splash):
if config.ENABLE_LOGGER is True:
model.startLogger()
# init OSC
model.startReceiveOSC(createDictOSCReceiveParameters())
# init OSC receive
model.startReceiveOSC()
splash.toProgress(3) # Last one.

80
model.py
View File

@@ -17,7 +17,7 @@ from typing import Callable
from flashtext import KeywordProcessor
from models.translation.translation_translator import Translator
from models.transcription.transcription_utils import getInputDevices, getOutputDevices
from models.osc.osc_tools import sendTyping, sendMessage, sendTestAction, receiveOscParameters
from models.osc.osc_tools import sendTyping, sendMessage, receiveOscParameters
from models.transcription.transcription_recorder import SelectedMicEnergyAndAudioRecorder, SelectedSpeakerEnergyAndAudioRecorder
from models.transcription.transcription_recorder import SelectedMicEnergyRecorder, SelectedSpeakerEnergyRecorder
from models.transcription.transcription_transcriber import AudioTranscriber
@@ -46,6 +46,18 @@ class threadFnc(Thread):
return
self.fnc(*self._args, **self._kwargs)
class ConditionalQueue(Queue):
def __init__(self, flag=True, *args, **kwargs):
super().__init__(*args, **kwargs)
self.flag = flag
def put(self, item, block=True, timeout=None):
if self.flag is True:
super().put(item, block, timeout)
def set_flag(self, value):
self.flag = value
class Model:
_instance = None
@@ -67,6 +79,9 @@ class Model:
self.speaker_energy_plot_progressbar = None
self.translator = Translator()
self.keyword_processor = KeywordProcessor()
self.mic_audio_queue = ConditionalQueue()
# self.mic_energy_queue = ConditionalQueue()
self.mute_status = False
def checkCTranslatorCTranslate2ModelWeight(self):
return checkCTranslate2Weight(config.PATH_LOCAL, config.CTRANSLATE2_WEIGHT_TYPE)
@@ -243,8 +258,39 @@ class Model:
# if self.is_valid_osc is False:
# fnc()
def startReceiveOSC(self, fnc):
self.listening_server = receiveOscParameters(fnc)
# def startReceiveOSC(self, fnc):
# th_osc_server = threadFnc(receiveOscParameters, args=(fnc,))
# th_osc_server.daemon = True
# th_osc_server.start()
def startReceiveOSC(self):
osc_parameter_prefix = "/avatar/parameters/"
param_MuteSelf = "MuteSelf"
param_Voice = "Voice"
def change_handler_mute(address, osc_arguments):
if config.ENABLE_MUTE_DETECT is True:
if osc_arguments is True and self.mute_status is False:
self.stopPutQueueMicAudio()
self.mute_status = True
elif osc_arguments is False and self.mute_status is True:
self.startPutQueueMicAudio()
self.mute_status = False
def change_handler_voice(address, osc_arguments):
if config.ENABLE_MUTE_DETECT is True:
if self.mute_status is True:
self.startPutQueueMicAudio()
self.mute_status = False
dict_filter_and_target = {
osc_parameter_prefix + param_MuteSelf: change_handler_mute,
osc_parameter_prefix + param_Voice: change_handler_voice,
}
th_osc_server = threadFnc(receiveOscParameters, args=(dict_filter_and_target,))
th_osc_server.daemon = True
th_osc_server.start()
@staticmethod
def checkSoftwareUpdated():
@@ -321,8 +367,8 @@ class Model:
pass
return
mic_audio_queue = Queue()
# mic_energy_queue = Queue()
self.mic_audio_queue = ConditionalQueue()
# self.mic_energy_queue = ConditionalQueue()
mic_device = choice_mic_device[0]
record_timeout = config.INPUT_MIC_RECORD_TIMEOUT
phase_timeout = config.INPUT_MIC_PHRASE_TIMEOUT
@@ -335,8 +381,8 @@ class Model:
dynamic_energy_threshold=config.INPUT_MIC_DYNAMIC_ENERGY_THRESHOLD,
record_timeout=record_timeout,
)
# self.mic_audio_recorder.recordIntoQueue(mic_audio_queue, mic_energy_queue)
self.mic_audio_recorder.recordIntoQueue(mic_audio_queue, None)
# self.mic_audio_recorder.recordIntoQueue(self.mic_audio_queue, mic_energy_queue)
self.mic_audio_recorder.recordIntoQueue(self.mic_audio_queue, None)
self.mic_transcriber = AudioTranscriber(
speaker=False,
source=self.mic_audio_recorder.source,
@@ -348,15 +394,17 @@ class Model:
)
def sendMicTranscript():
try:
self.mic_transcriber.transcribeAudioQueue(mic_audio_queue, config.SOURCE_LANGUAGE, config.SOURCE_COUNTRY)
self.mic_transcriber.transcribeAudioQueue(self.mic_audio_queue, config.SOURCE_LANGUAGE, config.SOURCE_COUNTRY)
message = self.mic_transcriber.getTranscript()
fnc(message)
except Exception:
pass
def endMicTranscript():
mic_audio_queue.queue.clear()
# mic_energy_queue.queue.clear()
while not self.mic_audio_queue.empty():
self.mic_audio_queue.get()
# while not self.mic_energy_queue.empty():
# self.mic_energy_queue.get()
del self.mic_transcriber
gc.collect()
@@ -378,6 +426,18 @@ class Model:
# self.mic_get_energy.daemon = True
# self.mic_get_energy.start()
def startPutQueueMicAudio(self):
while not self.mic_audio_queue.empty():
self.mic_audio_queue.get()
self.mic_audio_queue.set_flag(True)
# self.mic_energy_queue.set_flag(True)
def stopPutQueueMicAudio(self):
self.mic_audio_queue.set_flag(False)
while not self.mic_audio_queue.empty():
self.mic_audio_queue.get()
# self.mic_energy_queue.set_flag(False)
def stopMicTranscript(self):
if isinstance(self.mic_print_transcript, threadFnc):
self.mic_print_transcript.stop()

6
models/osc/osc_tools.py
View File

@@ -1,5 +1,5 @@
from time import sleep
import threading
from threading import Thread
from pythonosc import osc_message_builder
from pythonosc import udp_client
from pythonosc import dispatcher
@@ -55,14 +55,14 @@ def receiveOscParameters(dict_filter_and_target, ip_address="127.0.0.1", title="
osc_dispatcher = dispatcher.Dispatcher()
for filter, target in dict_filter_and_target.items():
osc_dispatcher.map(filter, target)
osc_udp_server = osc_server.ThreadingOSCUDPServer((ip_address, osc_port), osc_dispatcher)
threading.Thread(target=osc_udp_server.serve_forever, daemon = True).start()
osc_client = OSCQueryService(title, http_port, osc_port)
for filter, target in dict_filter_and_target.items():
osc_client.advertise_endpoint(filter)
osc_udp_server.serve_forever()
if __name__ == "__main__":
osc_parameter_prefix = "/avatar/parameters/"
osc_avatar_change_path = "/avatar/change"