From e21661df1e08582399ed0ac477d35fe42157b6ea Mon Sep 17 00:00:00 2001 From: misyaguziya <53165965+misyaguziya@users.noreply.github.com> Date: Mon, 4 Nov 2024 06:27:08 +0900 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=91=8D=EF=B8=8F[Update]=20Model=20:?= =?UTF-8?q?=20OSC=20=E3=81=AE=E5=87=A6=E7=90=86=E3=82=92=E8=A6=8B=E7=9B=B4?= =?UTF-8?q?=E3=81=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 4 +- requirements_cuda.txt | 4 +- src-python/model.py | 46 ++++---- src-python/models/osc/osc_tools.py | 180 +++++++++++++---------------- 4 files changed, 108 insertions(+), 126 deletions(-) diff --git a/requirements.txt b/requirements.txt index bd049af7..8f111f27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ ctranslate2==4.3.1 transformers==4.40.2 pillow == 10.0.0 PyAudioWPatch == 0.2.12.6 -python-osc == 1.8.3 +python-osc == 1.9.0 deepl == 1.15.0 flashtext ==2.7 pyinstaller==6.10.0 @@ -17,4 +17,4 @@ pykakasi==2.3.0 pycaw==20240210 translators @ git+https://github.com/misyaguziya/translators@5.9.2.1 SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4.1 -tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.2 \ No newline at end of file +tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.3 \ No newline at end of file diff --git a/requirements_cuda.txt b/requirements_cuda.txt index 965dcaca..026f1331 100644 --- a/requirements_cuda.txt +++ b/requirements_cuda.txt @@ -5,7 +5,7 @@ ctranslate2==4.3.1 transformers==4.40.2 pillow == 10.0.0 PyAudioWPatch == 0.2.12.6 -python-osc == 1.8.3 +python-osc == 1.9.0 deepl == 1.15.0 flashtext ==2.7 pyinstaller==6.10.0 @@ -18,4 +18,4 @@ pykakasi==2.3.0 pycaw==20240210 translators @ git+https://github.com/misyaguziya/translators@5.9.2.1 SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4.1 -tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.2 \ No newline at end of file +tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.3 \ No newline at end of file diff --git a/src-python/model.py b/src-python/model.py index d4161e22..9ad5d3e0 100644 --- a/src-python/model.py +++ b/src-python/model.py @@ -17,7 +17,7 @@ from device_manager import device_manager from config import config from models.translation.translation_translator import Translator -from models.osc.osc_tools import sendTyping, sendMessage, receiveOscParameters, getOSCParameterValue +from models.osc.osc_tools import OSCHandler from models.transcription.transcription_recorder import SelectedMicEnergyAndAudioRecorder, SelectedSpeakerEnergyAndAudioRecorder from models.transcription.transcription_recorder import SelectedMicEnergyRecorder, SelectedSpeakerEnergyRecorder from models.transcription.transcription_transcriber import AudioTranscriber @@ -100,6 +100,7 @@ class Model: self.mic_mute_status_check = None self.kks = kakasi() self.watchdog = Watchdog(config.WATCHDOG_TIMEOUT, config.WATCHDOG_INTERVAL) + self.osc_handler = OSCHandler(config.OSC_IP_ADDRESS, config.OSC_PORT) def checkTranslatorCTranslate2ModelWeight(self, weight_type:str): return checkCTranslate2Weight(config.PATH_LOCAL, weight_type) @@ -286,28 +287,26 @@ class Model: filtered_list.append(filtered_item) return filtered_list - @staticmethod - def oscStartSendTyping(): - sendTyping(True, config.OSC_IP_ADDRESS, config.OSC_PORT) + def setOscIpAddress(self, ip_address): + self.osc_handler.setOscIpAddress(ip_address) - @staticmethod - def oscStopSendTyping(): - sendTyping(False, config.OSC_IP_ADDRESS, config.OSC_PORT) + def setOscPort(self, port): + self.osc_handler.setOscPort(port) - @staticmethod - def oscSendMessage(message): - sendMessage(message, config.OSC_IP_ADDRESS, config.OSC_PORT) + def oscStartSendTyping(self): + self.osc_handler.sendTyping(flag=True) - @staticmethod - def getMuteSelfStatus(): - return getOSCParameterValue(address="/avatar/parameters/MuteSelf") + def oscStopSendTyping(self): + self.osc_handler.sendTyping(flag=False) + + def oscSendMessage(self, message, notification=True): + self.osc_handler.sendMessage(message=message, notification=notification) + + def getMuteSelfStatus(self): + return self.osc_handler.getOSCParameterMuteSelf() def startCheckMuteSelfStatus(self): def checkMuteSelfStatus(): - if self.mic_mute_status is not None: - self.changeMicTranscriptStatus() - self.stopCheckMuteSelfStatus() - status = self.getMuteSelfStatus() if status is not None: self.mic_mute_status = status @@ -325,10 +324,7 @@ class Model: self.mic_mute_status_check = None def startReceiveOSC(self): - osc_parameter_prefix = "/avatar/parameters/" - param_MuteSelf = "MuteSelf" - - def change_handler_mute(address, osc_arguments): + def changeHandlerMute(address, osc_arguments): if osc_arguments is True and self.mic_mute_status is False: self.mic_mute_status = osc_arguments self.changeMicTranscriptStatus() @@ -337,12 +333,12 @@ class Model: self.changeMicTranscriptStatus() dict_filter_and_target = { - osc_parameter_prefix + param_MuteSelf: change_handler_mute, + self.osc_handler.osc_parameter_muteself: changeHandlerMute, } + self.osc_handler.receiveOscParameters(dict_filter_and_target) - th_osc_server = Thread(target=receiveOscParameters, args=(dict_filter_and_target,)) - th_osc_server.daemon = True - th_osc_server.start() + def stopReceiveOSC(self): + self.osc_handler.oscServerStop() @staticmethod def checkSoftwareUpdated(): diff --git a/src-python/models/osc/osc_tools.py b/src-python/models/osc/osc_tools.py index 52b65621..9b021f1d 100644 --- a/src-python/models/osc/osc_tools.py +++ b/src-python/models/osc/osc_tools.py @@ -1,112 +1,98 @@ +import asyncio +from typing import Any from time import sleep -from pythonosc import osc_message_builder, udp_client, dispatcher, osc_server +from threading import Thread +from pythonosc import udp_client, dispatcher, osc_server from tinyoscquery.queryservice import OSCQueryService from tinyoscquery.query import OSCQueryBrowser, OSCQueryClient from tinyoscquery.utility import get_open_udp_port, get_open_tcp_port -from psutil import process_iter +from tinyoscquery.shared.node import OSCAccess -# send OSC message typing -def sendTyping(flag=False, ip_address="127.0.0.1", port=9000): - typing = osc_message_builder.OscMessageBuilder(address="/chatbox/typing") - typing.add_arg(flag) - b_typing = typing.build() - client = udp_client.SimpleUDPClient(ip_address, port) - client.send(b_typing) +class OSCHandler: + def __init__(self, ip_address="127.0.0.1", port=9000) -> None: + self.osc_ip_address = ip_address + self.osc_port = port + self.osc_parameter_muteself = "/avatar/parameters/MuteSelf" + self.osc_parameter_chatbox_typing = "/chatbox/typing" + self.osc_parameter_chatbox_input = "/chatbox/input" + self.udp_client = udp_client.SimpleUDPClient(self.osc_ip_address, self.osc_port) + self.osc_server_name = "VRChat-Client" + self.osc_server = None + self.osc_query_service = None + self.osc_query_service_name = "VRCT" + self.osc_server_ip_address = ip_address + self.http_port = None + self.osc_server_port = None -# send OSC message -def sendMessage(message=None, ip_address="127.0.0.1", port=9000): - if message is not None: - msg = osc_message_builder.OscMessageBuilder(address="/chatbox/input") - msg.add_arg(f"{message}") - msg.add_arg(True) - msg.add_arg(True) - b_msg = msg.build() - client = udp_client.SimpleUDPClient(ip_address, port) - client.send(b_msg) + def setOscIpAddress(self, ip_address:str) -> None: + self.osc_ip_address = ip_address + self.udp_client = udp_client.SimpleUDPClient(self.osc_ip_address, self.osc_port) -def sendTestAction(ip_address="127.0.0.1", port=9000): - client = udp_client.SimpleUDPClient(ip_address, port) - client.send_message("/input/Vertical", 1) - sleep(0.01) - client.send_message("/input/Vertical", False) + def setOscPort(self, port:int) -> None: + self.osc_port = port + self.udp_client = udp_client.SimpleUDPClient(self.osc_ip_address, self.osc_port) -# send Input Voice -def sendInputVoice(flag=False, ip_address="127.0.0.1", port=9000): - input_voice = osc_message_builder.OscMessageBuilder(address="/input/Voice") - input_voice.add_arg(flag) - b_input_voice = input_voice.build() - client = udp_client.SimpleUDPClient(ip_address, port) - client.send(b_input_voice) + # send OSC message typing + def sendTyping(self, flag:bool=False) -> None: + self.udp_client.send_message(self.osc_parameter_chatbox_typing, [flag]) -def sendChangeVoice(ip_address="127.0.0.1", port=9000): - sendInputVoice(flag=0, ip_address=ip_address, port=port) - sleep(0.05) - sendInputVoice(flag=1, ip_address=ip_address, port=port) - sleep(0.05) - sendInputVoice(flag=0, ip_address=ip_address, port=port) - sleep(0.05) + # send OSC message + def sendMessage(self, message:str="", notification:bool=True) -> None: + if len(message) > 0: + self.udp_client.send_message(self.osc_parameter_chatbox_input, [f"{message}", True, notification]) -def getOSCParameterValue(address, server_name="VRChat-Client"): - value = None - try: - browser = OSCQueryBrowser() - sleep(1) - service = browser.find_service_by_name(server_name) - if service is not None: - oscq = OSCQueryClient(service) - mute_self_node = oscq.query_node(address) - value = mute_self_node.value[0] - browser.zc.close() - browser.browser.cancel() - - except Exception: - pass - return value - -def checkVRChatRunning() -> bool: - _proc_name = "VRChat.exe" - return _proc_name in (p.name() for p in process_iter()) - -def receiveOscParameters(dict_filter_and_target, ip_address="127.0.0.1", title="VRCT"): - while True: - if not checkVRChatRunning(): + def getOSCParameterValue(self, address:str) -> Any: + value = None + try: + browser = OSCQueryBrowser() sleep(1) - else: - try: - osc_port = get_open_udp_port() - http_port = get_open_tcp_port() - osc_dispatcher = dispatcher.Dispatcher() - for filter, target in dict_filter_and_target.items(): - osc_dispatcher.map(filter, target) - osc_udp_server = osc_server.ThreadingOSCUDPServer((ip_address, osc_port), osc_dispatcher) + service = browser.find_service_by_name(self.osc_server_name) + if service is not None: + osc_query_client = OSCQueryClient(service) + mute_self_node = osc_query_client.query_node(address) + value = mute_self_node.value[0] + browser.zc.close() + browser.browser.cancel() - osc_client = OSCQueryService(title, http_port, osc_port) - for filter, target in dict_filter_and_target.items(): - osc_client.advertise_endpoint(filter) + except Exception: + pass + return value - osc_udp_server.serve_forever() - except Exception: - pass + def getOSCParameterMuteSelf(self) -> bool: + return self.getOSCParameterValue(self.osc_parameter_muteself) + + def receiveOscParameters(self, dict_filter_and_target:dict) -> None: + self.osc_server_port = get_open_udp_port() + self.http_port = get_open_tcp_port() + osc_dispatcher = dispatcher.Dispatcher() + for filter, target in dict_filter_and_target.items(): + osc_dispatcher.map(filter, target) + self.osc_server = osc_server.ThreadingOSCUDPServer((self.osc_server_ip_address, self.osc_server_port), osc_dispatcher, asyncio.get_event_loop()) + Thread(target=self.oscServerServe, daemon=True).start() + + self.osc_query_service = OSCQueryService(self.osc_query_service_name, self.http_port, self.osc_server_port) + for filter, target in dict_filter_and_target.items(): + self.osc_query_service.advertise_endpoint(filter, access=OSCAccess.READWRITE_VALUE) + + def oscServerServe(self) -> None: + self.osc_server.serve_forever(100) + + def oscServerStop(self) -> None: + if isinstance(self.osc_server, osc_server.ThreadingOSCUDPServer): + self.osc_server.shutdown() + self.osc_server = None + if isinstance(self.osc_query_service, OSCQueryService): + self.osc_query_service.http_server.shutdown() + self.osc_query_service = None if __name__ == "__main__": - osc_parameter_prefix = "/avatar/parameters/" - osc_avatar_change_path = "/avatar/change" - param_MuteSelf = "MuteSelf" - param_Voice = "Voice" - - def print_handler_all(address, *args): - print(f"all {address}: {args}") - - def print_handler_muteself(address, *args): - print(f"muteself {address}: {args}") - - def print_handler_voice(address, *args): - print(f"voice {address}: {args}") - - dict_filter_and_target = { - # osc_parameter_prefix + "*": print_handler_all, - osc_parameter_prefix + param_MuteSelf: print_handler_muteself, - osc_parameter_prefix + param_Voice: print_handler_voice, - } - - receiveOscParameters(dict_filter_and_target) \ No newline at end of file + handler = OSCHandler() + handler.receiveOscParameters({ + "/avatar/parameters/MuteSelf": print, + }) + sleep(5) + handler.sendTyping(True) + sleep(1) + handler.sendMessage(message="Hello World", notification=True) + sleep(60) + handler.oscServerStop() \ No newline at end of file From f79735f13c1df8a0f6ae45872ad9441bd615adc7 Mon Sep 17 00:00:00 2001 From: misyaguziya <53165965+misyaguziya@users.noreply.github.com> Date: Mon, 4 Nov 2024 06:29:36 +0900 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=91=8D=EF=B8=8F[Update]=20Model=20:?= =?UTF-8?q?=20rename=20osc=5Ftools.py=20->=20osc.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src-python/model.py | 2 +- src-python/models/osc/{osc_tools.py => osc.py} | 0 src-python/webui_controller.py | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) rename src-python/models/osc/{osc_tools.py => osc.py} (100%) diff --git a/src-python/model.py b/src-python/model.py index 9ad5d3e0..6dbcf10a 100644 --- a/src-python/model.py +++ b/src-python/model.py @@ -17,7 +17,7 @@ from device_manager import device_manager from config import config from models.translation.translation_translator import Translator -from models.osc.osc_tools import OSCHandler +from models.osc.osc import OSCHandler from models.transcription.transcription_recorder import SelectedMicEnergyAndAudioRecorder, SelectedSpeakerEnergyAndAudioRecorder from models.transcription.transcription_recorder import SelectedMicEnergyRecorder, SelectedSpeakerEnergyRecorder from models.transcription.transcription_transcriber import AudioTranscriber diff --git a/src-python/models/osc/osc_tools.py b/src-python/models/osc/osc.py similarity index 100% rename from src-python/models/osc/osc_tools.py rename to src-python/models/osc/osc.py diff --git a/src-python/webui_controller.py b/src-python/webui_controller.py index eb900729..ff688291 100644 --- a/src-python/webui_controller.py +++ b/src-python/webui_controller.py @@ -1069,6 +1069,7 @@ class Controller: @staticmethod def setOscIpAddress(data, *args, **kwargs) -> dict: config.OSC_IP_ADDRESS = data + model.setOscIpAddress(config.OSC_IP_ADDRESS) return {"status":200, "result":config.OSC_IP_ADDRESS} @staticmethod @@ -1078,6 +1079,7 @@ class Controller: @staticmethod def setOscPort(data, *args, **kwargs) -> dict: config.OSC_PORT = int(data) + model.setOscPort(config.OSC_PORT) return {"status":200, "result":config.OSC_PORT} @staticmethod From 7350910c7b80404bd505bd8deb331c25545e9047 Mon Sep 17 00:00:00 2001 From: misyaguziya <53165965+misyaguziya@users.noreply.github.com> Date: Mon, 4 Nov 2024 22:44:01 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=91=8D=EF=B8=8F[Update]=20Model=20:?= =?UTF-8?q?=20=E3=83=9F=E3=83=A5=E3=83=BC=E3=83=88=E5=90=8C=E6=9C=9F?= =?UTF-8?q?=E3=81=AE=E5=87=A6=E7=90=86=E3=82=92=E8=A6=8B=E7=9B=B4=E3=81=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src-python/model.py | 33 +++++++++------------------------ src-python/models/osc/osc.py | 2 +- src-python/webui_controller.py | 5 ++--- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/src-python/model.py b/src-python/model.py index 6dbcf10a..ec23039d 100644 --- a/src-python/model.py +++ b/src-python/model.py @@ -97,7 +97,6 @@ class Model: self.th_overlay = None self.mic_audio_queue = None self.mic_mute_status = None - self.mic_mute_status_check = None self.kks = kakasi() self.watchdog = Watchdog(config.WATCHDOG_TIMEOUT, config.WATCHDOG_INTERVAL) self.osc_handler = OSCHandler(config.OSC_IP_ADDRESS, config.OSC_PORT) @@ -305,32 +304,18 @@ class Model: def getMuteSelfStatus(self): return self.osc_handler.getOSCParameterMuteSelf() - def startCheckMuteSelfStatus(self): - def checkMuteSelfStatus(): - status = self.getMuteSelfStatus() - if status is not None: - self.mic_mute_status = status - self.changeMicTranscriptStatus() - self.stopCheckMuteSelfStatus() - - if not isinstance(self.mic_mute_status_check, threadFnc): - self.mic_mute_status_check = threadFnc(checkMuteSelfStatus) - self.mic_mute_status_check.daemon = True - self.mic_mute_status_check.start() - - def stopCheckMuteSelfStatus(self): - if isinstance(self.mic_mute_status_check, threadFnc): - self.mic_mute_status_check.stop() - self.mic_mute_status_check = None + def setMuteSelfStatus(self): + self.mic_mute_status = self.getMuteSelfStatus() def startReceiveOSC(self): def changeHandlerMute(address, osc_arguments): - if osc_arguments is True and self.mic_mute_status is False: - self.mic_mute_status = osc_arguments - self.changeMicTranscriptStatus() - elif osc_arguments is False and self.mic_mute_status is True: - self.mic_mute_status = osc_arguments - self.changeMicTranscriptStatus() + if config.ENABLE_TRANSCRIPTION_SEND is True: + if osc_arguments is True and self.mic_mute_status is False: + self.mic_mute_status = osc_arguments + self.changeMicTranscriptStatus() + elif osc_arguments is False and self.mic_mute_status is True: + self.mic_mute_status = osc_arguments + self.changeMicTranscriptStatus() dict_filter_and_target = { self.osc_handler.osc_parameter_muteself: changeHandlerMute, diff --git a/src-python/models/osc/osc.py b/src-python/models/osc/osc.py index 9b021f1d..f01ce3cd 100644 --- a/src-python/models/osc/osc.py +++ b/src-python/models/osc/osc.py @@ -75,7 +75,7 @@ class OSCHandler: self.osc_query_service.advertise_endpoint(filter, access=OSCAccess.READWRITE_VALUE) def oscServerServe(self) -> None: - self.osc_server.serve_forever(100) + self.osc_server.serve_forever(2) def oscServerStop(self) -> None: if isinstance(self.osc_server, osc_server.ThreadingOSCUDPServer): diff --git a/src-python/webui_controller.py b/src-python/webui_controller.py index ff688291..c35c5743 100644 --- a/src-python/webui_controller.py +++ b/src-python/webui_controller.py @@ -1303,14 +1303,13 @@ class Controller: @staticmethod def setEnableVrcMicMuteSync(*args, **kwargs) -> dict: config.VRC_MIC_MUTE_SYNC = True - model.startCheckMuteSelfStatus() + model.setMuteSelfStatus() model.changeMicTranscriptStatus() return {"status":200, "result":config.VRC_MIC_MUTE_SYNC} @staticmethod def setDisableVrcMicMuteSync(*args, **kwargs) -> dict: config.VRC_MIC_MUTE_SYNC = False - model.stopCheckMuteSelfStatus() model.changeMicTranscriptStatus() return {"status":200, "result":config.VRC_MIC_MUTE_SYNC} @@ -1725,7 +1724,7 @@ class Controller: printLog("Init OSC Receive") model.startReceiveOSC() if config.VRC_MIC_MUTE_SYNC is True: - model.startCheckMuteSelfStatus() + self.setEnableVrcMicMuteSync() # init Auto device selection device_manager.setCallbackHostList(self.updateMicHostList)