every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

👍️[Update] Model : OSC の処理を見直し

Browse Source
This commit is contained in:
misyaguziya
2024-11-04 06:27:08 +09:00
parent f380616bdb
commit e21661df1e
4 changed files with 108 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
requirements.txt
View File

@@ -4,7 +4,7 @@ ctranslate2==4.3.1
transformers==4.40.2
pillow == 10.0.0
PyAudioWPatch == 0.2.12.6
python-osc == 1.8.3
python-osc == 1.9.0
deepl == 1.15.0
flashtext ==2.7
pyinstaller==6.10.0
@@ -17,4 +17,4 @@ pykakasi==2.3.0
pycaw==20240210
translators @ git+https://github.com/misyaguziya/translators@5.9.2.1
SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4.1
tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.2
tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.3

4
requirements_cuda.txt
View File

@@ -5,7 +5,7 @@ ctranslate2==4.3.1
transformers==4.40.2
pillow == 10.0.0
PyAudioWPatch == 0.2.12.6
python-osc == 1.8.3
python-osc == 1.9.0
deepl == 1.15.0
flashtext ==2.7
pyinstaller==6.10.0
@@ -18,4 +18,4 @@ pykakasi==2.3.0
pycaw==20240210
translators @ git+https://github.com/misyaguziya/translators@5.9.2.1
SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4.1
tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.2
tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.3

46
src-python/model.py
View File

@@ -17,7 +17,7 @@ from device_manager import device_manager
from config import config
from models.translation.translation_translator import Translator
from models.osc.osc_tools import sendTyping, sendMessage, receiveOscParameters, getOSCParameterValue
from models.osc.osc_tools import OSCHandler
from models.transcription.transcription_recorder import SelectedMicEnergyAndAudioRecorder, SelectedSpeakerEnergyAndAudioRecorder
from models.transcription.transcription_recorder import SelectedMicEnergyRecorder, SelectedSpeakerEnergyRecorder
from models.transcription.transcription_transcriber import AudioTranscriber
@@ -100,6 +100,7 @@ class Model:
self.mic_mute_status_check = None
self.kks = kakasi()
self.watchdog = Watchdog(config.WATCHDOG_TIMEOUT, config.WATCHDOG_INTERVAL)
self.osc_handler = OSCHandler(config.OSC_IP_ADDRESS, config.OSC_PORT)
def checkTranslatorCTranslate2ModelWeight(self, weight_type:str):
return checkCTranslate2Weight(config.PATH_LOCAL, weight_type)
@@ -286,28 +287,26 @@ class Model:
filtered_list.append(filtered_item)
return filtered_list
@staticmethod
def oscStartSendTyping():
sendTyping(True, config.OSC_IP_ADDRESS, config.OSC_PORT)
def setOscIpAddress(self, ip_address):
self.osc_handler.setOscIpAddress(ip_address)
@staticmethod
def oscStopSendTyping():
sendTyping(False, config.OSC_IP_ADDRESS, config.OSC_PORT)
def setOscPort(self, port):
self.osc_handler.setOscPort(port)
@staticmethod
def oscSendMessage(message):
sendMessage(message, config.OSC_IP_ADDRESS, config.OSC_PORT)
def oscStartSendTyping(self):
self.osc_handler.sendTyping(flag=True)
@staticmethod
def getMuteSelfStatus():
return getOSCParameterValue(address="/avatar/parameters/MuteSelf")
def oscStopSendTyping(self):
self.osc_handler.sendTyping(flag=False)
def oscSendMessage(self, message, notification=True):
self.osc_handler.sendMessage(message=message, notification=notification)
def getMuteSelfStatus(self):
return self.osc_handler.getOSCParameterMuteSelf()
def startCheckMuteSelfStatus(self):
def checkMuteSelfStatus():
if self.mic_mute_status is not None:
self.changeMicTranscriptStatus()
self.stopCheckMuteSelfStatus()
status = self.getMuteSelfStatus()
if status is not None:
self.mic_mute_status = status
@@ -325,10 +324,7 @@ class Model:
self.mic_mute_status_check = None
def startReceiveOSC(self):
osc_parameter_prefix = "/avatar/parameters/"
param_MuteSelf = "MuteSelf"
def change_handler_mute(address, osc_arguments):
def changeHandlerMute(address, osc_arguments):
if osc_arguments is True and self.mic_mute_status is False:
self.mic_mute_status = osc_arguments
self.changeMicTranscriptStatus()
@@ -337,12 +333,12 @@ class Model:
self.changeMicTranscriptStatus()
dict_filter_and_target = {
osc_parameter_prefix + param_MuteSelf: change_handler_mute,
self.osc_handler.osc_parameter_muteself: changeHandlerMute,
}
self.osc_handler.receiveOscParameters(dict_filter_and_target)
th_osc_server = Thread(target=receiveOscParameters, args=(dict_filter_and_target,))
th_osc_server.daemon = True
th_osc_server.start()
def stopReceiveOSC(self):
self.osc_handler.oscServerStop()
@staticmethod
def checkSoftwareUpdated():

180
src-python/models/osc/osc_tools.py
View File

@@ -1,112 +1,98 @@
import asyncio
from typing import Any
from time import sleep
from pythonosc import osc_message_builder, udp_client, dispatcher, osc_server
from threading import Thread
from pythonosc import udp_client, dispatcher, osc_server
from tinyoscquery.queryservice import OSCQueryService
from tinyoscquery.query import OSCQueryBrowser, OSCQueryClient
from tinyoscquery.utility import get_open_udp_port, get_open_tcp_port
from psutil import process_iter
from tinyoscquery.shared.node import OSCAccess
# send OSC message typing
def sendTyping(flag=False, ip_address="127.0.0.1", port=9000):
typing = osc_message_builder.OscMessageBuilder(address="/chatbox/typing")
typing.add_arg(flag)
b_typing = typing.build()
client = udp_client.SimpleUDPClient(ip_address, port)
client.send(b_typing)
class OSCHandler:
def __init__(self, ip_address="127.0.0.1", port=9000) -> None:
self.osc_ip_address = ip_address
self.osc_port = port
self.osc_parameter_muteself = "/avatar/parameters/MuteSelf"
self.osc_parameter_chatbox_typing = "/chatbox/typing"
self.osc_parameter_chatbox_input = "/chatbox/input"
self.udp_client = udp_client.SimpleUDPClient(self.osc_ip_address, self.osc_port)
self.osc_server_name = "VRChat-Client"
self.osc_server = None
self.osc_query_service = None
self.osc_query_service_name = "VRCT"
self.osc_server_ip_address = ip_address
self.http_port = None
self.osc_server_port = None
# send OSC message
def sendMessage(message=None, ip_address="127.0.0.1", port=9000):
if message is not None:
msg = osc_message_builder.OscMessageBuilder(address="/chatbox/input")
msg.add_arg(f"{message}")
msg.add_arg(True)
msg.add_arg(True)
b_msg = msg.build()
client = udp_client.SimpleUDPClient(ip_address, port)
client.send(b_msg)
def setOscIpAddress(self, ip_address:str) -> None:
self.osc_ip_address = ip_address
self.udp_client = udp_client.SimpleUDPClient(self.osc_ip_address, self.osc_port)
def sendTestAction(ip_address="127.0.0.1", port=9000):
client = udp_client.SimpleUDPClient(ip_address, port)
client.send_message("/input/Vertical", 1)
sleep(0.01)
client.send_message("/input/Vertical", False)
def setOscPort(self, port:int) -> None:
self.osc_port = port
self.udp_client = udp_client.SimpleUDPClient(self.osc_ip_address, self.osc_port)
# send Input Voice
def sendInputVoice(flag=False, ip_address="127.0.0.1", port=9000):
input_voice = osc_message_builder.OscMessageBuilder(address="/input/Voice")
input_voice.add_arg(flag)
b_input_voice = input_voice.build()
client = udp_client.SimpleUDPClient(ip_address, port)
client.send(b_input_voice)
# send OSC message typing
def sendTyping(self, flag:bool=False) -> None:
self.udp_client.send_message(self.osc_parameter_chatbox_typing, [flag])
def sendChangeVoice(ip_address="127.0.0.1", port=9000):
sendInputVoice(flag=0, ip_address=ip_address, port=port)
sleep(0.05)
sendInputVoice(flag=1, ip_address=ip_address, port=port)
sleep(0.05)
sendInputVoice(flag=0, ip_address=ip_address, port=port)
sleep(0.05)
# send OSC message
def sendMessage(self, message:str="", notification:bool=True) -> None:
if len(message) > 0:
self.udp_client.send_message(self.osc_parameter_chatbox_input, [f"{message}", True, notification])
def getOSCParameterValue(address, server_name="VRChat-Client"):
value = None
try:
browser = OSCQueryBrowser()
sleep(1)
service = browser.find_service_by_name(server_name)
if service is not None:
oscq = OSCQueryClient(service)
mute_self_node = oscq.query_node(address)
value = mute_self_node.value[0]
browser.zc.close()
browser.browser.cancel()
except Exception:
pass
return value
def checkVRChatRunning() -> bool:
_proc_name = "VRChat.exe"
return _proc_name in (p.name() for p in process_iter())
def receiveOscParameters(dict_filter_and_target, ip_address="127.0.0.1", title="VRCT"):
while True:
if not checkVRChatRunning():
def getOSCParameterValue(self, address:str) -> Any:
value = None
try:
browser = OSCQueryBrowser()
sleep(1)
else:
try:
osc_port = get_open_udp_port()
http_port = get_open_tcp_port()
osc_dispatcher = dispatcher.Dispatcher()
for filter, target in dict_filter_and_target.items():
osc_dispatcher.map(filter, target)
osc_udp_server = osc_server.ThreadingOSCUDPServer((ip_address, osc_port), osc_dispatcher)
service = browser.find_service_by_name(self.osc_server_name)
if service is not None:
osc_query_client = OSCQueryClient(service)
mute_self_node = osc_query_client.query_node(address)
value = mute_self_node.value[0]
browser.zc.close()
browser.browser.cancel()
osc_client = OSCQueryService(title, http_port, osc_port)
for filter, target in dict_filter_and_target.items():
osc_client.advertise_endpoint(filter)
except Exception:
pass
return value
osc_udp_server.serve_forever()
except Exception:
pass
def getOSCParameterMuteSelf(self) -> bool:
return self.getOSCParameterValue(self.osc_parameter_muteself)
def receiveOscParameters(self, dict_filter_and_target:dict) -> None:
self.osc_server_port = get_open_udp_port()
self.http_port = get_open_tcp_port()
osc_dispatcher = dispatcher.Dispatcher()
for filter, target in dict_filter_and_target.items():
osc_dispatcher.map(filter, target)
self.osc_server = osc_server.ThreadingOSCUDPServer((self.osc_server_ip_address, self.osc_server_port), osc_dispatcher, asyncio.get_event_loop())
Thread(target=self.oscServerServe, daemon=True).start()
self.osc_query_service = OSCQueryService(self.osc_query_service_name, self.http_port, self.osc_server_port)
for filter, target in dict_filter_and_target.items():
self.osc_query_service.advertise_endpoint(filter, access=OSCAccess.READWRITE_VALUE)
def oscServerServe(self) -> None:
self.osc_server.serve_forever(100)
def oscServerStop(self) -> None:
if isinstance(self.osc_server, osc_server.ThreadingOSCUDPServer):
self.osc_server.shutdown()
self.osc_server = None
if isinstance(self.osc_query_service, OSCQueryService):
self.osc_query_service.http_server.shutdown()
self.osc_query_service = None
if __name__ == "__main__":
osc_parameter_prefix = "/avatar/parameters/"
osc_avatar_change_path = "/avatar/change"
param_MuteSelf = "MuteSelf"
param_Voice = "Voice"
def print_handler_all(address, *args):
print(f"all {address}: {args}")
def print_handler_muteself(address, *args):
print(f"muteself {address}: {args}")
def print_handler_voice(address, *args):
print(f"voice {address}: {args}")
dict_filter_and_target = {
# osc_parameter_prefix + "*": print_handler_all,
osc_parameter_prefix + param_MuteSelf: print_handler_muteself,
osc_parameter_prefix + param_Voice: print_handler_voice,
}
receiveOscParameters(dict_filter_and_target)
handler = OSCHandler()
handler.receiveOscParameters({
"/avatar/parameters/MuteSelf": print,
})
sleep(5)
handler.sendTyping(True)
sleep(1)
handler.sendMessage(message="Hello World", notification=True)
sleep(60)
handler.oscServerStop()