every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

👍️[Update] Model : overlayの機能をSpeaker2Logで表示できるように変更

Browse Source
This commit is contained in:
misyaguziya
2024-03-27 10:39:24 +09:00
parent 14bb05ad4e
commit c3b2f08b9b
3 changed files with 129 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
controller.py
View File

@@ -146,6 +146,11 @@ def receiveSpeakerMessage(message):
xsoverlay_message = messageFormatter("RECEIVED", translation, message)
model.notificationXSOverlay(xsoverlay_message)
if model.th_overlay is None:
model.startOverlay()
overlay_image = model.createOverlayImage(message, translation)
model.setOverlayImage(overlay_image)
# ------------Speaker2Chatbox------------
if config.ENABLE_SPEAKER2CHATBOX is True:
# send OSC message

27
model.py
View File

@@ -26,6 +26,7 @@ from models.translation.translation_languages import translation_lang
from models.transcription.transcription_languages import transcription_lang
from models.translation.translation_utils import checkCTranslate2Weight
from models.transcription.transcription_whisper import checkWhisperWeight
from models.overlay.overlay import create_overlay_image, Overlay
from config import config
class threadFnc(Thread):
@@ -37,7 +38,7 @@ class threadFnc(Thread):
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
return self._stop.is_set()
def run(self):
while True:
if self.stopped():
@@ -67,6 +68,8 @@ class Model:
self.speaker_energy_plot_progressbar = None
self.translator = Translator()
self.keyword_processor = KeywordProcessor()
self.overlay = Overlay()
self.th_overlay = None
def checkCTranslatorCTranslate2ModelWeight(self):
return checkCTranslate2Weight(config.PATH_LOCAL, config.CTRANSLATE2_WEIGHT_TYPE)
@@ -533,4 +536,26 @@ class Model:
def notificationXSOverlay(self, message):
xsoverlayForVRCT(content=f"{message}")
def createOverlayImage(self, message, translation):
your_language = config.TARGET_LANGUAGE
target_language = config.SOURCE_LANGUAGE
return create_overlay_image(message, your_language, translation, target_language)
def setOverlayImage(self, img):
if self.overlay.initFlag is True:
self.overlay.uiMan.uiUpdate(img)
def startOverlay(self):
if self.overlay.initFlag is False:
self.overlay.init()
if self.overlay.initFlag is True:
self.th_overlay = threadFnc(self.overlay.startOverlay)
self.th_overlay.daemon = True
self.th_overlay.start()
def stopOverlay(self):
if isinstance(self.th_overlay, threadFnc):
self.th_overlay.stop()
self.th_overlay = None
model = Model()

128
models/overlay/overlay.py
View File

@@ -63,6 +63,8 @@ settings = {
"Normalised icon Y position": -0.41,
"Icon plane depth": 1,
"Normalised icon width": 1,
"Fade time": 5,
"Fade interval": 2,
}
def mat34Id():
@@ -73,7 +75,7 @@ def mat34Id():
return arr
class UIElement:
def __init__(self, overlayRoot, key, name, pos, img, flip = False) -> None:
def __init__(self, overlayRoot, key, name, pos, flip = False) -> None:
"""
pos is a 2-tuple representing (x, y) normalised position of the overlay on the screen
"""
@@ -84,12 +86,7 @@ class UIElement:
self.handle = self.overlay.createOverlay(self.overlayKey, self.overlayName)
# configure overlay appearance
width, height = img.size
img = img.tobytes()
img = (ctypes.c_char * len(img)).from_buffer_copy(img)
self.setImage(img, width, height) # blank image for default
self.setImage(Image.new("RGBA", (1, 1), (0, 0, 0, 0))) # blank image for default
self.setColour(settings['Colour'])
self.setTransparency(settings['Transparency'])
self.overlay.setOverlayWidthInMeters(
@@ -101,7 +98,11 @@ class UIElement:
self.overlay.showOverlay(self.handle)
def setImage(self, img, width, height):
def setImage(self, img):
# configure overlay appearance
width, height = img.size
img = img.tobytes()
img = (ctypes.c_char * len(img)).from_buffer_copy(img)
self.overlay.setOverlayRaw(self.handle, img, width, height, 4)
def setColour(self, col):
@@ -131,41 +132,108 @@ class UIElement:
)
class UIManager:
def __init__(self, img) -> None:
def __init__(self):
self.overlay = openvr.IVROverlay()
self.UI = UIElement(
self.overlayUI = UIElement(
self.overlay,
"VRCT",
"Receive UI Element",
(settings['Normalised icon X position'], settings['Normalised icon Y position']),
img,
)
self.lastUpdate = time.monotonic()
async def mainLoop(uiMan):
time.sleep(10)
def update(self):
currTime = time.monotonic()
if settings['Fade interval'] != 0:
self.evaluateTransparencyFade(self.overlayUI, self.lastUpdate, currTime)
async def init_main(img):
uiMan = UIManager(img)
await mainLoop(uiMan)
def uiUpdate(self, img):
self.overlayUI.setImage(img)
self.overlayUI.setTransparency(settings['Transparency'])
self.lastUpdate = time.monotonic()
def evaluateTransparencyFade(self, ui, lastUpdate, currentTime):
if (currentTime - lastUpdate) > settings['Fade time']:
timeThroughInterval = currentTime - lastUpdate - settings['Fade time']
fadeRatio = 1 - timeThroughInterval / settings['Fade interval']
if fadeRatio < 0:
fadeRatio = 0
ui.setTransparency(fadeRatio * settings['Transparency'])
class Overlay:
def __init__(self):
openvr.init(openvr.VRApplication_Overlay)
self.initFlag = False
def plot_overlay(message, your_language="Japanese", translation=None, target_language=None):
img = create_overlay_image(message, your_language, translation, target_language)
UIManager(img)
def checkHMD(self):
return openvr.isHmdPresent()
def checkRuntime(self):
return openvr.isRuntimeInstalled()
def init(self):
try:
openvr.init(openvr.VRApplication_Overlay)
self.initFlag = True
except Exception as e:
print("Could not initialise OpenVR")
async def mainLoop(self):
while True:
startTime = time.monotonic()
self.uiMan.update()
sleepTime = (1 / 60) - (time.monotonic() - startTime)
if sleepTime > 0:
await asyncio.sleep(sleepTime)
async def init_main(self):
self.uiMan = UIManager()
await self.mainLoop()
def startOverlay(self):
asyncio.run(self.init_main())
if __name__ == '__main__':
message = "Hello,World!Goodbye こんにちは、世界！さようなら안녕하세요, 세계!안녕 你好世界！再见 你好世界！再見 Hello,World!Goodbye こんにちは、世界！さようなら안녕하세요, 세계!안녕 你好世界！再见 你好世界！再見 Hello,World!Goodbye こんにちは、世界！さようなら안녕하세요, 세계!안녕 你好世界！再见 你好世界！再見"
translation = "Hello,World!Goodbye こんにちは、世界！さようなら안녕하세요, 세계!안녕 你好世界！再见 你好世界！再見 Hello,World!Goodbye こんにちは、世界！さようなら안녕하세요, 세계!안녕 你好世界！再见 你好世界！再見 Hello,World!Goodbye こんにちは、世界！さようなら안녕하세요, 세계!안녕 你好世界！再见 你好世界！再見"
your_language = "Japanese"
target_language = "Chinese Simplified"
# font_family = "NotoSansJP-Regular"
# img = create_text_image(message, font_family)
from threading import Thread, Event
class threadFnc(Thread):
def __init__(self, fnc, end_fnc=None, daemon=True, *args, **kwargs):
super(threadFnc, self).__init__(daemon=daemon, *args, **kwargs)
self.fnc = fnc
self.end_fnc = end_fnc
self._stop = Event()
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.is_set()
def run(self):
while True:
if self.stopped():
if callable(self.end_fnc):
self.end_fnc()
return
self.fnc(*self._args, **self._kwargs)
# openvr.init(openvr.VRApplication_Overlay)
# asyncio.run(init_main(img))
o = Overlay()
o.plot_overlay(message, your_language, translation, target_language)
overlay = Overlay()
if overlay.initFlag is False:
overlay.init()
if overlay.initFlag is True:
t = threadFnc(overlay.startOverlay)
t.start()
img = create_overlay_image("こんにちは、世界！さようなら", "Japanese", "Hello,World!Goodbye", "Japanese")
if overlay.initFlag is True:
overlay.uiMan.uiUpdate(img)
time.sleep(10)
img = create_overlay_image("こんにちは、世界！さようなら", "Japanese", "안녕하세요, 세계!안녕", "Korean")
if overlay.initFlag is True:
overlay.uiMan.uiUpdate(img)
time.sleep(10)
img = create_overlay_image("こんにちは、世界！さようなら", "Japanese", "你好世界！再见", "Chinese Simplified")
if overlay.initFlag is True:
overlay.uiMan.uiUpdate(img)
time.sleep(10)