diff --git a/controller.py b/controller.py index 329f0fc3..9447cadf 100644 --- a/controller.py +++ b/controller.py @@ -164,11 +164,10 @@ def receiveSpeakerMessage(message): if config.ENABLE_OVERLAY_SMALL_LOG is True: if model.overlay.initialized is False: model.startOverlay() - if model.overlay.initialized is True: - overlay_image = model.createOverlayImageShort(message, translation) - model.updateOverlay(overlay_image) - # overlay_image = model.createOverlayImageLong("receive", message, translation) - # model.updateOverlay(overlay_image) + overlay_image = model.createOverlayImageShort(message, translation) + model.updateOverlay(overlay_image) + # overlay_image = model.createOverlayImageLong("receive", message, translation) + # model.updateOverlay(overlay_image) # ------------Speaker2Chatbox------------ if config.ENABLE_SPEAKER2CHATBOX is True: @@ -191,6 +190,7 @@ def startTranscriptionReceiveMessage(): def stopTranscriptionReceiveMessage(): model.stopSpeakerTranscript() + model.shutdownOverlay() view.setMainWindowAllWidgetsStatusToNormal() def startThreadingTranscriptionReceiveMessage(): @@ -874,11 +874,8 @@ def callbackSetEnableOverlaySmallLog(value): print("callbackSetEnableOverlaySmallLog", value) config.ENABLE_OVERLAY_SMALL_LOG = value - if config.ENABLE_OVERLAY_SMALL_LOG is True: - pass - else: - if model.overlay.initialized is True: - model.clearOverlayImage() + if config.ENABLE_OVERLAY_SMALL_LOG is False: + model.clearOverlayImage() def callbackSetOverlaySmallLogSettings(value, set_type:str): print("callbackSetOverlaySmallLogSettings", value, set_type) diff --git a/model.py b/model.py index 8095bd94..e18a05eb 100644 --- a/model.py +++ b/model.py @@ -26,7 +26,7 @@ from models.translation.translation_languages import translation_lang from models.transcription.transcription_languages import transcription_lang from models.translation.translation_utils import checkCTranslate2Weight from models.transcription.transcription_whisper import checkWhisperWeight -from models.overlay.overlay import Overlay +from models.overlay.overlay_2 import Overlay from models.overlay.overlay_image import OverlayImage from config import config @@ -603,46 +603,35 @@ class Model: # return self.overlay_image.create_overlay_image_long(message_type, message, your_language, translation, target_language) def clearOverlayImage(self): - if self.overlay.initialized is True: - self.overlay.uiManager.uiClear() + self.overlay.clearImage() def updateOverlay(self, img): - if self.overlay.initialized is True: - self.overlay.uiManager.uiUpdate(img) + self.overlay.setImage(img) def startOverlay(self): - if self.overlay.initialized is False: - self.overlay.init() - - if self.overlay.initialized is True and self.th_overlay is None: - self.th_overlay = Thread(target=self.overlay.startOverlay) - self.th_overlay.daemon = True - self.th_overlay.start() + self.overlay.startOverlay() def updateOverlayPosition(self): - if self.overlay.initialized is True: - pos = (config.OVERLAY_SMALL_LOG_SETTINGS["x_pos"], config.OVERLAY_SMALL_LOG_SETTINGS["y_pos"]) - self.overlay.uiManager.setPosition(pos) - depth = config.OVERLAY_SMALL_LOG_SETTINGS["depth"] - self.overlay.uiManager.setDepth(depth) - self.overlay.uiManager.posUpdate() + pos = (config.OVERLAY_SMALL_LOG_SETTINGS["x_pos"], config.OVERLAY_SMALL_LOG_SETTINGS["y_pos"]) + self.overlay.setPosition(pos) + depth = config.OVERLAY_SMALL_LOG_SETTINGS["depth"] + self.overlay.setDepth(depth) def updateOverlayTimes(self): - if self.overlay.initialized is True: - display_duration = config.OVERLAY_SMALL_LOG_SETTINGS["display_duration"] - self.overlay.uiManager.setFadeTime(display_duration) - fadeout_duration = config.OVERLAY_SMALL_LOG_SETTINGS["fadeout_duration"] - self.overlay.uiManager.setFadeInterval(fadeout_duration) - self.overlay.uiManager.update() + display_duration = config.OVERLAY_SMALL_LOG_SETTINGS["display_duration"] + self.overlay.setFadeTime(display_duration) + fadeout_duration = config.OVERLAY_SMALL_LOG_SETTINGS["fadeout_duration"] + self.overlay.setFadeInterval(fadeout_duration) def updateOverlayImageOpacity(self): - if self.overlay.initialized is True: - opacity = config.OVERLAY_SETTINGS["opacity"] - self.overlay.uiManager.setTransparency(opacity) + opacity = config.OVERLAY_SETTINGS["opacity"] + self.overlay.setTransparency(opacity) def updateOverlayImageUiScaling(self): - if self.overlay.initialized is True: - ui_scaling = config.OVERLAY_SETTINGS["ui_scaling"] - self.overlay.uiManager.setUiScaling(ui_scaling) + ui_scaling = config.OVERLAY_SETTINGS["ui_scaling"] + self.overlay.setUiScaling(ui_scaling) + + def shutdownOverlay(self): + self.overlay.shutdown() model = Model() \ No newline at end of file diff --git a/models/overlay/overlay.py b/models/overlay/overlay.py index 87cd12b1..a2787151 100644 --- a/models/overlay/overlay.py +++ b/models/overlay/overlay.py @@ -229,7 +229,7 @@ if __name__ == '__main__': return self.fnc(*self._args, **self._kwargs) - overlay = Overlay() + overlay = Overlay(0, 0, 1, 1, 1, 1, 1) overlay_image = OverlayImage() if overlay.initialized is False: diff --git a/models/overlay/overlay_2.py b/models/overlay/overlay_2.py new file mode 100644 index 00000000..7462f365 --- /dev/null +++ b/models/overlay/overlay_2.py @@ -0,0 +1,232 @@ +import ctypes +import psutil +import asyncio +import ctypes +import time +import openvr +from PIL import Image +from queue import Queue +from threading import Thread + +def checkSteamvrRunning() -> bool: + for proc in psutil.process_iter(): + if "vrserver.exe" == proc.name().lower(): + return True + return False + +def mat34Id(): + arr = openvr.HmdMatrix34_t() + arr[0][0] = 1 + arr[1][1] = 1 + arr[2][2] = 1 + return arr + +class Overlay: + def __init__(self, x, y , depth, fade_time, fade_interval, transparency, ui_scaling): + self.initialized = False + settings = { + "Color": [1, 1, 1], + "Transparency": transparency, + "Normalized_icon_X_position": x, + "Normalized_icon_Y_position": y, + "Icon_plane_depth": depth, + "Fade_time": fade_time, + "Fade_interval": fade_interval, + "Ui_scaling": ui_scaling, + } + self.settings = settings + self.system = None + self.overlay = None + self.handle = None + self.image_queue = Queue() + self.lastUpdate = time.monotonic() + self.thread_overlay = None + + def init(self): + try: + if checkSteamvrRunning() is True: + self.system = openvr.init(openvr.VRApplication_Background) + self.overlay = openvr.IVROverlay() + self.handle = self.overlay.createOverlay("Overlay_Speaker2log", "SOverlay_Speaker2log_UI") + + self.setImage(Image.new("RGBA", (1, 1), (0, 0, 0, 0))) + self.updateImage() + self.setColor(self.settings['Color']) + self.updateColor() + self.setTransparency(self.settings['Transparency']) + self.updateTransparency() + self.setUiScaling(self.settings['Ui_scaling']) + self.updateUiScaling() + self.setPosition((self.settings["Normalized_icon_X_position"], self.settings["Normalized_icon_Y_position"])) + self.updatePosition() + self.overlay.showOverlay(self.handle) + self.initialized = True + except Exception as e: + print("Could not initialise OpenVR", e) + + def setImage(self, img): + self.image_queue.put(img) + + def updateImage(self): + img = self.image_queue.get() + width, height = img.size + img = img.tobytes() + img = (ctypes.c_char * len(img)).from_buffer_copy(img) + self.overlay.setOverlayRaw(self.handle, img, width, height, 4) + + def clearImage(self): + while self.image_queue.empty() is False: + self.image_queue.get() + self.setImage(Image.new("RGBA", (1, 1), (0, 0, 0, 0))) + + def setColor(self, col): + """ + col is a 3-tuple representing (r, g, b) + """ + self.settings["Color"] = col + + def updateColor(self): + r, g, b = self.settings["Color"] + self.overlay.setOverlayColor(self.handle, r, g, b) + + def setTransparency(self, transparency): + self.settings['Transparency'] = transparency + + def updateTransparency(self): + self.overlay.setOverlayAlpha(self.handle, self.settings['Transparency']) + + def setUiScaling(self, ui_scaling): + self.settings['Ui_scaling'] = ui_scaling + + def updateUiScaling(self): + self.overlay.setOverlayWidthInMeters(self.handle, self.settings['Ui_scaling']) + + def setPosition(self, pos): + """ + pos is a 2-tuple representing normalized (x, y) + """ + self.settings["Normalized_icon_X_position"] = pos[0] + self.settings["Normalized_icon_Y_position"] = pos[1] + + def setDepth(self, depth): + self.settings["Icon_plane_depth"] = depth + + def updatePosition(self): + self.transform = mat34Id() # no rotation required for HMD attachment + + # assign position + self.transform[0][3] = self.settings["Normalized_icon_X_position"] * self.settings['Icon_plane_depth'] + self.transform[1][3] = self.settings["Normalized_icon_Y_position"] * self.settings['Icon_plane_depth'] + self.transform[2][3] = - self.settings['Icon_plane_depth'] + + self.overlay.setOverlayTransformTrackedDeviceRelative( + self.handle, + openvr.k_unTrackedDeviceIndex_Hmd, + self.transform + ) + + def setFadeTime(self, fade_time): + self.settings['Fade_time'] = fade_time + + def setFadeInterval(self, fade_interval): + self.settings['Fade_interval'] = fade_interval + + def checkActive(self): + try: + if self.system is not None and self.initialized is True: + new_event = openvr.VREvent_t() + while self.system.pollNextEvent(new_event): + if new_event.eventType == openvr.VREvent_Quit: + return False + return True + except Exception as e: + print("Could not check SteamVR running") + print(e) + return False + + def evaluateTransparencyFade(self, lastUpdate, currentTime): + if (currentTime - lastUpdate) > self.settings['Fade_time']: + timeThroughInterval = currentTime - lastUpdate - self.settings['Fade_time'] + self.fadeRatio = 1 - timeThroughInterval / self.settings['Fade_interval'] + if self.fadeRatio < 0: + self.fadeRatio = 0 + self.overlay.setOverlayAlpha(self.handle, self.fadeRatio * self.settings['Transparency']) + + def update(self): + if self.image_queue.empty() is False: + self.updateImage() + self.updateTransparency() + self.lastUpdate = time.monotonic() + + self.updateUiScaling() + self.updatePosition() + + currTime = time.monotonic() + if self.settings['Fade_interval'] != 0: + self.evaluateTransparencyFade(self.lastUpdate, currTime) + else: + self.updateTransparency() + + async def mainloop(self): + while self.checkActive() is True: + startTime = time.monotonic() + self.update() + sleepTime = (1 / 60) - (time.monotonic() - startTime) + if sleepTime > 0: + await asyncio.sleep(sleepTime) + + async def initMain(self): + await self.mainloop() + + def main(self): + self.init() + if self.initialized is True: + asyncio.run(self.initMain()) + + def startOverlay(self): + self.thread_overlay = Thread(target=self.main) + self.thread_overlay.daemon = True + self.thread_overlay.start() + + def shutdown(self): + if self.thread_overlay is not None: + ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(self.thread_overlay.ident), ctypes.py_object(SystemExit)) + if self.overlay is not None: + self.overlay.destroyOverlay(self.handle) + if self.system is not None: + openvr.shutdown() + self.initialized = False + +if __name__ == '__main__': + import threading + from overlay_image import OverlayImage + + overlay_image = OverlayImage() + overlay = Overlay(0, 0, 1, 1, 1, 1, 1) + thread = threading.Thread(target=overlay.startOverlay) + thread.start() + + for i in range(10): + print(f"time sleep {i}s") + time.sleep(1) + + # Example usage + img = overlay_image.createOverlayImageShort("こんにちは、世界！さようなら", "Japanese", "Hello,World!Goodbye", "Japanese", ui_type="sakura") + overlay.setImage(img) + for i in range(10): + overlay.setPosition((i/10, i/10)) + time.sleep(0.1) + + img = overlay_image.createOverlayImageShort("こんにちは、世界！さようなら", "Japanese", "Hello,World!Goodbye", "Japanese") + overlay.setImage(img) + + for i in range(10): + overlay.setPosition((i/10, i/10)) + time.sleep(0.1) + + time.sleep(10) + + overlay.shutdown() + for i in range(10): + print(f"time sleep {i}s") + time.sleep(1)