From 34c81cec18304abd098a60f9179ba6e53217996f Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Fri, 29 Mar 2024 23:34:31 +0900 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=8D=EF=B8=8F[Update]=20Model=20:=20ove?= =?UTF-8?q?rlay=E6=A9=9F=E8=83=BD=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 画像生成をoverlay_img.pyにまとめた --- config.py | 12 +++ controller.py | 22 +++- model.py | 14 ++- models/overlay/overlay.py | 135 ++++++++----------------- models/overlay/overlay_image.py | 174 ++++++++++++++++++++++++++++++++ 5 files changed, 258 insertions(+), 99 deletions(-) create mode 100644 models/overlay/overlay_image.py diff --git a/config.py b/config.py index 8c7d8f85..931c4df6 100644 --- a/config.py +++ b/config.py @@ -716,6 +716,17 @@ class Config: self._ENABLE_NOTICE_XSOVERLAY = value saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value) + @property + @json_serializable('ENABLE_NOTICE_OVERLAY') + def ENABLE_NOTICE_OVERLAY(self): + return self._ENABLE_NOTICE_OVERLAY + + @ENABLE_NOTICE_OVERLAY.setter + def ENABLE_NOTICE_OVERLAY(self, value): + if isinstance(value, bool): + self._ENABLE_NOTICE_OVERLAY = value + saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value) + @property @json_serializable('ENABLE_SEND_MESSAGE_TO_VRC') def ENABLE_SEND_MESSAGE_TO_VRC(self): @@ -979,6 +990,7 @@ class Config: self._ENABLE_SEND_ONLY_TRANSLATED_MESSAGES = False self._SEND_MESSAGE_BUTTON_TYPE = "show" self._ENABLE_NOTICE_XSOVERLAY = False + self._ENABLE_NOTICE_OVERLAY = False self._ENABLE_SEND_MESSAGE_TO_VRC = True self._ENABLE_SEND_RECEIVED_MESSAGE_TO_VRC = False # Speaker2Chatbox self._ENABLE_SPEAKER2CHATBOX_PASS = "000000000" diff --git a/controller.py b/controller.py index cc8e9782..d8f1deee 100644 --- a/controller.py +++ b/controller.py @@ -94,6 +94,12 @@ def sendMicMessage(message): translation = f" ({translation})" model.logger.info(f"[SENT] {message}{translation}") + # if config.ENABLE_NOTICE_OVERLAY is True: + # overlay_image = model.createOverlayImageShort(message, translation) + # model.setOverlayImage(overlay_image) + # overlay_image = model.createOverlayImageLong("send", message, translation) + # model.setOverlayImage(overlay_image) + def startTranscriptionSendMessage(): model.startMicTranscript(sendMicMessage, view.printToTextbox_TranscriptionSendNoDeviceError) view.setMainWindowAllWidgetsStatusToNormal() @@ -148,8 +154,12 @@ def receiveSpeakerMessage(message): if model.th_overlay is None: model.startOverlay() - overlay_image = model.createOverlayImage(message, translation) - model.setOverlayImage(overlay_image) + + if config.ENABLE_NOTICE_OVERLAY is True: + overlay_image = model.createOverlayImageShort(message, translation) + model.setOverlayImage(overlay_image) + # overlay_image = model.createOverlayImageLong("receive", message, translation) + # model.setOverlayImage(overlay_image) # ------------Speaker2Chatbox------------ if config.ENABLE_SPEAKER2CHATBOX is True: @@ -157,7 +167,7 @@ def receiveSpeakerMessage(message): if config.ENABLE_SEND_RECEIVED_MESSAGE_TO_VRC is True: osc_message = messageFormatter("RECEIVED", translation, message) model.oscSendMessage(osc_message) - # ------------Speaker2Chatbox------------ + # ------------Speaker2Chatbox------------ # update textbox message log (Received) view.printToTextbox_ReceivedMessage(message, translation) @@ -226,6 +236,12 @@ def sendChatMessage(message): osc_message = messageFormatter("SEND", translation, message) model.oscSendMessage(osc_message) + # if config.ENABLE_NOTICE_OVERLAY is True: + # overlay_image = model.createOverlayImageShort(message, translation) + # model.setOverlayImage(overlay_image) + # overlay_image = model.createOverlayImageLong("send", message, translation) + # model.setOverlayImage(overlay_image) + # update textbox message log (Sent) view.printToTextbox_SentMessage(message, translation) if config.ENABLE_LOGGER is True: diff --git a/model.py b/model.py index 4216a23e..fda9c7a6 100644 --- a/model.py +++ b/model.py @@ -26,7 +26,9 @@ from models.translation.translation_languages import translation_lang from models.transcription.transcription_languages import transcription_lang from models.translation.translation_utils import checkCTranslate2Weight from models.transcription.transcription_whisper import checkWhisperWeight -from models.overlay.overlay import create_overlay_image, Overlay +from models.overlay.overlay import Overlay +from models.overlay.overlay_image import OverlayImage + from config import config class threadFnc(Thread): @@ -69,6 +71,7 @@ class Model: self.translator = Translator() self.keyword_processor = KeywordProcessor() self.overlay = Overlay() + self.overlay_image = OverlayImage() self.th_overlay = None def checkCTranslatorCTranslate2ModelWeight(self): @@ -536,10 +539,15 @@ class Model: def notificationXSOverlay(self, message): xsoverlayForVRCT(content=f"{message}") - def createOverlayImage(self, message, translation): + def createOverlayImageShort(self, message, translation): your_language = config.TARGET_LANGUAGE target_language = config.SOURCE_LANGUAGE - return create_overlay_image(message, your_language, translation, target_language) + return self.overlay_image.create_overlay_image_short(message, your_language, translation, target_language) + + def createOverlayImageLong(self, message_type, message, translation): + your_language = config.TARGET_LANGUAGE if message_type == "receive" else config.SOURCE_LANGUAGE + target_language = config.SOURCE_LANGUAGE if message_type == "receive" else config.TARGET_LANGUAGE + return self.overlay_image.create_overlay_image_long(message_type, message, your_language, translation, target_language) def setOverlayImage(self, img): if self.overlay.initFlag is True: diff --git a/models/overlay/overlay.py b/models/overlay/overlay.py index 5861695c..1bb8d89d 100644 --- a/models/overlay/overlay.py +++ b/models/overlay/overlay.py @@ -2,71 +2,10 @@ import ctypes import time import asyncio import openvr -from PIL import Image, ImageDraw, ImageFont -from os import path as os_path - -def create_overlay_image(message, your_language, translation=None, target_language=None): - width, height = (1920, 46) - background_color = (54, 63, 77) - text_color = (223, 223, 224) - font_size = 46 - languages = { - "Japanese" : "NotoSansJP-Regular", - "Korean" : "NotoSansKR-Regular", - "Chinese Simplified" : "NotoSansSC-Regular", - "Chinese Traditional" : "NotoSansTC-Regular", - } - - def get_concat_v(im1, im2): - dst = Image.new('RGBA', (im1.width, im1.height + im2.height)) - dst.paste(im1, (0, 0)) - dst.paste(im2, (0, im1.height)) - return dst - - def create_textbox(text, language): - font_family = languages.get(language, "NotoSansJP-Regular") - img = Image.new("RGBA", (width, height), (0, 0, 0, 0)) - draw = ImageDraw.Draw(img) - font = ImageFont.truetype(os_path.join(os_path.dirname(__file__), "fonts", f"{font_family}.ttf"), font_size) - text_width = draw.textlength(text, font) - character_width = text_width // len(text) - character_line_num = int((width - 40) // character_width) - if len(text) > character_line_num: - text = "\n".join([text[i:i+character_line_num] for i in range(0, len(text), character_line_num)]) - text_height = font_size * (len(text.split("\n")) + 1) + 20 - img = Image.new("RGBA", (width, text_height), (0, 0, 0, 0)) - draw = ImageDraw.Draw(img) - # draw.rounded_rectangle([(0, 0), (width, text_height)], radius=30, fill=background_color, outline=background_color, width=5) - - text_x = width // 2 - text_y = text_height // 2 - draw.text((text_x, text_y), text, text_color, anchor="mm", stroke_width=0, font=font, align="center") - return img - - img = create_textbox(message, your_language) - if translation is not None and target_language is not None: - translation_img = create_textbox(translation, target_language) - img = get_concat_v(img, translation_img) - - width, height = img.size - background = Image.new("RGBA", (width, height), (0, 0, 0, 0)) - draw = ImageDraw.Draw(background) - draw.rounded_rectangle([(0, 0), (width, height)], radius=30, fill=background_color, outline=background_color, width=5) - img = Image.alpha_composite(background, img) - - return img - -settings = { - "Colour": [1, 1, 1], - "Transparency": 1, - "Normalised icon X position": 0.0, - "Normalised icon Y position": -0.41, - "Icon plane depth": 1, - "Normalised icon width": 1, - "Fade time": 5, - "Fade interval": 2, -} +from PIL import Image +# This code is based on the following source: +# [GOpy](https://github.com/MeroFune/GOpy) def mat34Id(): arr = openvr.HmdMatrix34_t() arr[0][0] = 1 @@ -75,27 +14,26 @@ def mat34Id(): return arr class UIElement: - def __init__(self, overlayRoot, key, name, pos, flip = False) -> None: + def __init__(self, overlayRoot, key: str, name: str, settings: dict = None) -> None: """ pos is a 2-tuple representing (x, y) normalised position of the overlay on the screen """ self.overlay = overlayRoot self.overlayKey = key self.overlayName = name - self.flip = flip - + self.settings = settings + pos = (self.settings['Normalised icon X position'], self.settings['Normalised icon Y position']) self.handle = self.overlay.createOverlay(self.overlayKey, self.overlayName) self.setImage(Image.new("RGBA", (1, 1), (0, 0, 0, 0))) # blank image for default - self.setColour(settings['Colour']) - self.setTransparency(settings['Transparency']) + self.setColour(self.settings['Colour']) + self.setTransparency(self.settings['Transparency']) self.overlay.setOverlayWidthInMeters( self.handle, - settings['Normalised icon width'] * settings['Icon plane depth'] + self.settings['Normalised icon width'] * self.settings['Icon plane depth'] ) self.setPosition(pos) - self.overlay.showOverlay(self.handle) def setImage(self, img): @@ -121,9 +59,9 @@ class UIElement: self.transform = mat34Id() # no rotation required for HMD attachment # assign position - self.transform[0][3] = pos[0] * settings['Icon plane depth'] - self.transform[1][3] = pos[1] * settings['Icon plane depth'] - self.transform[2][3] = -settings['Icon plane depth'] + self.transform[0][3] = pos[0] * self.settings['Icon plane depth'] + self.transform[1][3] = pos[1] * self.settings['Icon plane depth'] + self.transform[2][3] = - self.settings['Icon plane depth'] self.overlay.setOverlayTransformTrackedDeviceRelative( self.handle, @@ -132,45 +70,53 @@ class UIElement: ) class UIManager: - def __init__(self): + def __init__(self, settings): self.overlay = openvr.IVROverlay() - + self.settings = settings self.overlayUI = UIElement( self.overlay, "VRCT", "Receive UI Element", - (settings['Normalised icon X position'], settings['Normalised icon Y position']), + self.settings, ) self.lastUpdate = time.monotonic() def update(self): currTime = time.monotonic() - if settings['Fade interval'] != 0: + if self.settings['Fade interval'] != 0: self.evaluateTransparencyFade(self.overlayUI, self.lastUpdate, currTime) def uiUpdate(self, img): self.overlayUI.setImage(img) - self.overlayUI.setTransparency(settings['Transparency']) + self.overlayUI.setTransparency(self.settings['Transparency']) self.lastUpdate = time.monotonic() def evaluateTransparencyFade(self, ui, lastUpdate, currentTime): - if (currentTime - lastUpdate) > settings['Fade time']: - timeThroughInterval = currentTime - lastUpdate - settings['Fade time'] - fadeRatio = 1 - timeThroughInterval / settings['Fade interval'] + if (currentTime - lastUpdate) > self.settings['Fade time']: + timeThroughInterval = currentTime - lastUpdate - self.settings['Fade time'] + fadeRatio = 1 - timeThroughInterval / self.settings['Fade interval'] if fadeRatio < 0: fadeRatio = 0 - ui.setTransparency(fadeRatio * settings['Transparency']) + ui.setTransparency(fadeRatio * self.settings['Transparency']) + + def posUpdate(self, pos): + self.overlayUI.setPosition(pos) class Overlay: def __init__(self): self.initFlag = False - - def checkHMD(self): - return openvr.isHmdPresent() - - def checkRuntime(self): - return openvr.isRuntimeInstalled() + settings = { + "Colour": [1, 1, 1], + "Transparency": 1, + "Normalised icon X position": 0.0, + "Normalised icon Y position": -0.41, + "Icon plane depth": 1, + "Normalised icon width": 1, + "Fade time": 5, + "Fade interval": 2, + } + self.settings = settings def init(self): try: @@ -189,13 +135,14 @@ class Overlay: await asyncio.sleep(sleepTime) async def init_main(self): - self.uiMan = UIManager() + self.uiMan = UIManager(self.settings) await self.mainLoop() def startOverlay(self): asyncio.run(self.init_main()) if __name__ == '__main__': + from overlay_image import OverlayImage from threading import Thread, Event class threadFnc(Thread): def __init__(self, fnc, end_fnc=None, daemon=True, *args, **kwargs): @@ -216,6 +163,7 @@ if __name__ == '__main__': self.fnc(*self._args, **self._kwargs) overlay = Overlay() + overlay_image = OverlayImage() if overlay.initFlag is False: overlay.init() @@ -223,17 +171,18 @@ if __name__ == '__main__': t = threadFnc(overlay.startOverlay) t.start() - img = create_overlay_image("こんにちは、世界！さようなら", "Japanese", "Hello,World!Goodbye", "Japanese") + time.sleep(1) + img = overlay_image.create_overlay_image_short("こんにちは、世界！さようなら", "Japanese", "Hello,World!Goodbye", "Japanese") if overlay.initFlag is True: overlay.uiMan.uiUpdate(img) time.sleep(10) - img = create_overlay_image("こんにちは、世界！さようなら", "Japanese", "안녕하세요, 세계!안녕", "Korean") + img = overlay_image.create_overlay_image_short("こんにちは、世界！さようなら", "Japanese", "안녕하세요, 세계!안녕", "Korean") if overlay.initFlag is True: overlay.uiMan.uiUpdate(img) time.sleep(10) - img = create_overlay_image("こんにちは、世界！さようなら", "Japanese", "你好世界！再见", "Chinese Simplified") + img = overlay_image.create_overlay_image_short("こんにちは、世界！さようなら", "Japanese", "你好世界！再见", "Chinese Simplified") if overlay.initFlag is True: overlay.uiMan.uiUpdate(img) time.sleep(10) \ No newline at end of file diff --git a/models/overlay/overlay_image.py b/models/overlay/overlay_image.py new file mode 100644 index 00000000..bc4ec1eb --- /dev/null +++ b/models/overlay/overlay_image.py @@ -0,0 +1,174 @@ +from os import path as os_path +from datetime import datetime +from typing import Tuple +from PIL import Image, ImageDraw, ImageFont + +class OverlayImage: + WIDTH = 1920//2 + HEIGHT = 46//2 + BACKGROUND_COLOR = (41, 42, 45) + TEXT_COLOR_LARGE = (223, 223, 223) + TEXT_COLOR_SMALL = (190, 190, 190) + TEXT_COLOR_SEND = (70, 161, 146) + TEXT_COLOR_RECEIVE = (220, 20, 60) + TEXT_COLOR_TIME = (120, 120, 120) + FONT_SIZE_LARGE = HEIGHT + FONT_SIZE_SMALL = int(FONT_SIZE_LARGE * 2 / 3) + LANGUAGES = { + "Japanese": "NotoSansJP-Regular", + "Korean": "NotoSansKR-Regular", + "Chinese Simplified": "NotoSansSC-Regular", + "Chinese Traditional": "NotoSansTC-Regular", + } + + def __init__(self): + self.log_data = [] + + @staticmethod + def concatenate_images_vertically(img1: Image, img2: Image) -> Image: + dst = Image.new('RGBA', (img1.width, img1.height + img2.height)) + dst.paste(img1, (0, 0)) + dst.paste(img2, (0, img1.height)) + return dst + + @staticmethod + def add_image_margin(image: Image, top: int, right: int, bottom: int, left: int, color: Tuple[int, int, int, int]) -> Image: + width, height = image.size + new_width = width + right + left + new_height = height + top + bottom + result = Image.new(image.mode, (new_width, new_height), color) + result.paste(image, (left, top)) + return result + + def create_textimage(self, message_type, size, text, language): + font_size = self.FONT_SIZE_LARGE if size == "large" else self.FONT_SIZE_SMALL + text_color = self.TEXT_COLOR_LARGE if size == "large" else self.TEXT_COLOR_SMALL + anchor = "lm" if message_type == "receive" else "rm" + text_x = 0 if message_type == "receive" else self.WIDTH + align = "left" if message_type == "receive" else "right" + + font_family = self.LANGUAGES.get(language, "NotoSansJP-Regular") + img = Image.new("RGBA", (0, 0), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + font = ImageFont.truetype(os_path.join(os_path.dirname(__file__), "fonts", f"{font_family}.ttf"), font_size) + # font = ImageFont.truetype(os_path.join("./fonts", f"{font_family}.ttf"), font_size) + text_width = draw.textlength(text, font) + character_width = text_width // len(text) + character_line_num = int(self.WIDTH // character_width) + if len(text) > character_line_num: + text = "\n".join([text[i:i+character_line_num] for i in range(0, len(text), character_line_num)]) + + n_num = len(text.split("\n")) - 1 + text_height = int(font_size*(n_num+2)) + + img = Image.new("RGBA", (self.WIDTH, text_height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + + text_y = text_height // 2 + + draw.multiline_text((text_x, text_y), text, text_color, anchor=anchor, stroke_width=0, font=font, align=align) + return img + + def create_textimage_message_type(self, message_type): + anchor = "lm" if message_type == "receive" else "rm" + text = "Receive" if message_type == "receive" else "Send" + text_color = self.TEXT_COLOR_RECEIVE if message_type == "receive" else self.TEXT_COLOR_SEND + text_color_time = self.TEXT_COLOR_TIME + + now = datetime.now() + formatted_time = now.strftime("%H:%M") + font_size = self.FONT_SIZE_SMALL + img = Image.new("RGBA", (0, 0), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + font = ImageFont.truetype(os_path.join(os_path.dirname(__file__), "fonts", "NotoSansJP-Regular.ttf"), font_size) + # font = ImageFont.truetype(os_path.join("./fonts", "NotoSansJP-Regular.ttf"), font_size) + text_height = font_size*2 + text_width = draw.textlength(formatted_time, font) + character_width = text_width // len(formatted_time) + img = Image.new("RGBA", (self.WIDTH, text_height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + text_y = text_height // 2 + text_time_x = 0 if message_type == "receive" else self.WIDTH - (text_width + character_width) + text_x = (text_width + character_width) if message_type == "receive" else self.WIDTH + + draw.text((text_time_x, text_y), formatted_time, text_color_time, anchor=anchor, stroke_width=0, font=font) + draw.text((text_x, text_y), text, text_color, anchor=anchor, stroke_width=0, font=font) + return img + + def create_textbox(self, message_type, message, your_language, translation, target_language): + message_type_img = self.create_textimage_message_type(message_type) + if len(translation) > 0 and target_language is not None: + img = self.create_textimage(message_type, "small", message, your_language) + translation_img = self.create_textimage(message_type, "large",translation, target_language) + img = self.concatenate_images_vertically(img, translation_img) + else: + img = self.create_textimage(message_type, "large", message, your_language) + return self.concatenate_images_vertically(message_type_img, img) + + def create_overlay_image_long(self, message_type, message, your_language, translation="", target_language=None): + if len(self.log_data) > 10: + self.log_data.pop(0) + + self.log_data.append( + { + "message_type":message_type, + "message":message, + "your_language":your_language, + "translation":translation, + "target_language":target_language, + } + ) + + imgs = [] + for log in self.log_data: + message_type = log["message_type"] + message = log["message"] + your_language = log["your_language"] + translation = log["translation"] + target_language = log["target_language"] + img = self.create_textbox(message_type, message, your_language, translation, target_language) + imgs.append(img) + + img = imgs[0] + for i in imgs[1:]: + img = self.concatenate_images_vertically(img, i) + img = self.add_image_margin(img, 0, 20, 0, 20, (0, 0, 0, 0)) + + width, height = img.size + background = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(background) + draw.rounded_rectangle([(0, 0), (width, height)], radius=15, fill=self.BACKGROUND_COLOR, outline=self.BACKGROUND_COLOR, width=5) + img = Image.alpha_composite(background, img) + return img + + def create_overlay_image_short(self, message, your_language, translation="", target_language=None): + def create_textbox(text, language): + font_family = self.LANGUAGES.get(language, "NotoSansJP-Regular") + img = Image.new("RGBA", (self.WIDTH, self.HEIGHT), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + font = ImageFont.truetype(os_path.join(os_path.dirname(__file__), "fonts", f"{font_family}.ttf"), self.FONT_SIZE_LARGE) + text_width = draw.textlength(text, font) + character_width = text_width // len(text) + character_line_num = int((self.WIDTH - 40) // character_width) + if len(text) > character_line_num: + text = "\n".join([text[i:i+character_line_num] for i in range(0, len(text), character_line_num)]) + text_height = self.FONT_SIZE_LARGE * (len(text.split("\n")) + 1) + 20 + img = Image.new("RGBA", (self.WIDTH, text_height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + + text_x = self.WIDTH // 2 + text_y = text_height // 2 + draw.text((text_x, text_y), text, self.TEXT_COLOR_LARGE, anchor="mm", stroke_width=0, font=font, align="center") + return img + + img = create_textbox(message, your_language) + if len(translation) > 0 and target_language is not None: + translation_img = create_textbox(translation, target_language) + img = self.concatenate_images_vertically(img, translation_img) + + width, height = img.size + background = Image.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(background) + draw.rounded_rectangle([(0, 0), (width, height)], radius=30, fill=self.BACKGROUND_COLOR, outline=self.BACKGROUND_COLOR, width=5) + img = Image.alpha_composite(background, img) + return img \ No newline at end of file