👍️[Update] Model : overlay機能を修正

- 画像生成をoverlay_img.pyにまとめた
This commit is contained in:
misyaguziya
2024-03-29 23:34:31 +09:00
parent c3b2f08b9b
commit 34c81cec18
5 changed files with 258 additions and 99 deletions

View File

@@ -716,6 +716,17 @@ class Config:
self._ENABLE_NOTICE_XSOVERLAY = value
saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value)
@property
@json_serializable('ENABLE_NOTICE_OVERLAY')
def ENABLE_NOTICE_OVERLAY(self):
return self._ENABLE_NOTICE_OVERLAY
@ENABLE_NOTICE_OVERLAY.setter
def ENABLE_NOTICE_OVERLAY(self, value):
if isinstance(value, bool):
self._ENABLE_NOTICE_OVERLAY = value
saveJson(self.PATH_CONFIG, inspect.currentframe().f_code.co_name, value)
@property
@json_serializable('ENABLE_SEND_MESSAGE_TO_VRC')
def ENABLE_SEND_MESSAGE_TO_VRC(self):
@@ -979,6 +990,7 @@ class Config:
self._ENABLE_SEND_ONLY_TRANSLATED_MESSAGES = False
self._SEND_MESSAGE_BUTTON_TYPE = "show"
self._ENABLE_NOTICE_XSOVERLAY = False
self._ENABLE_NOTICE_OVERLAY = False
self._ENABLE_SEND_MESSAGE_TO_VRC = True
self._ENABLE_SEND_RECEIVED_MESSAGE_TO_VRC = False # Speaker2Chatbox
self._ENABLE_SPEAKER2CHATBOX_PASS = "000000000"

View File

@@ -94,6 +94,12 @@ def sendMicMessage(message):
translation = f" ({translation})"
model.logger.info(f"[SENT] {message}{translation}")
# if config.ENABLE_NOTICE_OVERLAY is True:
# overlay_image = model.createOverlayImageShort(message, translation)
# model.setOverlayImage(overlay_image)
# overlay_image = model.createOverlayImageLong("send", message, translation)
# model.setOverlayImage(overlay_image)
def startTranscriptionSendMessage():
model.startMicTranscript(sendMicMessage, view.printToTextbox_TranscriptionSendNoDeviceError)
view.setMainWindowAllWidgetsStatusToNormal()
@@ -148,8 +154,12 @@ def receiveSpeakerMessage(message):
if model.th_overlay is None:
model.startOverlay()
overlay_image = model.createOverlayImage(message, translation)
model.setOverlayImage(overlay_image)
if config.ENABLE_NOTICE_OVERLAY is True:
overlay_image = model.createOverlayImageShort(message, translation)
model.setOverlayImage(overlay_image)
# overlay_image = model.createOverlayImageLong("receive", message, translation)
# model.setOverlayImage(overlay_image)
# ------------Speaker2Chatbox------------
if config.ENABLE_SPEAKER2CHATBOX is True:
@@ -157,7 +167,7 @@ def receiveSpeakerMessage(message):
if config.ENABLE_SEND_RECEIVED_MESSAGE_TO_VRC is True:
osc_message = messageFormatter("RECEIVED", translation, message)
model.oscSendMessage(osc_message)
# ------------Speaker2Chatbox------------
# ------------Speaker2Chatbox------------
# update textbox message log (Received)
view.printToTextbox_ReceivedMessage(message, translation)
@@ -226,6 +236,12 @@ def sendChatMessage(message):
osc_message = messageFormatter("SEND", translation, message)
model.oscSendMessage(osc_message)
# if config.ENABLE_NOTICE_OVERLAY is True:
# overlay_image = model.createOverlayImageShort(message, translation)
# model.setOverlayImage(overlay_image)
# overlay_image = model.createOverlayImageLong("send", message, translation)
# model.setOverlayImage(overlay_image)
# update textbox message log (Sent)
view.printToTextbox_SentMessage(message, translation)
if config.ENABLE_LOGGER is True:

View File

@@ -26,7 +26,9 @@ from models.translation.translation_languages import translation_lang
from models.transcription.transcription_languages import transcription_lang
from models.translation.translation_utils import checkCTranslate2Weight
from models.transcription.transcription_whisper import checkWhisperWeight
from models.overlay.overlay import create_overlay_image, Overlay
from models.overlay.overlay import Overlay
from models.overlay.overlay_image import OverlayImage
from config import config
class threadFnc(Thread):
@@ -69,6 +71,7 @@ class Model:
self.translator = Translator()
self.keyword_processor = KeywordProcessor()
self.overlay = Overlay()
self.overlay_image = OverlayImage()
self.th_overlay = None
def checkCTranslatorCTranslate2ModelWeight(self):
@@ -536,10 +539,15 @@ class Model:
def notificationXSOverlay(self, message):
xsoverlayForVRCT(content=f"{message}")
def createOverlayImage(self, message, translation):
def createOverlayImageShort(self, message, translation):
your_language = config.TARGET_LANGUAGE
target_language = config.SOURCE_LANGUAGE
return create_overlay_image(message, your_language, translation, target_language)
return self.overlay_image.create_overlay_image_short(message, your_language, translation, target_language)
def createOverlayImageLong(self, message_type, message, translation):
your_language = config.TARGET_LANGUAGE if message_type == "receive" else config.SOURCE_LANGUAGE
target_language = config.SOURCE_LANGUAGE if message_type == "receive" else config.TARGET_LANGUAGE
return self.overlay_image.create_overlay_image_long(message_type, message, your_language, translation, target_language)
def setOverlayImage(self, img):
if self.overlay.initFlag is True:

View File

@@ -2,71 +2,10 @@ import ctypes
import time
import asyncio
import openvr
from PIL import Image, ImageDraw, ImageFont
from os import path as os_path
def create_overlay_image(message, your_language, translation=None, target_language=None):
width, height = (1920, 46)
background_color = (54, 63, 77)
text_color = (223, 223, 224)
font_size = 46
languages = {
"Japanese" : "NotoSansJP-Regular",
"Korean" : "NotoSansKR-Regular",
"Chinese Simplified" : "NotoSansSC-Regular",
"Chinese Traditional" : "NotoSansTC-Regular",
}
def get_concat_v(im1, im2):
dst = Image.new('RGBA', (im1.width, im1.height + im2.height))
dst.paste(im1, (0, 0))
dst.paste(im2, (0, im1.height))
return dst
def create_textbox(text, language):
font_family = languages.get(language, "NotoSansJP-Regular")
img = Image.new("RGBA", (width, height), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
font = ImageFont.truetype(os_path.join(os_path.dirname(__file__), "fonts", f"{font_family}.ttf"), font_size)
text_width = draw.textlength(text, font)
character_width = text_width // len(text)
character_line_num = int((width - 40) // character_width)
if len(text) > character_line_num:
text = "\n".join([text[i:i+character_line_num] for i in range(0, len(text), character_line_num)])
text_height = font_size * (len(text.split("\n")) + 1) + 20
img = Image.new("RGBA", (width, text_height), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
# draw.rounded_rectangle([(0, 0), (width, text_height)], radius=30, fill=background_color, outline=background_color, width=5)
text_x = width // 2
text_y = text_height // 2
draw.text((text_x, text_y), text, text_color, anchor="mm", stroke_width=0, font=font, align="center")
return img
img = create_textbox(message, your_language)
if translation is not None and target_language is not None:
translation_img = create_textbox(translation, target_language)
img = get_concat_v(img, translation_img)
width, height = img.size
background = Image.new("RGBA", (width, height), (0, 0, 0, 0))
draw = ImageDraw.Draw(background)
draw.rounded_rectangle([(0, 0), (width, height)], radius=30, fill=background_color, outline=background_color, width=5)
img = Image.alpha_composite(background, img)
return img
settings = {
"Colour": [1, 1, 1],
"Transparency": 1,
"Normalised icon X position": 0.0,
"Normalised icon Y position": -0.41,
"Icon plane depth": 1,
"Normalised icon width": 1,
"Fade time": 5,
"Fade interval": 2,
}
from PIL import Image
# This code is based on the following source:
# [GOpy](https://github.com/MeroFune/GOpy)
def mat34Id():
arr = openvr.HmdMatrix34_t()
arr[0][0] = 1
@@ -75,27 +14,26 @@ def mat34Id():
return arr
class UIElement:
def __init__(self, overlayRoot, key, name, pos, flip = False) -> None:
def __init__(self, overlayRoot, key: str, name: str, settings: dict = None) -> None:
"""
pos is a 2-tuple representing (x, y) normalised position of the overlay on the screen
"""
self.overlay = overlayRoot
self.overlayKey = key
self.overlayName = name
self.flip = flip
self.settings = settings
pos = (self.settings['Normalised icon X position'], self.settings['Normalised icon Y position'])
self.handle = self.overlay.createOverlay(self.overlayKey, self.overlayName)
self.setImage(Image.new("RGBA", (1, 1), (0, 0, 0, 0))) # blank image for default
self.setColour(settings['Colour'])
self.setTransparency(settings['Transparency'])
self.setColour(self.settings['Colour'])
self.setTransparency(self.settings['Transparency'])
self.overlay.setOverlayWidthInMeters(
self.handle,
settings['Normalised icon width'] * settings['Icon plane depth']
self.settings['Normalised icon width'] * self.settings['Icon plane depth']
)
self.setPosition(pos)
self.overlay.showOverlay(self.handle)
def setImage(self, img):
@@ -121,9 +59,9 @@ class UIElement:
self.transform = mat34Id() # no rotation required for HMD attachment
# assign position
self.transform[0][3] = pos[0] * settings['Icon plane depth']
self.transform[1][3] = pos[1] * settings['Icon plane depth']
self.transform[2][3] = -settings['Icon plane depth']
self.transform[0][3] = pos[0] * self.settings['Icon plane depth']
self.transform[1][3] = pos[1] * self.settings['Icon plane depth']
self.transform[2][3] = - self.settings['Icon plane depth']
self.overlay.setOverlayTransformTrackedDeviceRelative(
self.handle,
@@ -132,45 +70,53 @@ class UIElement:
)
class UIManager:
def __init__(self):
def __init__(self, settings):
self.overlay = openvr.IVROverlay()
self.settings = settings
self.overlayUI = UIElement(
self.overlay,
"VRCT",
"Receive UI Element",
(settings['Normalised icon X position'], settings['Normalised icon Y position']),
self.settings,
)
self.lastUpdate = time.monotonic()
def update(self):
currTime = time.monotonic()
if settings['Fade interval'] != 0:
if self.settings['Fade interval'] != 0:
self.evaluateTransparencyFade(self.overlayUI, self.lastUpdate, currTime)
def uiUpdate(self, img):
self.overlayUI.setImage(img)
self.overlayUI.setTransparency(settings['Transparency'])
self.overlayUI.setTransparency(self.settings['Transparency'])
self.lastUpdate = time.monotonic()
def evaluateTransparencyFade(self, ui, lastUpdate, currentTime):
if (currentTime - lastUpdate) > settings['Fade time']:
timeThroughInterval = currentTime - lastUpdate - settings['Fade time']
fadeRatio = 1 - timeThroughInterval / settings['Fade interval']
if (currentTime - lastUpdate) > self.settings['Fade time']:
timeThroughInterval = currentTime - lastUpdate - self.settings['Fade time']
fadeRatio = 1 - timeThroughInterval / self.settings['Fade interval']
if fadeRatio < 0:
fadeRatio = 0
ui.setTransparency(fadeRatio * settings['Transparency'])
ui.setTransparency(fadeRatio * self.settings['Transparency'])
def posUpdate(self, pos):
self.overlayUI.setPosition(pos)
class Overlay:
def __init__(self):
self.initFlag = False
def checkHMD(self):
return openvr.isHmdPresent()
def checkRuntime(self):
return openvr.isRuntimeInstalled()
settings = {
"Colour": [1, 1, 1],
"Transparency": 1,
"Normalised icon X position": 0.0,
"Normalised icon Y position": -0.41,
"Icon plane depth": 1,
"Normalised icon width": 1,
"Fade time": 5,
"Fade interval": 2,
}
self.settings = settings
def init(self):
try:
@@ -189,13 +135,14 @@ class Overlay:
await asyncio.sleep(sleepTime)
async def init_main(self):
self.uiMan = UIManager()
self.uiMan = UIManager(self.settings)
await self.mainLoop()
def startOverlay(self):
asyncio.run(self.init_main())
if __name__ == '__main__':
from overlay_image import OverlayImage
from threading import Thread, Event
class threadFnc(Thread):
def __init__(self, fnc, end_fnc=None, daemon=True, *args, **kwargs):
@@ -216,6 +163,7 @@ if __name__ == '__main__':
self.fnc(*self._args, **self._kwargs)
overlay = Overlay()
overlay_image = OverlayImage()
if overlay.initFlag is False:
overlay.init()
@@ -223,17 +171,18 @@ if __name__ == '__main__':
t = threadFnc(overlay.startOverlay)
t.start()
img = create_overlay_image("こんにちは、世界!さようなら", "Japanese", "Hello,World!Goodbye", "Japanese")
time.sleep(1)
img = overlay_image.create_overlay_image_short("こんにちは、世界!さようなら", "Japanese", "Hello,World!Goodbye", "Japanese")
if overlay.initFlag is True:
overlay.uiMan.uiUpdate(img)
time.sleep(10)
img = create_overlay_image("こんにちは、世界!さようなら", "Japanese", "안녕하세요, 세계!안녕", "Korean")
img = overlay_image.create_overlay_image_short("こんにちは、世界!さようなら", "Japanese", "안녕하세요, 세계!안녕", "Korean")
if overlay.initFlag is True:
overlay.uiMan.uiUpdate(img)
time.sleep(10)
img = create_overlay_image("こんにちは、世界!さようなら", "Japanese", "你好世界!再见", "Chinese Simplified")
img = overlay_image.create_overlay_image_short("こんにちは、世界!さようなら", "Japanese", "你好世界!再见", "Chinese Simplified")
if overlay.initFlag is True:
overlay.uiMan.uiUpdate(img)
time.sleep(10)

View File

@@ -0,0 +1,174 @@
from os import path as os_path
from datetime import datetime
from typing import Tuple
from PIL import Image, ImageDraw, ImageFont
class OverlayImage:
WIDTH = 1920//2
HEIGHT = 46//2
BACKGROUND_COLOR = (41, 42, 45)
TEXT_COLOR_LARGE = (223, 223, 223)
TEXT_COLOR_SMALL = (190, 190, 190)
TEXT_COLOR_SEND = (70, 161, 146)
TEXT_COLOR_RECEIVE = (220, 20, 60)
TEXT_COLOR_TIME = (120, 120, 120)
FONT_SIZE_LARGE = HEIGHT
FONT_SIZE_SMALL = int(FONT_SIZE_LARGE * 2 / 3)
LANGUAGES = {
"Japanese": "NotoSansJP-Regular",
"Korean": "NotoSansKR-Regular",
"Chinese Simplified": "NotoSansSC-Regular",
"Chinese Traditional": "NotoSansTC-Regular",
}
def __init__(self):
self.log_data = []
@staticmethod
def concatenate_images_vertically(img1: Image, img2: Image) -> Image:
dst = Image.new('RGBA', (img1.width, img1.height + img2.height))
dst.paste(img1, (0, 0))
dst.paste(img2, (0, img1.height))
return dst
@staticmethod
def add_image_margin(image: Image, top: int, right: int, bottom: int, left: int, color: Tuple[int, int, int, int]) -> Image:
width, height = image.size
new_width = width + right + left
new_height = height + top + bottom
result = Image.new(image.mode, (new_width, new_height), color)
result.paste(image, (left, top))
return result
def create_textimage(self, message_type, size, text, language):
font_size = self.FONT_SIZE_LARGE if size == "large" else self.FONT_SIZE_SMALL
text_color = self.TEXT_COLOR_LARGE if size == "large" else self.TEXT_COLOR_SMALL
anchor = "lm" if message_type == "receive" else "rm"
text_x = 0 if message_type == "receive" else self.WIDTH
align = "left" if message_type == "receive" else "right"
font_family = self.LANGUAGES.get(language, "NotoSansJP-Regular")
img = Image.new("RGBA", (0, 0), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
font = ImageFont.truetype(os_path.join(os_path.dirname(__file__), "fonts", f"{font_family}.ttf"), font_size)
# font = ImageFont.truetype(os_path.join("./fonts", f"{font_family}.ttf"), font_size)
text_width = draw.textlength(text, font)
character_width = text_width // len(text)
character_line_num = int(self.WIDTH // character_width)
if len(text) > character_line_num:
text = "\n".join([text[i:i+character_line_num] for i in range(0, len(text), character_line_num)])
n_num = len(text.split("\n")) - 1
text_height = int(font_size*(n_num+2))
img = Image.new("RGBA", (self.WIDTH, text_height), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
text_y = text_height // 2
draw.multiline_text((text_x, text_y), text, text_color, anchor=anchor, stroke_width=0, font=font, align=align)
return img
def create_textimage_message_type(self, message_type):
anchor = "lm" if message_type == "receive" else "rm"
text = "Receive" if message_type == "receive" else "Send"
text_color = self.TEXT_COLOR_RECEIVE if message_type == "receive" else self.TEXT_COLOR_SEND
text_color_time = self.TEXT_COLOR_TIME
now = datetime.now()
formatted_time = now.strftime("%H:%M")
font_size = self.FONT_SIZE_SMALL
img = Image.new("RGBA", (0, 0), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
font = ImageFont.truetype(os_path.join(os_path.dirname(__file__), "fonts", "NotoSansJP-Regular.ttf"), font_size)
# font = ImageFont.truetype(os_path.join("./fonts", "NotoSansJP-Regular.ttf"), font_size)
text_height = font_size*2
text_width = draw.textlength(formatted_time, font)
character_width = text_width // len(formatted_time)
img = Image.new("RGBA", (self.WIDTH, text_height), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
text_y = text_height // 2
text_time_x = 0 if message_type == "receive" else self.WIDTH - (text_width + character_width)
text_x = (text_width + character_width) if message_type == "receive" else self.WIDTH
draw.text((text_time_x, text_y), formatted_time, text_color_time, anchor=anchor, stroke_width=0, font=font)
draw.text((text_x, text_y), text, text_color, anchor=anchor, stroke_width=0, font=font)
return img
def create_textbox(self, message_type, message, your_language, translation, target_language):
message_type_img = self.create_textimage_message_type(message_type)
if len(translation) > 0 and target_language is not None:
img = self.create_textimage(message_type, "small", message, your_language)
translation_img = self.create_textimage(message_type, "large",translation, target_language)
img = self.concatenate_images_vertically(img, translation_img)
else:
img = self.create_textimage(message_type, "large", message, your_language)
return self.concatenate_images_vertically(message_type_img, img)
def create_overlay_image_long(self, message_type, message, your_language, translation="", target_language=None):
if len(self.log_data) > 10:
self.log_data.pop(0)
self.log_data.append(
{
"message_type":message_type,
"message":message,
"your_language":your_language,
"translation":translation,
"target_language":target_language,
}
)
imgs = []
for log in self.log_data:
message_type = log["message_type"]
message = log["message"]
your_language = log["your_language"]
translation = log["translation"]
target_language = log["target_language"]
img = self.create_textbox(message_type, message, your_language, translation, target_language)
imgs.append(img)
img = imgs[0]
for i in imgs[1:]:
img = self.concatenate_images_vertically(img, i)
img = self.add_image_margin(img, 0, 20, 0, 20, (0, 0, 0, 0))
width, height = img.size
background = Image.new("RGBA", (width, height), (0, 0, 0, 0))
draw = ImageDraw.Draw(background)
draw.rounded_rectangle([(0, 0), (width, height)], radius=15, fill=self.BACKGROUND_COLOR, outline=self.BACKGROUND_COLOR, width=5)
img = Image.alpha_composite(background, img)
return img
def create_overlay_image_short(self, message, your_language, translation="", target_language=None):
def create_textbox(text, language):
font_family = self.LANGUAGES.get(language, "NotoSansJP-Regular")
img = Image.new("RGBA", (self.WIDTH, self.HEIGHT), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
font = ImageFont.truetype(os_path.join(os_path.dirname(__file__), "fonts", f"{font_family}.ttf"), self.FONT_SIZE_LARGE)
text_width = draw.textlength(text, font)
character_width = text_width // len(text)
character_line_num = int((self.WIDTH - 40) // character_width)
if len(text) > character_line_num:
text = "\n".join([text[i:i+character_line_num] for i in range(0, len(text), character_line_num)])
text_height = self.FONT_SIZE_LARGE * (len(text.split("\n")) + 1) + 20
img = Image.new("RGBA", (self.WIDTH, text_height), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
text_x = self.WIDTH // 2
text_y = text_height // 2
draw.text((text_x, text_y), text, self.TEXT_COLOR_LARGE, anchor="mm", stroke_width=0, font=font, align="center")
return img
img = create_textbox(message, your_language)
if len(translation) > 0 and target_language is not None:
translation_img = create_textbox(translation, target_language)
img = self.concatenate_images_vertically(img, translation_img)
width, height = img.size
background = Image.new("RGBA", (width, height), (0, 0, 0, 0))
draw = ImageDraw.Draw(background)
draw.rounded_rectangle([(0, 0), (width, height)], radius=30, fill=self.BACKGROUND_COLOR, outline=self.BACKGROUND_COLOR, width=5)
img = Image.alpha_composite(background, img)
return img