👍️[Update] Model : overlay 左手周りの処理を追加

- HMD,左手の初期値を追加
- 初期値からの変化量をUIから調整できるように変更
- quaternionは不要なため、削除

config.py

@@ -1072,8 +1072,8 @@ class Config:
         self._ENABLE_OVERLAY_SMALL_LOG = False
         self._OVERLAY_SMALL_LOG_SETTINGS = {
             "x_pos": 0.0,
-            "y_pos": -0.41,
+            "y_pos": 0.0,
-            "depth": 1.0,
+            "depth": 0.0,
             "x_rotation": 0.0,
             "y_rotation": 0.0,
             "z_rotation": 0.0,

models/overlay/overlay.py

@@ -1,19 +1,56 @@
 import os
 import ctypes
-from psutil import process_iter
 import time
-import openvr
+from psutil import process_iter
-from PIL import Image
 from threading import Thread
+import openvr
 import numpy as np
-import quaternion
+from PIL import Image
+try:
+    from . import overlay_utils as utils
+except ImportError:
+    import overlay_utils as utils
 
-def mat34Id():
+def mat34Id(array):
     arr = openvr.HmdMatrix34_t()
-    # arr[0][0] = 1
+    for i in range(3):
-    # arr[1][1] = 1
+        for j in range(4):
-    # arr[2][2] = 1
+            arr[i][j] = array[i][j]
-    # print(arr)
+    return arr
+
+def getBaseMatrix(x_pos, y_pos, depth, x_rotation, y_rotation, z_rotation):
+    arr = np.zeros((3, 4))
+    rot = utils.euler_to_rotation_matrix((x_rotation, y_rotation, z_rotation))
+
+    for i in range(3):
+        for j in range(3):
+            arr[i][j] = rot[i][j]
+
+    arr[0][3] = x_pos * depth
+    arr[1][3] = y_pos * depth
+    arr[2][3] = - depth
+    return arr
+
+def getHMDBaseMatrix():
+    x_pos = 0.0
+    y_pos = -0.4
+    depth = 1.0
+    x_rotation = 0.0
+    y_rotation = 0.0
+    z_rotation = 0.0
+
+    arr = getBaseMatrix(x_pos, y_pos, depth, x_rotation, y_rotation, z_rotation)
+    return arr
+
+def getLeftHandBaseMatrix():
+    x_pos = 0.0
+    y_pos = -0.06
+    depth = -0.14
+    x_rotation = -62.0
+    y_rotation = 154.0
+    z_rotation = 71.0
+
+    arr = getBaseMatrix(x_pos, y_pos, depth, x_rotation, y_rotation, z_rotation)
     return arr
 
 class Overlay:
@@ -108,6 +145,7 @@ class Overlay:
         depth is a float representing the depth of the icon plane
         x_rotation, y_rotation, z_rotation are floats representing the rotation of the icon plane
         """
+
         self.settings["x_pos"] = pos[0]
         self.settings["y_pos"] = pos[1]
         self.settings["depth"] = depth
@@ -115,26 +153,20 @@ class Overlay:
         self.settings["y_rotation"] = y_rotation
         self.settings["z_rotation"] = z_rotation
 
-        self.transform = mat34Id() # no rotation required for HMD attachment
+        base_matrix = getHMDBaseMatrix()
+        # base_matrix = getLeftHandBaseMatrix()
+        translation = (self.settings["x_pos"], self.settings["y_pos"], - self.settings['depth'])
+        rotation = (self.settings["x_rotation"], self.settings["y_rotation"], self.settings["z_rotation"])
+        transform = utils.transform_matrix(base_matrix, translation, rotation)
+        self.transform = mat34Id(transform)
 
-        # assign rotation
+        hmdIndex = openvr.k_unTrackedDeviceIndex_Hmd
-        rot = np.quaternion(1, self.settings["x_rotation"], self.settings["y_rotation"], self.settings["z_rotation"])
+        # leftControllerIndex = self.overlay_system.getTrackedDeviceIndexForControllerRole(openvr.TrackedControllerRole_LeftHand)
-        rot = quaternion.as_rotation_matrix(rot)
+        # rightControllerIndex = self.overlay_system.getTrackedDeviceIndexForControllerRole(openvr.TrackedControllerRole_RightHand)
-        # self.transform[:3, :3] = rot
-        for i in range(3):
-            for j in range(3):
-                self.transform[i][j] = rot[i][j]
-
-        # assign position
-        self.transform[0][3] = self.settings["x_pos"] * self.settings['depth']
-        self.transform[1][3] = self.settings["y_pos"] * self.settings['depth']
-        self.transform[2][3] = - self.settings['depth']
-
-        leftControllerIndex = self.overlay_system.getTrackedDeviceIndexForControllerRole(openvr.TrackedControllerRole_LeftHand)
         if self.initialized is True:
             self.overlay.setOverlayTransformTrackedDeviceRelative(
                 self.handle,
-                leftControllerIndex, #openvr.k_unTrackedDeviceIndex_Hmd,
+                hmdIndex,
                 self.transform
             )
 
@@ -210,17 +242,17 @@ class Overlay:
         return _proc_name in (p.name() for p in process_iter())
 
 if __name__ == '__main__':
-    from overlay_image import OverlayImage
+    # from overlay_image import OverlayImage
-    overlay_image = OverlayImage()
+    # overlay_image = OverlayImage()
 
-    overlay = Overlay(0, 0, 1, 1, 0, 1, 1)
+    # overlay = Overlay(0, 0, 1, 1, 0, 1, 1)
-    overlay.startOverlay()
+    # overlay.startOverlay()
-    time.sleep(1)
+    # time.sleep(1)
 
-    # Example usage
+    # # Example usage
-    img = overlay_image.createOverlayImageShort("こんにちは、世界！さようなら", "Japanese", "Hello,World!Goodbye", "Japanese")
+    # img = overlay_image.createOverlayImageShort("こんにちは、世界！さようなら", "Japanese", "Hello,World!Goodbye", "Japanese")
-    overlay.updateImage(img)
+    # overlay.updateImage(img)
-    time.sleep(100000)
+    # time.sleep(100000)
     
     # for i in range(100):
     #     print(i)
@@ -238,3 +270,17 @@ if __name__ == '__main__':
     #    time.sleep(0.5)
 
     #    overlay.shutdownOverlay()
+
+    x_pos = 0
+    y_pos = 0
+    depth = 0
+    x_rotation = 0
+    y_rotation = 0
+    z_rotation = 0
+
+    base_matrix = getLeftHandBaseMatrix()
+    translation = (x_pos * depth, y_pos * depth, depth)
+    rotation = (x_rotation, y_rotation, z_rotation)
+    transform = utils.transform_matrix(base_matrix, translation, rotation)
+    transform = mat34Id(transform)
+    print(transform)

models/overlay/overlay_utils.py

@@ -0,0 +1,87 @@
+import numpy as np
+
+def toHomogeneous(matrix):
+    homogeneous_matrix = np.vstack([matrix, [0, 0, 0, 1]])
+    return homogeneous_matrix
+
+# 移動行列を生成する関数
+def calcTranslationMatrix(translation):
+    tx, ty, tz = translation
+    return np.array([
+        [1, 0, 0, tx],
+        [0, 1, 0, ty],
+        [0, 0, 1, tz],
+        [0, 0, 0, 1]
+    ])
+
+# X軸周りの回転行列を生成する関数
+def calcRotationMatrixX(angle):
+    c = np.cos(np.pi/180*angle)
+    s = np.sin(np.pi/180*angle)
+    return np.array([
+        [1, 0, 0, 0],
+        [0, c, -s, 0],
+        [0, s, c, 0],
+        [0, 0, 0, 1]
+    ])
+
+# Y軸周りの回転行列を生成する関数
+def calcRotationMatrixY(angle):
+    c = np.cos(np.pi/180*angle)
+    s = np.sin(np.pi/180*angle)
+    return np.array([
+        [c, 0, s, 0],
+        [0, 1, 0, 0],
+        [-s, 0, c, 0],
+        [0, 0, 0, 1]
+    ])
+
+# Z軸周りの回転行列を生成する関数
+def calcRotationMatrixZ(angle):
+    c = np.cos(np.pi/180*angle)
+    s = np.sin(np.pi/180*angle)
+    return np.array([
+        [c, -s, 0, 0],
+        [s, c, 0, 0],
+        [0, 0, 1, 0],
+        [0, 0, 0, 1]
+    ])
+
+# 3x4行列の座標を基準として回転や移動を行う関数
+def transform_matrix(base_matrix, translation, rotation):
+    homogeneous_base_matrix = toHomogeneous(base_matrix)
+    translation_matrix = calcTranslationMatrix(translation)
+    rotation_matrix_x = calcRotationMatrixX(rotation[0])
+    rotation_matrix_y = calcRotationMatrixY(rotation[1])
+    rotation_matrix_z = calcRotationMatrixZ(rotation[2])
+    rotation_matrix = np.dot(rotation_matrix_z, np.dot(rotation_matrix_y, rotation_matrix_x))
+    transformation_matrix = translation_matrix.copy()
+    transformation_matrix[:3, :3] = rotation_matrix[:3, :3]
+    result_matrix = np.dot(homogeneous_base_matrix, transformation_matrix)
+    return result_matrix[:3, :]
+
+def euler_to_rotation_matrix(angles):
+    phi = angles[0] * np.pi / 180
+    theta = angles[1] * np.pi / 180
+    psi = angles[2]* np.pi / 180
+    R_x = np.array([[1, 0, 0],
+                    [0, np.cos(phi), -np.sin(phi)],
+                    [0, np.sin(phi), np.cos(phi)]])
+    R_y = np.array([[np.cos(theta), 0, np.sin(theta)],
+                    [0, 1, 0],
+                    [-np.sin(theta), 0, np.cos(theta)]])
+    R_z = np.array([[np.cos(psi), -np.sin(psi), 0],
+                    [np.sin(psi), np.cos(psi), 0],
+                    [0, 0, 1]])
+    return np.dot(R_z, np.dot(R_y, R_x))
+
+if __name__ == "__main__":
+    base_matrix = np.array([
+        [1, 0, 0, 1],
+        [0, 1, 0, 1],
+        [0, 0, 1, 1]
+    ])
+    translation = [1, 2, 3]
+    rotation = [0, 0, 90]
+    result_matrix = transform_matrix(base_matrix, translation, rotation)
+    print(result_matrix)

requirements.txt

@@ -13,7 +13,6 @@ sentencepiece==0.1.99
 ctranslate2==4.1.0
 faster-whisper==1.0.1
 openvr==1.26.701
-numpy-quaternion==2023.0.3
 translators @ git+https://github.com/misyaguziya/translators@5.8.9
 SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4
 tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.2

view.py

@@ -169,38 +169,38 @@ class View():
             CALLBACK_SET_OVERLAY_SMALL_LOG_SETTINGS=None,
 
             VAR_LABEL_OVERLAY_SMALL_LOG_X_POS=StringVar(value=i18n.t("overlay_settings.x_position")),
-            SLIDER_RANGE_OVERLAY_SMALL_LOG_X_POS=(-2, 2),
+            SLIDER_RANGE_OVERLAY_SMALL_LOG_X_POS=(-5, 5),
-            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_X_POS=100,
+            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_X_POS=1000,
             VAR_OVERLAY_SMALL_LOG_X_POS=DoubleVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["x_pos"]),
             VAR_CURRENT_VALUE_OVERLAY_SMALL_LOG_X_POS=StringVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["x_pos"]),
 
             VAR_LABEL_OVERLAY_SMALL_LOG_Y_POS=StringVar(value=i18n.t("overlay_settings.y_position")),
-            SLIDER_RANGE_OVERLAY_SMALL_LOG_Y_POS=(-2, 2),
+            SLIDER_RANGE_OVERLAY_SMALL_LOG_Y_POS=(-5, 5),
-            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_Y_POS=160,
+            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_Y_POS=1000,
             VAR_OVERLAY_SMALL_LOG_Y_POS=DoubleVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["y_pos"]),
             VAR_CURRENT_VALUE_OVERLAY_SMALL_LOG_Y_POS=StringVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["y_pos"]),
 
             VAR_LABEL_OVERLAY_SMALL_LOG_DEPTH=StringVar(value=i18n.t("overlay_settings.depth")),
-            SLIDER_RANGE_OVERLAY_SMALL_LOG_DEPTH=(0, 2),
+            SLIDER_RANGE_OVERLAY_SMALL_LOG_DEPTH=(-5, 5),
-            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_DEPTH=100,
+            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_DEPTH=1000,
             VAR_OVERLAY_SMALL_LOG_DEPTH=DoubleVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["depth"]),
             VAR_CURRENT_VALUE_OVERLAY_SMALL_LOG_DEPTH=StringVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["depth"]),
 
             VAR_LABEL_OVERLAY_SMALL_LOG_X_ROTATION=StringVar(value="x_rotation"),
-            SLIDER_RANGE_OVERLAY_SMALL_LOG_X_ROTATION=(-1, 1),
+            SLIDER_RANGE_OVERLAY_SMALL_LOG_X_ROTATION=(-180, 180),
-            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_X_ROTATION=200,
+            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_X_ROTATION=360,
             VAR_OVERLAY_SMALL_LOG_X_ROTATION=DoubleVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["x_rotation"]),
             VAR_CURRENT_VALUE_OVERLAY_SMALL_LOG_X_ROTATION=StringVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["x_rotation"]),
 
             VAR_LABEL_OVERLAY_SMALL_LOG_Y_ROTATION=StringVar(value="y_rotation"),
-            SLIDER_RANGE_OVERLAY_SMALL_LOG_Y_ROTATION=(-1, 1),
+            SLIDER_RANGE_OVERLAY_SMALL_LOG_Y_ROTATION=(-180, 180),
-            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_Y_ROTATION=200,
+            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_Y_ROTATION=360,
             VAR_OVERLAY_SMALL_LOG_Y_ROTATION=DoubleVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["y_rotation"]),
             VAR_CURRENT_VALUE_OVERLAY_SMALL_LOG_Y_ROTATION=StringVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["y_rotation"]),
 
             VAR_LABEL_OVERLAY_SMALL_LOG_Z_ROTATION=StringVar(value="z_rotation"),
-            SLIDER_RANGE_OVERLAY_SMALL_LOG_Z_ROTATION=(-1, 1),
+            SLIDER_RANGE_OVERLAY_SMALL_LOG_Z_ROTATION=(-180, 180),
-            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_Z_ROTATION=200,
+            NUMBER_OF_STEPS_OVERLAY_SMALL_LOG_Z_ROTATION=360,
             VAR_OVERLAY_SMALL_LOG_Z_ROTATION=DoubleVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["z_rotation"]),
             VAR_CURRENT_VALUE_OVERLAY_SMALL_LOG_Z_ROTATION=StringVar(value=config.OVERLAY_SMALL_LOG_SETTINGS["z_rotation"]),
 
@@ -1165,8 +1165,8 @@ class View():
         }
         INIT_OVERLAY_SMALL_LOG_SETTINGS = {
             "x_pos": 0.0,
-            "y_pos": -0.41,
+            "y_pos": 0.0,
-            "depth": 1.0,
+            "depth": 0.0,
             "display_duration": 5,
             "fadeout_duration": 2,
         }