Merge branch 'encode' into for_webui

package-lock.json

@@ -19,6 +19,7 @@
         "eslint-plugin-react": "^7.34.2",
         "i18next": "^23.11.5",
         "jotai": "^2.8.3",
+        "js-base64": "^3.7.7",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
         "react-i18next": "^14.1.2",
@@ -4360,6 +4361,11 @@
         }
       }
     },
+    "node_modules/js-base64": {
+      "version": "3.7.7",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.7.tgz",
+      "integrity": "sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw=="
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",

package.json

@@ -27,6 +27,7 @@
     "eslint-plugin-react": "^7.34.2",
     "i18next": "^23.11.5",
     "jotai": "^2.8.3",
+    "js-base64": "^3.7.7",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-i18next": "^14.1.2",

src-python/utils.py

@@ -1,3 +1,4 @@
+import base64
 from typing import Any
 import json
 import random
@@ -73,13 +74,31 @@ def splitList(lst:list, split_count:int, to_shuffle:bool=False):
         split_lists.append(sub_list)
     return split_lists
 
-def encodeUtf8(data:str) -> dict:
+def encodeBase64(data:str) -> dict:
-    data = {i: byte for i, byte in enumerate(data.encode('UTF-8'))}
+    return json.loads(base64.b64decode(data).decode('utf-8'))
-    return data
-
-def decodeUtf8(data:dict) -> str:
-    data = bytes(data.values()).decode("UTF-8")
-    return data
 
 def printLog(log:str, data:Any=None) -> None:
-    print(json.dumps({"status":348, "log":log, "data":str(data)}), flush=True)
+    response = {
+        "status": 348,
+        "log": log,
+        "data": data,
+    }
+
+    with open('process.log', 'a', encoding="utf-8") as f:
+        f.write(f"log: {response}\n")
+
+    response = json.dumps(response)
+    print(response, flush=True)
+
+def printResponse(status:int, endpoint:str, result:Any=None) -> None:
+    response = {
+        "status": status,
+        "endpoint": endpoint,
+        "result": result,
+    }
+
+    with open('process.log', 'a', encoding="utf-8") as f:
+        f.write(f"log: {response}\n")
+
+    response = json.dumps(response)
+    print(response, flush=True)

src-python/webui_controller.py

@@ -5,7 +5,7 @@ from subprocess import Popen
 from threading import Thread
 from config import config
 from model import model
-from utils import getKeyByValue, isUniqueStrings, decodeUtf8, encodeUtf8, printLog
+from utils import getKeyByValue, isUniqueStrings, printLog, printResponse
 
 # Common
 class DownloadSoftwareProgressBar:
@@ -289,7 +289,7 @@ class ChatMessage:
 
     def send(self, data):
         id = data["id"]
-        message = decodeUtf8(data["message"])
+        message = data["message"]
         if len(message) > 0:
             # addSentMessageLog(message)
             translation = ""
@@ -329,8 +329,6 @@ class ChatMessage:
                     translation = f" ({translation})"
                 model.logger.info(f"[SENT] {message}{translation}")
 
-        message = encodeUtf8(message)
-        translation = encodeUtf8(translation)
         return {"status":200,
                 "result":{
                     "id":id,
@@ -1284,13 +1282,7 @@ def init(endpoints:dict, *args, **kwargs) -> None:
     printLog("Check Downloaded CTranslate2 Model Weight")
     if config.USE_TRANSLATION_FEATURE is True and model.checkCTranslatorCTranslate2ModelWeight() is False:
         def callback(progress):
-            print(json.dumps({
+            printResponse(200, endpoints["ctranslate2"], {"progress":progress})
-                "endpoint":endpoints["ctranslate2"],
-                "status":200,
-                "result":{
-                    "progress":progress
-                    }
-            }), flush=True)
         printLog("Download CTranslate2 Model Weight")
         model.downloadCTranslate2ModelWeight(callback)
 
@@ -1305,13 +1297,7 @@ def init(endpoints:dict, *args, **kwargs) -> None:
     printLog("Check Downloaded Whisper Model Weight")
     if config.USE_WHISPER_FEATURE is True and model.checkTranscriptionWhisperModelWeight() is False:
         def callback(progress):
-            print(json.dumps({
+            printResponse(200, endpoints["whisper"], {"progress":progress})
-                "endpoint":endpoints["whisper"],
-                "status":200,
-                "result":{
-                    "progress":progress
-                    }
-            }), flush=True)
         model.downloadWhisperModelWeight(callback)
 
     # set word filter

src-python/webui_mainloop.py

@@ -3,7 +3,7 @@ import json
 import time
 from config import config
 import webui_controller as controller
-from utils import printLog, encodeUtf8
+from utils import printLog, printResponse, encodeBase64
 
 config_mapping = {
     "/config/version": "VERSION",
@@ -270,51 +270,30 @@ class Action:
     def transmit(self, key:str, data:dict) -> None:
         status = data.get("status", None)
         result = data.get("result", None)
-        response = {
+        printResponse(status, self.endpoints[key], result)
-            "endpoint": self.endpoints[key],
-            "status": status,
-            "result": result,
-        }
-        response = json.dumps(response)
-        print(response, flush=True)
 
 def main():
     received_data = sys.stdin.readline().strip()
     received_data = json.loads(received_data)
 
-    with open('process.log', 'a') as f:
-        f.write(f"received_data: {received_data}\n")
-
     if received_data:
         endpoint = received_data.get("endpoint", None)
         data = received_data.get("data", None)
-
+        data = encodeBase64(data) if data is not None else None
-        with open('process.log', 'a') as f:
+        printLog(endpoint, data)
-            f.write(f"received_data : endpoint: {endpoint}, data:{data}\n")
 
         try:
             match endpoint.split("/")[1]:
                 case "config":
-                    result_data, status = handleConfigRequest(endpoint)
+                    result, status = handleConfigRequest(endpoint)
                 case "controller":
-                    result_data, status = handleControllerRequest(endpoint, data)
+                    result, status = handleControllerRequest(endpoint, data)
                 case _:
                     pass
         except Exception as e:
-            result_data = str(e)
+            result = str(e)
             status = 500
-
+        printResponse(status, endpoint, result)
-        response = {
-            "status": status,
-            "endpoint": endpoint,
-            "result": result_data,
-        }
-
-        response = json.dumps(response)
-        with open('process.log', 'a') as f:
-            f.write(f"response: {response}\n")
-
-        print(response, flush=True)
 
 if __name__ == "__main__":
     controller.init({
@@ -334,32 +313,18 @@ if __name__ == "__main__":
                     traceback.print_exc(file=f)
 
         case "test":
-            response_data, status = handleControllerRequest("/controller/callback_download_ctranslate2_weight")
+            endpoint = "/controller/callback_download_ctranslate2_weight"
-            response = {
+            result, status = handleControllerRequest(endpoint)
-                "status": status,
+            printResponse(status, endpoint, result)
-                "endpoint": "/controller/callback_download_ctranslate2_weight",
+            endpoint = "/controller/callback_download_whisper_weight"
-                "result": response_data,
+            result, status = handleControllerRequest(endpoint)
-            }
+            printResponse(status, endpoint, result)
-            response = json.dumps(response)
-            response_data, status = handleControllerRequest("/controller/callback_download_whisper_weight")
-            response = {
-                "status": status,
-                "endpoint": "/controller/callback_download_whisper_weight",
-                "result": response_data,
-            }
-            response = json.dumps(response)
 
         case "test_all":
             import time
             for endpoint, value in config_mapping.items():
-                response_data, status = handleConfigRequest(endpoint)
+                result, status = handleConfigRequest(endpoint)
-                response = {
+                printResponse(status, endpoint, result)
-                    "status": status,
-                    "endpoint": endpoint,
-                    "result": response_data,
-                }
-                response = json.dumps(response)
-                print(response, flush=True)
                 time.sleep(0.1)
 
             for endpoint, value in controller_mapping.items():
@@ -367,7 +332,7 @@ if __name__ == "__main__":
 
                 match endpoint:
                     case  "/controller/callback_messagebox_send":
-                        data = {"id":"123456", "message":encodeUtf8("テスト")}
+                        data = {"id":"123456", "message":"テスト"}
                     case "/controller/set_your_language_and_country":
                         data = {"language": "English", "country": "Hong Kong"}
                     case "/controller/set_target_language_and_country":
@@ -451,12 +416,6 @@ if __name__ == "__main__":
                     case _:
                         data = None
 
-                response_data, status = handleControllerRequest(endpoint, data)
+                result, status = handleControllerRequest(endpoint, data)
-                response = {
+                printResponse(status, endpoint, result)
-                    "status": status,
-                    "endpoint": endpoint,
-                    "result": response_data,
-                }
-                response = json.dumps(response)
-                print(response, flush=True)
                 time.sleep(0.5)

src-ui/logics/useMessage.js

@@ -7,14 +7,13 @@ import { useStdoutToPython } from "./useStdoutToPython";
 export const useMessage = () => {
     const { currentMessageLogsStatus, addMessageLogsStatus, updateMessageLogsStatus } = useMessageLogsStatus();
     const { asyncStdoutToPython } = useStdoutToPython();
-    const encoder = new TextEncoder();
 
     return {
         sendMessage: (message) => {
             const uuid = crypto.randomUUID();
             const send_message_object = {
                 id: uuid,
-                message: encoder.encode(message),
+                message: message,
             };
             asyncStdoutToPython("/controller/callback_messagebox_send", send_message_object);
 

src-ui/logics/useStdoutToPython.js

@@ -1,9 +1,10 @@
 import { store } from "@store";
+import { encode } from 'js-base64'
 
 export const useStdoutToPython = () => {
     const asyncStdoutToPython = async (path, value) => {
         let send_object = { endpoint: path };
-        if (value) send_object.data = value;
+        if (value) send_object.data = encode(JSON.stringify(value));
 
         // send to python
         const backend_subprocess = store.backend_subprocess;