From fb0a4a8f26016b1b83b84f59d13579618a95610c Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Tue, 3 Sep 2024 14:47:22 +0900 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=91=8D=EF=B8=8F[Update]=20Model=20:?= =?UTF-8?q?=20tauri=20->=20python=20=E3=81=AE=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E9=83=A8=E5=88=86=E3=82=92base64=E3=81=A7encode=E3=81=99?= =?UTF-8?q?=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 6 ++++++ package.json | 1 + src-python/utils.py | 8 -------- src-python/webui_controller.py | 6 ++---- src-python/webui_mainloop.py | 7 +++++-- src-ui/logics/useMessage.js | 3 +-- src-ui/logics/useStdoutToPython.js | 3 ++- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/package-lock.json b/package-lock.json index b69e0b4c..f6d3e2e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "eslint-plugin-react": "^7.34.2", "i18next": "^23.11.5", "jotai": "^2.8.3", + "js-base64": "^3.7.7", "react": "^18.2.0", "react-dom": "^18.2.0", "react-i18next": "^14.1.2", @@ -4360,6 +4361,11 @@ } } }, + "node_modules/js-base64": { + "version": "3.7.7", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.7.tgz", + "integrity": "sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw==" + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", diff --git a/package.json b/package.json index d3e91e6f..6834071a 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "eslint-plugin-react": "^7.34.2", "i18next": "^23.11.5", "jotai": "^2.8.3", + "js-base64": "^3.7.7", "react": "^18.2.0", "react-dom": "^18.2.0", "react-i18next": "^14.1.2", diff --git a/src-python/utils.py b/src-python/utils.py index 05a41b7e..f1ab1c5e 100644 --- a/src-python/utils.py +++ b/src-python/utils.py @@ -73,13 +73,5 @@ def splitList(lst:list, split_count:int, to_shuffle:bool=False): split_lists.append(sub_list) return split_lists -def encodeUtf8(data:str) -> dict: - data = {i: byte for i, byte in enumerate(data.encode('UTF-8'))} - return data - -def decodeUtf8(data:dict) -> str: - data = bytes(data.values()).decode("UTF-8") - return data - def printLog(log:str, data:Any=None) -> None: print(json.dumps({"status":348, "log":log, "data":str(data)}), flush=True) \ No newline at end of file diff --git a/src-python/webui_controller.py b/src-python/webui_controller.py index f10a63fb..12660461 100644 --- a/src-python/webui_controller.py +++ b/src-python/webui_controller.py @@ -5,7 +5,7 @@ from subprocess import Popen from threading import Thread from config import config from model import model -from utils import getKeyByValue, isUniqueStrings, decodeUtf8, encodeUtf8, printLog +from utils import getKeyByValue, isUniqueStrings, printLog # Common class DownloadSoftwareProgressBar: @@ -289,7 +289,7 @@ class ChatMessage: def send(self, data): id = data["id"] - message = decodeUtf8(data["message"]) + message = data["message"] if len(message) > 0: # addSentMessageLog(message) translation = "" @@ -329,8 +329,6 @@ class ChatMessage: translation = f" ({translation})" model.logger.info(f"[SENT] {message}{translation}") - message = encodeUtf8(message) - translation = encodeUtf8(translation) return {"status":200, "result":{ "id":id, diff --git a/src-python/webui_mainloop.py b/src-python/webui_mainloop.py index c38453cc..9adb6b8c 100644 --- a/src-python/webui_mainloop.py +++ b/src-python/webui_mainloop.py @@ -3,7 +3,8 @@ import json import time from config import config import webui_controller as controller -from utils import printLog, encodeUtf8 +from utils import printLog +import base64 config_mapping = { "/config/version": "VERSION", @@ -288,6 +289,8 @@ def main(): if received_data: endpoint = received_data.get("endpoint", None) data = received_data.get("data", None) + if data is not None: + data = json.loads(base64.b64decode(data).decode('utf-8')) with open('process.log', 'a') as f: f.write(f"received_data : endpoint: {endpoint}, data:{data}\n") @@ -367,7 +370,7 @@ if __name__ == "__main__": match endpoint: case "/controller/callback_messagebox_send": - data = {"id":"123456", "message":encodeUtf8("テスト")} + data = {"id":"123456", "message":"テスト"} case "/controller/set_your_language_and_country": data = {"language": "English", "country": "Hong Kong"} case "/controller/set_target_language_and_country": diff --git a/src-ui/logics/useMessage.js b/src-ui/logics/useMessage.js index 8e74c081..ebdb1b19 100644 --- a/src-ui/logics/useMessage.js +++ b/src-ui/logics/useMessage.js @@ -7,14 +7,13 @@ import { useStdoutToPython } from "./useStdoutToPython"; export const useMessage = () => { const { currentMessageLogsStatus, addMessageLogsStatus, updateMessageLogsStatus } = useMessageLogsStatus(); const { asyncStdoutToPython } = useStdoutToPython(); - const encoder = new TextEncoder(); return { sendMessage: (message) => { const uuid = crypto.randomUUID(); const send_message_object = { id: uuid, - message: encoder.encode(message), + message: message, }; asyncStdoutToPython("/controller/callback_messagebox_send", send_message_object); diff --git a/src-ui/logics/useStdoutToPython.js b/src-ui/logics/useStdoutToPython.js index cde6c121..553cf207 100644 --- a/src-ui/logics/useStdoutToPython.js +++ b/src-ui/logics/useStdoutToPython.js @@ -1,9 +1,10 @@ import { store } from "@store"; +import { encode } from 'js-base64' export const useStdoutToPython = () => { const asyncStdoutToPython = async (path, value) => { let send_object = { endpoint: path }; - if (value) send_object.data = value; + if (value) send_object.data = encode(value); // send to python const backend_subprocess = store.backend_subprocess; From 7ae86268a9df40708e6fde85cc68d297bf2111eb Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Tue, 3 Sep 2024 18:18:58 +0900 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=91=8D=EF=B8=8F[Update]=20Model=20:?= =?UTF-8?q?=20tauri=20->=20python=20=E3=81=AEencode=E5=87=A6=E7=90=86?= =?UTF-8?q?=E3=82=92=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src-python/utils.py | 29 ++++++++++- src-python/webui_controller.py | 18 ++----- src-python/webui_mainloop.py | 82 +++++++----------------------- src-ui/logics/useStdoutToPython.js | 2 +- 4 files changed, 51 insertions(+), 80 deletions(-) diff --git a/src-python/utils.py b/src-python/utils.py index f1ab1c5e..2ff40bb8 100644 --- a/src-python/utils.py +++ b/src-python/utils.py @@ -1,3 +1,4 @@ +import base64 from typing import Any import json import random @@ -73,5 +74,31 @@ def splitList(lst:list, split_count:int, to_shuffle:bool=False): split_lists.append(sub_list) return split_lists +def encodeBase64(data:str) -> dict: + return json.loads(base64.b64decode(data).decode('utf-8')) + def printLog(log:str, data:Any=None) -> None: - print(json.dumps({"status":348, "log":log, "data":str(data)}), flush=True) \ No newline at end of file + response = { + "status": 348, + "log": log, + "data": data, + } + + with open('process.log', 'a', encoding="utf-8") as f: + f.write(f"log: {response}\n") + + response = json.dumps(response) + print(response, flush=True) + +def printResponse(status:int, endpoint:str, result:Any=None) -> None: + response = { + "status": status, + "endpoint": endpoint, + "result": result, + } + + with open('process.log', 'a', encoding="utf-8") as f: + f.write(f"log: {response}\n") + + response = json.dumps(response) + print(response, flush=True) \ No newline at end of file diff --git a/src-python/webui_controller.py b/src-python/webui_controller.py index 12660461..afef99ad 100644 --- a/src-python/webui_controller.py +++ b/src-python/webui_controller.py @@ -5,7 +5,7 @@ from subprocess import Popen from threading import Thread from config import config from model import model -from utils import getKeyByValue, isUniqueStrings, printLog +from utils import getKeyByValue, isUniqueStrings, printLog, printResponse # Common class DownloadSoftwareProgressBar: @@ -1282,13 +1282,7 @@ def init(endpoints:dict, *args, **kwargs) -> None: printLog("Check Downloaded CTranslate2 Model Weight") if config.USE_TRANSLATION_FEATURE is True and model.checkCTranslatorCTranslate2ModelWeight() is False: def callback(progress): - print(json.dumps({ - "endpoint":endpoints["ctranslate2"], - "status":200, - "result":{ - "progress":progress - } - }), flush=True) + printResponse(200, endpoints["ctranslate2"], {"progress":progress}) printLog("Download CTranslate2 Model Weight") model.downloadCTranslate2ModelWeight(callback) @@ -1303,13 +1297,7 @@ def init(endpoints:dict, *args, **kwargs) -> None: printLog("Check Downloaded Whisper Model Weight") if config.USE_WHISPER_FEATURE is True and model.checkTranscriptionWhisperModelWeight() is False: def callback(progress): - print(json.dumps({ - "endpoint":endpoints["whisper"], - "status":200, - "result":{ - "progress":progress - } - }), flush=True) + printResponse(200, endpoints["whisper"], {"progress":progress}) model.downloadWhisperModelWeight(callback) # set word filter diff --git a/src-python/webui_mainloop.py b/src-python/webui_mainloop.py index 9adb6b8c..70f87475 100644 --- a/src-python/webui_mainloop.py +++ b/src-python/webui_mainloop.py @@ -3,8 +3,7 @@ import json import time from config import config import webui_controller as controller -from utils import printLog -import base64 +from utils import printLog, printResponse, encodeBase64 config_mapping = { "/config/version": "VERSION", @@ -271,53 +270,30 @@ class Action: def transmit(self, key:str, data:dict) -> None: status = data.get("status", None) result = data.get("result", None) - response = { - "endpoint": self.endpoints[key], - "status": status, - "result": result, - } - response = json.dumps(response) - print(response, flush=True) + printResponse(status, self.endpoints[key], result) def main(): received_data = sys.stdin.readline().strip() received_data = json.loads(received_data) - with open('process.log', 'a') as f: - f.write(f"received_data: {received_data}\n") - if received_data: endpoint = received_data.get("endpoint", None) data = received_data.get("data", None) - if data is not None: - data = json.loads(base64.b64decode(data).decode('utf-8')) - - with open('process.log', 'a') as f: - f.write(f"received_data : endpoint: {endpoint}, data:{data}\n") + data = encodeBase64(data) if data is not None else None + printLog(endpoint, data) try: match endpoint.split("/")[1]: case "config": - result_data, status = handleConfigRequest(endpoint) + result, status = handleConfigRequest(endpoint) case "controller": - result_data, status = handleControllerRequest(endpoint, data) + result, status = handleControllerRequest(endpoint, data) case _: pass except Exception as e: - result_data = str(e) + result = str(e) status = 500 - - response = { - "status": status, - "endpoint": endpoint, - "result": result_data, - } - - response = json.dumps(response) - with open('process.log', 'a') as f: - f.write(f"response: {response}\n") - - print(response, flush=True) + printResponse(status, endpoint, result) if __name__ == "__main__": controller.init({ @@ -325,7 +301,7 @@ if __name__ == "__main__": "whisper": action_mapping["/controller/callback_download_whisper_weight"]["download"], }) - process = "main" + process = "test_all" match process: case "main": try: @@ -337,32 +313,18 @@ if __name__ == "__main__": traceback.print_exc(file=f) case "test": - response_data, status = handleControllerRequest("/controller/callback_download_ctranslate2_weight") - response = { - "status": status, - "endpoint": "/controller/callback_download_ctranslate2_weight", - "result": response_data, - } - response = json.dumps(response) - response_data, status = handleControllerRequest("/controller/callback_download_whisper_weight") - response = { - "status": status, - "endpoint": "/controller/callback_download_whisper_weight", - "result": response_data, - } - response = json.dumps(response) + endpoint = "/controller/callback_download_ctranslate2_weight" + result, status = handleControllerRequest(endpoint) + printResponse(status, endpoint, result) + endpoint = "/controller/callback_download_whisper_weight" + result, status = handleControllerRequest(endpoint) + printResponse(status, endpoint, result) case "test_all": import time for endpoint, value in config_mapping.items(): - response_data, status = handleConfigRequest(endpoint) - response = { - "status": status, - "endpoint": endpoint, - "result": response_data, - } - response = json.dumps(response) - print(response, flush=True) + result, status = handleConfigRequest(endpoint) + printResponse(status, endpoint, result) time.sleep(0.1) for endpoint, value in controller_mapping.items(): @@ -454,12 +416,6 @@ if __name__ == "__main__": case _: data = None - response_data, status = handleControllerRequest(endpoint, data) - response = { - "status": status, - "endpoint": endpoint, - "result": response_data, - } - response = json.dumps(response) - print(response, flush=True) + result, status = handleControllerRequest(endpoint, data) + printResponse(status, endpoint, result) time.sleep(0.5) \ No newline at end of file diff --git a/src-ui/logics/useStdoutToPython.js b/src-ui/logics/useStdoutToPython.js index 553cf207..03dc39a7 100644 --- a/src-ui/logics/useStdoutToPython.js +++ b/src-ui/logics/useStdoutToPython.js @@ -4,7 +4,7 @@ import { encode } from 'js-base64' export const useStdoutToPython = () => { const asyncStdoutToPython = async (path, value) => { let send_object = { endpoint: path }; - if (value) send_object.data = encode(value); + if (value) send_object.data = encode(JSON.stringify(value)); // send to python const backend_subprocess = store.backend_subprocess; From 9b7a614b8b4df35fb74420ef6de47d6f742fe7fd Mon Sep 17 00:00:00 2001 From: misyaguziya Date: Tue, 3 Sep 2024 19:36:26 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=90=9B[bugfix]=20Model=20:=20typo?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src-python/webui_mainloop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-python/webui_mainloop.py b/src-python/webui_mainloop.py index 70f87475..3eb3f567 100644 --- a/src-python/webui_mainloop.py +++ b/src-python/webui_mainloop.py @@ -301,7 +301,7 @@ if __name__ == "__main__": "whisper": action_mapping["/controller/callback_download_whisper_weight"]["download"], }) - process = "test_all" + process = "main" match process: case "main": try: