every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'encode' into for_webui

Browse Source
This commit is contained in:
misyaguziya
2024-09-03 19:36:40 +09:00
parent 87d2ecef34 9b7a614b8b
commit 1fc2ad6824
7 changed files with 60 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
package-lock.json generated
View File

@@ -19,6 +19,7 @@
"eslint-plugin-react": "^7.34.2",
"i18next": "^23.11.5",
"jotai": "^2.8.3",
"js-base64": "^3.7.7",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-i18next": "^14.1.2",
@@ -4360,6 +4361,11 @@
}
}
},
"node_modules/js-base64": {
"version": "3.7.7",
"resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.7.tgz",
"integrity": "sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw=="
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",

1
package.json
View File

@@ -27,6 +27,7 @@
"eslint-plugin-react": "^7.34.2",
"i18next": "^23.11.5",
"jotai": "^2.8.3",
"js-base64": "^3.7.7",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-i18next": "^14.1.2",

35
src-python/utils.py
View File

@@ -1,3 +1,4 @@
import base64
from typing import Any
import json
import random
@@ -73,13 +74,31 @@ def splitList(lst:list, split_count:int, to_shuffle:bool=False):
split_lists.append(sub_list)
return split_lists
def encodeUtf8(data:str) -> dict:
data = {i: byte for i, byte in enumerate(data.encode('UTF-8'))}
return data
def decodeUtf8(data:dict) -> str:
data = bytes(data.values()).decode("UTF-8")
return data
def encodeBase64(data:str) -> dict:
return json.loads(base64.b64decode(data).decode('utf-8'))
def printLog(log:str, data:Any=None) -> None:
print(json.dumps({"status":348, "log":log, "data":str(data)}), flush=True)
response = {
"status": 348,
"log": log,
"data": data,
}
with open('process.log', 'a', encoding="utf-8") as f:
f.write(f"log: {response}\n")
response = json.dumps(response)
print(response, flush=True)
def printResponse(status:int, endpoint:str, result:Any=None) -> None:
response = {
"status": status,
"endpoint": endpoint,
"result": result,
}
with open('process.log', 'a', encoding="utf-8") as f:
f.write(f"log: {response}\n")
response = json.dumps(response)
print(response, flush=True)

22
src-python/webui_controller.py
View File

@@ -5,7 +5,7 @@ from subprocess import Popen
from threading import Thread
from config import config
from model import model
from utils import getKeyByValue, isUniqueStrings, decodeUtf8, encodeUtf8, printLog
from utils import getKeyByValue, isUniqueStrings, printLog, printResponse
# Common
class DownloadSoftwareProgressBar:
@@ -289,7 +289,7 @@ class ChatMessage:
def send(self, data):
id = data["id"]
message = decodeUtf8(data["message"])
message = data["message"]
if len(message) > 0:
# addSentMessageLog(message)
translation = ""
@@ -329,8 +329,6 @@ class ChatMessage:
translation = f" ({translation})"
model.logger.info(f"[SENT] {message}{translation}")
message = encodeUtf8(message)
translation = encodeUtf8(translation)
return {"status":200,
"result":{
"id":id,
@@ -1284,13 +1282,7 @@ def init(endpoints:dict, *args, **kwargs) -> None:
printLog("Check Downloaded CTranslate2 Model Weight")
if config.USE_TRANSLATION_FEATURE is True and model.checkCTranslatorCTranslate2ModelWeight() is False:
def callback(progress):
print(json.dumps({
"endpoint":endpoints["ctranslate2"],
"status":200,
"result":{
"progress":progress
}
}), flush=True)
printResponse(200, endpoints["ctranslate2"], {"progress":progress})
printLog("Download CTranslate2 Model Weight")
model.downloadCTranslate2ModelWeight(callback)
@@ -1305,13 +1297,7 @@ def init(endpoints:dict, *args, **kwargs) -> None:
printLog("Check Downloaded Whisper Model Weight")
if config.USE_WHISPER_FEATURE is True and model.checkTranscriptionWhisperModelWeight() is False:
def callback(progress):
print(json.dumps({
"endpoint":endpoints["whisper"],
"status":200,
"result":{
"progress":progress
}
}), flush=True)
printResponse(200, endpoints["whisper"], {"progress":progress})
model.downloadWhisperModelWeight(callback)
# set word filter

79
src-python/webui_mainloop.py
View File

@@ -3,7 +3,7 @@ import json
import time
from config import config
import webui_controller as controller
from utils import printLog, encodeUtf8
from utils import printLog, printResponse, encodeBase64
config_mapping = {
"/config/version": "VERSION",
@@ -270,51 +270,30 @@ class Action:
def transmit(self, key:str, data:dict) -> None:
status = data.get("status", None)
result = data.get("result", None)
response = {
"endpoint": self.endpoints[key],
"status": status,
"result": result,
}
response = json.dumps(response)
print(response, flush=True)
printResponse(status, self.endpoints[key], result)
def main():
received_data = sys.stdin.readline().strip()
received_data = json.loads(received_data)
with open('process.log', 'a') as f:
f.write(f"received_data: {received_data}\n")
if received_data:
endpoint = received_data.get("endpoint", None)
data = received_data.get("data", None)
with open('process.log', 'a') as f:
f.write(f"received_data : endpoint: {endpoint}, data:{data}\n")
data = encodeBase64(data) if data is not None else None
printLog(endpoint, data)
try:
match endpoint.split("/")[1]:
case "config":
result_data, status = handleConfigRequest(endpoint)
result, status = handleConfigRequest(endpoint)
case "controller":
result_data, status = handleControllerRequest(endpoint, data)
result, status = handleControllerRequest(endpoint, data)
case _:
pass
except Exception as e:
result_data = str(e)
result = str(e)
status = 500
response = {
"status": status,
"endpoint": endpoint,
"result": result_data,
}
response = json.dumps(response)
with open('process.log', 'a') as f:
f.write(f"response: {response}\n")
print(response, flush=True)
printResponse(status, endpoint, result)
if __name__ == "__main__":
controller.init({
@@ -334,32 +313,18 @@ if __name__ == "__main__":
traceback.print_exc(file=f)
case "test":
response_data, status = handleControllerRequest("/controller/callback_download_ctranslate2_weight")
response = {
"status": status,
"endpoint": "/controller/callback_download_ctranslate2_weight",
"result": response_data,
}
response = json.dumps(response)
response_data, status = handleControllerRequest("/controller/callback_download_whisper_weight")
response = {
"status": status,
"endpoint": "/controller/callback_download_whisper_weight",
"result": response_data,
}
response = json.dumps(response)
endpoint = "/controller/callback_download_ctranslate2_weight"
result, status = handleControllerRequest(endpoint)
printResponse(status, endpoint, result)
endpoint = "/controller/callback_download_whisper_weight"
result, status = handleControllerRequest(endpoint)
printResponse(status, endpoint, result)
case "test_all":
import time
for endpoint, value in config_mapping.items():
response_data, status = handleConfigRequest(endpoint)
response = {
"status": status,
"endpoint": endpoint,
"result": response_data,
}
response = json.dumps(response)
print(response, flush=True)
result, status = handleConfigRequest(endpoint)
printResponse(status, endpoint, result)
time.sleep(0.1)
for endpoint, value in controller_mapping.items():
@@ -367,7 +332,7 @@ if __name__ == "__main__":
match endpoint:
case "/controller/callback_messagebox_send":
data = {"id":"123456", "message":encodeUtf8("テスト")}
data = {"id":"123456", "message":"テスト"}
case "/controller/set_your_language_and_country":
data = {"language": "English", "country": "Hong Kong"}
case "/controller/set_target_language_and_country":
@@ -451,12 +416,6 @@ if __name__ == "__main__":
case _:
data = None
response_data, status = handleControllerRequest(endpoint, data)
response = {
"status": status,
"endpoint": endpoint,
"result": response_data,
}
response = json.dumps(response)
print(response, flush=True)
result, status = handleControllerRequest(endpoint, data)
printResponse(status, endpoint, result)
time.sleep(0.5)

3
src-ui/logics/useMessage.js
View File

@@ -7,14 +7,13 @@ import { useStdoutToPython } from "./useStdoutToPython";
export const useMessage = () => {
const { currentMessageLogsStatus, addMessageLogsStatus, updateMessageLogsStatus } = useMessageLogsStatus();
const { asyncStdoutToPython } = useStdoutToPython();
const encoder = new TextEncoder();
return {
sendMessage: (message) => {
const uuid = crypto.randomUUID();
const send_message_object = {
id: uuid,
message: encoder.encode(message),
message: message,
};
asyncStdoutToPython("/controller/callback_messagebox_send", send_message_object);

3
src-ui/logics/useStdoutToPython.js
View File

@@ -1,9 +1,10 @@
import { store } from "@store";
import { encode } from 'js-base64'
export const useStdoutToPython = () => {
const asyncStdoutToPython = async (path, value) => {
let send_object = { endpoint: path };
if (value) send_object.data = value;
if (value) send_object.data = encode(JSON.stringify(value));
// send to python
const backend_subprocess = store.backend_subprocess;