From 8dac3da93357645fc9b086cd7090a9cc5bcf77bd Mon Sep 17 00:00:00 2001 From: misyaguziya <53165965+misyaguziya@users.noreply.github.com> Date: Tue, 27 May 2025 14:12:20 +0900 Subject: [PATCH 1/3] [Update] Add hf_xet and huggingface-hub to requirements; enhance mainloop.py for Hugging Face caching --- backend.spec | 2 +- backend_cuda.spec | 2 +- install.bat | 11 +++++++++++ requirements.txt | 2 ++ requirements_cuda.txt | 2 ++ src-python/mainloop.py | 9 ++++++++- 6 files changed, 25 insertions(+), 3 deletions(-) diff --git a/backend.spec b/backend.spec index 33ec3cae..3933d942 100644 --- a/backend.spec +++ b/backend.spec @@ -5,7 +5,7 @@ a = Analysis( ['src-python\\mainloop.py'], pathex=[], binaries=[], - datas=[('./fonts', 'fonts/'), ('.venv/Lib/site-packages/zeroconf', 'zeroconf/'), ('.venv/Lib/site-packages/openvr', 'openvr/'), ('.venv/Lib/site-packages/pykakasi', 'pykakasi/'), ('.venv/Lib/site-packages/faster_whisper', 'faster_whisper/')], + datas=[('./fonts', 'fonts/'), ('.venv/Lib/site-packages/zeroconf', 'zeroconf/'), ('.venv/Lib/site-packages/openvr', 'openvr/'), ('.venv/Lib/site-packages/pykakasi', 'pykakasi/'), ('.venv/Lib/site-packages/faster_whisper', 'faster_whisper/'), ('.venv/Lib/site-packages/hf_xet', 'hf_xet/')], hiddenimports=[], hookspath=[], hooksconfig={}, diff --git a/backend_cuda.spec b/backend_cuda.spec index 34ed248f..08ba5fd7 100644 --- a/backend_cuda.spec +++ b/backend_cuda.spec @@ -5,7 +5,7 @@ a = Analysis( ['src-python\\mainloop.py'], pathex=[], binaries=[], - datas=[('./fonts', 'fonts/'), ('.venv_cuda/Lib/site-packages/zeroconf', 'zeroconf/'), ('.venv_cuda/Lib/site-packages/openvr', 'openvr/'), ('.venv_cuda/Lib/site-packages/pykakasi', 'pykakasi/'), ('.venv_cuda/Lib/site-packages/faster_whisper', 'faster_whisper/')], + datas=[('./fonts', 'fonts/'), ('.venv_cuda/Lib/site-packages/zeroconf', 'zeroconf/'), ('.venv_cuda/Lib/site-packages/openvr', 'openvr/'), ('.venv_cuda/Lib/site-packages/pykakasi', 'pykakasi/'), ('.venv_cuda/Lib/site-packages/faster_whisper', 'faster_whisper/'), ('.venv/Lib/site-packages/hf_xet', 'hf_xet/')], hiddenimports=[], hookspath=[], hooksconfig={}, diff --git a/install.bat b/install.bat index 5a57e817..69c5152b 100644 --- a/install.bat +++ b/install.bat @@ -1,6 +1,17 @@ +REM .venv .venv_cuda があれば削除 +if exist .venv ( + rmdir /s /q .venv +) + +if exist .venv_cuda ( + rmdir /s /q .venv_cuda +) + +REM .venv .venv_cuda を作成 python -m venv .venv python -m venv .venv_cuda +REM .venv .venv_cuda に必要なパッケージをインストール call .venv/Scripts/activate python.exe -m pip install --upgrade pip pip install --no-cache-dir --force-reinstall -r requirements.txt diff --git a/requirements.txt b/requirements.txt index 95501ea2..1f5cc9ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,8 @@ psutil==5.9.8 pykakasi==2.3.0 pycaw==20240210 websockets==15.0.1 +huggingface-hub==0.31.2 +hf-xet==1.1.2 translators @ git+https://github.com/misyaguziya/translators@5.9.2.1 SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4.1 tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.3 \ No newline at end of file diff --git a/requirements_cuda.txt b/requirements_cuda.txt index 768f45ba..347031cd 100644 --- a/requirements_cuda.txt +++ b/requirements_cuda.txt @@ -17,6 +17,8 @@ psutil==5.9.8 pykakasi==2.3.0 pycaw==20240210 websockets==15.0.1 +huggingface-hub==0.31.2 +hf-xet==1.1.2 translators @ git+https://github.com/misyaguziya/translators@5.9.2.1 SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4.1 tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.3 \ No newline at end of file diff --git a/src-python/mainloop.py b/src-python/mainloop.py index d8ad52e7..f5e2318b 100644 --- a/src-python/mainloop.py +++ b/src-python/mainloop.py @@ -1,9 +1,16 @@ +import os import sys import json import time from typing import Any from threading import Thread from queue import Queue +import warnings +warnings.filterwarnings('ignore') +huggingface_cache_dir = os.path.join(os.path.dirname(__file__), ".cache") +os.makedirs(huggingface_cache_dir, exist_ok=True) +os.environ["HF_HOME"] = os.path.join(huggingface_cache_dir) + from controller import Controller from utils import printLog, printResponse, errorLogging, encodeBase64 @@ -376,7 +383,7 @@ class Main: if status == 423: self.queue.put((endpoint, data)) else: - printLog(endpoint, {"send_data":result}) + printLog(endpoint, {"status": status, "send_data": result}) printResponse(status, endpoint, result) time.sleep(0.1) From 238bd41109b085d63b8fc32febc470169ebc1332 Mon Sep 17 00:00:00 2001 From: misyaguziya <53165965+misyaguziya@users.noreply.github.com> Date: Wed, 28 May 2025 11:14:39 +0900 Subject: [PATCH 2/3] [Update] Refactor Hugging Face dependencies in requirements files and enhance cache directory handling in mainloop.py --- requirements.txt | 4 ++-- requirements_cuda.txt | 4 ++-- src-python/mainloop.py | 14 +++++++++++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1f5cc9ed..c6b7ba57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,8 +16,8 @@ psutil==5.9.8 pykakasi==2.3.0 pycaw==20240210 websockets==15.0.1 -huggingface-hub==0.31.2 -hf-xet==1.1.2 +huggingface_hub[hf-xet] +setuptools==80.8.0 translators @ git+https://github.com/misyaguziya/translators@5.9.2.1 SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4.1 tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.3 \ No newline at end of file diff --git a/requirements_cuda.txt b/requirements_cuda.txt index 347031cd..00293ea3 100644 --- a/requirements_cuda.txt +++ b/requirements_cuda.txt @@ -17,8 +17,8 @@ psutil==5.9.8 pykakasi==2.3.0 pycaw==20240210 websockets==15.0.1 -huggingface-hub==0.31.2 -hf-xet==1.1.2 +huggingface[hub] +setuptools==80.8.0 translators @ git+https://github.com/misyaguziya/translators@5.9.2.1 SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4.1 tinyoscquery @ git+https://github.com/cyberkitsune/tinyoscquery@0.1.3 \ No newline at end of file diff --git a/src-python/mainloop.py b/src-python/mainloop.py index f5e2318b..c2429a58 100644 --- a/src-python/mainloop.py +++ b/src-python/mainloop.py @@ -7,9 +7,17 @@ from threading import Thread from queue import Queue import warnings warnings.filterwarnings('ignore') -huggingface_cache_dir = os.path.join(os.path.dirname(__file__), ".cache") -os.makedirs(huggingface_cache_dir, exist_ok=True) -os.environ["HF_HOME"] = os.path.join(huggingface_cache_dir) + +if getattr(sys, 'frozen', False): + cache_dir = os.path.join(os.path.dirname(sys.executable), ".cache") + hub_dir = os.path.join(os.path.dirname(sys.executable), ".cache", "hub") +else: + cache_dir = os.path.join(os.path.dirname(__file__), ".cache") + hub_dir = os.path.join(os.path.dirname(__file__), ".cache", "hub") + +os.makedirs(cache_dir, exist_ok=True) +os.makedirs(hub_dir, exist_ok=True) +os.environ["HF_HOME"] = os.path.join(cache_dir) from controller import Controller from utils import printLog, printResponse, errorLogging, encodeBase64 From 67d06ab1e1859e34b1fd4ea0939b4d91769801fd Mon Sep 17 00:00:00 2001 From: misyaguziya <53165965+misyaguziya@users.noreply.github.com> Date: Thu, 29 May 2025 08:04:06 +0900 Subject: [PATCH 3/3] [Update] Enhance build scripts and requirements for Hugging Face integration; improve logging and error handling --- build.bat | 2 +- build_cuda.bat | 2 +- install.bat | 20 ++++++++++++-------- requirements.txt | 3 ++- requirements_cuda.txt | 3 ++- src-python/controller.py | 2 +- src-python/mainloop.py | 20 ++++---------------- 7 files changed, 23 insertions(+), 29 deletions(-) diff --git a/build.bat b/build.bat index 89ee8cbb..3c08629c 100644 --- a/build.bat +++ b/build.bat @@ -1,2 +1,2 @@ call .venv/Scripts/activate -pyinstaller backend.spec --distpath src-tauri/bin --clean --noconfirm \ No newline at end of file +pyinstaller backend.spec --distpath src-tauri/bin --clean --noconfirm --log-level ERROR \ No newline at end of file diff --git a/build_cuda.bat b/build_cuda.bat index 73c85676..308b9174 100644 --- a/build_cuda.bat +++ b/build_cuda.bat @@ -1,2 +1,2 @@ call .venv_cuda/Scripts/activate -pyinstaller backend_cuda.spec --distpath src-tauri/bin --clean --noconfirm \ No newline at end of file +pyinstaller backend_cuda.spec --distpath src-tauri/bin --clean --noconfirm --log-level ERROR \ No newline at end of file diff --git a/install.bat b/install.bat index 69c5152b..aeb3d9b5 100644 --- a/install.bat +++ b/install.bat @@ -1,21 +1,25 @@ -REM .venv .venv_cuda があれば削除 +REM .venv exists if exist .venv ( rmdir /s /q .venv ) -if exist .venv_cuda ( - rmdir /s /q .venv_cuda -) - -REM .venv .venv_cuda を作成 +REM make .venv python -m venv .venv -python -m venv .venv_cuda -REM .venv .venv_cuda に必要なパッケージをインストール +REM install packages for .venv call .venv/Scripts/activate python.exe -m pip install --upgrade pip pip install --no-cache-dir --force-reinstall -r requirements.txt +REM if .venv_cuda exists +if exist .venv_cuda ( + rmdir /s /q .venv_cuda +) + +REM make .venv_cuda +python -m venv .venv_cuda + +REM install packages for .venv_cuda call .venv_cuda/Scripts/activate python.exe -m pip install --upgrade pip pip install --no-cache-dir --force-reinstall -r requirements_cuda.txt \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c6b7ba57..244a9238 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,8 @@ psutil==5.9.8 pykakasi==2.3.0 pycaw==20240210 websockets==15.0.1 -huggingface_hub[hf-xet] +huggingface_hub==0.32.2 +hf-xet==1.1.2 setuptools==80.8.0 translators @ git+https://github.com/misyaguziya/translators@5.9.2.1 SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4.1 diff --git a/requirements_cuda.txt b/requirements_cuda.txt index 00293ea3..4dadf192 100644 --- a/requirements_cuda.txt +++ b/requirements_cuda.txt @@ -17,7 +17,8 @@ psutil==5.9.8 pykakasi==2.3.0 pycaw==20240210 websockets==15.0.1 -huggingface[hub] +huggingface_hub==0.32.2 +hf-xet==1.1.2 setuptools==80.8.0 translators @ git+https://github.com/misyaguziya/translators@5.9.2.1 SpeechRecognition @ git+https://github.com/misyaguziya/custom_speech_recognition@3.10.4.1 diff --git a/src-python/controller.py b/src-python/controller.py index c2db0f45..f74b01ca 100644 --- a/src-python/controller.py +++ b/src-python/controller.py @@ -1,4 +1,4 @@ -from typing import Callable, Union, Any +from typing import Callable, Any from time import sleep from subprocess import Popen from threading import Thread diff --git a/src-python/mainloop.py b/src-python/mainloop.py index c2429a58..6b684b38 100644 --- a/src-python/mainloop.py +++ b/src-python/mainloop.py @@ -1,26 +1,14 @@ -import os import sys import json import time from typing import Any from threading import Thread from queue import Queue -import warnings -warnings.filterwarnings('ignore') +import logging +from controller import Controller # noqa: E402 +from utils import printLog, printResponse, errorLogging, encodeBase64 # noqa: E402 -if getattr(sys, 'frozen', False): - cache_dir = os.path.join(os.path.dirname(sys.executable), ".cache") - hub_dir = os.path.join(os.path.dirname(sys.executable), ".cache", "hub") -else: - cache_dir = os.path.join(os.path.dirname(__file__), ".cache") - hub_dir = os.path.join(os.path.dirname(__file__), ".cache", "hub") - -os.makedirs(cache_dir, exist_ok=True) -os.makedirs(hub_dir, exist_ok=True) -os.environ["HF_HOME"] = os.path.join(cache_dir) - -from controller import Controller -from utils import printLog, printResponse, errorLogging, encodeBase64 +logging.getLogger("huggingface_hub").setLevel(logging.ERROR) run_mapping = { "connected_network":"/run/connected_network",