every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into plugins_system

Browse Source 
# Conflicts:
#	src-tauri/tauri.conf.json
This commit is contained in:
Sakamoto Shiina
2025-04-24 02:20:00 +09:00
parent 33f1e54f4d 5aa5c0068e
commit e9aa3aa8ac
42 changed files with 716 additions and 589 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src-python/config.py
View File

@@ -956,7 +956,7 @@ class Config:
def init_config(self):
# Read Only
self._VERSION = "3.0.4"
self._VERSION = "3.0.5"
if getattr(sys, 'frozen', False):
self._PATH_LOCAL = os_path.dirname(sys.executable)
else:

30
src-python/controller.py
View File

@@ -6,7 +6,7 @@ import re
from device_manager import device_manager
from config import config
from model import model
from utils import removeLog, printLog, errorLogging, isConnectedNetwork
from utils import removeLog, printLog, errorLogging, isConnectedNetwork, isValidIpAddress
class Controller:
def __init__(self) -> None:
@@ -150,7 +150,7 @@ class Controller:
400,
self.run_mapping["error_device"],
{
"message":"No Speaker device detected",
"message":"No speaker device detected",
"data": None
},
)
@@ -1094,9 +1094,29 @@ class Controller:
@staticmethod
def setOscIpAddress(data, *args, **kwargs) -> dict:
config.OSC_IP_ADDRESS = data
model.setOscIpAddress(config.OSC_IP_ADDRESS)
return {"status":200, "result":config.OSC_IP_ADDRESS}
if isValidIpAddress(data) is False:
response = {
"status":400,
"result":{
"message":"Invalid IP address",
"data": config.OSC_IP_ADDRESS
}
}
else:
try:
model.setOscIpAddress(data)
config.OSC_IP_ADDRESS = data
response = {"status":200, "result":config.OSC_IP_ADDRESS}
except Exception:
model.setOscIpAddress(config.OSC_IP_ADDRESS)
response = {
"status":400,
"result":{
"message":"Cannot set IP address",
"data": config.OSC_IP_ADDRESS
}
}
return response
@staticmethod
def getOscPort(*args, **kwargs) -> dict:

357
src-python/model.py
View File

@@ -400,83 +400,83 @@ class Model:
mic_device_list = device_manager.getMicDevices().get(mic_host_name, [{"name": "NoDevice"}])
selected_mic_device = [device for device in mic_device_list if device["name"] == mic_device_name]
if len(selected_mic_device) == 0:
return False
if len(selected_mic_device) == 0 or mic_device_name == "NoDevice":
fnc({"text": False, "language": None})
else:
self.mic_audio_queue = Queue()
# self.mic_energy_queue = Queue()
self.mic_audio_queue = Queue()
# self.mic_energy_queue = Queue()
mic_device = selected_mic_device[0]
record_timeout = config.MIC_RECORD_TIMEOUT
phrase_timeout = config.MIC_PHRASE_TIMEOUT
if record_timeout > phrase_timeout:
record_timeout = phrase_timeout
mic_device = selected_mic_device[0]
record_timeout = config.MIC_RECORD_TIMEOUT
phrase_timeout = config.MIC_PHRASE_TIMEOUT
if record_timeout > phrase_timeout:
record_timeout = phrase_timeout
self.mic_audio_recorder = SelectedMicEnergyAndAudioRecorder(
device=mic_device,
energy_threshold=config.MIC_THRESHOLD,
dynamic_energy_threshold=config.MIC_AUTOMATIC_THRESHOLD,
phrase_time_limit=record_timeout,
)
# self.mic_audio_recorder.recordIntoQueue(self.mic_audio_queue, mic_energy_queue)
self.mic_audio_recorder.recordIntoQueue(self.mic_audio_queue, None)
self.mic_transcriber = AudioTranscriber(
speaker=False,
source=self.mic_audio_recorder.source,
phrase_timeout=phrase_timeout,
max_phrases=config.MIC_MAX_PHRASES,
transcription_engine=config.SELECTED_TRANSCRIPTION_ENGINE,
root=config.PATH_LOCAL,
whisper_weight_type=config.WHISPER_WEIGHT_TYPE,
device=config.SELECTED_TRANSCRIPTION_COMPUTE_DEVICE["device"],
device_index=config.SELECTED_TRANSCRIPTION_COMPUTE_DEVICE["device_index"],
)
def sendMicTranscript():
try:
selected_your_languages = config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]
languages = [data["language"] for data in selected_your_languages.values() if data["enable"] is True]
countries = [data["country"] for data in selected_your_languages.values() if data["enable"] is True]
if isinstance(self.mic_transcriber, AudioTranscriber) is True:
res = self.mic_transcriber.transcribeAudioQueue(
self.mic_audio_queue,
languages,
countries,
config.MIC_AVG_LOGPROB,
config.MIC_NO_SPEECH_PROB
)
if res:
result = self.mic_transcriber.getTranscript()
fnc(result)
except Exception:
errorLogging()
self.mic_audio_recorder = SelectedMicEnergyAndAudioRecorder(
device=mic_device,
energy_threshold=config.MIC_THRESHOLD,
dynamic_energy_threshold=config.MIC_AUTOMATIC_THRESHOLD,
phrase_time_limit=record_timeout,
)
# self.mic_audio_recorder.recordIntoQueue(self.mic_audio_queue, mic_energy_queue)
self.mic_audio_recorder.recordIntoQueue(self.mic_audio_queue, None)
self.mic_transcriber = AudioTranscriber(
speaker=False,
source=self.mic_audio_recorder.source,
phrase_timeout=phrase_timeout,
max_phrases=config.MIC_MAX_PHRASES,
transcription_engine=config.SELECTED_TRANSCRIPTION_ENGINE,
root=config.PATH_LOCAL,
whisper_weight_type=config.WHISPER_WEIGHT_TYPE,
device=config.SELECTED_TRANSCRIPTION_COMPUTE_DEVICE["device"],
device_index=config.SELECTED_TRANSCRIPTION_COMPUTE_DEVICE["device_index"],
)
def sendMicTranscript():
try:
selected_your_languages = config.SELECTED_YOUR_LANGUAGES[config.SELECTED_TAB_NO]
languages = [data["language"] for data in selected_your_languages.values() if data["enable"] is True]
countries = [data["country"] for data in selected_your_languages.values() if data["enable"] is True]
if isinstance(self.mic_transcriber, AudioTranscriber) is True:
res = self.mic_transcriber.transcribeAudioQueue(
self.mic_audio_queue,
languages,
countries,
config.MIC_AVG_LOGPROB,
config.MIC_NO_SPEECH_PROB
)
if res:
result = self.mic_transcriber.getTranscript()
fnc(result)
except Exception:
errorLogging()
def endMicTranscript():
while not self.mic_audio_queue.empty():
self.mic_audio_queue.get()
# while not self.mic_energy_queue.empty():
# self.mic_energy_queue.get()
self.mic_transcriber = None
gc.collect()
def endMicTranscript():
while not self.mic_audio_queue.empty():
self.mic_audio_queue.get()
# while not self.mic_energy_queue.empty():
# self.mic_energy_queue.get()
self.mic_transcriber = None
gc.collect()
# def sendMicEnergy():
# if mic_energy_queue.empty() is False:
# energy = mic_energy_queue.get()
# # print("mic energy:", energy)
# try:
# fnc(energy)
# except Exception:
# pass
# sleep(0.01)
# def sendMicEnergy():
# if mic_energy_queue.empty() is False:
# energy = mic_energy_queue.get()
# # print("mic energy:", energy)
# try:
# fnc(energy)
# except Exception:
# pass
# sleep(0.01)
self.mic_print_transcript = threadFnc(sendMicTranscript, end_fnc=endMicTranscript)
self.mic_print_transcript.daemon = True
self.mic_print_transcript.start()
self.mic_print_transcript = threadFnc(sendMicTranscript, end_fnc=endMicTranscript)
self.mic_print_transcript.daemon = True
self.mic_print_transcript.start()
# self.mic_get_energy = threadFnc(sendMicEnergy)
# self.mic_get_energy.daemon = True
# self.mic_get_energy.start()
# self.mic_get_energy = threadFnc(sendMicEnergy)
# self.mic_get_energy.daemon = True
# self.mic_get_energy.start()
self.changeMicTranscriptStatus()
self.changeMicTranscriptStatus()
def resumeMicTranscript(self):
# キューをクリア
@@ -535,25 +535,25 @@ class Model:
mic_device_list = device_manager.getMicDevices().get(mic_host_name, [{"name": "NoDevice"}])
selected_mic_device = [device for device in mic_device_list if device["name"] == mic_device_name]
if len(selected_mic_device) == 0:
return False
if len(selected_mic_device) == 0 or mic_device_name == "NoDevice":
self.check_mic_energy_fnc(False)
else:
def sendMicEnergy():
if mic_energy_queue.empty() is False:
energy = mic_energy_queue.get()
try:
self.check_mic_energy_fnc(energy)
except Exception:
errorLogging()
sleep(0.01)
def sendMicEnergy():
if mic_energy_queue.empty() is False:
energy = mic_energy_queue.get()
try:
self.check_mic_energy_fnc(energy)
except Exception:
errorLogging()
sleep(0.01)
mic_energy_queue = Queue()
mic_device = selected_mic_device[0]
self.mic_energy_recorder = SelectedMicEnergyRecorder(mic_device)
self.mic_energy_recorder.recordIntoQueue(mic_energy_queue)
self.mic_energy_plot_progressbar = threadFnc(sendMicEnergy)
self.mic_energy_plot_progressbar.daemon = True
self.mic_energy_plot_progressbar.start()
mic_energy_queue = Queue()
mic_device = selected_mic_device[0]
self.mic_energy_recorder = SelectedMicEnergyRecorder(mic_device)
self.mic_energy_recorder.recordIntoQueue(mic_energy_queue)
self.mic_energy_plot_progressbar = threadFnc(sendMicEnergy)
self.mic_energy_plot_progressbar.daemon = True
self.mic_energy_plot_progressbar.start()
def stopCheckMicEnergy(self):
if isinstance(self.mic_energy_plot_progressbar, threadFnc):
@@ -566,83 +566,85 @@ class Model:
self.mic_energy_recorder = None
def startSpeakerTranscript(self, fnc):
speaker_device_name = config.SELECTED_SPEAKER_DEVICE
speaker_device_list = device_manager.getSpeakerDevices()
selected_speaker_device = [device for device in speaker_device_list if device["name"] == config.SELECTED_SPEAKER_DEVICE]
selected_speaker_device = [device for device in speaker_device_list if device["name"] == speaker_device_name]
if len(selected_speaker_device) == 0:
return False
if len(selected_speaker_device) == 0 or speaker_device_name == "NoDevice":
fnc({"text": False, "language": None})
else:
speaker_audio_queue = Queue()
# speaker_energy_queue = Queue()
speaker_device = selected_speaker_device[0]
record_timeout = config.SPEAKER_RECORD_TIMEOUT
phrase_timeout = config.SPEAKER_PHRASE_TIMEOUT
if record_timeout > phrase_timeout:
record_timeout = phrase_timeout
speaker_audio_queue = Queue()
# speaker_energy_queue = Queue()
speaker_device = selected_speaker_device[0]
record_timeout = config.SPEAKER_RECORD_TIMEOUT
phrase_timeout = config.SPEAKER_PHRASE_TIMEOUT
if record_timeout > phrase_timeout:
record_timeout = phrase_timeout
self.speaker_audio_recorder = SelectedSpeakerEnergyAndAudioRecorder(
device=speaker_device,
energy_threshold=config.SPEAKER_THRESHOLD,
dynamic_energy_threshold=config.SPEAKER_AUTOMATIC_THRESHOLD,
phrase_time_limit=record_timeout,
)
# self.speaker_audio_recorder.recordIntoQueue(speaker_audio_queue, speaker_energy_queue)
self.speaker_audio_recorder.recordIntoQueue(speaker_audio_queue, None)
self.speaker_transcriber = AudioTranscriber(
speaker=True,
source=self.speaker_audio_recorder.source,
phrase_timeout=phrase_timeout,
max_phrases=config.SPEAKER_MAX_PHRASES,
transcription_engine=config.SELECTED_TRANSCRIPTION_ENGINE,
root=config.PATH_LOCAL,
whisper_weight_type=config.WHISPER_WEIGHT_TYPE,
device=config.SELECTED_TRANSCRIPTION_COMPUTE_DEVICE["device"],
device_index=config.SELECTED_TRANSCRIPTION_COMPUTE_DEVICE["device_index"],
)
def sendSpeakerTranscript():
try:
selected_target_languages = config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]
languages = [data["language"] for data in selected_target_languages.values() if data["enable"] is True]
countries = [data["country"] for data in selected_target_languages.values() if data["enable"] is True]
if isinstance(self.speaker_transcriber, AudioTranscriber) is True:
res = self.speaker_transcriber.transcribeAudioQueue(
speaker_audio_queue,
languages,
countries,
config.SPEAKER_AVG_LOGPROB,
config.SPEAKER_NO_SPEECH_PROB
)
if res:
result = self.speaker_transcriber.getTranscript()
fnc(result)
except Exception:
errorLogging()
self.speaker_audio_recorder = SelectedSpeakerEnergyAndAudioRecorder(
device=speaker_device,
energy_threshold=config.SPEAKER_THRESHOLD,
dynamic_energy_threshold=config.SPEAKER_AUTOMATIC_THRESHOLD,
phrase_time_limit=record_timeout,
)
# self.speaker_audio_recorder.recordIntoQueue(speaker_audio_queue, speaker_energy_queue)
self.speaker_audio_recorder.recordIntoQueue(speaker_audio_queue, None)
self.speaker_transcriber = AudioTranscriber(
speaker=True,
source=self.speaker_audio_recorder.source,
phrase_timeout=phrase_timeout,
max_phrases=config.SPEAKER_MAX_PHRASES,
transcription_engine=config.SELECTED_TRANSCRIPTION_ENGINE,
root=config.PATH_LOCAL,
whisper_weight_type=config.WHISPER_WEIGHT_TYPE,
device=config.SELECTED_TRANSCRIPTION_COMPUTE_DEVICE["device"],
device_index=config.SELECTED_TRANSCRIPTION_COMPUTE_DEVICE["device_index"],
)
def sendSpeakerTranscript():
try:
selected_target_languages = config.SELECTED_TARGET_LANGUAGES[config.SELECTED_TAB_NO]
languages = [data["language"] for data in selected_target_languages.values() if data["enable"] is True]
countries = [data["country"] for data in selected_target_languages.values() if data["enable"] is True]
if isinstance(self.speaker_transcriber, AudioTranscriber) is True:
res = self.speaker_transcriber.transcribeAudioQueue(
speaker_audio_queue,
languages,
countries,
config.SPEAKER_AVG_LOGPROB,
config.SPEAKER_NO_SPEECH_PROB
)
if res:
result = self.speaker_transcriber.getTranscript()
fnc(result)
except Exception:
errorLogging()
def endSpeakerTranscript():
while not speaker_audio_queue.empty():
speaker_audio_queue.get()
# while not speaker_energy_queue.empty():
# speaker_energy_queue.get()
self.speaker_transcriber = None
gc.collect()
def endSpeakerTranscript():
while not speaker_audio_queue.empty():
speaker_audio_queue.get()
# while not speaker_energy_queue.empty():
# speaker_energy_queue.get()
self.speaker_transcriber = None
gc.collect()
# def sendSpeakerEnergy():
# if speaker_energy_queue.empty() is False:
# energy = speaker_energy_queue.get()
# # print("speaker energy:", energy)
# try:
# fnc(energy)
# except Exception:
# pass
# sleep(0.01)
# def sendSpeakerEnergy():
# if speaker_energy_queue.empty() is False:
# energy = speaker_energy_queue.get()
# # print("speaker energy:", energy)
# try:
# fnc(energy)
# except Exception:
# pass
# sleep(0.01)
self.speaker_print_transcript = threadFnc(sendSpeakerTranscript, end_fnc=endSpeakerTranscript)
self.speaker_print_transcript.daemon = True
self.speaker_print_transcript.start()
self.speaker_print_transcript = threadFnc(sendSpeakerTranscript, end_fnc=endSpeakerTranscript)
self.speaker_print_transcript.daemon = True
self.speaker_print_transcript.start()
# self.speaker_get_energy = threadFnc(sendSpeakerEnergy)
# self.speaker_get_energy.daemon = True
# self.speaker_get_energy.start()
# self.speaker_get_energy = threadFnc(sendSpeakerEnergy)
# self.speaker_get_energy.daemon = True
# self.speaker_get_energy.start()
def stopSpeakerTranscript(self):
if isinstance(self.speaker_print_transcript, threadFnc):
@@ -660,28 +662,29 @@ class Model:
if isinstance(fnc, Callable):
self.check_speaker_energy_fnc = fnc
speaker_device_name = config.SELECTED_SPEAKER_DEVICE
speaker_device_list = device_manager.getSpeakerDevices()
selected_speaker_device = [device for device in speaker_device_list if device["name"] == config.SELECTED_SPEAKER_DEVICE]
selected_speaker_device = [device for device in speaker_device_list if device["name"] == speaker_device_name]
if len(selected_speaker_device) == 0:
return False
if len(selected_speaker_device) == 0 or speaker_device_name == "NoDevice":
self.check_speaker_energy_fnc(False)
else:
def sendSpeakerEnergy():
if speaker_energy_queue.empty() is False:
energy = speaker_energy_queue.get()
try:
self.check_speaker_energy_fnc(energy)
except Exception:
errorLogging()
sleep(0.01)
def sendSpeakerEnergy():
if speaker_energy_queue.empty() is False:
energy = speaker_energy_queue.get()
try:
self.check_speaker_energy_fnc(energy)
except Exception:
errorLogging()
sleep(0.01)
speaker_energy_queue = Queue()
speaker_device = selected_speaker_device[0]
self.speaker_energy_recorder = SelectedSpeakerEnergyRecorder(speaker_device)
self.speaker_energy_recorder.recordIntoQueue(speaker_energy_queue)
self.speaker_energy_plot_progressbar = threadFnc(sendSpeakerEnergy)
self.speaker_energy_plot_progressbar.daemon = True
self.speaker_energy_plot_progressbar.start()
speaker_energy_queue = Queue()
speaker_device = selected_speaker_device[0]
self.speaker_energy_recorder = SelectedSpeakerEnergyRecorder(speaker_device)
self.speaker_energy_recorder.recordIntoQueue(speaker_energy_queue)
self.speaker_energy_plot_progressbar = threadFnc(sendSpeakerEnergy)
self.speaker_energy_plot_progressbar.daemon = True
self.speaker_energy_plot_progressbar.start()
def stopCheckSpeakerEnergy(self):
if isinstance(self.speaker_energy_plot_progressbar, threadFnc):

2
src-python/models/transcription/transcription_whisper.py
View File

@@ -17,6 +17,8 @@ _MODELS = {
"large-v1": "Systran/faster-whisper-large-v1",
"large-v2": "Systran/faster-whisper-large-v2",
"large-v3": "Systran/faster-whisper-large-v3",
"large-v3-turbo-int8": "Zoont/faster-whisper-large-v3-turbo-int8-ct2", #794MB
"large-v3-turbo": "deepdml/faster-whisper-large-v3-turbo-ct2", #1.58GB
}
_FILENAMES = [

10
src-python/utils.py
View File

@@ -7,14 +7,22 @@ from logging.handlers import RotatingFileHandler
from ctranslate2 import get_supported_compute_types
import requests
import ipaddress
def isConnectedNetwork(url="http://www.google.com", timeout=3):
def isConnectedNetwork(url="http://www.google.com", timeout=3) -> bool:
try:
response = requests.get(url, timeout=timeout)
return response.status_code == 200
except requests.RequestException:
return False
def isValidIpAddress(ip_address: str) -> bool:
try:
ipaddress.ip_address(ip_address)
return True
except ValueError:
return False
def getBestComputeType(device, device_index) -> str:
compute_types = get_supported_compute_types(device, device_index)
compute_types = set(compute_types)