every_holiday/VRCT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'transcription' into develop

Browse Source
This commit is contained in:
misyaguziya
2023-06-11 02:41:19 +09:00
parent a2b62699e0 21a9edd43e
commit 1b28499972
1 changed files with 247 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

249
VRCT.py
View File

@@ -1,5 +1,6 @@
import os
import json
import threading
import deepl
import deepl_translate
import translators as ts
@@ -8,6 +9,8 @@ from pythonosc import udp_client
import tkinter as tk
import customtkinter
from PIL import Image
import pyaudio
import speech_recognition as sr
def save_json(path, key, value):
with open(path, "r") as fp:
@@ -87,6 +90,74 @@ class Translator():
pass
return result
# VoiceRecognizer
class VoiceRecognizer():
def __init__(self):
self.input_device_dict = self.search_input_device()
self.r = sr.Recognizer()
self.mic = None
self.languages = [
"ja-JP","en-US","en-GB","af-ZA","ar-DZ","ar-BH","ar-EG","ar-IL","ar-IQ","ar-JO","ar-KW","ar-LB","ar-MA",
"ar-OM","ar-PS","ar-QA","ar-SA","ar-TN","ar-AE","eu-ES","bg-BG","ca-ES","cmn-Hans-CN","cmn-Hans-HK",
"cmn-Hant-TW","yue-Hant-HK","hr_HR","cs-CZ","da-DK","en-AU","en-CA","en-IN","en-IE","en-NZ","en-PH",
"en-ZA","fa-IR","fr-FR","fil-PH","gl-ES","de-DE","el-GR","fi-FI","he-IL","hi-IN","hu-HU","id-ID","is-IS",
"it-IT","it-CH","ko-KR","lt-LT","ms-MY","nl-NL","nb-NO","pl-PL","pt-BR","pt-PT","ro-RO","ru-RU","sr-RS",
"sk-SK","sl-SI","es-AR","es-BO","es-CL","es-CO","es-CR","es-DO","es-EC","es-SV","es-GT","es-HN","es-MX",
"es-NI","es-PA","es-PY","es-PE","es-PR","es-ES","es-UY","es-US","es-VE","sv-SE","th-TH","tr-TR","uk-UA",
"vi-VN","zu-ZA"
]
def search_input_device(self):
pa = pyaudio.PyAudio()
input_device_dict = {}
mic_cnt = 1
for i in range(pa.get_device_count()):
device = pa.get_device_info_by_index(i)
try:
device["name"] = device["name"].encode('shift_jis').decode('utf-8')
except:
device["name"] = device["name"].encode('utf-8').decode('utf-8')
if device["maxInputChannels"] > 0:
input_device_dict[f'No.{mic_cnt}:{device["name"]}'] = device["index"]
mic_cnt += 1
pa.terminate()
return input_device_dict
def set_mic(self, device_name, threshold=50, is_dynamic=False):
if device_name in [v for v in self.input_device_dict.keys()]:
index = self.input_device_dict[device_name]
self.mic = sr.Microphone(device_index=index)
self.r.energy_threshold = threshold
if is_dynamic:
with self.mic as source:
self.r.adjust_for_ambient_noise(source, 3.0)
return True
else:
return False
def init_mic(self, threshold=50, is_dynamic=False):
if isinstance(self.mic, sr.Microphone):
self.r.energy_threshold = threshold
if is_dynamic:
with self.mic as source:
self.r.adjust_for_ambient_noise(source, 3.0)
return True
else:
return False
def listen_voice(self, language):
if self.mic != None:
with self.mic as source:
audio = self.r.listen(source)
try:
text = self.r.recognize_google(audio, language=language)
return text
except:
return ""
else:
return False
class ToplevelWindowInformation(customtkinter.CTkToplevel):
def __init__(self, parent, *args, **kwargs):
super().__init__(parent, *args, **kwargs)
@@ -194,8 +265,10 @@ class ToplevelWindowConfig(customtkinter.CTkToplevel):
self.tabview_config = customtkinter.CTkTabview(self)
self.tabview_config.grid(row=0, column=0, padx=5, pady=5, sticky="nsew")
self.tabview_config.add("GUI")
self.tabview_config.add("Transcription")
self.tabview_config.add("Parameter")
self.tabview_config.tab("GUI").grid_columnconfigure(2, weight=1)
self.tabview_config.tab("Transcription").grid_columnconfigure(2, weight=1)
self.tabview_config.tab("Parameter").grid_columnconfigure(1, weight=1)
self.tabview_config._segmented_button.configure(font=customtkinter.CTkFont(family=self.parent.FONT_FAMILY))
@@ -320,6 +393,79 @@ class ToplevelWindowConfig(customtkinter.CTkToplevel):
)
self.optionmenu_font_family.grid(row=5, column=1, columnspan=3, padx=5, pady=5, sticky="nsew")
# optionmenu mic device
self.label_mic_device = customtkinter.CTkLabel(
self.tabview_config.tab("Transcription"),
text="Select Mic Device:",
fg_color="transparent",
font=customtkinter.CTkFont(family=self.parent.FONT_FAMILY)
)
self.label_mic_device.grid(row=0, column=0, columnspan=1, padx=5, pady=5, sticky="nsw")
self.optionmenu_mic_device = customtkinter.CTkOptionMenu(
self.tabview_config.tab("Transcription"),
values=list(self.parent.vr.input_device_dict.keys()),
command=self.optionmenu_mic_device_callback,
font=customtkinter.CTkFont(family=self.parent.FONT_FAMILY),
variable=customtkinter.StringVar(value=self.parent.CHOICE_MIC_DEVICE)
)
self.optionmenu_mic_device.grid(row=0, column=1, columnspan=3 ,padx=5, pady=5, sticky="nsew")
# optionmenu input voice language
self.label_input_voice_language = customtkinter.CTkLabel(
self.tabview_config.tab("Transcription"),
text="Select Input Voice Language:",
fg_color="transparent",
font=customtkinter.CTkFont(family=self.parent.FONT_FAMILY)
)
self.label_input_voice_language.grid(row=1, column=0, columnspan=1, padx=5, pady=5, sticky="nsw")
self.optionmenu_input_voice_language = customtkinter.CTkOptionMenu(
self.tabview_config.tab("Transcription"),
values=list(self.parent.vr.languages),
# command=self.optionmenu_input_voice_language_callback,
font=customtkinter.CTkFont(family=self.parent.FONT_FAMILY),
variable=customtkinter.StringVar(value=self.parent.INPUT_VOICE_LANGUAGE)
)
self.optionmenu_input_voice_language.grid(row=1, column=1, columnspan=3 ,padx=5, pady=5, sticky="nsew")
# switch mic in dynamic
self.label_mic_is_dynamic = customtkinter.CTkLabel(
self.tabview_config.tab("Transcription"),
text="Select Mic IsDynamic:",
fg_color="transparent",
font=customtkinter.CTkFont(family=self.parent.FONT_FAMILY)
)
self.label_mic_is_dynamic.grid(row=2, column=0, columnspan=1, padx=5, pady=5, sticky="nsw")
self.checkbox_mic_is_dynamic = customtkinter.CTkCheckBox(
self.tabview_config.tab("Transcription"),
text="",
onvalue=True,
offvalue=False,
command=self.checkbox_mic_is_dynamic_callback,
font=customtkinter.CTkFont(family=self.parent.FONT_FAMILY)
)
self.checkbox_mic_is_dynamic.grid(row=2, column=1, columnspan=3 ,padx=5, pady=5, sticky="nsew")
if self.parent.ENABLE_MIC_IS_DYNAMIC is True:
self.checkbox_mic_is_dynamic.select()
else:
self.checkbox_mic_is_dynamic.deselect()
# switch mic threshold
self.label_mic_threshold = customtkinter.CTkLabel(
self.tabview_config.tab("Transcription"),
text="Select Mic Threshold:",
fg_color="transparent",
font=customtkinter.CTkFont(family=self.parent.FONT_FAMILY)
)
self.label_mic_threshold.grid(row=3, column=0, columnspan=1, padx=5, pady=5, sticky="nsw")
self.slider_mic_threshold = customtkinter.CTkSlider(
self.tabview_config.tab("Transcription"),
from_=0,
to=300,
command=self.slider_mic_threshold_callback,
variable=tk.DoubleVar(value=self.parent.MIC_THRESHOLD),
)
self.slider_mic_threshold.grid(row=3, column=1, columnspan=3 ,padx=5, pady=10, sticky="nsew")
# entry ip address
self.label_ip_address = customtkinter.CTkLabel(
self.tabview_config.tab("Parameter"),
@@ -478,6 +624,22 @@ class ToplevelWindowConfig(customtkinter.CTkToplevel):
self.parent.TARGET_LANG = choice
save_json(self.parent.PATH_CONFIG, "TARGET_LANG", self.parent.TARGET_LANG)
def optionmenu_mic_device_callback(self, choice):
self.parent.CHOICE_MIC_DEVICE = choice
save_json(self.parent.PATH_CONFIG, "CHOICE_MIC_DEVICE", self.parent.CHOICE_MIC_DEVICE)
self.parent.vr.set_mic(choice)
def checkbox_mic_is_dynamic_callback(self):
value = self.checkbox_mic_is_dynamic.get()
self.parent.ENABLE_MIC_IS_DYNAMIC = value
save_json(self.parent.PATH_CONFIG, "ENABLE_MIC_IS_DYNAMIC", self.parent.ENABLE_MIC_IS_DYNAMIC)
self.parent.vr.init_mic(threshold=self.parent.MIC_THRESHOLD, is_dynamic=self.parent.ENABLE_MIC_IS_DYNAMIC)
def slider_mic_threshold_callback(self, value):
self.parent.MIC_THRESHOLD = value
save_json(self.parent.PATH_CONFIG, "MIC_THRESHOLD", self.parent.MIC_THRESHOLD)
self.parent.vr.init_mic(threshold=self.parent.MIC_THRESHOLD, is_dynamic=self.parent.ENABLE_MIC_IS_DYNAMIC)
def optionmenu_theme_callback(self, choice):
customtkinter.set_appearance_mode(choice)
@@ -513,6 +675,12 @@ class ToplevelWindowConfig(customtkinter.CTkToplevel):
self.optionmenu_ui_scaling.configure(font=customtkinter.CTkFont(family=choice))
self.label_font_family.configure(font=customtkinter.CTkFont(family=choice))
self.optionmenu_font_family.configure(font=customtkinter.CTkFont(family=choice))
self.label_mic_device.configure(font=customtkinter.CTkFont(family=choice))
self.optionmenu_mic_device.configure(font=customtkinter.CTkFont(family=choice))
self.label_input_voice_language.configure(font=customtkinter.CTkFont(family=choice))
self.optionmenu_input_voice_language.configure(font=customtkinter.CTkFont(family=choice))
self.label_mic_is_dynamic.configure(font=customtkinter.CTkFont(family=choice))
self.label_mic_threshold.configure(font=customtkinter.CTkFont(family=choice))
self.label_ip_address.configure(font=customtkinter.CTkFont(family=choice))
self.entry_ip_address.configure(font=customtkinter.CTkFont(family=choice))
self.label_port.configure(font=customtkinter.CTkFont(family=choice))
@@ -545,6 +713,11 @@ class App(customtkinter.CTk):
"Google(web)": None,
}
self.MESSAGE_FORMAT = "[message]([translation])"
self.ENABLE_VOICE2CHAT = False
self.CHOICE_MIC_DEVICE = None
self.ENABLE_MIC_IS_DYNAMIC = False
self.MIC_THRESHOLD = 300
self.INPUT_VOICE_LANGUAGE = "ja-JP"
self.FONT_FAMILY = "Yu Gothic UI"
self.TRANSPARENCY = 100
self.APPEARANCE_THEME = "System"
@@ -574,6 +747,14 @@ class App(customtkinter.CTk):
self.MESSAGE_FORMAT = config["MESSAGE_FORMAT"]
if "FONT_FAMILY" in config.keys():
self.FONT_FAMILY = config["FONT_FAMILY"]
if "ENABLE_VOICE2CHAT" in config.keys():
self.ENABLE_VOICE2CHAT = config["ENABLE_VOICE2CHAT"]
if "CHOICE_MIC_DEVICE" in config.keys():
self.CHOICE_MIC_DEVICE = config["CHOICE_MIC_DEVICE"]
if "MIC_THRESHOLD" in config.keys():
self.MIC_THRESHOLD = config["MIC_THRESHOLD"]
if "INPUT_VOICE_LANGUAGE" in config.keys():
self.INPUT_VOICE_LANGUAGE = config["INPUT_VOICE_LANGUAGE"]
if "TRANSPARENCY" in config.keys():
self.TRANSPARENCY = config["TRANSPARENCY"]
if "APPEARANCE_THEME" in config.keys():
@@ -593,6 +774,9 @@ class App(customtkinter.CTk):
"AUTH_KEYS": self.AUTH_KEYS,
"MESSAGE_FORMAT": self.MESSAGE_FORMAT,
"FONT_FAMILY": self.FONT_FAMILY,
"ENABLE_VOICE2CHAT": self.ENABLE_VOICE2CHAT,
"CHOICE_MIC_DEVICE": self.CHOICE_MIC_DEVICE,
"INPUT_VOICE_LANGUAGE": self.INPUT_VOICE_LANGUAGE,
"TRANSPARENCY": self.TRANSPARENCY,
"APPEARANCE_THEME": self.APPEARANCE_THEME,
"UI_SCALING": self.UI_SCALING,
@@ -602,7 +786,7 @@ class App(customtkinter.CTk):
# init main window
self.iconbitmap(os.path.join(os.path.dirname(__file__), "img", "app.ico"))
self.title("VRCT")
self.geometry(f"{400}x{110}")
self.geometry(f"{400}x{140}")
self.minsize(400, 110)
self.grid_columnconfigure(1, weight=1)
self.grid_rowconfigure(0, weight=1)
@@ -623,6 +807,17 @@ class App(customtkinter.CTk):
)
self.checkbox_translation.grid(row=0, column=0, columnspan=2 ,padx=10, pady=(5, 5), sticky="we")
# add checkbox transcription
self.checkbox_transcription = customtkinter.CTkCheckBox(
self.sidebar_frame,
text="Transcription",
onvalue=True,
offvalue=False,
command=self.checkbox_transcription_callback,
font=customtkinter.CTkFont(family=self.FONT_FAMILY)
)
self.checkbox_transcription.grid(row=1, column=0, columnspan=2 ,padx=10, pady=(5, 5), sticky="we")
# add checkbox foreground
self.checkbox_foreground = customtkinter.CTkCheckBox(
self.sidebar_frame,
@@ -632,7 +827,7 @@ class App(customtkinter.CTk):
command=self.checkbox_foreground_callback,
font=customtkinter.CTkFont(family=self.FONT_FAMILY)
)
self.checkbox_foreground.grid(row=1, column=0, columnspan=2 ,padx=10, pady=(5, 5), sticky="we")
self.checkbox_foreground.grid(row=2, column=0, columnspan=2 ,padx=10, pady=(5, 5), sticky="we")
# add button information
self.button_information = customtkinter.CTkButton(
@@ -700,6 +895,10 @@ class App(customtkinter.CTk):
self.textbox_message_log.insert("0.0", f"Auth Keyを設定してないか間違っています\n")
self.textbox_message_log.configure(state='disabled')
## set voice2chat:
self.vr = VoiceRecognizer()
self.CHOICE_MIC_DEVICE = self.CHOICE_MIC_DEVICE if self.CHOICE_MIC_DEVICE is not None else list(self.vr.input_device_dict.keys())[0]
## set transparency for main window
self.wm_attributes("-alpha", self.TRANSPARENCY/100)
@@ -725,6 +924,52 @@ class App(customtkinter.CTk):
self.ENABLE_TRANSLATION = self.checkbox_translation.get()
save_json(self.PATH_CONFIG, "ENABLE_TRANSLATION", self.ENABLE_TRANSLATION)
def checkbox_transcription_callback(self):
if self.checkbox_transcription.get() is True:
# start threading
th = threading.Thread(target = self.voice_input)
th.start()
def voice_input(self):
mic_status = self.vr.set_mic(self.CHOICE_MIC_DEVICE)
self.vr.init_mic(threshold=self.MIC_THRESHOLD, is_dynamic=self.ENABLE_MIC_IS_DYNAMIC)
if mic_status:
# start voice_input
while self.checkbox_transcription.get() is True:
message = self.vr.listen_voice(language=self.INPUT_VOICE_LANGUAGE)
if len(message) > 0:
# translate
if self.checkbox_translation.get() is False:
chat_message = f"{message}"
elif (self.translator.translator_status[self.CHOICE_TRANSLATOR] is False) or (self.SOURCE_LANG == "None") or (self.TARGET_LANG == "None"):
self.textbox_message_log.configure(state='normal')
self.textbox_message_log.insert("0.0", f"Auth Keyもしくは言語の設定が間違っています\n")
self.textbox_message_log.configure(state='disabled')
chat_message = f"{message}"
else:
result = self.translator.translate(
translator_name=self.CHOICE_TRANSLATOR,
source_language=self.SOURCE_LANG,
target_language=self.TARGET_LANG,
message=message
)
chat_message = self.MESSAGE_FORMAT.replace("[message]", message).replace("[translation]", result)
# send OSC message
message = osc_message_builder.OscMessageBuilder(address="/chatbox/input")
message.add_arg(f"{chat_message}")
message.add_arg(True)
message.add_arg(True)
message = message.build()
client = udp_client.SimpleUDPClient(self.OSC_IP_ADDRESS, self.OSC_PORT)
client.send(message)
# update textbox message log
self.textbox_message_log.configure(state='normal')
self.textbox_message_log.insert("0.0", f"{chat_message}\n")
self.textbox_message_log.configure(state='disabled')
def checkbox_foreground_callback(self):
value = self.checkbox_foreground.get()