diff --git a/.gitignore b/.gitignore index 2d0f243..2848c67 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ avatar3d/models_archivio/ # Varie *.log .DS_Store +.venv-chatterbox/ +tts_chatterbox/test_*.py diff --git a/avatar/assistant.py b/avatar/assistant.py index ce08624..c14cbf6 100644 --- a/avatar/assistant.py +++ b/avatar/assistant.py @@ -233,8 +233,9 @@ class Assistant: self.settings.set("kokoro_voice", voice) self.settings.save() else: - label = "Sistema" if self.settings.get("tts_engine") == "system" else \ - {"if_sara": "Sara", "im_nicola": "Nicola"}.get(self.settings.get("kokoro_voice"), "Sara") + eng = self.settings.get("tts_engine") + label = "Sistema" if eng == "system" else ("Sara" if eng == "chatterbox" else + {"if_sara": "Sara", "im_nicola": "Nicola"}.get(self.settings.get("kokoro_voice"), "Sara")) try: if cm.get_voice() != label: cm.save_voice(label) diff --git a/avatar/settings.py b/avatar/settings.py index d7891c4..d10d9f4 100644 --- a/avatar/settings.py +++ b/avatar/settings.py @@ -24,6 +24,9 @@ DEFAULTS: dict[str, Any] = { "claudecode_path": "", "tts_engine": "kokoro", # kokoro | system "kokoro_voice": "if_sara", + "chatterbox_exaggeration": 0.6, # 0.3 sobria … 0.9 molto enfatica + "chatterbox_cfg": 0.3, # più basso = più veloce e meno aderente al testo + "chatterbox_ref": "", # wav di riferimento per clonare una voce (facoltativo) "system_voice": "", # "" = automatica "stt_model": "mlx-community/whisper-small-mlx", "vad_threshold": 0.08, diff --git a/avatar/settings_dialog.py b/avatar/settings_dialog.py index 637acab..9081241 100644 --- a/avatar/settings_dialog.py +++ b/avatar/settings_dialog.py @@ -99,11 +99,16 @@ class SettingsDialog(QDialog): self.stack.setCurrentIndex(self.provider.currentIndex()) form2 = QFormLayout() - self.tts_engine = _combo([("kokoro", "Kokoro, voce neurale in locale"), ("system", "Voce di sistema (macOS)")], s.get("tts_engine")) + self.tts_engine = _combo([("kokoro", "Kokoro, voce neurale in locale"), ("chatterbox", "Chatterbox, espressiva in locale (lenta)"), ("system", "Voce di sistema (macOS)")], s.get("tts_engine")) form2.addRow("Motore voce", self.tts_engine) self.kokoro_voice = _combo(list(KOKORO_VOICES.items()), s.get("kokoro_voice")); form2.addRow("Voce Kokoro", self.kokoro_voice) self.system_voice = _combo([("", "Automatica (Alice)")] + [(v, v) for v in SystemVoice.list_voices()], s.get("system_voice")) form2.addRow("Voce di sistema", self.system_voice) + row = QHBoxLayout() + self.cb_exag = QDoubleSpinBox(); self.cb_exag.setRange(0.2, 1.0); self.cb_exag.setSingleStep(0.1); self.cb_exag.setValue(float(s.get("chatterbox_exaggeration", 0.6))); row.addWidget(QLabel("enfasi")); row.addWidget(self.cb_exag) + self.cb_cfg = QDoubleSpinBox(); self.cb_cfg.setRange(0.0, 1.0); self.cb_cfg.setSingleStep(0.1); self.cb_cfg.setValue(float(s.get("chatterbox_cfg", 0.3))); row.addWidget(QLabel("aderenza (0 = più veloce)")); row.addWidget(self.cb_cfg) + self.cb_ref = QLineEdit(s.get("chatterbox_ref") or ""); self.cb_ref.setPlaceholderText("wav di riferimento per clonare una voce (facoltativo)"); row.addWidget(self.cb_ref, 1) + form2.addRow("Chatterbox", row) self.stt_model = _combo(STT_MODELS, s.get("stt_model")); form2.addRow("Riconoscimento vocale", self.stt_model) from .avatar3d import list_models models = list_models() @@ -341,6 +346,7 @@ class SettingsDialog(QDialog): "claudecode_config_dir": self.cc_config.currentText().strip(), "claudecode_path": self.cc_path.text().strip(), "tts_engine": self.tts_engine.currentData(), "kokoro_voice": self.kokoro_voice.currentData(), "system_voice": self.system_voice.currentData(), "stt_model": self.stt_model.currentData(), + "chatterbox_exaggeration": float(self.cb_exag.value()), "chatterbox_cfg": float(self.cb_cfg.value()), "chatterbox_ref": self.cb_ref.text().strip(), "vad_threshold": float(self.vad.value()), "avatar_model": self.avatar_model.currentData() or "", "telegram_api_id": self.tg_id.text().strip(), diff --git a/avatar/tts.py b/avatar/tts.py index 9c5bc0c..3994d3b 100644 --- a/avatar/tts.py +++ b/avatar/tts.py @@ -132,7 +132,76 @@ class SystemVoice: return data +class ChatterboxVoice: + """Chatterbox (Resemble AI) in un ambiente separato, avviato come servizio locale.""" + + PORT = 8792 + _proc = None + + def __init__(self, exaggeration: float = 0.6, cfg: float = 0.3, ref_audio: str = "", on_status=None) -> None: + self.exaggeration, self.cfg, self.ref_audio = exaggeration, cfg, ref_audio + self.voice = f"chatterbox:{exaggeration}:{cfg}:{ref_audio}" + self._on_status = on_status or (lambda m: None) + + @classmethod + def _start(cls) -> None: + import subprocess + from pathlib import Path as _P + if cls._proc is not None and cls._proc.poll() is None: + return + base = _P(__file__).resolve().parent.parent + python = base / ".venv-chatterbox" / "bin" / "python" + if not python.exists(): + raise RuntimeError("Chatterbox non installato: esegui `uv venv .venv-chatterbox --python 3.12 && uv pip install --python .venv-chatterbox/bin/python chatterbox-tts 'setuptools<81'`.") + log = open(base / "data" / "chatterbox.log", "a") + cls._proc = subprocess.Popen([str(python), str(base / "tts_chatterbox" / "server.py"), "--port", str(cls.PORT)], + stdout=log, stderr=subprocess.STDOUT, cwd=str(base)) + + def load(self) -> None: + import requests + import time as _t + self._start() + self._on_status("Avvio Chatterbox (la prima volta scarica il modello, circa 2 GB)…") + try: + for _ in range(600): # fino a 10 minuti (download iniziale) + try: + st = requests.get(f"http://127.0.0.1:{self.PORT}/", timeout=3).json() + if st.get("ready"): + return + if st.get("error"): + raise RuntimeError(st["error"]) + except requests.RequestException: + pass + if self._proc is not None and self._proc.poll() is not None: + raise RuntimeError("Il servizio Chatterbox si è chiuso: vedi data/chatterbox.log.") + _t.sleep(1) + raise RuntimeError("Chatterbox non è pronto.") + finally: + self._on_status(None) + + def synthesize(self, text: str) -> np.ndarray: + import requests + r = requests.post(f"http://127.0.0.1:{self.PORT}/tts", json={"text": text, "language": "it", "exaggeration": self.exaggeration, + "cfg": self.cfg, "ref": self.ref_audio or None}, timeout=600) + if r.status_code != 200: + raise RuntimeError(r.json().get("error", r.text)) + sr = int(r.headers.get("X-Sample-Rate", "24000")) + data = np.frombuffer(r.content, dtype=np.float32) + if sr != OUT_RATE: + data = np.interp(np.arange(0, data.size, sr / OUT_RATE), np.arange(data.size), data).astype(np.float32) + return data + + @classmethod + def stop(cls) -> None: + if cls._proc is not None and cls._proc.poll() is None: + cls._proc.terminate() + cls._proc = None + + def make_voice(settings, on_status=None): + if settings.get("tts_engine") == "chatterbox": + return ChatterboxVoice(float(settings.get("chatterbox_exaggeration", 0.6)), float(settings.get("chatterbox_cfg", 0.3)), + str(settings.get("chatterbox_ref", "") or ""), on_status=on_status) if settings.get("tts_engine") == "system": return SystemVoice(settings.get("system_voice", "")) return KokoroVoice(settings.get("kokoro_voice", "if_sara"), on_status=on_status) diff --git a/samples/chatterbox-it-0.5.wav b/samples/chatterbox-it-0.5.wav new file mode 100644 index 0000000..467496a Binary files /dev/null and b/samples/chatterbox-it-0.5.wav differ diff --git a/samples/chatterbox-it-0.8.wav b/samples/chatterbox-it-0.8.wav new file mode 100644 index 0000000..268c96e Binary files /dev/null and b/samples/chatterbox-it-0.8.wav differ diff --git a/samples/chatterbox-it-cfg0.0.wav b/samples/chatterbox-it-cfg0.0.wav new file mode 100644 index 0000000..f4e145d Binary files /dev/null and b/samples/chatterbox-it-cfg0.0.wav differ diff --git a/samples/chatterbox-it-cfg0.3.wav b/samples/chatterbox-it-cfg0.3.wav new file mode 100644 index 0000000..28a4984 Binary files /dev/null and b/samples/chatterbox-it-cfg0.3.wav differ diff --git a/tts_chatterbox/server.py b/tts_chatterbox/server.py new file mode 100644 index 0000000..edfc68f --- /dev/null +++ b/tts_chatterbox/server.py @@ -0,0 +1,76 @@ +"""Servizio Chatterbox (ambiente isolato .venv-chatterbox): sintesi vocale espressiva via HTTP locale.""" +from __future__ import annotations + +import io +import json +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import torch + +PORT = int(sys.argv[sys.argv.index("--port") + 1]) if "--port" in sys.argv else 8792 +DEVICE = "mps" if torch.backends.mps.is_available() else "cpu" +_load = torch.load + + +def _patched(*a, **k): + k.setdefault("map_location", torch.device(DEVICE)) + return _load(*a, **k) + + +torch.load = _patched +state = {"ready": False, "error": None} +model = None +lock = threading.Lock() + + +def load(): + global model + try: + from chatterbox.mtl_tts import ChatterboxMultilingualTTS + m = ChatterboxMultilingualTTS.from_pretrained(device=DEVICE) + m.generate("ciao", language_id="it") # riscaldamento + model = m + state["ready"] = True + except Exception as err: + state["error"] = str(err) + + +class H(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _json(self, code, obj): + body = json.dumps(obj).encode() + self.send_response(code); self.send_header("Content-Type", "application/json"); self.send_header("Content-Length", str(len(body))); self.end_headers(); self.wfile.write(body) + + def do_GET(self): + self._json(200, {"ready": state["ready"], "error": state["error"], "device": DEVICE, "sr": model.sr if model else None}) + + def do_POST(self): + if not state["ready"]: + return self._json(503, {"error": state["error"] or "modello in caricamento"}) + n = int(self.headers.get("Content-Length", "0")) + req = json.loads(self.rfile.read(n) or b"{}") + text = str(req.get("text", "")).strip() + if not text: + return self._json(400, {"error": "testo vuoto"}) + kw = {"language_id": req.get("language", "it"), "exaggeration": float(req.get("exaggeration", 0.6)), + "cfg_weight": float(req.get("cfg", 0.3)), "temperature": float(req.get("temperature", 0.8))} + if req.get("ref"): + kw["audio_prompt_path"] = req["ref"] + try: + with lock: + wav = model.generate(text, **kw) + data = wav.squeeze().cpu().numpy().astype("float32").tobytes() + except Exception as err: + return self._json(500, {"error": str(err)}) + self.send_response(200); self.send_header("Content-Type", "application/octet-stream") + self.send_header("X-Sample-Rate", str(model.sr)); self.send_header("Content-Length", str(len(data))); self.end_headers() + self.wfile.write(data) + + +threading.Thread(target=load, daemon=True).start() +print(f"[chatterbox] in ascolto su 127.0.0.1:{PORT}, dispositivo {DEVICE}", flush=True) +ThreadingHTTPServer(("127.0.0.1", PORT), H).serve_forever()