From 7931fcb7d09454b078bdb937cf437a17d2e7daeb Mon Sep 17 00:00:00 2001 From: ZhikangNiu Date: Sat, 14 Dec 2024 10:41:03 +0800 Subject: [PATCH 1/9] save every line wer results and update utmos evaluation --- .../eval/eval_librispeech_test_clean.py | 9 +++- src/f5_tts/eval/eval_seedtts_testset.py | 9 +++- src/f5_tts/eval/eval_utmos.py | 47 +++++++++++++++++++ src/f5_tts/eval/utils_eval.py | 11 ++++- 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 src/f5_tts/eval/eval_utmos.py diff --git a/src/f5_tts/eval/eval_librispeech_test_clean.py b/src/f5_tts/eval/eval_librispeech_test_clean.py index a5f76e0..631f284 100644 --- a/src/f5_tts/eval/eval_librispeech_test_clean.py +++ b/src/f5_tts/eval/eval_librispeech_test_clean.py @@ -10,7 +10,7 @@ import multiprocessing as mp from importlib.resources import files import numpy as np - +import json from f5_tts.eval.utils_eval import ( get_librispeech_test, run_asr_wer, @@ -56,12 +56,19 @@ def main(): # --------------------------- WER --------------------------- if eval_task == "wer": wers = [] + wer_results = [] with mp.Pool(processes=len(gpus)) as pool: args = [(rank, lang, sub_test_set, asr_ckpt_dir) for (rank, sub_test_set) in test_set] results = pool.map(run_asr_wer, args) for wers_ in results: wers.extend(wers_) + with open(f"{gen_wav_dir}/{lang}_wer_results.jsonl", "w") as f: + for line in wers: + wer_results.append(line["wer"]) + json_line = json.dumps(line, ensure_ascii=False) + f.write(json_line + "\n") + wer = round(np.mean(wers) * 100, 3) print(f"\nTotal {len(wers)} samples") print(f"WER : {wer}%") diff --git a/src/f5_tts/eval/eval_seedtts_testset.py b/src/f5_tts/eval/eval_seedtts_testset.py index 5cc1987..967a919 100644 --- a/src/f5_tts/eval/eval_seedtts_testset.py +++ b/src/f5_tts/eval/eval_seedtts_testset.py @@ -10,7 +10,7 @@ import multiprocessing as mp from importlib.resources import files import numpy as np - +import json from f5_tts.eval.utils_eval import ( get_seed_tts_test, run_asr_wer, @@ -56,12 +56,19 @@ def main(): if eval_task == "wer": wers = [] + wer_results = [] with mp.Pool(processes=len(gpus)) as pool: args = [(rank, lang, sub_test_set, asr_ckpt_dir) for (rank, sub_test_set) in test_set] results = pool.map(run_asr_wer, args) for wers_ in results: wers.extend(wers_) + with open(f"{gen_wav_dir}/{lang}_wer_results.jsonl", "w") as f: + for line in wers: + wer_results.append(line["wer"]) + json_line = json.dumps(line, ensure_ascii=False) + f.write(json_line + "\n") + wer = round(np.mean(wers) * 100, 3) print(f"\nTotal {len(wers)} samples") print(f"WER : {wer}%") diff --git a/src/f5_tts/eval/eval_utmos.py b/src/f5_tts/eval/eval_utmos.py new file mode 100644 index 0000000..65196ce --- /dev/null +++ b/src/f5_tts/eval/eval_utmos.py @@ -0,0 +1,47 @@ +import torch +import librosa +from pathlib import Path +import json +from tqdm import tqdm +import argparse + + +def main(): + parser = argparse.ArgumentParser(description="Evaluate UTMOS scores for audio files.") + parser.add_argument( + "--audio_dir", type=str, required=True, help="Path to the directory containing WAV audio files." + ) + parser.add_argument("--ext", type=str, default="wav", help="audio extension.") + parser.add_argument("--device", type=str, default="cuda", help="Device to run inference on (e.g. 'cuda' or 'cpu').") + + args = parser.parse_args() + + device = "cuda" if args.device and torch.cuda.is_available() else "cpu" + + predictor = torch.hub.load("tarepan/SpeechMOS:v1.2.0", "utmos22_strong", trust_repo=True) + predictor = predictor.to(device) + + lines = list(Path(args.audio_dir).rglob(f"*.{args.ext}")) + results = {} + utmos_result = 0 + + for line in tqdm(lines, desc="Processing"): + wave_name = line.stem + wave, sr = librosa.load(line, sr=None, mono=True) + wave_tensor = torch.from_numpy(wave).to(device).unsqueeze(0) + score = predictor(wave_tensor, sr) + results[str(wave_name)] = score.item() + utmos_result += score.item() + + avg_score = utmos_result / len(lines) if len(lines) > 0 else 0 + print(f"UTMOS: {avg_score}") + + output_path = Path(args.audio_dir) / "utmos_results.json" + with open(output_path, "w", encoding="utf-8") as f: + json.dump(results, f, ensure_ascii=False, indent=4) + + print(f"Results have been saved to {output_path}") + + +if __name__ == "__main__": + main() diff --git a/src/f5_tts/eval/utils_eval.py b/src/f5_tts/eval/utils_eval.py index 00cd97a..72f1759 100644 --- a/src/f5_tts/eval/utils_eval.py +++ b/src/f5_tts/eval/utils_eval.py @@ -7,7 +7,7 @@ import torch import torch.nn.functional as F import torchaudio from tqdm import tqdm - +from pathlib import Path from f5_tts.eval.ecapa_tdnn import ECAPA_TDNN_SMALL from f5_tts.model.modules import MelSpec from f5_tts.model.utils import convert_char_to_pinyin @@ -360,7 +360,14 @@ def run_asr_wer(args): # dele = measures["deletions"] / len(ref_list) # inse = measures["insertions"] / len(ref_list) - wers.append(wer) + wers.append( + { + "wav": Path(gen_wav).stem, # wav name + "truth": truth, # raw_truth + "hypo": hypo, # raw_hypo + "wer": wer, # wer score + } + ) return wers From 3c60f99714ae6a3f30793c29ffbfe7f705f8e1c7 Mon Sep 17 00:00:00 2001 From: ZhikangNiu Date: Sat, 14 Dec 2024 10:41:19 +0800 Subject: [PATCH 2/9] fix #524 #529 --- src/f5_tts/infer/SHARED.md | 17 +++++++++++++++++ src/f5_tts/infer/infer_cli.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/f5_tts/infer/SHARED.md b/src/f5_tts/infer/SHARED.md index 3401780..c67f60f 100644 --- a/src/f5_tts/infer/SHARED.md +++ b/src/f5_tts/infer/SHARED.md @@ -26,6 +26,8 @@ - [F5-TTS Italian @ finetune @ it](#f5-tts-italian--finetune--it) - [Japanese](#japanese) - [F5-TTS Japanese @ pretrain/finetune @ ja](#f5-tts-japanese--pretrainfinetune--ja) +- [Hindi](#hindi) + - [F5-TTS Small @ pretrain @ hi](#f5-tts-small--pretrain--hi) - [Mandarin](#mandarin) - [Spanish](#spanish) - [F5-TTS Spanish @ pretrain/finetune @ es](#f5-tts-spanish--pretrainfinetune--es) @@ -108,6 +110,21 @@ MODEL_CKPT: hf://Jmica/F5TTS/JA_8500000/model_8499660.pt VOCAB_FILE: hf://Jmica/F5TTS/JA_8500000/vocab_updated.txt ``` +## Hindi + +#### F5-TTS Small @ pretrain @ hi +|Model|🤗Hugging Face|Data (Hours)|Model License| +|:---:|:------------:|:-----------:|:-------------:| +|F5-TTS Small|[ckpt & vocab](https://huggingface.co/SPRINGLab/F5-Hindi-24KHz)|[IndicTTS Hi](https://huggingface.co/datasets/SPRINGLab/IndicTTS-Hindi) & [IndicVoices-R Hi](https://huggingface.co/datasets/SPRINGLab/IndicVoices-R_Hindi) |cc-by-4.0| + +```bash +MODEL_CKPT: hf://SPRINGLab/F5-Hindi-24KHz/model_2500000.safetensors +VOCAB_FILE: hf://SPRINGLab/F5-Hindi-24KHz/vocab.txt +``` + +Authors: SPRING Lab, Indian Institute of Technology, Madras +
+Website: https://asr.iitm.ac.in/ ## Mandarin diff --git a/src/f5_tts/infer/infer_cli.py b/src/f5_tts/infer/infer_cli.py index abf6ece..cc28182 100644 --- a/src/f5_tts/infer/infer_cli.py +++ b/src/f5_tts/infer/infer_cli.py @@ -71,6 +71,11 @@ parser.add_argument( type=str, help="Filename of output file..", ) +parser.add_argument( + "--save_chunk", + action="store_true", + help="Save chunk audio if your text is too long.", +) parser.add_argument( "--remove_silence", help="Remove silence.", @@ -87,6 +92,12 @@ parser.add_argument( default=1.0, help="Adjust the speed of the audio generation (default: 1.0)", ) +parser.add_argument( + "--nfe_step", + type=int, + default=32, + help="Set the number of denoising steps (default: 32)", +) args = parser.parse_args() config = tomli.load(open(args.config, "rb")) @@ -95,6 +106,7 @@ ref_audio = args.ref_audio if args.ref_audio else config["ref_audio"] ref_text = args.ref_text if args.ref_text != "666" else config["ref_text"] gen_text = args.gen_text if args.gen_text else config["gen_text"] gen_file = args.gen_file if args.gen_file else config["gen_file"] +save_chunk = args.save_chunk if args.save_chunk else False # patches for pip pkg user if "infer/examples/" in ref_audio: @@ -116,6 +128,7 @@ ckpt_file = args.ckpt_file if args.ckpt_file else "" vocab_file = args.vocab_file if args.vocab_file else "" remove_silence = args.remove_silence if args.remove_silence else config["remove_silence"] speed = args.speed +nfe_step = args.nfe_step wave_path = Path(output_dir) / output_file # spectrogram_path = Path(output_dir) / "infer_cli_out.png" @@ -200,7 +213,14 @@ def main_process(ref_audio, ref_text, text_gen, model_obj, mel_spec_type, remove ref_text = voices[voice]["ref_text"] print(f"Voice: {voice}") audio, final_sample_rate, spectragram = infer_process( - ref_audio, ref_text, gen_text, model_obj, vocoder, mel_spec_type=mel_spec_type, speed=speed + ref_audio, + ref_text, + gen_text, + model_obj, + vocoder, + mel_spec_type=mel_spec_type, + speed=speed, + nfe_step=nfe_step, ) generated_audio_segments.append(audio) @@ -216,6 +236,18 @@ def main_process(ref_audio, ref_text, text_gen, model_obj, mel_spec_type, remove if remove_silence: remove_silence_for_generated_wav(f.name) print(f.name) + # Ensure the gen_text chunk directory exists + + if save_chunk: + gen_text_chunk_dir = os.path.join(output_dir, "chunks") + if not os.path.exists(gen_text_chunk_dir): # if Not create directory + os.makedirs(gen_text_chunk_dir) + + # Save individual chunks as separate files + for idx, segment in enumerate(generated_audio_segments): + gen_text_chunk_path = os.path.join(output_dir, gen_text_chunk_dir, f"chunk_{idx}.wav") + sf.write(gen_text_chunk_path, segment, final_sample_rate) + print(f"Saved gen_text chunk {idx} at {gen_text_chunk_path}") def main(): From b7bc6419e7dad834bc91c7594f73dd844672d2f0 Mon Sep 17 00:00:00 2001 From: SWivid Date: Sun, 15 Dec 2024 22:49:31 +0800 Subject: [PATCH 3/9] reorganize infer_cli and stuff --- README.md | 4 +- src/f5_tts/eval/README.md | 15 +- .../eval/eval_librispeech_test_clean.py | 33 +-- src/f5_tts/eval/eval_seedtts_testset.py | 32 +-- src/f5_tts/eval/eval_utmos.py | 53 ++-- src/f5_tts/eval/utils_eval.py | 27 +- src/f5_tts/infer/README.md | 3 + src/f5_tts/infer/SHARED.md | 36 +-- src/f5_tts/infer/examples/basic/basic.toml | 2 +- src/f5_tts/infer/examples/multi/story.toml | 1 + src/f5_tts/infer/infer_cli.py | 233 ++++++++++++------ 11 files changed, 272 insertions(+), 167 deletions(-) diff --git a/README.md b/README.md index 3023f26..dcbfb75 100644 --- a/README.md +++ b/README.md @@ -147,11 +147,11 @@ Note: Some model components have linting exceptions for E722 to accommodate tens ## Acknowledgements - [E2-TTS](https://arxiv.org/abs/2406.18009) brilliant work, simple and effective -- [Emilia](https://arxiv.org/abs/2407.05361), [WenetSpeech4TTS](https://arxiv.org/abs/2406.05763) valuable datasets +- [Emilia](https://arxiv.org/abs/2407.05361), [WenetSpeech4TTS](https://arxiv.org/abs/2406.05763), [LibriTTS](https://arxiv.org/abs/1904.02882), [LJSpeech](https://keithito.com/LJ-Speech-Dataset/) valuable datasets - [lucidrains](https://github.com/lucidrains) initial CFM structure with also [bfs18](https://github.com/bfs18) for discussion - [SD3](https://arxiv.org/abs/2403.03206) & [Hugging Face diffusers](https://github.com/huggingface/diffusers) DiT and MMDiT code structure - [torchdiffeq](https://github.com/rtqichen/torchdiffeq) as ODE solver, [Vocos](https://huggingface.co/charactr/vocos-mel-24khz) as vocoder -- [FunASR](https://github.com/modelscope/FunASR), [faster-whisper](https://github.com/SYSTRAN/faster-whisper), [UniSpeech](https://github.com/microsoft/UniSpeech) for evaluation tools +- [FunASR](https://github.com/modelscope/FunASR), [faster-whisper](https://github.com/SYSTRAN/faster-whisper), [UniSpeech](https://github.com/microsoft/UniSpeech), [SpeechMOS](https://github.com/tarepan/SpeechMOS) for evaluation tools - [ctc-forced-aligner](https://github.com/MahmoudAshraf97/ctc-forced-aligner) for speech edit test - [mrfakename](https://x.com/realmrfakename) huggingface space demo ~ - [f5-tts-mlx](https://github.com/lucasnewman/f5-tts-mlx/tree/main) Implementation with MLX framework by [Lucas Newman](https://github.com/lucasnewman) diff --git a/src/f5_tts/eval/README.md b/src/f5_tts/eval/README.md index ff324a1..c33ef92 100644 --- a/src/f5_tts/eval/README.md +++ b/src/f5_tts/eval/README.md @@ -39,11 +39,14 @@ Then update in the following scripts with the paths you put evaluation model ckp ### Objective Evaluation -Update the path with your batch-inferenced results, and carry out WER / SIM evaluations: +Update the path with your batch-inferenced results, and carry out WER / SIM / UTMOS evaluations: ```bash -# Evaluation for Seed-TTS test set -python src/f5_tts/eval/eval_seedtts_testset.py --gen_wav_dir +# Evaluation [WER] for Seed-TTS test [ZH] set +python src/f5_tts/eval/eval_seedtts_testset.py --eval_task wer --lang zh --gen_wav_dir --gpu_nums 8 -# Evaluation for LibriSpeech-PC test-clean (cross-sentence) -python src/f5_tts/eval/eval_librispeech_test_clean.py --gen_wav_dir --librispeech_test_clean_path -``` \ No newline at end of file +# Evaluation [SIM] for LibriSpeech-PC test-clean (cross-sentence) +python src/f5_tts/eval/eval_librispeech_test_clean.py --eval_task sim --gen_wav_dir --librispeech_test_clean_path + +# Evaluation [UTMOS]. --ext: Audio extension +python src/f5_tts/eval/eval_utmos.py --audio_dir --ext wav +``` diff --git a/src/f5_tts/eval/eval_librispeech_test_clean.py b/src/f5_tts/eval/eval_librispeech_test_clean.py index 631f284..f172286 100644 --- a/src/f5_tts/eval/eval_librispeech_test_clean.py +++ b/src/f5_tts/eval/eval_librispeech_test_clean.py @@ -1,8 +1,9 @@ # Evaluate with Librispeech test-clean, ~3s prompt to generate 4-10s audio (the way of valle/voicebox evaluation) -import sys -import os import argparse +import json +import os +import sys sys.path.append(os.getcwd()) @@ -10,7 +11,6 @@ import multiprocessing as mp from importlib.resources import files import numpy as np -import json from f5_tts.eval.utils_eval import ( get_librispeech_test, run_asr_wer, @@ -54,36 +54,41 @@ def main(): wavlm_ckpt_dir = "../checkpoints/UniSpeech/wavlm_large_finetune.pth" # --------------------------- WER --------------------------- + if eval_task == "wer": - wers = [] wer_results = [] + wers = [] + with mp.Pool(processes=len(gpus)) as pool: args = [(rank, lang, sub_test_set, asr_ckpt_dir) for (rank, sub_test_set) in test_set] results = pool.map(run_asr_wer, args) - for wers_ in results: - wers.extend(wers_) + for r in results: + wer_results.extend(r) - with open(f"{gen_wav_dir}/{lang}_wer_results.jsonl", "w") as f: - for line in wers: - wer_results.append(line["wer"]) + wer_result_path = f"{gen_wav_dir}/{lang}_wer_results.jsonl" + with open(wer_result_path, "w") as f: + for line in wer_results: + wers.append(line["wer"]) json_line = json.dumps(line, ensure_ascii=False) f.write(json_line + "\n") wer = round(np.mean(wers) * 100, 3) print(f"\nTotal {len(wers)} samples") print(f"WER : {wer}%") + print(f"Results have been saved to {wer_result_path}") # --------------------------- SIM --------------------------- + if eval_task == "sim": - sim_list = [] + sims = [] with mp.Pool(processes=len(gpus)) as pool: args = [(rank, sub_test_set, wavlm_ckpt_dir) for (rank, sub_test_set) in test_set] results = pool.map(run_sim, args) - for sim_ in results: - sim_list.extend(sim_) + for r in results: + sims.extend(r) - sim = round(sum(sim_list) / len(sim_list), 3) - print(f"\nTotal {len(sim_list)} samples") + sim = round(sum(sims) / len(sims), 3) + print(f"\nTotal {len(sims)} samples") print(f"SIM : {sim}") diff --git a/src/f5_tts/eval/eval_seedtts_testset.py b/src/f5_tts/eval/eval_seedtts_testset.py index 967a919..95a5f44 100644 --- a/src/f5_tts/eval/eval_seedtts_testset.py +++ b/src/f5_tts/eval/eval_seedtts_testset.py @@ -1,8 +1,9 @@ # Evaluate with Seed-TTS testset -import sys -import os import argparse +import json +import os +import sys sys.path.append(os.getcwd()) @@ -10,7 +11,6 @@ import multiprocessing as mp from importlib.resources import files import numpy as np -import json from f5_tts.eval.utils_eval import ( get_seed_tts_test, run_asr_wer, @@ -55,35 +55,39 @@ def main(): # --------------------------- WER --------------------------- if eval_task == "wer": - wers = [] wer_results = [] + wers = [] + with mp.Pool(processes=len(gpus)) as pool: args = [(rank, lang, sub_test_set, asr_ckpt_dir) for (rank, sub_test_set) in test_set] results = pool.map(run_asr_wer, args) - for wers_ in results: - wers.extend(wers_) + for r in results: + wer_results.extend(r) - with open(f"{gen_wav_dir}/{lang}_wer_results.jsonl", "w") as f: - for line in wers: - wer_results.append(line["wer"]) + wer_result_path = f"{gen_wav_dir}/{lang}_wer_results.jsonl" + with open(wer_result_path, "w") as f: + for line in wer_results: + wers.append(line["wer"]) json_line = json.dumps(line, ensure_ascii=False) f.write(json_line + "\n") wer = round(np.mean(wers) * 100, 3) print(f"\nTotal {len(wers)} samples") print(f"WER : {wer}%") + print(f"Results have been saved to {wer_result_path}") # --------------------------- SIM --------------------------- + if eval_task == "sim": - sim_list = [] + sims = [] with mp.Pool(processes=len(gpus)) as pool: args = [(rank, sub_test_set, wavlm_ckpt_dir) for (rank, sub_test_set) in test_set] results = pool.map(run_sim, args) - for sim_ in results: - sim_list.extend(sim_) + for r in results: + sims.extend(r) - sim = round(sum(sim_list) / len(sim_list), 3) - print(f"\nTotal {len(sim_list)} samples") + sim = round(sum(sims) / len(sims), 3) + print(f"\nTotal {len(sims)} samples") print(f"SIM : {sim}") diff --git a/src/f5_tts/eval/eval_utmos.py b/src/f5_tts/eval/eval_utmos.py index 65196ce..9b069cd 100644 --- a/src/f5_tts/eval/eval_utmos.py +++ b/src/f5_tts/eval/eval_utmos.py @@ -1,46 +1,43 @@ -import torch -import librosa -from pathlib import Path -import json -from tqdm import tqdm import argparse +import json +from pathlib import Path + +import librosa +import torch +from tqdm import tqdm def main(): - parser = argparse.ArgumentParser(description="Evaluate UTMOS scores for audio files.") - parser.add_argument( - "--audio_dir", type=str, required=True, help="Path to the directory containing WAV audio files." - ) - parser.add_argument("--ext", type=str, default="wav", help="audio extension.") - parser.add_argument("--device", type=str, default="cuda", help="Device to run inference on (e.g. 'cuda' or 'cpu').") - + parser = argparse.ArgumentParser(description="UTMOS Evaluation") + parser.add_argument("--audio_dir", type=str, required=True, help="Audio file path.") + parser.add_argument("--ext", type=str, default="wav", help="Audio extension.") args = parser.parse_args() - device = "cuda" if args.device and torch.cuda.is_available() else "cpu" + device = "cuda" if torch.cuda.is_available() else "cpu" predictor = torch.hub.load("tarepan/SpeechMOS:v1.2.0", "utmos22_strong", trust_repo=True) predictor = predictor.to(device) - lines = list(Path(args.audio_dir).rglob(f"*.{args.ext}")) - results = {} - utmos_result = 0 + audio_paths = list(Path(args.audio_dir).rglob(f"*.{args.ext}")) + utmos_results = {} + utmos_score = 0 - for line in tqdm(lines, desc="Processing"): - wave_name = line.stem - wave, sr = librosa.load(line, sr=None, mono=True) - wave_tensor = torch.from_numpy(wave).to(device).unsqueeze(0) - score = predictor(wave_tensor, sr) - results[str(wave_name)] = score.item() - utmos_result += score.item() + for audio_path in tqdm(audio_paths, desc="Processing"): + wav_name = audio_path.stem + wav, sr = librosa.load(audio_path, sr=None, mono=True) + wav_tensor = torch.from_numpy(wav).to(device).unsqueeze(0) + score = predictor(wav_tensor, sr) + utmos_results[str(wav_name)] = score.item() + utmos_score += score.item() - avg_score = utmos_result / len(lines) if len(lines) > 0 else 0 + avg_score = utmos_score / len(audio_paths) if len(audio_paths) > 0 else 0 print(f"UTMOS: {avg_score}") - output_path = Path(args.audio_dir) / "utmos_results.json" - with open(output_path, "w", encoding="utf-8") as f: - json.dump(results, f, ensure_ascii=False, indent=4) + utmos_result_path = Path(args.audio_dir) / "utmos_results.json" + with open(utmos_result_path, "w", encoding="utf-8") as f: + json.dump(utmos_results, f, ensure_ascii=False, indent=4) - print(f"Results have been saved to {output_path}") + print(f"Results have been saved to {utmos_result_path}") if __name__ == "__main__": diff --git a/src/f5_tts/eval/utils_eval.py b/src/f5_tts/eval/utils_eval.py index 72f1759..7c0a8a8 100644 --- a/src/f5_tts/eval/utils_eval.py +++ b/src/f5_tts/eval/utils_eval.py @@ -2,12 +2,13 @@ import math import os import random import string +from pathlib import Path import torch import torch.nn.functional as F import torchaudio from tqdm import tqdm -from pathlib import Path + from f5_tts.eval.ecapa_tdnn import ECAPA_TDNN_SMALL from f5_tts.model.modules import MelSpec from f5_tts.model.utils import convert_char_to_pinyin @@ -320,7 +321,7 @@ def run_asr_wer(args): from zhon.hanzi import punctuation punctuation_all = punctuation + string.punctuation - wers = [] + wer_results = [] from jiwer import compute_measures @@ -335,8 +336,8 @@ def run_asr_wer(args): for segment in segments: hypo = hypo + " " + segment.text - # raw_truth = truth - # raw_hypo = hypo + raw_truth = truth + raw_hypo = hypo for x in punctuation_all: truth = truth.replace(x, "") @@ -360,16 +361,16 @@ def run_asr_wer(args): # dele = measures["deletions"] / len(ref_list) # inse = measures["insertions"] / len(ref_list) - wers.append( + wer_results.append( { - "wav": Path(gen_wav).stem, # wav name - "truth": truth, # raw_truth - "hypo": hypo, # raw_hypo - "wer": wer, # wer score + "wav": Path(gen_wav).stem, + "truth": raw_truth, + "hypo": raw_hypo, + "wer": wer, } ) - return wers + return wer_results # SIM Evaluation @@ -388,7 +389,7 @@ def run_sim(args): model = model.cuda(device) model.eval() - sim_list = [] + sims = [] for wav1, wav2, truth in tqdm(test_set): wav1, sr1 = torchaudio.load(wav1) wav2, sr2 = torchaudio.load(wav2) @@ -407,6 +408,6 @@ def run_sim(args): sim = F.cosine_similarity(emb1, emb2)[0].item() # print(f"VSim score between two audios: {sim:.4f} (-1.0, 1.0).") - sim_list.append(sim) + sims.append(sim) - return sim_list + return sims diff --git a/src/f5_tts/infer/README.md b/src/f5_tts/infer/README.md index fe48a78..0c706b5 100644 --- a/src/f5_tts/infer/README.md +++ b/src/f5_tts/infer/README.md @@ -64,6 +64,9 @@ f5-tts_infer-cli \ # Choose Vocoder f5-tts_infer-cli --vocoder_name bigvgan --load_vocoder_from_local --ckpt_file f5-tts_infer-cli --vocoder_name vocos --load_vocoder_from_local --ckpt_file + +# More instructions +f5-tts_infer-cli --help ``` And a `.toml` file would help with more flexible usage. diff --git a/src/f5_tts/infer/SHARED.md b/src/f5_tts/infer/SHARED.md index c67f60f..a09bef9 100644 --- a/src/f5_tts/infer/SHARED.md +++ b/src/f5_tts/infer/SHARED.md @@ -22,12 +22,12 @@ - [Finnish Common\_Voice Vox\_Populi @ finetune @ fi](#finnish-common_voice-vox_populi--finetune--fi) - [French](#french) - [French LibriVox @ finetune @ fr](#french-librivox--finetune--fr) +- [Hindi](#hindi) + - [F5-TTS Small @ pretrain @ hi](#f5-tts-small--pretrain--hi) - [Italian](#italian) - [F5-TTS Italian @ finetune @ it](#f5-tts-italian--finetune--it) - [Japanese](#japanese) - [F5-TTS Japanese @ pretrain/finetune @ ja](#f5-tts-japanese--pretrainfinetune--ja) -- [Hindi](#hindi) - - [F5-TTS Small @ pretrain @ hi](#f5-tts-small--pretrain--hi) - [Mandarin](#mandarin) - [Spanish](#spanish) - [F5-TTS Spanish @ pretrain/finetune @ es](#f5-tts-spanish--pretrainfinetune--es) @@ -81,6 +81,23 @@ VOCAB_FILE: hf://RASPIAUDIO/F5-French-MixedSpeakers-reduced/vocab.txt - [Discussion about this training can be found here](https://github.com/SWivid/F5-TTS/issues/434). +## Hindi + +#### F5-TTS Small @ pretrain @ hi +|Model|🤗Hugging Face|Data (Hours)|Model License| +|:---:|:------------:|:-----------:|:-------------:| +|F5-TTS Small|[ckpt & vocab](https://huggingface.co/SPRINGLab/F5-Hindi-24KHz)|[IndicTTS Hi](https://huggingface.co/datasets/SPRINGLab/IndicTTS-Hindi) & [IndicVoices-R Hi](https://huggingface.co/datasets/SPRINGLab/IndicVoices-R_Hindi) |cc-by-4.0| + +```bash +MODEL_CKPT: hf://SPRINGLab/F5-Hindi-24KHz/model_2500000.safetensors +VOCAB_FILE: hf://SPRINGLab/F5-Hindi-24KHz/vocab.txt +``` + +Authors: SPRING Lab, Indian Institute of Technology, Madras +
+Website: https://asr.iitm.ac.in/ + + ## Italian #### F5-TTS Italian @ finetune @ it @@ -110,21 +127,6 @@ MODEL_CKPT: hf://Jmica/F5TTS/JA_8500000/model_8499660.pt VOCAB_FILE: hf://Jmica/F5TTS/JA_8500000/vocab_updated.txt ``` -## Hindi - -#### F5-TTS Small @ pretrain @ hi -|Model|🤗Hugging Face|Data (Hours)|Model License| -|:---:|:------------:|:-----------:|:-------------:| -|F5-TTS Small|[ckpt & vocab](https://huggingface.co/SPRINGLab/F5-Hindi-24KHz)|[IndicTTS Hi](https://huggingface.co/datasets/SPRINGLab/IndicTTS-Hindi) & [IndicVoices-R Hi](https://huggingface.co/datasets/SPRINGLab/IndicVoices-R_Hindi) |cc-by-4.0| - -```bash -MODEL_CKPT: hf://SPRINGLab/F5-Hindi-24KHz/model_2500000.safetensors -VOCAB_FILE: hf://SPRINGLab/F5-Hindi-24KHz/vocab.txt -``` - -Authors: SPRING Lab, Indian Institute of Technology, Madras -
-Website: https://asr.iitm.ac.in/ ## Mandarin diff --git a/src/f5_tts/infer/examples/basic/basic.toml b/src/f5_tts/infer/examples/basic/basic.toml index 4c594c7..c43af38 100644 --- a/src/f5_tts/infer/examples/basic/basic.toml +++ b/src/f5_tts/infer/examples/basic/basic.toml @@ -8,4 +8,4 @@ gen_text = "I don't really care what you call me. I've been a silent spectator, gen_file = "" remove_silence = false output_dir = "tests" -output_file = "infer_cli_out.wav" +output_file = "infer_cli_basic.wav" diff --git a/src/f5_tts/infer/examples/multi/story.toml b/src/f5_tts/infer/examples/multi/story.toml index c637062..10ba3fc 100644 --- a/src/f5_tts/infer/examples/multi/story.toml +++ b/src/f5_tts/infer/examples/multi/story.toml @@ -8,6 +8,7 @@ gen_text = "" gen_file = "infer/examples/multi/story.txt" remove_silence = true output_dir = "tests" +output_file = "infer_cli_story.wav" [voices.town] ref_audio = "infer/examples/multi/town.flac" diff --git a/src/f5_tts/infer/infer_cli.py b/src/f5_tts/infer/infer_cli.py index cc28182..47a71a5 100644 --- a/src/f5_tts/infer/infer_cli.py +++ b/src/f5_tts/infer/infer_cli.py @@ -2,6 +2,7 @@ import argparse import codecs import os import re +from datetime import datetime from importlib.resources import files from pathlib import Path @@ -11,6 +12,14 @@ import tomli from cached_path import cached_path from f5_tts.infer.utils_infer import ( + mel_spec_type, + target_rms, + cross_fade_duration, + nfe_step, + cfg_strength, + sway_sampling_coef, + speed, + fix_duration, infer_process, load_model, load_vocoder, @@ -19,6 +28,7 @@ from f5_tts.infer.utils_infer import ( ) from f5_tts.model import DiT, UNetT + parser = argparse.ArgumentParser( prog="python3 infer-cli.py", description="Commandline interface for E2/F5 TTS with Advanced Batch Processing.", @@ -27,86 +37,161 @@ parser = argparse.ArgumentParser( parser.add_argument( "-c", "--config", - help="Configuration file. Default=infer/examples/basic/basic.toml", + type=str, default=os.path.join(files("f5_tts").joinpath("infer/examples/basic"), "basic.toml"), + help="The configuration file, default see infer/examples/basic/basic.toml", ) + + +# Note. Not to provide default value here in order to read default from config file + parser.add_argument( "-m", "--model", - help="F5-TTS | E2-TTS", + type=str, + help="The model name: F5-TTS | E2-TTS", ) parser.add_argument( "-p", "--ckpt_file", - help="The Checkpoint .pt", + type=str, + help="The path to model checkpoint .pt, leave blank to use default", ) parser.add_argument( "-v", "--vocab_file", - help="The vocab .txt", + type=str, + help="The path to vocab file .txt, leave blank to use default", +) +parser.add_argument( + "-r", + "--ref_audio", + type=str, + help="The reference audio file.", +) +parser.add_argument( + "-s", + "--ref_text", + type=str, + help="The transcript/subtitle for the reference audio", ) -parser.add_argument("-r", "--ref_audio", type=str, help="Reference audio file < 15 seconds.") -parser.add_argument("-s", "--ref_text", type=str, default="666", help="Subtitle for the reference audio.") parser.add_argument( "-t", "--gen_text", type=str, - help="Text to generate.", + help="The text to make model synthesize a speech", ) parser.add_argument( "-f", "--gen_file", type=str, - help="File with text to generate. Ignores --gen_text", + help="The file with text to generate, will ignore --gen_text", ) parser.add_argument( "-o", "--output_dir", type=str, - help="Path to output folder..", + help="The path to output folder", ) parser.add_argument( "-w", "--output_file", type=str, - help="Filename of output file..", + help="The name of output file", ) parser.add_argument( "--save_chunk", action="store_true", - help="Save chunk audio if your text is too long.", + help="To save each audio chunks during inference", ) parser.add_argument( "--remove_silence", - help="Remove silence.", + action="store_true", + help="To remove long silence found in ouput", ) -parser.add_argument("--vocoder_name", type=str, default="vocos", choices=["vocos", "bigvgan"], help="vocoder name") parser.add_argument( "--load_vocoder_from_local", action="store_true", - help="load vocoder from local. Default: ../checkpoints/charactr/vocos-mel-24khz", + help="To load vocoder from local dir, default to ../checkpoints/charactr/vocos-mel-24khz", ) parser.add_argument( - "--speed", + "--vocoder_name", + type=str, + choices=["vocos", "bigvgan"], + help=f"Used vocoder name: vocos | bigvgan, default {mel_spec_type}", +) +parser.add_argument( + "--target_rms", type=float, - default=1.0, - help="Adjust the speed of the audio generation (default: 1.0)", + help=f"Target output speech loudness normalization value, default {target_rms}", +) +parser.add_argument( + "--cross_fade_duration", + type=float, + help=f"Duration of cross-fade between audio segments in seconds, default {cross_fade_duration}", ) parser.add_argument( "--nfe_step", type=int, - default=32, - help="Set the number of denoising steps (default: 32)", + help=f"The number of function evaluation (denoising steps), default {nfe_step}", +) +parser.add_argument( + "--cfg_strength", + type=float, + help=f"Classifier-free guidance strength, default {cfg_strength}", +) +parser.add_argument( + "--sway_sampling_coef", + type=float, + help=f"Sway Sampling coefficient, default {sway_sampling_coef}", +) +parser.add_argument( + "--speed", + type=float, + help=f"The speed of the generated audio, default {speed}", +) +parser.add_argument( + "--fix_duration", + type=float, + help=f"Fix the total duration (ref and gen audios) in seconds, default {fix_duration}", ) args = parser.parse_args() + +# config file + config = tomli.load(open(args.config, "rb")) -ref_audio = args.ref_audio if args.ref_audio else config["ref_audio"] -ref_text = args.ref_text if args.ref_text != "666" else config["ref_text"] -gen_text = args.gen_text if args.gen_text else config["gen_text"] -gen_file = args.gen_file if args.gen_file else config["gen_file"] -save_chunk = args.save_chunk if args.save_chunk else False + +# command-line interface parameters + +model = args.model or config.get("model", "F5-TTS") +ckpt_file = args.ckpt_file or config.get("ckpt_file", "") +vocab_file = args.vocab_file or config.get("vocab_file", "") + +ref_audio = args.ref_audio or config.get("ref_audio", "infer/examples/basic/basic_ref_en.wav") +ref_text = args.ref_text or config.get("ref_text", "Some call me nature, others call me mother nature.") +gen_text = args.gen_text or config.get("gen_text", "Here we generate something just for test.") +gen_file = args.gen_file or config.get("gen_file", "") + +output_dir = args.output_dir or config.get("output_dir", "tests") +output_file = args.output_file or config.get( + "output_file", f"infer_cli_{datetime.now().strftime(r'%Y%m%d_%H%M%S')}.wav" +) + +save_chunk = args.save_chunk +remove_silence = args.remove_silence +load_vocoder_from_local = args.load_vocoder_from_local + +vocoder_name = args.vocoder_name or config.get("vocoder_name", mel_spec_type) +target_rms = args.target_rms or config.get("target_rms", target_rms) +cross_fade_duration = args.cross_fade_duration or config.get("cross_fade_duration", cross_fade_duration) +nfe_step = args.nfe_step or config.get("nfe_step", nfe_step) +cfg_strength = args.cfg_strength or config.get("cfg_strength", cfg_strength) +sway_sampling_coef = args.sway_sampling_coef or config.get("sway_sampling_coef", sway_sampling_coef) +speed = args.speed or config.get("speed", speed) +fix_duration = args.fix_duration or config.get("fix_duration", fix_duration) + # patches for pip pkg user if "infer/examples/" in ref_audio: @@ -119,35 +204,39 @@ if "voices" in config: if "infer/examples/" in voice_ref_audio: config["voices"][voice]["ref_audio"] = str(files("f5_tts").joinpath(f"{voice_ref_audio}")) + +# ignore gen_text if gen_file provided + if gen_file: gen_text = codecs.open(gen_file, "r", "utf-8").read() -output_dir = args.output_dir if args.output_dir else config["output_dir"] -output_file = args.output_file if args.output_file else config["output_file"] -model = args.model if args.model else config["model"] -ckpt_file = args.ckpt_file if args.ckpt_file else "" -vocab_file = args.vocab_file if args.vocab_file else "" -remove_silence = args.remove_silence if args.remove_silence else config["remove_silence"] -speed = args.speed -nfe_step = args.nfe_step + + +# output path wave_path = Path(output_dir) / output_file # spectrogram_path = Path(output_dir) / "infer_cli_out.png" +if save_chunk: + output_chunk_dir = os.path.join(output_dir, f"{Path(output_file).stem}_chunks") + if not os.path.exists(output_chunk_dir): + os.makedirs(output_chunk_dir) + + +# load vocoder -vocoder_name = args.vocoder_name -mel_spec_type = args.vocoder_name if vocoder_name == "vocos": vocoder_local_path = "../checkpoints/vocos-mel-24khz" elif vocoder_name == "bigvgan": vocoder_local_path = "../checkpoints/bigvgan_v2_24khz_100band_256x" -vocoder = load_vocoder(vocoder_name=mel_spec_type, is_local=args.load_vocoder_from_local, local_path=vocoder_local_path) +vocoder = load_vocoder(vocoder_name=vocoder_name, is_local=load_vocoder_from_local, local_path=vocoder_local_path) -# load models +# load TTS model + if model == "F5-TTS": model_cls = DiT model_cfg = dict(dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4) - if ckpt_file == "": + if not ckpt_file: # path not specified, download from repo if vocoder_name == "vocos": repo_name = "F5-TTS" exp_name = "F5TTS_Base" @@ -164,19 +253,21 @@ elif model == "E2-TTS": assert vocoder_name == "vocos", "E2-TTS only supports vocoder vocos" model_cls = UNetT model_cfg = dict(dim=1024, depth=24, heads=16, ff_mult=4) - if ckpt_file == "": + if not ckpt_file: # path not specified, download from repo repo_name = "E2-TTS" exp_name = "E2TTS_Base" ckpt_step = 1200000 ckpt_file = str(cached_path(f"hf://SWivid/{repo_name}/{exp_name}/model_{ckpt_step}.safetensors")) # ckpt_file = f"ckpts/{exp_name}/model_{ckpt_step}.pt" # .pt | .safetensors; local path - print(f"Using {model}...") -ema_model = load_model(model_cls, model_cfg, ckpt_file, mel_spec_type=mel_spec_type, vocab_file=vocab_file) +ema_model = load_model(model_cls, model_cfg, ckpt_file, mel_spec_type=vocoder_name, vocab_file=vocab_file) -def main_process(ref_audio, ref_text, text_gen, model_obj, mel_spec_type, remove_silence, speed): +# inference process + + +def main(): main_voice = {"ref_audio": ref_audio, "ref_text": ref_text} if "voices" not in config: voices = {"main": main_voice} @@ -184,16 +275,16 @@ def main_process(ref_audio, ref_text, text_gen, model_obj, mel_spec_type, remove voices = config["voices"] voices["main"] = main_voice for voice in voices: + print("Voice:", voice) + print("ref_audio ", voices[voice]["ref_audio"]) voices[voice]["ref_audio"], voices[voice]["ref_text"] = preprocess_ref_audio_text( voices[voice]["ref_audio"], voices[voice]["ref_text"] ) - print("Voice:", voice) - print("Ref_audio:", voices[voice]["ref_audio"]) - print("Ref_text:", voices[voice]["ref_text"]) + print("ref_audio_", voices[voice]["ref_audio"], "\n\n") generated_audio_segments = [] reg1 = r"(?=\[\w+\])" - chunks = re.split(reg1, text_gen) + chunks = re.split(reg1, gen_text) reg2 = r"\[(\w+)\]" for text in chunks: if not text.strip(): @@ -208,21 +299,35 @@ def main_process(ref_audio, ref_text, text_gen, model_obj, mel_spec_type, remove print(f"Voice {voice} not found, using main.") voice = "main" text = re.sub(reg2, "", text) - gen_text = text.strip() - ref_audio = voices[voice]["ref_audio"] - ref_text = voices[voice]["ref_text"] + ref_audio_ = voices[voice]["ref_audio"] + ref_text_ = voices[voice]["ref_text"] + gen_text_ = text.strip() print(f"Voice: {voice}") - audio, final_sample_rate, spectragram = infer_process( - ref_audio, - ref_text, - gen_text, - model_obj, + audio_segment, final_sample_rate, spectragram = infer_process( + ref_audio_, + ref_text_, + gen_text_, + ema_model, vocoder, - mel_spec_type=mel_spec_type, - speed=speed, + mel_spec_type=vocoder_name, + target_rms=target_rms, + cross_fade_duration=cross_fade_duration, nfe_step=nfe_step, + cfg_strength=cfg_strength, + sway_sampling_coef=sway_sampling_coef, + speed=speed, + fix_duration=fix_duration, ) - generated_audio_segments.append(audio) + generated_audio_segments.append(audio_segment) + + if save_chunk: + if len(gen_text_) > 200: + gen_text_ = gen_text_[:200] + " ... " + sf.write( + os.path.join(output_chunk_dir, f"{len(generated_audio_segments)-1}_{gen_text_}.wav"), + audio_segment, + final_sample_rate, + ) if generated_audio_segments: final_wave = np.concatenate(generated_audio_segments) @@ -236,22 +341,6 @@ def main_process(ref_audio, ref_text, text_gen, model_obj, mel_spec_type, remove if remove_silence: remove_silence_for_generated_wav(f.name) print(f.name) - # Ensure the gen_text chunk directory exists - - if save_chunk: - gen_text_chunk_dir = os.path.join(output_dir, "chunks") - if not os.path.exists(gen_text_chunk_dir): # if Not create directory - os.makedirs(gen_text_chunk_dir) - - # Save individual chunks as separate files - for idx, segment in enumerate(generated_audio_segments): - gen_text_chunk_path = os.path.join(output_dir, gen_text_chunk_dir, f"chunk_{idx}.wav") - sf.write(gen_text_chunk_path, segment, final_sample_rate) - print(f"Saved gen_text chunk {idx} at {gen_text_chunk_path}") - - -def main(): - main_process(ref_audio, ref_text, gen_text, ema_model, mel_spec_type, remove_silence, speed) if __name__ == "__main__": From c6e96d0c8380fef6fc19b1bd1b094a07f7533730 Mon Sep 17 00:00:00 2001 From: SWivid Date: Sun, 15 Dec 2024 23:21:00 +0800 Subject: [PATCH 4/9] Fixed #631 --- src/f5_tts/model/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/f5_tts/model/trainer.py b/src/f5_tts/model/trainer.py index 0825b02..9d0c79a 100644 --- a/src/f5_tts/model/trainer.py +++ b/src/f5_tts/model/trainer.py @@ -315,7 +315,7 @@ class Trainer: self.scheduler.step() self.optimizer.zero_grad() - if self.is_main: + if self.is_main and self.accelerator.sync_gradients: self.ema_model.update() global_step += 1 From 7c84e91a007f73b608c4e978310bcbfbcc234283 Mon Sep 17 00:00:00 2001 From: ZhikangNiu Date: Mon, 16 Dec 2024 13:24:58 +0800 Subject: [PATCH 5/9] update dit checkpoint_activations and fix#399 #400 --- src/f5_tts/configs/F5TTS_Base_train.yaml | 1 + src/f5_tts/configs/F5TTS_Small_train.yaml | 1 + src/f5_tts/model/backbones/dit.py | 17 ++++++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/f5_tts/configs/F5TTS_Base_train.yaml b/src/f5_tts/configs/F5TTS_Base_train.yaml index d757c5b..fe93332 100644 --- a/src/f5_tts/configs/F5TTS_Base_train.yaml +++ b/src/f5_tts/configs/F5TTS_Base_train.yaml @@ -28,6 +28,7 @@ model: ff_mult: 2 text_dim: 512 conv_layers: 4 + checkpoint_activations: False # recompute activations and save memory for extra compute mel_spec: target_sample_rate: 24000 n_mel_channels: 100 diff --git a/src/f5_tts/configs/F5TTS_Small_train.yaml b/src/f5_tts/configs/F5TTS_Small_train.yaml index 833c6af..466ee29 100644 --- a/src/f5_tts/configs/F5TTS_Small_train.yaml +++ b/src/f5_tts/configs/F5TTS_Small_train.yaml @@ -28,6 +28,7 @@ model: ff_mult: 2 text_dim: 512 conv_layers: 4 + checkpoint_activations: False # recompute activations and save memory for extra compute mel_spec: target_sample_rate: 24000 n_mel_channels: 100 diff --git a/src/f5_tts/model/backbones/dit.py b/src/f5_tts/model/backbones/dit.py index 391752a..472af28 100644 --- a/src/f5_tts/model/backbones/dit.py +++ b/src/f5_tts/model/backbones/dit.py @@ -105,6 +105,7 @@ class DiT(nn.Module): text_dim=None, conv_layers=0, long_skip_connection=False, + checkpoint_activations=False, ): super().__init__() @@ -127,6 +128,17 @@ class DiT(nn.Module): self.norm_out = AdaLayerNormZero_Final(dim) # final modulation self.proj_out = nn.Linear(dim, mel_dim) + self.checkpoint_activations = checkpoint_activations + + def ckpt_wrapper(self, module): + """Code from https://github.com/chuanyangjin/fast-DiT/blob/1a8ecce58f346f877749f2dc67cdb190d295e4dc/models.py#L233-L237""" + + def ckpt_forward(*inputs): + outputs = module(*inputs) + return outputs + + return ckpt_forward + def forward( self, x: float["b n d"], # nosied input audio # noqa: F722 @@ -152,7 +164,10 @@ class DiT(nn.Module): residual = x for block in self.transformer_blocks: - x = block(x, t, mask=mask, rope=rope) + if self.checkpoint_activations: + x = torch.utils.checkpoint.checkpoint(self.ckpt_wrapper(block), x, t, mask, rope) + else: + x = block(x, t, mask=mask, rope=rope) if self.long_skip_connection is not None: x = self.long_skip_connection(torch.cat((x, residual), dim=-1)) From 649b46e4cb9340190bfd1aa44947835171975944 Mon Sep 17 00:00:00 2001 From: ZhikangNiu Date: Mon, 16 Dec 2024 14:41:10 +0800 Subject: [PATCH 6/9] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dcbfb75..96cbb57 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Note: Some model components have linting exceptions for E722 to accommodate tens - [Emilia](https://arxiv.org/abs/2407.05361), [WenetSpeech4TTS](https://arxiv.org/abs/2406.05763), [LibriTTS](https://arxiv.org/abs/1904.02882), [LJSpeech](https://keithito.com/LJ-Speech-Dataset/) valuable datasets - [lucidrains](https://github.com/lucidrains) initial CFM structure with also [bfs18](https://github.com/bfs18) for discussion - [SD3](https://arxiv.org/abs/2403.03206) & [Hugging Face diffusers](https://github.com/huggingface/diffusers) DiT and MMDiT code structure -- [torchdiffeq](https://github.com/rtqichen/torchdiffeq) as ODE solver, [Vocos](https://huggingface.co/charactr/vocos-mel-24khz) as vocoder +- [torchdiffeq](https://github.com/rtqichen/torchdiffeq) as ODE solver, [Vocos](https://huggingface.co/charactr/vocos-mel-24khz) and [BigVGAN](https://github.com/NVIDIA/BigVGAN/tree/main) as vocoder - [FunASR](https://github.com/modelscope/FunASR), [faster-whisper](https://github.com/SYSTRAN/faster-whisper), [UniSpeech](https://github.com/microsoft/UniSpeech), [SpeechMOS](https://github.com/tarepan/SpeechMOS) for evaluation tools - [ctc-forced-aligner](https://github.com/MahmoudAshraf97/ctc-forced-aligner) for speech edit test - [mrfakename](https://x.com/realmrfakename) huggingface space demo ~ From c93462fc68f93241ed023b5074b5c575e082c8a6 Mon Sep 17 00:00:00 2001 From: SWivid Date: Mon, 16 Dec 2024 16:25:23 +0800 Subject: [PATCH 7/9] feature. allow custom model config for gradio infer --- README.md | 2 +- src/f5_tts/configs/F5TTS_Base_train.yaml | 2 +- src/f5_tts/configs/F5TTS_Small_train.yaml | 2 +- src/f5_tts/infer/SHARED.md | 73 +++++++++-------- src/f5_tts/infer/infer_cli.py | 19 +++-- src/f5_tts/infer/infer_gradio.py | 95 ++++++++++++++++------- src/f5_tts/model/backbones/dit.py | 3 +- 7 files changed, 126 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 96cbb57..9dc301d 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Note: Some model components have linting exceptions for E722 to accommodate tens - [Emilia](https://arxiv.org/abs/2407.05361), [WenetSpeech4TTS](https://arxiv.org/abs/2406.05763), [LibriTTS](https://arxiv.org/abs/1904.02882), [LJSpeech](https://keithito.com/LJ-Speech-Dataset/) valuable datasets - [lucidrains](https://github.com/lucidrains) initial CFM structure with also [bfs18](https://github.com/bfs18) for discussion - [SD3](https://arxiv.org/abs/2403.03206) & [Hugging Face diffusers](https://github.com/huggingface/diffusers) DiT and MMDiT code structure -- [torchdiffeq](https://github.com/rtqichen/torchdiffeq) as ODE solver, [Vocos](https://huggingface.co/charactr/vocos-mel-24khz) and [BigVGAN](https://github.com/NVIDIA/BigVGAN/tree/main) as vocoder +- [torchdiffeq](https://github.com/rtqichen/torchdiffeq) as ODE solver, [Vocos](https://huggingface.co/charactr/vocos-mel-24khz) and [BigVGAN](https://github.com/NVIDIA/BigVGAN) as vocoder - [FunASR](https://github.com/modelscope/FunASR), [faster-whisper](https://github.com/SYSTRAN/faster-whisper), [UniSpeech](https://github.com/microsoft/UniSpeech), [SpeechMOS](https://github.com/tarepan/SpeechMOS) for evaluation tools - [ctc-forced-aligner](https://github.com/MahmoudAshraf97/ctc-forced-aligner) for speech edit test - [mrfakename](https://x.com/realmrfakename) huggingface space demo ~ diff --git a/src/f5_tts/configs/F5TTS_Base_train.yaml b/src/f5_tts/configs/F5TTS_Base_train.yaml index fe93332..381843d 100644 --- a/src/f5_tts/configs/F5TTS_Base_train.yaml +++ b/src/f5_tts/configs/F5TTS_Base_train.yaml @@ -28,7 +28,7 @@ model: ff_mult: 2 text_dim: 512 conv_layers: 4 - checkpoint_activations: False # recompute activations and save memory for extra compute + checkpoint_activations: False # recompute activations and save memory for extra compute mel_spec: target_sample_rate: 24000 n_mel_channels: 100 diff --git a/src/f5_tts/configs/F5TTS_Small_train.yaml b/src/f5_tts/configs/F5TTS_Small_train.yaml index 466ee29..d261b18 100644 --- a/src/f5_tts/configs/F5TTS_Small_train.yaml +++ b/src/f5_tts/configs/F5TTS_Small_train.yaml @@ -28,7 +28,7 @@ model: ff_mult: 2 text_dim: 512 conv_layers: 4 - checkpoint_activations: False # recompute activations and save memory for extra compute + checkpoint_activations: False # recompute activations and save memory for extra compute mel_spec: target_sample_rate: 24000 n_mel_channels: 100 diff --git a/src/f5_tts/infer/SHARED.md b/src/f5_tts/infer/SHARED.md index a09bef9..88883fa 100644 --- a/src/f5_tts/infer/SHARED.md +++ b/src/f5_tts/infer/SHARED.md @@ -16,33 +16,34 @@ ### Supported Languages - [Multilingual](#multilingual) - - [F5-TTS Base @ pretrain @ zh \& en](#f5-tts-base--pretrain--zh--en) + - [F5-TTS Base @ zh \& en @ F5-TTS](#f5-tts-base--zh--en--f5-tts) - [English](#english) - [Finnish](#finnish) - - [Finnish Common\_Voice Vox\_Populi @ finetune @ fi](#finnish-common_voice-vox_populi--finetune--fi) + - [F5-TTS Base @ fi @ AsmoKoskinen](#f5-tts-base--fi--asmokoskinen) - [French](#french) - - [French LibriVox @ finetune @ fr](#french-librivox--finetune--fr) + - [F5-TTS Base @ fr @ RASPIAUDIO](#f5-tts-base--fr--raspiaudio) - [Hindi](#hindi) - - [F5-TTS Small @ pretrain @ hi](#f5-tts-small--pretrain--hi) + - [F5-TTS Small @ hi @ SPRINGLab](#f5-tts-small--hi--springlab) - [Italian](#italian) - - [F5-TTS Italian @ finetune @ it](#f5-tts-italian--finetune--it) + - [F5-TTS Base @ it @ alien79](#f5-tts-base--it--alien79) - [Japanese](#japanese) - - [F5-TTS Japanese @ pretrain/finetune @ ja](#f5-tts-japanese--pretrainfinetune--ja) + - [F5-TTS Base @ ja @ Jmica](#f5-tts-base--ja--jmica) - [Mandarin](#mandarin) - [Spanish](#spanish) - - [F5-TTS Spanish @ pretrain/finetune @ es](#f5-tts-spanish--pretrainfinetune--es) + - [F5-TTS Base @ es @ jpgallegoar](#f5-tts-base--es--jpgallegoar) ## Multilingual -#### F5-TTS Base @ pretrain @ zh & en +#### F5-TTS Base @ zh & en @ F5-TTS |Model|🤗Hugging Face|Data (Hours)|Model License| |:---:|:------------:|:-----------:|:-------------:| |F5-TTS Base|[ckpt & vocab](https://huggingface.co/SWivid/F5-TTS/tree/main/F5TTS_Base)|[Emilia 95K zh&en](https://huggingface.co/datasets/amphion/Emilia-Dataset/tree/fc71e07)|cc-by-nc-4.0| ```bash -MODEL_CKPT: hf://SWivid/F5-TTS/F5TTS_Base/model_1200000.safetensors -VOCAB_FILE: hf://SWivid/F5-TTS/F5TTS_Base/vocab.txt +Model: hf://SWivid/F5-TTS/F5TTS_Base/model_1200000.safetensors +Vocab: hf://SWivid/F5-TTS/F5TTS_Base/vocab.txt +Config: {"dim": 1024, "depth": 22, "heads": 16, "ff_mult": 2, "text_dim": 512, "conv_layers": 4} ``` *Other infos, e.g. Author info, Github repo, Link to some sampled results, Usage instruction, Tutorial (Blog, Video, etc.) ...* @@ -53,27 +54,29 @@ VOCAB_FILE: hf://SWivid/F5-TTS/F5TTS_Base/vocab.txt ## Finnish -#### Finnish Common_Voice Vox_Populi @ finetune @ fi +#### F5-TTS Base @ fi @ AsmoKoskinen |Model|🤗Hugging Face|Data|Model License| |:---:|:------------:|:-----------:|:-------------:| -|F5-TTS Finnish|[ckpt & vocab](https://huggingface.co/AsmoKoskinen/F5-TTS_Finnish_Model)|[Common Voice](https://huggingface.co/datasets/mozilla-foundation/common_voice_17_0), [Vox Populi](https://huggingface.co/datasets/facebook/voxpopuli)|cc-by-nc-4.0| +|F5-TTS Base|[ckpt & vocab](https://huggingface.co/AsmoKoskinen/F5-TTS_Finnish_Model)|[Common Voice](https://huggingface.co/datasets/mozilla-foundation/common_voice_17_0), [Vox Populi](https://huggingface.co/datasets/facebook/voxpopuli)|cc-by-nc-4.0| ```bash -MODEL_CKPT: hf://AsmoKoskinen/F5-TTS_Finnish_Model/model_common_voice_fi_vox_populi_fi_20241206.safetensors -VOCAB_FILE: hf://AsmoKoskinen/F5-TTS_Finnish_Model/vocab.txt +Model: hf://AsmoKoskinen/F5-TTS_Finnish_Model/model_common_voice_fi_vox_populi_fi_20241206.safetensors +Vocab: hf://AsmoKoskinen/F5-TTS_Finnish_Model/vocab.txt +Config: {"dim": 1024, "depth": 22, "heads": 16, "ff_mult": 2, "text_dim": 512, "conv_layers": 4} ``` ## French -#### French LibriVox @ finetune @ fr +#### F5-TTS Base @ fr @ RASPIAUDIO |Model|🤗Hugging Face|Data (Hours)|Model License| |:---:|:------------:|:-----------:|:-------------:| -|F5-TTS French|[ckpt & vocab](https://huggingface.co/RASPIAUDIO/F5-French-MixedSpeakers-reduced)|[LibriVox](https://librivox.org/)|cc-by-nc-4.0| +|F5-TTS Base|[ckpt & vocab](https://huggingface.co/RASPIAUDIO/F5-French-MixedSpeakers-reduced)|[LibriVox](https://librivox.org/)|cc-by-nc-4.0| ```bash -MODEL_CKPT: hf://RASPIAUDIO/F5-French-MixedSpeakers-reduced/model_last_reduced.pt -VOCAB_FILE: hf://RASPIAUDIO/F5-French-MixedSpeakers-reduced/vocab.txt +Model: hf://RASPIAUDIO/F5-French-MixedSpeakers-reduced/model_last_reduced.pt +Vocab: hf://RASPIAUDIO/F5-French-MixedSpeakers-reduced/vocab.txt +Config: {"dim": 1024, "depth": 22, "heads": 16, "ff_mult": 2, "text_dim": 512, "conv_layers": 4} ``` - [Online Inference with Hugging Face Space](https://huggingface.co/spaces/RASPIAUDIO/f5-tts_french). @@ -83,31 +86,32 @@ VOCAB_FILE: hf://RASPIAUDIO/F5-French-MixedSpeakers-reduced/vocab.txt ## Hindi -#### F5-TTS Small @ pretrain @ hi +#### F5-TTS Small @ hi @ SPRINGLab |Model|🤗Hugging Face|Data (Hours)|Model License| |:---:|:------------:|:-----------:|:-------------:| |F5-TTS Small|[ckpt & vocab](https://huggingface.co/SPRINGLab/F5-Hindi-24KHz)|[IndicTTS Hi](https://huggingface.co/datasets/SPRINGLab/IndicTTS-Hindi) & [IndicVoices-R Hi](https://huggingface.co/datasets/SPRINGLab/IndicVoices-R_Hindi) |cc-by-4.0| ```bash -MODEL_CKPT: hf://SPRINGLab/F5-Hindi-24KHz/model_2500000.safetensors -VOCAB_FILE: hf://SPRINGLab/F5-Hindi-24KHz/vocab.txt +Model: hf://SPRINGLab/F5-Hindi-24KHz/model_2500000.safetensors +Vocab: hf://SPRINGLab/F5-Hindi-24KHz/vocab.txt +Config: {"dim": 768, "depth": 18, "heads": 12, "ff_mult": 2, "text_dim": 512, "conv_layers": 4} ``` -Authors: SPRING Lab, Indian Institute of Technology, Madras -
-Website: https://asr.iitm.ac.in/ +- Authors: SPRING Lab, Indian Institute of Technology, Madras +- Website: https://asr.iitm.ac.in/ ## Italian -#### F5-TTS Italian @ finetune @ it +#### F5-TTS Base @ it @ alien79 |Model|🤗Hugging Face|Data|Model License| |:---:|:------------:|:-----------:|:-------------:| -|F5-TTS Italian|[ckpt & vocab](https://huggingface.co/alien79/F5-TTS-italian)|[ylacombe/cml-tts](https://huggingface.co/datasets/ylacombe/cml-tts) |cc-by-nc-4.0| +|F5-TTS Base|[ckpt & vocab](https://huggingface.co/alien79/F5-TTS-italian)|[ylacombe/cml-tts](https://huggingface.co/datasets/ylacombe/cml-tts) |cc-by-nc-4.0| ```bash -MODEL_CKPT: hf://alien79/F5-TTS-italian/model_159600.safetensors -VOCAB_FILE: hf://alien79/F5-TTS-italian/vocab.txt +Model: hf://alien79/F5-TTS-italian/model_159600.safetensors +Vocab: hf://alien79/F5-TTS-italian/vocab.txt +Config: {"dim": 1024, "depth": 22, "heads": 16, "ff_mult": 2, "text_dim": 512, "conv_layers": 4} ``` - Trained by [Mithril Man](https://github.com/MithrilMan) @@ -117,14 +121,15 @@ VOCAB_FILE: hf://alien79/F5-TTS-italian/vocab.txt ## Japanese -#### F5-TTS Japanese @ pretrain/finetune @ ja +#### F5-TTS Base @ ja @ Jmica |Model|🤗Hugging Face|Data (Hours)|Model License| |:---:|:------------:|:-----------:|:-------------:| -|F5-TTS Japanese|[ckpt & vocab](https://huggingface.co/Jmica/F5TTS/tree/main/JA_8500000)|[Emilia 1.7k JA](https://huggingface.co/datasets/amphion/Emilia-Dataset/tree/fc71e07) & [Galgame Dataset 5.4k](https://huggingface.co/datasets/OOPPEENN/Galgame_Dataset)|cc-by-nc-4.0| +|F5-TTS Base|[ckpt & vocab](https://huggingface.co/Jmica/F5TTS/tree/main/JA_8500000)|[Emilia 1.7k JA](https://huggingface.co/datasets/amphion/Emilia-Dataset/tree/fc71e07) & [Galgame Dataset 5.4k](https://huggingface.co/datasets/OOPPEENN/Galgame_Dataset)|cc-by-nc-4.0| ```bash -MODEL_CKPT: hf://Jmica/F5TTS/JA_8500000/model_8499660.pt -VOCAB_FILE: hf://Jmica/F5TTS/JA_8500000/vocab_updated.txt +Model: hf://Jmica/F5TTS/JA_8500000/model_8499660.pt +Vocab: hf://Jmica/F5TTS/JA_8500000/vocab_updated.txt +Config: {"dim": 1024, "depth": 22, "heads": 16, "ff_mult": 2, "text_dim": 512, "conv_layers": 4} ``` @@ -133,9 +138,9 @@ VOCAB_FILE: hf://Jmica/F5TTS/JA_8500000/vocab_updated.txt ## Spanish -#### F5-TTS Spanish @ pretrain/finetune @ es +#### F5-TTS Base @ es @ jpgallegoar |Model|🤗Hugging Face|Data (Hours)|Model License| |:---:|:------------:|:-----------:|:-------------:| -|F5-TTS Spanish|[ckpt & vocab](https://huggingface.co/jpgallegoar/F5-Spanish)|[Voxpopuli](https://huggingface.co/datasets/facebook/voxpopuli) & Crowdsourced & TEDx, 218 hours|cc0-1.0| +|F5-TTS Base|[ckpt & vocab](https://huggingface.co/jpgallegoar/F5-Spanish)|[Voxpopuli](https://huggingface.co/datasets/facebook/voxpopuli) & Crowdsourced & TEDx, 218 hours|cc0-1.0| - @jpgallegoar [GitHub repo](https://github.com/jpgallegoar/Spanish-F5), Jupyter Notebook and Gradio usage for Spanish model. diff --git a/src/f5_tts/infer/infer_cli.py b/src/f5_tts/infer/infer_cli.py index 47a71a5..69e7464 100644 --- a/src/f5_tts/infer/infer_cli.py +++ b/src/f5_tts/infer/infer_cli.py @@ -10,6 +10,7 @@ import numpy as np import soundfile as sf import tomli from cached_path import cached_path +from omegaconf import OmegaConf from f5_tts.infer.utils_infer import ( mel_spec_type, @@ -51,6 +52,12 @@ parser.add_argument( type=str, help="The model name: F5-TTS | E2-TTS", ) +parser.add_argument( + "-mc", + "--model_cfg", + type=str, + help="The path to F5-TTS model config file .yaml", +) parser.add_argument( "-p", "--ckpt_file", @@ -166,6 +173,7 @@ config = tomli.load(open(args.config, "rb")) # command-line interface parameters model = args.model or config.get("model", "F5-TTS") +model_cfg = args.model_cfg or config.get("model_cfg", str(files("f5_tts").joinpath("configs/F5TTS_Base_train.yaml"))) ckpt_file = args.ckpt_file or config.get("ckpt_file", "") vocab_file = args.vocab_file or config.get("vocab_file", "") @@ -179,9 +187,9 @@ output_file = args.output_file or config.get( "output_file", f"infer_cli_{datetime.now().strftime(r'%Y%m%d_%H%M%S')}.wav" ) -save_chunk = args.save_chunk -remove_silence = args.remove_silence -load_vocoder_from_local = args.load_vocoder_from_local +save_chunk = args.save_chunk or config.get("save_chunk", False) +remove_silence = args.remove_silence or config.get("remove_silence", False) +load_vocoder_from_local = args.load_vocoder_from_local or config.get("load_vocoder_from_local", False) vocoder_name = args.vocoder_name or config.get("vocoder_name", mel_spec_type) target_rms = args.target_rms or config.get("target_rms", target_rms) @@ -235,7 +243,7 @@ vocoder = load_vocoder(vocoder_name=vocoder_name, is_local=load_vocoder_from_loc if model == "F5-TTS": model_cls = DiT - model_cfg = dict(dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4) + model_cfg = OmegaConf.load(model_cfg).model.arch if not ckpt_file: # path not specified, download from repo if vocoder_name == "vocos": repo_name = "F5-TTS" @@ -250,7 +258,8 @@ if model == "F5-TTS": ckpt_file = str(cached_path(f"hf://SWivid/{repo_name}/{exp_name}/model_{ckpt_step}.pt")) elif model == "E2-TTS": - assert vocoder_name == "vocos", "E2-TTS only supports vocoder vocos" + assert args.model_cfg is None, "E2-TTS does not support custom model_cfg yet" + assert vocoder_name == "vocos", "E2-TTS only supports vocoder vocos yet" model_cls = UNetT model_cfg = dict(dim=1024, depth=24, heads=16, ff_mult=4) if not ckpt_file: # path not specified, download from repo diff --git a/src/f5_tts/infer/infer_gradio.py b/src/f5_tts/infer/infer_gradio.py index 869a736..86aa443 100644 --- a/src/f5_tts/infer/infer_gradio.py +++ b/src/f5_tts/infer/infer_gradio.py @@ -1,6 +1,7 @@ # ruff: noqa: E402 # Above allows ruff to ignore E402: module level import not at top of file +import json import re import tempfile from collections import OrderedDict @@ -43,6 +44,12 @@ from f5_tts.infer.utils_infer import ( DEFAULT_TTS_MODEL = "F5-TTS" tts_model_choice = DEFAULT_TTS_MODEL +DEFAULT_TTS_MODEL_CFG = [ + "hf://SWivid/F5-TTS/F5TTS_Base/model_1200000.safetensors", + "hf://SWivid/F5-TTS/F5TTS_Base/vocab.txt", + json.dumps(dict(dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4)), +] + # load models @@ -103,7 +110,15 @@ def generate_response(messages, model, tokenizer): @gpu_decorator def infer( - ref_audio_orig, ref_text, gen_text, model, remove_silence, cross_fade_duration=0.15, speed=1, show_info=gr.Info + ref_audio_orig, + ref_text, + gen_text, + model, + remove_silence, + cross_fade_duration=0.15, + nfe_step=32, + speed=1, + show_info=gr.Info, ): ref_audio, ref_text = preprocess_ref_audio_text(ref_audio_orig, ref_text, show_info=show_info) @@ -120,7 +135,7 @@ def infer( global custom_ema_model, pre_custom_path if pre_custom_path != model[1]: show_info("Loading Custom TTS model...") - custom_ema_model = load_custom(model[1], vocab_path=model[2]) + custom_ema_model = load_custom(model[1], vocab_path=model[2], model_cfg=model[3]) pre_custom_path = model[1] ema_model = custom_ema_model @@ -131,6 +146,7 @@ def infer( ema_model, vocoder, cross_fade_duration=cross_fade_duration, + nfe_step=nfe_step, speed=speed, show_info=show_info, progress=gr.Progress(), @@ -184,6 +200,14 @@ with gr.Blocks() as app_tts: step=0.1, info="Adjust the speed of the audio.", ) + nfe_slider = gr.Slider( + label="NFE Steps", + minimum=4, + maximum=64, + value=32, + step=2, + info="Set the number of denoising steps.", + ) cross_fade_duration_slider = gr.Slider( label="Cross-Fade Duration (s)", minimum=0.0, @@ -203,6 +227,7 @@ with gr.Blocks() as app_tts: gen_text_input, remove_silence, cross_fade_duration_slider, + nfe_slider, speed_slider, ): audio_out, spectrogram_path, ref_text_out = infer( @@ -211,8 +236,9 @@ with gr.Blocks() as app_tts: gen_text_input, tts_model_choice, remove_silence, - cross_fade_duration_slider, - speed_slider, + cross_fade_duration=cross_fade_duration_slider, + nfe_step=nfe_slider, + speed=speed_slider, ) return audio_out, spectrogram_path, gr.update(value=ref_text_out) @@ -224,6 +250,7 @@ with gr.Blocks() as app_tts: gen_text_input, remove_silence, cross_fade_duration_slider, + nfe_slider, speed_slider, ], outputs=[audio_output, spectrogram_output, ref_text_input], @@ -744,34 +771,38 @@ If you're having issues, try converting your reference audio to WAV or MP3, clip """ ) - last_used_custom = files("f5_tts").joinpath("infer/.cache/last_used_custom.txt") + last_used_custom = files("f5_tts").joinpath("infer/.cache/last_used_custom_model_info.txt") def load_last_used_custom(): try: - with open(last_used_custom, "r") as f: - return f.read().split(",") + custom = [] + with open(last_used_custom, "r", encoding='utf-8') as f: + for line in f: + custom.append(line.strip()) + return custom except FileNotFoundError: last_used_custom.parent.mkdir(parents=True, exist_ok=True) - return [ - "hf://SWivid/F5-TTS/F5TTS_Base/model_1200000.safetensors", - "hf://SWivid/F5-TTS/F5TTS_Base/vocab.txt", - ] + return DEFAULT_TTS_MODEL_CFG def switch_tts_model(new_choice): global tts_model_choice if new_choice == "Custom": # override in case webpage is refreshed - custom_ckpt_path, custom_vocab_path = load_last_used_custom() - tts_model_choice = ["Custom", custom_ckpt_path, custom_vocab_path] - return gr.update(visible=True, value=custom_ckpt_path), gr.update(visible=True, value=custom_vocab_path) + custom_ckpt_path, custom_vocab_path, custom_model_cfg = load_last_used_custom() + tts_model_choice = ["Custom", custom_ckpt_path, custom_vocab_path, json.loads(custom_model_cfg)] + return ( + gr.update(visible=True, value=custom_ckpt_path), + gr.update(visible=True, value=custom_vocab_path), + gr.update(visible=True, value=custom_model_cfg), + ) else: tts_model_choice = new_choice - return gr.update(visible=False), gr.update(visible=False) + return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False) - def set_custom_model(custom_ckpt_path, custom_vocab_path): + def set_custom_model(custom_ckpt_path, custom_vocab_path, custom_model_cfg): global tts_model_choice - tts_model_choice = ["Custom", custom_ckpt_path, custom_vocab_path] - with open(last_used_custom, "w") as f: - f.write(f"{custom_ckpt_path},{custom_vocab_path}") + tts_model_choice = ["Custom", custom_ckpt_path, custom_vocab_path, json.loads(custom_model_cfg)] + with open(last_used_custom, "w", encoding='utf-8') as f: + f.write(custom_ckpt_path + "\n" + custom_vocab_path + "\n" + custom_model_cfg + "\n") with gr.Row(): if not USING_SPACES: @@ -783,34 +814,46 @@ If you're having issues, try converting your reference audio to WAV or MP3, clip choices=[DEFAULT_TTS_MODEL, "E2-TTS"], label="Choose TTS Model", value=DEFAULT_TTS_MODEL ) custom_ckpt_path = gr.Dropdown( - choices=["hf://SWivid/F5-TTS/F5TTS_Base/model_1200000.safetensors"], + choices=[DEFAULT_TTS_MODEL_CFG[0]], value=load_last_used_custom()[0], allow_custom_value=True, - label="MODEL CKPT: local_path | hf://user_id/repo_id/model_ckpt", + label="Model: local_path | hf://user_id/repo_id/model_ckpt", visible=False, ) custom_vocab_path = gr.Dropdown( - choices=["hf://SWivid/F5-TTS/F5TTS_Base/vocab.txt"], + choices=[DEFAULT_TTS_MODEL_CFG[1]], value=load_last_used_custom()[1], allow_custom_value=True, - label="VOCAB FILE: local_path | hf://user_id/repo_id/vocab_file", + label="Vocab: local_path | hf://user_id/repo_id/vocab_file", + visible=False, + ) + custom_model_cfg = gr.Dropdown( + choices=[DEFAULT_TTS_MODEL_CFG[2]], + value=load_last_used_custom()[2], + allow_custom_value=True, + label="Config: in a dictionary form", visible=False, ) choose_tts_model.change( switch_tts_model, inputs=[choose_tts_model], - outputs=[custom_ckpt_path, custom_vocab_path], + outputs=[custom_ckpt_path, custom_vocab_path, custom_model_cfg], show_progress="hidden", ) custom_ckpt_path.change( set_custom_model, - inputs=[custom_ckpt_path, custom_vocab_path], + inputs=[custom_ckpt_path, custom_vocab_path, custom_model_cfg], show_progress="hidden", ) custom_vocab_path.change( set_custom_model, - inputs=[custom_ckpt_path, custom_vocab_path], + inputs=[custom_ckpt_path, custom_vocab_path, custom_model_cfg], + show_progress="hidden", + ) + custom_model_cfg.change( + set_custom_model, + inputs=[custom_ckpt_path, custom_vocab_path, custom_model_cfg], show_progress="hidden", ) diff --git a/src/f5_tts/model/backbones/dit.py b/src/f5_tts/model/backbones/dit.py index 472af28..1ecd10e 100644 --- a/src/f5_tts/model/backbones/dit.py +++ b/src/f5_tts/model/backbones/dit.py @@ -131,8 +131,7 @@ class DiT(nn.Module): self.checkpoint_activations = checkpoint_activations def ckpt_wrapper(self, module): - """Code from https://github.com/chuanyangjin/fast-DiT/blob/1a8ecce58f346f877749f2dc67cdb190d295e4dc/models.py#L233-L237""" - + # https://github.com/chuanyangjin/fast-DiT/blob/main/models.py def ckpt_forward(*inputs): outputs = module(*inputs) return outputs From 17087321f5792ca4a602b2f192a2ec34b233a626 Mon Sep 17 00:00:00 2001 From: SWivid Date: Mon, 16 Dec 2024 16:26:21 +0800 Subject: [PATCH 8/9] formatting --- src/f5_tts/infer/infer_gradio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/f5_tts/infer/infer_gradio.py b/src/f5_tts/infer/infer_gradio.py index 86aa443..9bd9780 100644 --- a/src/f5_tts/infer/infer_gradio.py +++ b/src/f5_tts/infer/infer_gradio.py @@ -776,7 +776,7 @@ If you're having issues, try converting your reference audio to WAV or MP3, clip def load_last_used_custom(): try: custom = [] - with open(last_used_custom, "r", encoding='utf-8') as f: + with open(last_used_custom, "r", encoding="utf-8") as f: for line in f: custom.append(line.strip()) return custom @@ -801,7 +801,7 @@ If you're having issues, try converting your reference audio to WAV or MP3, clip def set_custom_model(custom_ckpt_path, custom_vocab_path, custom_model_cfg): global tts_model_choice tts_model_choice = ["Custom", custom_ckpt_path, custom_vocab_path, json.loads(custom_model_cfg)] - with open(last_used_custom, "w", encoding='utf-8') as f: + with open(last_used_custom, "w", encoding="utf-8") as f: f.write(custom_ckpt_path + "\n" + custom_vocab_path + "\n" + custom_model_cfg + "\n") with gr.Row(): From 81f69e57b72567375fc5945e29ebbfd41ef55fc6 Mon Sep 17 00:00:00 2001 From: Yushen CHEN <45333109+SWivid@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:31:23 +0800 Subject: [PATCH 9/9] Update infer_cli.py; typo --- src/f5_tts/infer/infer_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/f5_tts/infer/infer_cli.py b/src/f5_tts/infer/infer_cli.py index 69e7464..8cf3f5b 100644 --- a/src/f5_tts/infer/infer_cli.py +++ b/src/f5_tts/infer/infer_cli.py @@ -119,7 +119,7 @@ parser.add_argument( parser.add_argument( "--load_vocoder_from_local", action="store_true", - help="To load vocoder from local dir, default to ../checkpoints/charactr/vocos-mel-24khz", + help="To load vocoder from local dir, default to ../checkpoints/vocos-mel-24khz", ) parser.add_argument( "--vocoder_name",