update

.gitignore

@@ -130,3 +130,4 @@ dmypy.json
 .pyre/
 
 .idea/*
+.models/*

.models/clip.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8ab5a7751b6098b7e57528b5d812ea2ffbaa16f1b36c02e143c501c74900140d
-size 271601435

api.py

@@ -23,9 +23,11 @@ from utils.tokenizer import VoiceBpeTokenizer, lev_distance
 pbar = None
 def download_models():
     MODELS = {
-        'clip.pth': 'https://huggingface.co/jbetker/tortoise-tts-clip/resolve/main/pytorch-model.bin',
-        'diffusion.pth': 'https://huggingface.co/jbetker/tortoise-tts-diffusion-v1/resolve/main/pytorch-model.bin',
-        'autoregressive.pth': 'https://huggingface.co/jbetker/tortoise-tts-autoregressive/resolve/main/pytorch-model.bin'
+        'autoregressive.pth': 'https://huggingface.co/jbetker/tortoise-tts-v2/resolve/main/.models/autoregressive.pth',
+        'clvp.pth': 'https://huggingface.co/jbetker/tortoise-tts-v2/resolve/main/.models/clip.pth',
+        'cvvp.pth': 'https://huggingface.co/jbetker/tortoise-tts-v2/resolve/main/.models/cvvp.pth',
+        'diffusion_decoder.pth': 'https://huggingface.co/jbetker/tortoise-tts-v2/resolve/main/.models/diffusion_decoder.pth',
+        'vocoder.pth': 'https://huggingface.co/jbetker/tortoise-tts-v2/resolve/main/.models/vocoder.pth',
     }
     os.makedirs('.models', exist_ok=True)
     def show_progress(block_num, block_size, total_size):
@@ -162,25 +164,12 @@ class TextToSpeech:
                                       train_solo_embeddings=False,
                                       average_conditioning_embeddings=True).cpu().eval()
         self.autoregressive.load_state_dict(torch.load('.models/autoregressive.pth'))
-        '''
-        self.autoregressive = UnifiedVoice(max_mel_tokens=2048, max_text_tokens=1024, max_conditioning_inputs=1, layers=42,
-                                      model_dim=1152, heads=18, number_text_tokens=256, train_solo_embeddings=False,
-                                      average_conditioning_embeddings=True, types=2).cpu().eval()
-        self.autoregressive.load_state_dict(torch.load('X:\\dlas\\experiments\\train_gpt_tts_xl\\models\\15250_gpt_ema.pth'))
-        '''
-
-        self.autoregressive_for_diffusion = UnifiedVoice(max_mel_tokens=604, max_text_tokens=402, max_conditioning_inputs=2, layers=30,
-                                      model_dim=1024,
-                                      heads=16, number_text_tokens=255, start_text_token=255, checkpointing=False,
-                                      train_solo_embeddings=False,
-                                      average_conditioning_embeddings=True).cpu().eval()
-        self.autoregressive_for_diffusion.load_state_dict(torch.load('.models/autoregressive.pth'))
 
         self.clvp = CLVP(dim_text=512, dim_speech=512, dim_latent=512, num_text_tokens=256, text_enc_depth=12,
                          text_seq_len=350, text_heads=8,
                          num_speech_tokens=8192, speech_enc_depth=12, speech_heads=8, speech_seq_len=430,
                          use_xformers=True).cpu().eval()
-        self.clvp.load_state_dict(torch.load('.models/clip.pth'))
+        self.clvp.load_state_dict(torch.load('.models/clvp.pth'))
 
         self.cvvp = CVVP(model_dim=512, transformer_heads=8, dropout=0, mel_codes=8192, conditioning_enc_depth=8, cond_mask_percentage=0,
                          speech_enc_depth=8, speech_mask_percentage=0, latent_multiplier=1).cpu().eval()
@@ -213,7 +202,7 @@ class TextToSpeech:
             'ultra_fast': {'num_autoregressive_samples': 32, 'diffusion_iterations': 16, 'cond_free': False},
             'fast': {'num_autoregressive_samples': 96, 'diffusion_iterations': 32},
             'standard': {'num_autoregressive_samples': 256, 'diffusion_iterations': 128},
-            'high_quality': {'num_autoregressive_samples': 512, 'diffusion_iterations': 2048},
+            'high_quality': {'num_autoregressive_samples': 512, 'diffusion_iterations': 1024},
         }
         kwargs.update(presets[preset])
         return self.tts(text, voice_samples, **kwargs)
@@ -281,11 +270,11 @@ class TextToSpeech:
             # The diffusion model actually wants the last hidden layer from the autoregressive model as conditioning
             # inputs. Re-produce those for the top results. This could be made more efficient by storing all of these
             # results, but will increase memory usage.
-            self.autoregressive_for_diffusion = self.autoregressive_for_diffusion.cuda()
-            best_latents = self.autoregressive_for_diffusion(conds, text, torch.tensor([text.shape[-1]], device=conds.device), best_results,
-                                               torch.tensor([best_results.shape[-1]*self.autoregressive_for_diffusion.mel_length_compression], device=conds.device),
+            self.autoregressive = self.autoregressive.cuda()
+            best_latents = self.autoregressive(conds, text, torch.tensor([text.shape[-1]], device=conds.device), best_results,
+                                               torch.tensor([best_results.shape[-1]*self.autoregressive.mel_length_compression], device=conds.device),
                                                return_latent=True, clip_inputs=False)
-            self.autoregressive_for_diffusion = self.autoregressive_for_diffusion.cpu()
+            self.autoregressive = self.autoregressive.cpu()
 
             print("Performing vocoding..")
             wav_candidates = []

do_tts.py

@@ -11,6 +11,10 @@ if __name__ == '__main__':
     parser.add_argument('--text', type=str, help='Text to speak.', default="I am a language model that has learned to speak.")
     parser.add_argument('--voice', type=str, help='Selects the voice to use for generation. See options in voices/ directory (and add your own!) '
                                                  'Use the & character to join two voices together. Use a comma to perform inference on multiple voices.', default='patrick_stewart')
+    parser.add_argument('--preset', type=str, help='Which voice preset to use.', default='standard')
+    parser.add_argument('--voice_diversity_intelligibility_slider', type=float,
+                        help='How to balance vocal diversity with the quality/intelligibility of the spoken text. 0 means highly diverse voice (not recommended), 1 means maximize intellibility',
+                        default=.5)
     parser.add_argument('--output_path', type=str, help='Where to store outputs.', default='results/')
     args = parser.parse_args()
     os.makedirs(args.output_path, exist_ok=True)
@@ -25,6 +29,6 @@ if __name__ == '__main__':
         for cond_path in cond_paths:
             c = load_audio(cond_path, 22050)
             conds.append(c)
-        gen = tts.tts_with_preset(args.text, conds, preset='standard')
+        gen = tts.tts_with_preset(args.text, conds, preset=args.preset, clvp_cvvp_slider=args.voice_diversity_intelligibility_slider)
         torchaudio.save(os.path.join(args.output_path, f'{voice}.wav'), gen.squeeze(0).cpu(), 24000)
 

models/arch_util.py

@@ -5,8 +5,7 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 import torchaudio
-from x_transformers import ContinuousTransformerWrapper
-from x_transformers.x_transformers import RelativePositionBias
+from models.xtransformers import ContinuousTransformerWrapper, RelativePositionBias
 
 
 def zero_module(module):

models/clvp.py

@@ -2,10 +2,10 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 from torch import einsum
-from x_transformers import Encoder
 
 from models.arch_util import CheckpointedXTransformerEncoder
 from models.transformer import Transformer
+from models.xtransformers import Encoder
 
 
 def exists(val):

models/xtransformers.py

@@ -1253,50 +1253,3 @@ class ContinuousTransformerWrapper(nn.Module):
             return tuple(res)
         return res[0]
 
-
-class XTransformer(nn.Module):
-    def __init__(
-            self,
-            *,
-            dim,
-            tie_token_emb=False,
-            **kwargs
-    ):
-        super().__init__()
-        enc_kwargs, kwargs = groupby_prefix_and_trim('enc_', kwargs)
-        dec_kwargs, kwargs = groupby_prefix_and_trim('dec_', kwargs)
-
-        assert 'dim' not in enc_kwargs and 'dim' not in dec_kwargs, 'dimension of either encoder or decoder must be set with `dim` keyword'
-        enc_transformer_kwargs = pick_and_pop(['num_tokens', 'max_seq_len'], enc_kwargs)
-        enc_transformer_kwargs['emb_dropout'] = enc_kwargs.pop('emb_dropout', 0)
-        enc_transformer_kwargs['num_memory_tokens'] = enc_kwargs.pop('num_memory_tokens', None)
-        enc_transformer_kwargs['use_pos_emb'] = enc_kwargs.pop('use_pos_emb', True)
-
-        dec_transformer_kwargs = pick_and_pop(['num_tokens', 'max_seq_len'], dec_kwargs)
-        dec_transformer_kwargs['emb_dropout'] = dec_kwargs.pop('emb_dropout', 0)
-        dec_transformer_kwargs['use_pos_emb'] = dec_kwargs.pop('use_pos_emb', True)
-
-        self.encoder = TransformerWrapper(
-            **enc_transformer_kwargs,
-            attn_layers=Encoder(dim=dim, **enc_kwargs)
-        )
-
-        self.decoder = TransformerWrapper(
-            **dec_transformer_kwargs,
-            attn_layers=Decoder(dim=dim, cross_attend=True, **dec_kwargs)
-        )
-
-        if tie_token_emb:
-            self.decoder.token_emb = self.encoder.token_emb
-
-        self.decoder = AutoregressiveWrapper(self.decoder)
-
-    @torch.no_grad()
-    def generate(self, seq_in, seq_out_start, seq_len, src_mask=None, src_attn_mask=None, **kwargs):
-        encodings = self.encoder(seq_in, mask=src_mask, attn_mask=src_attn_mask, return_embeddings=True)
-        return self.decoder.generate(seq_out_start, seq_len, context=encodings, context_mask=src_mask, **kwargs)
-
-    def forward(self, src, tgt, src_mask=None, tgt_mask=None, src_attn_mask=None):
-        enc = self.encoder(src, mask=src_mask, attn_mask=src_attn_mask, return_embeddings=True)
-        out = self.decoder(tgt, context=enc, mask=tgt_mask, context_mask=src_mask)
-        return out

read.py

@@ -28,11 +28,14 @@ def split_and_recombine_text(texts, desired_length=200, max_len=300):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument('--textfile', type=str, help='A file containing the text to read.', default="data/riding_hood2.txt")
+    parser.add_argument('--textfile', type=str, help='A file containing the text to read.', default="data/riding_hood.txt")
     parser.add_argument('--voice', type=str, help='Selects the voice to use for generation. See options in voices/ directory (and add your own!) '
                                                  'Use the & character to join two voices together. Use a comma to perform inference on multiple voices.', default='patrick_stewart')
     parser.add_argument('--output_path', type=str, help='Where to store outputs.', default='results/longform/')
-    parser.add_argument('--generation_preset', type=str, help='Preset to use for generation', default='standard')
+    parser.add_argument('--preset', type=str, help='Which voice preset to use.', default='standard')
+    parser.add_argument('--voice_diversity_intelligibility_slider', type=float,
+                        help='How to balance vocal diversity with the quality/intelligibility of the spoken text. 0 means highly diverse voice (not recommended), 1 means maximize intellibility',
+                        default=.5)
     args = parser.parse_args()
 
     outpath = args.output_path
@@ -60,16 +63,11 @@ if __name__ == '__main__':
         if not cond_paths:
             print('Error: no valid voices specified. Try again.')
 
-        priors = []
+        conds = []
+        for cond_path in cond_paths:
+            c = load_audio(cond_path, 22050)
+            conds.append(c)
         for j, text in enumerate(texts):
-            conds = priors.copy()
-            for cond_path in cond_paths:
-                c = load_audio(cond_path, 22050)
-                conds.append(c)
-            gen = tts.tts_with_preset(text, conds, preset=args.generation_preset)
+            gen = tts.tts_with_preset(text, conds, preset=args.preset, clvp_cvvp_slider=args.voice_diversity_intelligibility_slider)
             torchaudio.save(os.path.join(voice_outpath, f'{j}.wav'), gen.squeeze(0).cpu(), 24000)
 
-            priors.append(torchaudio.functional.resample(gen, 24000, 22050).squeeze(0))
-            while len(priors) > 2:
-                priors.pop(0)
-

requirements.txt

@@ -6,5 +6,4 @@ tokenizers
 inflect
 progressbar
 einops
-unidecode
-x-transformers
+unidecode