update

examples/silero_vad_by_webrtcvad/step_1_prepare_data.py

@@ -43,7 +43,7 @@ def get_args():
     return args
 
 
-def target_second_signal_generator(data_dir: str, duration: int = 2, sample_rate: int = 8000, max_epoch: int = 20000):
+def target_second_signal_generator(data_dir: str, duration: int = 6, sample_rate: int = 8000, max_epoch: int = 20000):
     data_dir = Path(data_dir)
     for epoch_idx in range(max_epoch):
         for filename in data_dir.glob("**/*.wav"):

examples/silero_vad_by_webrtcvad/step_2_make_vad_segments.py

@@ -30,7 +30,7 @@ def get_args():
     parser.add_argument("--agg", default=3, type=int)
     parser.add_argument("--frame_duration_ms", default=30, type=int)
     parser.add_argument("--padding_duration_ms", default=30, type=int)
-    parser.add_argument("--silence_duration_threshold", default=0.3, type=float)
+    parser.add_argument("--silence_duration_threshold", default=0.0, type=float)
 
     args = parser.parse_args()
     return args

toolbox/torchaudio/models/vad/fsmn_vad/configuration_fsmn_vad.py

@@ -0,0 +1,66 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from typing import Tuple
+
+from toolbox.torchaudio.configuration_utils import PretrainedConfig
+
+
+class FSMNVadConfig(PretrainedConfig):
+    def __init__(self,
+                 sample_rate: int = 8000,
+                 nfft: int = 512,
+                 win_size: int = 240,
+                 hop_size: int = 80,
+                 win_type: str = "hann",
+
+                 in_channels: int = 64,
+                 hidden_size: int = 128,
+
+                 lr: float = 0.001,
+                 lr_scheduler: str = "CosineAnnealingLR",
+                 lr_scheduler_kwargs: dict = None,
+
+                 max_epochs: int = 100,
+                 clip_grad_norm: float = 10.,
+                 seed: int = 1234,
+
+                 num_workers: int = 4,
+                 batch_size: int = 4,
+                 eval_steps: int = 25000,
+
+                 **kwargs
+                 ):
+        super(FSMNVadConfig, self).__init__(**kwargs)
+        # transform
+        self.sample_rate = sample_rate
+        self.nfft = nfft
+        self.win_size = win_size
+        self.hop_size = hop_size
+        self.win_type = win_type
+
+        # encoder
+        self.in_channels = in_channels
+        self.hidden_size = hidden_size
+
+        # train
+        self.lr = lr
+        self.lr_scheduler = lr_scheduler
+        self.lr_scheduler_kwargs = lr_scheduler_kwargs or dict()
+
+        self.max_epochs = max_epochs
+        self.clip_grad_norm = clip_grad_norm
+        self.seed = seed
+
+        self.num_workers = num_workers
+        self.batch_size = batch_size
+        self.eval_steps = eval_steps
+
+
+def main():
+    config = FSMNVadConfig()
+    config.to_yaml_file("config.yaml")
+    return
+
+
+if __name__ == "__main__":
+    main()

toolbox/torchaudio/models/vad/fsmn_vad/fsmn_encoder.py

@@ -1,10 +1,11 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
+"""
+https://github.com/modelscope/FunASR/blob/main/funasr/models/fsmn_vad_streaming/encoder.py
+
+"""
 from typing import Tuple, Dict, List
-import copy
-import os
 
-import numpy as np
 import torch
 import torch.nn as nn
 import torch.nn.functional as F

toolbox/torchaudio/models/vad/fsmn_vad/modeling_fsmn_vad.py

@@ -10,8 +10,50 @@ https://github.com/lovemefan/fsmn-vad
 https://github.com/modelscope/FunASR/blob/main/funasr/models/fsmn_vad_streaming/encoder.py
 
 """
+import os
+from typing import Optional, Union
 
+import torch
+import torch.nn as nn
 
+from toolbox.torchaudio.configuration_utils import CONFIG_FILE
+from toolbox.torchaudio.models.vad.fsmn_vad.configuration_fsmn_vad import FSMNVadConfig
+from toolbox.torchaudio.modules.conv_stft import ConvSTFT
+from toolbox.torchaudio.models.vad.fsmn_vad.fsmn_encoder import FSMN
+
+
+MODEL_FILE = "model.pt"
+
+
+class FSMNVadModel(nn.Module):
+    def __init__(self, config: FSMNVadConfig):
+        super(FSMNVadModel, self).__init__()
+        self.config = config
+        self.eps = 1e-12
+
+        self.stft = ConvSTFT(
+            nfft=config.nfft,
+            win_size=config.win_size,
+            hop_size=config.hop_size,
+            win_type=config.win_type,
+            power=1,
+            requires_grad=False
+        )
+
+        self.fsmn_encoder = FSMN(
+            input_size=400,
+            input_affine_size=140,
+            hidden_size=250,
+            basic_block_layers=4,
+            basic_block_hidden_size=128,
+            basic_block_lorder=20,
+            basic_block_rorder=0,
+            basic_block_lstride=1,
+            basic_block_rstride=0,
+            output_affine_size=140,
+            output_size=248,
+            use_softmax=True,
+        )
 
 
 if __name__ == "__main__":

toolbox/torchaudio/modules/freq_bands/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == "__main__":
+    pass

toolbox/torchaudio/modules/freq_bands/erb_bands.py

@@ -0,0 +1,176 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+
+class ErbBandsNumpy(object):
+
+    @staticmethod
+    def freq2erb(freq_hz: float) -> float:
+        """
+        https://www.cnblogs.com/LXP-Never/p/16011229.html
+        1 / (24.7 * 9.265) = 0.00436976
+        """
+        return 9.265 * math.log(freq_hz / (24.7 * 9.265) + 1)
+
+    @staticmethod
+    def erb2freq(n_erb: float) -> float:
+        return 24.7 * 9.265 * (math.exp(n_erb / 9.265) - 1)
+
+    @classmethod
+    def get_erb_widths(cls, sample_rate: int, nfft: int, erb_bins: int, min_freq_bins_for_erb: int) -> np.ndarray:
+        """
+        https://github.com/Rikorose/DeepFilterNet/blob/main/libDF/src/lib.rs
+        :param sample_rate:
+        :param nfft:
+        :param erb_bins: erb (Equivalent Rectangular Bandwidth) 等效矩形带宽的通道数.
+        :param min_freq_bins_for_erb: Minimum number of frequency bands per erb band
+        :return:
+        """
+        nyq_freq = sample_rate / 2.
+        freq_width: float = sample_rate / nfft
+
+        min_erb: float = cls.freq2erb(0.)
+        max_erb: float = cls.freq2erb(nyq_freq)
+
+        erb = [0] * erb_bins
+        step = (max_erb - min_erb) / erb_bins
+
+        prev_freq_bin = 0
+        freq_over = 0
+        for i in range(1, erb_bins + 1):
+            f = cls.erb2freq(min_erb + i * step)
+            freq_bin = int(round(f / freq_width))
+            freq_bins = freq_bin - prev_freq_bin - freq_over
+
+            if freq_bins < min_freq_bins_for_erb:
+                freq_over = min_freq_bins_for_erb - freq_bins
+                freq_bins = min_freq_bins_for_erb
+            else:
+                freq_over = 0
+            erb[i - 1] = freq_bins
+            prev_freq_bin = freq_bin
+
+        erb[erb_bins - 1] += 1
+        too_large = sum(erb) - (nfft / 2 + 1)
+        if too_large > 0:
+            erb[erb_bins - 1] -= too_large
+        return np.array(erb, dtype=np.uint64)
+
+    @staticmethod
+    def get_erb_filter_bank(erb_widths: np.ndarray,
+                            normalized: bool = True,
+                            inverse: bool = False,
+                            ):
+        num_freq_bins = int(np.sum(erb_widths))
+        num_erb_bins = len(erb_widths)
+
+        fb: np.ndarray = np.zeros(shape=(num_freq_bins, num_erb_bins))
+
+        points = np.cumsum([0] + erb_widths.tolist()).astype(int)[:-1]
+        for i, (b, w) in enumerate(zip(points.tolist(), erb_widths.tolist())):
+            fb[b: b + w, i] = 1
+
+        if inverse:
+            fb = fb.T
+            if not normalized:
+                fb /= np.sum(fb, axis=1, keepdims=True)
+        else:
+            if normalized:
+                fb /= np.sum(fb, axis=0)
+        return fb
+
+    @staticmethod
+    def spec2erb(spec: np.ndarray, erb_fb: np.ndarray, db: bool = True):
+        """
+        ERB filterbank and transform to decibel scale.
+
+        :param spec: Spectrum of shape [B, C, T, F].
+        :param erb_fb: ERB filterbank array of shape [B] containing the ERB widths,
+                where B are the number of ERB bins.
+        :param db: Whether to transform the output into decibel scale. Defaults to `True`.
+        :return:
+        """
+        # complex spec to power spec. (real * real + image * image)
+        spec_ = np.abs(spec) ** 2
+
+        # spec to erb feature.
+        erb_feat = np.matmul(spec_, erb_fb)
+
+        if db:
+            erb_feat = 10 * np.log10(erb_feat + 1e-10)
+
+        erb_feat = np.array(erb_feat, dtype=np.float32)
+        return erb_feat
+
+
+class ErbBands(nn.Module):
+    def __init__(self,
+                 sample_rate: int = 8000,
+                 nfft: int = 512,
+                 erb_bins: int = 32,
+                 min_freq_bins_for_erb: int = 2,
+                 ):
+        super().__init__()
+        self.sample_rate = sample_rate
+        self.nfft = nfft
+        self.erb_bins = erb_bins
+        self.min_freq_bins_for_erb = min_freq_bins_for_erb
+
+        erb_fb, erb_fb_inv = self.init_erb_fb()
+        erb_fb = torch.tensor(erb_fb, dtype=torch.float32, requires_grad=False)
+        erb_fb_inv = torch.tensor(erb_fb_inv, dtype=torch.float32, requires_grad=False)
+        self.erb_fb = nn.Parameter(erb_fb, requires_grad=False)
+        self.erb_fb_inv = nn.Parameter(erb_fb_inv, requires_grad=False)
+
+    def init_erb_fb(self):
+        erb_widths = ErbBandsNumpy.get_erb_widths(
+            sample_rate=self.sample_rate,
+            nfft=self.nfft,
+            erb_bins=self.erb_bins,
+            min_freq_bins_for_erb=self.min_freq_bins_for_erb,
+        )
+        erb_fb = ErbBandsNumpy.get_erb_filter_bank(
+            erb_widths=erb_widths,
+            normalized=True,
+            inverse=False,
+        )
+        erb_fb_inv = ErbBandsNumpy.get_erb_filter_bank(
+            erb_widths=erb_widths,
+            normalized=True,
+            inverse=True,
+        )
+        return erb_fb, erb_fb_inv
+
+    def erb_scale(self, spec: torch.Tensor, db: bool = True):
+        # spec shape: (b, t, f)
+        spec_erb = torch.matmul(spec, self.erb_fb)
+        if db:
+            spec_erb = 10 * torch.log10(spec_erb + 1e-10)
+        return spec_erb
+
+    def erb_scale_inv(self, spec_erb: torch.Tensor):
+        spec = torch.matmul(spec_erb, self.erb_fb_inv)
+        return spec
+
+
+def main():
+
+    erb_bands = ErbBands()
+
+    spec = torch.randn(size=(2, 199, 257), dtype=torch.float32)
+    spec_erb = erb_bands.erb_scale(spec)
+    print(spec_erb.shape)
+
+    spec = erb_bands.erb_scale_inv(spec_erb)
+    print(spec.shape)
+
+    return
+
+
+if __name__ == "__main__":
+    main()

toolbox/torchaudio/modules/freq_bands/mel_bands.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == "__main__":
+    pass

toolbox/webrtcvad/vad.py

@@ -24,13 +24,15 @@ class WebRTCVad(object):
                  frame_duration_ms: int = 30,
                  padding_duration_ms: int = 300,
                  silence_duration_threshold: float = 0.3,
-                 sample_rate: int = 8000
+                 sample_rate: int = 8000,
+                 ring_buffer_activity_threshold: float = 0.9,
                  ):
         self.agg = agg
         self.frame_duration_ms = frame_duration_ms
         self.padding_duration_ms = padding_duration_ms
         self.silence_duration_threshold = silence_duration_threshold
         self.sample_rate = sample_rate
+        self.ring_buffer_activity_threshold = ring_buffer_activity_threshold
 
         self._vad = webrtcvad.Vad(mode=agg)
 
@@ -110,7 +112,7 @@ class WebRTCVad(object):
                 self.ring_buffer.append((frame, is_speech))
                 num_voiced = len([f for f, speech in self.ring_buffer if speech])
 
-                if num_voiced > 0.9 * self.ring_buffer.maxlen:
+                if num_voiced > self.ring_buffer_activity_threshold * self.ring_buffer.maxlen:
                     self.triggered = True
 
                     for f, _ in self.ring_buffer:
@@ -120,7 +122,7 @@ class WebRTCVad(object):
                 self.voiced_frames.append(frame)
                 self.ring_buffer.append((frame, is_speech))
                 num_unvoiced = len([f for f, speech in self.ring_buffer if not speech])
-                if num_unvoiced > 0.9 * self.ring_buffer.maxlen:
+                if num_unvoiced > self.ring_buffer_activity_threshold * self.ring_buffer.maxlen:
                     self.triggered = False
                     segment = [
                         np.concatenate([f.signal for f in self.voiced_frames]),
@@ -204,12 +206,12 @@ def get_args():
     )
     parser.add_argument(
         "--padding_duration_ms",
-        default=300,
+        default=30,
         type=int,
     )
     parser.add_argument(
         "--silence_duration_threshold",
-        default=0.3,
+        default=0.0,
         type=float,
         help="minimum silence duration, in seconds."
     )