fixed rmvpe infer pipelines

vc_infer_pipeline.py

@@ -1,8 +1,6 @@
 import numpy as np, parselmouth, torch, pdb, sys, os
 from time import time as ttime
 import torch.nn.functional as F
-import torchcrepe  # Fork feature. Use the crepe f0 algorithm. New dependency (pip install torchcrepe)
-from torch import Tensor
 import scipy.signal as signal
 import pyworld, os, traceback, faiss, librosa, torchcrepe
 from scipy import signal
@@ -71,186 +69,6 @@ class VC(object):
         self.t_max = self.sr * self.x_max  # 免查询时长阈值
         self.device = config.device
 
-    # Fork Feature: Get the best torch device to use for f0 algorithms that require a torch device. Will return the type (torch.device)
-    def get_optimal_torch_device(self, index: int = 0) -> torch.device:
-        # Get cuda device
-        if torch.cuda.is_available():
-            return torch.device(
-                f"cuda:{index % torch.cuda.device_count()}"
-            )  # Very fast
-        elif torch.backends.mps.is_available():
-            return torch.device("mps")
-        # Insert an else here to grab "xla" devices if available. TO DO later. Requires the torch_xla.core.xla_model library
-        # Else wise return the "cpu" as a torch device,
-        return torch.device("cpu")
-
-    # Fork Feature: Compute f0 with the crepe method
-    def get_f0_crepe_computation(
-        self,
-        x,
-        f0_min,
-        f0_max,
-        p_len,
-        hop_length=160,  # 512 before. Hop length changes the speed that the voice jumps to a different dramatic pitch. Lower hop lengths means more pitch accuracy but longer inference time.
-        model="full",  # Either use crepe-tiny "tiny" or crepe "full". Default is full
-    ):
-        x = x.astype(
-            np.float32
-        )  # fixes the F.conv2D exception. We needed to convert double to float.
-        x /= np.quantile(np.abs(x), 0.999)
-        torch_device = self.get_optimal_torch_device()
-        audio = torch.from_numpy(x).to(torch_device, copy=True)
-        audio = torch.unsqueeze(audio, dim=0)
-        if audio.ndim == 2 and audio.shape[0] > 1:
-            audio = torch.mean(audio, dim=0, keepdim=True).detach()
-        audio = audio.detach()
-        print("Initiating prediction with a crepe_hop_length of: " + str(hop_length))
-        pitch: Tensor = torchcrepe.predict(
-            audio,
-            self.sr,
-            hop_length,
-            f0_min,
-            f0_max,
-            model,
-            batch_size=hop_length * 2,
-            device=torch_device,
-            pad=True,
-        )
-        p_len = p_len or x.shape[0] // hop_length
-        # Resize the pitch for final f0
-        source = np.array(pitch.squeeze(0).cpu().float().numpy())
-        source[source < 0.001] = np.nan
-        target = np.interp(
-            np.arange(0, len(source) * p_len, len(source)) / p_len,
-            np.arange(0, len(source)),
-            source,
-        )
-        f0 = np.nan_to_num(target)
-        return f0  # Resized f0
-
-    def get_f0_official_crepe_computation(
-        self,
-        x,
-        f0_min,
-        f0_max,
-        model="full",
-    ):
-        # Pick a batch size that doesn't cause memory errors on your gpu
-        batch_size = 512
-        # Compute pitch using first gpu
-        audio = torch.tensor(np.copy(x))[None].float()
-        f0, pd = torchcrepe.predict(
-            audio,
-            self.sr,
-            self.window,
-            f0_min,
-            f0_max,
-            model,
-            batch_size=batch_size,
-            device=self.device,
-            return_periodicity=True,
-        )
-        pd = torchcrepe.filter.median(pd, 3)
-        f0 = torchcrepe.filter.mean(f0, 3)
-        f0[pd < 0.1] = 0
-        f0 = f0[0].cpu().numpy()
-        return f0
-
-    # Fork Feature: Compute pYIN f0 method
-    def get_f0_pyin_computation(self, x, f0_min, f0_max):
-        y, sr = librosa.load("saudio/Sidney.wav", self.sr, mono=True)
-        f0, _, _ = librosa.pyin(y, sr=self.sr, fmin=f0_min, fmax=f0_max)
-        f0 = f0[1:]  # Get rid of extra first frame
-        return f0
-
-    # Fork Feature: Acquire median hybrid f0 estimation calculation
-    def get_f0_hybrid_computation(
-        self,
-        methods_str,
-        input_audio_path,
-        x,
-        f0_min,
-        f0_max,
-        p_len,
-        filter_radius,
-        crepe_hop_length,
-        time_step,
-    ):
-        # Get various f0 methods from input to use in the computation stack
-        s = methods_str
-        s = s.split("hybrid")[1]
-        s = s.replace("[", "").replace("]", "")
-        methods = s.split("+")
-        f0_computation_stack = []
-
-        print("Calculating f0 pitch estimations for methods: %s" % str(methods))
-        x = x.astype(np.float32)
-        x /= np.quantile(np.abs(x), 0.999)
-        # Get f0 calculations for all methods specified
-        for method in methods:
-            f0 = None
-            if method == "pm":
-                f0 = (
-                    parselmouth.Sound(x, self.sr)
-                    .to_pitch_ac(
-                        time_step=time_step / 1000,
-                        voicing_threshold=0.6,
-                        pitch_floor=f0_min,
-                        pitch_ceiling=f0_max,
-                    )
-                    .selected_array["frequency"]
-                )
-                pad_size = (p_len - len(f0) + 1) // 2
-                if pad_size > 0 or p_len - len(f0) - pad_size > 0:
-                    f0 = np.pad(
-                        f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant"
-                    )
-            elif method == "crepe":
-                f0 = self.get_f0_official_crepe_computation(x, f0_min, f0_max)
-                f0 = f0[1:]  # Get rid of extra first frame
-            elif method == "crepe-tiny":
-                f0 = self.get_f0_official_crepe_computation(x, f0_min, f0_max, "tiny")
-                f0 = f0[1:]  # Get rid of extra first frame
-            elif method == "mangio-crepe":
-                f0 = self.get_f0_crepe_computation(
-                    x, f0_min, f0_max, p_len, crepe_hop_length
-                )
-            elif method == "mangio-crepe-tiny":
-                f0 = self.get_f0_crepe_computation(
-                    x, f0_min, f0_max, p_len, crepe_hop_length, "tiny"
-                )
-            elif method == "harvest":
-                f0 = cache_harvest_f0(input_audio_path, self.sr, f0_max, f0_min, 10)
-                if filter_radius > 2:
-                    f0 = signal.medfilt(f0, 3)
-                f0 = f0[1:]  # Get rid of first frame.
-            elif method == "dio":  # Potentially buggy?
-                f0, t = pyworld.dio(
-                    x.astype(np.double),
-                    fs=self.sr,
-                    f0_ceil=f0_max,
-                    f0_floor=f0_min,
-                    frame_period=10,
-                )
-                f0 = pyworld.stonemask(x.astype(np.double), f0, t, self.sr)
-                f0 = signal.medfilt(f0, 3)
-                f0 = f0[1:]
-            # elif method == "pyin": Not Working just yet
-            #    f0 = self.get_f0_pyin_computation(x, f0_min, f0_max)
-            # Push method to the stack
-            f0_computation_stack.append(f0)
-
-        for fc in f0_computation_stack:
-            print(len(fc))
-
-        print("Calculating hybrid median f0 from the stack of: %s" % str(methods))
-        f0_median_hybrid = None
-        if len(f0_computation_stack) == 1:
-            f0_median_hybrid = f0_computation_stack[0]
-        else:
-            f0_median_hybrid = np.nanmedian(f0_computation_stack, axis=0)
-        return f0_median_hybrid
-
     def get_f0(
         self,
         input_audio_path,
@@ -259,7 +77,6 @@ class VC(object):
         f0_up_key,
         f0_method,
         filter_radius,
-        crepe_hop_length,
         inp_f0=None,
     ):
         global input_audio_path2wav
@@ -289,28 +106,27 @@ class VC(object):
             f0 = cache_harvest_f0(input_audio_path, self.sr, f0_max, f0_min, 10)
             if filter_radius > 2:
                 f0 = signal.medfilt(f0, 3)
-        elif f0_method == "dio":  # Potentially Buggy?
-            f0, t = pyworld.dio(
-                x.astype(np.double),
-                fs=self.sr,
-                f0_ceil=f0_max,
-                f0_floor=f0_min,
-                frame_period=10,
-            )
-            f0 = pyworld.stonemask(x.astype(np.double), f0, t, self.sr)
-            f0 = signal.medfilt(f0, 3)
         elif f0_method == "crepe":
-            f0 = self.get_f0_official_crepe_computation(x, f0_min, f0_max)
-        elif f0_method == "crepe-tiny":
-            f0 = self.get_f0_official_crepe_computation(x, f0_min, f0_max, "tiny")
-        elif f0_method == "mangio-crepe":
-            f0 = self.get_f0_crepe_computation(
-                x, f0_min, f0_max, p_len, crepe_hop_length
-            )
-        elif f0_method == "mangio-crepe-tiny":
-            f0 = self.get_f0_crepe_computation(
-                x, f0_min, f0_max, p_len, crepe_hop_length, "tiny"
+            model = "full"
+            # Pick a batch size that doesn't cause memory errors on your gpu
+            batch_size = 512
+            # Compute pitch using first gpu
+            audio = torch.tensor(np.copy(x))[None].float()
+            f0, pd = torchcrepe.predict(
+                audio,
+                self.sr,
+                self.window,
+                f0_min,
+                f0_max,
+                model,
+                batch_size=batch_size,
+                device=self.device,
+                return_periodicity=True,
             )
+            pd = torchcrepe.filter.median(pd, 3)
+            f0 = torchcrepe.filter.mean(f0, 3)
+            f0[pd < 0.1] = 0
+            f0 = f0[0].cpu().numpy()
         elif f0_method == "rmvpe":
             if hasattr(self, "model_rmvpe") == False:
                 from rmvpe import RMVPE
@@ -320,22 +136,6 @@ class VC(object):
                     "rmvpe.pt", is_half=self.is_half, device=self.device
                 )
             f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
-
-        elif "hybrid" in f0_method:
-            # Perform hybrid median pitch estimation
-            input_audio_path2wav[input_audio_path] = x.astype(np.double)
-            f0 = self.get_f0_hybrid_computation(
-                f0_method,
-                input_audio_path,
-                x,
-                f0_min,
-                f0_max,
-                p_len,
-                filter_radius,
-                crepe_hop_length,
-                time_step,
-            )
-
         f0 *= pow(2, f0_up_key / 12)
         # with open("test.txt","w")as f:f.write("\n".join([str(i)for i in f0.tolist()]))
         tf0 = self.sr // self.window  # 每秒f0点数
@@ -359,7 +159,6 @@ class VC(object):
         f0_mel[f0_mel <= 1] = 1
         f0_mel[f0_mel > 255] = 255
         f0_coarse = np.rint(f0_mel).astype(np.int)
-
         return f0_coarse, f0bak  # 1-0
 
     def vc(
@@ -484,7 +283,6 @@ class VC(object):
         rms_mix_rate,
         version,
         protect,
-        crepe_hop_length,
         f0_file=None,
     ):
         if (
@@ -546,7 +344,6 @@ class VC(object):
                 f0_up_key,
                 f0_method,
                 filter_radius,
-                crepe_hop_length,
                 inp_f0,
             )
             pitch = pitch[:p_len]
@@ -643,4 +440,4 @@ class VC(object):
         del pitch, pitchf, sid
         if torch.cuda.is_available():
             torch.cuda.empty_cache()
-        return audio_opt
+        return audio_opt