Fix Docker failing to start due to missing PortAudio, and adjust the code to stabilize latency to prevent audio instability when using a VAD

Author: PhamHuynhAnh16

Dockerfile

@@ -10,6 +10,7 @@ WORKDIR /app
 # Install system dependencies, clean up cache to keep image size small
 RUN apt update && \
     apt install -y -qq ffmpeg && \
+    apt install -y -qq libportaudio2 && \
     apt clean && rm -rf /var/lib/apt/lists/*
 
 # Copy application files into the container

rvc/realtime/core.py

@@ -86,7 +86,7 @@ def __init__(
         # noise reduce
         self.reduced_noise = (
             TorchGate(
-                AUDIO_SAMPLE_RATE,
+                self.pipeline.tgt_sr,
                 prop_decrease=clean_strength,
             ).to(self.device)
             if clean_audio
@@ -244,7 +244,7 @@ def inference(
                 # Busy wait to keep power manager happy and clocks stable. Running pipeline on-demand seems to lag when the delay between
                 # voice changer activation is too high.
                 # https://forums.developer.nvidia.com/t/why-kernel-calculate-speed-got-slower-after-waiting-for-a-while/221059/9
-                self.pipeline.voice_conversion(
+                audio_model = self.pipeline.voice_conversion(
                     self.convert_buffer,
                     self.pitch_buffer,
                     self.pitchf_buffer,
@@ -260,14 +260,14 @@ def inference(
                     f0_autotune_strength,
                     proposed_pitch,
                     proposed_pitch_threshold,
+                    self.reduced_noise,
+                    self.board,
                 )
-                return None, vol
+
+                return torch.zeros(audio_model.shape, dtype=self.dtype, device=self.device), vol
 
         if vol < self.input_sensitivity:
-            # Busy wait to keep power manager happy and clocks stable. Running pipeline on-demand seems to lag when the delay between
-            # voice changer activation is too high.
-            # https://forums.developer.nvidia.com/t/why-kernel-calculate-speed-got-slower-after-waiting-for-a-while/221059/9
-            self.pipeline.voice_conversion(
+            audio_model = self.pipeline.voice_conversion(
                 self.convert_buffer,
                 self.pitch_buffer,
                 self.pitchf_buffer,
@@ -283,9 +283,11 @@ def inference(
                 f0_autotune_strength,
                 proposed_pitch,
                 proposed_pitch_threshold,
+                self.reduced_noise,
+                self.board,
             )
 
-            return None, vol
+            return torch.zeros(audio_model.shape, dtype=self.dtype, device=self.device), vol
 
         circular_write(audio_input_16k, self.convert_buffer)
 
@@ -305,18 +307,11 @@ def inference(
             f0_autotune_strength,
             proposed_pitch,
             proposed_pitch_threshold,
+            self.reduced_noise,
+            self.board,
         )
 
         audio_out: torch.Tensor = self.resample_out(audio_model * torch.sqrt(vol_t))
-
-        if self.reduced_noise is not None:
-            audio_out = self.reduced_noise(audio_out.unsqueeze(0)).squeeze(0)
-        if self.board is not None:
-            audio_out = torch.as_tensor(
-                self.board(audio_out.cpu().numpy(), AUDIO_SAMPLE_RATE),
-                device=self.device,
-            )
-
         return audio_out, vol
 
     def __del__(self):
@@ -424,9 +419,9 @@ def process_audio(
             proposed_pitch_threshold,
         )
 
-        if audio is None:
+        # if audio is None:
             # In case there's an actual silence - send full block with zeros
-            return np.zeros(block_size, dtype=np.float32), vol
+            # return np.zeros(block_size, dtype=np.float32), vol
 
         conv_input = audio[None, None, : self.crossfade_frame + self.sola_search_frame]
         cor_nom = F.conv1d(conv_input, self.sola_buffer[None, None, :])

rvc/realtime/pipeline.py

@@ -228,6 +228,8 @@ def voice_conversion(
         f0_autotune_strength: float = 1,
         proposed_pitch: bool = False,
         proposed_pitch_threshold: float = 155.0,
+        reduced_noise = None,
+        board = None,
     ):
         """
         Performs realtime voice conversion on a given audio segment.
@@ -321,6 +323,14 @@ def voice_conversion(
                     out_audio[: return_length * scaled_window]
                 )
 
+            if reduced_noise is not None:
+                out_audio = reduced_noise(out_audio.unsqueeze(0)).squeeze(0)
+            if board is not None:
+                out_audio = torch.as_tensor(
+                    board(out_audio.cpu().numpy(), self.tgt_sr),
+                    device=self.device,
+                )
+
         return out_audio
 
     def _retrieve_speaker_embeddings(