cancel
Showing results for 
Search instead for 
Did you mean: 

Recording sound from mic during Oculus LipSync.

YanaArtis
Protege

I use Oculus LipSync in my application for Quest and Rift. Articulation works fine. I can record facial blendshapes with my code and then playback the recorded articulations via C# code later. Now I am trying to record the speech while Oculus asset performs lipsync. But I cannot get the sound from microphone because OVRLipSyncMicInput.cs starts capturing sound from microphone into temporary audioclip with 1-second duration, therefore my recording from microphone conflicts with Oculus code.

 

How can I record the sound during Oculus lipsync to play it later simultaneously with recorded lipsync?

 

I use Unity 2019.4.25f1, Oculus Integration 28.0.

1 ACCEPTED SOLUTION

Accepted Solutions

I have found the solution working on Rift and Quest:

 

 

// I have created my own class OculusLipSyncMicInput, copied into it source code from
// Oculus' OVRLipSyncMicInput.cs, and made several modifications:
public class OculusLipSyncMicInput : MonoBehaviour
{
...
    private int _micNdx = 0;
    private AudioClip _recordedClip;
    private AudioClip _croppedClip;
    private bool _isRecordingSpeech = false;
...
    void Start()
    {
      audioSource.loop = true;
      audioSource.mute = false;

      bool micSelected = false;
      _micNdx = 0;
      if (Microphone.devices.Length > 1)
      {
        for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++)
        {
          if (Microphone.devices[i].ToString().Contains("Rift"))
          {
            _micNdx = i;
            micSelected = true;
          }
        }

        for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++)
        {
          if (Microphone.devices[i].ToString().Contains("Oculus"))
          {
            _micNdx = i;
            micSelected = true;
          }
        }
      }
      InitializeMicrophone();
    }
...
    private void InitializeMicrophone()
    {
      if (initialized)
      {
        return;
      }
      if (Microphone.devices.Length == 0)
      {
        return;
      }

//      selectedDevice = Microphone.devices[0].ToString(); // Oculus' code
      selectedDevice = Microphone.devices[_micNdx].ToString(); // My code
      micSelected = true;
      GetMicCaps();
      initialized = true;
    }
...
//    public void StartMicrophone() // Oculus' code
    public void StartMicrophone(int soundLenSec = 1) // My code
    {
      if (micSelected == false) return;

      //Starts recording
//      audioSource.clip = Microphone.Start(selectedDevice, true, 1, micFrequency); // Oculus' code
      audioSource.clip = Microphone.Start(selectedDevice, true, soundLenSec, micFrequency); // My code

      Stopwatch timer = Stopwatch.StartNew();

      // Wait until the recording has started
      while (!(Microphone.GetPosition(selectedDevice) > 0) && timer.Elapsed.TotalMilliseconds < 1000)
      {
        Thread.Sleep(50);
      }

      if (Microphone.GetPosition(selectedDevice) <= 0)
      {
        throw new Exception("Timeout initializing microphone " + selectedDevice);
      }
      // Play the audio source
      audioSource.Play();
    }
...
// And I added several methods at the end
    public void StartMicrophoneRecord (int soundLenSec)
    {
      StopMicrophone();
      StartMicrophone(soundLenSec);
      _isRecordingSpeech = true;
    }

    public void EndMicrophoneRecord ()
    {
      int recordedSamples = Microphone.GetPosition(null);
      _recordedClip = (audioSource == null) ? null : audioSource.clip;
      StopMicrophone();

      if (_isRecordingSpeech)
      {
        if ((audioSource == null) || (audioSource.clip == null))
        {
          _recordedClip = null;
        }
        else
        {
          float[] croppedData = new float[recordedSamples * _recordedClip.channels];
          _recordedClip.GetData(croppedData, 0);
          if (_croppedClip != null)
          {
            _croppedClip.UnloadAudioData();
            Destroy(_croppedClip);
          }
          _croppedClip = AudioClip.Create(_recordedClip.name, recordedSamples, _recordedClip.channels, _recordedClip.frequency, false);
          _croppedClip.SetData(croppedData, 0);
        }
        _isRecordingSpeech = false;
      }

      StartMicrophone(1);
    }

    public AudioClip GetMicrophoneRecord ()
    {
      return _croppedClip;
    }

    public void Mute ()
    {
      micInputVolume = 0;
    }

    public void Unmute ()
    {
      micInputVolume = 100;
    }
}

 

View solution in original post

9 REPLIES 9

YanaArtis
Protege

I have found the solution. I created OculusLipSyncMicInput.cs - my own version of OVRLipSyncMicInput.cs that permits me to record the sound.

jynxiwin
Explorer

Hi, I am interested in what you ended up doing. Can you share?

// I have created my own class OculusLipSyncMicInput, copied into it source code from
// Oculus' OVRLipSyncMicInput.cs, and made several modifications:
public class OculusLipSyncMicInput : MonoBehaviour
{
...
    private int _micNdx = 0;
    private AudioClip _recordedClip;
    private AudioClip _croppedClip;
    private bool _isRecordingSpeech = false;
...
    void Start()
    {
      audioSource.loop = true;     // Set the AudioClip to loop
      audioSource.mute = false;

      // Automatically choose suitable microphone. Required for Oculus Rift.
      _micNdx = 0;
      if (Microphone.devices.Length > 1)
      {
        for (int i = 0; i < Microphone.devices.Length; i++)
        {
          if (Microphone.devices[i].ToString().Contains("Oculus"))
          {
            _micNdx = i;
            break;
          }
        }
      }
      InitializeMicrophone();
    }
...
    private void InitializeMicrophone()
    {
      if (initialized)
      {
        return;
      }
      if (Microphone.devices.Length == 0)
      {
        return;
      }

//      selectedDevice = Microphone.devices[0].ToString(); // Oculus' code
      selectedDevice = Microphone.devices[_micNdx].ToString(); // My code
      micSelected = true;
      GetMicCaps();
      initialized = true;
    }
...
//    public void StartMicrophone() // Oculus' code
    public void StartMicrophone(int soundLenSec = 1) // My code
    {
      if (micSelected == false) return;

      //Starts recording
//      audioSource.clip = Microphone.Start(selectedDevice, true, 1, micFrequency); // Oculus' code
      audioSource.clip = Microphone.Start(selectedDevice, true, soundLenSec, micFrequency); // My code

      Stopwatch timer = Stopwatch.StartNew();

      // Wait until the recording has started
      while (!(Microphone.GetPosition(selectedDevice) > 0) && timer.Elapsed.TotalMilliseconds < 1000)
      {
        Thread.Sleep(50);
      }

      if (Microphone.GetPosition(selectedDevice) <= 0)
      {
        throw new Exception("Timeout initializing microphone " + selectedDevice);
      }
      // Play the audio source
      audioSource.Play();
    }
...
// And I added several methods at the end
    public void StartMicrophoneRecord (int soundLenSec)
    {
      StopMicrophone();
      StartMicrophone(soundLenSec);
      _isRecordingSpeech = true;
    }

    public void EndMicrophoneRecord ()
    {
      int recordedSamples = Microphone.GetPosition(null);
      _recordedClip = (audioSource == null) ? null : audioSource.clip;
      StopMicrophone();

      if (_isRecordingSpeech)
      {
        if ((audioSource == null) || (audioSource.clip == null))
        {
          _recordedClip = null;
        }
        else
        {
          float[] croppedData = new float[recordedSamples * _recordedClip.channels];
          _recordedClip.GetData(croppedData, 0);
          if (_croppedClip != null)
          {
            _croppedClip.UnloadAudioData();
            Destroy(_croppedClip);
          }
          _croppedClip = AudioClip.Create(_recordedClip.name, recordedSamples, _recordedClip.channels, _recordedClip.frequency, false);
          _croppedClip.SetData(croppedData, 0);
        }
        _isRecordingSpeech = false;
      }

      StartMicrophone(1);
    }

    public AudioClip GetMicrophoneRecord ()
    {
      return _croppedClip;
    }

    public void Mute ()
    {
      micInputVolume = 0;
    }

    public void Unmute ()
    {
      micInputVolume = 100;
    }
}

jynxiwin
Explorer

Thank you. I am doing something similar and getting 10 seconds of silence in a .wav I'm saving. Are you creating a file with this voice data?

Yes, and I saved real voice. I even process it raising the pitch 🙂 There is the result of my voice capturing/processing:

https://www.youtube.com/watch?v=GBD0wwxVZtI

jynxiwin
Explorer

Is this on Quest2?

Yes. On Quest 1 and Quest 2 it works fine both as standalone Quest app, and as PC VR app. I just received message from my Rift tester. There are errors on the end of recording when running with Rift:

 

C:\buildslave\unity\build\Modules/Audio/Public/sound/SoundManager.cpp(729) : Error executing instance->m_Sound->unlock(ptr1, ptr2, len1, len2) (An invalid parameter was passed to this function. )
net.kiborgov.vrss.OculusLipSyncMicInput:EndMicrophoneRecord()

 

ArgumentException: Length of created clip must be larger than 0
at UnityEngine.AudioClip.Create (System.String name, System.Int32 lengthSamples, System.Int32 channels, System.Int32 frequency, System.Boolean stream, UnityEngine.AudioClip+PCMReaderCallback pcmreadercallback, UnityEngine.AudioClip+PCMSetPositionCallback pcmsetpositioncallback) [0x00000] in <00000000000000000000000000000000>:0
at UnityEngine.AudioClip.Create (System.String name, System.Int32 lengthSamples, System.Int32 channels, System.Int32 frequency, System.Boolean stream) [0x00000] in <00000000000000000000000000000000>:0
at net.kiborgov.vrss.OculusLipSyncMicInput.EndMicrophoneRecord () [0x00000] in <00000000000000000000000000000000>:0

I have found the solution working on Rift and Quest:

 

 

// I have created my own class OculusLipSyncMicInput, copied into it source code from
// Oculus' OVRLipSyncMicInput.cs, and made several modifications:
public class OculusLipSyncMicInput : MonoBehaviour
{
...
    private int _micNdx = 0;
    private AudioClip _recordedClip;
    private AudioClip _croppedClip;
    private bool _isRecordingSpeech = false;
...
    void Start()
    {
      audioSource.loop = true;
      audioSource.mute = false;

      bool micSelected = false;
      _micNdx = 0;
      if (Microphone.devices.Length > 1)
      {
        for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++)
        {
          if (Microphone.devices[i].ToString().Contains("Rift"))
          {
            _micNdx = i;
            micSelected = true;
          }
        }

        for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++)
        {
          if (Microphone.devices[i].ToString().Contains("Oculus"))
          {
            _micNdx = i;
            micSelected = true;
          }
        }
      }
      InitializeMicrophone();
    }
...
    private void InitializeMicrophone()
    {
      if (initialized)
      {
        return;
      }
      if (Microphone.devices.Length == 0)
      {
        return;
      }

//      selectedDevice = Microphone.devices[0].ToString(); // Oculus' code
      selectedDevice = Microphone.devices[_micNdx].ToString(); // My code
      micSelected = true;
      GetMicCaps();
      initialized = true;
    }
...
//    public void StartMicrophone() // Oculus' code
    public void StartMicrophone(int soundLenSec = 1) // My code
    {
      if (micSelected == false) return;

      //Starts recording
//      audioSource.clip = Microphone.Start(selectedDevice, true, 1, micFrequency); // Oculus' code
      audioSource.clip = Microphone.Start(selectedDevice, true, soundLenSec, micFrequency); // My code

      Stopwatch timer = Stopwatch.StartNew();

      // Wait until the recording has started
      while (!(Microphone.GetPosition(selectedDevice) > 0) && timer.Elapsed.TotalMilliseconds < 1000)
      {
        Thread.Sleep(50);
      }

      if (Microphone.GetPosition(selectedDevice) <= 0)
      {
        throw new Exception("Timeout initializing microphone " + selectedDevice);
      }
      // Play the audio source
      audioSource.Play();
    }
...
// And I added several methods at the end
    public void StartMicrophoneRecord (int soundLenSec)
    {
      StopMicrophone();
      StartMicrophone(soundLenSec);
      _isRecordingSpeech = true;
    }

    public void EndMicrophoneRecord ()
    {
      int recordedSamples = Microphone.GetPosition(null);
      _recordedClip = (audioSource == null) ? null : audioSource.clip;
      StopMicrophone();

      if (_isRecordingSpeech)
      {
        if ((audioSource == null) || (audioSource.clip == null))
        {
          _recordedClip = null;
        }
        else
        {
          float[] croppedData = new float[recordedSamples * _recordedClip.channels];
          _recordedClip.GetData(croppedData, 0);
          if (_croppedClip != null)
          {
            _croppedClip.UnloadAudioData();
            Destroy(_croppedClip);
          }
          _croppedClip = AudioClip.Create(_recordedClip.name, recordedSamples, _recordedClip.channels, _recordedClip.frequency, false);
          _croppedClip.SetData(croppedData, 0);
        }
        _isRecordingSpeech = false;
      }

      StartMicrophone(1);
    }

    public AudioClip GetMicrophoneRecord ()
    {
      return _croppedClip;
    }

    public void Mute ()
    {
      micInputVolume = 0;
    }

    public void Unmute ()
    {
      micInputVolume = 100;
    }
}

 

jynxiwin
Explorer

Yea, I worked out a similar solution. Thanks for the help....

Now to get TTS wotking - got RTVoice