05-03-2021 03:03 PM - edited 05-03-2021 03:29 PM
I use Oculus LipSync in my application for Quest and Rift. Articulation works fine. I can record facial blendshapes with my code and then playback the recorded articulations via C# code later. Now I am trying to record the speech while Oculus asset performs lipsync. But I cannot get the sound from microphone because OVRLipSyncMicInput.cs starts capturing sound from microphone into temporary audioclip with 1-second duration, therefore my recording from microphone conflicts with Oculus code.
How can I record the sound during Oculus lipsync to play it later simultaneously with recorded lipsync?
I use Unity 2019.4.25f1, Oculus Integration 28.0.
Solved! Go to Solution.
05-22-2021 10:15 PM - edited 05-23-2021 03:32 AM
I have found the solution working on Rift and Quest:
// I have created my own class OculusLipSyncMicInput, copied into it source code from
// Oculus' OVRLipSyncMicInput.cs, and made several modifications:
public class OculusLipSyncMicInput : MonoBehaviour
{
...
private int _micNdx = 0;
private AudioClip _recordedClip;
private AudioClip _croppedClip;
private bool _isRecordingSpeech = false;
...
void Start()
{
audioSource.loop = true;
audioSource.mute = false;
bool micSelected = false;
_micNdx = 0;
if (Microphone.devices.Length > 1)
{
for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++)
{
if (Microphone.devices[i].ToString().Contains("Rift"))
{
_micNdx = i;
micSelected = true;
}
}
for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++)
{
if (Microphone.devices[i].ToString().Contains("Oculus"))
{
_micNdx = i;
micSelected = true;
}
}
}
InitializeMicrophone();
}
...
private void InitializeMicrophone()
{
if (initialized)
{
return;
}
if (Microphone.devices.Length == 0)
{
return;
}
// selectedDevice = Microphone.devices[0].ToString(); // Oculus' code
selectedDevice = Microphone.devices[_micNdx].ToString(); // My code
micSelected = true;
GetMicCaps();
initialized = true;
}
...
// public void StartMicrophone() // Oculus' code
public void StartMicrophone(int soundLenSec = 1) // My code
{
if (micSelected == false) return;
//Starts recording
// audioSource.clip = Microphone.Start(selectedDevice, true, 1, micFrequency); // Oculus' code
audioSource.clip = Microphone.Start(selectedDevice, true, soundLenSec, micFrequency); // My code
Stopwatch timer = Stopwatch.StartNew();
// Wait until the recording has started
while (!(Microphone.GetPosition(selectedDevice) > 0) && timer.Elapsed.TotalMilliseconds < 1000)
{
Thread.Sleep(50);
}
if (Microphone.GetPosition(selectedDevice) <= 0)
{
throw new Exception("Timeout initializing microphone " + selectedDevice);
}
// Play the audio source
audioSource.Play();
}
...
// And I added several methods at the end
public void StartMicrophoneRecord (int soundLenSec)
{
StopMicrophone();
StartMicrophone(soundLenSec);
_isRecordingSpeech = true;
}
public void EndMicrophoneRecord ()
{
int recordedSamples = Microphone.GetPosition(null);
_recordedClip = (audioSource == null) ? null : audioSource.clip;
StopMicrophone();
if (_isRecordingSpeech)
{
if ((audioSource == null) || (audioSource.clip == null))
{
_recordedClip = null;
}
else
{
float[] croppedData = new float[recordedSamples * _recordedClip.channels];
_recordedClip.GetData(croppedData, 0);
if (_croppedClip != null)
{
_croppedClip.UnloadAudioData();
Destroy(_croppedClip);
}
_croppedClip = AudioClip.Create(_recordedClip.name, recordedSamples, _recordedClip.channels, _recordedClip.frequency, false);
_croppedClip.SetData(croppedData, 0);
}
_isRecordingSpeech = false;
}
StartMicrophone(1);
}
public AudioClip GetMicrophoneRecord ()
{
return _croppedClip;
}
public void Mute ()
{
micInputVolume = 0;
}
public void Unmute ()
{
micInputVolume = 100;
}
}
05-03-2021 05:58 PM
I have found the solution. I created OculusLipSyncMicInput.cs - my own version of OVRLipSyncMicInput.cs that permits me to record the sound.
05-11-2021 12:28 PM
Hi, I am interested in what you ended up doing. Can you share?
05-17-2021 07:13 AM
// I have created my own class OculusLipSyncMicInput, copied into it source code from
// Oculus' OVRLipSyncMicInput.cs, and made several modifications:
public class OculusLipSyncMicInput : MonoBehaviour
{
...
private int _micNdx = 0;
private AudioClip _recordedClip;
private AudioClip _croppedClip;
private bool _isRecordingSpeech = false;
...
void Start()
{
audioSource.loop = true; // Set the AudioClip to loop
audioSource.mute = false;
// Automatically choose suitable microphone. Required for Oculus Rift.
_micNdx = 0;
if (Microphone.devices.Length > 1)
{
for (int i = 0; i < Microphone.devices.Length; i++)
{
if (Microphone.devices[i].ToString().Contains("Oculus"))
{
_micNdx = i;
break;
}
}
}
InitializeMicrophone();
}
...
private void InitializeMicrophone()
{
if (initialized)
{
return;
}
if (Microphone.devices.Length == 0)
{
return;
}
// selectedDevice = Microphone.devices[0].ToString(); // Oculus' code
selectedDevice = Microphone.devices[_micNdx].ToString(); // My code
micSelected = true;
GetMicCaps();
initialized = true;
}
...
// public void StartMicrophone() // Oculus' code
public void StartMicrophone(int soundLenSec = 1) // My code
{
if (micSelected == false) return;
//Starts recording
// audioSource.clip = Microphone.Start(selectedDevice, true, 1, micFrequency); // Oculus' code
audioSource.clip = Microphone.Start(selectedDevice, true, soundLenSec, micFrequency); // My code
Stopwatch timer = Stopwatch.StartNew();
// Wait until the recording has started
while (!(Microphone.GetPosition(selectedDevice) > 0) && timer.Elapsed.TotalMilliseconds < 1000)
{
Thread.Sleep(50);
}
if (Microphone.GetPosition(selectedDevice) <= 0)
{
throw new Exception("Timeout initializing microphone " + selectedDevice);
}
// Play the audio source
audioSource.Play();
}
...
// And I added several methods at the end
public void StartMicrophoneRecord (int soundLenSec)
{
StopMicrophone();
StartMicrophone(soundLenSec);
_isRecordingSpeech = true;
}
public void EndMicrophoneRecord ()
{
int recordedSamples = Microphone.GetPosition(null);
_recordedClip = (audioSource == null) ? null : audioSource.clip;
StopMicrophone();
if (_isRecordingSpeech)
{
if ((audioSource == null) || (audioSource.clip == null))
{
_recordedClip = null;
}
else
{
float[] croppedData = new float[recordedSamples * _recordedClip.channels];
_recordedClip.GetData(croppedData, 0);
if (_croppedClip != null)
{
_croppedClip.UnloadAudioData();
Destroy(_croppedClip);
}
_croppedClip = AudioClip.Create(_recordedClip.name, recordedSamples, _recordedClip.channels, _recordedClip.frequency, false);
_croppedClip.SetData(croppedData, 0);
}
_isRecordingSpeech = false;
}
StartMicrophone(1);
}
public AudioClip GetMicrophoneRecord ()
{
return _croppedClip;
}
public void Mute ()
{
micInputVolume = 0;
}
public void Unmute ()
{
micInputVolume = 100;
}
}
05-18-2021 08:58 AM - edited 05-18-2021 08:59 AM
Thank you. I am doing something similar and getting 10 seconds of silence in a .wav I'm saving. Are you creating a file with this voice data?
05-19-2021 07:55 AM
Yes, and I saved real voice. I even process it raising the pitch 🙂 There is the result of my voice capturing/processing:
05-19-2021 09:35 AM
Is this on Quest2?
05-19-2021 09:54 AM
Yes. On Quest 1 and Quest 2 it works fine both as standalone Quest app, and as PC VR app. I just received message from my Rift tester. There are errors on the end of recording when running with Rift:
C:\buildslave\unity\build\Modules/Audio/Public/sound/SoundManager.cpp(729) : Error executing instance->m_Sound->unlock(ptr1, ptr2, len1, len2) (An invalid parameter was passed to this function. )
net.kiborgov.vrss.OculusLipSyncMicInput:EndMicrophoneRecord()
ArgumentException: Length of created clip must be larger than 0
at UnityEngine.AudioClip.Create (System.String name, System.Int32 lengthSamples, System.Int32 channels, System.Int32 frequency, System.Boolean stream, UnityEngine.AudioClip+PCMReaderCallback pcmreadercallback, UnityEngine.AudioClip+PCMSetPositionCallback pcmsetpositioncallback) [0x00000] in <00000000000000000000000000000000>:0
at UnityEngine.AudioClip.Create (System.String name, System.Int32 lengthSamples, System.Int32 channels, System.Int32 frequency, System.Boolean stream) [0x00000] in <00000000000000000000000000000000>:0
at net.kiborgov.vrss.OculusLipSyncMicInput.EndMicrophoneRecord () [0x00000] in <00000000000000000000000000000000>:0
05-22-2021 10:15 PM - edited 05-23-2021 03:32 AM
I have found the solution working on Rift and Quest:
// I have created my own class OculusLipSyncMicInput, copied into it source code from
// Oculus' OVRLipSyncMicInput.cs, and made several modifications:
public class OculusLipSyncMicInput : MonoBehaviour
{
...
private int _micNdx = 0;
private AudioClip _recordedClip;
private AudioClip _croppedClip;
private bool _isRecordingSpeech = false;
...
void Start()
{
audioSource.loop = true;
audioSource.mute = false;
bool micSelected = false;
_micNdx = 0;
if (Microphone.devices.Length > 1)
{
for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++)
{
if (Microphone.devices[i].ToString().Contains("Rift"))
{
_micNdx = i;
micSelected = true;
}
}
for (int i = 0; !micSelected && (i < Microphone.devices.Length); i++)
{
if (Microphone.devices[i].ToString().Contains("Oculus"))
{
_micNdx = i;
micSelected = true;
}
}
}
InitializeMicrophone();
}
...
private void InitializeMicrophone()
{
if (initialized)
{
return;
}
if (Microphone.devices.Length == 0)
{
return;
}
// selectedDevice = Microphone.devices[0].ToString(); // Oculus' code
selectedDevice = Microphone.devices[_micNdx].ToString(); // My code
micSelected = true;
GetMicCaps();
initialized = true;
}
...
// public void StartMicrophone() // Oculus' code
public void StartMicrophone(int soundLenSec = 1) // My code
{
if (micSelected == false) return;
//Starts recording
// audioSource.clip = Microphone.Start(selectedDevice, true, 1, micFrequency); // Oculus' code
audioSource.clip = Microphone.Start(selectedDevice, true, soundLenSec, micFrequency); // My code
Stopwatch timer = Stopwatch.StartNew();
// Wait until the recording has started
while (!(Microphone.GetPosition(selectedDevice) > 0) && timer.Elapsed.TotalMilliseconds < 1000)
{
Thread.Sleep(50);
}
if (Microphone.GetPosition(selectedDevice) <= 0)
{
throw new Exception("Timeout initializing microphone " + selectedDevice);
}
// Play the audio source
audioSource.Play();
}
...
// And I added several methods at the end
public void StartMicrophoneRecord (int soundLenSec)
{
StopMicrophone();
StartMicrophone(soundLenSec);
_isRecordingSpeech = true;
}
public void EndMicrophoneRecord ()
{
int recordedSamples = Microphone.GetPosition(null);
_recordedClip = (audioSource == null) ? null : audioSource.clip;
StopMicrophone();
if (_isRecordingSpeech)
{
if ((audioSource == null) || (audioSource.clip == null))
{
_recordedClip = null;
}
else
{
float[] croppedData = new float[recordedSamples * _recordedClip.channels];
_recordedClip.GetData(croppedData, 0);
if (_croppedClip != null)
{
_croppedClip.UnloadAudioData();
Destroy(_croppedClip);
}
_croppedClip = AudioClip.Create(_recordedClip.name, recordedSamples, _recordedClip.channels, _recordedClip.frequency, false);
_croppedClip.SetData(croppedData, 0);
}
_isRecordingSpeech = false;
}
StartMicrophone(1);
}
public AudioClip GetMicrophoneRecord ()
{
return _croppedClip;
}
public void Mute ()
{
micInputVolume = 0;
}
public void Unmute ()
{
micInputVolume = 100;
}
}
05-23-2021 05:16 AM
Yea, I worked out a similar solution. Thanks for the help....
Now to get TTS wotking - got RTVoice