Record and playback avatar animations — Oculus
Welcome to the Oculus Developer Forums!

Your participation on the forum is subject to the Oculus Code of Conduct.

In general, please be respectful and kind. If you violate the Oculus Code of Conduct, your access to the developer forums may be revoked at the discretion of Oculus staff.

Record and playback avatar animations

Hello, 

so since I finally got some time today and since the topic is pretty straightforward, I hereby post my first take on recording and playing back (canned) avatar animations in Unity. 

The script is super simple, and should be added to the RemoteLoopback Unity Sample scene. It writes out a file with animation data (default is "C:\temp\Recording.avs", see the script paramters). A boolean variable called "Recording" switches between recording (=True) and playback (=False). 

I want to flesh this out  in order to use Avatars as trainers and holograms.

The next stop is to be able to record voice data from the microphone to a WAV file (code is available on the net), but ideally with added cue-points in the WAV in order to sync up the animation frames with the sound. 

Also changing the default avatar would be nice, without having to provide an avatar ID of a recorded profile. Also, getting everything nicer and smoother, and so on and so forth, eventually I'd also like to add LZ4 compression to make the animation frames smaller. 

For those who don't want to record anything, I provided a sample demo file which can be downloaded at 

http://old.multimedial.de/share/Recording.avs (580Kb)

Download it, then either place it in the default directory (C:\temp\) or point the property called "F Name" (for fileName, I should rename that...) to the new place of the file. 

Enjoy, feedback is much appreciated. 

using UnityEngine;using System.Collections;using System.Collections.Generic;using System;using System.IO;

public class RemoteLoopbackManager : MonoBehaviour {
    public OvrAvatar LocalAvatar;    public OvrAvatar LoopbackAvatar;    public String fName = "C:\\temp\\Recording.avs";    public Boolean recording = true;    int offset = 0;    float startTime;    FileStream file;    Dictionary<string, object> animationFrame;        int packetSequence = 0;
// Use this for initialization void Start () {
        LocalAvatar.RecordPackets = recording;                        if (recording)        {   // recording mode            LocalAvatar.PacketRecorded += OnLocalAvatarPacketRecorded;            File.Delete(fName);            file = new FileStream(fName, FileMode.CreateNew);        }        else        {   // playback mode            file = new FileStream(fName, FileMode.Open);        }
        initPlayback();
    }        void Update()    {        if (recording) return; // bail out, nothing to record
        //////////////////////////////////////////////////////        // if we are still here, then we are playing back!        //////////////////////////////////////////////////////        if (animationFrame == null)            animationFrame = ReadAnimationFrame();
        var currentTime = Time.time - startTime;        var animationTime = (double)animationFrame["time"];
        if (animationTime > currentTime)        {            Debug.Log("Not playing yet " + animationTime + " " + currentTime);            return;        }

        PlaybackPacketData((byte[])animationFrame["data"]);        // reset frame        animationFrame = null;
    }

    Dictionary<string, object> ReadAnimationFrame()    {        /////////////////////////////////////////////////////////////////////////////////////////////        // from hereon, we play back...        /////////////////////////////////////////////////////////////////////////////////////////////        // read frameSize        /////////////////////////////////////////////////////////////////////////////////////////////        var frameSize = new byte[sizeof(int)];        offset += file.Read(frameSize, 0, frameSize.Length);
        /////////////////////////////////////////////////////////////////////////////////////////////        // read frameTime        /////////////////////////////////////////////////////////////////////////////////////////////        var frameTime = new byte[sizeof(double)];        offset += file.Read(frameTime, 0, frameTime.Length);
        var animationTime = BitConverter.ToDouble(frameTime, 0);
        var frameContent = new byte[BitConverter.ToInt32(frameSize, 0)];
        // read in frameSize bytes of data        offset += file.Read(frameContent, 0, frameContent.Length);                // wrap around            if (offset >= file.Length)        {            initPlayback();            return null;        }                var animationFrame = new Dictionary<string, object>();        animationFrame.Add("time", animationTime);        animationFrame.Add("data", frameContent);
        return animationFrame;
    }
    void initPlayback()    {        // reset filepointer to the beginning of file        file.Position = offset = 0;        // reset time        startTime = Time.time;        // reset animationFrame        animationFrame = null;    }
    void OnLocalAvatarPacketRecorded(object sender, OvrAvatar.PacketEventArgs args)    {        using (MemoryStream outputStream = new MemoryStream())        {
            BinaryWriter writer = new BinaryWriter(outputStream);                        writer.Write(packetSequence);            args.Packet.Write(outputStream);                       var dataBytes = outputStream.ToArray();
            ////////////////////////////////////////////////////////////            // write frameSize            ////////////////////////////////////////////////////////////            var frameSize = BitConverter.GetBytes(dataBytes.Length);            file.Write(frameSize, 0, frameSize.Length);// write size of frame
            ////////////////////////////////////////////////////////////            // write currentTime            ////////////////////////////////////////////////////////////            var diffTime = (double)(Time.time - startTime);            var currentTime = BitConverter.GetBytes(diffTime);            file.Write(currentTime, 0, currentTime.Length); // write relative time
            Debug.Log(diffTime);
            ////////////////////////////////////////////////////////////            // write framedata            ////////////////////////////////////////////////////////////            file.Write(dataBytes, 0, dataBytes.Length);// write data
            file.Flush();                        SendPacketData(outputStream.ToArray());
        }    }
    void SendPacketData(byte[] data)    {        // Loopback by just "receiving" the data        ReceivePacketData(data);    }
    void ReceivePacketData(byte[] data)    {        using (MemoryStream inputStream = new MemoryStream(data))        {            BinaryReader reader = new BinaryReader(inputStream);            int sequence = reader.ReadInt32();            OvrAvatarPacket packet = OvrAvatarPacket.Read(inputStream);            LoopbackAvatar.GetComponent<OvrAvatarRemoteDriver>().QueuePacket(sequence, packet);        }    }
    void PlaybackPacketData(byte[] data)    {        using (MemoryStream inputStream = new MemoryStream(data))        {            BinaryReader reader = new BinaryReader(inputStream);            LoopbackAvatar.GetComponent<OvrAvatarRemoteDriver>().QueuePacket(reader.ReadInt32(), OvrAvatarPacket.Read(inputStream));        }    }}

Comments

  • multimedialmultimedial Posts: 32
    Brain Burst
    Still looking forward to your feedback... I am looking forward to lip and face animations for the avatar sdk. Any news when this will be coming? I am looking into recording and playback sound. 

    Also, I would like to be able to show other avatars other than the default one if possible. Right now, one can only show preconfigured avatars....
  • innerspaceappinnerspaceapp Posts: 4
    NerveGear
    Still looking forward to your feedback... I am looking forward to lip and face animations for the avatar sdk. Any news when this will be coming? I am looking into recording and playback sound. 

    Also, I would like to be able to show other avatars other than the default one if possible. Right now, one can only show preconfigured avatars....
    Hi Multimedial,

    thanks for your contribution. Tried it out and worked perfectly. Had to do some modifications due to the new 1.14 Version of the SDK. I guess there is already Facial-Animation built in, because the OVRAvatar has a Method called UpdateVoiceVisualization where you should able to pass audio samples in it. I just did not get it to work. Do you have any idea how to do this as there is no documentation around this.
  • multimedialmultimedial Posts: 32
    Brain Burst
    Hi innerspaceapp, 

    would you care to elaborate what changes you did? I haven't looked into the new SDK yet, this is just a side project for me. We would like to turn it into an asset for the Unity Asset Store if possible. 

    I am mostly interested in getting other avatars to work other than the standard one. We also got a lot of different ideas on how to flesh this out, like encrypting the performance and multiplexing the whole data. 

    As far as I can tell, there also seems to be audio support in the works?

    I'd love to chat with someone at Oculus about the opportunity that this offers - my feeling is that there are only a few modifications necessary to turn this into a valuable training tool in VR.
  • innerspaceappinnerspaceapp Posts: 4
    NerveGear
    Hi @multimedial,

    good to read, that you're looking for similar things to do with the Avatar SDK (VR Training Tool). I tried to have different Avatars just by having different Oculus Accounts, where you can user their Avatar Designer. You can then apply the design of those by identifying the Oculus ID of the User and then get the avatar specs from the web. I guess it is also possible to write those Avatar Specs in a file and load it from there, which is great, when you don't have web access.

    For the changes for the newer version of the SDK just have a look in their RemoteLoopbackSample. There the Methods "ReceivePacketData"  and "OnLocalAvatarPacketRecorded" changed.

    If you have news about the facial Animation, let me know.
  • KevinDerksen1992KevinDerksen1992 Posts: 1
    NerveGear
    Hi @innerspaceapp,

    I wanted to thank you for this amazing contribution, it is exactly what I Am looking for. I am working on games, and I need a tool to explain some mechanics simply to the user, by recording the required movements I am saving a lot of time.

    Thanks! 

  • owenwpowenwp Posts: 681 Oculus Start Member
    I was planning to implement something just like this since my app will also allow you to play with people who do not have VR, plus local AI, and I wanted to do an acted out tutorial sequence as well.  Having a recorded avatar agent that can physically point things out to you seems like a great feature.
  • multimedialmultimedial Posts: 32
    Brain Burst
    This is an on and off side project for me. We have audio sound recording in the works, with embedded cue points that allow for triggering events during playback. As for facial animations, I haven't looked into it yet.
  • hazelshihazelshi Posts: 2
    NerveGear
    edited August 2017
    Hi!
    Thanks for your contribution, I'm currently doing the pretty same thing.

    to meet the new SDK, I changed my playbackPacketData into:

    void PlaybackPacketData(byte[] data)
    {
    using (MemoryStream inputStream = new MemoryStream(data))
    {
    Debug.Log("Playing");
    BinaryReader reader = new BinaryReader(inputStream);
    int sequence = reader.ReadInt32();
    int size = reader.ReadInt32();
    byte[] sdkData = reader.ReadBytes(size);
    IntPtr packet = CAPI.ovrAvatarPacket_Read((UInt32)data.Length, sdkData);
    LoopbackAvatar.GetComponent<OvrAvatarRemoteDriver>().QueuePacket(sequence, new OvrAvatarPacket { ovrNativePacket = packet });
    }
    }

    But I don'k know for what reason, the avatar won't move, could you help?

  • GabrielVAGabrielVA Posts: 1
    NerveGear
    Working code as of Unity 2017.1.1f1 + SDK 3.4.1 (I think, at least it's what's written on LICENSE.txt). This is not optimized or organized at all. It's the bare minimum I wrote to make it work.

    Made on top of RemoteLoopbackManager and inspired by OP.

    To record: start the scene with the "record" box toggled and untoggle it before stopping the scene
    To playback: start the scene with the "playback" box toggled or toggle it after starting

    using UnityEngine;
    using System.Collections;
    using System;
    using System.IO;
    using Oculus.Avatar;
    using System.Runtime.InteropServices;
    using System.Collections.Generic;
    
    public class Playback : MonoBehaviour
    {
        [Serializable]
        class PacketsFile
        {
            public LinkedList<Packet> packetList;
        };
    
        [Serializable]
        class Packet
        {
            public byte[] PacketData;
        };
    
        public OvrAvatar LocalAvatar;
        public OvrAvatar LoopbackAvatar;
    
        private int PacketSequence = 0;
    
        LinkedList<Packet> packetQueue = new LinkedList<Packet>();
    
        public bool record = false;
        bool _lastRecord = false;
        public bool playback = false;
        public string fileName = "C:\\temp\\Recording.avs";
        LinkedList<Packet> _recordedQueue = new LinkedList<Packet>();
    
    
        void Start()
        {
            LocalAvatar.RecordPackets = true;
            LocalAvatar.PacketRecorded += OnLocalAvatarPacketRecorded;
            _lastRecord = record;
            if (playback)
            {
                ReadFile();
            }
        }
    
        void OnLocalAvatarPacketRecorded(object sender, OvrAvatar.PacketEventArgs args)
        {
            if (playback)
            {
                LinkedListNode<Packet> packet = _recordedQueue.First;
                if (packet == null)
                {
                    ReadFile();
                    packet = _recordedQueue.First;
                }
                SendPacketData(packet.Value.PacketData);
                _recordedQueue.RemoveFirst();
                
            }
    
            else using (MemoryStream outputStream = new MemoryStream())
            {
                BinaryWriter writer = new BinaryWriter(outputStream);
    
                var size = CAPI.ovrAvatarPacket_GetSize(args.Packet.ovrNativePacket);
                byte[] data = new byte[size];
                CAPI.ovrAvatarPacket_Write(args.Packet.ovrNativePacket, size, data);
    
                writer.Write(PacketSequence++);
                writer.Write(size);
                writer.Write(data);
    
                SendPacketData(outputStream.ToArray());
            }
        }
    
        void Update()
        {
            if (!record && _lastRecord)
            {
                WriteToFile();
                _lastRecord = record;
            }
    
            if (packetQueue.Count > 0)
            {
                List<Packet> deadList = new List<Packet>();
                foreach (Packet packet in packetQueue)
                {
                    ReceivePacketData(packet.PacketData);
                    deadList.Add(packet);
                }
    
                foreach (var packet in deadList)
                {
                    packetQueue.Remove(packet);
                }
            }
        }
    
        void SendPacketData(byte[] data)
        {
            Packet packet = new Packet();
            packet.PacketData = data;
    
            packetQueue.AddLast(packet);
            if (record && !playback) _recordedQueue.AddLast(packet);
        }
    
        void ReceivePacketData(byte[] data)
        {
            using (MemoryStream inputStream = new MemoryStream(data))
            {
                BinaryReader reader = new BinaryReader(inputStream);
                int sequence = reader.ReadInt32();
    
                int size = reader.ReadInt32();
                byte[] sdkData = reader.ReadBytes(size);
    
                IntPtr packet = CAPI.ovrAvatarPacket_Read((UInt32)data.Length, sdkData);
                LoopbackAvatar.GetComponent<OvrAvatarRemoteDriver>().QueuePacket(sequence, new OvrAvatarPacket { ovrNativePacket = packet });
            }
        }
    
        void WriteToFile()
        {
            using (Stream stream = File.Open(fileName, FileMode.Create))
            {
                new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter().Serialize(stream, new PacketsFile { packetList = _recordedQueue });
            }
            Debug.Log("File written");
        }
        
        void ReadFile()
        {
            using (Stream stream = File.Open(fileName, FileMode.Open))
            {
                _recordedQueue = (new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter().Deserialize(stream) as PacketsFile).packetList;
            }
            Debug.Log("File read");
        }
    }

  • edinkyedinky Posts: 14
    NerveGear
    This works great. Now I need to work out a clean way to record objects the avatar interacts with!
  • zapfanaticzapfanatic Posts: 2
    NerveGear
    Hi guys,

    in case someone needs avatar recording functionality in 2019, you are welcome to check out our (VR Labs) free tool for recording and playing back Oculus avatars:

    https://medium.com/telerik-ar-vr/motion-recording-for-oculus-avatars-game-objects-in-unity-with-vr-labs-free-motiontool-1299aee5223f
Sign In or Register to comment.