diff --git a/Runtime/FrameMetadata.cs b/Runtime/FrameMetadata.cs index 3b2bc03..df63360 100644 --- a/Runtime/FrameMetadata.cs +++ b/Runtime/FrameMetadata.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using Unity.Collections; -using UnityEngine.Profiling.Memory.Experimental; using UnityEngine; using UnityEngine.Playables; using Unity.WebRTC; @@ -10,152 +9,153 @@ using UnityEngine.UIElements; namespace Dolby.Millicast { - public class TransformableFrameInfo { - internal TransformableFrameInfo(RTCEncodedFrame frame) { - this.ssrc = frame.Ssrc; - this.timestamp = frame.Timestamp; - this._transformedData = null; - this.data = frame.GetData(); - this.length = this.data.Length; - } - - /// - /// the source identifier that generated the frame. - /// This distinguishes the source that generated the frame. - /// - public readonly uint ssrc; - - /// - /// The timestamp of receiving the video frame. - /// - public readonly uint timestamp; - - /// - /// This contains the encoded frame data without any modification. - /// - /// When Publishing, this should be filled with data that will end up - /// being attached to the frame after encoding. This does not contain - /// The frame data. - /// - /// When Subscribing, this will contain data extracted from the encoded frame buffer - /// before decoding. - /// - public readonly NativeArray.ReadOnly data; - - public void SetData(NativeArray.ReadOnly data, int length) { - if (length > data.Length) { - throw new Exception("Invalid length provided"); - } - this.length = length; - _transformedData = data; - } - - /// - /// Transformed data in case the user called SetData(); - /// - private NativeArray.ReadOnly? _transformedData; - - /// - /// The length of the frame to be sent down to webrtc. If the frame - /// is not transformed, then this will reflect the original frame's size. - /// - internal int length { get; private set; } - - /// - /// Internally used to either get the original frame or the transformed - /// frame if it was set. - /// - /// - internal NativeArray.ReadOnly GetData() { - if (_transformedData?.IsCreated ?? false) { - return (NativeArray.ReadOnly)_transformedData; - } - return data; - } +public class TransformableFrameInfo { + internal TransformableFrameInfo(RTCEncodedFrame frame) { + this.ssrc = frame.Ssrc; + this.timestamp = frame.Timestamp; + this._transformedData = null; + this.data = frame.GetData(); + this.length = this.data.Length; } - + /// - /// This class contains Encoded Video Frames that are optionally - /// transformed before being packetized for sending or decoded. Use - /// this class to get information about the video frames as well as - /// optionally transform the encoded frames. + /// the source identifier that generated the frame. + /// This distinguishes the source that generated the frame. /// - public class TransformableVideoFrameInfo: TransformableFrameInfo { + public readonly uint ssrc; + /// + /// The timestamp of receiving the video frame. + /// + public readonly uint timestamp; - public enum FrameType { - Empty, - Key, - Delta - }; + /// + /// This contains the encoded frame data without any modification. + /// + /// When Publishing, this should be filled with data that will end up + /// being attached to the frame after encoding. This does not contain + /// The frame data. + /// + /// When Subscribing, this will contain data extracted from the encoded frame + /// buffer before decoding. + /// + public readonly NativeArray.ReadOnly data; - internal TransformableVideoFrameInfo( - RTCEncodedVideoFrame frame): base(frame) { - var metaData = frame.GetMetadata(); - if (metaData.frameId > 0) this.frameId = metaData.frameId; - if (metaData.width > 0) this.width = metaData.width; - if (metaData.height > 0) this.height = metaData.height; - this.spatialIndex = metaData.spatialIndex; - this.temporalIndex = metaData.temporalIndex; - this.Type = ConvertToFrameType(frame.Type); + public void SetData(NativeArray.ReadOnly data, int length) { + if (length > data.Length) { + throw new Exception("Invalid length provided"); } + this.length = length; + _transformedData = data; + } + /// + /// Transformed data in case the user called SetData(); + /// + private NativeArray.ReadOnly? _transformedData; + + /// + /// The length of the frame to be sent down to webrtc. If the frame + /// is not transformed, then this will reflect the original frame's size. + /// + internal int length { get; private set; } - /// - /// - /// - public readonly FrameType Type; - - /// - /// - /// - public readonly long? frameId; - - /// - /// The width of the encoded video frame - /// - public readonly ushort? width; - - /// - /// The height of the encoded video frame - /// - public readonly ushort? height; - - /// - /// This specifies the SVC/simulcast spatial layer index - /// - public readonly long? spatialIndex; - - /// - /// This specifies the SVC/simulcast temporal layer index - /// - public readonly long? temporalIndex; - - - private FrameType ConvertToFrameType(RTCEncodedVideoFrameType type) { - switch (type) { - case RTCEncodedVideoFrameType.Empty: return FrameType.Empty; - case RTCEncodedVideoFrameType.Key: return FrameType.Key; - case RTCEncodedVideoFrameType.Delta: return FrameType.Delta; - default: return FrameType.Empty; - }; + /// + /// Internally used to either get the original frame or the transformed + /// frame if it was set. + /// + /// + internal NativeArray.ReadOnly GetData() { + if (_transformedData?.IsCreated ?? false) { + return (NativeArray.ReadOnly)_transformedData; } + return data; + } +} + +/// +/// This class contains Encoded Video Frames that are optionally +/// transformed before being packetized for sending or decoded. Use +/// this class to get information about the video frames as well as +/// optionally transform the encoded frames. +/// +public class TransformableVideoFrameInfo : TransformableFrameInfo { + + public enum FrameType { Empty, Key, Delta } + ; + + internal TransformableVideoFrameInfo(RTCEncodedVideoFrame frame) + : base(frame) { + var metaData = frame.GetMetadata(); + if (metaData.frameId > 0) + this.frameId = metaData.frameId; + if (metaData.width > 0) + this.width = metaData.width; + if (metaData.height > 0) + this.height = metaData.height; + this.spatialIndex = metaData.spatialIndex; + this.temporalIndex = metaData.temporalIndex; + this.Type = ConvertToFrameType(frame.Type); } + /// + /// + /// + public readonly FrameType Type; /// - /// This class contains Encoded Audio Frames that are optionally - /// transformed before being packetized for sending or decoded. Use - /// this class to optionally transform the encoded frames or attach metadata. + /// /// - public class TransformableAudioFrameInfo : TransformableFrameInfo { - internal TransformableAudioFrameInfo(RTCEncodedAudioFrame frame) : base(frame) { - } - } + public readonly long? frameId; + + /// + /// The width of the encoded video frame + /// + public readonly ushort? width; + + /// + /// The height of the encoded video frame + /// + public readonly ushort? height; - public class FrameMetadata { - public delegate void DelegateOnTransformableVideoFrame(TransformableVideoFrameInfo info); - public delegate void DelegateOnTransformableAudioFrame(TransformableAudioFrameInfo info); + /// + /// This specifies the SVC/simulcast spatial layer index + /// + public readonly long? spatialIndex; + + /// + /// This specifies the SVC/simulcast temporal layer index + /// + public readonly long? temporalIndex; + + private FrameType ConvertToFrameType(RTCEncodedVideoFrameType type) { + switch (type) { + case RTCEncodedVideoFrameType.Empty: + return FrameType.Empty; + case RTCEncodedVideoFrameType.Key: + return FrameType.Key; + case RTCEncodedVideoFrameType.Delta: + return FrameType.Delta; + default: + return FrameType.Empty; + }; } } +/// +/// This class contains Encoded Audio Frames that are optionally +/// transformed before being packetized for sending or decoded. Use +/// this class to optionally transform the encoded frames or attach metadata. +/// +public class TransformableAudioFrameInfo : TransformableFrameInfo { + internal TransformableAudioFrameInfo(RTCEncodedAudioFrame frame) + : base(frame) {} +} + +public class FrameMetadata { + public delegate void + DelegateOnTransformableVideoFrame(TransformableVideoFrameInfo info); + public delegate void + DelegateOnTransformableAudioFrame(TransformableAudioFrameInfo info); +} +} diff --git a/Runtime/Internal/FrameTransformer.cs b/Runtime/Internal/FrameTransformer.cs index b6eba6e..f9f3d65 100644 --- a/Runtime/Internal/FrameTransformer.cs +++ b/Runtime/Internal/FrameTransformer.cs @@ -2,84 +2,88 @@ using System.Collections.Generic; using Unity.Collections; using Unity.WebRTC; -using UnityEngine.Profiling.Memory.Experimental; using UnityEngine; using UnityEngine.Playables; using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("Dolby.Millicast.RuntimeTests")] -namespace Dolby.Millicast -{ - internal abstract class IFrameTransformer { - public abstract RTCRtpTransform GetTransform(); - public abstract void SetTransform(RTCRtpScriptTransform transform); - } - internal class SendFrameTransformer: IFrameTransformer { - private RTCRtpSender _rTCRtpSender; - - public SendFrameTransformer(RTCRtpSender rTCRtpSender) { - _rTCRtpSender = rTCRtpSender; - } +[assembly:InternalsVisibleTo("Dolby.Millicast.RuntimeTests")] +namespace Dolby.Millicast { +internal abstract class IFrameTransformer { + public abstract RTCRtpTransform GetTransform(); + public abstract void SetTransform(RTCRtpScriptTransform transform); +} +internal class SendFrameTransformer : IFrameTransformer { + private RTCRtpSender _rTCRtpSender; - public override RTCRtpTransform GetTransform() { - return _rTCRtpSender.Transform; - } + public SendFrameTransformer(RTCRtpSender rTCRtpSender) { + _rTCRtpSender = rTCRtpSender; + } - public override void SetTransform(RTCRtpScriptTransform transform) { - _rTCRtpSender.Transform = transform; - } + public override RTCRtpTransform GetTransform() { + return _rTCRtpSender.Transform; } - internal class ReceiveFrameTransformer: IFrameTransformer { - private RTCRtpReceiver _rTCRtpReceiver; + public override void SetTransform(RTCRtpScriptTransform transform) { + _rTCRtpSender.Transform = transform; + } +} - public ReceiveFrameTransformer(RTCRtpReceiver rTCRtpReceiver) { - _rTCRtpReceiver = rTCRtpReceiver; - } +internal class ReceiveFrameTransformer : IFrameTransformer { + private RTCRtpReceiver _rTCRtpReceiver; - public override RTCRtpTransform GetTransform() { - return _rTCRtpReceiver.Transform; - } + public ReceiveFrameTransformer(RTCRtpReceiver rTCRtpReceiver) { + _rTCRtpReceiver = rTCRtpReceiver; + } - public override void SetTransform(RTCRtpScriptTransform transform) { - _rTCRtpReceiver.Transform = transform; - } + public override RTCRtpTransform GetTransform() { + return _rTCRtpReceiver.Transform; } - internal class VideoConfigurator { - private IFrameTransformer _iFrameTransformer; - private FrameMetadata.DelegateOnTransformableVideoFrame _onTransformableVideoFrame; - internal VideoConfigurator(IFrameTransformer iFrameTransformer) { - _iFrameTransformer = iFrameTransformer; - } + public override void SetTransform(RTCRtpScriptTransform transform) { + _rTCRtpReceiver.Transform = transform; + } +} - public void SetTransform(FrameMetadata.DelegateOnTransformableVideoFrame callback) { - _onTransformableVideoFrame = callback; - _iFrameTransformer.SetTransform(new RTCRtpScriptTransform(TrackKind.Video, (RTCTransformEvent e) => { - TransformableVideoFrameInfo info = new TransformableVideoFrameInfo(e.Frame as RTCEncodedVideoFrame); - this._onTransformableVideoFrame?.Invoke(info); - e.Frame.SetData(info.GetData(), 0, info.length); - _iFrameTransformer.GetTransform().Write(e.Frame); - })); - } +internal class VideoConfigurator { + private IFrameTransformer _iFrameTransformer; + private FrameMetadata + .DelegateOnTransformableVideoFrame _onTransformableVideoFrame; + internal VideoConfigurator(IFrameTransformer iFrameTransformer) { + _iFrameTransformer = iFrameTransformer; } - internal class AudioConfigurator { - private IFrameTransformer _iFrameTransformer; - private FrameMetadata.DelegateOnTransformableAudioFrame _onTransformableAudioFrame; - internal AudioConfigurator(IFrameTransformer iFrameTransformer) { - _iFrameTransformer = iFrameTransformer; - } - public void SetTransform(FrameMetadata.DelegateOnTransformableAudioFrame callback) { - _onTransformableAudioFrame = callback; - _iFrameTransformer.SetTransform(new RTCRtpScriptTransform(TrackKind.Audio, (RTCTransformEvent e) => { - TransformableAudioFrameInfo info = new TransformableAudioFrameInfo(e.Frame as RTCEncodedAudioFrame); - this._onTransformableAudioFrame?.Invoke(info); - e.Frame.SetData(info.GetData(), 0, info.length); - _iFrameTransformer.GetTransform().Write(e.Frame); - })); - } + public void + SetTransform(FrameMetadata.DelegateOnTransformableVideoFrame callback) { + _onTransformableVideoFrame = callback; + _iFrameTransformer.SetTransform( + new RTCRtpScriptTransform(TrackKind.Video, (RTCTransformEvent e) => { + TransformableVideoFrameInfo info = + new TransformableVideoFrameInfo(e.Frame as RTCEncodedVideoFrame); + this._onTransformableVideoFrame?.Invoke(info); + e.Frame.SetData(info.GetData(), 0, info.length); + _iFrameTransformer.GetTransform().Write(e.Frame); + })); } } - +internal class AudioConfigurator { + private IFrameTransformer _iFrameTransformer; + private FrameMetadata + .DelegateOnTransformableAudioFrame _onTransformableAudioFrame; + internal AudioConfigurator(IFrameTransformer iFrameTransformer) { + _iFrameTransformer = iFrameTransformer; + } + public void + SetTransform(FrameMetadata.DelegateOnTransformableAudioFrame callback) { + _onTransformableAudioFrame = callback; + _iFrameTransformer.SetTransform( + new RTCRtpScriptTransform(TrackKind.Audio, (RTCTransformEvent e) => { + TransformableAudioFrameInfo info = + new TransformableAudioFrameInfo(e.Frame as RTCEncodedAudioFrame); + this._onTransformableAudioFrame?.Invoke(info); + e.Frame.SetData(info.GetData(), 0, info.length); + _iFrameTransformer.GetTransform().Write(e.Frame); + })); + } +} +} diff --git a/package.json b/package.json index 8b32353..24fcd30 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "licensesUrl": "https://example.com/licensing.html", "dependencies": { "com.unity.nuget.newtonsoft-json": "3.0.2", - "com.unity.inputsystem": "1.0.2" + "com.unity.inputsystem": "1.0.2", "com.unity.editorcoroutines": "1.0.0" }, "keywords": [