From d29bfc5c12cbae9dab2acc8b9764cbadf10d2a15 Mon Sep 17 00:00:00 2001 From: Pravus Date: Wed, 30 Oct 2024 20:35:26 +0100 Subject: [PATCH] fix: sdk video player SEEK + VS_ERROR video event (#2563) --- .../Components/MediaPlayerComponent.cs | 16 +++++--- .../MediaStream/MediaPlayerExtensions.cs | 23 +++++++++-- .../Systems/CreateMediaPlayerSystem.cs | 21 +++++++--- .../Systems/UpdateMediaPlayerSystem.cs | 22 +++++----- .../MediaStream/Systems/VideoEventsSystem.cs | 41 +++++++++++-------- .../Wrapper/MediaPlayerPluginWrapper.cs | 2 +- .../WebRequestControllerExtensions.cs | 1 + .../Assets/DCL/WebRequests/WebRequestUtils.cs | 3 +- 8 files changed, 85 insertions(+), 44 deletions(-) diff --git a/Explorer/Assets/DCL/SDKComponents/MediaStream/Components/MediaPlayerComponent.cs b/Explorer/Assets/DCL/SDKComponents/MediaStream/Components/MediaPlayerComponent.cs index f88da35c14..bbba779e5a 100644 --- a/Explorer/Assets/DCL/SDKComponents/MediaStream/Components/MediaPlayerComponent.cs +++ b/Explorer/Assets/DCL/SDKComponents/MediaStream/Components/MediaPlayerComponent.cs @@ -1,7 +1,5 @@ using DCL.ECSComponents; -using DCL.Optimization.Pools; using RenderHeads.Media.AVProVideo; -using System; using System.Threading; using Utility; @@ -17,9 +15,10 @@ public struct MediaPlayerComponent public string URL; public bool IsFromContentServer; - public VideoState State; - public double PreviousPlayingTimeCheck; - public float LastStateChangeTime; + public VideoState State { get; private set; } + public VideoState LastPropagatedState; + public double PreviousCurrentTimeChecked; + public float LastStateChangeTime { get; private set; } public CancellationTokenSource Cts; public OpenMediaPromise OpenMediaPromise; @@ -28,6 +27,13 @@ public struct MediaPlayerComponent public float CurrentTime => (float)MediaPlayer.Control.GetCurrentTime(); public float Duration => (float)MediaPlayer.Info.GetDuration(); + public void SetState(VideoState newState) + { + if (State == newState) return; + State = newState; + LastStateChangeTime = UnityEngine.Time.realtimeSinceStartup; + } + public void Dispose() { MediaPlayer = null; diff --git a/Explorer/Assets/DCL/SDKComponents/MediaStream/MediaPlayerExtensions.cs b/Explorer/Assets/DCL/SDKComponents/MediaStream/MediaPlayerExtensions.cs index e281a904e6..c85b2de153 100644 --- a/Explorer/Assets/DCL/SDKComponents/MediaStream/MediaPlayerExtensions.cs +++ b/Explorer/Assets/DCL/SDKComponents/MediaStream/MediaPlayerExtensions.cs @@ -1,5 +1,7 @@ +using Cysharp.Threading.Tasks; using DCL.ECSComponents; using RenderHeads.Media.AVProVideo; +using System; using UnityEngine; namespace DCL.SDKComponents.MediaStream @@ -62,12 +64,25 @@ public static MediaPlayer UpdatePlayback(this MediaPlayer mediaPlayer, bool hasP public static void SetPlaybackProperties(this MediaPlayer mediaPlayer, PBVideoPlayer sdkVideoPlayer) { if (!mediaPlayer.MediaOpened) return; + SetPlaybackPropertiesAsync(mediaPlayer.Control, sdkVideoPlayer).Forget(); + } - IMediaControl control = mediaPlayer.Control; + private static async UniTask SetPlaybackPropertiesAsync(IMediaControl control, PBVideoPlayer sdkVideoPlayer) + { + // If there are no seekable/buffered times, and we try to seek, AVPro may mistakenly play it from the start. + await UniTask.WaitUntil(() => control.GetBufferedTimes().Count > 0); - control.SetLooping(sdkVideoPlayer.HasLoop && sdkVideoPlayer.Loop); // default: false +#if (UNITY_EDITOR_OSX || UNITY_STANDALONE_OSX) + // The only way found to make the video initialization consistent and reliable on MacOS even after a scene reload + await UniTask.Delay(TimeSpan.FromSeconds(1f)); +#endif + + control.SetLooping(sdkVideoPlayer is { HasLoop: true, Loop: true }); // default: false control.SetPlaybackRate(sdkVideoPlayer.HasPlaybackRate ? sdkVideoPlayer.PlaybackRate : MediaPlayerComponent.DEFAULT_PLAYBACK_RATE); - control.SeekFast(sdkVideoPlayer.HasPosition ? sdkVideoPlayer.Position : MediaPlayerComponent.DEFAULT_POSITION); + control.Seek(sdkVideoPlayer.HasPosition ? sdkVideoPlayer.Position : MediaPlayerComponent.DEFAULT_POSITION); + + if (sdkVideoPlayer is { HasPlaying: true, Playing: true }) + control.Play(); } public static void UpdatePlaybackProperties(this MediaPlayer mediaPlayer, PBVideoPlayer sdkVideoPlayer) @@ -83,7 +98,7 @@ public static void UpdatePlaybackProperties(this MediaPlayer mediaPlayer, PBVide control.SetPlaybackRate(sdkVideoPlayer.PlaybackRate); if (sdkVideoPlayer.HasPosition) - control.SeekFast(sdkVideoPlayer.Position); + control.Seek(sdkVideoPlayer.Position); } } } diff --git a/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/CreateMediaPlayerSystem.cs b/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/CreateMediaPlayerSystem.cs index 3ad6784eab..a451a1ffa8 100644 --- a/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/CreateMediaPlayerSystem.cs +++ b/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/CreateMediaPlayerSystem.cs @@ -24,6 +24,8 @@ namespace DCL.SDKComponents.MediaStream [ThrottlingEnabled] public partial class CreateMediaPlayerSystem : BaseUnityLoopSystem { + private static string CONTENT_SERVER_PREFIX = "/content/contents"; + private readonly ISceneStateProvider sceneStateProvider; private readonly IPerformanceBudget frameTimeBudget; private readonly IComponentPool mediaPlayerPool; @@ -76,20 +78,27 @@ private void CreateMediaPlayer(Entity entity, string url, bool hasVolume, float private MediaPlayerComponent CreateMediaPlayerComponent(Entity entity, string url, bool hasVolume, float volume) { // if it is not valid, we try get it as a scene local video - if (!url.IsValidUrl() && sceneData.TryGetMediaUrl(url, out URLAddress mediaUrl)) - url = mediaUrl; + bool isValidStreamUrl = url.IsValidUrl(); + bool isValidLocalPath = false; + + if (!isValidStreamUrl) + { + isValidLocalPath = sceneData.TryGetMediaUrl(url, out URLAddress mediaUrl); + if(isValidLocalPath) + url = mediaUrl; + } var component = new MediaPlayerComponent { MediaPlayer = mediaPlayerPool.Get(), URL = url, - IsFromContentServer = url.Contains("/content/contents"), - State = url.IsValidUrl() ? VideoState.VsNone : VideoState.VsError, - PreviousPlayingTimeCheck = -1, - LastStateChangeTime = -1, + IsFromContentServer = url.Contains(CONTENT_SERVER_PREFIX), + PreviousCurrentTimeChecked = -1, + LastPropagatedState = VideoState.VsPaused, Cts = new CancellationTokenSource(), OpenMediaPromise = new OpenMediaPromise(), }; + component.SetState(isValidStreamUrl || isValidLocalPath || string.IsNullOrEmpty(url) ? VideoState.VsNone : VideoState.VsError); #if UNITY_EDITOR component.MediaPlayer.gameObject.name = $"MediaPlayer_Entity_{entity}"; diff --git a/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/UpdateMediaPlayerSystem.cs b/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/UpdateMediaPlayerSystem.cs index fdb3d75599..b39f161104 100644 --- a/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/UpdateMediaPlayerSystem.cs +++ b/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/UpdateMediaPlayerSystem.cs @@ -106,7 +106,7 @@ private void UpdateVideoStream(ref MediaPlayerComponent component, PBVideoPlayer } HandleComponentChange(ref component, sdkComponent, sdkComponent.Src, sdkComponent.HasPlaying, sdkComponent.Playing, sdkComponent, static (mediaPlayer, sdk) => mediaPlayer.UpdatePlaybackProperties(sdk)); - ConsumePromise(ref component, sdkComponent.HasPlaying && sdkComponent.Playing, sdkComponent, static (mediaPlayer, sdk) => mediaPlayer.SetPlaybackProperties(sdk)); + ConsumePromise(ref component, false, sdkComponent, static (mediaPlayer, sdk) => mediaPlayer.SetPlaybackProperties(sdk)); } [Query] @@ -134,9 +134,7 @@ private void HandleComponentChange(ref MediaPlayerComponent component, IDirtyMar { if (!sdkComponent.IsDirty) return; - sceneData.TryGetMediaUrl(url, out URLAddress localMediaUrl); - - if (component.URL != url && component.URL != localMediaUrl) + if (component.URL != url && (!sceneData.TryGetMediaUrl(url, out URLAddress localMediaUrl) || component.URL != localMediaUrl)) { component.MediaPlayer.CloseCurrentStream(); @@ -175,20 +173,24 @@ private static void ConsumePromise(ref MediaPlayerComponent component, bool auto } else { - component.State = VideoState.VsError; + component.SetState(string.IsNullOrEmpty(component.URL) ? VideoState.VsNone : VideoState.VsError); component.MediaPlayer.CloseCurrentStream(); } } private void UpdateStreamUrl(ref MediaPlayerComponent component, string url) { - component.MediaPlayer.CloseCurrentStream(); - - if (!url.IsValidUrl() && sceneData.TryGetMediaUrl(url, out URLAddress mediaUrl)) - url = mediaUrl; + bool isValidStreamUrl = url.IsValidUrl(); + bool isValidLocalPath = false; + if (!isValidStreamUrl) + { + isValidLocalPath = sceneData.TryGetMediaUrl(url, out URLAddress mediaUrl); + if(isValidLocalPath) + url = mediaUrl; + } component.URL = url; - component.State = url.IsValidUrl() ? VideoState.VsNone : VideoState.VsError; + component.SetState(isValidStreamUrl || isValidLocalPath || string.IsNullOrEmpty(url) ? VideoState.VsNone : VideoState.VsError); } public override void Dispose() diff --git a/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/VideoEventsSystem.cs b/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/VideoEventsSystem.cs index 7f715f77cc..b3babf0036 100644 --- a/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/VideoEventsSystem.cs +++ b/Explorer/Assets/DCL/SDKComponents/MediaStream/Systems/VideoEventsSystem.cs @@ -1,13 +1,11 @@ using Arch.Core; using Arch.System; using Arch.SystemGroups; -using Arch.SystemGroups.Throttling; using CRDT; using CrdtEcsBridge.ECSToCRDTWriter; using DCL.Diagnostics; using DCL.ECSComponents; using DCL.Optimization.PerformanceBudgeting; -using DCL.Optimization.Pools; using ECS.Abstract; using ECS.Groups; using RenderHeads.Media.AVProVideo; @@ -18,21 +16,18 @@ namespace DCL.SDKComponents.MediaStream { [UpdateInGroup(typeof(SyncedPreRenderingSystemGroup))] [LogCategory(ReportCategory.MEDIA_STREAM)] - [ThrottlingEnabled] public partial class VideoEventsSystem : BaseUnityLoopSystem { private const float MAX_VIDEO_FROZEN_SECONDS_BEFORE_ERROR = 10f; private readonly IECSToCRDTWriter ecsToCRDTWriter; private readonly ISceneStateProvider sceneStateProvider; - private readonly IComponentPool componentPool; private readonly IPerformanceBudget frameTimeBudget; - private VideoEventsSystem(World world, IECSToCRDTWriter ecsToCrdtWriter, ISceneStateProvider sceneStateProvider, IComponentPool componentPool, IPerformanceBudget frameTimeBudget) : base(world) + private VideoEventsSystem(World world, IECSToCRDTWriter ecsToCrdtWriter, ISceneStateProvider sceneStateProvider, IPerformanceBudget frameTimeBudget) : base(world) { ecsToCRDTWriter = ecsToCrdtWriter; this.sceneStateProvider = sceneStateProvider; - this.componentPool = componentPool; this.frameTimeBudget = frameTimeBudget; } @@ -43,22 +38,30 @@ protected override void Update(float t) [Query] [All(typeof(PBVideoPlayer))] - private void PropagateVideoEvents(ref CRDTEntity sdkEntity, ref MediaPlayerComponent mediaPlayer) + private void PropagateVideoEvents(in CRDTEntity sdkEntity, ref MediaPlayerComponent mediaPlayer) { if (!frameTimeBudget.TrySpendBudget()) return; - VideoState newState = GetCurrentVideoState(mediaPlayer.MediaPlayer.Control, mediaPlayer.PreviousPlayingTimeCheck, mediaPlayer.LastStateChangeTime); + // The Media Player could already been flagged with errors detected on the video promise, those have to be propagated. + if (mediaPlayer.State != VideoState.VsError) + { + VideoState newState = GetCurrentVideoState(mediaPlayer); - if (mediaPlayer.State == newState) return; - mediaPlayer.LastStateChangeTime = Time.realtimeSinceStartup; - mediaPlayer.PreviousPlayingTimeCheck = mediaPlayer.MediaPlayer.Control.GetCurrentTime(); - mediaPlayer.State = newState; + if (mediaPlayer.State != newState) + { + mediaPlayer.PreviousCurrentTimeChecked = mediaPlayer.MediaPlayer.Control.GetCurrentTime(); + mediaPlayer.SetState(newState); + } + } - AppendMessage(in sdkEntity, in mediaPlayer); + PropagateStateInVideoEvent(in sdkEntity, ref mediaPlayer); } - private void AppendMessage(in CRDTEntity sdkEntity, in MediaPlayerComponent mediaPlayer) + private void PropagateStateInVideoEvent(in CRDTEntity sdkEntity, ref MediaPlayerComponent mediaPlayer) { + if (mediaPlayer.LastPropagatedState == mediaPlayer.State) return; + + mediaPlayer.LastPropagatedState = mediaPlayer.State; ecsToCRDTWriter.AppendMessage ( prepareMessage: static (pbVideoEvent, data) => @@ -74,9 +77,12 @@ private void AppendMessage(in CRDTEntity sdkEntity, in MediaPlayerComponent medi ); } - private static VideoState GetCurrentVideoState(IMediaControl mediaPlayerControl, double previousPlayingTimeCheck, float lastStateChangeTime) + private static VideoState GetCurrentVideoState(in MediaPlayerComponent mediaPlayer) { + if (string.IsNullOrEmpty(mediaPlayer.URL)) return VideoState.VsNone; + // Important: while PLAYING or PAUSED, MediaPlayerControl may also be BUFFERING and/or SEEKING. + var mediaPlayerControl = mediaPlayer.MediaPlayer.Control; if (mediaPlayerControl.IsFinished()) return VideoState.VsNone; if (mediaPlayerControl.GetLastError() != ErrorCode.None) return VideoState.VsError; @@ -87,15 +93,16 @@ private static VideoState GetCurrentVideoState(IMediaControl mediaPlayerControl, { state = VideoState.VsPlaying; - if (mediaPlayerControl.GetCurrentTime().Equals(previousPlayingTimeCheck)) // Video is frozen + if (mediaPlayerControl.GetCurrentTime().Equals(mediaPlayer.PreviousCurrentTimeChecked)) // Video is frozen { state = mediaPlayerControl.IsSeeking() ? VideoState.VsSeeking : VideoState.VsBuffering; // If the seeking/buffering never ends, update state with error so the scene can react - if ((Time.realtimeSinceStartup - lastStateChangeTime) > MAX_VIDEO_FROZEN_SECONDS_BEFORE_ERROR) + if ((Time.realtimeSinceStartup - mediaPlayer.LastStateChangeTime) > MAX_VIDEO_FROZEN_SECONDS_BEFORE_ERROR) state = VideoState.VsError; } } + return state; } } diff --git a/Explorer/Assets/DCL/SDKComponents/MediaStream/Wrapper/MediaPlayerPluginWrapper.cs b/Explorer/Assets/DCL/SDKComponents/MediaStream/Wrapper/MediaPlayerPluginWrapper.cs index 373d859a19..de6159a680 100644 --- a/Explorer/Assets/DCL/SDKComponents/MediaStream/Wrapper/MediaPlayerPluginWrapper.cs +++ b/Explorer/Assets/DCL/SDKComponents/MediaStream/Wrapper/MediaPlayerPluginWrapper.cs @@ -73,7 +73,7 @@ public void InjectToWorld(ref ArchSystemsWorldBuilder builder, ISceneData CreateMediaPlayerSystem.InjectToWorld(ref builder, webRequestController, sceneData, mediaPlayerPool, sceneStateProvider, frameTimeBudget); UpdateMediaPlayerSystem.InjectToWorld(ref builder, webRequestController, sceneData, sceneStateProvider, frameTimeBudget, worldVolumeMacBus); - VideoEventsSystem.InjectToWorld(ref builder, ecsToCrdtWriter, sceneStateProvider, componentPoolsRegistry.GetReferenceTypePool(), frameTimeBudget); + VideoEventsSystem.InjectToWorld(ref builder, ecsToCrdtWriter, sceneStateProvider, frameTimeBudget); finalizeWorldSystems.Add(CleanUpMediaPlayerSystem.InjectToWorld(ref builder, mediaPlayerPool, videoTexturePool)); #endif diff --git a/Explorer/Assets/DCL/WebRequests/WebRequestControllerExtensions.cs b/Explorer/Assets/DCL/WebRequests/WebRequestControllerExtensions.cs index 866734ac22..e00581411e 100644 --- a/Explorer/Assets/DCL/WebRequests/WebRequestControllerExtensions.cs +++ b/Explorer/Assets/DCL/WebRequests/WebRequestControllerExtensions.cs @@ -165,6 +165,7 @@ public static async UniTask IsHeadReachableAsync(this IWebRequestControlle { // It means there is no such end-point at all case WebRequestUtils.BAD_REQUEST: + case WebRequestUtils.FORBIDDEN_ACCESS: case WebRequestUtils.NOT_FOUND: return false; } diff --git a/Explorer/Assets/DCL/WebRequests/WebRequestUtils.cs b/Explorer/Assets/DCL/WebRequests/WebRequestUtils.cs index f0f3a9291f..75b9a9cfd8 100644 --- a/Explorer/Assets/DCL/WebRequests/WebRequestUtils.cs +++ b/Explorer/Assets/DCL/WebRequests/WebRequestUtils.cs @@ -11,8 +11,9 @@ namespace DCL.WebRequests public static class WebRequestUtils { public static string CANNOT_CONNECT_ERROR = "Cannot connect to destination host"; - + public const int BAD_REQUEST = 400; + public const int FORBIDDEN_ACCESS = 403; public const int NOT_FOUND = 404; public static SuppressExceptionWithFallback SuppressExceptionsWithFallback(this TCoreOp coreOp, TResult fallbackValue, SuppressExceptionWithFallback.Behaviour behaviour = SuppressExceptionWithFallback.Behaviour.Default, ReportData? reportContext = null) where TWebRequest: struct, ITypedWebRequest where TCoreOp: IWebRequestOp =>