Audio Device single thread contract, improvements

ServiceProvider/Services/Audio/JitterBuffer.cs

@@ -16,7 +16,8 @@ class JitterBuffer
         public int NumLostPackages = 0;
         public int BufferLatency;
         public int MinBufferLatency = 60;
-        public int Duration => numSeqBuffered * 20;
+        public int CaptureInterval = 20;
+        public int Duration => numSeqBuffered * CaptureInterval;
         private ConcurrentDictionary<DateTime, AudioSample> samples = new ConcurrentDictionary<DateTime, AudioSample>();
         private readonly object locker = new object();
         private MemoryStream sampleStream = new MemoryStream();
@@ -57,7 +58,7 @@ public void StartPublushing2()
                     //int toTake = Math.Min(2, numSeqBuffered) + (samplesOrdered_.Count() - (BufferLatency / 20));
                     IEnumerable<KeyValuePair<DateTime, AudioSample>> samplesOrdered;
                     // jittr buffer reached max duration, we have to take even if seq is broken
-                    if (numSeqBuffered >= BufferLatency / 20)
+                    if (numSeqBuffered >= BufferLatency / CaptureInterval)
                     {
                         int toTake = Math.Max(2, consequtiveSeqLenght+1);
                         samplesOrdered = samplesOrdered_.Take(Math.Min(toTake, samplesOrdered_.Count()));

ServiceProvider/Services/Network/MessageHandler.cs

@@ -338,12 +338,17 @@ public Dictionary<Guid, double> GetTcpPingStatus()
 
         public void Disconnect()
         {
-            foreach (var item in registeredPeers)
+            try
             {
-                OnPeerUnregistered?.Invoke(item.Key);
+                foreach (var item in registeredPeers)
+                {
+                    OnPeerUnregistered?.Invoke(item.Key);
+                }
+                registeredPeers.Clear();
+                client.Disconnect();
             }
-            registeredPeers.Clear();
-            client.Disconnect();
+            catch { }
+
             //client.Dispose();
             //client = null;
             //InitializeClient();

ServiceProvider/Services/ScreenShare/ScreenShareHandlerH264.cs

@@ -1,5 +1,4 @@
-using OpenCvSharp;
-using System;
+using System;
 using Videocall.Services.Video;
 using System.Threading;
 using System.Collections.Concurrent;
@@ -69,7 +68,7 @@ void SetupTranscoder()
         {
             transcoder = new H264Transcoder(targetFps, 2_000_000);
             transcoder.SetupTranscoder(0, 0, configType);
-            transcoder.DecodedFrameAvailable = (f) => { incomingFrames++; RemoteImageAvailable?.Invoke(ImageReference.FromRgbImage(f)); };
+            transcoder.DecodedFrameAvailable = (f) => { incomingFrames++; RemoteImageAvailable?.Invoke(f); };
             transcoder.EncodedFrameAvailable = (b, o, k) => { bytesSent += o; OnBytesAvailable?.Invoke(b, o, k); };
 
             //transcoder.EncodedFrameAvailable2 = (action, size, isKeyFrame) => 
@@ -114,13 +113,11 @@ public void StartCapture()
             }
 
         }
-
+        ConcurrentBag<RgbImage> previewImgs = new ConcurrentBag<RgbImage>();
         private void BeginCaptureThread(float scale)
         {
-            Mat mainMat = null;
-            Mat Yuv420Mat = new Mat();
-            Mat smallMat = new Mat();
-            RgbImage smallThumb =null;
+
+           // RgbImage smallThumb =null;
             screenCapture.CaptureAuto(targetFps,
                (img) =>
                {
@@ -132,10 +129,6 @@ private void BeginCaptureThread(float scale)
                        transcoder.ApplyChanges(targetFps,TargetKBps * 1000, img.Width, img.Height, configType);
                    }
 
-
-
-
-
                    if (!parallel)
                    {
                        ImageData data_ = new ImageData(ImageType.Bgr,
@@ -181,6 +174,7 @@ private void BeginCaptureThread(float scale)
                                                      img.Width * img.Height * 3);
 
                    int mul = 8;
+                   previewImgs.TryTake(out RgbImage smallThumb);
                    if(smallThumb == null)
                    {
                        smallThumb = new RgbImage(data.Width / mul, data.Height / mul);
@@ -194,9 +188,9 @@ private void BeginCaptureThread(float scale)
 
                    transcoder.Downscale(data, smallThumb, mul);
                    sww.Stop();
-                   Trace.WriteLine("DS " + sww.ElapsedTicks);
-                   LocalImageAvailable?.Invoke(ImageReference.FromRgbImage(smallThumb));
 
+                   LocalImageAvailable?.Invoke(ImageReference.FromRgbImage(smallThumb, ReturnSmallImg));
+
                    //smallThumb.Dispose();
                    //smallThumb = null;
 
@@ -205,6 +199,10 @@ private void BeginCaptureThread(float scale)
                });
         }
 
+        private void ReturnSmallImg(ImageReference imRef)
+        {
+            previewImgs.Add((RgbImage)imRef.underlyingData);
+        }
 
 
         private void EncodeParallel()
@@ -370,10 +368,6 @@ public void HandleLtrRecovery(byte[] payload, int payloadOffset, int payloadCoun
             transcoder.SetLTRRecoverRequest(payload, payloadOffset, payloadCount);
         }
 
-        public void ReturnImage(ImageReference image)
-        {
-            if(image.underlyingData is RgbImage)
-                transcoder.ReturnImage((RgbImage)image.underlyingData);
-        }
+
     }
 }

ServiceProvider/Services/Video/Camera/ICameraProvider.cs

@@ -10,6 +10,6 @@ public interface ICameraProvider
         bool IsOpened();
         void Open(int camIdx);
         void Release();
-        bool Retrieve(ref ImageReference im);
+        bool Retrieve(out ImageReference im);
     }
 }

ServiceProvider/Services/Video/Camera/WindowsCameraProvider.cs

@@ -1,5 +1,6 @@
 using OpenCvSharp;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -9,7 +10,8 @@ namespace ServiceProvider.Services.Video.Camera
 {
     public class WindowsCameraProvider : ICameraProvider
     {
-        readonly Mat frame = new Mat();
+        private ConcurrentBag<Mat> mats = new ConcurrentBag<Mat>();
+        private ConcurrentBag<ImageReference> imrefs = new ConcurrentBag<ImageReference>();
         readonly VideoCapture capture;
         private int frameWidth => capture.FrameWidth;
         private int frameHeight => capture.FrameHeight;
@@ -48,8 +50,20 @@ public bool Grab()
             return capture.Grab();
         }
 
-        public bool Retrieve(ref ImageReference im)
+        public bool Retrieve(out ImageReference im)
         {
+            mats.TryTake(out var frame);
+            imrefs.TryTake(out im);
+
+            if (frame == null)
+            {
+                frame= new Mat();
+            }
+            if (frame.Width != capture.FrameWidth || frame.Height != capture.FrameHeight)
+            {
+                frame.Dispose();
+                frame =  new Mat();
+            }
 
             var res = capture.Retrieve(frame);
             if (res == false)
@@ -58,7 +72,7 @@ public bool Retrieve(ref ImageReference im)
             }
             if (im == null)
             {
-                im = ImageReference.FromMat(frame);
+                im = ImageReference.FromMat(frame,ReturnMat);
             }
             else
             {
@@ -68,5 +82,11 @@ public bool Retrieve(ref ImageReference im)
             return res;
 
         }
+
+        private void ReturnMat(ImageReference reference)
+        {
+            imrefs.Add(reference);
+            mats.Add((Mat)reference.underlyingData);
+        }
     }
 }

ServiceProvider/Services/Video/H264/H264Transcoder.cs

@@ -13,18 +13,20 @@
 using System.Threading.Tasks;
 using System.Runtime.InteropServices;
 using System.Diagnostics;
+using ServiceProvider.Services.Video;
 
 namespace Videocall.Services.Video.H264
 {
     internal class H264Transcoder:IDisposable
     {
         public Action<Action<PooledMemoryStream>, int,bool> EncodedFrameAvailable2;
         public Action<byte[], int, bool> EncodedFrameAvailable;
-        public Action<RgbImage> DecodedFrameAvailable;
+        public Action<ImageReference> DecodedFrameAvailable;
         public Action KeyFrameRequested;
         public Action<byte[],int, int> MarkingFeedback;
         public Action<byte[],int,int> LtrRecoveryRequest;
         public ConcurrentBag<RgbImage> imagePool = new ConcurrentBag<RgbImage>();
+        public ConcurrentBag<ImageReference> imrefPool = new ConcurrentBag<ImageReference>();
         private bool DecodeNoDelay = true;
         public bool InvokeAction => EncodedFrameAvailable2!=null;
         public double Duration => jitterBufffer.Duration;
@@ -49,13 +51,13 @@ internal class H264Transcoder:IDisposable
         private ushort decoderHeight;
         int consecutiveError = 0;
         Converter converter = new Converter();
-        public H264Transcoder(ConcurrentBag<RgbImage> matPool,int desiredFps,int desiredBps)
-        {
-            this.imagePool = matPool;
-            fps = desiredFps;
-            bps = desiredBps;
-            jitterBufffer.FrameAvailable += (f) => Decode(f.Data, f.Offset, f.Count,f.w,f.h);
-        }
+        //public H264Transcoder(ConcurrentBag<RgbImage> matPool,int desiredFps,int desiredBps)
+        //{
+        //    this.imagePool = matPool;
+        //    fps = desiredFps;
+        //    bps = desiredBps;
+        //    jitterBufffer.FrameAvailable += (f) => Decode(f.Data, f.Offset, f.Count,f.w,f.h);
+        //}
         public H264Transcoder(int desiredFps, int desiredBps)
         {
             fps = desiredFps;
@@ -292,7 +294,14 @@ private void Decode(byte[] payload, int payloadOffset, int payloadCount, int w,
                     if (succ)
                     {
                         ManageError(statusCode);
-                        DecodedFrameAvailable?.Invoke(rgbImage);
+
+                        imrefPool.TryTake(out var imref);
+                        if (imref == null)
+                            imref = ImageReference.FromRgbImage(rgbImage, ReturnImage);
+                        else
+                            imref.Update(rgbImage);
+
+                        DecodedFrameAvailable?.Invoke(imref);
 
                         keyReq = 0;
                     }
@@ -365,7 +374,13 @@ private unsafe void Decode2(byte[] payload, int payloadOffset, int payloadCount,
                     CheckMarkingFeedback(statusCode);
                     if (succ)
                     {
-                        DecodedFrameAvailable?.Invoke(rgbImage);
+                        imrefPool.TryTake(out var imref);
+                        if (imref == null)
+                            imref = ImageReference.FromRgbImage(rgbImage, ReturnImage);
+                        else
+                            imref.Update(rgbImage);
+
+                        DecodedFrameAvailable?.Invoke(imref);
                         keyReq = 0;
                         ManageError(statusCode);
 
@@ -554,12 +569,25 @@ public void Dispose()
 
         }
 
-        internal void ReturnImage(RgbImage image)
-        {
+        internal void ReturnImage(ImageReference imref)
+        {   
+            var image = (RgbImage)imref.underlyingData;
+
             if(imagePool.Count<2)
                 imagePool.Add(image);
             else
                 image.Dispose();
+
+            imrefPool.Add(imref);
+
+        }
+
+        internal void FlushPool()
+        {
+            while (imagePool.TryTake(out var img))
+                img?.Dispose();
+
+            imrefPool.Clear();
         }
     }
 }

ServiceProvider/Services/Video/ImageReference.cs

@@ -18,7 +18,7 @@ public class ImageReference
         public int Offset;
         public int Length;
         public int Width, Height, Stride;
-
+        public Action ReturnImage;
         public IntPtr DataStart {
             get
             {
@@ -39,17 +39,18 @@ public IntPtr DataStart {
             }
             set => dataStart = value; }
 
-        public ImageReference(object underlyingData, IntPtr dataPtr, int width, int height, int stride)
+        public ImageReference(object underlyingData, IntPtr dataPtr, int width, int height, int stride, Action<ImageReference> whenReturning)
         {
-            Create(underlyingData, dataPtr, width, height, stride);
+            Create(underlyingData, dataPtr, width, height, stride, () => whenReturning.Invoke(this));
         }
-        private void Create(object underlyingData, IntPtr dataPtr, int width, int height, int stride)
+        private void Create(object underlyingData, IntPtr dataPtr, int width, int height, int stride, Action whenReturning)
         {
             this.underlyingData = underlyingData;
             this.DataStart = dataPtr;
             Width = width;
             Height = height;
             Stride = stride;
+            ReturnImage = whenReturning;
         }
 
         public ImageReference( byte[] data, int offset, int length, int width, int height, int stride)
@@ -65,23 +66,29 @@ public ImageReference( byte[] data, int offset, int length, int width, int heigh
 
         public void Update(Mat mat)
         {
-             Create(mat, mat.DataStart, mat.Width, mat.Height, (int)mat.Step());
+
+             Create(mat, mat.DataStart, mat.Width, mat.Height, (int)mat.Step(), ReturnImage);
         }
-
-        public static ImageReference FromMat(Mat mat)
+        internal void Update(RgbImage mat)
         {
-            return new ImageReference(mat, mat.DataStart, mat.Width, mat.Height,(int)mat.Step());
+            Create(mat, mat.ImageBytes, mat.Width, mat.Height, mat.Stride, ReturnImage);
+        }
+        public static ImageReference FromMat(Mat mat,Action<ImageReference> whenReturning)
+        {
+            return new ImageReference(mat, mat.DataStart, mat.Width, mat.Height,(int)mat.Step(), whenReturning);
         }
 
-        public static ImageReference FromRgbImage(RgbImage rgb)
+        public static ImageReference FromRgbImage(RgbImage rgb, Action<ImageReference> whenReturning)
         {
-            return new ImageReference(rgb, rgb.ImageBytes, rgb.Width, rgb.Height, rgb.Stride);
+            return new ImageReference(rgb, rgb.ImageBytes, rgb.Width, rgb.Height, rgb.Stride, whenReturning);
         }
 
         public void Release()
         {
             if(underlyingData!=null && underlyingData is IDisposable)
                 ((IDisposable)underlyingData).Dispose();
         }
+
+
     }
 }