From d77bc29389b410fba80929b99912f42ca5e17c89 Mon Sep 17 00:00:00 2001 From: Johnny Z Date: Thu, 21 Dec 2017 04:41:36 +1000 Subject: [PATCH] Unsafe direct buffers. (#316) --- src/DotNetty.Buffers/AbstractByteBuffer.cs | 24 +- .../AbstractByteBufferAllocator.cs | 53 ++- .../AbstractPooledDerivedByteBuffer.cs | 4 + .../AbstractUnpooledSlicedByteBuffer.cs | 21 +- .../AdvancedLeakAwareByteBuffer.cs | 8 +- .../AdvancedLeakAwareCompositeByteBuffer.cs | 8 +- src/DotNetty.Buffers/ByteBufferUtil.cs | 8 +- src/DotNetty.Buffers/CompositeByteBuffer.cs | 100 ++++- src/DotNetty.Buffers/DotNetty.Buffers.csproj | 5 + src/DotNetty.Buffers/EmptyByteBuffer.cs | 125 ++---- src/DotNetty.Buffers/HeapByteBufferUtil.cs | 4 +- src/DotNetty.Buffers/IByteBuffer.cs | 37 +- src/DotNetty.Buffers/IByteBufferAllocator.cs | 12 + .../IByteBufferAllocatorMetric .cs | 7 +- src/DotNetty.Buffers/PoolArena.cs | 110 +++++- src/DotNetty.Buffers/PoolThreadCache.cs | 60 ++- src/DotNetty.Buffers/PooledByteBuffer.cs | 6 +- .../PooledByteBufferAllocator.cs | 102 ++++- .../PooledByteBufferAllocatorMetric.cs | 23 +- .../PooledDuplicatedByteBuffer.cs | 4 + src/DotNetty.Buffers/PooledHeapByteBuffer.cs | 12 + .../PooledSlicedByteBuffer.cs | 17 +- .../PooledUnsafeDirectByteBuffer.cs | 187 +++++++++ src/DotNetty.Buffers/ThrowHelper.cs | 4 + src/DotNetty.Buffers/Unpooled.cs | 13 +- .../UnpooledByteBufferAllocator.cs | 70 +++- .../UnpooledDuplicatedByteBuffer.cs | 13 +- .../UnpooledHeapByteBuffer.cs | 22 +- .../UnpooledUnsafeDirectByteBuffer.cs | 359 ++++++++++++++++++ src/DotNetty.Buffers/UnsafeByteBufferUtil .cs | 332 ++++++++++++++++ src/DotNetty.Buffers/WrappedByteBuffer.cs | 21 +- .../WrappedCompositeByteBuffer.cs | 8 +- src/DotNetty.Codecs.Redis/RedisDecoder.cs | 6 +- src/DotNetty.Codecs/LineBasedFrameDecoder.cs | 2 +- src/DotNetty.Common/Internal/MathUtil.cs | 29 +- .../Internal/PlatformDependent.cs | 85 ++++- .../Internal/PlatformDependent0.cs | 46 +++ .../Utilities/ByteProcessor.cs | 87 ++--- .../Native/ReadOperation.cs | 56 +-- src/DotNetty.Transport.Libuv/Native/Tcp.cs | 2 +- .../AbstractByteBufferAllocatorTests.cs | 37 +- .../AbstractByteBufferTests.cs | 55 ++- ...AbstractReferenceCountedByteBufferTests.cs | 8 + .../ByteBufferAllocatorTests.cs | 155 ++++++-- .../EmptyByteBufferTests.cs | 8 + test/DotNetty.Buffers.Tests/PoolArenaTests.cs | 8 +- .../PooledByteBufferAllocatorTests.cs | 34 +- .../PooledDirectByteBufferTests.cs | 10 + .../SimpleLeakAwareByteBufferTests.cs | 112 ++---- ...SimpleLeakAwareCompositeByteBufferTests.cs | 114 ++---- .../UnpooledByteBufferAllocatorTests.cs | 4 +- .../UnreleaseableByteBufferTests.cs | 4 +- .../UnsafeDirectByteBufferTest.cs | 20 + .../RoundTripTests.cs | 4 +- .../RoundTripTests.cs | 4 +- ... => PooledByteBufferAllocatorBenchmark.cs} | 6 +- ...> UnpooledByteBufferAllocatorBenchmark.cs} | 5 +- ...chmark.cs => PooledByteBufferBenchmark.cs} | 29 +- ...mark.cs => UnpooledByteBufferBenchmark.cs} | 24 +- test/DotNetty.Microbench/Program.cs | 8 +- .../SocketDatagramChannelUnicastTest.cs | 10 +- 61 files changed, 2142 insertions(+), 609 deletions(-) create mode 100644 src/DotNetty.Buffers/PooledUnsafeDirectByteBuffer.cs create mode 100644 src/DotNetty.Buffers/UnpooledUnsafeDirectByteBuffer.cs create mode 100644 src/DotNetty.Buffers/UnsafeByteBufferUtil .cs create mode 100644 src/DotNetty.Common/Internal/PlatformDependent0.cs create mode 100644 test/DotNetty.Buffers.Tests/PooledDirectByteBufferTests.cs create mode 100644 test/DotNetty.Buffers.Tests/UnsafeDirectByteBufferTest.cs rename test/DotNetty.Microbench/Allocators/{PooledHeapByteBufferAllocatorBenchmark.cs => PooledByteBufferAllocatorBenchmark.cs} (50%) rename test/DotNetty.Microbench/Allocators/{UnpooledHeapByteBufferAllocatorBenchmark.cs => UnpooledByteBufferAllocatorBenchmark.cs} (54%) rename test/DotNetty.Microbench/Buffers/{PooledHeapByteBufferBenchmark.cs => PooledByteBufferBenchmark.cs} (59%) rename test/DotNetty.Microbench/Buffers/{UnpooledHeapByteBufferBenchmark.cs => UnpooledByteBufferBenchmark.cs} (65%) diff --git a/src/DotNetty.Buffers/AbstractByteBuffer.cs b/src/DotNetty.Buffers/AbstractByteBuffer.cs index 0e0d9738c..b905aa2c3 100644 --- a/src/DotNetty.Buffers/AbstractByteBuffer.cs +++ b/src/DotNetty.Buffers/AbstractByteBuffer.cs @@ -243,7 +243,7 @@ public virtual IByteBuffer EnsureWritable(int minWritableBytes) return this; } - protected void EnsureWritable0(int minWritableBytes) + protected internal void EnsureWritable0(int minWritableBytes) { this.EnsureAccessible(); if (minWritableBytes <= this.WritableBytes) @@ -821,6 +821,7 @@ public virtual IByteBuffer ReadBytes(int length) public virtual IByteBuffer ReadSlice(int length) { + this.CheckReadableBytes(length); IByteBuffer slice = this.Slice(this.readerIndex, length); this.readerIndex += length; return slice; @@ -828,6 +829,7 @@ public virtual IByteBuffer ReadSlice(int length) public virtual IByteBuffer ReadRetainedSlice(int length) { + this.CheckReadableBytes(length); IByteBuffer slice = this.RetainedSlice(this.readerIndex, length); this.readerIndex += length; return slice; @@ -1145,19 +1147,19 @@ public virtual int BytesBefore(int index, int length, byte value) return endIndex - index; } - public virtual int ForEachByte(ByteProcessor processor) + public virtual int ForEachByte(IByteProcessor processor) { this.EnsureAccessible(); return this.ForEachByteAsc0(this.readerIndex, this.writerIndex, processor); } - public virtual int ForEachByte(int index, int length, ByteProcessor processor) + public virtual int ForEachByte(int index, int length, IByteProcessor processor) { this.CheckIndex(index, length); return this.ForEachByteAsc0(index, index + length, processor); } - int ForEachByteAsc0(int start, int end, ByteProcessor processor) + int ForEachByteAsc0(int start, int end, IByteProcessor processor) { for (; start < end; ++start) { @@ -1170,19 +1172,19 @@ int ForEachByteAsc0(int start, int end, ByteProcessor processor) return -1; } - public virtual int ForEachByteDesc(ByteProcessor processor) + public virtual int ForEachByteDesc(IByteProcessor processor) { this.EnsureAccessible(); return this.ForEachByteDesc0(this.writerIndex - 1, this.readerIndex, processor); } - public virtual int ForEachByteDesc(int index, int length, ByteProcessor processor) + public virtual int ForEachByteDesc(int index, int length, IByteProcessor processor) { this.CheckIndex(index, length); return this.ForEachByteDesc0(index + length - 1, index, processor); } - int ForEachByteDesc0(int rStart, int rEnd, ByteProcessor processor) + int ForEachByteDesc0(int rStart, int rEnd, IByteProcessor processor) { for (; rStart >= rEnd; --rStart) { @@ -1331,8 +1333,16 @@ protected void DiscardMarks() public abstract int ArrayOffset { get; } + public abstract bool HasMemoryAddress { get; } + + public abstract ref byte GetPinnableMemoryAddress(); + + public abstract IntPtr AddressOfPinnedMemory(); + public abstract IByteBuffer Unwrap(); + public abstract bool IsDirect { get; } + public abstract int ReferenceCount { get; } public abstract IReferenceCounted Retain(); diff --git a/src/DotNetty.Buffers/AbstractByteBufferAllocator.cs b/src/DotNetty.Buffers/AbstractByteBufferAllocator.cs index 69c513326..d44e38172 100644 --- a/src/DotNetty.Buffers/AbstractByteBufferAllocator.cs +++ b/src/DotNetty.Buffers/AbstractByteBufferAllocator.cs @@ -4,6 +4,7 @@ namespace DotNetty.Buffers { using System; + using System.Runtime.CompilerServices; using DotNetty.Common; /// @@ -75,6 +76,7 @@ protected static CompositeByteBuffer ToLeakAwareBuffer(CompositeByteBuffer buf) return buf; } + readonly bool directByDefault; readonly IByteBuffer emptyBuffer; protected AbstractByteBufferAllocator() @@ -82,11 +84,19 @@ protected AbstractByteBufferAllocator() this.emptyBuffer = new EmptyByteBuffer(this); } - public IByteBuffer Buffer() => this.HeapBuffer(); + protected AbstractByteBufferAllocator(bool preferDirect) + { + this.directByDefault = preferDirect; + this.emptyBuffer = new EmptyByteBuffer(this); + } - public IByteBuffer Buffer(int initialCapacity) => this.HeapBuffer(initialCapacity); + public IByteBuffer Buffer() => this.directByDefault ? this.DirectBuffer() : this.HeapBuffer(); - public IByteBuffer Buffer(int initialCapacity, int maxCapacity) => this.HeapBuffer(initialCapacity, maxCapacity); + public IByteBuffer Buffer(int initialCapacity) => + this.directByDefault ? this.DirectBuffer(initialCapacity) : this.HeapBuffer(initialCapacity); + + public IByteBuffer Buffer(int initialCapacity, int maxCapacity) => + this.directByDefault ? this.DirectBuffer(initialCapacity, maxCapacity) : this.HeapBuffer(initialCapacity, maxCapacity); public IByteBuffer HeapBuffer() => this.HeapBuffer(DefaultInitialCapacity, DefaultMaxCapacity); @@ -103,29 +113,56 @@ public IByteBuffer HeapBuffer(int initialCapacity, int maxCapacity) return this.NewHeapBuffer(initialCapacity, maxCapacity); } - public CompositeByteBuffer CompositeBuffer() => this.CompositeHeapBuffer(); + public IByteBuffer DirectBuffer() => this.DirectBuffer(DefaultInitialCapacity, DefaultMaxCapacity); + + public IByteBuffer DirectBuffer(int initialCapacity) => this.DirectBuffer(initialCapacity, DefaultMaxCapacity); + + public IByteBuffer DirectBuffer(int initialCapacity, int maxCapacity) + { + if (initialCapacity == 0 && maxCapacity == 0) + { + return this.emptyBuffer; + } + Validate(initialCapacity, maxCapacity); + return this.NewDirectBuffer(initialCapacity, maxCapacity); + } + + public CompositeByteBuffer CompositeBuffer() => + this.directByDefault ? this.CompositeDirectBuffer() : this.CompositeHeapBuffer(); - public CompositeByteBuffer CompositeBuffer(int maxComponents) => this.CompositeHeapBuffer(maxComponents); + public CompositeByteBuffer CompositeBuffer(int maxComponents) => + this.directByDefault ? this.CompositeDirectBuffer(maxComponents) : this.CompositeHeapBuffer(maxComponents); public CompositeByteBuffer CompositeHeapBuffer() => this.CompositeHeapBuffer(DefaultMaxComponents); - public virtual CompositeByteBuffer CompositeHeapBuffer(int maxNumComponents) => ToLeakAwareBuffer(new CompositeByteBuffer(this, maxNumComponents)); + public virtual CompositeByteBuffer CompositeHeapBuffer(int maxNumComponents) => + ToLeakAwareBuffer(new CompositeByteBuffer(this, false, maxNumComponents)); + public CompositeByteBuffer CompositeDirectBuffer() => this.CompositeDirectBuffer(DefaultMaxComponents); + + public virtual CompositeByteBuffer CompositeDirectBuffer(int maxNumComponents) => + ToLeakAwareBuffer(new CompositeByteBuffer(this, true, maxNumComponents)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] static void Validate(int initialCapacity, int maxCapacity) { if (initialCapacity < 0) { - throw new ArgumentOutOfRangeException(nameof(initialCapacity), "initialCapacity must be greater than zero"); + ThrowHelper.ThrowArgumentOutOfRangeException(nameof(initialCapacity), "initialCapacity must be greater than zero"); } if (initialCapacity > maxCapacity) { - throw new ArgumentOutOfRangeException(nameof(initialCapacity), $"initialCapacity ({initialCapacity}) must be greater than maxCapacity ({maxCapacity})"); + ThrowHelper.ThrowArgumentOutOfRangeException(nameof(initialCapacity), $"initialCapacity ({initialCapacity}) must be greater than maxCapacity ({maxCapacity})"); } } protected abstract IByteBuffer NewHeapBuffer(int initialCapacity, int maxCapacity); + protected abstract IByteBuffer NewDirectBuffer(int initialCapacity, int maxCapacity); + + public abstract bool IsDirectBufferPooled { get; } + public int CalculateNewCapacity(int minNewCapacity, int maxCapacity) { if (minNewCapacity < 0) diff --git a/src/DotNetty.Buffers/AbstractPooledDerivedByteBuffer.cs b/src/DotNetty.Buffers/AbstractPooledDerivedByteBuffer.cs index 5991b6ed0..9f1824878 100644 --- a/src/DotNetty.Buffers/AbstractPooledDerivedByteBuffer.cs +++ b/src/DotNetty.Buffers/AbstractPooledDerivedByteBuffer.cs @@ -74,10 +74,14 @@ protected internal sealed override void Deallocate() public sealed override IByteBufferAllocator Allocator => this.Unwrap().Allocator; + public sealed override bool IsDirect => this.Unwrap().IsDirect; + public override bool HasArray => this.Unwrap().HasArray; public override byte[] Array => this.Unwrap().Array; + public override bool HasMemoryAddress => this.Unwrap().HasMemoryAddress; + public sealed override int IoBufferCount => this.Unwrap().IoBufferCount; public sealed override IByteBuffer RetainedSlice() diff --git a/src/DotNetty.Buffers/AbstractUnpooledSlicedByteBuffer.cs b/src/DotNetty.Buffers/AbstractUnpooledSlicedByteBuffer.cs index d6ea8df0c..0f304e874 100644 --- a/src/DotNetty.Buffers/AbstractUnpooledSlicedByteBuffer.cs +++ b/src/DotNetty.Buffers/AbstractUnpooledSlicedByteBuffer.cs @@ -5,6 +5,7 @@ namespace DotNetty.Buffers { using System; using System.IO; + using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using DotNetty.Common.Internal; @@ -45,6 +46,8 @@ protected AbstractUnpooledSlicedByteBuffer(IByteBuffer buffer, int index, int le public override IByteBufferAllocator Allocator => this.Unwrap().Allocator; + public override bool IsDirect => this.Unwrap().IsDirect; + public override IByteBuffer AdjustCapacity(int newCapacity) => throw new NotSupportedException("sliced buffer"); public override bool HasArray => this.Unwrap().HasArray; @@ -53,6 +56,20 @@ protected AbstractUnpooledSlicedByteBuffer(IByteBuffer buffer, int index, int le public override int ArrayOffset => this.Idx(this.Unwrap().ArrayOffset); + public override bool HasMemoryAddress => this.Unwrap().HasMemoryAddress; + + public override ref byte GetPinnableMemoryAddress() => ref Unsafe.Add(ref this.Unwrap().GetPinnableMemoryAddress(), this.adjustment); + + public override IntPtr AddressOfPinnedMemory() + { + IntPtr ptr = this.Unwrap().AddressOfPinnedMemory(); + if (ptr == IntPtr.Zero) + { + return ptr; + } + return ptr + this.adjustment; + } + public override byte GetByte(int index) { this.CheckIndex0(index, 1); @@ -275,7 +292,7 @@ public override ArraySegment[] GetIoBuffers(int index, int length) return this.Unwrap().GetIoBuffers(index + this.adjustment, length); } - public override int ForEachByte(int index, int length, ByteProcessor processor) + public override int ForEachByte(int index, int length, IByteProcessor processor) { this.CheckIndex0(index, length); int ret = this.Unwrap().ForEachByte(this.Idx(index), length, processor); @@ -289,7 +306,7 @@ public override int ForEachByte(int index, int length, ByteProcessor processor) } } - public override int ForEachByteDesc(int index, int length, ByteProcessor processor) + public override int ForEachByteDesc(int index, int length, IByteProcessor processor) { this.CheckIndex0(index, length); int ret = this.Unwrap().ForEachByteDesc(this.Idx(index), length, processor); diff --git a/src/DotNetty.Buffers/AdvancedLeakAwareByteBuffer.cs b/src/DotNetty.Buffers/AdvancedLeakAwareByteBuffer.cs index 2fcc62719..4bae93d78 100644 --- a/src/DotNetty.Buffers/AdvancedLeakAwareByteBuffer.cs +++ b/src/DotNetty.Buffers/AdvancedLeakAwareByteBuffer.cs @@ -534,25 +534,25 @@ public override int BytesBefore(int index, int length, byte value) return base.BytesBefore(index, length, value); } - public override int ForEachByte(ByteProcessor processor) + public override int ForEachByte(IByteProcessor processor) { RecordLeakNonRefCountingOperation(this.Leak); return base.ForEachByte(processor); } - public override int ForEachByte(int index, int length, ByteProcessor processor) + public override int ForEachByte(int index, int length, IByteProcessor processor) { RecordLeakNonRefCountingOperation(this.Leak); return base.ForEachByte(index, length, processor); } - public override int ForEachByteDesc(ByteProcessor processor) + public override int ForEachByteDesc(IByteProcessor processor) { RecordLeakNonRefCountingOperation(this.Leak); return base.ForEachByteDesc(processor); } - public override int ForEachByteDesc(int index, int length, ByteProcessor processor) + public override int ForEachByteDesc(int index, int length, IByteProcessor processor) { RecordLeakNonRefCountingOperation(this.Leak); return base.ForEachByteDesc(index, length, processor); diff --git a/src/DotNetty.Buffers/AdvancedLeakAwareCompositeByteBuffer.cs b/src/DotNetty.Buffers/AdvancedLeakAwareCompositeByteBuffer.cs index 2386e9e90..ec8354877 100644 --- a/src/DotNetty.Buffers/AdvancedLeakAwareCompositeByteBuffer.cs +++ b/src/DotNetty.Buffers/AdvancedLeakAwareCompositeByteBuffer.cs @@ -416,25 +416,25 @@ public override int BytesBefore(int index, int length, byte value) return base.BytesBefore(index, length, value); } - public override int ForEachByte(ByteProcessor processor) + public override int ForEachByte(IByteProcessor processor) { RecordLeakNonRefCountingOperation(this.Leak); return base.ForEachByte(processor); } - public override int ForEachByte(int index, int length, ByteProcessor processor) + public override int ForEachByte(int index, int length, IByteProcessor processor) { RecordLeakNonRefCountingOperation(this.Leak); return base.ForEachByte(index, length, processor); } - public override int ForEachByteDesc(ByteProcessor processor) + public override int ForEachByteDesc(IByteProcessor processor) { RecordLeakNonRefCountingOperation(this.Leak); return base.ForEachByteDesc(processor); } - public override int ForEachByteDesc(int index, int length, ByteProcessor processor) + public override int ForEachByteDesc(int index, int length, IByteProcessor processor) { RecordLeakNonRefCountingOperation(this.Leak); return base.ForEachByteDesc(index, length, processor); diff --git a/src/DotNetty.Buffers/ByteBufferUtil.cs b/src/DotNetty.Buffers/ByteBufferUtil.cs index ff64b5a79..988786ddd 100644 --- a/src/DotNetty.Buffers/ByteBufferUtil.cs +++ b/src/DotNetty.Buffers/ByteBufferUtil.cs @@ -281,7 +281,7 @@ static int FirstIndexOf(IByteBuffer buffer, int fromIndex, int toIndex, byte val return -1; } - return buffer.ForEachByte(fromIndex, toIndex - fromIndex, new ByteProcessor.IndexOfProcessor(value)); + return buffer.ForEachByte(fromIndex, toIndex - fromIndex, new IndexOfProcessor(value)); } static int LastIndexOf(IByteBuffer buffer, int fromIndex, int toIndex, byte value) @@ -292,7 +292,7 @@ static int LastIndexOf(IByteBuffer buffer, int fromIndex, int toIndex, byte valu return -1; } - return buffer.ForEachByteDesc(toIndex, fromIndex - toIndex, new ByteProcessor.IndexOfProcessor(value)); + return buffer.ForEachByteDesc(toIndex, fromIndex - toIndex, new IndexOfProcessor(value)); } /// @@ -667,9 +667,9 @@ public static bool IsText(IByteBuffer buf, int index, int length, Encoding encod static readonly FindNonAscii AsciiByteProcessor = new FindNonAscii(); - sealed class FindNonAscii : ByteProcessor + sealed class FindNonAscii : IByteProcessor { - public override bool Process(byte value) => value < 0x80; + public bool Process(byte value) => value < 0x80; } static bool IsAscii(IByteBuffer buf, int index, int length) => buf.ForEachByte(index, length, AsciiByteProcessor) == -1; diff --git a/src/DotNetty.Buffers/CompositeByteBuffer.cs b/src/DotNetty.Buffers/CompositeByteBuffer.cs index a7a8b4e0f..503c218a3 100644 --- a/src/DotNetty.Buffers/CompositeByteBuffer.cs +++ b/src/DotNetty.Buffers/CompositeByteBuffer.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +// ReSharper disable ConvertToAutoProperty namespace DotNetty.Buffers { using System; @@ -37,34 +38,40 @@ public ComponentEntry(IByteBuffer buffer) } static readonly ArraySegment EmptyNioBuffer = Unpooled.Empty.GetIoBuffer(); + + readonly IByteBufferAllocator allocator; + readonly bool direct; readonly List components; + readonly int maxNumComponents; bool freed; - public CompositeByteBuffer(IByteBufferAllocator allocator, int maxNumComponents) + public CompositeByteBuffer(IByteBufferAllocator allocator, bool direct, int maxNumComponents) : base(AbstractByteBufferAllocator.DefaultMaxCapacity) { Contract.Requires(allocator != null); Contract.Requires(maxNumComponents >= 2); - this.Allocator = allocator; - this.MaxNumComponents = maxNumComponents; + this.allocator = allocator; + this.direct = direct; + this.maxNumComponents = maxNumComponents; this.components = NewList(maxNumComponents); } - public CompositeByteBuffer(IByteBufferAllocator allocator, int maxNumComponents, params IByteBuffer[] buffers) - : this(allocator, maxNumComponents, buffers, 0, buffers.Length) + public CompositeByteBuffer(IByteBufferAllocator allocator, bool direct, int maxNumComponents, params IByteBuffer[] buffers) + : this(allocator, direct, maxNumComponents, buffers, 0, buffers.Length) { } - internal CompositeByteBuffer(IByteBufferAllocator allocator, int maxNumComponents, IByteBuffer[] buffers, int offset, int length) + internal CompositeByteBuffer(IByteBufferAllocator allocator, bool direct, int maxNumComponents, IByteBuffer[] buffers, int offset, int length) : base(AbstractByteBufferAllocator.DefaultMaxCapacity) { Contract.Requires(allocator != null); Contract.Requires(maxNumComponents >= 2); - this.Allocator = allocator; - this.MaxNumComponents = maxNumComponents; + this.allocator = allocator; + this.direct = direct; + this.maxNumComponents = maxNumComponents; this.components = NewList(maxNumComponents); this.AddComponents0(false, 0, buffers, offset, length); @@ -73,15 +80,15 @@ internal CompositeByteBuffer(IByteBufferAllocator allocator, int maxNumComponent } public CompositeByteBuffer( - IByteBufferAllocator allocator, int maxNumComponents, IEnumerable buffers) + IByteBufferAllocator allocator, bool direct, int maxNumComponents, IEnumerable buffers) : base(AbstractByteBufferAllocator.DefaultMaxCapacity) { Contract.Requires(allocator != null); Contract.Requires(maxNumComponents >= 2); - this.Allocator = allocator; - this.MaxNumComponents = maxNumComponents; - + this.allocator = allocator; + this.direct = direct; + this.maxNumComponents = maxNumComponents; this.components = NewList(maxNumComponents); this.AddComponents0(false, 0, buffers); @@ -95,8 +102,9 @@ static List NewList(int maxNumComponents) => // Special constructor used by WrappedCompositeByteBuf internal CompositeByteBuffer(IByteBufferAllocator allocator) : base(int.MaxValue) { - this.Allocator = allocator; - this.MaxNumComponents = 0; + this.allocator = allocator; + this.direct = false; + this.maxNumComponents = 0; this.components = new List(0); } @@ -579,6 +587,27 @@ public override ArraySegment[] GetIoBuffers(int index, int length) return buffers.ToArray(); } + + public override bool IsDirect + { + get + { + int size = this.components.Count; + if (size == 0) + { + return false; + } + for (int i = 0; i < size; i++) + { + if (!this.components[i].Buffer.IsDirect) + { + return false; + } + } + return true; + } + } + public override bool HasArray { get @@ -627,6 +656,42 @@ public override int ArrayOffset } } + public override bool HasMemoryAddress + { + get + { + switch (this.components.Count) + { + case 1: + return this.components[0].Buffer.HasMemoryAddress; + default: + return false; + } + } + } + + public override ref byte GetPinnableMemoryAddress() + { + switch (this.components.Count) + { + case 1: + return ref this.components[0].Buffer.GetPinnableMemoryAddress(); + default: + throw new NotSupportedException(); + } + } + + public override IntPtr AddressOfPinnedMemory() + { + switch (this.components.Count) + { + case 1: + return this.components[0].Buffer.AddressOfPinnedMemory(); + default: + throw new NotSupportedException(); + } + } + public override int Capacity => GetCapacity(this.components); static int GetCapacity(List components) @@ -699,7 +764,7 @@ public override IByteBuffer AdjustCapacity(int newCapacity) return this; } - public override IByteBufferAllocator Allocator { get; } + public override IByteBufferAllocator Allocator => this.allocator; /// /// Return the current number of {@link IByteBuffer}'s that are composed in this instance @@ -709,7 +774,7 @@ public override IByteBuffer AdjustCapacity(int newCapacity) /// /// Return the max number of {@link IByteBuffer}'s that are composed in this instance /// - public virtual int MaxNumComponents { get; } + public virtual int MaxNumComponents => this.maxNumComponents; /// /// Return the index for the given offset @@ -1396,7 +1461,8 @@ public override IByteBuffer DiscardReadBytes() return this; } - IByteBuffer AllocateBuffer(int capacity) => this.Allocator.Buffer(capacity); + IByteBuffer AllocateBuffer(int capacity) => + this.direct ? this.Allocator.DirectBuffer(capacity) : this.Allocator.HeapBuffer(capacity); public override string ToString() { diff --git a/src/DotNetty.Buffers/DotNetty.Buffers.csproj b/src/DotNetty.Buffers/DotNetty.Buffers.csproj index aef168352..c3af3deeb 100644 --- a/src/DotNetty.Buffers/DotNetty.Buffers.csproj +++ b/src/DotNetty.Buffers/DotNetty.Buffers.csproj @@ -39,4 +39,9 @@ + + + C:\Users\Johnny\.nuget\packages\system.threading.tasks.extensions\4.3.0\lib\netstandard1.0\System.Threading.Tasks.Extensions.dll + + \ No newline at end of file diff --git a/src/DotNetty.Buffers/EmptyByteBuffer.cs b/src/DotNetty.Buffers/EmptyByteBuffer.cs index 1629636e9..1797ea0fa 100644 --- a/src/DotNetty.Buffers/EmptyByteBuffer.cs +++ b/src/DotNetty.Buffers/EmptyByteBuffer.cs @@ -38,6 +38,8 @@ public EmptyByteBuffer(IByteBufferAllocator allocator) public IByteBuffer Unwrap() => null; + public bool IsDirect => true; + public int ReaderIndex => 0; public IByteBuffer SetReaderIndex(int readerIndex) => this.CheckIndex(readerIndex); @@ -453,110 +455,47 @@ public double ReadDoubleLE() public IByteBuffer SkipBytes(int length) => this.CheckLength(length); - public IByteBuffer WriteBoolean(bool value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteBoolean(bool value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteByte(int value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteByte(int value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteShort(int value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteShort(int value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteShortLE(int value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteShortLE(int value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteUnsignedShort(ushort value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteUnsignedShort(ushort value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteUnsignedShortLE(ushort value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteUnsignedShortLE(ushort value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteMedium(int value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteMedium(int value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteMediumLE(int value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteMediumLE(int value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteUnsignedMedium(int value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteUnsignedMedium(int value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteUnsignedMediumLE(int value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteUnsignedMediumLE(int value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteInt(int value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteInt(int value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteIntLE(int value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteIntLE(int value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteUnsignedInt(uint value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteUnsignedInt(uint value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteUnsignedIntLE(uint value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteUnsignedIntLE(uint value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteLong(long value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteLong(long value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteLongLE(long value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteLongLE(long value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteChar(char value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteChar(char value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteFloat(float value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteFloat(float value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteFloatLE(float value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteFloatLE(float value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteDouble(double value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteDouble(double value) => throw new IndexOutOfRangeException(); - public IByteBuffer WriteDoubleLE(double value) - { - throw new IndexOutOfRangeException(); - } + public IByteBuffer WriteDoubleLE(double value) => throw new IndexOutOfRangeException(); public IByteBuffer WriteBytes(IByteBuffer src) => this.CheckLength(src.ReadableBytes); @@ -591,17 +530,17 @@ public int BytesBefore(int index, int length, byte value) return -1; } - public int ForEachByte(ByteProcessor processor) => -1; + public int ForEachByte(IByteProcessor processor) => -1; - public int ForEachByte(int index, int length, ByteProcessor processor) + public int ForEachByte(int index, int length, IByteProcessor processor) { this.CheckIndex(index, length); return -1; } - public int ForEachByteDesc(ByteProcessor processor) => -1; + public int ForEachByteDesc(IByteProcessor processor) => -1; - public int ForEachByteDesc(int index, int length, ByteProcessor processor) + public int ForEachByteDesc(int index, int length, IByteProcessor processor) { this.CheckIndex(index, length); return -1; @@ -651,6 +590,12 @@ public ArraySegment[] GetIoBuffers(int index, int length) public int ArrayOffset => 0; + public bool HasMemoryAddress => false; + + public ref byte GetPinnableMemoryAddress() => throw new NotSupportedException(); + + public IntPtr AddressOfPinnedMemory() => IntPtr.Zero; + public string ToString(Encoding encoding) => string.Empty; public string ToString(int index, int length, Encoding encoding) @@ -709,7 +654,7 @@ public Task WriteBytesAsync(Stream stream, int length, CancellationToken cancell return TaskEx.Completed; } - // ReSharper disable UnusedParameter.Local + // ReSharper disable ParameterOnlyUsedForPreconditionCheck.Local IByteBuffer CheckIndex(int index) { if (index != 0) @@ -732,7 +677,7 @@ IByteBuffer CheckIndex(int index, int length) return this; } - // ReSharper restore UnusedParameter.Local + // ReSharper restore ParameterOnlyUsedForPreconditionCheck.Local IByteBuffer CheckLength(int length) { diff --git a/src/DotNetty.Buffers/HeapByteBufferUtil.cs b/src/DotNetty.Buffers/HeapByteBufferUtil.cs index 5cba800b6..25f019718 100644 --- a/src/DotNetty.Buffers/HeapByteBufferUtil.cs +++ b/src/DotNetty.Buffers/HeapByteBufferUtil.cs @@ -16,9 +16,7 @@ internal static short GetShort(byte[] memory, int index) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static short GetShortLE(byte[] memory, int index) => - unchecked( - (short)(memory[index] | - memory[index + 1] << 8)); + unchecked((short)(memory[index] | memory[index + 1] << 8)); [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int GetUnsignedMedium(byte[] memory, int index) => diff --git a/src/DotNetty.Buffers/IByteBuffer.cs b/src/DotNetty.Buffers/IByteBuffer.cs index 55e5838ae..75beb1ea6 100644 --- a/src/DotNetty.Buffers/IByteBuffer.cs +++ b/src/DotNetty.Buffers/IByteBuffer.cs @@ -35,6 +35,8 @@ public interface IByteBuffer : IReferenceCounted, IComparable, IEqu /// IByteBufferAllocator Allocator { get; } + bool IsDirect { get; } + int ReaderIndex { get; } int WriterIndex { get; } @@ -1077,9 +1079,26 @@ public interface IByteBuffer : IReferenceCounted, IComparable, IEqu /// /// Grabs the underlying byte array for this buffer /// - /// byte[] Array { get; } + /// + /// Returns {@code true} if and only if this buffer has a reference to the low-level memory address that points + /// to the backing data. + /// + bool HasMemoryAddress { get; } + + /// + /// Returns the low-level memory address that point to the first byte of ths backing data. + /// + /// The low-level memory address + ref byte GetPinnableMemoryAddress(); + + /// + /// Returns the pointer address of the buffer if the memory is pinned. + /// + /// IntPtr.Zero if not pinned. + IntPtr AddressOfPinnedMemory(); + /// /// Creates a deep clone of the existing byte array and returns it /// @@ -1141,10 +1160,10 @@ public interface IByteBuffer : IReferenceCounted, IComparable, IEqu /// /// /// -1 if the processor iterated to or beyond the end of the readable bytes. - /// The last-visited index If the returned false. + /// The last-visited index If the returned false. /// /// Processor. - int ForEachByte(ByteProcessor processor); + int ForEachByte(IByteProcessor processor); /// /// Iterates over the specified area of this buffer with the specified in ascending order. @@ -1152,22 +1171,22 @@ public interface IByteBuffer : IReferenceCounted, IComparable, IEqu /// /// /// -1 if the processor iterated to or beyond the end of the specified area. - /// The last-visited index If the returned false. + /// The last-visited index If the returned false. /// /// Index. /// Length. /// Processor. - int ForEachByte(int index, int length, ByteProcessor processor); + int ForEachByte(int index, int length, IByteProcessor processor); /// /// Iterates over the readable bytes of this buffer with the specified in descending order. /// /// /// -1 if the processor iterated to or beyond the beginning of the readable bytes. - /// The last-visited index If the returned false. + /// The last-visited index If the returned false. /// /// Processor. - int ForEachByteDesc(ByteProcessor processor); + int ForEachByteDesc(IByteProcessor processor); /// /// Iterates over the specified area of this buffer with the specified in descending order. @@ -1175,11 +1194,11 @@ public interface IByteBuffer : IReferenceCounted, IComparable, IEqu /// /// /// -1 if the processor iterated to or beyond the beginning of the specified area. - /// The last-visited index If the returned false. + /// The last-visited index If the returned false. /// /// Index. /// Length. /// Processor. - int ForEachByteDesc(int index, int length, ByteProcessor processor); + int ForEachByteDesc(int index, int length, IByteProcessor processor); } } \ No newline at end of file diff --git a/src/DotNetty.Buffers/IByteBufferAllocator.cs b/src/DotNetty.Buffers/IByteBufferAllocator.cs index 0c814fdfd..548106b2c 100644 --- a/src/DotNetty.Buffers/IByteBufferAllocator.cs +++ b/src/DotNetty.Buffers/IByteBufferAllocator.cs @@ -20,6 +20,12 @@ public interface IByteBufferAllocator IByteBuffer HeapBuffer(int initialCapacity, int maxCapacity); + IByteBuffer DirectBuffer(); + + IByteBuffer DirectBuffer(int initialCapacity); + + IByteBuffer DirectBuffer(int initialCapacity, int maxCapacity); + CompositeByteBuffer CompositeBuffer(); CompositeByteBuffer CompositeBuffer(int maxComponents); @@ -28,6 +34,12 @@ public interface IByteBufferAllocator CompositeByteBuffer CompositeHeapBuffer(int maxComponents); + CompositeByteBuffer CompositeDirectBuffer(); + + CompositeByteBuffer CompositeDirectBuffer(int maxComponents); + + bool IsDirectBufferPooled { get; } + int CalculateNewCapacity(int minNewCapacity, int maxCapacity); } } \ No newline at end of file diff --git a/src/DotNetty.Buffers/IByteBufferAllocatorMetric .cs b/src/DotNetty.Buffers/IByteBufferAllocatorMetric .cs index f83026931..57abe9d4c 100644 --- a/src/DotNetty.Buffers/IByteBufferAllocatorMetric .cs +++ b/src/DotNetty.Buffers/IByteBufferAllocatorMetric .cs @@ -6,8 +6,13 @@ namespace DotNetty.Buffers public interface IByteBufferAllocatorMetric { /// - /// Returns the number of bytes of heap memory used by a {@link ByteBufAllocator} or {@code -1} if unknown. + /// Returns the number of bytes of heap memory used by a {@link ByteBufAllocator} or {@code -1} if unknown. /// long UsedHeapMemory { get; } + + /// + /// Returns the number of bytes of direct memory used by a {@link ByteBufAllocator} or {@code -1} if unknown. + /// + long UsedDirectMemory { get; } } } diff --git a/src/DotNetty.Buffers/PoolArena.cs b/src/DotNetty.Buffers/PoolArena.cs index 544045779..72cc6d813 100644 --- a/src/DotNetty.Buffers/PoolArena.cs +++ b/src/DotNetty.Buffers/PoolArena.cs @@ -7,6 +7,8 @@ namespace DotNetty.Buffers using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.Contracts; + using System.Runtime.CompilerServices; + using System.Runtime.InteropServices; using System.Text; using System.Threading; using DotNetty.Common.Internal; @@ -126,6 +128,8 @@ PoolSubpage NewSubpagePoolHead(int pageSize) PoolSubpage[] NewSubpagePoolArray(int size) => new PoolSubpage[size]; + internal abstract bool IsDirect { get; } + internal PooledByteBuffer Allocate(PoolThreadCache cache, int reqCapacity, int maxCapacity) { PooledByteBuffer buf = this.NewByteBuf(maxCapacity); @@ -474,7 +478,7 @@ static List SubPageMetricList(PoolSubpage[] pages) continue; } PoolSubpage s = head.Next; - for (; ;) + for (; ; ) { metrics.Add(s); s = s.Next; @@ -537,7 +541,7 @@ public long NumActiveAllocations { long val = this.NumTinyAllocations + this.NumSmallAllocations + this.NumHugeAllocations - this.NumHugeDeallocations; - lock(this) + lock (this) { val += this.allocationsNormal - (this.deallocationsTiny + this.deallocationsSmall + this.deallocationsNormal); } @@ -644,7 +648,7 @@ static void AppendPoolSubPages(StringBuilder buf, PoolSubpage[] subpages) .Append(i) .Append(": "); PoolSubpage s = head.Next; - for (;;) + for (; ;) { buf.Append(s); s = s.Next; @@ -689,10 +693,12 @@ public HeapArena(PooledByteBufferAllocator parent, int pageSize, int maxOrder, i static byte[] NewByteArray(int size) => new byte[size]; - protected override PoolChunk NewChunk(int pageSize, int maxOrder, int pageShifts, int chunkSize) => + internal override bool IsDirect => false; + + protected override PoolChunk NewChunk(int pageSize, int maxOrder, int pageShifts, int chunkSize) => new PoolChunk(this, NewByteArray(chunkSize), pageSize, maxOrder, pageShifts, chunkSize, 0); - protected override PoolChunk NewUnpooledChunk(int capacity) => + protected override PoolChunk NewUnpooledChunk(int capacity) => new PoolChunk(this, NewByteArray(capacity), capacity, 0); protected internal override void DestroyChunk(PoolChunk chunk) @@ -700,7 +706,8 @@ protected internal override void DestroyChunk(PoolChunk chunk) // Rely on GC. } - protected override PooledByteBuffer NewByteBuf(int maxCapacity) => PooledHeapByteBuffer.NewInstance(maxCapacity); + protected override PooledByteBuffer NewByteBuf(int maxCapacity) => + PooledHeapByteBuffer.NewInstance(maxCapacity); protected override void MemoryCopy(byte[] src, int srcOffset, byte[] dst, int dstOffset, int length) { @@ -712,4 +719,95 @@ protected override void MemoryCopy(byte[] src, int srcOffset, byte[] dst, int ds PlatformDependent.CopyMemory(src, srcOffset, dst, dstOffset, length); } } + + //TODO: Maybe use Memory or OwnedMemory as direct arena/byte buffer type parameter in NETStandard 2.0 + sealed class DirectArena : PoolArena + { + readonly List memoryChunks; + + public DirectArena(PooledByteBufferAllocator parent, int pageSize, int maxOrder, int pageShifts, int chunkSize) + : base(parent, pageSize, maxOrder, pageShifts, chunkSize) + { + this.memoryChunks = new List(); + } + + static MemoryChunk NewMemoryChunk(int size) => new MemoryChunk(size); + + internal override bool IsDirect => true; + + protected override PoolChunk NewChunk(int pageSize, int maxOrder, int pageShifts, int chunkSize) + { + MemoryChunk memoryChunk = NewMemoryChunk(chunkSize); + this.memoryChunks.Add(memoryChunk); + var chunk = new PoolChunk(this, memoryChunk.Bytes, pageSize, maxOrder, pageShifts, chunkSize, 0); + return chunk; + } + + protected override PoolChunk NewUnpooledChunk(int capacity) + { + MemoryChunk memoryChunk = NewMemoryChunk(capacity); + this.memoryChunks.Add(memoryChunk); + var chunk = new PoolChunk(this, memoryChunk.Bytes, capacity, 0); + return chunk; + } + + protected override PooledByteBuffer NewByteBuf(int maxCapacity) => + PooledUnsafeDirectByteBuffer.NewInstance(maxCapacity); + + protected override unsafe void MemoryCopy(byte[] src, int srcOffset, byte[] dst, int dstOffset, int length) => + PlatformDependent.CopyMemory((byte*)Unsafe.AsPointer(ref src[srcOffset]), (byte*)Unsafe.AsPointer(ref dst[dstOffset]), length); + + protected internal override void DestroyChunk(PoolChunk chunk) + { + for (int i = 0; i < this.memoryChunks.Count; i++) + { + MemoryChunk memoryChunk = this.memoryChunks[i]; + if (ReferenceEquals(chunk.Memory, memoryChunk.Bytes)) + { + this.memoryChunks.Remove(memoryChunk); + memoryChunk.Dispose(); + break; + } + } + } + + sealed class MemoryChunk : IDisposable + { + internal byte[] Bytes; + GCHandle handle; + + internal MemoryChunk(int size) + { + this.Bytes = new byte[size]; + this.handle = GCHandle.Alloc(this.Bytes, GCHandleType.Pinned); + } + + void Release() + { + if (this.handle.IsAllocated) + { + try + { + this.handle.Free(); + } + catch (InvalidOperationException) + { + // Free is not thread safe + } + } + this.Bytes = null; + } + + public void Dispose() + { + this.Release(); + GC.SuppressFinalize(this); + } + + ~MemoryChunk() + { + this.Release(); + } + } + } } \ No newline at end of file diff --git a/src/DotNetty.Buffers/PoolThreadCache.cs b/src/DotNetty.Buffers/PoolThreadCache.cs index 386968bdf..5c8e1cc3d 100644 --- a/src/DotNetty.Buffers/PoolThreadCache.cs +++ b/src/DotNetty.Buffers/PoolThreadCache.cs @@ -28,13 +28,18 @@ sealed class PoolThreadCache static readonly IInternalLogger Logger = InternalLoggerFactory.GetInstance>(); internal readonly PoolArena HeapArena; + internal readonly PoolArena DirectArena; // Hold the caches for the different size classes, which are tiny, small and normal. readonly MemoryRegionCache[] tinySubPageHeapCaches; readonly MemoryRegionCache[] smallSubPageHeapCaches; + readonly MemoryRegionCache[] tinySubPageDirectCaches; + readonly MemoryRegionCache[] smallSubPageDirectCaches; readonly MemoryRegionCache[] normalHeapCaches; + readonly MemoryRegionCache[] normalDirectCaches; // Used for bitshifting when calculate the index of normal caches later + readonly int numShiftsNormalDirect; readonly int numShiftsNormalHeap; readonly int freeSweepAllocationThreshold; @@ -46,7 +51,7 @@ sealed class PoolThreadCache // TODO: Test if adding padding helps under contention //private long pad0, pad1, pad2, pad3, pad4, pad5, pad6, pad7; - internal PoolThreadCache(PoolArena heapArena, + internal PoolThreadCache(PoolArena heapArena, PoolArena directArena, int tinyCacheSize, int smallCacheSize, int normalCacheSize, int maxCachedBufferCapacity, int freeSweepAllocationThreshold) { @@ -55,6 +60,28 @@ internal PoolThreadCache(PoolArena heapArena, this.freeSweepAllocationThreshold = freeSweepAllocationThreshold; this.HeapArena = heapArena; + this.DirectArena = directArena; + if (directArena != null) + { + this.tinySubPageDirectCaches = CreateSubPageCaches( + tinyCacheSize, PoolArena.NumTinySubpagePools, SizeClass.Tiny); + this.smallSubPageDirectCaches = CreateSubPageCaches( + smallCacheSize, directArena.NumSmallSubpagePools, SizeClass.Small); + + this.numShiftsNormalDirect = Log2(directArena.PageSize); + this.normalDirectCaches = CreateNormalCaches( + normalCacheSize, maxCachedBufferCapacity, directArena); + + directArena.IncrementNumThreadCaches(); + } + else + { + // No directArea is configured so just null out all caches + this.tinySubPageDirectCaches = null; + this.smallSubPageDirectCaches = null; + this.normalDirectCaches = null; + this.numShiftsNormalDirect = -1; + } if (heapArena != null) { // Create the caches for the heap allocations @@ -79,7 +106,8 @@ internal PoolThreadCache(PoolArena heapArena, } // We only need to watch the thread when any cache is used. - if (this.tinySubPageHeapCaches != null || this.smallSubPageHeapCaches != null || this.normalHeapCaches != null) + if (this.tinySubPageDirectCaches != null || this.smallSubPageDirectCaches != null || this.normalDirectCaches != null + || this.tinySubPageHeapCaches != null || this.smallSubPageHeapCaches != null || this.normalHeapCaches != null) { this.freeTask = this.Free0; this.deathWatchThread = Thread.CurrentThread; @@ -150,19 +178,19 @@ static int Log2(int val) /** * Try to allocate a tiny buffer out of the cache. Returns {@code true} if successful {@code false} otherwise */ - internal bool AllocateTiny(PoolArena area, PooledByteBuffer buf, int reqCapacity, int normCapacity) => + internal bool AllocateTiny(PoolArena area, PooledByteBuffer buf, int reqCapacity, int normCapacity) => this.Allocate(this.CacheForTiny(area, normCapacity), buf, reqCapacity); /** * Try to allocate a small buffer out of the cache. Returns {@code true} if successful {@code false} otherwise */ - internal bool AllocateSmall(PoolArena area, PooledByteBuffer buf, int reqCapacity, int normCapacity) => + internal bool AllocateSmall(PoolArena area, PooledByteBuffer buf, int reqCapacity, int normCapacity) => this.Allocate(this.CacheForSmall(area, normCapacity), buf, reqCapacity); /** * Try to allocate a small buffer out of the cache. Returns {@code true} if successful {@code false} otherwise */ - internal bool AllocateNormal(PoolArena area, PooledByteBuffer buf, int reqCapacity, int normCapacity) => + internal bool AllocateNormal(PoolArena area, PooledByteBuffer buf, int reqCapacity, int normCapacity) => this.Allocate(this.CacheForNormal(area, normCapacity), buf, reqCapacity); bool Allocate(MemoryRegionCache cache, PooledByteBuffer buf, int reqCapacity) @@ -226,7 +254,10 @@ internal void Free() void Free0() { - int numFreed = Free(this.tinySubPageHeapCaches) + + int numFreed = Free(this.tinySubPageDirectCaches) + + Free(this.smallSubPageDirectCaches) + + Free(this.normalDirectCaches) + + Free(this.tinySubPageHeapCaches) + Free(this.smallSubPageHeapCaches) + Free(this.normalHeapCaches); @@ -235,6 +266,7 @@ void Free0() Logger.Debug("Freed {} thread-local buffer(s) from thread: {}", numFreed, this.deathWatchThread.Name); } + this.DirectArena?.DecrementNumThreadCaches(); this.HeapArena?.DecrementNumThreadCaches(); } @@ -264,6 +296,9 @@ static int Free(MemoryRegionCache cache) internal void Trim() { + Trim(this.tinySubPageDirectCaches); + Trim(this.smallSubPageDirectCaches); + Trim(this.normalDirectCaches); Trim(this.tinySubPageHeapCaches); Trim(this.smallSubPageHeapCaches); Trim(this.normalHeapCaches); @@ -286,17 +321,22 @@ static void Trim(MemoryRegionCache[] caches) MemoryRegionCache CacheForTiny(PoolArena area, int normCapacity) { int idx = PoolArena.TinyIdx(normCapacity); - return Cache(this.tinySubPageHeapCaches, idx); + return Cache(area.IsDirect ? this.tinySubPageDirectCaches : this.tinySubPageHeapCaches, idx); } MemoryRegionCache CacheForSmall(PoolArena area, int normCapacity) { int idx = PoolArena.SmallIdx(normCapacity); - return Cache(this.smallSubPageHeapCaches, idx); + return Cache(area.IsDirect ? this.smallSubPageDirectCaches : this.smallSubPageHeapCaches, idx); } MemoryRegionCache CacheForNormal(PoolArena area, int normCapacity) { + if (area.IsDirect) + { + int idx = Log2(normCapacity >> this.numShiftsNormalDirect); + return Cache(this.normalDirectCaches, idx); + } int idx1 = Log2(normCapacity >> this.numShiftsNormalHeap); return Cache(this.normalHeapCaches, idx1); } @@ -321,7 +361,7 @@ internal SubPageMemoryRegionCache(int size, SizeClass sizeClass) } protected override void InitBuf( - PoolChunk chunk, long handle, PooledByteBuffer buf, int reqCapacity) => + PoolChunk chunk, long handle, PooledByteBuffer buf, int reqCapacity) => chunk.InitBufWithSubpage(buf, handle, reqCapacity); } @@ -336,7 +376,7 @@ internal NormalMemoryRegionCache(int size) } protected override void InitBuf( - PoolChunk chunk, long handle, PooledByteBuffer buf, int reqCapacity) => + PoolChunk chunk, long handle, PooledByteBuffer buf, int reqCapacity) => chunk.InitBuf(buf, handle, reqCapacity); } diff --git a/src/DotNetty.Buffers/PooledByteBuffer.cs b/src/DotNetty.Buffers/PooledByteBuffer.cs index 70c7adc21..9c8e19124 100644 --- a/src/DotNetty.Buffers/PooledByteBuffer.cs +++ b/src/DotNetty.Buffers/PooledByteBuffer.cs @@ -5,6 +5,7 @@ namespace DotNetty.Buffers { using System; using System.Diagnostics; + using System.Runtime.CompilerServices; using DotNetty.Common; using DotNetty.Common.Utilities; @@ -27,10 +28,10 @@ protected PooledByteBuffer(ThreadLocalPool.Handle recyclerHandle, int maxCapacit this.recyclerHandle = recyclerHandle; } - internal void Init(PoolChunk chunk, long handle, int offset, int length, int maxLength, PoolThreadCache cache) => + internal virtual void Init(PoolChunk chunk, long handle, int offset, int length, int maxLength, PoolThreadCache cache) => this.Init0(chunk, handle, offset, length, maxLength, cache); - internal void InitUnpooled(PoolChunk chunk, int length) => this.Init0(chunk, 0, 0, length, length, null); + internal virtual void InitUnpooled(PoolChunk chunk, int length) => this.Init0(chunk, 0, 0, length, length, null); void Init0(PoolChunk chunk, long handle, int offset, int length, int maxLength, PoolThreadCache cache) { @@ -144,6 +145,7 @@ protected internal sealed override void Deallocate() void Recycle() => this.recyclerHandle.Release(this); + [MethodImpl(MethodImplOptions.AggressiveInlining)] protected int Idx(int index) => this.Offset + index; } } \ No newline at end of file diff --git a/src/DotNetty.Buffers/PooledByteBufferAllocator.cs b/src/DotNetty.Buffers/PooledByteBufferAllocator.cs index ea6ad71e4..247e311bc 100644 --- a/src/DotNetty.Buffers/PooledByteBufferAllocator.cs +++ b/src/DotNetty.Buffers/PooledByteBufferAllocator.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +// ReSharper disable ConvertToAutoProperty +// ReSharper disable ConvertToAutoPropertyWhenPossible namespace DotNetty.Buffers { using System; @@ -17,6 +19,7 @@ public class PooledByteBufferAllocator : AbstractByteBufferAllocator, IByteBuffe static readonly IInternalLogger Logger = InternalLoggerFactory.GetInstance(); public static readonly int DefaultNumHeapArena; + public static readonly int DefaultNumDirectArena; public static readonly int DefaultPageSize; public static readonly int DefaultMaxOrder; // 8192 << 11 = 16 MiB per chunk @@ -67,6 +70,7 @@ static PooledByteBufferAllocator() // See https://github.com/netty/netty/issues/3888 int defaultMinNumArena = Environment.ProcessorCount * 2; DefaultNumHeapArena = Math.Max(0, SystemPropertyUtil.GetInt("io.netty.allocator.numHeapArenas", defaultMinNumArena)); + DefaultNumDirectArena = Math.Max(0, SystemPropertyUtil.GetInt("io.netty.allocator.numDirectArenas", defaultMinNumArena)); // cache sizes DefaultTinyCacheSize = SystemPropertyUtil.GetInt("io.netty.allocator.tinyCacheSize", 512); @@ -84,6 +88,7 @@ static PooledByteBufferAllocator() if (Logger.DebugEnabled) { Logger.Debug("-Dio.netty.allocator.numHeapArenas: {}", DefaultNumHeapArena); + Logger.Debug("-Dio.netty.allocator.numDirectArenas: {}", DefaultNumDirectArena); if (pageSizeFallbackCause == null) { Logger.Debug("-Dio.netty.allocator.pageSize: {}", DefaultPageSize); @@ -108,35 +113,48 @@ static PooledByteBufferAllocator() Logger.Debug("-Dio.netty.allocator.cacheTrimInterval: {}", DefaultCacheTrimInterval); } - Default = new PooledByteBufferAllocator(); + Default = new PooledByteBufferAllocator(PlatformDependent.DirectBufferPreferred); } public static readonly PooledByteBufferAllocator Default; readonly PoolArena[] heapArenas; + readonly PoolArena[] directArenas; readonly int tinyCacheSize; readonly int smallCacheSize; readonly int normalCacheSize; readonly IReadOnlyList heapArenaMetrics; + readonly IReadOnlyList directArenaMetrics; readonly PoolThreadLocalCache threadCache; readonly int chunkSize; readonly PooledByteBufferAllocatorMetric metric; - public PooledByteBufferAllocator() - : this(DefaultNumHeapArena, DefaultPageSize, DefaultMaxOrder) + public PooledByteBufferAllocator() : this(false) { } - public PooledByteBufferAllocator(int nHeapArena, int pageSize, int maxOrder) - : this(nHeapArena, pageSize, maxOrder, + public PooledByteBufferAllocator(bool preferDirect) + : this(preferDirect, DefaultNumHeapArena, DefaultNumDirectArena, DefaultPageSize, DefaultMaxOrder) + { + } + + public PooledByteBufferAllocator(int nHeapArena, int nDirectArena, int pageSize, int maxOrder) + : this(false, nHeapArena, nDirectArena, pageSize, maxOrder) + { + } + + public PooledByteBufferAllocator(bool preferDirect, int nHeapArena, int nDirectArena, int pageSize, int maxOrder) + : this(preferDirect, nHeapArena, nDirectArena, pageSize, maxOrder, DefaultTinyCacheSize, DefaultSmallCacheSize, DefaultNormalCacheSize) { } - public PooledByteBufferAllocator(int nHeapArena, int pageSize, int maxOrder, + public PooledByteBufferAllocator(bool preferDirect, int nHeapArena, int nDirectArena, int pageSize, int maxOrder, int tinyCacheSize, int smallCacheSize, int normalCacheSize) + : base(preferDirect) { Contract.Requires(nHeapArena >= 0); + Contract.Requires(nDirectArena >= 0); this.threadCache = new PoolThreadLocalCache(this); this.tinyCacheSize = tinyCacheSize; @@ -164,6 +182,24 @@ public PooledByteBufferAllocator(int nHeapArena, int pageSize, int maxOrder, this.heapArenaMetrics = new IPoolArenaMetric[0]; } + if (nDirectArena > 0) + { + this.directArenas = NewArenaArray(nDirectArena); + var metrics = new List(this.directArenas.Length); + for (int i = 0; i < this.directArenas.Length; i++) + { + var arena = new DirectArena(this, pageSize, maxOrder, pageShifts, this.chunkSize); + this.directArenas[i] = arena; + metrics.Add(arena); + } + this.directArenaMetrics = metrics.AsReadOnly(); + } + else + { + this.directArenas = null; + this.directArenaMetrics = new IPoolArenaMetric[0]; + } + this.metric = new PooledByteBufferAllocatorMetric(this); } @@ -175,7 +211,7 @@ static int ValidateAndCalculatePageShifts(int pageSize) Contract.Requires((pageSize & pageSize - 1) == 0, "Expected power of 2"); // Logarithm base 2. At this point we know that pageSize is a power of two. - return (sizeof(int) * 8 - 1) - pageSize.NumberOfLeadingZeros(); + return (sizeof(int) * 8 - 1) - pageSize.NumberOfLeadingZeros(); } static int ValidateAndCalculateChunkSize(int pageSize, int maxOrder) @@ -213,6 +249,28 @@ protected override IByteBuffer NewHeapBuffer(int initialCapacity, int maxCapacit return ToLeakAwareBuffer(buf); } + protected override IByteBuffer NewDirectBuffer(int initialCapacity, int maxCapacity) + { + PoolThreadCache cache = this.threadCache.Value; + PoolArena directArena = cache.DirectArena; + + IByteBuffer buf; + if (directArena != null) + { + buf = directArena.Allocate(cache, initialCapacity, maxCapacity); + } + else + { + buf = UnsafeByteBufferUtil.NewUnsafeDirectByteBuffer(this, initialCapacity, maxCapacity); + } + + return ToLeakAwareBuffer(buf); + } + + public static bool DefaultPreferDirect => PlatformDependent.DirectBufferPreferred; + + public override bool IsDirectBufferPooled => this.heapArenas != null; + sealed class PoolThreadLocalCache : FastThreadLocal> { readonly PooledByteBufferAllocator owner; @@ -227,8 +285,11 @@ protected override PoolThreadCache GetInitialValue() lock (this) { PoolArena heapArena = this.LeastUsedArena(this.owner.heapArenas); - return new PoolThreadCache( - heapArena, this.owner.tinyCacheSize, this.owner.smallCacheSize, this.owner.normalCacheSize, + PoolArena directArena = this.LeastUsedArena(this.owner.directArenas); + + return new PoolThreadCache( + heapArena, directArena, + this.owner.tinyCacheSize, this.owner.smallCacheSize, this.owner.normalCacheSize, DefaultMaxCachedBufferCapacity, DefaultCacheTrimInterval); } } @@ -258,7 +319,8 @@ PoolArena LeastUsedArena(PoolArena[] arenas) internal IReadOnlyList HeapArenas() => this.heapArenaMetrics; - // ReSharper disable ConvertToAutoPropertyWhenPossible + internal IReadOnlyList DirectArenas() => this.directArenaMetrics; + internal int TinyCacheSize => this.tinyCacheSize; internal int SmallCacheSize => this.smallCacheSize; @@ -266,16 +328,15 @@ PoolArena LeastUsedArena(PoolArena[] arenas) internal int NormalCacheSize => this.normalCacheSize; internal int ChunkSize => this.chunkSize; - // ReSharper restore ConvertToAutoPropertyWhenPossible - // ReSharper disable ConvertToAutoProperty public PooledByteBufferAllocatorMetric Metric => this.metric; - // ReSharper restore ConvertToAutoProperty IByteBufferAllocatorMetric IByteBufferAllocatorMetricProvider.Metric => this.Metric; - + internal long UsedHeapMemory => UsedMemory(this.heapArenas); + internal long UsedDirectMemory => UsedMemory(this.directArenas); + static long UsedMemory(PoolArena[] arenas) { if (arenas == null) @@ -315,6 +376,19 @@ public string DumpStats() } } + int directArenasLen = this.directArenas?.Length ?? 0; + buf.Append(directArenasLen) + .Append(" direct arena(s):") + .Append(StringUtil.Newline); + if (directArenasLen > 0) + { + // ReSharper disable once PossibleNullReferenceException + foreach (PoolArena a in this.directArenas) + { + buf.Append(a); + } + } + return buf.ToString(); } } diff --git a/src/DotNetty.Buffers/PooledByteBufferAllocatorMetric.cs b/src/DotNetty.Buffers/PooledByteBufferAllocatorMetric.cs index 76167ad4e..03e7a99bc 100644 --- a/src/DotNetty.Buffers/PooledByteBufferAllocatorMetric.cs +++ b/src/DotNetty.Buffers/PooledByteBufferAllocatorMetric.cs @@ -18,6 +18,8 @@ internal PooledByteBufferAllocatorMetric(PooledByteBufferAllocator allocator) public IReadOnlyList HeapArenas() => this.allocator.HeapArenas(); + public IReadOnlyList DirectArenas() => this.allocator.DirectArenas(); + public int TinyCacheSize => this.allocator.TinyCacheSize; public int SmallCacheSize => this.allocator.SmallCacheSize; @@ -28,18 +30,27 @@ internal PooledByteBufferAllocatorMetric(PooledByteBufferAllocator allocator) public long UsedHeapMemory => this.allocator.UsedHeapMemory; + public long UsedDirectMemory => this.allocator.UsedDirectMemory; + public int NumThreadLocalCaches() { + int total = 0; IReadOnlyList arenas = this.HeapArenas(); - if (arenas == null) + if (arenas != null) { - return 0; + foreach (IPoolArenaMetric metric in arenas) + { + total += metric.NumThreadCaches; + } } - int total = 0; - foreach (IPoolArenaMetric metric in arenas) + arenas = this.DirectArenas(); + if (arenas != null) { - total += metric.NumThreadCaches; + foreach (IPoolArenaMetric metric in arenas) + { + total += metric.NumThreadCaches; + } } return total; @@ -50,7 +61,9 @@ public override string ToString() var sb = new StringBuilder(256); sb.Append(StringUtil.SimpleClassName(this)) .Append("(usedHeapMemory: ").Append(this.UsedHeapMemory) + .Append("; usedDirectMemory: ").Append(this.UsedDirectMemory) .Append("; numHeapArenas: ").Append(this.HeapArenas().Count) + .Append("; numDirectArenas: ").Append(this.DirectArenas().Count) .Append("; tinyCacheSize: ").Append(this.TinyCacheSize) .Append("; smallCacheSize: ").Append(this.SmallCacheSize) .Append("; normalCacheSize: ").Append(this.NormalCacheSize) diff --git a/src/DotNetty.Buffers/PooledDuplicatedByteBuffer.cs b/src/DotNetty.Buffers/PooledDuplicatedByteBuffer.cs index 5acb94762..d3d51e45a 100644 --- a/src/DotNetty.Buffers/PooledDuplicatedByteBuffer.cs +++ b/src/DotNetty.Buffers/PooledDuplicatedByteBuffer.cs @@ -38,6 +38,10 @@ public override IByteBuffer AdjustCapacity(int newCapacity) public override int ArrayOffset => this.Unwrap().ArrayOffset; + public override ref byte GetPinnableMemoryAddress() => ref this.Unwrap().GetPinnableMemoryAddress(); + + public override IntPtr AddressOfPinnedMemory() => this.Unwrap().AddressOfPinnedMemory(); + public override ArraySegment GetIoBuffer(int index, int length) => this.Unwrap().GetIoBuffer(index, length); public override ArraySegment[] GetIoBuffers(int index, int length) => this.Unwrap().GetIoBuffers(index, length); diff --git a/src/DotNetty.Buffers/PooledHeapByteBuffer.cs b/src/DotNetty.Buffers/PooledHeapByteBuffer.cs index ced464a0e..35558a1ee 100644 --- a/src/DotNetty.Buffers/PooledHeapByteBuffer.cs +++ b/src/DotNetty.Buffers/PooledHeapByteBuffer.cs @@ -26,6 +26,8 @@ internal PooledHeapByteBuffer(ThreadLocalPool.Handle recyclerHandle, int maxCapa { } + public override bool IsDirect => false; + protected internal override byte _GetByte(int index) => HeapByteBufferUtil.GetByte(this.Memory, this.Idx(index)); protected internal override short _GetShort(int index) => HeapByteBufferUtil.GetShort(this.Memory, this.Idx(index)); @@ -165,5 +167,15 @@ public override byte[] Array } public override int ArrayOffset => this.Offset; + + public override bool HasMemoryAddress => true; + + public override ref byte GetPinnableMemoryAddress() + { + this.EnsureAccessible(); + return ref this.Memory[this.Offset]; + } + + public override IntPtr AddressOfPinnedMemory() => IntPtr.Zero; } } \ No newline at end of file diff --git a/src/DotNetty.Buffers/PooledSlicedByteBuffer.cs b/src/DotNetty.Buffers/PooledSlicedByteBuffer.cs index 8453e1aac..3a58227d7 100644 --- a/src/DotNetty.Buffers/PooledSlicedByteBuffer.cs +++ b/src/DotNetty.Buffers/PooledSlicedByteBuffer.cs @@ -5,6 +5,7 @@ namespace DotNetty.Buffers { using System; using System.IO; + using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using DotNetty.Common; @@ -44,6 +45,18 @@ static PooledSlicedByteBuffer NewInstance0(AbstractByteBuffer unwrapped, IByteBu public override int ArrayOffset => this.Idx(this.Unwrap().ArrayOffset); + public override ref byte GetPinnableMemoryAddress() => ref Unsafe.Add(ref this.Unwrap().GetPinnableMemoryAddress(), this.adjustment); + + public override IntPtr AddressOfPinnedMemory() + { + IntPtr ptr = this.Unwrap().AddressOfPinnedMemory(); + if (ptr == IntPtr.Zero) + { + return ptr; + } + return ptr + this.adjustment; + } + public override ArraySegment GetIoBuffer(int index, int length) { this.CheckIndex0(index, length); @@ -271,7 +284,7 @@ public override Task SetBytesAsync(int index, Stream src, int length, Cance return this.Unwrap().SetBytesAsync(this.Idx(index), src, length, cancellationToken); } - public override int ForEachByte(int index, int length, ByteProcessor processor) + public override int ForEachByte(int index, int length, IByteProcessor processor) { this.CheckIndex0(index, length); int ret = this.Unwrap().ForEachByte(this.Idx(index), length, processor); @@ -282,7 +295,7 @@ public override int ForEachByte(int index, int length, ByteProcessor processor) return ret - this.adjustment; } - public override int ForEachByteDesc(int index, int length, ByteProcessor processor) + public override int ForEachByteDesc(int index, int length, IByteProcessor processor) { this.CheckIndex0(index, length); int ret = this.Unwrap().ForEachByteDesc(this.Idx(index), length, processor); diff --git a/src/DotNetty.Buffers/PooledUnsafeDirectByteBuffer.cs b/src/DotNetty.Buffers/PooledUnsafeDirectByteBuffer.cs new file mode 100644 index 000000000..8eb636a64 --- /dev/null +++ b/src/DotNetty.Buffers/PooledUnsafeDirectByteBuffer.cs @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace DotNetty.Buffers +{ + using System; + using System.IO; + using System.Runtime.CompilerServices; + using System.Threading; + using System.Threading.Tasks; + using DotNetty.Common; + + sealed unsafe class PooledUnsafeDirectByteBuffer : PooledByteBuffer + { + static readonly ThreadLocalPool Recycler = new ThreadLocalPool(handle => new PooledUnsafeDirectByteBuffer(handle, 0)); + + byte* memoryAddress; + + internal static PooledUnsafeDirectByteBuffer NewInstance(int maxCapacity) + { + PooledUnsafeDirectByteBuffer buf = Recycler.Take(); + buf.Reuse(maxCapacity); + return buf; + } + + PooledUnsafeDirectByteBuffer(ThreadLocalPool.Handle recyclerHandle, int maxCapacity) + : base(recyclerHandle, maxCapacity) + { + } + + internal override void Init(PoolChunk chunk, long handle, int offset, int length, int maxLength, + PoolThreadCache cache) + { + base.Init(chunk, handle, offset, length, maxLength, cache); + this.InitMemoryAddress(); + } + + internal override void InitUnpooled(PoolChunk chunk, int length) + { + base.InitUnpooled(chunk, length); + this.InitMemoryAddress(); + } + + void InitMemoryAddress() + { + this.memoryAddress = (byte*)Unsafe.AsPointer(ref this.Memory[this.Offset]); + } + + public override bool IsDirect => true; + + protected internal override byte _GetByte(int index) => *(this.memoryAddress + index); + + protected internal override short _GetShort(int index) => UnsafeByteBufferUtil.GetShort(this.Addr(index)); + + protected internal override short _GetShortLE(int index) => UnsafeByteBufferUtil.GetShortLE(this.Addr(index)); + + protected internal override int _GetUnsignedMedium(int index) => UnsafeByteBufferUtil.GetUnsignedMedium(this.Addr(index)); + + protected internal override int _GetUnsignedMediumLE(int index) => UnsafeByteBufferUtil.GetUnsignedMediumLE(this.Addr(index)); + + protected internal override int _GetInt(int index) => UnsafeByteBufferUtil.GetInt(this.Addr(index)); + + protected internal override int _GetIntLE(int index) => UnsafeByteBufferUtil.GetIntLE(this.Addr(index)); + + protected internal override long _GetLong(int index) => UnsafeByteBufferUtil.GetLong(this.Addr(index)); + + protected internal override long _GetLongLE(int index) => UnsafeByteBufferUtil.GetLongLE(this.Addr(index)); + + public override IByteBuffer GetBytes(int index, IByteBuffer dst, int dstIndex, int length) + { + this.CheckIndex(index, length); + UnsafeByteBufferUtil.GetBytes(this, this.Addr(index), index, dst, dstIndex, length); + return this; + } + + public override IByteBuffer GetBytes(int index, byte[] dst, int dstIndex, int length) + { + this.CheckIndex(index, length); + UnsafeByteBufferUtil.GetBytes(this, this.Addr(index), index, dst, dstIndex, length); + return this; + } + + public override IByteBuffer GetBytes(int index, Stream output, int length) + { + UnsafeByteBufferUtil.GetBytes(this, this.Addr(index), index, output, length); + return this; + } + + protected internal override void _SetByte(int index, int value) => *(this.memoryAddress + index) = unchecked((byte)value); + + protected internal override void _SetShort(int index, int value) => UnsafeByteBufferUtil.SetShort(this.Addr(index), value); + + protected internal override void _SetShortLE(int index, int value) => UnsafeByteBufferUtil.SetShortLE(this.Addr(index), value); + + protected internal override void _SetMedium(int index, int value) => UnsafeByteBufferUtil.SetMedium(this.Addr(index), value); + + protected internal override void _SetMediumLE(int index, int value) => UnsafeByteBufferUtil.SetMediumLE(this.Addr(index), value); + + protected internal override void _SetInt(int index, int value) => UnsafeByteBufferUtil.SetInt(this.Addr(index), value); + + protected internal override void _SetIntLE(int index, int value) => UnsafeByteBufferUtil.SetIntLE(this.Addr(index), value); + + protected internal override void _SetLong(int index, long value) => UnsafeByteBufferUtil.SetLong(this.Addr(index), value); + + protected internal override void _SetLongLE(int index, long value) => UnsafeByteBufferUtil.SetLongLE(this.Addr(index), value); + + public override IByteBuffer SetBytes(int index, IByteBuffer src, int srcIndex, int length) + { + this.CheckIndex(index, length); + UnsafeByteBufferUtil.SetBytes(this, this.Addr(index), index, src, srcIndex, length); + return this; + } + + public override IByteBuffer SetBytes(int index, byte[] src, int srcIndex, int length) + { + this.CheckIndex(index, length); + UnsafeByteBufferUtil.SetBytes(this, this.Addr(index), index, src, srcIndex, length); + return this; + } + + public override Task SetBytesAsync(int index, Stream src, int length, CancellationToken cancellationToken) + { + this.CheckIndex(index, length); + int read = UnsafeByteBufferUtil.SetBytes(this, this.Addr(index), index, src, length); + return Task.FromResult(read); + } + + public override IByteBuffer Copy(int index, int length) + { + this.CheckIndex(index, length); + return UnsafeByteBufferUtil.Copy(this, this.Addr(index), index, length); + } + + public override int IoBufferCount => 1; + + public override ArraySegment GetIoBuffer(int index, int length) + { + this.CheckIndex(index, length); + index = this.Idx(index); + return new ArraySegment(this.Memory, index, length); + } + + public override ArraySegment[] GetIoBuffers(int index, int length) => new[] { this.GetIoBuffer(index, length) }; + + public override bool HasArray => true; + + public override byte[] Array + { + get + { + this.EnsureAccessible(); + return this.Memory; + } + } + + public override int ArrayOffset => this.Offset; + + public override bool HasMemoryAddress => true; + + public override ref byte GetPinnableMemoryAddress() + { + this.EnsureAccessible(); + return ref this.Memory[this.Offset]; + } + + public override IntPtr AddressOfPinnedMemory() => (IntPtr)this.memoryAddress; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + byte* Addr(int index) => this.memoryAddress + index; + + public override IByteBuffer SetZero(int index, int length) + { + this.CheckIndex(index, length); + UnsafeByteBufferUtil.SetZero(this.Addr(index), length); + return this; + } + + public override IByteBuffer WriteZero(int length) + { + this.EnsureWritable(length); + int wIndex = this.WriterIndex; + UnsafeByteBufferUtil.SetZero(this.Addr(wIndex), length); + this.SetWriterIndex(wIndex + length); + return this; + } + } +} diff --git a/src/DotNetty.Buffers/ThrowHelper.cs b/src/DotNetty.Buffers/ThrowHelper.cs index 013361cd7..e7f75d173 100644 --- a/src/DotNetty.Buffers/ThrowHelper.cs +++ b/src/DotNetty.Buffers/ThrowHelper.cs @@ -10,5 +10,9 @@ static class ThrowHelper public static void ThrowIndexOutOfRangeException(string message) => throw new IndexOutOfRangeException(message); public static void ThrowIllegalReferenceCountException(int count = 0) => throw new IllegalReferenceCountException(count); + + public static void ThrowArgumentNullException(string message) => throw new ArgumentNullException(message); + + public static void ThrowArgumentOutOfRangeException(string name, string message) => throw new ArgumentOutOfRangeException(name, message); } } diff --git a/src/DotNetty.Buffers/Unpooled.cs b/src/DotNetty.Buffers/Unpooled.cs index f070adbfc..d832dc764 100644 --- a/src/DotNetty.Buffers/Unpooled.cs +++ b/src/DotNetty.Buffers/Unpooled.cs @@ -20,11 +20,18 @@ public static class Unpooled public static IByteBuffer Buffer() => Allocator.HeapBuffer(); + public static IByteBuffer DirectBuffer() => Allocator.DirectBuffer(); + public static IByteBuffer Buffer(int initialCapacity) => Allocator.HeapBuffer(initialCapacity); + public static IByteBuffer DirectBuffer(int initialCapacity) => Allocator.DirectBuffer(initialCapacity); + public static IByteBuffer Buffer(int initialCapacity, int maxCapacity) => Allocator.HeapBuffer(initialCapacity, maxCapacity); + public static IByteBuffer DirectBuffer(int initialCapacity, int maxCapacity) => + Allocator.DirectBuffer(initialCapacity, maxCapacity); + /// /// Creates a new big-endian buffer which wraps the specified array. /// A modification on the specified array's content will be visible to the returned buffer. @@ -118,7 +125,7 @@ public static IByteBuffer WrappedBuffer(int maxNumComponents, params byte[][] ar if (components.Count > 0) { - return new CompositeByteBuffer(Allocator, maxNumComponents, components); + return new CompositeByteBuffer(Allocator, false, maxNumComponents, components); } break; } @@ -151,7 +158,7 @@ public static IByteBuffer WrappedBuffer(int maxNumComponents, params IByteBuffer { IByteBuffer buf = buffers[i]; if (buf.IsReadable()) - return new CompositeByteBuffer(Allocator, maxNumComponents, buffers, i, buffers.Length); + return new CompositeByteBuffer(Allocator, false, maxNumComponents, buffers, i, buffers.Length); else buf.Release(); } @@ -163,7 +170,7 @@ public static IByteBuffer WrappedBuffer(int maxNumComponents, params IByteBuffer public static CompositeByteBuffer CompositeBuffer() => CompositeBuffer(AbstractByteBufferAllocator.DefaultMaxComponents); - public static CompositeByteBuffer CompositeBuffer(int maxNumComponents) => new CompositeByteBuffer(Allocator, maxNumComponents); + public static CompositeByteBuffer CompositeBuffer(int maxNumComponents) => new CompositeByteBuffer(Allocator, false, maxNumComponents); /// /// Creates a new big-endian buffer whose content is a copy of the specified array diff --git a/src/DotNetty.Buffers/UnpooledByteBufferAllocator.cs b/src/DotNetty.Buffers/UnpooledByteBufferAllocator.cs index 25221d2b8..df12782ac 100644 --- a/src/DotNetty.Buffers/UnpooledByteBufferAllocator.cs +++ b/src/DotNetty.Buffers/UnpooledByteBufferAllocator.cs @@ -4,6 +4,7 @@ namespace DotNetty.Buffers { using System.Threading; + using DotNetty.Common.Internal; using DotNetty.Common.Utilities; /// @@ -14,9 +15,21 @@ public sealed class UnpooledByteBufferAllocator : AbstractByteBufferAllocator, I readonly UnpooledByteBufferAllocatorMetric metric = new UnpooledByteBufferAllocatorMetric(); readonly bool disableLeakDetector; - public static readonly UnpooledByteBufferAllocator Default = new UnpooledByteBufferAllocator(); + public static readonly UnpooledByteBufferAllocator Default = + new UnpooledByteBufferAllocator(PlatformDependent.DirectBufferPreferred); - public UnpooledByteBufferAllocator(bool disableLeakDetector = false) + public UnpooledByteBufferAllocator() + : this(PlatformDependent.DirectBufferPreferred, false) + { + } + + public UnpooledByteBufferAllocator(bool preferDirect) + : this(preferDirect, false) + { + } + + public UnpooledByteBufferAllocator(bool preferDirect, bool disableLeakDetector) + : base(preferDirect) { this.disableLeakDetector = disableLeakDetector; } @@ -24,16 +37,34 @@ public UnpooledByteBufferAllocator(bool disableLeakDetector = false) protected override IByteBuffer NewHeapBuffer(int initialCapacity, int maxCapacity) => new InstrumentedUnpooledHeapByteBuffer(this, initialCapacity, maxCapacity); + protected override IByteBuffer NewDirectBuffer(int initialCapacity, int maxCapacity) + { + IByteBuffer buf = new InstrumentedUnpooledUnsafeDirectByteBuffer(this, initialCapacity, maxCapacity); + return this.disableLeakDetector ? buf : ToLeakAwareBuffer(buf); + } + public override CompositeByteBuffer CompositeHeapBuffer(int maxNumComponents) { - var buf = new CompositeByteBuffer(this, maxNumComponents); + var buf = new CompositeByteBuffer(this, false, maxNumComponents); + return this.disableLeakDetector ? buf : ToLeakAwareBuffer(buf); + } + + public override CompositeByteBuffer CompositeDirectBuffer(int maxNumComponents) + { + var buf = new CompositeByteBuffer(this, true, maxNumComponents); return this.disableLeakDetector ? buf : ToLeakAwareBuffer(buf); } + public override bool IsDirectBufferPooled => false; + public IByteBufferAllocatorMetric Metric => this.metric; + internal void IncrementDirect(int amount) => this.metric.DirectCounter(amount); + + internal void DecrementDirect(int amount) => this.metric.DirectCounter(-amount); + internal void IncrementHeap(int amount) => this.metric.HeapCounter(amount); - + internal void DecrementHeap(int amount) => this.metric.HeapCounter(-amount); sealed class InstrumentedUnpooledHeapByteBuffer : UnpooledHeapByteBuffer @@ -60,15 +91,44 @@ protected override void FreeArray(byte[] bytes) } } + sealed class InstrumentedUnpooledUnsafeDirectByteBuffer : UnpooledUnsafeDirectByteBuffer + { + internal InstrumentedUnpooledUnsafeDirectByteBuffer( + UnpooledByteBufferAllocator alloc, int initialCapacity, int maxCapacity) + : base(alloc, initialCapacity, maxCapacity) + { + ((UnpooledByteBufferAllocator)this.Allocator).IncrementDirect(initialCapacity); + } + + protected override byte[] AllocateDirect(int initialCapacity) + { + byte[] bytes = base.AllocateDirect(initialCapacity); + ((UnpooledByteBufferAllocator)this.Allocator).IncrementDirect(bytes.Length); + return bytes; + } + + protected override void FreeDirect(byte[] array) + { + int capacity = array.Length; + base.FreeDirect(array); + ((UnpooledByteBufferAllocator)this.Allocator).DecrementDirect(capacity); + } + } + sealed class UnpooledByteBufferAllocatorMetric : IByteBufferAllocatorMetric { long usedHeapMemory; + long userDirectMemory; public long UsedHeapMemory => Volatile.Read(ref this.usedHeapMemory); + public long UsedDirectMemory => Volatile.Read(ref this.userDirectMemory); + public void HeapCounter(int amount) => Interlocked.Add(ref this.usedHeapMemory, amount); - public override string ToString() => $"{StringUtil.SimpleClassName(this)} (usedHeapMemory: {this.UsedHeapMemory})"; + public void DirectCounter(int amount) => Interlocked.Add(ref this.userDirectMemory, amount); + + public override string ToString() => $"{StringUtil.SimpleClassName(this)} (usedHeapMemory: {this.UsedHeapMemory}; usedDirectMemory: {this.UsedDirectMemory})"; } } } \ No newline at end of file diff --git a/src/DotNetty.Buffers/UnpooledDuplicatedByteBuffer.cs b/src/DotNetty.Buffers/UnpooledDuplicatedByteBuffer.cs index a93cb533d..9495200a2 100644 --- a/src/DotNetty.Buffers/UnpooledDuplicatedByteBuffer.cs +++ b/src/DotNetty.Buffers/UnpooledDuplicatedByteBuffer.cs @@ -3,6 +3,7 @@ namespace DotNetty.Buffers { + using System; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -45,6 +46,8 @@ internal UnpooledDuplicatedByteBuffer(AbstractByteBuffer buffer, int readerIndex public override IByteBufferAllocator Allocator => this.Unwrap().Allocator; + public override bool IsDirect => this.Unwrap().IsDirect; + public override int Capacity => this.Unwrap().Capacity; public override IByteBuffer AdjustCapacity(int newCapacity) => this.Unwrap().AdjustCapacity(newCapacity); @@ -57,6 +60,12 @@ internal UnpooledDuplicatedByteBuffer(AbstractByteBuffer buffer, int readerIndex public override int ArrayOffset => this.Unwrap().ArrayOffset; + public override bool HasMemoryAddress => this.Unwrap().HasMemoryAddress; + + public override ref byte GetPinnableMemoryAddress() => ref this.Unwrap().GetPinnableMemoryAddress(); + + public override IntPtr AddressOfPinnedMemory() => this.Unwrap().AddressOfPinnedMemory(); + protected internal override byte _GetByte(int index) => this.UnwrapCore()._GetByte(index); protected internal override short _GetShort(int index) => this.UnwrapCore()._GetShort(index); @@ -105,8 +114,8 @@ internal UnpooledDuplicatedByteBuffer(AbstractByteBuffer buffer, int readerIndex protected internal override void _SetLongLE(int index, long value) => this.UnwrapCore()._SetLongLE(index, value); - public override int ForEachByte(int index, int length, ByteProcessor processor) => this.Unwrap().ForEachByte(index, length, processor); + public override int ForEachByte(int index, int length, IByteProcessor processor) => this.Unwrap().ForEachByte(index, length, processor); - public override int ForEachByteDesc(int index, int length, ByteProcessor processor) => this.Unwrap().ForEachByteDesc(index, length, processor); + public override int ForEachByteDesc(int index, int length, IByteProcessor processor) => this.Unwrap().ForEachByteDesc(index, length, processor); } } diff --git a/src/DotNetty.Buffers/UnpooledHeapByteBuffer.cs b/src/DotNetty.Buffers/UnpooledHeapByteBuffer.cs index 9cbcd9bb2..917e4f9a8 100644 --- a/src/DotNetty.Buffers/UnpooledHeapByteBuffer.cs +++ b/src/DotNetty.Buffers/UnpooledHeapByteBuffer.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +// ReSharper disable ConvertToAutoProperty namespace DotNetty.Buffers { using System; @@ -12,6 +13,7 @@ namespace DotNetty.Buffers public class UnpooledHeapByteBuffer : AbstractReferenceCountedByteBuffer { + readonly IByteBufferAllocator allocator; byte[] array; protected internal UnpooledHeapByteBuffer(IByteBufferAllocator alloc, int initialCapacity, int maxCapacity) @@ -20,7 +22,7 @@ protected internal UnpooledHeapByteBuffer(IByteBufferAllocator alloc, int initia Contract.Requires(alloc != null); Contract.Requires(initialCapacity <= maxCapacity); - this.Allocator = alloc; + this.allocator = alloc; this.SetArray(this.NewArray(initialCapacity)); this.SetIndex0(0, 0); } @@ -36,7 +38,7 @@ protected internal UnpooledHeapByteBuffer(IByteBufferAllocator alloc, byte[] ini throw new ArgumentException($"initialCapacity({initialArray.Length}) > maxCapacity({maxCapacity})"); } - this.Allocator = alloc; + this.allocator = alloc; this.SetArray(initialArray); this.SetIndex0(0, initialArray.Length); } @@ -52,7 +54,9 @@ protected virtual void FreeArray(byte[] bytes) protected void SetArray(byte[] initialArray) => this.array = initialArray; - public override IByteBufferAllocator Allocator { get; } + public override IByteBufferAllocator Allocator => this.allocator; + + public override bool IsDirect => false; public override int Capacity { @@ -72,7 +76,7 @@ public override IByteBuffer AdjustCapacity(int newCapacity) if (newCapacity > oldCapacity) { byte[] newArray = this.AllocateArray(newCapacity); - PlatformDependent.CopyMemory(this.array, 0, newArray, 0, this.array.Length); + PlatformDependent.CopyMemory(this.array, 0, newArray, 0, oldCapacity); this.SetArray(newArray); this.FreeArray(oldArray); @@ -115,6 +119,16 @@ public override byte[] Array public override int ArrayOffset => 0; + public override bool HasMemoryAddress => true; + + public override ref byte GetPinnableMemoryAddress() + { + this.EnsureAccessible(); + return ref this.array[0]; + } + + public override IntPtr AddressOfPinnedMemory() => IntPtr.Zero; + public override IByteBuffer GetBytes(int index, IByteBuffer dst, int dstIndex, int length) { this.CheckDstIndex(index, length, dstIndex, dst.Capacity); diff --git a/src/DotNetty.Buffers/UnpooledUnsafeDirectByteBuffer.cs b/src/DotNetty.Buffers/UnpooledUnsafeDirectByteBuffer.cs new file mode 100644 index 000000000..27bb7c88d --- /dev/null +++ b/src/DotNetty.Buffers/UnpooledUnsafeDirectByteBuffer.cs @@ -0,0 +1,359 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// ReSharper disable ConvertToAutoProperty +namespace DotNetty.Buffers +{ + using System; + using System.Diagnostics.Contracts; + using System.IO; + using System.Runtime.CompilerServices; + using System.Threading; + using System.Threading.Tasks; + using DotNetty.Common.Internal; + + public unsafe class UnpooledUnsafeDirectByteBuffer : AbstractReferenceCountedByteBuffer + { + readonly IByteBufferAllocator allocator; + + int capacity; + bool doNotFree; + byte[] buffer; + + public UnpooledUnsafeDirectByteBuffer(IByteBufferAllocator alloc, int initialCapacity, int maxCapacity) + : base(maxCapacity) + { + Contract.Requires(alloc != null); + Contract.Requires(initialCapacity >= 0); + Contract.Requires(maxCapacity >= 0); + + if (initialCapacity > maxCapacity) + { + throw new ArgumentException($"initialCapacity({initialCapacity}) > maxCapacity({maxCapacity})"); + } + + this.allocator = alloc; + this.SetByteBuffer(this.NewArray(initialCapacity), false); + } + + protected UnpooledUnsafeDirectByteBuffer(IByteBufferAllocator alloc, byte[] initialBuffer, int maxCapacity, bool doFree) + : base(maxCapacity) + { + Contract.Requires(alloc != null); + Contract.Requires(initialBuffer != null); + + int initialCapacity = initialBuffer.Length; + if (initialCapacity > maxCapacity) + { + throw new ArgumentException($"initialCapacity({initialCapacity}) > maxCapacity({maxCapacity})"); + } + + this.allocator = alloc; + this.doNotFree = !doFree; + this.SetByteBuffer(initialBuffer, false); + } + + protected virtual byte[] AllocateDirect(int initialCapacity) => this.NewArray(initialCapacity); + + protected byte[] NewArray(int initialCapacity) => new byte[initialCapacity]; + + protected virtual void FreeDirect(byte[] array) + { + // NOOP rely on GC. + } + + void SetByteBuffer(byte[] array, bool tryFree) + { + if (tryFree) + { + byte[] oldBuffer = this.buffer; + if (oldBuffer != null) + { + if (this.doNotFree) + { + this.doNotFree = false; + } + else + { + this.FreeDirect(oldBuffer); + } + } + } + this.buffer = array; + this.capacity = array.Length; + } + + public override bool IsDirect => true; + + public override int Capacity => this.capacity; + + public override IByteBuffer AdjustCapacity(int newCapacity) + { + this.CheckNewCapacity(newCapacity); + + int rIdx = this.ReaderIndex; + int wIdx = this.WriterIndex; + + int oldCapacity = this.capacity; + if (newCapacity > oldCapacity) + { + byte[] oldBuffer = this.buffer; + byte[] newBuffer = this.AllocateDirect(newCapacity); + PlatformDependent.CopyMemory(oldBuffer, 0, newBuffer, 0, oldCapacity); + this.SetByteBuffer(newBuffer, true); + } + else if (newCapacity < oldCapacity) + { + byte[] oldBuffer = this.buffer; + byte[] newBuffer = this.AllocateDirect(newCapacity); + if (rIdx < newCapacity) + { + if (wIdx > newCapacity) + { + this.SetWriterIndex(wIdx = newCapacity); + } + PlatformDependent.CopyMemory(oldBuffer, rIdx, newBuffer, 0, wIdx - rIdx); + } + else + { + this.SetIndex(newCapacity, newCapacity); + } + this.SetByteBuffer(newBuffer, true); + } + return this; + } + + public override IByteBufferAllocator Allocator => this.allocator; + + public override bool HasArray => true; + + public override byte[] Array + { + get + { + this.EnsureAccessible(); + return this.buffer; + } + } + + public override int ArrayOffset => 0; + + public override bool HasMemoryAddress => true; + + public override ref byte GetPinnableMemoryAddress() + { + this.EnsureAccessible(); + return ref this.buffer[0]; + } + + public override IntPtr AddressOfPinnedMemory() => IntPtr.Zero; + + protected internal override byte _GetByte(int index) => this.buffer[index]; + + protected internal override short _GetShort(int index) + { + fixed (byte* addr = &this.Addr(index)) + return UnsafeByteBufferUtil.GetShort(addr); + } + + protected internal override short _GetShortLE(int index) + { + fixed (byte* addr = &this.Addr(index)) + return UnsafeByteBufferUtil.GetShortLE(addr); + } + + protected internal override int _GetUnsignedMedium(int index) + { + fixed (byte* addr = &this.Addr(index)) + return UnsafeByteBufferUtil.GetUnsignedMedium(addr); + } + + protected internal override int _GetUnsignedMediumLE(int index) + { + fixed (byte* addr = &this.Addr(index)) + return UnsafeByteBufferUtil.GetUnsignedMediumLE(addr); + } + + protected internal override int _GetInt(int index) + { + fixed (byte* addr = &this.Addr(index)) + return UnsafeByteBufferUtil.GetInt(addr); + } + + protected internal override int _GetIntLE(int index) + { + fixed (byte* addr = &this.Addr(index)) + return UnsafeByteBufferUtil.GetIntLE(addr); + } + + protected internal override long _GetLong(int index) + { + fixed (byte* addr = &this.Addr(index)) + return UnsafeByteBufferUtil.GetLong(addr); + } + + protected internal override long _GetLongLE(int index) + { + fixed (byte* addr = &this.Addr(index)) + return UnsafeByteBufferUtil.GetLongLE(addr); + } + + public override IByteBuffer GetBytes(int index, IByteBuffer dst, int dstIndex, int length) + { + this.CheckIndex(index, length); + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.GetBytes(this, addr, index, dst, dstIndex, length); + return this; + } + + public override IByteBuffer GetBytes(int index, byte[] dst, int dstIndex, int length) + { + this.CheckIndex(index, length); + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.GetBytes(this, addr, index, dst, dstIndex, length); + return this; + } + + protected internal override void _SetByte(int index, int value) => this.buffer[index] = unchecked((byte)value); + + protected internal override void _SetShort(int index, int value) + { + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetShort(addr, value); + } + + protected internal override void _SetShortLE(int index, int value) + { + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetShortLE(addr, value); + } + + protected internal override void _SetMedium(int index, int value) + { + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetMedium(addr, value); + } + + protected internal override void _SetMediumLE(int index, int value) + { + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetMediumLE(addr, value); + } + + protected internal override void _SetInt(int index, int value) + { + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetInt(addr, value); + } + + protected internal override void _SetIntLE(int index, int value) + { + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetIntLE(addr, value); + } + + protected internal override void _SetLong(int index, long value) + { + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetLong(addr, value); + } + + protected internal override void _SetLongLE(int index, long value) + { + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetLongLE(addr, value); + } + + public override IByteBuffer SetBytes(int index, IByteBuffer src, int srcIndex, int length) + { + this.CheckIndex(index, length); + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetBytes(this, addr, index, src, srcIndex, length); + return this; + } + + public override IByteBuffer SetBytes(int index, byte[] src, int srcIndex, int length) + { + this.CheckIndex(index, length); + if (length != 0) + { + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetBytes(this, addr, index, src, srcIndex, length); + } + return this; + } + + public override IByteBuffer GetBytes(int index, Stream output, int length) + { + this.CheckIndex(index, length); + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.GetBytes(this, addr, index, output, length); + return this; + } + + public override Task SetBytesAsync(int index, Stream src, int length, CancellationToken cancellationToken) + { + this.CheckIndex(index, length); + int read; + fixed (byte* addr = &this.Addr(index)) + read = UnsafeByteBufferUtil.SetBytes(this, addr, index, src, length); + return Task.FromResult(read); + } + + public override int IoBufferCount => 1; + + public override ArraySegment GetIoBuffer(int index, int length) + { + this.CheckIndex(index, length); + return new ArraySegment(this.buffer, index, length); + } + + public override ArraySegment[] GetIoBuffers(int index, int length) => new[] { this.GetIoBuffer(index, length) }; + + public override IByteBuffer Copy(int index, int length) + { + this.CheckIndex(index, length); + fixed (byte* addr = &this.Addr(index)) + return UnsafeByteBufferUtil.Copy(this, addr, index, length); + } + + protected internal override void Deallocate() + { + byte[] buf = this.buffer; + if (buf == null) + { + return; + } + + this.buffer = null; + + if (!this.doNotFree) + { + this.FreeDirect(buf); + } + } + + public override IByteBuffer Unwrap() => null; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + ref byte Addr(int index) => ref this.buffer[index]; + + public override IByteBuffer SetZero(int index, int length) + { + this.CheckIndex(index, length); + fixed (byte* addr = &this.Addr(index)) + UnsafeByteBufferUtil.SetZero(addr, length); + return this; + } + + public override IByteBuffer WriteZero(int length) + { + this.EnsureWritable(length); + int wIndex = this.WriterIndex; + fixed (byte* addr = &this.Addr(wIndex)) + UnsafeByteBufferUtil.SetZero(addr, length); + this.SetWriterIndex(wIndex + length); + return this; + } + } +} diff --git a/src/DotNetty.Buffers/UnsafeByteBufferUtil .cs b/src/DotNetty.Buffers/UnsafeByteBufferUtil .cs new file mode 100644 index 000000000..c876459d5 --- /dev/null +++ b/src/DotNetty.Buffers/UnsafeByteBufferUtil .cs @@ -0,0 +1,332 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace DotNetty.Buffers +{ + using System; + using System.Diagnostics.Contracts; + using System.IO; + using System.Runtime.CompilerServices; + using DotNetty.Common.Internal; + + static unsafe class UnsafeByteBufferUtil + { + const byte Zero = 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static short GetShort(byte* bytes) => + unchecked((short)(((*bytes) << 8) | *(bytes + 1))); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static short GetShortLE(byte* bytes) => + unchecked((short)((*bytes) | (*(bytes + 1) << 8))); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int GetUnsignedMedium(byte* bytes) => + *bytes << 16 | + *(bytes + 1) << 8 | + *(bytes + 2); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int GetUnsignedMediumLE(byte* bytes) => + *bytes | + *(bytes + 1) << 8 | + *(bytes + 2) << 16; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int GetInt(byte* bytes) => + (*bytes << 24) | + (*(bytes + 1) << 16) | + (*(bytes + 2) << 8) | + (*(bytes + 3)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int GetIntLE(byte* bytes) => + *bytes | + (*(bytes + 1) << 8) | + (*(bytes + 2) << 16) | + (*(bytes + 3) << 24); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static long GetLong(byte* bytes) + { + unchecked + { + int i1 = (*bytes << 24) | (*(bytes + 1) << 16) | (*(bytes + 2) << 8) | (*(bytes + 3)); + int i2 = (*(bytes + 4) << 24) | (*(bytes + 5) << 16) | (*(bytes + 6) << 8) | *(bytes + 7); + return (uint)i2 | ((long)i1 << 32); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static long GetLongLE(byte* bytes) + { + unchecked + { + int i1 = *bytes | (*(bytes + 1) << 8) | (*(bytes + 2) << 16) | (*(bytes + 3) << 24); + int i2 = *(bytes + 4) | (*(bytes + 5) << 8) | (*(bytes + 6) << 16) | (*(bytes + 7) << 24); + return (uint)i1 | ((long)i2 << 32); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void SetShort(byte* bytes, int value) + { + unchecked + { + *bytes = (byte)((ushort)value >> 8); + *(bytes + 1) = (byte)value; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void SetShortLE(byte* bytes, int value) + { + unchecked + { + *bytes = (byte)value; + *(bytes + 1) = (byte)((ushort)value >> 8); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void SetMedium(byte* bytes, int value) + { + unchecked + { + uint unsignedValue = (uint)value; + *bytes = (byte)(unsignedValue >> 16); + *(bytes + 1) = (byte)(unsignedValue >> 8); + *(bytes + 2) = (byte)unsignedValue; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void SetMediumLE(byte* bytes, int value) + { + unchecked + { + uint unsignedValue = (uint)value; + *bytes = (byte)unsignedValue; + *(bytes + 1) = (byte)(unsignedValue >> 8); + *(bytes + 2) = (byte)(unsignedValue >> 16); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void SetInt(byte* bytes, int value) + { + unchecked + { + uint unsignedValue = (uint)value; + *bytes = (byte)(unsignedValue >> 24); + *(bytes + 1) = (byte)(unsignedValue >> 16); + *(bytes + 2) = (byte)(unsignedValue >> 8); + *(bytes + 3) = (byte)unsignedValue; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void SetIntLE(byte* bytes, int value) + { + unchecked + { + uint unsignedValue = (uint)value; + *bytes = (byte)unsignedValue; + *(bytes + 1) = (byte)(unsignedValue >> 8); + *(bytes + 2) = (byte)(unsignedValue >> 16); + *(bytes + 3) = (byte)(unsignedValue >> 24); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void SetLong(byte* bytes, long value) + { + unchecked + { + ulong unsignedValue = (ulong)value; + *bytes = (byte)(unsignedValue >> 56); + *(bytes + 1) = (byte)(unsignedValue >> 48); + *(bytes + 2) = (byte)(unsignedValue >> 40); + *(bytes + 3) = (byte)(unsignedValue >> 32); + *(bytes + 4) = (byte)(unsignedValue >> 24); + *(bytes + 5) = (byte)(unsignedValue >> 16); + *(bytes + 6) = (byte)(unsignedValue >> 8); + *(bytes + 7) = (byte)unsignedValue; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void SetLongLE(byte* bytes, long value) + { + unchecked + { + ulong unsignedValue = (ulong)value; + *bytes = (byte)unsignedValue; + *(bytes + 1) = (byte)(unsignedValue >> 8); + *(bytes + 2) = (byte)(unsignedValue >> 16); + *(bytes + 3) = (byte)(unsignedValue >> 24); + *(bytes + 4) = (byte)(unsignedValue >> 32); + *(bytes + 5) = (byte)(unsignedValue >> 40); + *(bytes + 6) = (byte)(unsignedValue >> 48); + *(bytes + 7) = (byte)(unsignedValue >> 56); + } + } + + internal static void SetZero(byte[] array, int index, int length) + { + if (length == 0) + { + return; + } + PlatformDependent.SetMemory(array, index, length, Zero); + } + + internal static IByteBuffer Copy(AbstractByteBuffer buf, byte* addr, int index, int length) + { + IByteBuffer copy = buf.Allocator.Buffer(length, buf.MaxCapacity); + if (length != 0) + { + if (copy.HasMemoryAddress) + { + fixed (byte* dst = ©.GetPinnableMemoryAddress()) + { + PlatformDependent.CopyMemory(addr, dst, length); + } + copy.SetIndex(0, length); + } + else + { + copy.WriteBytes(buf, index, length); + } + } + return copy; + } + + internal static int SetBytes(AbstractByteBuffer buf, byte* addr, int index, Stream input, int length) + { + IByteBuffer tmpBuf = buf.Allocator.HeapBuffer(length); + try + { + byte[] tmp = tmpBuf.Array; + int offset = tmpBuf.ArrayOffset; + int readBytes = input.Read(tmp, offset, length); + if (readBytes > 0) + { + PlatformDependent.CopyMemory(tmp, offset, addr, readBytes); + } + + return readBytes; + } + finally + { + tmpBuf.Release(); + } + } + + internal static void GetBytes(AbstractByteBuffer buf, byte* addr, int index, IByteBuffer dst, int dstIndex, int length) + { + Contract.Requires(dst != null); + + if (MathUtil.IsOutOfBounds(dstIndex, length, dst.Capacity)) + { + throw new IndexOutOfRangeException($"dstIndex: {dstIndex}"); + } + + if (dst.HasMemoryAddress) + { + fixed (byte* destination = &dst.GetPinnableMemoryAddress()) + { + PlatformDependent.CopyMemory(addr, destination + dstIndex, length); + } + } + else if (dst.HasArray) + { + PlatformDependent.CopyMemory(addr, dst.Array, dst.ArrayOffset + dstIndex, length); + } + else + { + dst.SetBytes(dstIndex, buf, index, length); + } + } + + internal static void GetBytes(AbstractByteBuffer buf, byte* addr, int index, byte[] dst, int dstIndex, int length) + { + Contract.Requires(dst != null); + + if (MathUtil.IsOutOfBounds(dstIndex, length, dst.Length)) + { + throw new IndexOutOfRangeException($"dstIndex: {dstIndex}"); + } + if (length != 0) + { + PlatformDependent.CopyMemory(addr, dst, dstIndex, length); + } + } + + internal static void SetBytes(AbstractByteBuffer buf, byte* addr, int index, IByteBuffer src, int srcIndex, int length) + { + Contract.Requires(src != null); + + if (MathUtil.IsOutOfBounds(srcIndex, length, src.Capacity)) + { + throw new IndexOutOfRangeException($"srcIndex: {srcIndex}"); + } + + if (length != 0) + { + if (src.HasMemoryAddress) + { + fixed (byte* source = &src.GetPinnableMemoryAddress()) + { + PlatformDependent.CopyMemory(source + srcIndex, addr, length); + } + } + else if (src.HasArray) + { + PlatformDependent.CopyMemory(src.Array, src.ArrayOffset + srcIndex, addr, length); + } + else + { + src.GetBytes(srcIndex, buf, index, length); + } + } + } + + // No need to check length zero, the calling method already done it + internal static void SetBytes(AbstractByteBuffer buf, byte* addr, int index, byte[] src, int srcIndex, int length) => + PlatformDependent.CopyMemory(src, srcIndex, addr, length); + + internal static void GetBytes(AbstractByteBuffer buf, byte* addr, int index, Stream output, int length) + { + if (length != 0) + { + IByteBuffer tmpBuf = buf.Allocator.HeapBuffer(length); + try + { + byte[] tmp = tmpBuf.Array; + int offset = tmpBuf.ArrayOffset; + PlatformDependent.CopyMemory(addr, tmp, offset, length); + output.Write(tmp, offset, length); + } + finally + { + tmpBuf.Release(); + } + } + } + + internal static void SetZero(byte* addr, int length) + { + if (length == 0) + { + return; + } + PlatformDependent.SetMemory(addr, length, Zero); + } + + internal static UnpooledUnsafeDirectByteBuffer NewUnsafeDirectByteBuffer( + IByteBufferAllocator alloc, int initialCapacity, int maxCapacity) => + new UnpooledUnsafeDirectByteBuffer(alloc, initialCapacity, maxCapacity); + } +} diff --git a/src/DotNetty.Buffers/WrappedByteBuffer.cs b/src/DotNetty.Buffers/WrappedByteBuffer.cs index 729865b98..173c8cf15 100644 --- a/src/DotNetty.Buffers/WrappedByteBuffer.cs +++ b/src/DotNetty.Buffers/WrappedByteBuffer.cs @@ -12,6 +12,11 @@ namespace DotNetty.Buffers using DotNetty.Common; using DotNetty.Common.Utilities; + /// Wraps another . + /// + /// It's important that the {@link #readerIndex()} and {@link #writerIndex()} will not do any adjustments on the + /// indices on the fly because of internal optimizations made by {@link ByteBufUtil#writeAscii(ByteBuf, CharSequence)} + /// and {@link ByteBufUtil#writeUtf8(ByteBuf, CharSequence)}. class WrappedByteBuffer : IByteBuffer { protected readonly IByteBuffer Buf; @@ -23,6 +28,12 @@ protected WrappedByteBuffer(IByteBuffer buf) this.Buf = buf; } + public bool HasMemoryAddress => this.Buf.HasMemoryAddress; + + public ref byte GetPinnableMemoryAddress() => ref this.Buf.GetPinnableMemoryAddress(); + + public IntPtr AddressOfPinnedMemory() => this.Buf.AddressOfPinnedMemory(); + public int Capacity => this.Buf.Capacity; public virtual IByteBuffer AdjustCapacity(int newCapacity) @@ -37,6 +48,8 @@ public virtual IByteBuffer AdjustCapacity(int newCapacity) public IByteBuffer Unwrap() => this.Buf; + public bool IsDirect => this.Buf.IsDirect; + public int ReaderIndex => this.Buf.ReaderIndex; public IByteBuffer SetReaderIndex(int readerIndex) @@ -565,13 +578,13 @@ public virtual IByteBuffer WriteZero(int length) public virtual int BytesBefore(int index, int length, byte value) => this.Buf.BytesBefore(index, length, value); - public virtual int ForEachByte(ByteProcessor processor) => this.Buf.ForEachByte(processor); + public virtual int ForEachByte(IByteProcessor processor) => this.Buf.ForEachByte(processor); - public virtual int ForEachByte(int index, int length, ByteProcessor processor) => this.Buf.ForEachByte(index, length, processor); + public virtual int ForEachByte(int index, int length, IByteProcessor processor) => this.Buf.ForEachByte(index, length, processor); - public virtual int ForEachByteDesc(ByteProcessor processor) => this.Buf.ForEachByteDesc(processor); + public virtual int ForEachByteDesc(IByteProcessor processor) => this.Buf.ForEachByteDesc(processor); - public virtual int ForEachByteDesc(int index, int length, ByteProcessor processor) => this.Buf.ForEachByteDesc(index, length, processor); + public virtual int ForEachByteDesc(int index, int length, IByteProcessor processor) => this.Buf.ForEachByteDesc(index, length, processor); public virtual IByteBuffer Copy() => this.Buf.Copy(); diff --git a/src/DotNetty.Buffers/WrappedCompositeByteBuffer.cs b/src/DotNetty.Buffers/WrappedCompositeByteBuffer.cs index c2a66b74a..68de47d24 100644 --- a/src/DotNetty.Buffers/WrappedCompositeByteBuffer.cs +++ b/src/DotNetty.Buffers/WrappedCompositeByteBuffer.cs @@ -102,13 +102,13 @@ internal WrappedCompositeByteBuffer(CompositeByteBuffer wrapped) : base(wrapped. public override int BytesBefore(int index, int length, byte value) => this.wrapped.BytesBefore(index, length, value); - public override int ForEachByte(ByteProcessor processor) => this.wrapped.ForEachByte(processor); + public override int ForEachByte(IByteProcessor processor) => this.wrapped.ForEachByte(processor); - public override int ForEachByte(int index, int length, ByteProcessor processor) => this.wrapped.ForEachByte(index, length, processor); + public override int ForEachByte(int index, int length, IByteProcessor processor) => this.wrapped.ForEachByte(index, length, processor); - public override int ForEachByteDesc(ByteProcessor processor) => this.wrapped.ForEachByteDesc(processor); + public override int ForEachByteDesc(IByteProcessor processor) => this.wrapped.ForEachByteDesc(processor); - public override int ForEachByteDesc(int index, int length, ByteProcessor processor) => this.wrapped.ForEachByteDesc(index, length, processor); + public override int ForEachByteDesc(int index, int length, IByteProcessor processor) => this.wrapped.ForEachByteDesc(index, length, processor); public override int GetHashCode() => this.wrapped.GetHashCode(); diff --git a/src/DotNetty.Codecs.Redis/RedisDecoder.cs b/src/DotNetty.Codecs.Redis/RedisDecoder.cs index ccdd99590..b3afaabe1 100644 --- a/src/DotNetty.Codecs.Redis/RedisDecoder.cs +++ b/src/DotNetty.Codecs.Redis/RedisDecoder.cs @@ -333,7 +333,7 @@ static IByteBuffer ReadLine(IByteBuffer byteBuffer) return null; } - int lfIndex = byteBuffer.ForEachByte(ByteProcessor.FIND_LF); + int lfIndex = byteBuffer.ForEachByte(ByteProcessor.FindLF); if (lfIndex < 0) { return null; @@ -400,9 +400,9 @@ long ParsePositiveNumber(IByteBuffer byteBuffer) return this.toPositiveLongProcessor.Content; } - class ToPositiveLongProcessor : ByteProcessor + class ToPositiveLongProcessor : IByteProcessor { - public override bool Process(byte value) + public bool Process(byte value) { if (!char.IsDigit((char)value)) { diff --git a/src/DotNetty.Codecs/LineBasedFrameDecoder.cs b/src/DotNetty.Codecs/LineBasedFrameDecoder.cs index 1033e9960..66f6e3dd4 100644 --- a/src/DotNetty.Codecs/LineBasedFrameDecoder.cs +++ b/src/DotNetty.Codecs/LineBasedFrameDecoder.cs @@ -160,7 +160,7 @@ void Fail(IChannelHandlerContext ctx, string length) int FindEndOfLine(IByteBuffer buffer) { - int i = buffer.ForEachByte(ByteProcessor.FIND_LF); + int i = buffer.ForEachByte(ByteProcessor.FindLF); if (i > 0 && buffer.GetByte(i - 1) == '\r') { i--; diff --git a/src/DotNetty.Common/Internal/MathUtil.cs b/src/DotNetty.Common/Internal/MathUtil.cs index 64be2e256..bebeb6406 100644 --- a/src/DotNetty.Common/Internal/MathUtil.cs +++ b/src/DotNetty.Common/Internal/MathUtil.cs @@ -3,26 +3,26 @@ namespace DotNetty.Common.Internal { + using System.Diagnostics; using System.Runtime.CompilerServices; public static class MathUtil { [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsOutOfBounds(int index, int length, int capacity) - { - return (index | length | (index + length) | (capacity - (index + length))) < 0; - } + public static bool IsOutOfBounds(int index, int length, int capacity) => + (index | length | (index + length) | (capacity - (index + length))) < 0; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int SafeFindNextPositivePowerOfTwo(int value) + public static int FindNextPositivePowerOfTwo(int value) { - return value <= 0 - ? 1 - : value >= 0x40000000 - ? 0x40000000 - : 1 << (32 - NumberOfLeadingZeros(value - 1)); + Debug.Assert(value > int.MinValue && value < 0x40000000); + return 1 << (32 - NumberOfLeadingZeros(value - 1)); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SafeFindNextPositivePowerOfTwo(int value) => + value <= 0 ? 1 : value >= 0x40000000 ? 0x40000000 : FindNextPositivePowerOfTwo(value); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int NumberOfLeadingZeros(this int i) { @@ -31,19 +31,14 @@ public static int NumberOfLeadingZeros(this int i) i |= i >> 4; i |= i >> 8; i |= i >> 16; + i = ~i; - return BitCount(~i); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static int BitCount(int i) - { + //bit count i -= ((i >> 1) & 0x55555555); i = (i & 0x33333333) + ((i >> 2) & 0x33333333); i = (((i >> 4) + i) & 0x0F0F0F0F); i += (i >> 8); i += (i >> 16); - return (i & 0x0000003F); } } diff --git a/src/DotNetty.Common/Internal/PlatformDependent.cs b/src/DotNetty.Common/Internal/PlatformDependent.cs index 1cb06e94f..8b242971d 100644 --- a/src/DotNetty.Common/Internal/PlatformDependent.cs +++ b/src/DotNetty.Common/Internal/PlatformDependent.cs @@ -1,15 +1,34 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +// ReSharper disable ConvertToAutoPropertyWhenPossible namespace DotNetty.Common.Internal { using System; + using System.Collections.Concurrent; + using System.Collections.Generic; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Threading; + using DotNetty.Common.Internal.Logging; public static class PlatformDependent { + static readonly IInternalLogger Logger = InternalLoggerFactory.GetInstance(typeof(PlatformDependent)); + + static readonly bool UseDirectBuffer; + + static PlatformDependent() + { + UseDirectBuffer = !SystemPropertyUtil.GetBoolean("io.netty.noPreferDirect", false); + if (Logger.DebugEnabled) + { + Logger.Debug("-Dio.netty.noPreferDirect: {}", !UseDirectBuffer); + } + } + + public static bool DirectBufferPreferred => UseDirectBuffer; + static int seed = (int)(Stopwatch.GetTimestamp() & 0xFFFFFFFF); //used to safly cast long to int, because the timestamp returned is long and it doesn't fit into an int static readonly ThreadLocal ThreadLocalRandom = new ThreadLocal(() => new Random(Interlocked.Increment(ref seed))); //used to simulate java ThreadLocalRandom @@ -17,31 +36,79 @@ public static class PlatformDependent public static IQueue NewMpscQueue() where T : class => new CompatibleConcurrentQueue(); - public static ILinkedQueue SpscLinkedQueue() where T : class => new SpscLinkedQueue(); + public static IDictionary NewConcurrentHashMap() => new ConcurrentDictionary(); + + public static ILinkedQueue NewSpscLinkedQueue() where T : class => new SpscLinkedQueue(); public static Random GetThreadLocalRandom() => ThreadLocalRandom.Value; + public static unsafe bool ByteArrayEquals(byte[] bytes1, int startPos1, byte[] bytes2, int startPos2, int length) + { + fixed (byte* array1 = bytes1) + fixed (byte* array2 = bytes2) + return PlatformDependent0.ByteArrayEquals(array1, startPos1, array2, startPos2, length); + } + public static unsafe void CopyMemory(byte[] src, int srcIndex, byte[] dst, int dstIndex, int length) { - if (length == 0) + if (length > 0) { - return; + fixed (byte* source = &src[srcIndex]) + fixed (byte* destination = &dst[dstIndex]) + Unsafe.CopyBlock(destination, source, unchecked((uint)length)); } + } + + public static unsafe void CopyMemory(byte* src, byte* dst, int length) + { + if (length > 0) + { + Unsafe.CopyBlock(dst, src, unchecked((uint)length)); + } + } - fixed (byte* source = &src[srcIndex]) + public static unsafe void CopyMemory(byte* src, byte[] dst, int dstIndex, int length) + { + if (length > 0) + { fixed (byte* destination = &dst[dstIndex]) - Unsafe.CopyBlock(destination, source, (uint)length); + Unsafe.CopyBlock(destination, src, unchecked((uint)length)); + } + } + + public static unsafe void CopyMemory(byte[] src, int srcIndex, byte* dst, int length) + { + if (length > 0) + { + fixed (byte* source = &src[srcIndex]) + Unsafe.CopyBlock(dst, source, unchecked((uint)length)); + } } public static unsafe void Clear(byte[] src, int srcIndex, int length) { - if (length == 0) + if (length > 0) { - return; + fixed (void* source = &src[srcIndex]) + Unsafe.InitBlock(source, default(byte), unchecked((uint)length)); } + } - fixed (void* source = &src[srcIndex]) - Unsafe.InitBlock(source, default(byte), (uint)length); + public static unsafe void SetMemory(byte* src, int length, byte value) + { + if (length > 0) + { + Unsafe.InitBlock(src, value, unchecked((uint)length)); + } + } + + public static unsafe void SetMemory(byte[] src, int srcIndex, int length, byte value) + { + if (length > 0) + { + fixed (byte* source = &src[srcIndex]) + Unsafe.InitBlock(source, value, unchecked((uint)length)); + } } } } \ No newline at end of file diff --git a/src/DotNetty.Common/Internal/PlatformDependent0.cs b/src/DotNetty.Common/Internal/PlatformDependent0.cs new file mode 100644 index 000000000..233b91ab9 --- /dev/null +++ b/src/DotNetty.Common/Internal/PlatformDependent0.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace DotNetty.Common.Internal +{ + using System.Runtime.CompilerServices; + + static class PlatformDependent0 + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe bool ByteArrayEquals(byte* bytes1, int startPos1, byte* bytes2, int startPos2, int length) + { + if (length <= 0) + { + return true; + } + + byte* baseOffset1 = bytes1 + startPos1; + byte* baseOffset2 = bytes2 + startPos2; + int remainingBytes = length & 7; + byte* end = baseOffset1 + remainingBytes; + for (byte* i = baseOffset1 - 8 + length, j = baseOffset2 - 8 + length; i >= end; i -= 8, j -= 8) + { + if (Unsafe.Read(i) != Unsafe.Read(j)) + { + return false; + } + } + + if (remainingBytes >= 4) + { + remainingBytes -= 4; + if (Unsafe.Read(baseOffset1 + remainingBytes) != Unsafe.Read(baseOffset2 + remainingBytes)) + { + return false; + } + } + if (remainingBytes >= 2) + { + return Unsafe.Read(baseOffset1) == Unsafe.Read(baseOffset2) + && (remainingBytes == 2 || *(bytes1 + startPos1 + 2) == *(bytes2 + startPos2 + 2)); + } + return *baseOffset1 == *baseOffset2; + } + } +} diff --git a/src/DotNetty.Common/Utilities/ByteProcessor.cs b/src/DotNetty.Common/Utilities/ByteProcessor.cs index ecb997b79..b13abfead 100644 --- a/src/DotNetty.Common/Utilities/ByteProcessor.cs +++ b/src/DotNetty.Common/Utilities/ByteProcessor.cs @@ -9,108 +9,101 @@ namespace DotNetty.Common.Utilities /// /// Provides a mechanism to iterate over a collection of bytes. /// - public abstract class ByteProcessor + public interface IByteProcessor { - /// - /// A which finds the first appearance of a specific byte. - /// - public sealed class IndexOfProcessor : ByteProcessor - { - readonly byte byteToFind; + bool Process(byte value); + } - public IndexOfProcessor(byte byteToFind) - { - this.byteToFind = byteToFind; - } + public sealed class IndexOfProcessor : IByteProcessor + { + readonly byte byteToFind; - public override bool Process(byte value) => value != this.byteToFind; + public IndexOfProcessor(byte byteToFind) + { + this.byteToFind = byteToFind; } - public sealed class IndexNotOfProcessor : ByteProcessor - { - readonly byte byteToNotFind; + public bool Process(byte value) => value != this.byteToFind; + } - public IndexNotOfProcessor(byte byteToNotFind) - { - this.byteToNotFind = byteToNotFind; - } + public sealed class IndexNotOfProcessor : IByteProcessor + { + readonly byte byteToNotFind; - public override bool Process(byte value) => value == this.byteToNotFind; + public IndexNotOfProcessor(byte byteToNotFind) + { + this.byteToNotFind = byteToNotFind; } - public sealed class CustomProcessor : ByteProcessor - { - readonly Func customHandler; + public bool Process(byte value) => value == this.byteToNotFind; + } - public CustomProcessor(Func customHandler) - { - Contract.Assert(customHandler != null, "'customHandler' is required parameter."); - this.customHandler = customHandler; - } + public sealed class ByteProcessor : IByteProcessor + { + readonly Func customHandler; - public override bool Process(byte value) => this.customHandler(value); + public ByteProcessor(Func customHandler) + { + Contract.Assert(customHandler != null, "'customHandler' is required parameter."); + this.customHandler = customHandler; } + public bool Process(byte value) => this.customHandler(value); + + /// /// Aborts on a NUL (0x00). /// - public static ByteProcessor FIND_NUL = new IndexOfProcessor(0); + public static IByteProcessor FindNul = new IndexOfProcessor(0); /// /// Aborts on a non-{@code NUL (0x00)}. /// - public static ByteProcessor FIND_NON_NUL = new IndexNotOfProcessor(0); + public static IByteProcessor FindNonNul = new IndexNotOfProcessor(0); /// /// Aborts on a {@code CR ('\r')}. /// - public static ByteProcessor FIND_CR = new IndexOfProcessor((byte)'\r'); + public static IByteProcessor FindCR = new IndexOfProcessor((byte)'\r'); /// /// Aborts on a non-{@code CR ('\r')}. /// - public static ByteProcessor FIND_NON_CR = new IndexNotOfProcessor((byte)'\r'); + public static IByteProcessor FindNonCR = new IndexNotOfProcessor((byte)'\r'); /// /// Aborts on a {@code LF ('\n')}. /// - public static ByteProcessor FIND_LF = new IndexOfProcessor((byte)'\n'); + public static IByteProcessor FindLF = new IndexOfProcessor((byte)'\n'); /// /// Aborts on a non-{@code LF ('\n')}. /// - public static ByteProcessor FIND_NON_LF = new IndexNotOfProcessor((byte)'\n'); + public static IByteProcessor FindNonLF = new IndexNotOfProcessor((byte)'\n'); /// /// Aborts on a {@code CR (';')}. /// - public static ByteProcessor FIND_SEMI_COLON = new IndexOfProcessor((byte)';'); + public static IByteProcessor FindSemiCOLON = new IndexOfProcessor((byte)';'); /// /// Aborts on a {@code CR ('\r')} or a {@code LF ('\n')}. /// - public static ByteProcessor FIND_CRLF = new CustomProcessor(new Func(value => value != '\r' && value != '\n')); + public static IByteProcessor FindCrlf = new ByteProcessor(new Func(value => value != '\r' && value != '\n')); /// /// Aborts on a byte which is neither a {@code CR ('\r')} nor a {@code LF ('\n')}. /// - public static ByteProcessor FIND_NON_CRLF = new CustomProcessor(new Func(value => value == '\r' || value == '\n')); + public static IByteProcessor FindNonCrlf = new ByteProcessor(new Func(value => value == '\r' || value == '\n')); /// /// Aborts on a linear whitespace (a ({@code ' '} or a {@code '\t'}). /// - public static ByteProcessor FIND_LINEAR_WHITESPACE = new CustomProcessor(new Func(value => value != ' ' && value != '\t')); + public static IByteProcessor FindLinearWhitespace = new ByteProcessor(new Func(value => value != ' ' && value != '\t')); /// /// Aborts on a byte which is not a linear whitespace (neither {@code ' '} nor {@code '\t'}). /// - public static ByteProcessor FIND_NON_LINEAR_WHITESPACE = new CustomProcessor(new Func(value => value == ' ' || value == '\t')); - - /* - * @return {@code true} if the processor wants to continue the loop and handle the next byte in the buffer. - * {@code false} if the processor wants to stop handling bytes and abort the loop. - */ - - public abstract bool Process(byte value); + public static IByteProcessor FindNonLinearWhitespace = new ByteProcessor(new Func(value => value == ' ' || value == '\t')); } } \ No newline at end of file diff --git a/src/DotNetty.Transport.Libuv/Native/ReadOperation.cs b/src/DotNetty.Transport.Libuv/Native/ReadOperation.cs index 07b767eb7..3d4a83b0e 100644 --- a/src/DotNetty.Transport.Libuv/Native/ReadOperation.cs +++ b/src/DotNetty.Transport.Libuv/Native/ReadOperation.cs @@ -1,56 +1,68 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +// ReSharper disable ConvertToAutoProperty +// ReSharper disable ConvertToAutoPropertyWithPrivateSetter namespace DotNetty.Transport.Libuv.Native { using System; + using System.Diagnostics; using System.Runtime.InteropServices; using DotNetty.Buffers; public sealed class ReadOperation : IDisposable { readonly INativeUnsafe nativeUnsafe; - GCHandle array; + readonly IByteBuffer buffer; + + OperationException error; + int status; + bool endOfStream; + + GCHandle pin; internal ReadOperation(INativeUnsafe nativeUnsafe, IByteBuffer buffer) { this.nativeUnsafe = nativeUnsafe; - this.Buffer = buffer; - this.Status = 0; - this.EndOfStream = false; + this.buffer = buffer; + this.status = 0; + this.endOfStream = false; } - internal IByteBuffer Buffer { get; } + internal IByteBuffer Buffer => this.buffer; - internal OperationException Error { get; private set; } + internal OperationException Error => this.error; - internal int Status { get; private set; } + internal int Status => this.status; - internal bool EndOfStream { get; private set; } + internal bool EndOfStream => this.endOfStream; - internal void Complete(int status, OperationException error) + internal void Complete(int statusCode, OperationException exception) { this.Release(); - this.Status = status; - this.EndOfStream = status == (int)uv_err_code.UV_EOF; - this.Error = error; + this.status = statusCode; + this.endOfStream = statusCode == (int)uv_err_code.UV_EOF; + this.error = exception; this.nativeUnsafe.FinishRead(this); } internal uv_buf_t GetBuffer() { - if (this.array.IsAllocated) - { - throw new InvalidOperationException( - $"{nameof(ReadOperation)} has already been initialized and not released yet."); - } + Debug.Assert(!this.pin.IsAllocated); IByteBuffer buf = this.Buffer; - this.array = GCHandle.Alloc(buf.Array, GCHandleType.Pinned); - IntPtr arrayHandle = this.array.AddrOfPinnedObject(); - int index = buf.ArrayOffset + buf.WriterIndex; + // Do not pin the buffer again if it is already pinned + IntPtr arrayHandle = buf.AddressOfPinnedMemory(); + int index = buf.WriterIndex; + + if (arrayHandle == IntPtr.Zero) + { + this.pin = GCHandle.Alloc(buf.Array, GCHandleType.Pinned); + arrayHandle = this.pin.AddrOfPinnedObject(); + index += buf.ArrayOffset; + } int length = buf.WritableBytes; return new uv_buf_t(arrayHandle + index, length); @@ -58,9 +70,9 @@ internal uv_buf_t GetBuffer() void Release() { - if (this.array.IsAllocated) + if (this.pin.IsAllocated) { - this.array.Free(); + this.pin.Free(); } } diff --git a/src/DotNetty.Transport.Libuv/Native/Tcp.cs b/src/DotNetty.Transport.Libuv/Native/Tcp.cs index 1ae3afe5e..9b210b075 100644 --- a/src/DotNetty.Transport.Libuv/Native/Tcp.cs +++ b/src/DotNetty.Transport.Libuv/Native/Tcp.cs @@ -19,7 +19,7 @@ sealed class Tcp : TcpHandle public Tcp(Loop loop) : base(loop) { - this.pendingReads = PlatformDependent.SpscLinkedQueue(); + this.pendingReads = PlatformDependent.NewSpscLinkedQueue(); } public void ReadStart(INativeUnsafe channel) diff --git a/test/DotNetty.Buffers.Tests/AbstractByteBufferAllocatorTests.cs b/test/DotNetty.Buffers.Tests/AbstractByteBufferAllocatorTests.cs index d114eb273..ff53e3a12 100644 --- a/test/DotNetty.Buffers.Tests/AbstractByteBufferAllocatorTests.cs +++ b/test/DotNetty.Buffers.Tests/AbstractByteBufferAllocatorTests.cs @@ -10,14 +10,18 @@ public abstract class AbstractByteBufferAllocatorTests : ByteBufferAllocatorTest { protected abstract IByteBufferAllocator NewUnpooledAllocator(); + protected override bool IsDirectExpected(bool preferDirect) => preferDirect; + protected sealed override int DefaultMaxCapacity => AbstractByteBufferAllocator.DefaultMaxCapacity; protected sealed override int DefaultMaxComponents => AbstractByteBufferAllocator.DefaultMaxComponents; - [Fact] - public void CalculateNewCapacity() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void CalculateNewCapacity(bool preferDirect) { - IByteBufferAllocator allocator = this.NewAllocator(); + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); Assert.Equal(8, allocator.CalculateNewCapacity(1, 8)); Assert.Equal(7, allocator.CalculateNewCapacity(1, 7)); Assert.Equal(64, allocator.CalculateNewCapacity(1, 129)); @@ -26,7 +30,20 @@ public void CalculateNewCapacity() Assert.Throws(() => allocator.CalculateNewCapacity(-1, 8)); } - protected static void AssertInstanceOf(T buffer) where T : IByteBuffer + [Fact] + public void UnsafeHeapBufferAndUnsafeDirectBuffer() + { + IByteBufferAllocator allocator = this.NewUnpooledAllocator(); + IByteBuffer directBuffer = allocator.DirectBuffer(); + AssertInstanceOf(directBuffer); + directBuffer.Release(); + + IByteBuffer heapBuffer = allocator.HeapBuffer(); + AssertInstanceOf(heapBuffer); + heapBuffer.Release(); + } + + protected static void AssertInstanceOf(IByteBuffer buffer) where T : IByteBuffer { Assert.IsAssignableFrom(buffer is SimpleLeakAwareByteBuffer ? buffer.Unwrap() : buffer); } @@ -34,7 +51,7 @@ protected static void AssertInstanceOf(T buffer) where T : IByteBuffer [Fact] public void UsedHeapMemory() { - IByteBufferAllocator allocator = this.NewAllocator(); + IByteBufferAllocator allocator = this.NewAllocator(true); IByteBufferAllocatorMetric metric = ((IByteBufferAllocatorMetricProvider)allocator).Metric; Assert.Equal(0, metric.UsedHeapMemory); @@ -51,14 +68,8 @@ public void UsedHeapMemory() Assert.Equal(this.ExpectedUsedMemoryAfterRelease(allocator, capacity), metric.UsedHeapMemory); } - protected virtual long ExpectedUsedMemory(IByteBufferAllocator allocator, int capacity) - { - return capacity; - } + protected virtual long ExpectedUsedMemory(IByteBufferAllocator allocator, int capacity) => capacity; - protected virtual long ExpectedUsedMemoryAfterRelease(IByteBufferAllocator allocator, int capacity) - { - return 0; - } + protected virtual long ExpectedUsedMemoryAfterRelease(IByteBufferAllocator allocator, int capacity) => 0; } } diff --git a/test/DotNetty.Buffers.Tests/AbstractByteBufferTests.cs b/test/DotNetty.Buffers.Tests/AbstractByteBufferTests.cs index dd1077dd1..f029dc002 100644 --- a/test/DotNetty.Buffers.Tests/AbstractByteBufferTests.cs +++ b/test/DotNetty.Buffers.Tests/AbstractByteBufferTests.cs @@ -1999,7 +1999,7 @@ public void ForEachByte() this.buffer.SetIndex(Capacity / 4, Capacity * 3 / 4); int i1 = Capacity / 4; Assert.Equal(-1, - this.buffer.ForEachByte(new ByteProcessor.CustomProcessor( + this.buffer.ForEachByte(new ByteProcessor( value => { Assert.Equal(value, (byte)(i1 + 1)); @@ -2022,7 +2022,7 @@ public void ForEachByteAbort() int stop = Capacity / 2; int i1 = Capacity / 3; - Assert.Equal(stop, this.buffer.ForEachByte(Capacity / 3, Capacity / 3, new ByteProcessor.CustomProcessor(value => + Assert.Equal(stop, this.buffer.ForEachByte(Capacity / 3, Capacity / 3, new ByteProcessor(value => { Assert.Equal((byte)(i1 + 1), value); if (i1 == stop) @@ -2046,7 +2046,7 @@ public void ForEachByteDesc() int lastIndex = 0; int i1 = Capacity * 3 / 4 - 1; - Assert.Equal(-1, this.buffer.ForEachByteDesc(Capacity / 4, Capacity * 2 / 4, new ByteProcessor.CustomProcessor(value => + Assert.Equal(-1, this.buffer.ForEachByteDesc(Capacity / 4, Capacity * 2 / 4, new ByteProcessor(value => { Assert.Equal((byte)(i1 + 1), value); Volatile.Write(ref lastIndex, i1); @@ -2446,6 +2446,16 @@ public void ArrayAfterRelease() } } + [Fact] + public void MemoryAddressAfterRelease() + { + IByteBuffer buf = this.ReleasedBuffer(); + if (buf.HasMemoryAddress) + { + Assert.Throws(() => buf.GetPinnableMemoryAddress()); + } + } + [Fact] public void SliceRelease() { @@ -2455,6 +2465,33 @@ public void SliceRelease() Assert.Equal(0, buf.ReferenceCount); } + [Fact] + public void ReadSliceOutOfBounds() => Assert.Throws(() => this.ReadSliceOutOfBounds(false)); + + [Fact] + public void ReadRetainedSliceOutOfBounds() => Assert.Throws(() => this.ReadSliceOutOfBounds(true)); + + void ReadSliceOutOfBounds(bool retainedSlice) + { + IByteBuffer buf = this.NewBuffer(100); + try + { + buf.WriteZero(50); + if (retainedSlice) + { + buf.ReadRetainedSlice(51); + } + else + { + buf.ReadSlice(51); + } + } + finally + { + buf.Release(); + } + } + [Fact] public void RetainedSliceIndexOutOfBounds() => Assert.Throws(() => this.SliceOutOfBounds(true, true, true)); @@ -3097,7 +3134,7 @@ public virtual void ForEachByteDesc2() } } - sealed class ForEachByteDesc2Processor : ByteProcessor + sealed class ForEachByteDesc2Processor : IByteProcessor { int index; @@ -3109,7 +3146,7 @@ public ForEachByteDesc2Processor(int length) public byte[] Bytes { get; } - public override bool Process(byte value) + public bool Process(byte value) { this.Bytes[this.index--] = value; return true; @@ -3136,7 +3173,7 @@ public virtual void ForEachByte2() } } - sealed class ForEachByte2Processor : ByteProcessor + sealed class ForEachByte2Processor : IByteProcessor { int index; @@ -3148,7 +3185,7 @@ public ForEachByte2Processor(int length) public byte[] Bytes { get; } - public override bool Process(byte value) + public bool Process(byte value) { this.Bytes[this.index++] = value; return true; @@ -3301,9 +3338,9 @@ protected IByteBuffer ReleaseLater(IByteBuffer buf) return buf; } - sealed class TestByteProcessor : ByteProcessor + sealed class TestByteProcessor : IByteProcessor { - public override bool Process(byte value) => true; + public bool Process(byte value) => true; } } } \ No newline at end of file diff --git a/test/DotNetty.Buffers.Tests/AbstractReferenceCountedByteBufferTests.cs b/test/DotNetty.Buffers.Tests/AbstractReferenceCountedByteBufferTests.cs index 84c39e21a..2abddffa9 100644 --- a/test/DotNetty.Buffers.Tests/AbstractReferenceCountedByteBufferTests.cs +++ b/test/DotNetty.Buffers.Tests/AbstractReferenceCountedByteBufferTests.cs @@ -126,8 +126,16 @@ public ReferenceCountedByteBuffer() public override int ArrayOffset => throw new NotSupportedException(); + public override bool HasMemoryAddress => throw new NotSupportedException(); + + public override ref byte GetPinnableMemoryAddress() => throw new NotSupportedException(); + + public override IntPtr AddressOfPinnedMemory() => throw new NotSupportedException(); + public override IByteBuffer Unwrap() => throw new NotSupportedException(); + public override bool IsDirect => throw new NotSupportedException(); + public override IByteBuffer Copy(int index, int length) => throw new NotSupportedException(); public override IByteBuffer SetBytes(int index, byte[] src, int srcIndex, int length) => throw new NotSupportedException(); diff --git a/test/DotNetty.Buffers.Tests/ByteBufferAllocatorTests.cs b/test/DotNetty.Buffers.Tests/ByteBufferAllocatorTests.cs index 145b0ff32..ff13d9c3a 100644 --- a/test/DotNetty.Buffers.Tests/ByteBufferAllocatorTests.cs +++ b/test/DotNetty.Buffers.Tests/ByteBufferAllocatorTests.cs @@ -13,16 +13,18 @@ public abstract class ByteBufferAllocatorTests protected abstract int DefaultMaxComponents { get; } - protected abstract IByteBufferAllocator NewAllocator(); + protected abstract IByteBufferAllocator NewAllocator(bool preferDirect); - [Fact] - public void Buffer() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Buffer(bool preferDirect) { - IByteBufferAllocator allocator = this.NewAllocator(); + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); IByteBuffer buffer = allocator.Buffer(1); try { - AssertBuffer(buffer, 1, this.DefaultMaxCapacity); + AssertBuffer(buffer, this.IsDirectExpected(preferDirect), 1, this.DefaultMaxCapacity); } finally { @@ -30,14 +32,16 @@ public void Buffer() } } - [Fact] - public void BufferWithCapacity() + [Theory] + [InlineData(true, 8)] + [InlineData(false, 8)] + public void BufferWithCapacity(bool preferDirect, int maxCapacity) { - IByteBufferAllocator allocator = this.NewAllocator(); - IByteBuffer buffer = allocator.Buffer(1, 8); + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); + IByteBuffer buffer = allocator.Buffer(1, maxCapacity); try { - AssertBuffer(buffer, 1, 8); + AssertBuffer(buffer, this.IsDirectExpected(preferDirect), 1, maxCapacity); } finally { @@ -45,14 +49,16 @@ public void BufferWithCapacity() } } - [Fact] - public void HeapBuffer() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void HeapBuffer(bool preferDirect) { - IByteBufferAllocator allocator = this.NewAllocator(); + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); IByteBuffer buffer = allocator.HeapBuffer(1); try { - AssertBuffer(buffer, 1, this.DefaultMaxCapacity); + AssertBuffer(buffer, false, 1, this.DefaultMaxCapacity); } finally { @@ -60,14 +66,18 @@ public void HeapBuffer() } } - [Fact] - public void HeapBufferWithCapacity() + protected abstract bool IsDirectExpected(bool preferDirect); + + [Theory] + [InlineData(true, 8)] + [InlineData(false, 8)] + public void HeapBufferWithCapacity(bool preferDirect, int maxCapacity) { - IByteBufferAllocator allocator = this.NewAllocator(); - IByteBuffer buffer = allocator.HeapBuffer(1, 8); + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); + IByteBuffer buffer = allocator.HeapBuffer(1, maxCapacity); try { - AssertBuffer(buffer, 1, 8); + AssertBuffer(buffer, false, 1, maxCapacity); } finally { @@ -75,14 +85,33 @@ public void HeapBufferWithCapacity() } } - [Fact] - public void CompositeBuffer() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void DirectBuffer(bool preferDirect) { - IByteBufferAllocator allocator = this.NewAllocator(); - CompositeByteBuffer buffer = allocator.CompositeBuffer(); + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); + IByteBuffer buffer = allocator.DirectBuffer(1); try { - this.AssertCompositeByteBuffer(buffer, this.DefaultMaxComponents); + AssertBuffer(buffer, true, 1, this.DefaultMaxCapacity); + } + finally + { + buffer.Release(); + } + } + + [Theory] + [InlineData(true, 8)] + [InlineData(false, 8)] + public void DirectBufferWithCapacity(bool preferDirect, int maxCapacity) + { + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); + IByteBuffer buffer = allocator.DirectBuffer(1, maxCapacity); + try + { + AssertBuffer(buffer, true, 1, maxCapacity); } finally { @@ -90,14 +119,16 @@ public void CompositeBuffer() } } - [Fact] - public void CompositeBufferWithCapacity() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void CompositeBuffer(bool preferDirect) { - IByteBufferAllocator allocator = this.NewAllocator(); - CompositeByteBuffer buffer = allocator.CompositeBuffer(8); + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); + CompositeByteBuffer buffer = allocator.CompositeBuffer(); try { - this.AssertCompositeByteBuffer(buffer, 8); + this.AssertCompositeByteBuffer(buffer, this.DefaultMaxComponents); } finally { @@ -105,10 +136,17 @@ public void CompositeBufferWithCapacity() } } - [Fact] - public void CompositeHeapBuffer() + [Theory] + [InlineData(true, 8)] + [InlineData(false, 8)] + public void CompositeBufferWithCapacity(bool preferDirect, int maxNumComponents) => this.TestCompositeHeapBufferWithCapacity(preferDirect, maxNumComponents); + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void CompositeHeapBuffer(bool preferDirect) { - IByteBufferAllocator allocator = this.NewAllocator(); + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); CompositeByteBuffer buffer = allocator.CompositeHeapBuffer(); try { @@ -120,14 +158,18 @@ public void CompositeHeapBuffer() } } - [Fact] - public void CompositeHeapBufferWithCapacity() + [Theory] + [InlineData(true, 8)] + [InlineData(false, 8)] + public void CompositeHeapBufferWithCapacity(bool preferDirect, int maxNumComponents) => this.TestCompositeHeapBufferWithCapacity(preferDirect, maxNumComponents); + + void TestCompositeHeapBufferWithCapacity(bool preferDirect, int maxNumComponents) { - IByteBufferAllocator allocator = this.NewAllocator(); - CompositeByteBuffer buffer = allocator.CompositeHeapBuffer(8); + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); + CompositeByteBuffer buffer = allocator.CompositeHeapBuffer(maxNumComponents); try { - this.AssertCompositeByteBuffer(buffer, 8); + this.AssertCompositeByteBuffer(buffer, maxNumComponents); } finally { @@ -135,14 +177,43 @@ public void CompositeHeapBufferWithCapacity() } } - static void AssertBuffer(IByteBuffer buffer, int expectedCapacity, int expectedMaxCapacity) + [Theory] + [InlineData(true)] + [InlineData(false)] + public void CompositeDirectBuffer(bool preferDirect) { - if (!(buffer is CompositeByteBuffer)) + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); + CompositeByteBuffer buffer = allocator.CompositeDirectBuffer(); + try { - Assert.True(buffer is UnpooledHeapByteBuffer || buffer is PooledHeapByteBuffer, - $"Wrong byte buffer type{buffer.GetType().FullName}"); + this.AssertCompositeByteBuffer(buffer, this.DefaultMaxComponents); + } + finally + { + buffer.Release(); } + } + [Theory] + [InlineData(true, 8)] + [InlineData(false, 8)] + public void CompositeDirectBufferWithCapacity(bool preferDirect, int maxNumComponents) + { + IByteBufferAllocator allocator = this.NewAllocator(preferDirect); + CompositeByteBuffer buffer = allocator.CompositeDirectBuffer(maxNumComponents); + try + { + this.AssertCompositeByteBuffer(buffer, maxNumComponents); + } + finally + { + buffer.Release(); + } + } + + static void AssertBuffer(IByteBuffer buffer, bool expectedDirect, int expectedCapacity, int expectedMaxCapacity) + { + Assert.Equal(expectedDirect, buffer.IsDirect); Assert.Equal(expectedCapacity, buffer.Capacity); Assert.Equal(expectedMaxCapacity, buffer.MaxCapacity); } @@ -151,7 +222,7 @@ void AssertCompositeByteBuffer(CompositeByteBuffer buffer, int expectedMaxNumCom { Assert.Equal(0, buffer.NumComponents); Assert.Equal(expectedMaxNumComponents, buffer.MaxNumComponents); - AssertBuffer(buffer, 0, this.DefaultMaxCapacity); + AssertBuffer(buffer, false, 0, this.DefaultMaxCapacity); } } } diff --git a/test/DotNetty.Buffers.Tests/EmptyByteBufferTests.cs b/test/DotNetty.Buffers.Tests/EmptyByteBufferTests.cs index 710f3bdb5..7ccff5831 100644 --- a/test/DotNetty.Buffers.Tests/EmptyByteBufferTests.cs +++ b/test/DotNetty.Buffers.Tests/EmptyByteBufferTests.cs @@ -42,5 +42,13 @@ public void Array() Assert.Equal(0, empty.Array.Length); Assert.Equal(0, empty.ArrayOffset); } + + [Fact] + public void MemoryAddress() + { + var empty = new EmptyByteBuffer(UnpooledByteBufferAllocator.Default); + Assert.False(empty.HasMemoryAddress); + Assert.Throws(() => empty.GetPinnableMemoryAddress()); + } } } diff --git a/test/DotNetty.Buffers.Tests/PoolArenaTests.cs b/test/DotNetty.Buffers.Tests/PoolArenaTests.cs index f6d0948f0..023d7f8ec 100644 --- a/test/DotNetty.Buffers.Tests/PoolArenaTests.cs +++ b/test/DotNetty.Buffers.Tests/PoolArenaTests.cs @@ -23,7 +23,9 @@ public void NormalizeCapacity() public void AllocationCounter() { var allocator = new PooledByteBufferAllocator( - 1, // nHeapArena + true, // preferDirect + 0, // nHeapArena + 1, // nDirectArena 8192, // pageSize 11, // maxOrder 0, // tinyCacheSize @@ -48,8 +50,8 @@ public void AllocationCounter() Assert.True(b2.Release()); Assert.True(b3.Release()); - Assert.True(allocator.HeapArenas().Count >= 1); - IPoolArenaMetric metric = allocator.HeapArenas()[0]; + Assert.True(allocator.DirectArenas().Count >= 1); + IPoolArenaMetric metric = allocator.DirectArenas()[0]; Assert.Equal(3, metric.NumDeallocations); Assert.Equal(3, metric.NumAllocations); diff --git a/test/DotNetty.Buffers.Tests/PooledByteBufferAllocatorTests.cs b/test/DotNetty.Buffers.Tests/PooledByteBufferAllocatorTests.cs index d4cfc1649..dc526c25f 100644 --- a/test/DotNetty.Buffers.Tests/PooledByteBufferAllocatorTests.cs +++ b/test/DotNetty.Buffers.Tests/PooledByteBufferAllocatorTests.cs @@ -10,9 +10,9 @@ namespace DotNetty.Buffers.Tests public class PooledByteBufferAllocatorTests : AbstractByteBufferAllocatorTests { - protected override IByteBufferAllocator NewAllocator() => new PooledByteBufferAllocator(); + protected override IByteBufferAllocator NewAllocator(bool preferDirect) => new PooledByteBufferAllocator(preferDirect); - protected override IByteBufferAllocator NewUnpooledAllocator() => new PooledByteBufferAllocator(0, 8192, 1); + protected override IByteBufferAllocator NewUnpooledAllocator() => new PooledByteBufferAllocator(0, 0, 8192, 1); protected override long ExpectedUsedMemory(IByteBufferAllocator allocator, int capacity) { @@ -28,27 +28,23 @@ protected override long ExpectedUsedMemoryAfterRelease(IByteBufferAllocator allo } [Fact] - public void PooledHeapBuffer() + public void PooledUnsafeHeapBufferAndUnsafeDirectBuffer() { - IByteBufferAllocator allocator = this.NewAllocator(); + var allocator = (PooledByteBufferAllocator)this.NewAllocator(true); + IByteBuffer directBuffer = allocator.DirectBuffer(); + AssertInstanceOf(directBuffer); + directBuffer.Release(); IByteBuffer heapBuffer = allocator.HeapBuffer(); - - try - { - Assert.IsAssignableFrom(heapBuffer); - } - finally - { - heapBuffer.Release(); - } + AssertInstanceOf(heapBuffer); + heapBuffer.Release(); } [Fact] - public void ArenaMetricsNoCache() => ArenaMetrics0(new PooledByteBufferAllocator(2, 8192, 11, 0, 0, 0), 100, 0, 100, 100); + public void ArenaMetricsNoCache() => ArenaMetrics0(new PooledByteBufferAllocator(true, 2, 2, 8192, 11, 0, 0, 0), 100, 0, 100, 100); [Fact] - public void ArenaMetricsCache() => ArenaMetrics0(new PooledByteBufferAllocator(2, 8192, 11, 1000, 1000, 1000), 100, 1, 1, 0); + public void ArenaMetricsCache() => ArenaMetrics0(new PooledByteBufferAllocator(true, 2, 2, 8192, 11, 1000, 1000, 1000), 100, 1, 1, 0); static void ArenaMetrics0(PooledByteBufferAllocator allocator, int num, int expectedActive, int expectedAlloc, int expectedDealloc) { @@ -106,7 +102,7 @@ static void AssertPoolChunkListMetric(IPoolChunkListMetric m, int min, int max) [Fact] public void SmallSubpageMetric() { - var allocator = new PooledByteBufferAllocator(1, 8192, 11, 0, 0, 0); + var allocator = new PooledByteBufferAllocator(true, 1, 1, 8192, 11, 0, 0, 0); IByteBuffer buffer = allocator.HeapBuffer(500); try { @@ -123,7 +119,7 @@ public void SmallSubpageMetric() [Fact] public void TinySubpageMetric() { - var allocator = new PooledByteBufferAllocator(1, 8192, 11, 0, 0, 0); + var allocator = new PooledByteBufferAllocator(true, 1, 1, 8192, 11, 0, 0, 0); IByteBuffer buffer = allocator.HeapBuffer(1); try { @@ -140,7 +136,7 @@ public void TinySubpageMetric() [Fact] public void AllocNotNull() { - var allocator = new PooledByteBufferAllocator(1, 8192, 11, 0, 0, 0); + var allocator = new PooledByteBufferAllocator(true, 1, 1, 8192, 11, 0, 0, 0); // Huge allocation AllocNotNull(allocator, allocator.Metric.ChunkSize + 1); // Normal allocation @@ -163,7 +159,7 @@ static void AllocNotNull(PooledByteBufferAllocator allocator, int capacity) public void FreePoolChunk() { const int ChunkSize = 16 * 1024 * 1024; - var allocator = new PooledByteBufferAllocator(1, 8192, 11, 0, 0, 0); + var allocator = new PooledByteBufferAllocator(true, 1, 0, 8192, 11, 0, 0, 0); IByteBuffer buffer = allocator.HeapBuffer(ChunkSize); IReadOnlyList arenas = allocator.Metric.HeapArenas(); Assert.Equal(1, arenas.Count); diff --git a/test/DotNetty.Buffers.Tests/PooledDirectByteBufferTests.cs b/test/DotNetty.Buffers.Tests/PooledDirectByteBufferTests.cs new file mode 100644 index 000000000..c6aa07e74 --- /dev/null +++ b/test/DotNetty.Buffers.Tests/PooledDirectByteBufferTests.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace DotNetty.Buffers.Tests +{ + public sealed class PooledDirectByteBufferTests : AbstractPooledByteBufferTests + { + protected override IByteBuffer Alloc(int length, int maxCapacity) => PooledByteBufferAllocator.Default.DirectBuffer(length, maxCapacity); + } +} diff --git a/test/DotNetty.Buffers.Tests/SimpleLeakAwareByteBufferTests.cs b/test/DotNetty.Buffers.Tests/SimpleLeakAwareByteBufferTests.cs index f48fde9dc..c87d55e2a 100644 --- a/test/DotNetty.Buffers.Tests/SimpleLeakAwareByteBufferTests.cs +++ b/test/DotNetty.Buffers.Tests/SimpleLeakAwareByteBufferTests.cs @@ -48,134 +48,76 @@ public override void Dispose() } [Fact] - public void WrapSlice() - { - IByteBuffer buf = null; - - try - { - buf = this.NewBuffer(8).Slice(); - Assert.IsType(this.ByteBufferType, buf); - } - finally - { - buf?.Release(); - } - } + public void WrapSlice() => this.AssertWrapped(this.NewBuffer(8).Slice()); [Fact] - public void WrapSlice2() - { - IByteBuffer buf = null; - - try - { - buf = this.NewBuffer(8).Slice(0, 1); - Assert.IsType(this.ByteBufferType, buf); - } - finally - { - buf?.Release(); - } - } + public void WrapSlice2() => this.AssertWrapped(this.NewBuffer(8).Slice(0, 1)); [Fact] public void WrapReadSlice() { - IByteBuffer buf = null; - - try + IByteBuffer buffer = this.NewBuffer(8); + if (buffer.IsReadable()) { - buf = this.NewBuffer(8).ReadSlice(1); - Assert.IsType(this.ByteBufferType, buf); + this.AssertWrapped(buffer.ReadSlice(1)); } - finally + else { - buf?.Release(); + Assert.True(buffer.Release()); } } [Fact] public void WrapRetainedSlice() { - IByteBuffer buf = null; - try - { - buf = this.NewBuffer(8).RetainedSlice(); - Assert.IsType(this.ByteBufferType, buf); - } - finally - { - buf?.Release(); - } - - Assert.True(buf.Release()); + IByteBuffer buffer = this.NewBuffer(8); + this.AssertWrapped(buffer.RetainedSlice()); + Assert.True(buffer.Release()); } [Fact] public void WrapRetainedSlice2() { - IByteBuffer buf = null; - try - { - buf = this.NewBuffer(8).RetainedSlice(0, 1); - Assert.IsType(this.ByteBufferType, buf); - } - finally + IByteBuffer buffer = this.NewBuffer(8); + if (buffer.IsReadable()) { - buf?.Release(); + this.AssertWrapped(buffer.RetainedSlice(0, 1)); } - - Assert.True(buf.Release()); + Assert.True(buffer.Release()); } [Fact] public void WrapReadRetainedSlice() { - IByteBuffer buf = null; - try - { - buf = this.NewBuffer(8).ReadRetainedSlice(1); - Assert.IsType(this.ByteBufferType, buf); - } - finally + IByteBuffer buffer = this.NewBuffer(8); + if (buffer.IsReadable()) { - buf?.Release(); + this.AssertWrapped(buffer.ReadRetainedSlice(1)); } - - Assert.True(buf.Release()); + Assert.True(buffer.Release()); } [Fact] - public void WrapDuplicate() - { - IByteBuffer buf = null; - try - { - buf = this.NewBuffer(8).Duplicate(); - Assert.IsType(this.ByteBufferType, buf); - } - finally - { - buf?.Release(); - } - } + public void WrapDuplicate() => this.AssertWrapped(this.NewBuffer(8).Duplicate()); [Fact] public void WrapRetainedDuplicate() { - IByteBuffer buf = null; + IByteBuffer buffer = this.NewBuffer(8); + this.AssertWrapped(buffer.RetainedDuplicate()); + Assert.True(buffer.Release()); + } + + protected void AssertWrapped(IByteBuffer buf) + { try { - buf = this.NewBuffer(8).RetainedDuplicate(); Assert.IsType(this.ByteBufferType, buf); } finally { - buf?.Release(); + buf.Release(); } - - Assert.True(buf.Release()); } } } diff --git a/test/DotNetty.Buffers.Tests/SimpleLeakAwareCompositeByteBufferTests.cs b/test/DotNetty.Buffers.Tests/SimpleLeakAwareCompositeByteBufferTests.cs index 113a7c221..37011af69 100644 --- a/test/DotNetty.Buffers.Tests/SimpleLeakAwareCompositeByteBufferTests.cs +++ b/test/DotNetty.Buffers.Tests/SimpleLeakAwareCompositeByteBufferTests.cs @@ -28,7 +28,7 @@ public override void Dispose() { base.Dispose(); - for (; ; ) + for (; ;) { NoopResourceLeakTracker tracker = null; if (this.trackers.Count > 0) @@ -46,134 +46,76 @@ public override void Dispose() } [Fact] - public void WrapSlice() - { - IByteBuffer buf = null; - - try - { - buf = this.NewBuffer(8).Slice(); - Assert.IsType(this.ByteBufferType, buf); - } - finally - { - buf?.Release(); - } - } + public void WrapSlice() => this.AssertWrapped(this.NewBuffer(8).Slice()); [Fact] - public void WrapSlice2() - { - IByteBuffer buf = null; - - try - { - buf = this.NewBuffer(8).Slice(0, 1); - Assert.IsType(this.ByteBufferType, buf); - } - finally - { - buf?.Release(); - } - } + public void WrapSlice2() => this.AssertWrapped(this.NewBuffer(8).Slice(0, 1)); [Fact] public void WrapReadSlice() { - IByteBuffer buf = null; - - try + IByteBuffer buffer = this.NewBuffer(8); + if (buffer.IsReadable()) { - buf = this.NewBuffer(8).ReadSlice(1); - Assert.IsType(this.ByteBufferType, buf); + this.AssertWrapped(buffer.ReadSlice(1)); } - finally + else { - buf?.Release(); + Assert.True(buffer.Release()); } } [Fact] public void WrapRetainedSlice() { - IByteBuffer buf = null; - try - { - buf = this.NewBuffer(8).RetainedSlice(); - Assert.IsType(this.ByteBufferType, buf); - } - finally - { - buf?.Release(); - } - - Assert.True(buf.Release()); + IByteBuffer buffer = this.NewBuffer(8); + this.AssertWrapped(buffer.RetainedSlice()); + Assert.True(buffer.Release()); } [Fact] public void WrapRetainedSlice2() { - IByteBuffer buf = null; - try - { - buf = this.NewBuffer(8).RetainedSlice(0, 1); - Assert.IsType(this.ByteBufferType, buf); - } - finally + IByteBuffer buffer = this.NewBuffer(8); + if (buffer.IsReadable()) { - buf?.Release(); + this.AssertWrapped(buffer.RetainedSlice(0, 1)); } - - Assert.True(buf.Release()); + Assert.True(buffer.Release()); } [Fact] public void WrapReadRetainedSlice() { - IByteBuffer buf = null; - try - { - buf = this.NewBuffer(8).ReadRetainedSlice(1); - Assert.IsType(this.ByteBufferType, buf); - } - finally + IByteBuffer buffer = this.NewBuffer(8); + if (buffer.IsReadable()) { - buf?.Release(); + this.AssertWrapped(buffer.ReadRetainedSlice(1)); } - - Assert.True(buf.Release()); + Assert.True(buffer.Release()); } [Fact] - public void WrapDuplicate() - { - IByteBuffer buf = null; - try - { - buf = this.NewBuffer(8).Duplicate(); - Assert.IsType(this.ByteBufferType, buf); - } - finally - { - buf?.Release(); - } - } + public void WrapDuplicate() => this.AssertWrapped(this.NewBuffer(8).Duplicate()); [Fact] public void WrapRetainedDuplicate() { - IByteBuffer buf = null; + IByteBuffer buffer = this.NewBuffer(8); + this.AssertWrapped(buffer.RetainedDuplicate()); + Assert.True(buffer.Release()); + } + + protected void AssertWrapped(IByteBuffer buf) + { try { - buf = this.NewBuffer(8).RetainedDuplicate(); Assert.IsType(this.ByteBufferType, buf); } finally { - buf?.Release(); + buf.Release(); } - - Assert.True(buf.Release()); } } } diff --git a/test/DotNetty.Buffers.Tests/UnpooledByteBufferAllocatorTests.cs b/test/DotNetty.Buffers.Tests/UnpooledByteBufferAllocatorTests.cs index 6f77e8e7d..7e338ac84 100644 --- a/test/DotNetty.Buffers.Tests/UnpooledByteBufferAllocatorTests.cs +++ b/test/DotNetty.Buffers.Tests/UnpooledByteBufferAllocatorTests.cs @@ -5,8 +5,8 @@ namespace DotNetty.Buffers.Tests { public class UnpooledByteBufferAllocatorTests : AbstractByteBufferAllocatorTests { - protected override IByteBufferAllocator NewAllocator() => new UnpooledByteBufferAllocator(); + protected override IByteBufferAllocator NewAllocator(bool preferDirect) => new UnpooledByteBufferAllocator(preferDirect); - protected override IByteBufferAllocator NewUnpooledAllocator() => new UnpooledByteBufferAllocator(); + protected override IByteBufferAllocator NewUnpooledAllocator() => new UnpooledByteBufferAllocator(false); } } diff --git a/test/DotNetty.Buffers.Tests/UnreleaseableByteBufferTests.cs b/test/DotNetty.Buffers.Tests/UnreleaseableByteBufferTests.cs index 3ddb21272..f9650d475 100644 --- a/test/DotNetty.Buffers.Tests/UnreleaseableByteBufferTests.cs +++ b/test/DotNetty.Buffers.Tests/UnreleaseableByteBufferTests.cs @@ -1,4 +1,6 @@ - +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + namespace DotNetty.Buffers.Tests { using Xunit; diff --git a/test/DotNetty.Buffers.Tests/UnsafeDirectByteBufferTest.cs b/test/DotNetty.Buffers.Tests/UnsafeDirectByteBufferTest.cs new file mode 100644 index 000000000..2ee85814f --- /dev/null +++ b/test/DotNetty.Buffers.Tests/UnsafeDirectByteBufferTest.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace DotNetty.Buffers.Tests +{ + using Xunit; + + public class UnsafeDirectByteBufferTest : AbstractByteBufferTests + { + protected override IByteBuffer NewBuffer(int length, int maxCapacity) + { + IByteBuffer buffer = this.NewDirectBuffer(length, maxCapacity); + Assert.Equal(0, buffer.WriterIndex); + return buffer; + } + + protected IByteBuffer NewDirectBuffer(int length, int maxCapacity) => + new UnpooledUnsafeDirectByteBuffer(UnpooledByteBufferAllocator.Default, length, maxCapacity); + } +} diff --git a/test/DotNetty.Codecs.Protobuf.Tests/RoundTripTests.cs b/test/DotNetty.Codecs.Protobuf.Tests/RoundTripTests.cs index e7c7bb3cd..611ed187d 100644 --- a/test/DotNetty.Codecs.Protobuf.Tests/RoundTripTests.cs +++ b/test/DotNetty.Codecs.Protobuf.Tests/RoundTripTests.cs @@ -208,7 +208,7 @@ public void Run1(AddressBook addressBook, bool isCompositeBuffer) IByteBuffer inputBuffer; if (isCompositeBuffer) { - inputBuffer = new CompositeByteBuffer(UnpooledByteBufferAllocator.Default, 2, + inputBuffer = Unpooled.WrappedBuffer( Unpooled.CopiedBuffer(data, 0, 2), Unpooled.CopiedBuffer(data, 2, data.Length - 2)); } @@ -274,7 +274,7 @@ public void Run2(Person.Types.PhoneType phoneType, string number, bool isComposi IByteBuffer inputBuffer; if (isCompositeBuffer) { - inputBuffer = new CompositeByteBuffer(UnpooledByteBufferAllocator.Default, 2, + inputBuffer = Unpooled.WrappedBuffer( Unpooled.CopiedBuffer(data, 0, 2), Unpooled.CopiedBuffer(data, 2, data.Length - 2)); } diff --git a/test/DotNetty.Codecs.ProtocolBuffers.Tests/RoundTripTests.cs b/test/DotNetty.Codecs.ProtocolBuffers.Tests/RoundTripTests.cs index 2b83104dd..72e3cd4cb 100644 --- a/test/DotNetty.Codecs.ProtocolBuffers.Tests/RoundTripTests.cs +++ b/test/DotNetty.Codecs.ProtocolBuffers.Tests/RoundTripTests.cs @@ -118,7 +118,7 @@ public void Run1(AddressBook addressBook, bool isCompositeBuffer) IByteBuffer inputBuffer; if (isCompositeBuffer) { - inputBuffer = new CompositeByteBuffer(UnpooledByteBufferAllocator.Default, 2, + inputBuffer = Unpooled.WrappedBuffer( Unpooled.CopiedBuffer(data, 0, 2), Unpooled.CopiedBuffer(data, 2, data.Length - 2)); } @@ -186,7 +186,7 @@ public void Run2(Person.Types.PhoneType phoneType, string number, bool isComposi IByteBuffer inputBuffer; if (isCompositeBuffer) { - inputBuffer = new CompositeByteBuffer(UnpooledByteBufferAllocator.Default, 2, + inputBuffer = Unpooled.WrappedBuffer( Unpooled.CopiedBuffer(data, 0, 2), Unpooled.CopiedBuffer(data, 2, data.Length - 2)); } diff --git a/test/DotNetty.Microbench/Allocators/PooledHeapByteBufferAllocatorBenchmark.cs b/test/DotNetty.Microbench/Allocators/PooledByteBufferAllocatorBenchmark.cs similarity index 50% rename from test/DotNetty.Microbench/Allocators/PooledHeapByteBufferAllocatorBenchmark.cs rename to test/DotNetty.Microbench/Allocators/PooledByteBufferAllocatorBenchmark.cs index 1953ab761..2a4a03c30 100644 --- a/test/DotNetty.Microbench/Allocators/PooledHeapByteBufferAllocatorBenchmark.cs +++ b/test/DotNetty.Microbench/Allocators/PooledByteBufferAllocatorBenchmark.cs @@ -5,10 +5,10 @@ namespace DotNetty.Microbench.Allocators { using DotNetty.Buffers; - public class PooledHeapByteBufferAllocatorBenchmark : AbstractByteBufferAllocatorBenchmark + public class PooledByteBufferAllocatorBenchmark : AbstractByteBufferAllocatorBenchmark { - public PooledHeapByteBufferAllocatorBenchmark() - : base( new PooledByteBufferAllocator(4, 8192, 11, 0, 0, 0)) // // Disable thread-local cache + public PooledByteBufferAllocatorBenchmark() + : base(new PooledByteBufferAllocator(true, 4, 4, 8192, 11, 0, 0, 0)) // Disable thread-local cache { } } diff --git a/test/DotNetty.Microbench/Allocators/UnpooledHeapByteBufferAllocatorBenchmark.cs b/test/DotNetty.Microbench/Allocators/UnpooledByteBufferAllocatorBenchmark.cs similarity index 54% rename from test/DotNetty.Microbench/Allocators/UnpooledHeapByteBufferAllocatorBenchmark.cs rename to test/DotNetty.Microbench/Allocators/UnpooledByteBufferAllocatorBenchmark.cs index ed32f1900..f0a8ce245 100644 --- a/test/DotNetty.Microbench/Allocators/UnpooledHeapByteBufferAllocatorBenchmark.cs +++ b/test/DotNetty.Microbench/Allocators/UnpooledByteBufferAllocatorBenchmark.cs @@ -5,10 +5,9 @@ namespace DotNetty.Microbench.Allocators { using DotNetty.Buffers; - public class UnpooledHeapByteBufferAllocatorBenchmark : AbstractByteBufferAllocatorBenchmark + public class UnpooledByteBufferAllocatorBenchmark : AbstractByteBufferAllocatorBenchmark { - public UnpooledHeapByteBufferAllocatorBenchmark() - : base(new UnpooledByteBufferAllocator(true)) + public UnpooledByteBufferAllocatorBenchmark() : base(new UnpooledByteBufferAllocator(true)) { } } diff --git a/test/DotNetty.Microbench/Buffers/PooledHeapByteBufferBenchmark.cs b/test/DotNetty.Microbench/Buffers/PooledByteBufferBenchmark.cs similarity index 59% rename from test/DotNetty.Microbench/Buffers/PooledHeapByteBufferBenchmark.cs rename to test/DotNetty.Microbench/Buffers/PooledByteBufferBenchmark.cs index 40f64bb10..4da127daa 100644 --- a/test/DotNetty.Microbench/Buffers/PooledHeapByteBufferBenchmark.cs +++ b/test/DotNetty.Microbench/Buffers/PooledByteBufferBenchmark.cs @@ -10,56 +10,65 @@ namespace DotNetty.Microbench.Buffers [CoreJob] [BenchmarkCategory("ByteBuffer")] - public class PooledHeapByteBufferBenchmark + public class PooledByteBufferBenchmark { - static PooledHeapByteBufferBenchmark() + static PooledByteBufferBenchmark() { ResourceLeakDetector.Level = ResourceLeakDetector.DetectionLevel.Disabled; } - PooledHeapByteBuffer buffer; + AbstractByteBuffer unsafeBuffer; + AbstractByteBuffer buffer; [GlobalSetup] public void GlobalSetup() { - this.buffer = (PooledHeapByteBuffer)PooledByteBufferAllocator.Default.HeapBuffer(8); + this.unsafeBuffer = (AbstractByteBuffer)PooledByteBufferAllocator.Default.DirectBuffer(8); + this.buffer = (AbstractByteBuffer)PooledByteBufferAllocator.Default.HeapBuffer(8); this.buffer.WriteLong(1L); } [GlobalCleanup] public void GlobalCleanup() { + this.unsafeBuffer.Release(); this.buffer.Release(); } + [Benchmark] + public void CheckIndexUnsafe() => this.unsafeBuffer.CheckIndex(0, 8); + [Benchmark] public void CheckIndex() => this.buffer.CheckIndex(0, 8); + [Benchmark] + public byte GetByteUnsafe() => this.unsafeBuffer.GetByte(0); + [Benchmark] public byte GetByte() => this.buffer.GetByte(0); + [Benchmark] + public short GetShortUnsafe() => this.unsafeBuffer.GetShort(0); + [Benchmark] public short GetShort() => this.buffer.GetShort(0); [Benchmark] - public short GetShortLE() => this.buffer.GetShortLE(0); + public int GetMediumUnsafe() => this.unsafeBuffer.GetMedium(0); [Benchmark] public int GetMedium() => this.buffer.GetMedium(0); [Benchmark] - public int GetMediumLE() => this.buffer.GetMediumLE(0); + public int GetIntUnsafe() => this.unsafeBuffer.GetInt(0); [Benchmark] public int GetInt() => this.buffer.GetInt(0); [Benchmark] - public int GetIntLE() => this.buffer.GetIntLE(0); + public long GetLongUnsafe() => this.unsafeBuffer.GetLong(0); [Benchmark] public long GetLong() => this.buffer.GetLong(0); - - [Benchmark] - public long GetLongLE() => this.buffer.GetLongLE(0); } } diff --git a/test/DotNetty.Microbench/Buffers/UnpooledHeapByteBufferBenchmark.cs b/test/DotNetty.Microbench/Buffers/UnpooledByteBufferBenchmark.cs similarity index 65% rename from test/DotNetty.Microbench/Buffers/UnpooledHeapByteBufferBenchmark.cs rename to test/DotNetty.Microbench/Buffers/UnpooledByteBufferBenchmark.cs index 31a97825e..bc1d3bdf5 100644 --- a/test/DotNetty.Microbench/Buffers/UnpooledHeapByteBufferBenchmark.cs +++ b/test/DotNetty.Microbench/Buffers/UnpooledByteBufferBenchmark.cs @@ -10,18 +10,20 @@ namespace DotNetty.Microbench.Buffers [CoreJob] [BenchmarkCategory("ByteBuffer")] - public class UnpooledHeapByteBufferBenchmark + public class UnpooledByteBufferBenchmark { - static UnpooledHeapByteBufferBenchmark() + static UnpooledByteBufferBenchmark() { ResourceLeakDetector.Level = ResourceLeakDetector.DetectionLevel.Disabled; } - UnpooledHeapByteBuffer buffer; + IByteBuffer unsafeBuffer; + IByteBuffer buffer; [GlobalSetup] public void GlobalSetup() { + this.unsafeBuffer = new UnpooledUnsafeDirectByteBuffer(UnpooledByteBufferAllocator.Default, 8, int.MaxValue); this.buffer = new UnpooledHeapByteBuffer(UnpooledByteBufferAllocator.Default, 8, int.MaxValue); this.buffer.WriteLong(1L); } @@ -29,34 +31,38 @@ public void GlobalSetup() [GlobalCleanup] public void GlobalCleanup() { + this.unsafeBuffer.Release(); this.buffer.Release(); } + [Benchmark] + public byte GetByteUnsafe() => this.unsafeBuffer.GetByte(0); + [Benchmark] public byte GetByte() => this.buffer.GetByte(0); + [Benchmark] + public short GetShortUnsafe() => this.unsafeBuffer.GetShort(0); + [Benchmark] public short GetShort() => this.buffer.GetShort(0); [Benchmark] - public short GetShortLE() => this.buffer.GetShortLE(0); + public int GetMediumUnsafe() => this.unsafeBuffer.GetMedium(0); [Benchmark] public int GetMedium() => this.buffer.GetMedium(0); [Benchmark] - public int GetMediumLE() => this.buffer.GetMediumLE(0); + public int GetIntUnsafe() => this.unsafeBuffer.GetInt(0); [Benchmark] public int GetInt() => this.buffer.GetInt(0); [Benchmark] - public int GetIntLE() => this.buffer.GetIntLE(0); + public long GetLongUnsafe() => this.unsafeBuffer.GetLong(0); [Benchmark] public long GetLong() => this.buffer.GetLong(0); - - [Benchmark] - public long GetLongLE() => this.buffer.GetLongLE(0); } } diff --git a/test/DotNetty.Microbench/Program.cs b/test/DotNetty.Microbench/Program.cs index c90542064..605a5c485 100644 --- a/test/DotNetty.Microbench/Program.cs +++ b/test/DotNetty.Microbench/Program.cs @@ -13,11 +13,11 @@ class Program { static readonly Type[] BenchmarkTypes = { - typeof(PooledHeapByteBufferAllocatorBenchmark), - typeof(UnpooledHeapByteBufferAllocatorBenchmark), + typeof(PooledByteBufferAllocatorBenchmark), + typeof(UnpooledByteBufferAllocatorBenchmark), typeof(ByteBufferBenchmark), - typeof(UnpooledHeapByteBufferBenchmark), - typeof(PooledHeapByteBufferBenchmark), + typeof(UnpooledByteBufferBenchmark), + typeof(PooledByteBufferBenchmark), typeof(FastThreadLocalBenchmark), typeof(SingleThreadEventExecutorBenchmark) }; diff --git a/test/DotNetty.Transport.Tests/Channel/Sockets/SocketDatagramChannelUnicastTest.cs b/test/DotNetty.Transport.Tests/Channel/Sockets/SocketDatagramChannelUnicastTest.cs index 46f8596fe..cce20f53f 100644 --- a/test/DotNetty.Transport.Tests/Channel/Sockets/SocketDatagramChannelUnicastTest.cs +++ b/test/DotNetty.Transport.Tests/Channel/Sockets/SocketDatagramChannelUnicastTest.cs @@ -120,9 +120,8 @@ static IEnumerable GetData() yield return new object[] { - new CompositeByteBuffer( - UnpooledByteBufferAllocator.Default, - 2, Unpooled.CopiedBuffer(Data, 0, 2), Unpooled.CopiedBuffer(Data, 2, 2)), + Unpooled.WrappedBuffer( + Unpooled.CopiedBuffer(Data, 0, 2), Unpooled.CopiedBuffer(Data, 2, 2)), bindClient, allocator, addressFamily, @@ -132,9 +131,8 @@ static IEnumerable GetData() yield return new object[] { - new CompositeByteBuffer( - UnpooledByteBufferAllocator.Default, - 2, Unpooled.CopiedBuffer(Data, 0, 2), Unpooled.CopiedBuffer(Data, 2, 2)), + Unpooled.WrappedBuffer( + Unpooled.CopiedBuffer(Data, 0, 2), Unpooled.CopiedBuffer(Data, 2, 2)), bindClient, allocator, addressFamily,