Samples (#154)

Updates samples, rework code coverage

.gitignore

@@ -13,3 +13,4 @@
 /src/Debug
 /src/Grpc/NativeGrpc/NativeGrpc.vcxproj.user
 **/coverage.net5.0.xml
+/src/Tests/Coverage/coverage.xml

README.md

@@ -106,6 +106,7 @@ FlatSharp supports some interesting features not covered here. Detailed document
 - [Indexed Vectors (Dictionary-like functionality)](samples/Example11-IndexedVectors/)
 - [Type Facades](samples/Example12-TypeFacades/)
 - [Fixed-Length Vectors](samples/Example13-StructVectors/)
+- [Write-Through -- update buffers in place](samples/Example14-WriteThrough/)
 
 ### Internals
 FlatSharp works by generating subclasses of your data contracts based on the schema that you define. That is, when you attempt to deserialize a ```MonsterTable``` object, you actually get back a subclass of ```MonsterTable```, which has properties defined in such a way as to index into the buffer, according to the deserialization mode specified (greedy, lazy, etc).

samples/Example10-SharedStrings/SharedStringsExample.cs

@@ -180,7 +180,7 @@ public void FlushWrites<TSpanWriter>(TSpanWriter writer, Span<byte> data, Serial
                 // Update all the pointers that need to point to that string.
                 foreach (var offset in offsets)
                 {
-                    writer.WriteUOffset(data, offset, stringOffset, context);
+                    writer.WriteUOffset(data, offset, stringOffset);
                 }
             }
         }

samples/Example14-WriteThrough/WriteThrough.cs

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2021 James Courtney
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Samples.WriteThrough
+{
+    using System;
+    using System.Diagnostics;
+    using FlatSharp;
+
+    /// <summary>
+    /// FlatSharp supports Write-Through in limited cases:
+    /// - Serialization method is VectorCacheMutable
+    /// - Struct field has been opted into write-through.
+    /// 
+    /// Write Through allows you to make updates to an already-serialized FlatBuffer in-place without a full parse or re-serialize.
+    /// This is extremely performant, especially for large buffers as it avoids series of copies and allows in-place updates.
+    /// </summary>
+    public class WriteThroughSample
+    {
+        public static void Run()
+        {
+            Roster roster = new Roster
+            {
+                Walkers = new[]
+                {
+                    new DogWalker { Name = "Beth", CurrentDog = new DogReference { Id = 7 }, Position = new Location { Latitude = 1.2, Longitude = 2.1, NoWriteThrough = 10 } },
+                    new DogWalker { Name = "Jerry", CurrentDog = new DogReference { Id = 127 }, Position = new Location { Latitude = 12.7, Longitude = 128 } },
+                    new DogWalker { Name = "Summer", CurrentDog = new DogReference { Id = 22 }, Position = new Location { Latitude = 36.4, Longitude = 32.1 } },
+                }
+            };
+
+            // Write the data to the buffer the first time.
+            byte[] data = new byte[1024];
+            Roster.Serializer.Write(data, roster);
+
+            // Parsed now refers to the buffer. Any changes we make to the fs_writeThrough fields will be reflected back.
+            Roster parsed = Roster.Serializer.Parse(data);
+            var walker = parsed.Walkers![0];
+
+            // Walking dog 8 in Tokyo now.
+            walker.CurrentDog!.Id = 8;
+            walker.Position!.Latitude = 35.683;
+            walker.Position.Longitude = 139.808;
+
+            // we can update this field, but since writethrough is disabled, the changes won't be reflected in the buffer.
+            walker.Position.NoWriteThrough = 1024; 
+
+            // Now let's parse again and see our changes.
+            // Notice that we never invoked .Write to rewrite the whole structure back.
+            // We only mutated a few fields.
+            parsed = Roster.Serializer.Parse(data);
+
+            walker = parsed.Walkers![0];
+
+            Debug.Assert(walker.CurrentDog!.Id == 8, "Dog has been updated. Was 7 originally.");
+            Debug.Assert(walker.Position!.Latitude == 35.683, "Latitude has been updated. Was 1.2 originally.");
+            Debug.Assert(walker.Position!.Longitude == 139.808, "Longitude has been updated. Was 2.1 originally.");
+            Debug.Assert(walker.Position!.NoWriteThrough == 10, "NoWriteThrough was unchanged.");
+        }
+    }
+}

samples/Example14-WriteThrough/WriteThrough.fbs

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2021 James Courtney
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+// WriteThrough is great when you only need to make small changes to a FlatBuffer and don't want to pay the cost
+// of a complete read-modify-write sequence. With WriteThrough enabled, you can make select updates directly back to the buffer.
+
+// WriteThrough is only applicable to structs, and is only available when using "VectorCacheMutable" as the serializer.
+
+namespace Samples.WriteThrough;
+
+// In this contrived example, imagine an animal shelter. There are lots of volunteers walking different dogs.
+// Each volunteer only has one dog at a time, but they can change dogs quite frequently. We'd like to
+// be able to update the current dog that each volunteer is walking without doing a full parse and re-serialize
+// of the entire list of volunteers. There are also lots of dog-nappers around, so we need to be able to update
+// the last known location of a given volunteer so we can call the police if they go missing.
+
+table Roster (fs_serializer:"VectorCacheMutable") {
+   Walkers : [DogWalker];
+}
+
+table DogWalker {
+    Name : string;
+    CurrentDog : DogReference;
+    Position : Location;
+}
+
+struct DogReference { Id : int (fs_writeThrough); }
+
+// Can be applied to all items in a struct.
+struct Location (fs_writeThrough) { 
+    Latitude : double; 
+    Longitude : double; 
+    NoWriteThrough : int (fs_writeThrough:"false"); 
+}

samples/Samples.csproj

@@ -21,16 +21,17 @@
     <FlatSharpSchema Include="Example10-SharedStrings\SharedStrings.fbs" />
     <FlatSharpSchema Include="Example11-IndexedVectors\IndexedVectors.fbs" />
     <FlatSharpSchema Include="Example13-StructVectors\StructVectors.fbs" />
+    <FlatSharpSchema Include="Example14-WriteThrough\WriteThrough.fbs" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="FlatSharp" Version="5.0.0" />
-    <PackageReference Include="FlatSharp.Compiler" Version="5.0.0">
+    <PackageReference Include="FlatSharp" Version="5.3.0" />
+    <PackageReference Include="FlatSharp.Compiler" Version="5.3.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="FlatSharp.Runtime" Version="5.0.0" />
-    <PackageReference Include="FlatSharp.Unsafe" Version="5.0.0" />
+    <PackageReference Include="FlatSharp.Runtime" Version="5.3.0" />
+    <PackageReference Include="FlatSharp.Unsafe" Version="5.3.0" />
     <PackageReference Include="Grpc" Version="2.36.1" />
   </ItemGroup>
 

src/FlatSharp.Compiler/TypeDefinitions/StructVectorDefinition.cs

@@ -121,11 +121,6 @@ public void EmitStructVector(TableOrStructDefinition parent, CodeWriter writer,
                     {
                         writer.AppendLine($"yield return thisItem.{this.PropertyNames[i]};");
                     }
-
-                    if (this.PropertyNames.Count == 0)
-                    {
-                        writer.AppendLine("yield break;");
-                    }
                 }
 
                 string arrayOrSpanType = $"{typeName}[]";

src/FlatSharp.Compiler/Visitors/FieldVisitor.cs

@@ -81,6 +81,10 @@ public override bool VisitField_decl([NotNull] FlatBuffersParser.Field_declConte
                 {
                     ErrorContext.Current.RegisterError("Only structs may contain fixed-length vectors");
                 }
+                else if (structVectorLength.Value <= 0)
+                {
+                    ErrorContext.Current.RegisterError("Struct vector lengths must be non-negative.");
+                }
                 else
                 {
                     List<string> groupNames = new List<string>();

src/FlatSharp/Serialization/DeserializeClassDefinition.cs

@@ -93,8 +93,6 @@ public DeserializeClassDefinition(
 
         public bool HasEmbeddedBufferReference => !this.options.GreedyDeserialize;
 
-        public bool HasEmbeddedVTableInfo => !this.options.GreedyDeserialize && this.typeModel.SchemaType == FlatBufferSchemaType.Table;
-
         public string ClassName { get; }
 
         public void AddProperty(ItemMemberModel itemModel, string readValueMethodName, string writeValueMethodName)
@@ -203,22 +201,19 @@ private void AddPropertyDefinitions(ItemMemberModel itemModel, string writeValue
                 }
             }
 
-            if (itemModel.PropertyInfo.SetMethod is not null)
+            MethodInfo? setMethod = itemModel.PropertyInfo.SetMethod;
+            if (setMethod is not null)
             {
                 string verb = "set";
                 string setterBody;
 
-                MethodInfo? methodInfo = itemModel.PropertyInfo.SetMethod;
-                if (methodInfo is not null)
+                // see if set is init-only.
+                bool isInitOnly = setMethod.ReturnParameter.GetRequiredCustomModifiers().Any(
+                    x => x.FullName == "System.Runtime.CompilerServices.IsExternalInit");
+
+                if (isInitOnly)
                 {
-                    // see if set is init-only.
-                    bool isInitOnly = methodInfo.ReturnParameter.GetRequiredCustomModifiers().Any(
-                        x => x.FullName == "System.Runtime.CompilerServices.IsExternalInit");
-
-                    if (isInitOnly)
-                    {
-                        verb = "init";
-                    }
+                    verb = "init";
                 }
 
                 if (!this.options.GenerateMutableObjects)
@@ -261,19 +256,15 @@ private void AddCtorStatements(ItemMemberModel itemModel)
                 assignment = $"this.{GetFieldName(itemModel)}";
             }
 
-            if (!this.options.GreedyDeserialize && itemModel.IsVirtual)
+            if (this.options.GreedyDeserialize || !itemModel.IsVirtual)
             {
-                if (itemModel.ItemTypeModel.ClassifyContextually(this.typeModel.SchemaType) == (ContextualTypeModelClassification.ReferenceType | ContextualTypeModelClassification.Required))
-                {
-                    this.ctorStatements.Add($"{assignment} = null!;");
-                }
-
-                return;
+                this.ctorStatements.Add($"{assignment} = {GetReadIndexMethodName(itemModel)}(buffer, offset, {this.vtableOffsetAccessor}, {this.vtableMaxIndexAccessor});");
             }
-            else 
+            else if (
+                !this.options.Lazy &&
+                itemModel.ItemTypeModel.ClassifyContextually(this.typeModel.SchemaType).IsRequiredReference())
             {
-
-                this.ctorStatements.Add($"{assignment} = {GetReadIndexMethodName(itemModel)}(buffer, offset, {this.vtableOffsetAccessor}, {this.vtableMaxIndexAccessor});");
+                this.ctorStatements.Add($"{assignment} = null!;");
             }
         }
 

src/FlatSharp/TypeModel/UnionTypeModel.cs

@@ -99,7 +99,7 @@ public override CodeGeneratedMethod CreateGetMaxSizeMethodBody(GetMaxSizeCodeGen
                 string @case =
 $@"
                     case {unionIndex}:
-                        return {context.MethodNameMap[unionMember.ClrType]}({context.ValueVariableName}.Item{unionIndex});";
+                        return {sizeof(uint) + SerializationHelpers.GetMaxPadding(sizeof(uint))} + {context.MethodNameMap[unionMember.ClrType]}({context.ValueVariableName}.Item{unionIndex});";
 
                 switchCases.Add(@case);
             }

src/Tests/Coverage/Coverage.csproj

@@ -0,0 +1,41 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\..\common.props" />
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <IsPackable>false</IsPackable>
+    <AssemblyName>FlatSharpCompilerTests</AssemblyName>
+    <RootNamespace>FlatSharpTests</RootNamespace>
+    <DelaySign>false</DelaySign>
+    <SignAssembly>true</SignAssembly>
+    <Nullable>annotations</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="..\FlatSharpTests\**\*.cs">
+      <Link>FS\%(RecursiveDir)%(Filename)%(Extension)</Link>
+    </Compile>
+    <Compile Include="..\FlatSharpCompilerTests\**\*.cs">
+      <Link>Compiler\%(RecursiveDir)%(Filename)%(Extension)</Link>
+    </Compile>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="AltCover" Version="6.7.750" />
+    <PackageReference Include="Grpc.Core" Version="2.27.0" />
+    <PackageReference Include="Grpc.Core.Api" Version="2.35.0" />
+    <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.3" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.1.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.1.2" />
+    <PackageReference Include="ReportGenerator" Version="4.1.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\FlatSharp.Compiler\FlatSharp.Compiler.csproj" />
+    <ProjectReference Include="..\..\FlatSharp.Runtime\FlatSharp.Runtime.csproj" />
+    <ProjectReference Include="..\..\FlatSharp.Unsafe\FlatSharp.Unsafe.csproj" />
+    <ProjectReference Include="..\..\FlatSharp\FlatSharp.csproj" />
+    <ProjectReference Include="..\..\Google.FlatBuffers\Google.FlatBuffers.csproj" />
+  </ItemGroup>
+</Project>

src/Tests/Coverage/testsWithCoverage.cmd

@@ -0,0 +1,2 @@
+dotnet test /p:AltCoverForce=true /p:AltCover=true /p:AltCoverAttributeFilter=ExcludeFromCodeCoverage /p:AltCoverStrongNameKey=..\..\..\misc\strongname.snk -v d -f net5.0
+dotnet %UserProfile%\.nuget\packages\reportgenerator\4.1.2\tools\netcoreapp2.1\ReportGenerator.dll -reports:coverage.xml -targetdir:.coverage\

src/Tests/FlatSharpCompilerTests/FlatSharpCompilerTests.csproj

@@ -13,14 +13,12 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AltCover" Version="6.7.750" />
     <PackageReference Include="Grpc.Core" Version="2.27.0" />
     <PackageReference Include="Grpc.Core.Api" Version="2.35.0" />
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.8.3" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.1.2" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.2" />
-    <PackageReference Include="ReportGenerator" Version="4.1.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.1.2" /> 
   </ItemGroup>
 
   <ItemGroup>

src/Tests/FlatSharpCompilerTests/StructVectorTests.cs

@@ -35,6 +35,10 @@ public class StructVectorTests
         public void StructVector_Byte_NegativeLength() 
             => Assert.ThrowsException<InvalidFbsFileException>(() => this.RunTest<byte>("ubyte", -3, FlatBufferDeserializationOption.Greedy));
 
+        [TestMethod]
+        public void StructVector_Byte_ZeroLength()
+            => Assert.ThrowsException<InvalidFbsFileException>(() => this.RunTest<byte>("ubyte", 0, FlatBufferDeserializationOption.Greedy));
+
         [TestMethod]
         public void StructVector_Byte() => this.RunTest<byte>("ubyte", 1, FlatBufferDeserializationOption.Greedy);