Define constants in a central place

georg-jung · Nov 27, 2023 · d2bc474 · d2bc474
1 parent 41ab5aa
commit d2bc474
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/src/FastBertTokenizer/BertTokenizer.cs b/src/FastBertTokenizer/BertTokenizer.cs
@@ -230,7 +230,7 @@ bool NeedsRemoval(ReadOnlySpan<char> formD)
         }
 
         int i = 0;
-        Span<char> span = normalizedString.Length < 1000
+        Span<char> span = normalizedString.Length < Constants.StackallocCharThreshold
             ? stackalloc char[normalizedString.Length]
             : new char[normalizedString.Length];
 

diff --git a/src/FastBertTokenizer/Constants.cs b/src/FastBertTokenizer/Constants.cs
@@ -0,0 +1,12 @@
+// Copyright (c) Georg Jung. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace FastBertTokenizer
+{
+    internal static class Constants
+    {
+        // https://github.com/dotnet/runtime/blob/6ed57ec3d26353549d7bffe5055a7100f443ff5b/src/libraries/System.Text.Json/Common/JsonConstants.cs#L12C33-L12C33
+        public const int StackallocByteThreshold = 256;
+        public const int StackallocCharThreshold = StackallocByteThreshold / 2;
+    }
+}