Merge pull request #8 from GOOD21/main

Update tokenizer.go
tiktoken-go · Nov 7, 2024 · 60becaa · 60becaa
2 parents 53141e4 + ad6a208
commit 60becaa
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/tokenizer.go b/tokenizer.go
@@ -72,6 +72,7 @@ package tokenizer
 //}
 
 import (
+	"strings"
 	"errors"
 
 	"github.com/tiktoken-go/tokenizer/codec"
@@ -200,7 +201,7 @@ func ForModel(model Model) (Codec, error) {
 		return Get(GPT2Enc)
 	default:
 		for prefix, enc := range modelPrefixToEncoding {
-			if string(model)[:len(prefix)] == string(prefix) {
+			if strings.HasPrefix(string(model), string(prefix)) {
 				return Get(enc)
 			}
 		}