Fix match model to encoding error.

aiqinxuancai · May 14, 2024 · 598b72e · 598b72e
1 parent c07d9d2
commit 598b72e
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 10 deletions.
diff --git a/TiktokenSharp.Test/Program.cs b/TiktokenSharp.Test/Program.cs
@@ -25,7 +25,8 @@ static void Main(string[] args)
 
         static void GPT4o()
         {
-            TikToken tikToken = TikToken.GetEncoding("o200k_base");
+            TikToken tikToken = TikToken.EncodingForModel("gpt-4o");
+            //TikToken tikToken = TikToken.GetEncoding("o200k_base");
             var i = tikToken.Encode("hello world");
             var d = tikToken.Decode(i);
 

diff --git a/TiktokenSharp/Services/EncodingManager.cs b/TiktokenSharp/Services/EncodingManager.cs
@@ -90,20 +90,23 @@ public static EncodingManager Instance
         /// <returns></returns>
         public EncodingSettingModel GetEncodingSetting(string modelOrEncodingName)
         {
-            var encodingName = MODEL_TO_ENCODING.FirstOrDefault(a => modelOrEncodingName.StartsWith(a.Key)).Value;
+            var encodingName = "";
 
-            if (string.IsNullOrEmpty(encodingName))
+
+            if (MODEL_TO_ENCODING.Any(a => a.Value == modelOrEncodingName))
+            {
+                //modelOrEncodingName is encoding name?
+                encodingName = modelOrEncodingName;
+            }
+
+            if (MODEL_TO_ENCODING.ContainsKey(modelOrEncodingName))
             {
-                encodingName = MODEL_TO_ENCODING.FirstOrDefault(a => a.Key.StartsWith(modelOrEncodingName)).Value;
+                encodingName = MODEL_TO_ENCODING[modelOrEncodingName];
             }
 
             if (string.IsNullOrEmpty(encodingName))
             {
-                if (MODEL_TO_ENCODING.Any(a => a.Value == modelOrEncodingName))
-                {
-                    //modelOrEncodingName is encoding name
-                    encodingName = modelOrEncodingName;
-                }
+                encodingName = MODEL_TO_ENCODING.FirstOrDefault(a => a.Key.StartsWith(modelOrEncodingName)).Value; //MODEL_TO_ENCODING.FirstOrDefault(a => modelOrEncodingName.StartsWith(a.Key)).Value;
             }
 
             return GetEncoding(encodingName);

diff --git a/TiktokenSharp/TiktokenSharp.csproj b/TiktokenSharp/TiktokenSharp.csproj
@@ -10,7 +10,7 @@
 	<RepositoryUrl>https://github.com/aiqinxuancai/TiktokenSharp</RepositoryUrl>
 	<PackageTags>chatgpt;openai</PackageTags>
 	<Title>TiktokenSharp</Title>
-	<VersionPrefix Condition=" '$(VersionPrefix)' == '' ">1.1.3</VersionPrefix>
+	<VersionPrefix Condition=" '$(VersionPrefix)' == '' ">1.1.4</VersionPrefix>
 	<Authors>aiqinxuancai</Authors>
 	<PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
 	<EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles>