Optimize model name matching encoding.

aiqinxuancai · Dec 24, 2024 · 90a1f35 · 90a1f35
1 parent 535350e
commit 90a1f35
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 5 deletions.
diff --git a/README.md b/README.md
@@ -86,6 +86,9 @@ public int TiktokenSharp()
 
 ## Update
 
+### 1.1.6 20241224
+* Optimize model name matching encoding.
+
 ### 1.1.5 20240913
 * Add Support o1 models(o200k_base).
 

diff --git a/TiktokenSharp.Test/Program.cs b/TiktokenSharp.Test/Program.cs
@@ -25,7 +25,7 @@ static void Main(string[] args)
 
         static void GPT4o()
         {
-            TikToken tikToken = TikToken.EncodingForModel("gpt-4o");
+            TikToken tikToken = TikToken.EncodingForModel("gpt-4o-vision");
             //TikToken tikToken = TikToken.GetEncoding("o200k_base");
             var i = tikToken.Encode("hello world");
             var d = tikToken.Decode(i);

diff --git a/TiktokenSharp/Services/EncodingManager.cs b/TiktokenSharp/Services/EncodingManager.cs
@@ -39,6 +39,8 @@ public static EncodingManager Instance
                                                             { "o1-preview", "o200k_base" }, // Auto match o1-preview-{date}
                                                             { "o1-mini", "o200k_base" },  // Auto match o1-mini-{date}
 
+                                                            //gpt-4-vision-preview
+
                                                             // chat
                                                             { "gpt-4", "cl100k_base" },
                                                             { "gpt-4o", "o200k_base" },
@@ -106,7 +108,7 @@ public EncodingSettingModel GetEncodingSetting(string modelOrEncodingName)
                 encodingName = modelOrEncodingName;
             }
 
-            if (MODEL_TO_ENCODING.ContainsKey(modelOrEncodingName))
+            if (string.IsNullOrEmpty(encodingName) && MODEL_TO_ENCODING.ContainsKey(modelOrEncodingName))
             {
                 encodingName = MODEL_TO_ENCODING[modelOrEncodingName];
             }
@@ -116,6 +118,14 @@ public EncodingSettingModel GetEncodingSetting(string modelOrEncodingName)
                 encodingName = MODEL_TO_ENCODING.FirstOrDefault(a => a.Key.StartsWith(modelOrEncodingName)).Value; //MODEL_TO_ENCODING.FirstOrDefault(a => modelOrEncodingName.StartsWith(a.Key)).Value;
             }
 
+            if (string.IsNullOrEmpty(encodingName))
+            {
+                encodingName = MODEL_TO_ENCODING
+                    .OrderByDescending(x => x.Key.Length)
+                    .FirstOrDefault(a => modelOrEncodingName.StartsWith(a.Key))
+                    .Value;
+            }
+
             return GetEncoding(encodingName);
         }
 

diff --git a/TiktokenSharp/TiktokenSharp.csproj b/TiktokenSharp/TiktokenSharp.csproj
@@ -1,16 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>.net8;.net7;netstandard2.1;netstandard2.0</TargetFrameworks>
+    <TargetFrameworks>.net9;.net8;.net7;netstandard2.1;netstandard2.0</TargetFrameworks>
 	<LangVersion>latest</LangVersion>
     <Nullable>enable</Nullable>
-	<Description>Token calculation for OpenAI gpt-3.5-turbo;gpt-4;gpt-4o models.</Description>
+	<Description>Token calculation for OpenAI gpt-3.5-turbo;gpt-4;gpt-4o;o1 models.</Description>
 	<PackageProjectUrl>https://github.com/aiqinxuancai/TiktokenSharp</PackageProjectUrl>
 	<PackageReadmeFile>README.md</PackageReadmeFile>
 	<RepositoryUrl>https://github.com/aiqinxuancai/TiktokenSharp</RepositoryUrl>
 	<PackageTags>chatgpt;openai</PackageTags>
 	<Title>TiktokenSharp</Title>
-	<VersionPrefix Condition=" '$(VersionPrefix)' == '' ">1.1.5</VersionPrefix>
+	<VersionPrefix Condition=" '$(VersionPrefix)' == '' ">1.1.6</VersionPrefix>
 	<Authors>aiqinxuancai</Authors>
 	<PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
 	<EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles>