Use lite version of @dqbd/tiktoken

langchain/scripts/check-tree-shaking.js

@@ -26,6 +26,7 @@ export function listExternals() {
     ...Object.keys(packageJson.dependencies),
     ...Object.keys(packageJson.peerDependencies),
     /node\:/,
+    /@dqbd\/tiktoken/,
     "axios", // axios is a dependency of openai
     "pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js",
     "@zilliz/milvus2-sdk-node/dist/milvus/const/Milvus.js",

langchain/src/base_language/count_tokens.ts

@@ -57,9 +57,26 @@ interface CalculateMaxTokenProps {
   modelName: TiktokenModel;
 }
 
-export const importTiktoken = async () => {
+export const importTiktoken = /* @__PURE__ */ async () => {
   try {
-    const { encoding_for_model } = await import("@dqbd/tiktoken");
+    const [{ Tiktoken }, { load }, { default: registry }, { default: models }] =
+      await Promise.all([
+        import("@dqbd/tiktoken/lite"),
+        import("@dqbd/tiktoken/load"),
+        import("@dqbd/tiktoken/registry.json"),
+        import("@dqbd/tiktoken/model_to_encoding.json"),
+      ]);
+
+    const encoding_for_model = async (modelName: TiktokenModel) => {
+      const model = await load(
+        registry[
+          models[modelName as keyof typeof models] as keyof typeof registry
+        ]
+      );
+
+      return new Tiktoken(model.bpe_ranks, model.special_tokens, model.pat_str);
+    };
+
     return { encoding_for_model };
   } catch (error) {
     console.log(error);
@@ -78,7 +95,9 @@ export const calculateMaxTokens = async ({
 
   try {
     if (encoding_for_model) {
-      const encoding = encoding_for_model(getModelNameForTiktoken(modelName));
+      const encoding = await encoding_for_model(
+        getModelNameForTiktoken(modelName)
+      );
 
       const tokenized = encoding.encode(prompt);
 

langchain/src/base_language/index.ts

@@ -97,7 +97,7 @@ export abstract class BaseLanguageModel
         // modelName only exists in openai subclasses, but tiktoken only supports
         // openai tokenisers anyway, so for other subclasses we default to gpt2
         if (encoding_for_model) {
-          this._encoding = encoding_for_model(
+          this._encoding = await encoding_for_model(
             "modelName" in this
               ? getModelNameForTiktoken(this.modelName as string)
               : "gpt2"

langchain/src/text_splitter.ts

@@ -254,8 +254,8 @@ export class TokenTextSplitter
 
   async splitText(text: string): Promise<string[]> {
     if (!this.tokenizer) {
-      const tiktoken = await TokenTextSplitter.imports();
-      this.tokenizer = tiktoken.get_encoding(this.encodingName);
+      const load = await TokenTextSplitter.imports();
+      this.tokenizer = await load(this.encodingName);
       // We need to register a finalizer to free the tokenizer when the
       // splitter is garbage collected.
       this.registry = new FinalizationRegistry((t) => t.free());
@@ -287,9 +287,29 @@ export class TokenTextSplitter
     return splits;
   }
 
-  static async imports(): Promise<typeof tiktoken> {
+  private static async imports(): Promise<
+    (encodingName: string) => Promise<tiktoken.Tiktoken>
+  > {
     try {
-      return await import("@dqbd/tiktoken");
+      const [{ Tiktoken }, { load }, { default: registry }] = await Promise.all(
+        [
+          import("@dqbd/tiktoken/lite"),
+          import("@dqbd/tiktoken/load"),
+          import("@dqbd/tiktoken/registry.json"),
+        ]
+      );
+
+      return async (encodingName: string) => {
+        const model = await load(
+          registry[encodingName as keyof typeof registry]
+        );
+
+        return new Tiktoken(
+          model.bpe_ranks,
+          model.special_tokens,
+          model.pat_str
+        );
+      };
     } catch (err) {
       console.error(err);
       throw new Error(

langchain/tsconfig.json

@@ -19,6 +19,7 @@
     "noUnusedParameters": true,
     "useDefineForClassFields": true,
     "strictPropertyInitialization": false,
+    "resolveJsonModule": true,
     "allowJs": true,
     "strict": true
   },