[build-ci-image] add tiktoken

huggingface · Jul 23, 2024 · 2595c02 · 2595c02
1 parent 7079d1f
commit 2595c02
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/setup.py b/setup.py
@@ -177,6 +177,7 @@
     "tensorflow-probability<0.24",
     "tf2onnx",
     "timeout-decorator",
+    "tiktoken",
     "timm<=0.9.16",
     "tokenizers>=0.19,<0.20",
     "torch",

diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
@@ -82,6 +82,7 @@
     "tensorflow-probability": "tensorflow-probability<0.24",
     "tf2onnx": "tf2onnx",
     "timeout-decorator": "timeout-decorator",
+    "tiktoken": "tiktoken",
     "timm": "timm<=0.9.16",
     "tokenizers": "tokenizers>=0.19,<0.20",
     "torch": "torch",