From 2595c0221a7d224c98ce207698af188a391819f5 Mon Sep 17 00:00:00 2001
From: Ita Zaporozhets <ita.zaporozhets@huggingface.co>
Date: Tue, 23 Jul 2024 11:10:28 +0200
Subject: [PATCH] [build-ci-image] add tiktoken

---
 setup.py                                      | 1 +
 src/transformers/dependency_versions_table.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/setup.py b/setup.py
index e29dece29b9d2f..b92ef154564ced 100644
--- a/setup.py
+++ b/setup.py
@@ -177,6 +177,7 @@
     "tensorflow-probability<0.24",
     "tf2onnx",
     "timeout-decorator",
+    "tiktoken",
     "timm<=0.9.16",
     "tokenizers>=0.19,<0.20",
     "torch",
diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
index 7179c1df716011..f6af2082b7d586 100644
--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@@ -82,6 +82,7 @@
     "tensorflow-probability": "tensorflow-probability<0.24",
     "tf2onnx": "tf2onnx",
     "timeout-decorator": "timeout-decorator",
+    "tiktoken": "tiktoken",
     "timm": "timm<=0.9.16",
     "tokenizers": "tokenizers>=0.19,<0.20",
     "torch": "torch",