From 84a22493f03689b4ca3c183e24798634236af18c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Damien=20Coupp=C3=A9?= <damien.couppe@gmail.com>
Date: Fri, 28 Jul 2023 14:09:53 +0200
Subject: [PATCH] feat: Add option TIKTOKEN_FORCE_CACHE

Add taking into account the env var TIKTOKEN_FORCE_CACHE. When TIKTOKEN_FORCE_CACHE is set to "1", tiktoken will read BPE files only from the local cache.

It allows us to have control over when BPE files are downloaded from the internet.
---
 tiktoken/load.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tiktoken/load.py b/tiktoken/load.py
index f30c5133..48c72c8b 100644
--- a/tiktoken/load.py
+++ b/tiktoken/load.py
@@ -44,7 +44,9 @@ def read_file_cached(blobpath: str) -> bytes:
     if os.path.exists(cache_path):
         with open(cache_path, "rb") as f:
             return f.read()
-
+    elif os.environ.get("TIKTOKEN_FORCE_CACHE") == "1":
+        raise FileNotFoundError(f"Cache for: {blobpath} not found in tiktoken cache dir: {cache_dir}")
+    
     contents = read_file(blobpath)
 
     os.makedirs(cache_dir, exist_ok=True)