Expand default cache limit to 30GB (#1850)

stanfordnlp · Nov 24, 2024 · 1b9b280 · 1b9b280
1 parent 2b6b204
commit 1b9b280
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.py
@@ -8,8 +8,15 @@
 from litellm.caching import Cache
 
 DISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
+DISK_CACHE_LIMIT = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10))  # 30 GB default
+
+# TODO: There's probably value in getting litellm to support FanoutCache and to separate the limit for
+# the LM cache from the embeddings cache. Then we can lower the default 30GB limit.
 litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")
+litellm.cache.cache.disk_cache.reset('size_limit', DISK_CACHE_LIMIT)
+
 litellm.telemetry = False
+
 # Turn off by default to avoid LiteLLM logging during every LM call.
 litellm.suppress_debug_info = True