Switch from pyzstd to zstandard

openai · Apr 2, 2024 · e5f4ea6 · e5f4ea6
1 parent 2420c62
commit e5f4ea6
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 38 deletions.
diff --git a/evals/data.py b/evals/data.py
@@ -12,12 +12,12 @@
 from collections.abc import Iterator
 from functools import partial
 from pathlib import Path
-from typing import Any, List, Optional, Sequence, Text, Union
+from typing import Any, BinaryIO, List, Optional, Sequence, Text, Union
 
 import blobfile as bf
 import lz4.frame
 import pydantic
-import pyzstd
+import zstandard
 
 logger = logging.getLogger(__name__)
 
@@ -37,11 +37,11 @@ def lz4_open(filename: str, mode: str = "rb", openhook: Any = open) -> lz4.frame
     return lz4.frame.LZ4FrameFile(openhook(filename, mode), mode=mode)
 
 
-def zstd_open(filename: str, mode: str = "rb", openhook: Any = open) -> pyzstd.ZstdFile:
+def zstd_open(filename: str, mode: str = "rb", openhook: Any = open) -> BinaryIO:
     if mode and "b" not in mode:
         mode += "b"
 
-    return pyzstd.ZstdFile(openhook(filename, mode), mode=mode)
+    return zstandard.open(openhook(filename, mode), mode=mode, closefd=True)
 
 
 def open_by_file_pattern(filename: Union[str, Path], mode: str = "r", **kwargs: Any) -> Any:

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,50 +4,50 @@ version = "2.0.0.post1"
 requires-python = ">=3.9"
 readme = "README.md"
 dependencies = [
-    "mypy",
-    "openai>=1.0.0",
-    "tiktoken",
-    "blobfile",
+    "aiolimiter",
+    "anthropic",
     "backoff",
-    "numpy",
-    "snowflake-connector-python[pandas]",
-    "pandas",
+    "beartype==0.12.0",
+    "blobfile",
+    "chess",
+    "dacite",
     "datasets",
-    "fire",
-    "pydantic",
-    "tqdm",
-    "types-tqdm",
-    "nltk",
+    "docker",
+    "evaluate",
     "filelock",
-    "mock",
+    "fire",
+    "flask",
+    "google-generativeai",
+    "gymnasium",
+    "jiwer",
+    "langchain",
     "langdetect",
-    'termcolor',
     "lz4",
-    "pyzstd",
-    "pyyaml",
-    "sacrebleu",
     "matplotlib",
-    "pytest",
-    "langchain",
+    "mock",
+    "mypy",
+    "networkx",
+    "nltk",
     "numexpr",
-    "types-PyYAML",
-    "spacy-universal-sentence-encoder",
-    "jiwer",
+    "numpy",
+    "openai>=1.0.0",
+    "pandas",
+    "playwright==1.32.1",
+    "pydantic",
+    "pytest",
+    "pyyaml",
+    "sacrebleu",
     "seaborn",
+    "snowflake-connector-python[pandas]",
+    "spacy-universal-sentence-encoder",
     "statsmodels",
+    "termcolor",
+    "tiktoken",
     "torch",
-    "dacite",
-    "playwright==1.32.1",
-    "evaluate",
-    "aiolimiter",
-    "beartype==0.12.0",
-    "flask",
-    "docker",
-    "gymnasium",
-    "networkx",
-    "chess",
-    "anthropic",
-    "google-generativeai",
+    "tqdm",
+    "types-PyYAML",
+    "types-tqdm",
+    "zstandard",
 ]
 
 [project.urls]