From a3766f538aff74aef4690570c733121543d78c0f Mon Sep 17 00:00:00 2001 From: Alex Cheema Date: Sun, 26 Jan 2025 20:32:48 +0000 Subject: [PATCH] add exception for mlx-community/DeepSeek-R1-3bit and mlx-community/DeepSeek-V3-3bit in tokenizers test --- test/test_tokenizers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_tokenizers.py b/test/test_tokenizers.py index 65b54cbe5..6e1b383e7 100644 --- a/test/test_tokenizers.py +++ b/test/test_tokenizers.py @@ -37,6 +37,6 @@ def test_tokenizer(name, tokenizer, verbose=False): for m in models: # TODO: figure out why use_fast=False is giving inconsistent behaviour (no spaces decoding invididual tokens) for Mistral-Large-Instruct-2407-4bit # test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=False), verbose) - if m not in ["mlx-community/DeepSeek-R1-4bit", "mlx-community/DeepSeek-V3-4bit"]: + if m not in ["mlx-community/DeepSeek-R1-4bit", "mlx-community/DeepSeek-R1-3bit", "mlx-community/DeepSeek-V3-4bit", "mlx-community/DeepSeek-V3-3bit"]: test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=True, trust_remote_code=True), verbose) test_tokenizer(m, AutoTokenizer.from_pretrained(m, trust_remote_code=True), verbose)