diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..0fb4f0d62 --- /dev/null +++ b/.env.example @@ -0,0 +1,11 @@ +# LOG_LEVEL= +# LOG_PATH= +# LOG_NAME= +# TIMEZONE= + +# NUM_PER_BATCH= +# DEFAULT_DATASET_URL= + +DATASET_LOCAL_DIR="/tmp/vector_db_bench/dataset" + +# DROP_OLD = True diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..f0e32dd9e --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*.sw[op] +*.egg-info +dist/ +__pycache__ +.env +.data/ +__MACOSX +.DS_Store diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 000000000..fb6b19c3d --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,49 @@ +# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. +# Enable flake8-bugbear (`B`) rules. +select = ["E", "F", "B"] +ignore = [ + "E501", # (line length violations) +] + +# Allow autofix for all enabled rules (when `--fix`) is provided. +fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", + "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", + "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT", +] +unfixable = [] + +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "__pycache__", + "__init__.py", +] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +# Assume Python 3.11. +target-version = "py311" + +[mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 10 diff --git a/.streamlit/config.toml b/.streamlit/config.toml new file mode 100644 index 000000000..aa282117e --- /dev/null +++ b/.streamlit/config.toml @@ -0,0 +1,4 @@ +[theme] +primaryColor="#3670F2" +secondaryBackgroundColor="#F0F2F6" +base="light" \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..bdf1e1b39 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Zilliztech + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 000000000..f5d81280e --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +requires: `python >= 3.11` + +## 1. Quick Start +### Installation +```shell +$ pip install vector_db_bench +``` + +### Run +```shell +$ init_bench +``` + +### View app in browser + +Local URL: http://localhost:8501 + +## 2. How to run test server + +### Install requirements +``` shell +pip install -e '.[test]' +``` + +### Run test server +``` +$ python -m vector_db_bench +``` + +OR: + +```shell +$ init_bench +``` + +## 3. How to check coding styles + +```shell +$ ruff check vector_db_bench +``` + +Add `--fix` if you want to fix the coding styles automatically +```shell +$ ruff check vector_db_bench --fix +``` + +## 4. How to run uinitest +``` +pytest -sv tests/ +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..c2c43aa3f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = ["setuptools>=67.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "vector_db_bench" +# authors = [ +# { name="", email="" }, +# ] +description = "" +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +dependencies = [ + "pytz", + "streamlit-autorefresh", + "streamlit>=1.23.0", + "streamlit_extras", + "grpcio==1.53.0", # for qdrant-client and pymilvus + "grpcio-tools==1.53.0", # for qdrant-client and pymilvus + "pymilvus", # with pandas, numpy, ujson + "qdrant-client", + "pinecone-client", + "weaviate-client", + "elasticsearch", + "plotly", + "pydantic==v1.10.7", # for qdrant-client + "environs", + "scikit-learn", + "s3fs", + "psutil", +] +version = "0.0.1" + +[project.optional-dependencies] +test = [ + "ruff", + "pytest", +] + + +# [project.urls] +# "Homepage" = "" +# "Docs: User Guide" = "" +# "Source Code" = "" + +[project.scripts] +init_bench = "vector_db_bench.__main__:main" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..4e7e2ebac --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,5 @@ +import os +import sys + +from os.path import dirname, abspath +sys.path.append(dirname(dirname(abspath(__file__)))) diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 000000000..b9e4bc334 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +filterwarnings = + ignore::UserWarning diff --git a/tests/test_bench_runner.py b/tests/test_bench_runner.py new file mode 100644 index 000000000..637b67cdf --- /dev/null +++ b/tests/test_bench_runner.py @@ -0,0 +1,60 @@ +import time +import logging +from vector_db_bench.interface import BenchMarkRunner +from vector_db_bench.models import ( + DB, IndexType, CaseType, TaskConfig, CaseConfig, +) + +log = logging.getLogger(__name__) + +class TestBenchRunner: + def test_get_results(self): + runner = BenchMarkRunner() + + result = runner.get_results() + log.info(f"test result: {result}") + + def test_performance_case_whole(self): + runner = BenchMarkRunner() + + task_config=TaskConfig( + db=DB.Milvus, + db_config=DB.Milvus.config(), + db_case_config=DB.Milvus.case_config_cls(index=IndexType.Flat)(), + case_config=CaseConfig(case_id=CaseType.PerformanceSZero), + ) + + runner.run([task_config]) + runner._sync_running_task() + result = runner.get_results() + log.info(f"test result: {result}") + + def test_performance_case_clean(self): + runner = BenchMarkRunner() + + task_config=TaskConfig( + db=DB.Milvus, + db_config=DB.Milvus.config(), + db_case_config=DB.Milvus.case_config_cls(index=IndexType.Flat)(), + case_config=CaseConfig(case_id=CaseType.PerformanceSZero), + ) + + runner.run([task_config]) + time.sleep(3) + runner.stop_running() + + def test_performance_case_no_error(self): + task_config=TaskConfig( + db=DB.ZillizCloud, + db_config=DB.ZillizCloud.config(uri="xxx", user="abc", password="1234"), + db_case_config=DB.ZillizCloud.case_config_cls()(), + case_config=CaseConfig(case_id=CaseType.PerformanceSZero), + ) + + t = task_config.copy() + d = t.json(exclude={'db_config': {'password', 'api_key'}}) + log.info(f"{d}") + + import ujson + loads = ujson.loads(d) + log.info(f"{loads}") diff --git a/tests/test_case.py b/tests/test_case.py new file mode 100644 index 000000000..4db1def28 --- /dev/null +++ b/tests/test_case.py @@ -0,0 +1,99 @@ +import pytest +import logging +import vector_db_bench.backend.dataset as ds +from vector_db_bench.models import DB, IndexType +from vector_db_bench.backend import cases +from vector_db_bench.backend.clients.milvus import Milvus +from vector_db_bench.backend.clients.weaviate import Weaviate + +log = logging.getLogger(__name__) +class TestCases: + def test_init_LoadCase(self): + c = cases.LoadSDimCase(run_id=1, db_class=Milvus) + log.debug(f"c: {c}, {c.dict().keys()}") + + def test_case_type(self): + from vector_db_bench.models import CaseType + log.debug(f"{CaseType.LoadLDim}") + + def test_performance_case_small_zero(self): + dataset = ds.get(ds.Name.Cohere, ds.Label.SMALL) + # milvus crash + # db_case_config = DB.Milvus.case_config_cls(IndexType.HNSW)( + # M=8, + # efConstruction=32, + # ef=8, + # ) + + db_case_config = DB.Milvus.case_config_cls(IndexType.Flat)() + db_case_config.metric_type = dataset.data.metric_type + c = cases.PerformanceSZero(run_id=1, db_configs=( + DB.Milvus.init_cls, + DB.Milvus.config().to_dict(), + db_case_config, + )) + c.run() + + @pytest.mark.skip(reason="replace url and api_key by real value") + def test_performance_case_small_zero_weaviate(self): + dataset = ds.get(ds.Name.Cohere, ds.Label.SMALL) + db_case_config = DB.WeaviateCloud.case_config_cls()() + db_case_config.metric_type = dataset.data.metric_type + + c = cases.PerformanceSZero(run_id=1, db_configs={ + DB.WeaviateCloud.init_cls, + DB.WeaviateCloud.config(url="", api_key="").to_dict(), + db_case_config, + }) + c.run() + + def test_performance_case_small_low_filter(self): + dataset = ds.get(ds.Name.Cohere, ds.Label.SMALL) + + db_case_config = DB.Milvus.case_config_cls(IndexType.Flat)() + db_case_config.metric_type = dataset.data.metric_type + c = cases.PerformanceSLow(run_id=2, db_configs=( + DB.Milvus.init_cls, + DB.Milvus.config().to_dict(), + db_case_config, + )) + c.run() + + def test_performance_case_small_high_filter(self): + dataset = ds.get(ds.Name.Cohere, ds.Label.SMALL) + db_case_config = DB.Milvus.case_config_cls(IndexType.Flat)() + db_case_config.metric_type = dataset.data.metric_type + + c = cases.PerformanceSHigh(run_id=3, db_configs=( + DB.Milvus.init_cls, + DB.Milvus.config().to_dict(), + db_case_config, + )) + c.run() + + def test_load_small_dim(self): + dataset = ds.get(ds.Name.SIFT, ds.Label.SMALL) + db_case_config = DB.Milvus.case_config_cls(IndexType.Flat)() + db_case_config.metric_type = dataset.data.metric_type + + c = cases.LoadSDimCase(run_id=1, db_configs=( + DB.Milvus.init_cls, + DB.Milvus.config().to_dict(), + db_case_config, + )) + c.run() + + def test_performance_case_medium_zero(self): + dataset = ds.get(ds.Name.Cohere, ds.Label.MEDIUM) + db_case_config = DB.Milvus.case_config_cls(IndexType.Flat)() + db_case_config.metric_type = dataset.data.metric_type + c = cases.PerformanceMZero(run_id=1, db_configs=( + DB.Milvus.init_cls, + DB.Milvus.config().to_dict(), + db_case_config, + )) + + # c.dataset.prepare(False) + # c._insert_train_data() + c.run() + diff --git a/tests/test_dataset.py b/tests/test_dataset.py new file mode 100644 index 000000000..c6c3abcee --- /dev/null +++ b/tests/test_dataset.py @@ -0,0 +1,53 @@ +import pytest +import logging + +from vector_db_bench.backend import dataset as ds + +log = logging.getLogger(__name__) +class TestDataSet: + @pytest.mark.skip("not ready in s3") + def test_init_dataset(self): + testdatasets = [ds.get(d, lb) for d in ds.Name for lb in ds.Label if ds.get(d, lb) is not None] + for t in testdatasets: + t._validate_local_file() + + @pytest.mark.skip("not ready in s3") + def test_init_gist(self): + g = ds.GIST_S() + log.debug(f"GIST SMALL: {g}") + assert g.name == "GIST" + assert g.label == "SMALL" + assert g.size == 100_000 + + gists = [ds.get(ds.Name.GIST, lb) for lb in ds.Label if ds.get(ds.Name.GIST, lb) is not None] + for t in gists: + t._validate_local_file() + + def test_init_cohere(self): + coheres = [ds.get(ds.Name.Cohere, lb) for lb in ds.Label if ds.get(ds.Name.Cohere, lb) is not None] + for t in coheres: + t._validate_local_file() + + def test_init_sift(self): + sifts = [ds.get(ds.Name.SIFT, lb) for lb in ds.Label if ds.get(ds.Name.SIFT, lb) is not None] + for t in sifts: + t._validate_local_file() + + @pytest.mark.skip("runs locally") + def test_iter_dataset_cohere(self): + cohere_s = ds.get(ds.Name.Cohere, ds.Label.SMALL) + assert cohere_s.prepare() + + for f in cohere_s: + log.debug(f"iter to: {f.columns}") + + # @pytest.mark.skip("runs locally") + def test_dataset_download(self): + cohere_s = ds.get(ds.Name.Cohere, ds.Label.SMALL) + assert cohere_s.prepare() + + + cohere_m = ds.get(ds.Name.Cohere, ds.Label.MEDIUM) + cohere_m._validate_local_file() + assert cohere_m.prepare() is True + assert cohere_m.prepare() is True diff --git a/tests/test_elasticsearch_cloud.py b/tests/test_elasticsearch_cloud.py new file mode 100644 index 000000000..161e35f6b --- /dev/null +++ b/tests/test_elasticsearch_cloud.py @@ -0,0 +1,86 @@ +import logging +from vector_db_bench.models import ( + DB, + MetricType, + ElasticsearchConfig, +) +import numpy as np + + +log = logging.getLogger(__name__) + +cloud_id = "" +password = "" + + +class TestModels: + def test_insert_and_search(self): + assert DB.ElasticCloud.value == "Elasticsearch" + assert DB.ElasticCloud.config == ElasticsearchConfig + + dbcls = DB.ElasticCloud.init_cls + dbConfig = dbcls.config_cls()(cloud_id=cloud_id, password=password) + dbCaseConfig = dbcls.case_config_cls()( + metric_type=MetricType.L2, efConstruction=64, M=16, num_candidates=100 + ) + + dim = 16 + es = dbcls( + dim=dim, + db_config=dbConfig.to_dict(), + db_case_config=dbCaseConfig, + indice="test_es_cloud", + drop_old=True, + ) + + count = 10_000 + filter_rate = 0.9 + embeddings = [[np.random.random() for _ in range(dim)] for _ in range(count)] + + # insert + with es.init(): + res = es.insert_embeddings(embeddings=embeddings, metadata=range(count)) + # bulk_insert return + assert ( + res == count + ), f"the return count of bulk insert ({res}) is not equal to count ({count})" + + # indice_count return + es.client.indices.refresh() + esCountRes = es.client.count(index=es.indice) + countResCount = esCountRes.raw["count"] + assert ( + countResCount == count + ), f"the return count of es client ({countResCount}) is not equal to count ({count})" + + # search + with es.init(): + test_id = np.random.randint(count) + log.info(f"test_id: {test_id}") + q = embeddings[test_id] + + res = es.search_embedding(query=q, k=100) + log.info(f"search_results_id: {res}") + assert ( + res[0] == test_id + ), f"the most nearest neighbor ({res[0]}) id is not test_id ({test_id})" + + # search with filters + with es.init(): + test_id = np.random.randint(count * filter_rate, count) + log.info(f"test_id: {test_id}") + q = embeddings[test_id] + + res = es.search_embedding( + query=q, k=100, filters={"id": count * filter_rate} + ) + log.info(f"search_results_id: {res}") + assert ( + res[0] == test_id + ), f"the most nearest neighbor ({res[0]}) id is not test_id ({test_id})" + isFilter = True + for id in res: + if id < count * filter_rate: + isFilter = False + break + assert isFilter, f"filters failed" diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 000000000..db4056aba --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,71 @@ +import pytest +import logging +from vector_db_bench.models import ( + TaskConfig, CaseConfig, + CaseResult, TestResult, + Metric, CaseType, ResultLabel +) +from vector_db_bench.backend.clients import ( + DB, + IndexType +) + +from vector_db_bench import config + + +log = logging.getLogger(__name__) + + +class TestModels: + @pytest.mark.skip("runs locally") + def test_test_result(self): + result = CaseResult( + task_config=TaskConfig( + db=DB.Milvus, + db_config=DB.Milvus.config(), + db_case_config=DB.Milvus.case_config_cls(index=IndexType.Flat)(), + case_config=CaseConfig(case_id=CaseType.PerformanceLZero), + ), + metrics=Metric(), + ) + + test_result = TestResult(run_id=10000, results=[result]) + test_result.write_file() + + with pytest.raises(ValueError): + result = TestResult.read_file('nosuchfile.json') + + def test_test_result_read_write(self): + result_dir = config.RESULTS_LOCAL_DIR + for json_file in result_dir.glob("*.json"): + res = TestResult.read_file(json_file) + res.task_label = f"Milvus-{res.run_id}" + res.write_file() + + def test_test_result_merge(self): + result_dir = config.RESULTS_LOCAL_DIR + all_results = [] + + first_result = None + for json_file in result_dir.glob("*.json"): + res = TestResult.read_file(json_file) + + for cr in res.results: + all_results.append(cr) + + if not first_result: + first_result = res + + tr = TestResult( + run_id=first_result.run_id, + task_label="standard", + results=all_results, + ) + tr.write_file() + + def test_test_result_display(self): + result_dir = config.RESULTS_LOCAL_DIR + for json_file in result_dir.glob("*.json"): + res = TestResult.read_file(json_file) + # res.display([DB.ZillizCloud]) + res.display() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 000000000..5e9cdfe6b --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,39 @@ +import pytest +import logging + +from vector_db_bench.backend import utils +from vector_db_bench.metric import calc_recall + +log = logging.getLogger(__name__) + +class TestUtils: + @pytest.mark.parametrize("testcases", [ + (1, '1'), + (10, '10'), + (100, '100'), + (1000, '1K'), + (2000, '2K'), + (30_000, '30K'), + (400_000, '400K'), + (5_000_000, '5M'), + (60_000_000, '60M'), + (1_000_000_000, '1B'), + (1_000_000_000_000, '1000B'), + ]) + def test_numerize(self, testcases): + t_in, expected = testcases + assert expected == utils.numerize(t_in) + + @pytest.mark.parametrize("got_expected", [ + ([1, 3, 5, 7, 9, 10], 1.0), + ([11, 12, 13, 14, 15, 16], 0.0), + ([1, 3, 5, 11, 12, 13], 0.5), + ([1, 3, 5], 0.5), + ]) + def test_recall(self, got_expected): + got, expected = got_expected + ground_truth = [1, 3, 5, 7, 9, 10] + res = calc_recall(6, ground_truth, got) + log.info(f"recall: {res}, expected: {expected}") + assert res == expected + diff --git a/vector_db_bench/__init__.py b/vector_db_bench/__init__.py new file mode 100644 index 000000000..1c858af7e --- /dev/null +++ b/vector_db_bench/__init__.py @@ -0,0 +1,29 @@ +import environs +import inspect +import pathlib +from . import log_util + + +env = environs.Env() +env.read_env(".env") + +class config: + LOG_LEVEL = env.str("LOG_LEVEL", "INFO") + + DEFAULT_DATASET_URL = env.str("DEFAULT_DATASET_URL", "assets.zilliz.com/benchmark/") + DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vector_db_bench/dataset") + NUM_PER_BATCH = env.int("NUM_PER_BATCH", 5000) + + DROP_OLD = env.bool("DROP_OLD", True) + USE_SHUFFLED_DATA = env.bool("USE_SHUFFLED_DATA", True) + + RESULTS_LOCAL_DIR = pathlib.Path(__file__).parent.joinpath("results") + + + def display(self) -> str: + tmp = [i for i in inspect.getmembers(self) + if not inspect.ismethod(i[1]) and not i[0].startswith('_') \ + ] + return tmp + +log_util.init(config.LOG_LEVEL) diff --git a/vector_db_bench/__main__.py b/vector_db_bench/__main__.py new file mode 100644 index 000000000..ede5004b5 --- /dev/null +++ b/vector_db_bench/__main__.py @@ -0,0 +1,25 @@ +import traceback +import logging +import subprocess +import os +from . import config + +log = logging.getLogger("vector_db_bench") + +def main(): + log.info(f"all configs: {config().display()}") + run_streamlit() + +def run_streamlit(): + cmd = ['streamlit', 'run', f'{os.path.dirname(__file__)}/frontend/vdb_benchmark.py', '--logger.level', 'info'] + log.debug(f"cmd: {cmd}") + try: + subprocess.run(cmd, check=True) + except KeyboardInterrupt: + log.info("exit streamlit...") + except Exception as e: + log.warning(f"exit, err={e}\nstack trace={traceback.format_exc(chain=True)}") + + +if __name__ == "__main__": + main() diff --git a/vector_db_bench/backend/__init__.py b/vector_db_bench/backend/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vector_db_bench/backend/assembler.py b/vector_db_bench/backend/assembler.py new file mode 100644 index 000000000..c3c33e4ec --- /dev/null +++ b/vector_db_bench/backend/assembler.py @@ -0,0 +1,57 @@ +from .cases import type2case, CaseLabel +from .task_runner import CaseRunner, RunningStatus, TaskRunner +from ..models import TaskConfig +from ..backend.clients import EmptyDBCaseConfig +import logging + + +log = logging.getLogger(__name__) + + +class Assembler: + @classmethod + def assemble(cls, run_id , task: TaskConfig) -> CaseRunner: + c_cls = type2case.get(task.case_config.case_id) + + c = c_cls() + if type(task.db_case_config) != EmptyDBCaseConfig: + task.db_case_config.metric_type = c.dataset.data.metric_type + + runner = CaseRunner( + run_id=run_id, + config=task, + ca=c, + status=RunningStatus.PENDING, + ) + + return runner + + @classmethod + def assemble_all(cls, run_id: str, task_label: str, tasks: list[TaskConfig]) -> TaskRunner: + """group by case type, db, and case dataset""" + runners = [cls.assemble(run_id, task) for task in tasks] + load_runners = [r for r in runners if r.ca.label == CaseLabel.Load] + perf_runners = [r for r in runners if r.ca.label == CaseLabel.Performance] + + # group by db + db2runner = {} + for r in perf_runners: + db = r.config.db + if db not in db2runner: + db2runner[db] = [] + db2runner[db].append(r) + + # sort by dataset size + for k in db2runner.keys(): + db2runner[k].sort(key=lambda x:x.ca.dataset.data.size) + + all_runners = [] + all_runners.extend(load_runners) + for v in db2runner.values(): + all_runners.extend(v) + + return TaskRunner( + run_id=run_id, + task_label=task_label, + case_runners=all_runners, + ) diff --git a/vector_db_bench/backend/cases.py b/vector_db_bench/backend/cases.py new file mode 100644 index 000000000..d389f401a --- /dev/null +++ b/vector_db_bench/backend/cases.py @@ -0,0 +1,124 @@ +import logging +from enum import Enum, auto + +from . import dataset as ds +from ..base import BaseModel +from ..models import CaseType + + +log = logging.getLogger(__name__) + + +class CaseLabel(Enum): + Load = auto() + Performance = auto() + + +class Case(BaseModel): + """ Undifined case + + Fields: + case_id(CaseType): default 11 case type plus one custom cases. + label(CaseLabel): performance or load. + dataset(DataSet): dataset for this case runner. + filter_rate(float | None): one of 99% | 1% | None + filters(dict | None): filters for search + """ + + case_id: CaseType + label: CaseLabel + dataset: ds.DataSet + + filter_rate: float | None + + @property + def filters(self) -> dict | None: + if self.filter_rate is not None: + ID = round(self.filter_rate * self.dataset.data.size) + return { + "metadata": f">={ID}", + "id": ID, + } + + return None + + +class LoadCase(Case, BaseModel): + label: CaseLabel = CaseLabel.Load + filter_rate: float | int | None = None + +class PerformanceCase(Case, BaseModel): + label: CaseLabel = CaseLabel.Performance + filter_rate: float | int | None = None + +class LoadLDimCase(LoadCase): + case_id: CaseType = CaseType.LoadLDim + dataset: ds.DataSet = ds.get(ds.Name.GIST, ds.Label.SMALL) + +class LoadSDimCase(LoadCase): + case_id: CaseType = CaseType.LoadSDim + dataset: ds.DataSet = ds.get(ds.Name.SIFT, ds.Label.SMALL) + +class PerformanceLZero(PerformanceCase): + case_id: CaseType = CaseType.PerformanceLZero + dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.LARGE) + +class PerformanceMZero(PerformanceCase): + case_id: CaseType = CaseType.PerformanceMZero + dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.MEDIUM) + +class PerformanceSZero(PerformanceCase): + case_id: CaseType = CaseType.PerformanceSZero + dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.SMALL) + +class PerformanceLLow(PerformanceCase): + case_id: CaseType = CaseType.PerformanceLLow + filter_rate: float | int | None = 0.01 + dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.LARGE) + +class PerformanceMLow(PerformanceCase): + case_id: CaseType = CaseType.PerformanceMLow + filter_rate: float | int | None = 0.01 + dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.MEDIUM) + +class PerformanceSLow(PerformanceCase): + case_id: CaseType = CaseType.PerformanceSLow + filter_rate: float | int | None = 0.01 + dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.SMALL) + +class PerformanceLHigh(PerformanceCase): + case_id: CaseType = CaseType.PerformanceLHigh + filter_rate: float | int | None = 0.99 + dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.LARGE) + +class PerformanceMHigh(PerformanceCase): + case_id: CaseType = CaseType.PerformanceMHigh + filter_rate: float | int | None = 0.99 + dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.MEDIUM) + +class PerformanceSHigh(PerformanceCase): + case_id: CaseType = CaseType.PerformanceSLow + filter_rate: float | int | None = 0.99 + dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.SMALL) + +class Performance100M(PerformanceCase): + case_id: CaseType = CaseType.Performance100M + filter_rate: float | int | None = None + dataset: ds.DataSet = ds.get(ds.Name.LAION, ds.Label.LARGE) + +type2case = { + CaseType.LoadLDim: LoadLDimCase, + CaseType.LoadSDim: LoadSDimCase, + + CaseType.PerformanceLZero: PerformanceLZero, + CaseType.PerformanceMZero: PerformanceMZero, + CaseType.PerformanceSZero: PerformanceSZero, + + CaseType.PerformanceLLow: PerformanceLLow, + CaseType.PerformanceMLow: PerformanceMLow, + CaseType.PerformanceSLow: PerformanceSLow, + CaseType.PerformanceLHigh: PerformanceLHigh, + CaseType.PerformanceMHigh: PerformanceMHigh, + CaseType.PerformanceSHigh: PerformanceSHigh, + CaseType.Performance100M: Performance100M, +} diff --git a/vector_db_bench/backend/clients/__init__.py b/vector_db_bench/backend/clients/__init__.py new file mode 100644 index 000000000..8728c0132 --- /dev/null +++ b/vector_db_bench/backend/clients/__init__.py @@ -0,0 +1,57 @@ +from enum import Enum +from typing import Type +from .api import ( + VectorDB, + DBConfig, + DBCaseConfig, + EmptyDBCaseConfig, + IndexType, + MetricType, +) + +from .milvus.milvus import Milvus +from .elastic_cloud.elastic_cloud import ElasticCloud +from .pinecone.pinecone import Pinecone +from .weaviate_cloud.weaviate_cloud import WeaviateCloud +from .qdrant_cloud.qdrant_cloud import QdrantCloud +from .zilliz_cloud.zilliz_cloud import ZillizCloud + + +class DB(Enum): + """Database types + + Examples: + >>> DB.Milvus + + >>> DB.Milvus.value + "Milvus" + >>> DB.Milvus.name + "Milvus" + """ + + Milvus = "Milvus" + ZillizCloud = "ZillizCloud" + Pinecone = "Pinecone" + ElasticCloud = "ElasticCloud" + QdrantCloud = "QdrantCloud" + WeaviateCloud = "WeaviateCloud" + + + @property + def init_cls(self) -> Type[VectorDB]: + return db2client.get(self) + + +db2client = { + DB.Milvus: Milvus, + DB.ZillizCloud: ZillizCloud, + DB.WeaviateCloud: WeaviateCloud, + DB.ElasticCloud: ElasticCloud, + DB.QdrantCloud: QdrantCloud, + DB.Pinecone: Pinecone, +} + + +__all__ = [ + "DB", "VectorDB", "DBConfig", "DBCaseConfig", "IndexType", "MetricType", "EmptyDBCaseConfig", +] diff --git a/vector_db_bench/backend/clients/api.py b/vector_db_bench/backend/clients/api.py new file mode 100644 index 000000000..cead897e6 --- /dev/null +++ b/vector_db_bench/backend/clients/api.py @@ -0,0 +1,179 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import Any, Type +from contextlib import contextmanager + +from pydantic import BaseModel + + +class MetricType(str, Enum): + L2 = "L2" + COSINE = "COSINE" + IP = "IP" + + +class IndexType(str, Enum): + HNSW = "HNSW" + DISKANN = "DISKANN" + IVFFlat = "IVF_FLAT" + Flat = "FLAT" + AUTOINDEX = "AUTOINDEX" + ES_HNSW = "hnsw" + + +class DBConfig(ABC, BaseModel): + """DBConfig contains the connection info of vector database + + Args: + db_label(str): label to distinguish different types of DB of the same database. + + MilvusConfig.db_label = 2c8g + MilvusConfig.db_label = 16c64g + ZillizCloudConfig.db_label = 1cu-perf + """ + + db_label: str | None = None + + @abstractmethod + def to_dict(self) -> dict: + raise NotImplementedError + + +class DBCaseConfig(ABC): + """Case specific vector database configs, usually uesed for index params like HNSW""" + @abstractmethod + def index_param(self) -> dict: + raise NotImplementedError + + @abstractmethod + def search_param(self) -> dict: + raise NotImplementedError + + +class EmptyDBCaseConfig(BaseModel, DBCaseConfig): + """EmptyDBCaseConfig will be used if the vector database has no case specific configs""" + null: str | None = None + def index_param(self) -> dict: + return {} + + def search_param(self) -> dict: + return {} + + +class VectorDB(ABC): + """Each VectorDB will be __init__ once for one case, the object will be copied into multiple processes. + + In each process, the benchmark cases ensure VectorDB.init() calls before any other methods operations + + insert_embeddings, search_embedding, and, ready_to_search will be timed for each call. + + Examples: + >>> milvus = Milvus() + >>> with milvus.init(): + >>> milvus.insert_embeddings() + >>> milvus.search_embedding() + """ + + @abstractmethod + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig | None, + collection_name: str, + drop_old: bool = False, + **kwargs + ) -> None: + """Initialize wrapper around the vector database client + + Args: + dim(int): the dimension of the dataset + db_config(dict): configs to establish connections with the vector database + db_case_config(DBCaseConfig | None): case specific configs for indexing and searching + drop_old(bool): whether to drop the existing collection of the dataset. + """ + raise NotImplementedError + + @classmethod + @abstractmethod + def config_cls(self) -> Type[DBConfig]: + raise NotImplementedError + + + @classmethod + @abstractmethod + def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseConfig]: + raise NotImplementedError + + + @abstractmethod + @contextmanager + def init(self) -> None: + """ create and destory connections to database. + + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + """ + raise NotImplementedError + + @abstractmethod + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + kwargs: Any, + ) -> int: + """Insert the embeddings to the vector database. The default number of embeddings for + each insert_embeddings is 5000. + + Args: + embeddings(list[list[float]]): list of embedding to add to the vector database. + metadatas(list[int]): metadata associated with the embeddings, for filtering. + kwargs(Any): vector database specific parameters. + + Returns: + int: inserted data count + """ + raise NotImplementedError + + @abstractmethod + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + ) -> list[int]: + """Get k most similar embeddings to query vector. + + Args: + query(list[float]): query embedding to look up documents similar to. + k(int): Number of most similar embeddings to return. Defaults to 100. + filters(dict, optional): filtering expression to filter the data while searching. + + Returns: + list[int]: list of k most similar embeddings IDs to the query embedding. + """ + raise NotImplementedError + + # TODO: remove + @abstractmethod + def ready_to_search(self): + """ready_to_search will be called between insertion and search in performance cases. + + Should be blocked until the vectorDB is ready to be tested on + heavy performance cases. + + Time(insert the dataset) + Time(ready_to_search) will be recorded as "load_duration" metric + """ + raise NotImplementedError + + # TODO: remove + @abstractmethod + def ready_to_load(self): + """ready_to_load will be called before load in load cases. + + Should be blocked until the vectorDB is ready to be tested on + heavy load cases. + """ + raise NotImplementedError diff --git a/vector_db_bench/backend/clients/elastic_cloud/config.py b/vector_db_bench/backend/clients/elastic_cloud/config.py new file mode 100644 index 000000000..55b0737c7 --- /dev/null +++ b/vector_db_bench/backend/clients/elastic_cloud/config.py @@ -0,0 +1,56 @@ +from enum import Enum +from pydantic import SecretStr, BaseModel + +from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + + +class ElasticsearchConfig(DBConfig, BaseModel): + cloud_id: SecretStr + password: SecretStr | None = None + + def to_dict(self) -> dict: + return { + "cloud_id": self.cloud_id.get_secret_value(), + "basic_auth": ("elastic", self.password.get_secret_value()), + } + + +class ESElementType(str, Enum): + float = "float" # 4 byte + byte = "byte" # 1 byte, -128 to 127 + + +class ElasticsearchIndexConfig(BaseModel, DBCaseConfig): + element_type: ESElementType = ESElementType.float + index: IndexType = IndexType.ES_HNSW # ES only support 'hnsw' + + metric_type: MetricType | None = None + efConstruction: int | None = None + M: int | None = None + num_candidates: int | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2_norm" + elif self.metric_type == MetricType.IP: + return "dot_product" + return "cosine" + + def index_param(self) -> dict: + params = { + "type": "dense_vector", + "index": True, + "element_type": self.element_type.value, + "similarity": self.parse_metric(), + "index_options": { + "type": self.index.value, + "m": self.M, + "ef_construction": self.efConstruction + } + } + return params + + def search_param(self) -> dict: + return { + "num_candidates": self.num_candidates, + } diff --git a/vector_db_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vector_db_bench/backend/clients/elastic_cloud/elastic_cloud.py new file mode 100644 index 000000000..a6cce9fb7 --- /dev/null +++ b/vector_db_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -0,0 +1,152 @@ +import logging +from contextlib import contextmanager +from typing import Iterable, Type +from ..api import VectorDB, DBCaseConfig, DBConfig, IndexType +from .config import ElasticsearchIndexConfig, ElasticsearchConfig +from elasticsearch.helpers import bulk + + +for logger in ("elasticsearch", "elastic_transport"): + logging.getLogger(logger).setLevel(logging.WARNING) + +log = logging.getLogger(__name__) + +class ElasticCloud(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: ElasticsearchIndexConfig, + indice: str = "vdb_bench_indice", # must be lowercase + id_col_name: str = "id", + vector_col_name: str = "vector", + drop_old: bool = False, + ): + self.dim = dim + self.db_config = db_config + self.case_config = db_case_config + self.indice = indice + self.id_col_name = id_col_name + self.vector_col_name = vector_col_name + + from elasticsearch import Elasticsearch + + client = Elasticsearch(**self.db_config) + + if drop_old: + log.info(f"Elasticsearch client drop_old indices: {self.indice}") + is_existed_res = client.indices.exists(index=self.indice) + if is_existed_res.raw: + client.indices.delete(index=self.indice) + self._create_indice(client) + + + @classmethod + def config_cls(cls) -> Type[DBConfig]: + return ElasticsearchConfig + + + @classmethod + def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: + return ElasticsearchIndexConfig + + + @contextmanager + def init(self) -> None: + """connect to elasticsearch""" + from elasticsearch import Elasticsearch + self.client = Elasticsearch(**self.db_config, request_timeout=30) + + yield + # self.client.transport.close() + self.client = None + del(self.client) + + def _create_indice(self, client) -> None: + mappings = { + "properties": { + self.id_col_name: {"type": "integer"}, + self.vector_col_name: { + "dims": self.dim, + **self.case_config.index_param(), + }, + } + } + + try: + client.indices.create(index=self.indice, mappings=mappings) + except Exception as e: + log.warning(f"Failed to create indice: {self.indice} error: {str(e)}") + raise e from None + + def insert_embeddings( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + ) -> int: + """Insert the embeddings to the elasticsearch.""" + assert self.client is not None, "should self.init() first" + + insert_data = [ + { + "_index": self.indice, + "_source": { + self.id_col_name: metadata[i], + self.vector_col_name: embeddings[i], + }, + } + for i in range(len(embeddings)) + ] + try: + bulk_insert_res = bulk(self.client, insert_data) + return bulk_insert_res[0] + except Exception as e: + log.warning(f"Failed to insert data: {self.indice} error: {str(e)}") + raise e from None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + ) -> list[int]: + """Get k most similar embeddings to query vector. + + Args: + query(list[float]): query embedding to look up documents similar to. + k(int): Number of most similar embeddings to return. Defaults to 100. + filters(dict, optional): filtering expression to filter the data while searching. + + Returns: + list[tuple[int, float]]: list of k most similar embeddings in (id, score) tuple to the query embedding. + """ + assert self.client is not None, "should self.init() first" + # is_existed_res = self.client.indices.exists(index=self.indice) + # assert is_existed_res.raw == True, "should self.init() first" + + knn = { + "field": self.vector_col_name, + "k": k, + "num_candidates": self.case_config.num_candidates, + "filter": [{"range": {self.id_col_name: {"gt": filters["id"]}}}] + if filters + else [], + "query_vector": query, + } + size = k + try: + search_res = self.client.search(index=self.indice, knn=knn, size=size) + res = [d["_source"][self.id_col_name] for d in search_res["hits"]["hits"]] + + return res + except Exception as e: + log.warning(f"Failed to search: {self.indice} error: {str(e)}") + raise e from None + + def ready_to_search(self): + """ready_to_search will be called between insertion and search in performance cases.""" + pass + + def ready_to_load(self): + """ready_to_load will be called before load in load cases.""" + pass diff --git a/vector_db_bench/backend/clients/milvus/config.py b/vector_db_bench/backend/clients/milvus/config.py new file mode 100644 index 000000000..fdcb0c313 --- /dev/null +++ b/vector_db_bench/backend/clients/milvus/config.py @@ -0,0 +1,123 @@ +from pydantic import BaseModel, SecretStr +from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + + +class MilvusConfig(DBConfig, BaseModel): + uri: SecretStr | None = "http://localhost:19530" + + def to_dict(self) -> dict: + return {"uri": self.uri.get_secret_value()} + + + +class MilvusIndexConfig(BaseModel): + """Base config for milvus""" + + index: IndexType + metric_type: MetricType | None = None + + def parse_metric(self) -> str: + if not self.metric_type: + return "" + + if self.metric_type == MetricType.COSINE: + return MetricType.L2.value + return self.metric_type.value + + +class AutoIndexConfig(MilvusIndexConfig, DBCaseConfig): + index: IndexType = IndexType.AUTOINDEX + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + } + +class HNSWConfig(MilvusIndexConfig, DBCaseConfig): + M: int + efConstruction: int + ef: int | None = None + index: IndexType = IndexType.HNSW + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"M": self.M, "efConstruction": self.efConstruction}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"ef": self.ef}, + } + + +class DISKANNConfig(MilvusIndexConfig, DBCaseConfig): + search_list: int | None = None + index: IndexType = IndexType.DISKANN + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"search_list": self.search_list}, + } + + +class IVFFlatConfig(MilvusIndexConfig, DBCaseConfig): + nlist: int + nprobe: int | None = None + index: IndexType = IndexType.IVFFlat + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"nlist": self.nlist}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"nprobe": self.nprobe}, + } + + +class FLATConfig(MilvusIndexConfig, DBCaseConfig): + index: IndexType = IndexType.Flat + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {}, + } + +_milvus_case_config = { + IndexType.AUTOINDEX: AutoIndexConfig, + IndexType.HNSW: HNSWConfig, + IndexType.DISKANN: DISKANNConfig, + IndexType.IVFFlat: IVFFlatConfig, + IndexType.Flat: FLATConfig, +} + diff --git a/vector_db_bench/backend/clients/milvus/milvus.py b/vector_db_bench/backend/clients/milvus/milvus.py new file mode 100644 index 000000000..e21f5f9fc --- /dev/null +++ b/vector_db_bench/backend/clients/milvus/milvus.py @@ -0,0 +1,182 @@ +"""Wrapper around the Milvus vector database over VectorDB""" + +import logging +from contextlib import contextmanager +from typing import Any, Iterable, Type + +from pymilvus import Collection, utility +from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusException + +from ..api import VectorDB, DBCaseConfig, DBConfig, IndexType +from .config import MilvusConfig, _milvus_case_config + + +log = logging.getLogger(__name__) + + +class Milvus(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + collection_name: str = "VectorDBBenchCollection", + drop_old: bool = False, + name: str = "Milvus", + ): + """Initialize wrapper around the milvus vector database.""" + self.name = name + self.db_config = db_config + self.case_config = db_case_config + self.collection_name = collection_name + + self._primary_field = "pk" + self._scalar_field = "id" + self._vector_field = "vector" + self._index_name = "vector_idx" + + from pymilvus import connections + connections.connect(**self.db_config, timeout=30) + if drop_old and utility.has_collection(self.collection_name): + log.info(f"{self.name} client drop_old collection: {self.collection_name}") + utility.drop_collection(self.collection_name) + + if not utility.has_collection(self.collection_name): + fields = [ + FieldSchema(self._primary_field, DataType.INT64, is_primary=True), + FieldSchema(self._scalar_field, DataType.INT64), + FieldSchema(self._vector_field, DataType.FLOAT_VECTOR, dim=dim) + ] + + log.info(f"{self.name} create collection: {self.collection_name}") + + # Create the collection + coll = Collection( + name=self.collection_name, + schema=CollectionSchema(fields), + consistency_level="Session", + ) + + # self._pre_load(coll) + + connections.disconnect("default") + + @classmethod + def config_cls(cls) -> Type[DBConfig]: + return MilvusConfig + + @classmethod + def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: + return _milvus_case_config.get(index_type) + + + @contextmanager + def init(self) -> None: + """ + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + >>> self.search_embedding() + """ + from pymilvus import connections + self.col: Collection | None = None + + connections.connect(**self.db_config, timeout=60) + # Grab the existing colection with connections + self.col = Collection(self.collection_name) + + yield + connections.disconnect("default") + + def _pre_load(self, coll: Collection): + if not coll.has_index(index_name=self._index_name): + log.info(f"{self.name} create index and load") + try: + coll.create_index( + self._vector_field, + self.case_config.index_param(), + index_name=self._index_name, + ) + + coll.load() + except Exception as e: + log.warning(f"{self.name} pre load error: {e}") + raise e from None + + def _optimize(self): + log.info(f"{self.name} optimizing before search") + try: + self.col.flush() + self.col.compact() + self.col.wait_for_compaction_completed() + + # wait for index done and load refresh + self.col.create_index( + self._vector_field, + self.case_config.index_param(), + index_name=self._index_name, + ) + utility.wait_for_index_building_complete(self.collection_name) + self.col.load() + # self.col.load(_refresh=True) + # utility.wait_for_loading_complete(self.collection_name) + # import time; time.sleep(10) + except Exception as e: + log.warning(f"{self.name} optimize error: {e}") + raise e from None + + def ready_to_load(self): + assert self.col, "Please call self.init() before" + self._pre_load(self.col) + pass + + def ready_to_search(self): + assert self.col, "Please call self.init() before" + self._optimize() + + def insert_embeddings( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> int: + """Insert embeddings into Milvus. should call self.init() first""" + # use the first insert_embeddings to init collection + assert self.col is not None + insert_data = [ + metadata, + metadata, + embeddings, + ] + + try: + res = self.col.insert(insert_data, **kwargs) + return len(res.primary_keys) + except MilvusException as e: + log.warning("Failed to insert data") + raise e from None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + """Perform a search on a query embedding and return results.""" + assert self.col is not None + + expr = f"{self._scalar_field} {filters.get('metadata')}" if filters else "" + + # Perform the search. + res = self.col.search( + data=[query], + anns_field=self._vector_field, + param=self.case_config.search_param(), + limit=k, + expr=expr, + ) + + # Organize results. + ret = [result.id for result in res[0]] + return ret diff --git a/vector_db_bench/backend/clients/pinecone/config.py b/vector_db_bench/backend/clients/pinecone/config.py new file mode 100644 index 000000000..27c86d78f --- /dev/null +++ b/vector_db_bench/backend/clients/pinecone/config.py @@ -0,0 +1,15 @@ +from pydantic import BaseModel, SecretStr +from ..api import DBConfig + + +class PineconeConfig(DBConfig, BaseModel): + api_key: SecretStr | None = None + environment: SecretStr | None = None + index_name: str + + def to_dict(self) -> dict: + return { + "api_key": self.api_key.get_secret_value(), + "environment": self.environment.get_secret_value(), + "index_name": self.index_name, + } diff --git a/vector_db_bench/backend/clients/pinecone/pinecone.py b/vector_db_bench/backend/clients/pinecone/pinecone.py new file mode 100644 index 000000000..e2820c42c --- /dev/null +++ b/vector_db_bench/backend/clients/pinecone/pinecone.py @@ -0,0 +1,112 @@ +"""Wrapper around the Pinecone vector database over VectorDB""" + +import logging +from contextlib import contextmanager +from typing import Any, Type + +import pinecone + +from ..api import VectorDB, DBConfig, DBCaseConfig, EmptyDBCaseConfig, IndexType +from .config import PineconeConfig + + +log = logging.getLogger(__name__) + +PINECONE_MAX_NUM_PER_BATCH = 1000 +PINECONE_MAX_SIZE_PER_BATCH = 2 * 1024 * 1024 # 2MB + +class Pinecone(VectorDB): + def __init__( + self, + dim, + db_config: dict, + db_case_config: DBCaseConfig, + drop_old: bool = False, + ): + """Initialize wrapper around the milvus vector database.""" + self.index_name = db_config["index_name"] + self.api_key = db_config["api_key"] + self.environment = db_config["environment"] + self.batch_size = int(min(PINECONE_MAX_SIZE_PER_BATCH / (dim * 5), PINECONE_MAX_NUM_PER_BATCH)) + pinecone.init( + api_key=self.api_key, environment=self.environment) + if drop_old: + list_indexes = pinecone.list_indexes() + if self.index_name in list_indexes: + index = pinecone.Index(self.index_name) + index_dim = index.describe_index_stats()["dimension"] + if (index_dim != dim): + raise ValueError( + f"Pinecone index {self.index_name} dimension mismatch, expected {index_dim} got {dim}") + log.info( + f"Pinecone client delete old index: {self.index_name}") + index.delete(delete_all=True) + index.close() + else: + raise ValueError( + f"Pinecone index {self.index_name} does not exist") + + self._metadata_key = "meta" + + @classmethod + def config_cls(cls) -> Type[DBConfig]: + return PineconeConfig + + @classmethod + def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: + return EmptyDBCaseConfig + + @contextmanager + def init(self) -> None: + pinecone.init( + api_key=self.api_key, environment=self.environment) + self.index = pinecone.Index(self.index_name) + yield + self.index.close() + + def ready_to_load(self): + pass + + def ready_to_search(self): + pass + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + ) -> list[str]: + assert len(embeddings) == len(metadata) + for batch_start_offset in range(0, len(embeddings), self.batch_size): + batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) + insert_datas = [] + for i in range(batch_start_offset, batch_end_offset): + insert_data = (str(metadata[i]), embeddings[i], { + self._metadata_key: metadata[i]}) + insert_datas.append(insert_data) + self.index.upsert(insert_datas) + return len(embeddings) + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[tuple[int, float]]: + if filters is None: + pinecone_filters = {} + else: + pinecone_filters = {self._metadata_key: {"$gte": filters["id"]}} + + try: + res = self.index.query( + top_k=k, + vector=query, + filter=pinecone_filters, + )['matches'] + except Exception as e: + print(f"Error querying index: {e}") + raise e + id_res = [int(one_res['id']) for one_res in res] + return id_res diff --git a/vector_db_bench/backend/clients/qdrant_cloud/config.py b/vector_db_bench/backend/clients/qdrant_cloud/config.py new file mode 100644 index 000000000..449a4a56a --- /dev/null +++ b/vector_db_bench/backend/clients/qdrant_cloud/config.py @@ -0,0 +1,16 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBConfig + + +class QdrantConfig(DBConfig, BaseModel): + url: SecretStr | None = None + api_key: SecretStr | None = None + prefer_grpc: bool = True + + def to_dict(self) -> dict: + return { + "url": self.url.get_secret_value(), + "api_key": self.api_key.get_secret_value(), + "prefer_grpc": self.prefer_grpc, + } diff --git a/vector_db_bench/backend/clients/qdrant_cloud/qdrant_cloud.py b/vector_db_bench/backend/clients/qdrant_cloud/qdrant_cloud.py new file mode 100644 index 000000000..db256eba0 --- /dev/null +++ b/vector_db_bench/backend/clients/qdrant_cloud/qdrant_cloud.py @@ -0,0 +1,169 @@ +"""Wrapper around the QdrantCloud vector database over VectorDB""" + +import logging +import time +from contextlib import contextmanager +from typing import Any, Type + +from ..api import VectorDB, DBConfig, DBCaseConfig, EmptyDBCaseConfig, IndexType +from .config import QdrantConfig +from qdrant_client.http.models import ( + CollectionStatus, + Distance, + VectorParams, + PayloadSchemaType, + Batch, + Filter, + FieldCondition, + Range, +) + +from qdrant_client import QdrantClient + + +log = logging.getLogger(__name__) + + +class QdrantCloud(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + collection_name: str = "QdrantCloudCollection", + drop_old: bool = False, + ): + """Initialize wrapper around the QdrantCloud vector database.""" + self.db_config = db_config + self.case_config = db_case_config + self.collection_name = collection_name + + self._primary_field = "pk" + self._vector_field = "vector" + + tmp_client = QdrantClient(**self.db_config) + if drop_old: + log.info(f"QdrantCloud client drop_old collection: {self.collection_name}") + tmp_client.delete_collection(self.collection_name) + + self._create_collection(dim, tmp_client) + tmp_client = None + + @classmethod + def config_cls(cls) -> Type[DBConfig]: + return QdrantConfig + + @classmethod + def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: + return EmptyDBCaseConfig + + @contextmanager + def init(self) -> None: + """ + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + >>> self.search_embedding() + """ + self.qdrant_client = QdrantClient(**self.db_config) + yield + self.qdrant_client = None + del(self.qdrant_client) + + def ready_to_load(self): + pass + + + def ready_to_search(self): + assert self.qdrant_client, "Please call self.init() before" + # wait for vectors to be fully indexed + SECONDS_WAITING_FOR_INDEXING_API_CALL = 5 + try: + while True: + info = self.qdrant_client.get_collection(self.collection_name) + time.sleep(SECONDS_WAITING_FOR_INDEXING_API_CALL) + if info.status != CollectionStatus.GREEN: + continue + if info.status == CollectionStatus.GREEN: + log.info(f"Stored vectors: {info.vectors_count}, Indexed vectors: {info.indexed_vectors_count}, Collection status: {info.indexed_vectors_count}") + return + except Exception as e: + log.warning(f"QdrantCloud ready to search error: {e}") + raise e from None + + def _create_collection(self, dim, qdrant_client: int): + log.info(f"Create collection: {self.collection_name}") + + try: + qdrant_client.create_collection( + collection_name=self.collection_name, + vectors_config=VectorParams(size=dim, distance=Distance.EUCLID) + ) + + qdrant_client.create_payload_index( + collection_name=self.collection_name, + field_name=self._primary_field, + field_schema=PayloadSchemaType.INTEGER, + ) + + except Exception as e: + if "already exists!" in str(e): + return + log.warning(f"Failed to create collection: {self.collection_name} error: {e}") + raise e from None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> list[str]: + """Insert embeddings into Milvus. should call self.init() first""" + assert self.qdrant_client is not None + try: + # TODO: counts + _ = self.qdrant_client.upsert( + collection_name=self.collection_name, + wait=True, + points=Batch(ids=metadata, payloads=[{self._primary_field: v} for v in metadata], vectors=embeddings) + ) + + return len(metadata) + except Exception as e: + log.info(f"Failed to insert data, {e}") + raise e from None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + """Perform a search on a query embedding and return results with score. + Should call self.init() first. + """ + assert self.qdrant_client is not None + + f = None + if filters: + f = Filter( + must=[FieldCondition( + key = self._primary_field, + range = Range( + gt=filters.get('id'), + ), + )] + ) + + res = self.qdrant_client.search( + collection_name=self.collection_name, + query_vector=query, + limit=k, + query_filter=f, + # with_payload=True, + ), + + ret = [result.id for result in res[0]] + return ret diff --git a/vector_db_bench/backend/clients/weaviate_cloud/config.py b/vector_db_bench/backend/clients/weaviate_cloud/config.py new file mode 100644 index 000000000..77fdff61d --- /dev/null +++ b/vector_db_bench/backend/clients/weaviate_cloud/config.py @@ -0,0 +1,45 @@ +from pydantic import BaseModel, SecretStr +import weaviate + +from ..api import DBConfig, DBCaseConfig, MetricType + + +class WeaviateConfig(DBConfig, BaseModel): + url: SecretStr | None = None + api_key: SecretStr | None = None + + def to_dict(self) -> dict: + return { + "url": self.url.get_secret_value(), + "auth_client_secret": weaviate.AuthApiKey(api_key=self.api_key.get_secret_value()), + } + + +class WeaviateIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + ef: int | None = -1 + efConstruction: int | None = None + maxConnections: int | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2-squared" + elif self.metric_type == MetricType.IP: + return "dot" + return "cosine" + + def index_param(self) -> dict: + if self.maxConnections is not None and self.efConstruction is not None: + params = { + "distance": self.parse_metric(), + "maxConnections": self.maxConnections, + "efConstruction": self.efConstruction, + } + else: + params = {"distance": self.parse_metric()} + return params + + def search_param(self) -> dict: + return { + "ef": self.ef, + } diff --git a/vector_db_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vector_db_bench/backend/clients/weaviate_cloud/weaviate_cloud.py new file mode 100644 index 000000000..b03063eaf --- /dev/null +++ b/vector_db_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -0,0 +1,151 @@ +"""Wrapper around the Weaviate vector database over VectorDB""" + +import logging +from typing import Any, Iterable, Type +from contextlib import contextmanager + +from weaviate.exceptions import WeaviateBaseError + +from ..api import VectorDB, DBConfig, DBCaseConfig, IndexType +from .config import WeaviateConfig, WeaviateIndexConfig + + +log = logging.getLogger(__name__) + + +class WeaviateCloud(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + collection_name: str = "VectorDBBenchCollection", + drop_old: bool = False, + ): + """Initialize wrapper around the weaviate vector database.""" + self.db_config = db_config + self.case_config = db_case_config + self.collection_name = collection_name + + self._scalar_field = "key" + self._vector_field = "vector" + self._index_name = "vector_idx" + + from weaviate import Client + client = Client(**db_config) + if drop_old: + try: + if client.schema.exists(self.collection_name): + log.info(f"weaviate client drop_old collection: {self.collection_name}") + client.schema.delete_class(self.collection_name) + except WeaviateBaseError as e: + log.warning(f"Failed to drop collection: {self.collection_name} error: {str(e)}") + raise e from None + self._create_collection(client) + client = None + + @classmethod + def config_cls(cls) -> Type[DBConfig]: + return WeaviateConfig + + @classmethod + def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: + return WeaviateIndexConfig + + @contextmanager + def init(self) -> None: + """ + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + >>> self.search_embedding() + """ + from weaviate import Client + self.client = Client(**self.db_config) + yield + self.client = None + del(self.client) + + def ready_to_load(self): + """Should call insert first, do nothing""" + pass + + def ready_to_search(self): + assert self.client.schema.exists(self.collection_name) + self.client.schema.update_config(self.collection_name, {"vectorIndexConfig": self.case_config.search_param() } ) + + def _create_collection(self, client): + if not client.schema.exists(self.collection_name): + log.info(f"Create collection: {self.collection_name}") + class_obj = { + "class": self.collection_name, + "vectorizer": "none", + "properties": [ + { + "dataType": ["int"], + "name": self._scalar_field, + }, + ] + } + class_obj["vectorIndexConfig"] = self.case_config.index_param() + try: + client.schema.create_class(class_obj) + except WeaviateBaseError as e: + log.warning(f"Failed to create collection: {self.collection_name} error: {str(e)}") + raise e from None + + def insert_embeddings( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> int: + """Insert embeddings into Weaviate""" + assert self.client.schema.exists(self.collection_name) + + try: + with self.client.batch as batch: + batch.batch_size = len(metadata) + batch.dynamic = True + res = [] + for i in range(len(metadata)): + res.append(batch.add_data_object( + {self._scalar_field: metadata[i]}, + class_name=self.collection_name, + vector=embeddings[i] + )) + return len(res) + except WeaviateBaseError as e: + log.warning(f"Failed to insert data, error: {str(e)}") + raise e from None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + """Perform a search on a query embedding and return results with distance. + Should call self.init() first. + """ + assert self.client.schema.exists(self.collection_name) + + query_obj = self.client.query.get(self.collection_name, [self._scalar_field]).with_additional("distance").with_near_vector({"vector": query}).with_limit(k) + if filters: + where_filter = { + "path": "key", + "operator": "GreaterThanEqual", + "valueInt": filters.get('id') + } + query_obj = query_obj.with_where(where_filter) + + # Perform the search. + res = query_obj.do() + + # Organize results. + ret = [result[self._scalar_field] for result in res["data"]["Get"][self.collection_name]] + + return ret + diff --git a/vector_db_bench/backend/clients/zilliz_cloud/config.py b/vector_db_bench/backend/clients/zilliz_cloud/config.py new file mode 100644 index 000000000..43df0044a --- /dev/null +++ b/vector_db_bench/backend/clients/zilliz_cloud/config.py @@ -0,0 +1,34 @@ +from pydantic import BaseModel, SecretStr +from ..api import DBCaseConfig, DBConfig +from ..milvus.config import MilvusIndexConfig, IndexType + + +class ZillizCloudConfig(DBConfig, BaseModel): + uri: SecretStr | None = None + user: str + password: SecretStr | None = None + + def to_dict(self) -> dict: + return { + "uri": self.uri.get_secret_value(), + "user": self.user, + "password": self.password.get_secret_value(), + } + + +class AutoIndexConfig(MilvusIndexConfig, DBCaseConfig): + index: IndexType = IndexType.AUTOINDEX + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + } + + diff --git a/vector_db_bench/backend/clients/zilliz_cloud/zilliz_cloud.py b/vector_db_bench/backend/clients/zilliz_cloud/zilliz_cloud.py new file mode 100644 index 000000000..396eabb61 --- /dev/null +++ b/vector_db_bench/backend/clients/zilliz_cloud/zilliz_cloud.py @@ -0,0 +1,35 @@ +"""Wrapper around the ZillizCloud vector database over VectorDB""" + +from typing import Type +from ..milvus.milvus import Milvus +from ..api import DBConfig, DBCaseConfig, IndexType +from .config import ZillizCloudConfig, AutoIndexConfig + + +class ZillizCloud(Milvus): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + collection_name: str = "ZillizCloudVectorDBBench", + drop_old: bool = False, + name: str = "ZillizCloud" + ): + super().__init__( + dim=dim, + db_config=db_config, + db_case_config=db_case_config, + collection_name=collection_name, + drop_old=drop_old, + name=name, + ) + + @classmethod + def config_cls(cls) -> Type[DBConfig]: + return ZillizCloudConfig + + + @classmethod + def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: + return AutoIndexConfig diff --git a/vector_db_bench/backend/dataset.py b/vector_db_bench/backend/dataset.py new file mode 100644 index 000000000..d4f9834b0 --- /dev/null +++ b/vector_db_bench/backend/dataset.py @@ -0,0 +1,393 @@ +""" +Usage: + >>> from xxx import dataset as ds + >>> gist_s = ds.get(ds.Name.GIST, ds.Label.SMALL) + >>> gist_s.dict() + dataset: {'data': {'name': 'GIST', 'dim': 128, 'metric_type': 'L2', 'label': 'SMALL', 'size': 50000000}, 'data_dir': 'xxx'} +""" + +import os +import logging +import pathlib +import math +from hashlib import md5 +from enum import Enum, auto +from typing import Any + +import s3fs +import pandas as pd +from tqdm import tqdm +from pydantic.dataclasses import dataclass + +from ..base import BaseModel +from .. import config +from ..backend.clients import MetricType +from . import utils + +log = logging.getLogger(__name__) + +@dataclass +class LAION: + name: str = "LAION" + dim: int = 768 + metric_type: MetricType = MetricType.COSINE + use_shuffled: bool = False + + @property + def dir_name(self) -> str: + return f"{self.name}_{self.label}_{utils.numerize(self.size)}".lower() + +@dataclass +class GIST: + name: str = "GIST" + dim: int = 960 + metric_type: MetricType = MetricType.L2 + use_shuffled: bool = False + + @property + def dir_name(self) -> str: + return f"{self.name}_{self.label}_{utils.numerize(self.size)}".lower() + +@dataclass +class Cohere: + name: str = "Cohere" + dim: int = 768 + metric_type: MetricType = MetricType.COSINE + use_shuffled: bool = config.USE_SHUFFLED_DATA + + @property + def dir_name(self) -> str: + return f"{self.name}_{self.label}_{utils.numerize(self.size)}".lower() + +@dataclass +class Glove: + name: str = "Glove" + dim: int = 200 + metric_type: MetricType = MetricType.COSINE + use_shuffled: bool = False + + @property + def dir_name(self) -> str: + return f"{self.name}_{self.label}_{utils.numerize(self.size)}".lower() + +@dataclass +class SIFT: + name: str = "SIFT" + dim: int = 128 + metric_type: MetricType = MetricType.COSINE + use_shuffled: bool = False + + @property + def dir_name(self) -> str: + return f"{self.name}_{self.label}_{utils.numerize(self.size)}".lower() + +@dataclass +class LAION_L(LAION): + label: str = "LARGE" + size: int = 100_000_000 + +@dataclass +class GIST_S(GIST): + label: str = "SMALL" + size: int = 100_000 + +@dataclass +class GIST_M(GIST): + label: str = "MEDIUM" + size: int = 1_000_000 + +@dataclass +class Cohere_S(Cohere): + label: str = "SMALL" + size: int = 100_000 + +@dataclass +class Cohere_M(Cohere): + label: str = "MEDIUM" + size: int = 1_000_000 + +@dataclass +class Cohere_L(Cohere): + label : str = "LARGE" + size : int = 10_000_000 + +@dataclass +class Glove_S(Glove): + label: str = "SMALL" + size : int = 100_000 + +@dataclass +class Glove_M(Glove): + label: str = "MEDIUM" + size : int = 1_000_000 + +@dataclass +class SIFT_S(SIFT): + label: str = "SMALL" + size : int = 500_000 + +@dataclass +class SIFT_M(SIFT): + label: str = "MEDIUM" + size : int = 5_000_000 + +@dataclass +class SIFT_L(SIFT): + label: str = "LARGE" + size : int = 50_000_000 + + +class DataSet(BaseModel): + """Download dataset if not int the local directory. Provide data for cases. + + DataSet is iterable, each iteration will return the next batch of data in pandas.DataFrame + + Examples: + >>> cohere_s = DataSet(data=Cohere_S) + >>> for data in cohere_s: + >>> print(data.columns) + """ + data: GIST | Cohere | Glove | SIFT | Any + test_data: pd.DataFrame | None = None + train_files : list[str] = [] + + def __eq__(self, obj): + if isinstance(obj, DataSet): + return self.data.name == obj.data.name and \ + self.data.label == obj.data.label + return False + + @property + def data_dir(self) -> pathlib.Path: + """ data local directory: config.DATASET_LOCAL_DIR/{dataset_name}/{dataset_dirname} + + Examples: + >>> sift_s = DataSet(data=SIFT_L()) + >>> sift_s.relative_path + '/tmp/vector_db_bench/dataset/sift/sift_small_500k/' + """ + return pathlib.Path(config.DATASET_LOCAL_DIR, self.data.name.lower(), self.data.dir_name.lower()) + + @property + def download_dir(self) -> str: + """ data s3 directory: config.DEFAULT_DATASET_URL/{dataset_dirname} + + Examples: + >>> sift_s = DataSet(data=SIFT_L()) + >>> sift_s.download_dir + 'assets.zilliz.com/benchmark/sift_small_500k' + """ + return f"{config.DEFAULT_DATASET_URL}{self.data.dir_name}" + + def __iter__(self): + return DataSetIterator(self) + + + def _validate_local_file(self): + if not self.data_dir.exists(): + log.info(f"local file path not exist, creating it: {self.data_dir}") + self.data_dir.mkdir(parents=True) + + fs = s3fs.S3FileSystem( + anon=True, + client_kwargs={'region_name': 'us-west-2'} + ) + dataset_info = fs.ls(self.download_dir, detail=True) + if len(dataset_info) == 0: + raise ValueError(f"No data in s3 for dataset: {self.download_dir}") + path2etag = {info['Key']: info['ETag'].split('"')[1] for info in dataset_info} + + perfix_to_filter = "train" if self.data.use_shuffled else "shuffle_train" + filtered_keys = [key for key in path2etag.keys() if key.split("/")[-1].startswith(perfix_to_filter)] + for k in filtered_keys: + path2etag.pop(k) + + # get local files ended with '.parquet' + file_names = [p.name for p in self.data_dir.glob("*.parquet")] + log.info(f"local files: {file_names}") + log.info(f"s3 files: {path2etag.keys()}") + downloads = [] + if len(file_names) == 0: + log.info("no local files, set all to downloading lists") + downloads = path2etag.keys() + else: + # if local file exists, check the etag of local file with s3, + # make sure data files aren't corrupted. + for name in tqdm([key.split("/")[-1] for key in path2etag.keys()]): + s3_path = f"{self.download_dir}/{name}" + local_path = self.data_dir.joinpath(name) + log.debug(f"s3 path: {s3_path}, local_path: {local_path}") + if not local_path.exists(): + log.info(f"local file not exists: {local_path}, add to downloading lists") + downloads.append(s3_path) + + elif not self.match_etag(path2etag.get(s3_path), local_path): + log.info(f"local file etag not match with s3 file: {local_path}, add to downloading lists") + downloads.append(s3_path) + + for s3_file in tqdm(downloads): + log.debug(f"downloading file {s3_file} to {self.data_dir}") + fs.download(s3_file, self.data_dir.as_posix()) + + def match_etag(self, expected_etag: str, local_file) -> bool: + """Check if local files' etag match with S3""" + def factor_of_1MB(filesize, num_parts): + x = filesize / int(num_parts) + y = x % 1048576 + return int(x + 1048576 - y) + + def calc_etag(inputfile, partsize): + md5_digests = [] + with open(inputfile, 'rb') as f: + for chunk in iter(lambda: f.read(partsize), b''): + md5_digests.append(md5(chunk).digest()) + return md5(b''.join(md5_digests)).hexdigest() + '-' + str(len(md5_digests)) + + def possible_partsizes(filesize, num_parts): + return lambda partsize: partsize < filesize and (float(filesize) / float(partsize)) <= num_parts + + filesize = os.path.getsize(local_file) + le = "" + if '-' not in expected_etag: # no spliting uploading + with open(local_file, 'rb') as f: + le = md5(f.read()).hexdigest() + log.debug(f"calculated local etag {le}, expected etag: {expected_etag}") + return expected_etag == le + else: + num_parts = int(expected_etag.split('-')[-1]) + partsizes = [ ## Default Partsizes Map + 8388608, # aws_cli/boto3 + 15728640, # s3cmd + factor_of_1MB(filesize, num_parts) # Used by many clients to upload large files + ] + + for partsize in filter(possible_partsizes(filesize, num_parts), partsizes): + le = calc_etag(local_file, partsize) + log.debug(f"calculated local etag {le}, expected etag: {expected_etag}") + if expected_etag == le: + return True + return False + + def prepare(self, check=True) -> bool: + """Download the dataset from S3 + url = f"{config.DEFAULT_DATASET_URL}/{self.data.dir_name}" + + download files from url to self.data_dir, there'll be 4 types of files in the data_dir + - train*.parquet: for training + - test.parquet: for testing + - neighbors.parquet: ground_truth of the test.parquet + - neighbors_90p.parquet: ground_truth of the test.parquet after filtering 90% data + - neighbors_head_1p.parquet: ground_truth of the test.parquet after filtering 1% data + - neighbors_99p.parquet: ground_truth of the test.parquet after filtering 99% data + """ + if check: + self._validate_local_file() + + prefix = "shuffle_train" if self.data.use_shuffled else "train" + self.train_files = sorted([f.name for f in self.data_dir.glob(f'{prefix}*.parquet')]) + log.debug(f"{self.data.name}: available train files {self.train_files}") + self.test_data = self._read_file("test.parquet") + return True + + def get_ground_truth(self, filters: int | float | None = None) -> pd.DataFrame: + + file_name = "" + if filters is None: + file_name = "neighbors.parquet" + elif filters == 0.01: + file_name = "neighbors_head_1p.parquet" + elif filters == 0.99: + file_name = "neighbors_tail_1p.parquet" + else: + raise ValueError(f"Filters not supported: {filters}") + return self._read_file(file_name) + + def _read_file(self, file_name: str) -> pd.DataFrame: + """read one file from disk into memory""" + import pyarrow.parquet as pq + + p = pathlib.Path(self.data_dir, file_name) + log.info(f"reading file into memory: {p}") + if not p.exists(): + log.warning(f"No such file: {p}") + return pd.DataFrame() + data = pq.read_table(p) + df = data.to_pandas() + return df + + +class DataSetIterator: + def __init__(self, dataset: DataSet): + self._ds = dataset + self._idx = 0 # file number + self._curr: pd.DataFrame | None = None + self._sub_idx = [0 for i in range(len(self._ds.train_files))] # iter num for each file + + def __next__(self) -> pd.DataFrame: + """return the data in the next file of the training list""" + if self._idx < len(self._ds.train_files): + _sub = self._sub_idx[self._idx] + if _sub == 0 and self._idx == 0: # init + file_name = self._ds.train_files[self._idx] + self._curr = self._ds._read_file(file_name) + self._iter_num = math.ceil(self._curr.shape[0]/100_000) + + if _sub == self._iter_num: + if self._idx == len(self._ds.train_files) - 1: + self._curr = None + raise StopIteration + else: + self._idx += 1 + _sub = self._sub_idx[self._idx] + + self._curr = None + file_name = self._ds.train_files[self._idx] + self._curr = self._ds._read_file(file_name) + + sub_df = self._curr[_sub*100_000: (_sub+1)*100_000] + self._sub_idx[self._idx] += 1 + log.info(f"Get the [{_sub+1}/{self._iter_num}] batch of {self._idx+1}/{len(self._ds.train_files)} train file") + return sub_df + self._curr = None + raise StopIteration + + +class Name(Enum): + GIST = auto() + Cohere = auto() + Glove = auto() + SIFT = auto() + LAION = auto() + + +class Label(Enum): + SMALL = auto() + MEDIUM = auto() + LARGE = auto() + +_global_ds_mapping = { + Name.GIST: { + Label.SMALL: DataSet(data=GIST_S()), + Label.MEDIUM: DataSet(data=GIST_M()), + }, + Name.Cohere: { + Label.SMALL: DataSet(data=Cohere_S()), + Label.MEDIUM: DataSet(data=Cohere_M()), + Label.LARGE: DataSet(data=Cohere_L()), + }, + Name.Glove:{ + Label.SMALL: DataSet(data=Glove_S()), + Label.MEDIUM: DataSet(data=Glove_M()), + }, + Name.SIFT: { + Label.SMALL: DataSet(data=SIFT_S()), + Label.MEDIUM: DataSet(data=SIFT_M()), + Label.LARGE: DataSet(data=SIFT_L()), + }, + Name.LAION: { + Label.LARGE: DataSet(data=LAION_L()), + }, +} + +def get(ds: Name, label: Label): + return _global_ds_mapping.get(ds, {}).get(label) diff --git a/vector_db_bench/backend/result_collector.py b/vector_db_bench/backend/result_collector.py new file mode 100644 index 000000000..e17ab0571 --- /dev/null +++ b/vector_db_bench/backend/result_collector.py @@ -0,0 +1,15 @@ +import pathlib +from ..models import TestResult + + +class ResultCollector: + @classmethod + def collect(cls, result_dir: pathlib.Path) -> list[TestResult]: + results = [] + if not result_dir.exists() or len(list(result_dir.glob("*.json"))) == 0: + return [] + + for json_file in result_dir.glob("*.json"): + results.append(TestResult.read_file(json_file, trans_unit=True)) + + return results diff --git a/vector_db_bench/backend/runner/__init__.py b/vector_db_bench/backend/runner/__init__.py new file mode 100644 index 000000000..77bb25d67 --- /dev/null +++ b/vector_db_bench/backend/runner/__init__.py @@ -0,0 +1,12 @@ +from .mp_runner import ( + MultiProcessingSearchRunner, +) + +from .serial_runner import SerialSearchRunner, SerialInsertRunner + + +__all__ = [ + 'MultiProcessingSearchRunner', + 'SerialSearchRunner', + 'SerialInsertRunner', +] diff --git a/vector_db_bench/backend/runner/mp_runner.py b/vector_db_bench/backend/runner/mp_runner.py new file mode 100644 index 000000000..1013f0ff1 --- /dev/null +++ b/vector_db_bench/backend/runner/mp_runner.py @@ -0,0 +1,124 @@ +import time +import traceback +import concurrent +import multiprocessing as mp +import logging +from typing import Iterable +import numpy as np +from ..clients import api +from .. import utils +from ... import config + + +NUM_PER_BATCH = config.NUM_PER_BATCH +log = logging.getLogger(__name__) + + +class MultiProcessingSearchRunner: + """ multiprocessing search runner + + Args: + k(int): search topk, default to 100 + concurrency(Iterable): concurrencies, default [1, 5, 10, 15, 20, 25, 30, 35] + duration(int): duration for each concurency, default to 30s + """ + def __init__( + self, + db: api.VectorDB, + test_data: np.ndarray, + k: int = 100, + filters: dict | None = None, + concurrencies: Iterable[int] = (1, 5, 10, 15, 20, 25, 30, 35), + duration: int = 30, + ): + self.db = db + self.k = k + self.filters = filters + self.concurrencies = concurrencies + self.duration = duration + + self.test_data = utils.SharedNumpyArray(test_data) + log.debug(f"test dataset columns: {len(test_data)}") + + def search(self, test_np: utils.SharedNumpyArray) -> tuple[int, float]: + with self.db.init(): + test_data = test_np.read().tolist() + num, idx = len(test_data), 0 + + start_time = time.perf_counter() + count = 0 + while time.perf_counter() < start_time + self.duration: + s = time.perf_counter() + try: + self.db.search_embedding( + test_data[idx], + self.k, + self.filters, + ) + except Exception as e: + log.warning(f"VectorDB search_embedding error: {e}") + traceback.print_exc(chain=True) + raise e from None + + count += 1 + # loop through the test data + idx = idx + 1 if idx < num - 1 else 0 + + if count % 500 == 0: + log.debug(f"({mp.current_process().name:16}) search_count: {count}, latest_latency={time.perf_counter()-s}") + + total_dur = round(time.perf_counter() - start_time, 4) + log.info( + f"{mp.current_process().name:16} search {self.duration}s: " + f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}" + ) + + return (count, total_dur) + + @staticmethod + def get_mp_context(): + mp_start_method = "forkserver" if "forkserver" in mp.get_all_start_methods() else "spawn" + log.debug(f"MultiProcessingSearchRunner get multiprocessing start method: {mp_start_method}") + return mp.get_context(mp_start_method) + + def _run_all_concurrencies_mem_efficient(self) -> float: + max_qps = 0 + try: + for conc in self.concurrencies: + with concurrent.futures.ProcessPoolExecutor(mp_context=self.get_mp_context(), max_workers=conc) as executor: + start = time.perf_counter() + log.info(f"start search {self.duration}s in concurrency {conc}, filters: {self.filters}") + future_iter = executor.map(self.search, [self.test_data for i in range(conc)]) + all_count = sum([r[0] for r in future_iter]) + + cost = time.perf_counter() - start + qps = round(all_count / cost, 4) + log.info(f"end search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") + + if qps > max_qps: + max_qps = qps + log.info(f"update largest qps with concurrency {conc}: current max_qps={max_qps}") + except Exception as e: + log.warning(f"fail to search all concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}") + traceback.print_exc() + + # No results available, raise exception + if max_qps == 0.0: + raise e from None + + finally: + self.stop() + + return max_qps + + def run(self) -> float: + """ + Returns: + float: largest qps + """ + return self._run_all_concurrencies_mem_efficient() + + def stop(self) -> None: + if self.test_data: + self.test_data.unlink() + self.test_data = None diff --git a/vector_db_bench/backend/runner/serial_runner.py b/vector_db_bench/backend/runner/serial_runner.py new file mode 100644 index 000000000..fdcd7b658 --- /dev/null +++ b/vector_db_bench/backend/runner/serial_runner.py @@ -0,0 +1,162 @@ +import time +import logging +import traceback +import concurrent +import multiprocessing as mp +import math +import numpy as np +import pandas as pd + +from ..clients import api +from ...metric import calc_recall +from ...models import LoadTimeoutError +from .. import utils +from ... import config + +NUM_PER_BATCH = config.NUM_PER_BATCH +LOAD_TIMEOUT = 12 * 60 * 60 + +log = logging.getLogger(__name__) + + +class SerialInsertRunner: + def __init__(self, db: api.VectorDB, train_emb: list[list[float]], train_id: list[int]): + log.debug(f"Dataset shape: {len(train_emb)}") + self.db = db + self.shared_emb = train_emb + self.train_id = train_id + + self.seq_batches = math.ceil(len(train_emb)/NUM_PER_BATCH) + + def insert_data(self, left_id: int = 0) -> int: + with self.db.init(): + all_embeddings = self.shared_emb + + # unique id for endlessness insertion + all_metadata = [i+left_id for i in self.train_id] + + num_conc_batches = math.ceil(len(all_embeddings)/NUM_PER_BATCH) + log.info(f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}") + count = 0 + for batch_id in range(self.seq_batches): + metadata = all_metadata[batch_id*NUM_PER_BATCH: (batch_id+1)*NUM_PER_BATCH] + embeddings = all_embeddings[batch_id*NUM_PER_BATCH: (batch_id+1)*NUM_PER_BATCH] + + log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_conc_batches}], Start inserting {len(metadata)} embeddings") + insert_count = self.db.insert_embeddings( + embeddings=embeddings, + metadata=metadata, + ) + log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_conc_batches}], Finish inserting {len(metadata)} embeddings") + + assert insert_count == len(metadata) + count += insert_count + log.info(f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}") + return count + + @utils.time_it + def _insert_all_batches(self) -> int: + """Performance case only""" + with concurrent.futures.ProcessPoolExecutor(mp_context=mp.get_context('spawn'), max_workers=1) as executor: + future = executor.submit(self.insert_data) + count = future.result() + return count + + + def run_endlessness(self) -> int: + """run forever util DB raises exception or crash""" + start_time = time.perf_counter() + max_load_count, times = 0, 0 + try: + with self.db.init(): + self.db.ready_to_load() + while time.perf_counter() - start_time < LOAD_TIMEOUT: + count = self.insert_data(left_id=max_load_count) + max_load_count += count + times += 1 + log.info(f"Loaded {times:3} entire dataset, current max load counts={utils.numerize(max_load_count)}, {max_load_count}") + raise LoadTimeoutError("load case timeout and stop") + except Exception as e: + log.info(f"load reach limit, insertion counts={utils.numerize(max_load_count)}, {max_load_count}, err={e}") + traceback.print_exc() + return max_load_count + + def run(self) -> int: + count, dur = self._insert_all_batches() + return count + + +class SerialSearchRunner: + def __init__( + self, + db: api.VectorDB, + test_data: list[list[float]], + ground_truth: pd.DataFrame, + k: int = 100, + filters: dict | None = None, + ): + self.db = db + self.k = k + self.filters = filters + + if isinstance(test_data[0], np.ndarray): + self.test_data = [query.tolist() for query in test_data] + else: + self.test_data = test_data + self.ground_truth = ground_truth + + def search(self, args: tuple[list, pd.DataFrame]): + log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") + with self.db.init(): + test_data, ground_truth = args + + log.debug(f"test dataset size: {len(test_data)}") + log.info(f"ground truth size: {ground_truth.columns}, shape: {ground_truth.shape}") + + latencies, recalls = [], [] + for idx, emb in enumerate(test_data): + s = time.perf_counter() + try: + results = self.db.search_embedding( + emb, + self.k, + self.filters, + ) + + except Exception as e: + log.warning(f"VectorDB search_embedding error: {e}") + traceback.print_exc(chain=True) + raise e from None + + latencies.append(time.perf_counter() - s) + + gt = ground_truth['neighbors_id'][idx] + recalls.append(calc_recall(self.k, gt[:self.k], results)) + + + if len(latencies) % 100 == 0: + log.debug(f"({mp.current_process().name:14}) search_count={len(latencies):3}, latest_latency={latencies[-1]}, latest recall={recalls[-1]}") + + avg_latency = round(np.mean(latencies), 4) + avg_recall = round(np.mean(recalls), 4) + cost = round(np.sum(latencies), 4) + p99 = round(np.percentile(latencies, 99), 4) + log.info( + f"{mp.current_process().name:14} search entire test_data: " + f"cost={cost}s, " + f"queries={len(latencies)}, " + f"avg_recall={avg_recall}, " + f"avg_latency={avg_latency}, " + f"p99={p99}" + ) + return (avg_recall, p99) + + + def _run_in_subprocess(self) -> tuple[float, float]: + with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: + future = executor.submit(self.search, (self.test_data, self.ground_truth)) + result = future.result() + return result + + def run(self) -> tuple[float, float]: + return self._run_in_subprocess() diff --git a/vector_db_bench/backend/task_runner.py b/vector_db_bench/backend/task_runner.py new file mode 100644 index 000000000..0cddbaf83 --- /dev/null +++ b/vector_db_bench/backend/task_runner.py @@ -0,0 +1,290 @@ +import logging +import traceback +import concurrent +import numpy as np +from enum import Enum, auto + +from . import utils +from .cases import Case, CaseLabel +from ..base import BaseModel +from ..models import TaskConfig + +from .clients import ( + api, + ZillizCloud, + Milvus, + MetricType +) +from ..metric import Metric +from .runner import MultiProcessingSearchRunner +from .runner import SerialSearchRunner, SerialInsertRunner + + +log = logging.getLogger(__name__) + + +class RunningStatus(Enum): + PENDING = auto() + FINISHED = auto() + + +class CaseRunner(BaseModel): + """ DataSet, filter_rate, db_class with db config + + Fields: + run_id(str): run_id of this case runner, + indicating which task does this case belong to. + config(TaskConfig): task configs of this case runner. + ca(Case): case for this case runner. + status(RunningStatus): RunningStatus of this case runner. + + db(api.VectorDB): The vector database for this case runner. + """ + + run_id: str + config: TaskConfig + ca: Case + status: RunningStatus + + db: api.VectorDB | None = None + test_emb: np.ndarray | None = None + search_runner: MultiProcessingSearchRunner | None = None + serial_search_runner: SerialSearchRunner | None = None + + def __eq__(self, obj): + if isinstance(obj, CaseRunner): + return self.ca.label == CaseLabel.Performance and \ + self.config.db == obj.config.db and \ + self.config.db_case_config == obj.config.db_case_config and \ + self.ca.dataset == obj.ca.dataset + return False + + def display(self) -> dict: + c_dict = self.ca.dict(include={'label':True, 'filters': True,'dataset':{'data': True} }) + c_dict['db'] = self.config.db_name + return c_dict + + @property + def normalize(self) -> bool: + assert self.db + return isinstance(self.db, (Milvus, ZillizCloud)) and \ + self.ca.dataset.data.metric_type == MetricType.COSINE + + def init_db(self, drop_old: bool = True) -> None: + db_cls = self.config.db.init_cls + + self.db = db_cls( + dim=self.ca.dataset.data.dim, + db_config=self.config.db_config.to_dict(), + db_case_config=self.config.db_case_config, + drop_old=drop_old, + ) + + def _pre_run(self, drop_old: bool = True): + try: + self.ca.dataset.prepare() + self.init_db(drop_old) + except Exception as e: + log.warning(f"pre run case error: {e}") + raise e from None + + def run(self, drop_old: bool = True) -> Metric: + self._pre_run(drop_old) + + if self.ca.label == CaseLabel.Load: + return self._run_load_case() + elif self.ca.label == CaseLabel.Performance: + return self._run_perf_case(drop_old) + else: + log.warning(f"unknown case type: {self.ca.label}") + raise ValueError(f"Unknown case type: {self.ca.label}") + + + def _run_load_case(self) -> Metric: + """ run load cases + + Returns: + Metric: the max load count + """ + log.info("start to run load case") + # datasets for load tests are quite small, can fit into memory + # only 1 file + data_df = [data_df for data_df in self.ca.dataset][0] + + all_embeddings, all_metadata = np.stack(data_df["emb"]).tolist(), data_df['id'].tolist() + runner = SerialInsertRunner(self.db, all_embeddings, all_metadata) + try: + count = runner.run_endlessness() + log.info(f"load reach limit: insertion counts={count}") + return Metric(max_load_count=count) + except Exception as e: + log.warning(f"run load case error: {e}") + raise e from None + log.info("end run load case") + + + def _run_perf_case(self, drop_old: bool = True) -> Metric: + try: + m = Metric() + if drop_old: + _, load_dur = self._load_train_data() + build_dur = self._optimize() + m.load_duration = round(load_dur+build_dur, 4) + + self._init_search_runner() + m.recall, m.serial_latency_p99 = self._serial_search() + m.qps = self._conc_search() + + log.info(f"got results: {m}") + return m + except Exception as e: + log.warning(f"performance case run error: {e}") + traceback.print_exc() + raise e + + @utils.time_it + def _load_train_data(self): + """Insert train data and get the insert_duration""" + for data_df in self.ca.dataset: + try: + all_metadata = data_df['id'].tolist() + + emb_np = np.stack(data_df['emb']) + if self.normalize: + log.debug("normalize the 100k train data") + all_embeddings = emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis].tolist() + else: + all_embeddings = emb_np.tolist() + + del(emb_np) + log.debug(f"normalized size: {len(all_embeddings)}, {len(all_metadata)}") + + runner = SerialInsertRunner(self.db, all_embeddings, all_metadata) + runner.run() + except Exception as e: + raise e from None + finally: + runner = None + + + def _serial_search(self) -> tuple[float, float]: + """Performance serial tests, search the entire test data once, + calculate the recall, serial_latency_p99 + + Returns: + tuple[float, float]: recall, serial_latency_p99 + """ + try: + return self.serial_search_runner.run() + except Exception as e: + log.warning(f"search error: {str(e)}, {e}") + self.stop() + raise e from None + + def _conc_search(self): + """Performance concurrency tests, search the test data endlessness + for 30s in several concurrencies + + Returns: + float: the largest qps in all concurrencies + """ + try: + return self.search_runner.run() + except Exception as e: + log.warning(f"search error: {str(e)}, {e}") + raise e from None + finally: + self.stop() + + @utils.time_it + def _task(self) -> None: + """""" + with self.db.init(): + self.db.ready_to_search() + + def _optimize(self) -> float: + with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: + future = executor.submit(self._task) + try: + return future.result()[1] + except Exception as e: + log.warning(f"VectorDB ready_to_search error: {e}") + raise e from None + + def _init_search_runner(self): + test_emb = np.stack(self.ca.dataset.test_data["emb"]) + if self.normalize: + test_emb = test_emb / np.linalg.norm(test_emb, axis=1)[:, np.newaxis] + self.test_emb = test_emb + + gt_df = self.ca.dataset.get_ground_truth(self.ca.filter_rate) + + self.serial_search_runner = SerialSearchRunner( + db=self.db, + test_data=self.test_emb.tolist(), + ground_truth=gt_df, + filters=self.ca.filters, + ) + + self.search_runner = MultiProcessingSearchRunner( + db=self.db, + test_data=self.test_emb, + filters=self.ca.filters, + ) + + def stop(self): + if self.search_runner: + self.search_runner.stop() + + +class TaskRunner(BaseModel): + run_id: str + task_label: str + case_runners: list[CaseRunner] + + def num_cases(self) -> int: + return len(self.case_runners) + + def num_finished(self) -> int: + return self._get_num_by_status(RunningStatus.FINISHED) + + def set_finished(self, idx: int) -> None: + self.case_runners[idx].status = RunningStatus.FINISHED + + def _get_num_by_status(self, status: RunningStatus) -> int: + return sum([1 for c in self.case_runners if c.status == status]) + + def display(self) -> None: + DATA_FORMAT = (" %-14s | %-12s %-20s %7s | %-10s") + TITLE_FORMAT = (" %-14s | %-12s %-20s %7s | %-10s") % ( + "DB", "CaseType", "Dataset", "Filter", "task_label") + + fmt = [TITLE_FORMAT] + fmt.append(DATA_FORMAT%( + "-"*11, + "-"*12, + "-"*20, + "-"*7, + "-"*7 + )) + + for f in self.case_runners: + if f.ca.filter_rate != 0.0: + filters = f.ca.filter_rate + elif f.ca.filter_size != 0: + filters = f.ca.filter_size + else: + filters = "None" + + ds_str = f"{f.ca.dataset.data.name}-{f.ca.dataset.data.label}-{utils.numerize(f.ca.dataset.data.size)}" + fmt.append(DATA_FORMAT%( + f.config.db_name, + f.ca.label.name, + ds_str, + filters, + self.task_label, + )) + + tmp_logger = logging.getLogger("no_color") + for f in fmt: + tmp_logger.info(f) diff --git a/vector_db_bench/backend/utils.py b/vector_db_bench/backend/utils.py new file mode 100644 index 000000000..927e997c0 --- /dev/null +++ b/vector_db_bench/backend/utils.py @@ -0,0 +1,85 @@ +import time +from functools import wraps +from multiprocessing.shared_memory import SharedMemory + +import numpy as np + + +def numerize(n) -> str: + """display positive number n for readability + + Examples: + >>> numerize(1_000) + '1K' + >>> numerize(1_000_000_000) + '1B' + """ + sufix2upbound = { + "EMPTY": 1e3, + "K": 1e6, + "M": 1e9, + "B": 1e12, + "END": float('inf'), + } + + display_n, sufix = n, "" + for s, base in sufix2upbound.items(): + # number >= 1000B will alway have sufix 'B' + if s == "END": + display_n = int(n/1e9) + sufix = "B" + break + + if n < base: + sufix = "" if s == "EMPTY" else s + display_n = int(n/(base/1e3)) + break + return f"{display_n}{sufix}" + + +def time_it(func): + @wraps(func) + def inner(*args, **kwargs): + pref = time.perf_counter() + result = func(*args, **kwargs) + delta = time.perf_counter() - pref + return result, delta + return inner + + +class SharedNumpyArray: + ''' Wraps a numpy array so that it can be shared quickly among processes, + avoiding unnecessary copying and (de)serializing. + ''' + def __init__(self, array: np.ndarray): + ''' + Creates the shared memory and copies the array therein + ''' + # create the shared memory location of the same size of the array + self._shared = SharedMemory(create=True, size=array.nbytes) + + # save data type and shape, necessary to read the data correctly + self._dtype, self._shape = array.dtype, array.shape + + # create a new numpy array that uses the shared memory we created. + # at first, it is filled with zeros + res = np.ndarray( + self._shape, dtype=self._dtype, buffer=self._shared.buf + ) + + # copy data from the array to the shared memory. numpy will + # take care of copying everything in the correct format + res[:] = array[:] + + def read(self) -> np.ndarray: + '''Reads the array from the shared memory without unnecessary copying. ''' + # simply create an array of the correct shape and type, + # using the shared memory location we created earlier + return np.ndarray(self._shape, self._dtype, buffer=self._shared.buf) + + def unlink(self) -> None: + ''' Releases the allocated memory. Call when finished using the data, + or when the data was copied somewhere else. + ''' + self._shared.close() + self._shared.unlink() diff --git a/vector_db_bench/base.py b/vector_db_bench/base.py new file mode 100644 index 000000000..3c71fb5a7 --- /dev/null +++ b/vector_db_bench/base.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel as PydanticBaseModel + + +class BaseModel(PydanticBaseModel, arbitrary_types_allowed=True): + pass + diff --git a/vector_db_bench/frontend/assets/chroma.png b/vector_db_bench/frontend/assets/chroma.png new file mode 100644 index 000000000..d395f8385 Binary files /dev/null and b/vector_db_bench/frontend/assets/chroma.png differ diff --git a/vector_db_bench/frontend/assets/elasticsearch.png b/vector_db_bench/frontend/assets/elasticsearch.png new file mode 100644 index 000000000..fabb7abc2 Binary files /dev/null and b/vector_db_bench/frontend/assets/elasticsearch.png differ diff --git a/vector_db_bench/frontend/assets/favicon.png b/vector_db_bench/frontend/assets/favicon.png new file mode 100644 index 000000000..1608b0c42 Binary files /dev/null and b/vector_db_bench/frontend/assets/favicon.png differ diff --git a/vector_db_bench/frontend/assets/milvus.png b/vector_db_bench/frontend/assets/milvus.png new file mode 100644 index 000000000..fd64e6eb8 Binary files /dev/null and b/vector_db_bench/frontend/assets/milvus.png differ diff --git a/vector_db_bench/frontend/assets/opensearch.png b/vector_db_bench/frontend/assets/opensearch.png new file mode 100644 index 000000000..34fe9ad88 Binary files /dev/null and b/vector_db_bench/frontend/assets/opensearch.png differ diff --git a/vector_db_bench/frontend/assets/pinecone.png b/vector_db_bench/frontend/assets/pinecone.png new file mode 100644 index 000000000..3a64143a1 Binary files /dev/null and b/vector_db_bench/frontend/assets/pinecone.png differ diff --git a/vector_db_bench/frontend/assets/qdrant.png b/vector_db_bench/frontend/assets/qdrant.png new file mode 100644 index 000000000..ba8a3552e Binary files /dev/null and b/vector_db_bench/frontend/assets/qdrant.png differ diff --git a/vector_db_bench/frontend/assets/vdb_benchmark.png b/vector_db_bench/frontend/assets/vdb_benchmark.png new file mode 100644 index 000000000..6a4f1c2b9 Binary files /dev/null and b/vector_db_bench/frontend/assets/vdb_benchmark.png differ diff --git a/vector_db_bench/frontend/assets/weaviate.png b/vector_db_bench/frontend/assets/weaviate.png new file mode 100644 index 000000000..446872991 Binary files /dev/null and b/vector_db_bench/frontend/assets/weaviate.png differ diff --git a/vector_db_bench/frontend/assets/zilliz.png b/vector_db_bench/frontend/assets/zilliz.png new file mode 100644 index 000000000..6117343a5 Binary files /dev/null and b/vector_db_bench/frontend/assets/zilliz.png differ diff --git a/vector_db_bench/frontend/components/check_results/charts.py b/vector_db_bench/frontend/components/check_results/charts.py new file mode 100644 index 000000000..3de1d6adb --- /dev/null +++ b/vector_db_bench/frontend/components/check_results/charts.py @@ -0,0 +1,245 @@ +from vector_db_bench.metric import metricOrder, isLowerIsBetterMetric, metricUnitMap +from vector_db_bench.frontend.const import * +import plotly.express as px +from vector_db_bench.frontend.utils import displayCaseText +from vector_db_bench.models import ResultLabel + + +def drawCharts(st, allData, failedTasks, cases): + st.markdown( + "", + unsafe_allow_html=True, + ) + st.markdown( + """""", + unsafe_allow_html=True, + ) + for case in cases: + chartContainer = st.expander(displayCaseText(case), True) + data = [data for data in allData if data["case"] == case] + drawChart(data, chartContainer) + + errorDBs = failedTasks[case] + showFailedDBs(chartContainer, errorDBs) + + +def showFailedDBs(st, errorDBs): + failedDBs = [db for db, label in errorDBs.items() if label == ResultLabel.FAILED] + timeoutDBs = [ + db for db, label in errorDBs.items() if label == ResultLabel.OUTOFRANGE + ] + + showFailedText(st, 'Failed', failedDBs) + showFailedText(st, 'Timeout', timeoutDBs) + + +def showFailedText(st, text, dbs): + if len(dbs) > 0: + st.markdown(f"
{text}:   {', '.join(dbs)}
", unsafe_allow_html=True) + + +def drawChart(data, st): + metricsSet = set() + for d in data: + metricsSet = metricsSet.union(d["metricsSet"]) + showMetrics = [metric for metric in metricOrder if metric in metricsSet] + + for i, metric in enumerate(showMetrics): + container = st.container() + drawMetricChart(data, metric, container) + + +def getLabelToShapeMap(data): + labelIndexMap = {} + + dbSet = {d["db"] for d in data} + for db in dbSet: + labelSet = {d["db_label"] for d in data if d["db"] == db} + labelList = list(labelSet) + + usedShapes = set() + i = 0 + for label in labelList: + if label not in labelIndexMap: + loopCount = 0 + while i % len(PATTERN_SHAPES) in usedShapes: + i += 1 + loopCount += 1 + if loopCount > len(PATTERN_SHAPES): + break + labelIndexMap[label] = i + i += 1 + else: + usedShapes.add(labelIndexMap[label] % len(PATTERN_SHAPES)) + + labelToShapeMap = { + label: getPatternShape(index) for label, index in labelIndexMap.items() + } + return labelToShapeMap + + +def drawMetricChart(data, metric, st): + dataWithMetric = [d for d in data if d.get(metric, 0) > 1e-7] + # dataWithMetric = data + if len(dataWithMetric) == 0: + return + + # title = st.container() + # title.markdown( + # f"**{metric}** ({'less' if isLowerIsBetterMetric(metric) else 'more'} is better)" + # ) + chart = st.container() + + height = len(dataWithMetric) * 24 + 48 + xmin = 0 + xmax = max([d.get(metric, 0) for d in dataWithMetric]) + xpadding = (xmax - xmin) / 16 + xpadding_multiplier = 1.6 + xrange = [xmin, xmax + xpadding * xpadding_multiplier] + unit = metricUnitMap.get(metric, "") + labelToShapeMap = getLabelToShapeMap(dataWithMetric) + categoryorder = ( + "total descending" if isLowerIsBetterMetric(metric) else "total ascending" + ) + fig = px.bar( + dataWithMetric, + x=metric, + y="db_name", + color="db", + height=height, + # pattern_shape="db_label", + # pattern_shape_sequence=SHAPES, + pattern_shape_map=labelToShapeMap, + orientation="h", + hover_data={ + "db": False, + "db_label": False, + "db_name": True, + }, + color_discrete_map=COLOR_MAP, + text_auto=True, + title=f"{metric.capitalize()} ({'less' if isLowerIsBetterMetric(metric) else 'more'} is better)", + ) + fig.update_xaxes(showticklabels=False, visible=False, range=xrange) + fig.update_yaxes( + # showticklabels=False, + # visible=False, + title=dict( + font=dict( + size=1, + ), + # text="", + ) + ) + fig.update_traces( + textposition="outside", + textfont=dict( + color="#333", + size=12, + ), + marker=dict( + pattern=dict(fillmode="overlay", fgcolor="#fff", fgopacity=1, size=7) + ), + texttemplate="%{x:,.4~r}" + unit, + ) + fig.update_layout( + margin=dict(l=0, r=0, t=48, b=12, pad=8), + bargap=0.25, + showlegend=False, + legend=dict( + orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title="" + ), + # legend=dict(orientation="v", title=""), + yaxis={"categoryorder": categoryorder}, + title=dict( + font=dict( + size=16, + color="#666", + # family="Arial, sans-serif", + ), + pad=dict(l=16), + # y=0.95, + # yanchor="top", + # yref="container", + ), + ) + + chart.plotly_chart(fig, use_container_width=True) + + +def drawChartQpsPerHour(metric, dataWithMetric, st): + title = st.container() + title.markdown( + f"**{metric}** ({'less' if isLowerIsBetterMetric(metric) else 'more'} is better)" + ) + chart = st.container() + + height = len(dataWithMetric) * 24 + xmin = 0 + xmax = max([d.get(metric, 0) for d in dataWithMetric]) + xpadding = (xmax - xmin) / 16 + xpadding_multiplier = 1.6 + xrange = [xmin, xmax + xpadding * xpadding_multiplier] + unit = metricUnitMap.get(metric, "") + labelToShapeMap = getLabelToShapeMap(dataWithMetric) + categoryorder = ( + "total descending" if isLowerIsBetterMetric(metric) else "total ascending" + ) + fig = px.bar( + dataWithMetric, + x=metric, + y="db_name", + color="db", + height=height, + pattern_shape="db_label", + # pattern_shape_sequence=SHAPES, + pattern_shape_map=labelToShapeMap, + orientation="h", + hover_data={ + "db": False, + "db_label": False, + "db_name": True, + }, + color_discrete_map=COLOR_MAP, + text_auto=True, + ) + fig.update_xaxes(showticklabels=False, visible=False, range=xrange) + fig.update_yaxes( + # showticklabels=False, + # visible=False, + title=dict( + font=dict( + size=1, + ), + # text="", + ) + ) + fig.update_traces( + textposition="outside", + textfont=dict( + # color="#fff", + size=14, + ), + marker=dict( + pattern=dict(fillmode="overlay", fgcolor="#fff", fgopacity=1, size=7) + ), + texttemplate="%{x:,.4~r}" + unit, + ) + fig.update_layout( + margin=dict(l=0, r=0, t=0, b=0, pad=8), + bargap=0.25, + showlegend=False, + legend=dict( + orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title="" + ), + # legend=dict(orientation="v", title=""), + yaxis={"categoryorder": categoryorder}, + ) + + chart.plotly_chart(fig, use_container_width=True) \ No newline at end of file diff --git a/vector_db_bench/frontend/components/check_results/data.py b/vector_db_bench/frontend/components/check_results/data.py new file mode 100644 index 000000000..be3112d4d --- /dev/null +++ b/vector_db_bench/frontend/components/check_results/data.py @@ -0,0 +1,86 @@ +from collections import defaultdict +from dataclasses import asdict +from vector_db_bench.metric import isLowerIsBetterMetric +from vector_db_bench.models import ResultLabel + + +def getChartData(tasks, dbNames, cases): + filterTasks = getFilterTasks(tasks, dbNames, cases) + mergedTasks, failedTasks = mergeTasks(filterTasks) + return mergedTasks, failedTasks + + +def getFilterTasks(tasks, dbNames, cases): + filterTasks = [ + task + for task in tasks + if task.task_config.db_name in dbNames + and task.task_config.case_config.case_id.value in cases + ] + return filterTasks + + +def mergeTasks(tasks): + dbCaseMetricsMap = defaultdict(lambda: defaultdict(dict)) + for task in tasks: + db_name = task.task_config.db_name + db = task.task_config.db.value + db_label = task.task_config.db_config.db_label or "" + case = task.task_config.case_config.case_id.value + dbCaseMetricsMap[db_name][case] = { + "db": db, + "db_label": db_label, + "metrics": mergeMetrics( + dbCaseMetricsMap[db_name][case].get("metrics", {}), asdict(task.metrics) + ), + "label": getBetterLabel(dbCaseMetricsMap[db_name][case].get("label", ResultLabel.FAILED), task.label) + } + + mergedTasks = [] + failedTasks = defaultdict(lambda: defaultdict(str)) + for db_name, caseMetricsMap in dbCaseMetricsMap.items(): + for case, metricInfo in caseMetricsMap.items(): + metrics = metricInfo["metrics"] + db = metricInfo["db"] + db_label = metricInfo["db_label"] + label = metricInfo["label"] + if label == ResultLabel.NORMAL: + mergedTasks.append( + { + "db_name": db_name, + "db": db, + "db_label": db_label, + "case": case, + "metricsSet": set(metrics.keys()), + **metrics, + } + ) + else: + failedTasks[case][db_name] = label + + return mergedTasks, failedTasks + + +def mergeMetrics(metrics_1: dict, metrics_2: dict) -> dict: + metrics = {**metrics_1} + for key, value in metrics_2.items(): + metrics[key] = ( + getBetterMetric(key, value, metrics[key]) if key in metrics else value + ) + + return metrics + + +def getBetterMetric(metric, value_1, value_2): + if value_1 < 1e-7: + return value_2 + if value_2 < 1e-7: + return value_1 + return ( + min(value_1, value_2) + if isLowerIsBetterMetric(metric) + else max(value_1, value_2) + ) + +def getBetterLabel(label_1: ResultLabel, label_2: ResultLabel): + return label_2 if label_1 != ResultLabel.NORMAL else label_1 diff --git a/vector_db_bench/frontend/components/check_results/filters.py b/vector_db_bench/frontend/components/check_results/filters.py new file mode 100644 index 000000000..a675c60a7 --- /dev/null +++ b/vector_db_bench/frontend/components/check_results/filters.py @@ -0,0 +1,98 @@ +from vector_db_bench.frontend.components.check_results.data import getChartData +from vector_db_bench.frontend.utils import displayCaseText +from vector_db_bench.frontend.const import * + + +def getshownData(results, st): + # hide the nav + st.markdown( + "", + unsafe_allow_html=True, + ) + + st.header("Filters") + + shownResults = getshownResults(results, st) + showDBNames, showCases = getShowDbsAndCases(shownResults, st) + + shownData, failedTasks = getChartData(shownResults, showDBNames, showCases) + + return shownData, failedTasks, showCases + + +def getshownResults(results, st): + resultSelectOptions = [ + result.task_label + if result.task_label != result.run_id + else f"res-{result.run_id[:4]}" + for result in results + ] + if len(resultSelectOptions) == 0: + st.write( + "There are no results to display. Please wait for the task to complete or run a new task." + ) + return [] + + selectedResultSelectedOptions = st.multiselect( + "Select the task results you need to analyze.", + resultSelectOptions, + # label_visibility="hidden", + default=resultSelectOptions, + ) + selectedResult = [] + for option in selectedResultSelectedOptions: + result = results[resultSelectOptions.index(option)].results + selectedResult += result + + return selectedResult + + +def getShowDbsAndCases(result, st): + # expanderStyles + st.markdown("", unsafe_allow_html=True,) + st.markdown( + "", + unsafe_allow_html=True, + ) + st.markdown( + "", + unsafe_allow_html=True, + ) + + allDbNames = list(set({res.task_config.db_name for res in result})) + allDbNames.sort() + allCasesSet = set({res.task_config.case_config.case_id for res in result}) + allCases = [case["name"].value for case in CASE_LIST if case["name"] in allCasesSet] + + # dbFilterContainer = st.container() + # dbFilterContainer.subheader("DB Filter") + dbFilterContainer = st.expander("DB Filter", True) + showDBNames = filterView(allDbNames, dbFilterContainer, col=1) + + # caseFilterContainer = st.container() + # caseFilterContainer.subheader("Case Filter") + caseFilterContainer = st.expander("Case Filter", True) + showCases = filterView( + allCases, + caseFilterContainer, + col=1, + optionLables=[displayCaseText(case) for case in allCases], + ) + + return showDBNames, showCases + + +def filterView(options, st, col, optionLables=None): + columns = st.columns( + col, + gap="small", + ) + isActive = {option: True for option in options} + if optionLables is None: + optionLables = options + for i, option in enumerate(options): + isActive[option] = columns[i % col].checkbox( + optionLables[i], value=isActive[option] + ) + + return [option for option in options if isActive[option]] diff --git a/vector_db_bench/frontend/components/check_results/headerIcon.py b/vector_db_bench/frontend/components/check_results/headerIcon.py new file mode 100644 index 000000000..fa5d3fd0f --- /dev/null +++ b/vector_db_bench/frontend/components/check_results/headerIcon.py @@ -0,0 +1,18 @@ +def drawHeaderIcon(st): + st.markdown(""" +
+ +") + if navClick: + switch_page("run test") + + +def NavToQPSWithPrice(st): + navClick = st.button("QPS with Price   >") + if navClick: + switch_page("qps with price") + + +def NavToResults(st): + navClick = st.button("<   Back to Results") + if navClick: + switch_page("vdb benchmark") diff --git a/vector_db_bench/frontend/components/run_test/autoRefresh.py b/vector_db_bench/frontend/components/run_test/autoRefresh.py new file mode 100644 index 000000000..116563816 --- /dev/null +++ b/vector_db_bench/frontend/components/run_test/autoRefresh.py @@ -0,0 +1,10 @@ +from streamlit_autorefresh import st_autorefresh +from vector_db_bench.frontend.const import * + + +def autoRefresh(): + auto_refresh_count = st_autorefresh( + interval=MAX_AUTO_REFRESH_INTERVAL, + limit=MAX_AUTO_REFRESH_COUNT, + key="streamlit-auto-refresh", + ) diff --git a/vector_db_bench/frontend/components/run_test/caseSelector.py b/vector_db_bench/frontend/components/run_test/caseSelector.py new file mode 100644 index 000000000..7e8d70bfc --- /dev/null +++ b/vector_db_bench/frontend/components/run_test/caseSelector.py @@ -0,0 +1,88 @@ +from vector_db_bench.frontend.const import * +from vector_db_bench.frontend.utils import displayCaseText + + +def caseSelector(st, activedDbList): + st.markdown( + "
", + unsafe_allow_html=True, + ) + st.subheader("STEP 2: Choose the case(s)") + st.markdown( + "
Choose at least one case you want to run the test for.
", + unsafe_allow_html=True, + ) + + caseIsActived = {case["name"]: False for case in CASE_LIST} + allCaseConfigs = {db: {case["name"]: {} for case in CASE_LIST} for db in DB_LIST} + for case in CASE_LIST: + caseItemContainer = st.container() + caseIsActived[case["name"]] = caseItem( + caseItemContainer, allCaseConfigs, case, activedDbList + ) + if case.get("divider"): + caseItemContainer.markdown( + "
", + unsafe_allow_html=True, + ) + activedCaseList = [ + case["name"] for case in CASE_LIST if caseIsActived[case["name"]] + ] + return activedCaseList, allCaseConfigs + +def caseItem(st, allCaseConfigs, case, activedDbList): + selected = st.checkbox(displayCaseText(case["name"].value)) + st.markdown( + f"
{case['intro']}
", + unsafe_allow_html=True, + ) + + if selected: + caseConfigSettingContainer = st.container() + caseConfigSetting( + caseConfigSettingContainer, allCaseConfigs, case["name"], activedDbList + ) + + return selected + + +def caseConfigSetting(st, allCaseConfigs, case, activedDbList): + for db in activedDbList: + columns = st.columns(1 + CASE_CONFIG_SETTING_COLUMNS) + # column 0 - title + dbColumn = columns[0] + dbColumn.markdown( + f"
{db.name}
", + unsafe_allow_html=True, + ) + caseConfig = allCaseConfigs[db][case] + k = 0 + for config in CASE_CONFIG_MAP.get(db, {}).get(case, []): + if config.isDisplayed(caseConfig): + column = columns[1 + k % CASE_CONFIG_SETTING_COLUMNS] + key = "%s-%s-%s" % (db, case, config.label.value) + if config.inputType == InputType.Text: + caseConfig[config.label] = column.text_input( + config.label.value, + key=key, + value=config.inputConfig["value"], + ) + elif config.inputType == InputType.Option: + caseConfig[config.label] = column.selectbox( + config.label.value, + config.inputConfig["options"], + key=key, + ) + elif config.inputType == InputType.Number: + caseConfig[config.label] = column.number_input( + config.label.value, + format="%d", + step=1, + min_value=config.inputConfig["min"], + max_value=config.inputConfig["max"], + key=key, + value=config.inputConfig["value"], + ) + k += 1 + if k == 0: + columns[1].write("Auto") diff --git a/vector_db_bench/frontend/components/run_test/dbConfigSetting.py b/vector_db_bench/frontend/components/run_test/dbConfigSetting.py new file mode 100644 index 000000000..074cf8c42 --- /dev/null +++ b/vector_db_bench/frontend/components/run_test/dbConfigSetting.py @@ -0,0 +1,47 @@ +from vector_db_bench.frontend.const import * +from vector_db_bench.frontend.utils import inputIsPassword + + +def dbConfigSettings(st, activedDbList): + st.markdown( + "", + unsafe_allow_html=True, + ) + expander = st.expander("Configurations for the selected databases", True) + + dbConfigs = {} + for activeDb in activedDbList: + dbConfigSettingItemContainer = expander.container() + dbConfig = dbConfigSettingItem(dbConfigSettingItemContainer, activeDb) + dbConfigs[activeDb] = dbConfig + + return dbConfigs + +def dbConfigSettingItem(st, activeDb): + st.markdown( + f"
{activeDb.value}
", + unsafe_allow_html=True, + ) + columns = st.columns(DB_CONFIG_SETTING_COLUMNS) + + activeDbCls = activeDb.init_cls + dbConfigClass = activeDbCls.config_cls() + properties = dbConfigClass.schema().get("properties") + propertiesItems = list(properties.items()) + moveDBLabelToLast(propertiesItems) + dbConfig = {} + for j, property in enumerate(propertiesItems): + column = columns[j % DB_CONFIG_SETTING_COLUMNS] + key, value = property + dbConfig[key] = column.text_input( + key, + key="%s-%s" % (activeDb, key), + value=value.get("default", ""), + type="password" if inputIsPassword(key) else "default", + ) + return dbConfigClass(**dbConfig) + + +def moveDBLabelToLast(propertiesItems): + propertiesItems.sort(key=lambda x: 1 if x[0] == 'db_label' else 0) + \ No newline at end of file diff --git a/vector_db_bench/frontend/components/run_test/dbSelector.py b/vector_db_bench/frontend/components/run_test/dbSelector.py new file mode 100644 index 000000000..f2b228600 --- /dev/null +++ b/vector_db_bench/frontend/components/run_test/dbSelector.py @@ -0,0 +1,36 @@ + +from vector_db_bench.frontend.const import * + + +def dbSelector(st): + st.markdown( + "
", + unsafe_allow_html=True, + ) + st.subheader("STEP 1: Select the database(s)") + st.markdown( + "
Choose at least one case you want to run the test for.
", + unsafe_allow_html=True, + ) + + dbContainerColumns = st.columns(DB_SELECTOR_COLUMNS, gap="small") + dbIsActived = {db: False for db in DB_LIST} + + # style - image; column gap; checkbox font; + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + for i, db in enumerate(DB_LIST): + column = dbContainerColumns[i % DB_SELECTOR_COLUMNS] + dbIsActived[db] = column.checkbox(db.name) + column.image(DB_TO_ICON.get(db, "")) + activedDbList = [db for db in DB_LIST if dbIsActived[db]] + + return activedDbList diff --git a/vector_db_bench/frontend/components/run_test/generateTasks.py b/vector_db_bench/frontend/components/run_test/generateTasks.py new file mode 100644 index 000000000..95c652416 --- /dev/null +++ b/vector_db_bench/frontend/components/run_test/generateTasks.py @@ -0,0 +1,21 @@ +from vector_db_bench.models import CaseConfig, CaseConfigParamType, TaskConfig + + +def generate_tasks(activedDbList, dbConfigs, activedCaseList, allCaseConfigs): + tasks = [ + TaskConfig( + db=db.value, + db_config=dbConfigs[db], + case_config=CaseConfig( + case_id=case.value, + custom_case={}, + ), + db_case_config=db.init_cls.case_config_cls( + allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None) + )(**{key.value: value for key, value in allCaseConfigs[db][case].items()}), + ) + for case in activedCaseList + for db in activedDbList + ] + + return tasks diff --git a/vector_db_bench/frontend/components/run_test/hideSidebar.py b/vector_db_bench/frontend/components/run_test/hideSidebar.py new file mode 100644 index 000000000..6cbcbfbae --- /dev/null +++ b/vector_db_bench/frontend/components/run_test/hideSidebar.py @@ -0,0 +1,10 @@ +def hideSidebar(st): + st.markdown( + "", + unsafe_allow_html=True, + ) + + st.markdown( + "", + unsafe_allow_html=True, + ) diff --git a/vector_db_bench/frontend/components/run_test/submitTask.py b/vector_db_bench/frontend/components/run_test/submitTask.py new file mode 100644 index 000000000..66ce0417b --- /dev/null +++ b/vector_db_bench/frontend/components/run_test/submitTask.py @@ -0,0 +1,69 @@ +from datetime import datetime +from vector_db_bench.frontend.const import * +from vector_db_bench.interface import benchMarkRunner + + +def submitTask(st, tasks): + st.markdown( + "
", + unsafe_allow_html=True, + ) + st.subheader("STEP 3: Task Label") + st.markdown( + "
This description is used to mark the result.
", + unsafe_allow_html=True, + ) + + taskLabel = taskLabelInput(st) + + st.markdown( + "
", + unsafe_allow_html=True, + ) + + controlPanelContainer = st.container() + controlPanel(controlPanelContainer, tasks, taskLabel) + + +def taskLabelInput(st): + defaultTaskLabel = datetime.now().strftime("%Y%m%d-%H%M%S") + columns = st.columns(TASK_LABEL_INPUT_COLUMNS) + taskLabel = columns[0].text_input("task_label", defaultTaskLabel, label_visibility="collapsed") + return taskLabel + + +def controlPanel(st, tasks, taskLabel): + isRunning = benchMarkRunner.has_running() + runHandler = lambda: benchMarkRunner.run(tasks, taskLabel) + stopHandler = lambda: benchMarkRunner.stop_running() + + if isRunning: + currentTaskId = benchMarkRunner.get_current_task_id() + tasksCount = benchMarkRunner.get_tasks_count() + text = f":running: Running Task {currentTaskId} / {tasksCount}" + st.progress(currentTaskId / tasksCount, text=text) + + columns = st.columns(6) + columns[0].button( + "Run Your Test", + disabled=True, + on_click=runHandler, + type="primary", + ) + columns[1].button( + "Stop", + on_click=stopHandler, + type="primary", + ) + + else: + errorText = benchMarkRunner.latest_error or "" + if len(errorText) > 0: + st.error(errorText) + disabled = True if len(tasks) == 0 else False + st.button( + "Run Your Test", + disabled=disabled, + on_click=runHandler, + type="primary", + ) diff --git a/vector_db_bench/frontend/const.py b/vector_db_bench/frontend/const.py new file mode 100644 index 000000000..012862d8a --- /dev/null +++ b/vector_db_bench/frontend/const.py @@ -0,0 +1,392 @@ +from enum import IntEnum +from vector_db_bench.models import DB, CaseType, IndexType, CaseConfigParamType +from pydantic import BaseModel +import typing + +# style const +DB_SELECTOR_COLUMNS = 6 +DB_CONFIG_SETTING_COLUMNS = 3 +CASE_CONFIG_SETTING_COLUMNS = 4 +CHECKBOX_INDENT = 30 +TASK_LABEL_INPUT_COLUMNS = 2 +CHECKBOX_MAX_COLUMNS = 4 +DB_CONFIG_INPUT_MAX_COLUMNS = 2 +CASE_CONFIG_INPUT_MAX_COLUMNS = 3 +DB_CONFIG_INPUT_WIDTH_RADIO = 2 +CASE_CONFIG_INPUT_WIDTH_RADIO = 0.98 +CASE_INTRO_RATIO = 3 +MAX_STREAMLIT_INT = (1 << 53) - 1 + +COLORS = [ + "#3B69FE", + "#66C8FF", + "#35CE73", + "#FDC513", + "#FE708D", + "#8773FB", +] +LEGEND_RECT_WIDTH = 24 +LEGEND_RECT_HEIGHT = 16 +LEGEND_TEXT_FONT_SIZE = 14 + +PATTERN_SHAPES = ["", "+", "\\", "x", ".", "|", "/", "-"] + + +def getPatternShape(i): + return PATTERN_SHAPES[i % len(PATTERN_SHAPES)] + + +MAX_AUTO_REFRESH_COUNT = 999999 +MAX_AUTO_REFRESH_INTERVAL = 5000 # 2s + + +DB_LIST = [d for d in DB] + +DB_TO_ICON = { + DB.Milvus: "https://assets.zilliz.com/milvus_c30b0d1994.png", + DB.ZillizCloud: "https://assets.zilliz.com/zilliz_5f4cc9b050.png", + DB.ElasticCloud: "https://assets.zilliz.com/elasticsearch_beffeadc29.png", + DB.Pinecone: "https://assets.zilliz.com/pinecone_94d8154979.png", + DB.QdrantCloud: "https://assets.zilliz.com/qdrant_b691674fcd.png", + DB.WeaviateCloud: "https://assets.zilliz.com/weaviate_4f6f171ebe.png", +} + +COLOR_MAP = {db.value: COLORS[i] for i, db in enumerate(DB_LIST)} + +CASE_LIST = [ + { + "name": CaseType.LoadSDim, + "intro": """This case tests the vector database's loading capacity by repeatedly inserting small-dimension vectors (SIFT 100K vectors, 128 dimensions) until it is fully loaded. +Number of inserted vectors will be reported.""", + }, + { + "name": CaseType.LoadLDim, + "divider": True, + "intro": """This case tests the vector database's loading capacity by repeatedly inserting large-dimension vectors (GIST 100K vectors, 960 dimensions) until it is fully loaded. +Number of inserted vectors will be reported. +""", + }, + { + "name": CaseType.PerformanceSZero, + "intro": """This case tests the search performance of a vector database with a small dataset (Cohere 100K vectors, 768 dimensions) at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""", + }, + { + "name": CaseType.PerformanceMZero, + "intro": """This case tests the search performance of a vector database with a medium dataset (Cohere 1M vectors, 768 dimensions) at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""", + }, + { + "name": CaseType.PerformanceLZero, + "intro": """This case tests the search performance of a vector database with a large dataset (Cohere 10M vectors, 768 dimensions) at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""", + }, + { + "name": CaseType.Performance100M, + "divider": True, + "intro": """This case tests the search performance of a vector database with a large 100M dataset (LAION 100M vectors, 768 dimensions), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""", + }, + { + "name": CaseType.PerformanceSLow, + "intro": """This case tests the search performance of a vector database with a small dataset (Cohere 100K vectors, 768 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""", + }, + { + "name": CaseType.PerformanceMLow, + "intro": """This case tests the search performance of a vector database with a medium dataset (Cohere 1M vectors, 768 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""", + }, + { + "name": CaseType.PerformanceLLow, + "intro": """This case tests the search performance of a vector database with a large dataset (Cohere 10M vectors, 768 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""", + }, + { + "name": CaseType.PerformanceSHigh, + "intro": """This case tests the search performance of a vector database with a small dataset (Cohere 100K vectors, 768 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""", + }, + { + "name": CaseType.PerformanceMHigh, + "intro": """This case tests the search performance of a vector database with a medium dataset (Cohere 1M vectors, 768 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""", + }, + { + "name": CaseType.PerformanceLHigh, + "intro": """This case tests the search performance of a vector database with a large dataset (Cohere 10M vectors, 768 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""", + }, +] + + +class InputType(IntEnum): + Text = 20001 + Number = 20002 + Option = 20003 + + +class CaseConfigInput(BaseModel): + label: CaseConfigParamType + inputType: InputType = InputType.Text + inputConfig: dict = {} + # todo type should be a function + isDisplayed: typing.Any = lambda x: True + + +CaseConfigParamInput_IndexType = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.HNSW.value, + IndexType.IVFFlat.value, + IndexType.DISKANN.value, + IndexType.Flat.value, + IndexType.AUTOINDEX.value, + ], + }, +) + +CaseConfigParamInput_M = CaseConfigInput( + label=CaseConfigParamType.M, + inputType=InputType.Number, + inputConfig={ + "min": 4, + "max": 64, + "value": 30, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.HNSW.value, +) + +CaseConfigParamInput_EFConstruction_Milvus = CaseConfigInput( + label=CaseConfigParamType.EFConstruction, + inputType=InputType.Number, + inputConfig={ + "min": 8, + "max": 512, + "value": 360, + }, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] + == IndexType.HNSW.value, +) + +CaseConfigParamInput_EFConstruction_Weaviate = CaseConfigInput( + label=CaseConfigParamType.EFConstruction, + inputType=InputType.Number, + inputConfig={ + "min": 8, + "max": 512, + "value": 128, + }, +) + +CaseConfigParamInput_EFConstruction_ES = CaseConfigInput( + label=CaseConfigParamType.EFConstruction, + inputType=InputType.Number, + inputConfig={ + "min": 8, + "max": 512, + "value": 360, + }, +) + +CaseConfigParamInput_M_ES = CaseConfigInput( + label=CaseConfigParamType.M, + inputType=InputType.Number, + inputConfig={ + "min": 4, + "max": 64, + "value": 30, + }, +) + +CaseConfigParamInput_NumCandidates_ES = CaseConfigInput( + label=CaseConfigParamType.numCandidates, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 10000, + "value": 100, + }, +) + +CaseConfigParamInput_EF_Milvus = CaseConfigInput( + label=CaseConfigParamType.EF, + inputType=InputType.Number, + inputConfig={ + "min": 100, + "max": MAX_STREAMLIT_INT, + "value": 100, + }, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] + == IndexType.HNSW.value, +) + +CaseConfigParamInput_EF_Weaviate = CaseConfigInput( + label=CaseConfigParamType.EF, + inputType=InputType.Number, + inputConfig={ + "min": -1, + "max": MAX_STREAMLIT_INT, + "value": -1, + }, +) + +CaseConfigParamInput_MaxConnections = CaseConfigInput( + label=CaseConfigParamType.MaxConnections, + inputType=InputType.Number, + inputConfig={"min": 1, "max": MAX_STREAMLIT_INT, "value": 64}, +) + +CaseConfigParamInput_SearchList = CaseConfigInput( + label=CaseConfigParamType.SearchList, + inputType=InputType.Number, + inputConfig={ + "min": 100, + "max": MAX_STREAMLIT_INT, + "value": 100, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.DISKANN.value, +) + +CaseConfigParamInput_Nlist = CaseConfigInput( + label=CaseConfigParamType.Nlist, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65536, + "value": 1000, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.IVFFlat.value, +) + +CaseConfigParamInput_Nprobe = CaseConfigInput( + label=CaseConfigParamType.Nprobe, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65536, + "value": 10, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.IVFFlat.value, +) + + +MilvusLoadConfig = [ + CaseConfigParamInput_IndexType, + CaseConfigParamInput_M, + CaseConfigParamInput_EFConstruction_Milvus, + CaseConfigParamInput_Nlist, +] + + +MilvusPerformanceConfig = [ + CaseConfigParamInput_IndexType, + CaseConfigParamInput_M, + CaseConfigParamInput_EFConstruction_Milvus, + CaseConfigParamInput_EF_Milvus, + CaseConfigParamInput_SearchList, + CaseConfigParamInput_Nlist, + CaseConfigParamInput_Nprobe, +] + +WeaviateLoadConfig = [ + CaseConfigParamInput_MaxConnections, + CaseConfigParamInput_EFConstruction_Weaviate, +] + +WeaviatePerformanceConfig = [ + CaseConfigParamInput_MaxConnections, + CaseConfigParamInput_EFConstruction_Weaviate, + CaseConfigParamInput_EF_Weaviate, +] + +ESLoadingConfig = [CaseConfigParamInput_EFConstruction_ES, CaseConfigParamInput_M_ES] + +ESPerformanceConfig = [ + CaseConfigParamInput_EFConstruction_ES, + CaseConfigParamInput_M_ES, + CaseConfigParamInput_NumCandidates_ES, +] + +CASE_CONFIG_MAP = { + DB.Milvus: { + CaseType.LoadLDim: MilvusLoadConfig, + CaseType.LoadSDim: MilvusLoadConfig, + CaseType.PerformanceLZero: MilvusPerformanceConfig, + CaseType.PerformanceMZero: MilvusPerformanceConfig, + CaseType.PerformanceSZero: MilvusPerformanceConfig, + CaseType.PerformanceLLow: MilvusPerformanceConfig, + CaseType.PerformanceMLow: MilvusPerformanceConfig, + CaseType.PerformanceSLow: MilvusPerformanceConfig, + CaseType.PerformanceLHigh: MilvusPerformanceConfig, + CaseType.PerformanceMHigh: MilvusPerformanceConfig, + CaseType.PerformanceSHigh: MilvusPerformanceConfig, + CaseType.Performance100M: MilvusPerformanceConfig, + }, + DB.WeaviateCloud: { + CaseType.LoadLDim: WeaviateLoadConfig, + CaseType.LoadSDim: WeaviateLoadConfig, + CaseType.PerformanceLZero: WeaviatePerformanceConfig, + CaseType.PerformanceMZero: WeaviatePerformanceConfig, + CaseType.PerformanceSZero: WeaviatePerformanceConfig, + CaseType.PerformanceLLow: WeaviatePerformanceConfig, + CaseType.PerformanceMLow: WeaviatePerformanceConfig, + CaseType.PerformanceSLow: WeaviatePerformanceConfig, + CaseType.PerformanceLHigh: WeaviatePerformanceConfig, + CaseType.PerformanceMHigh: WeaviatePerformanceConfig, + CaseType.PerformanceSHigh: WeaviatePerformanceConfig, + CaseType.Performance100M: WeaviatePerformanceConfig, + }, + DB.ElasticCloud: { + CaseType.LoadLDim: ESLoadingConfig, + CaseType.LoadSDim: ESLoadingConfig, + CaseType.PerformanceLZero: ESPerformanceConfig, + CaseType.PerformanceMZero: ESPerformanceConfig, + CaseType.PerformanceSZero: ESPerformanceConfig, + CaseType.PerformanceLLow: ESPerformanceConfig, + CaseType.PerformanceMLow: ESPerformanceConfig, + CaseType.PerformanceSLow: ESPerformanceConfig, + CaseType.PerformanceLHigh: ESPerformanceConfig, + CaseType.PerformanceMHigh: ESPerformanceConfig, + CaseType.PerformanceSHigh: ESPerformanceConfig, + CaseType.Performance100M: ESPerformanceConfig, + }, +} + +DB_DBLABEL_TO_PRICE = { + DB.Milvus.value: {}, + DB.ZillizCloud.value: { + "1cu-perf": 0.159, + "8cu-perf": 1.272, + "1cu-cap": 0.159, + "2cu-cap": 0.318, + }, + DB.WeaviateCloud.value: { + # "sandox": 0, # emmmm + "standard": 10.10, + "bus_crit": 32.60, + }, + DB.ElasticCloud.value: { + "free-5c8g": 0.260, + "upTo2.5c8g": 0.4793, + }, + DB.QdrantCloud.value: { + "0.5c4g-1node": 0.052, + "2c8g-1node": 0.166, + "4c16g-5node": 1.426, + }, + DB.Pinecone.value: { + "s1.x1": 0.0973, + "s1.x2": 0.194, + "p1.x1": 0.0973, + "p1.x5-2pod": 0.973, + "p2.x1": 0.146, + "p2.x5-2pod": 1.46, + }, +} diff --git a/vector_db_bench/frontend/pages/qps_with_price.py b/vector_db_bench/frontend/pages/qps_with_price.py new file mode 100644 index 000000000..22502808e --- /dev/null +++ b/vector_db_bench/frontend/pages/qps_with_price.py @@ -0,0 +1,56 @@ +import streamlit as st +from vector_db_bench.frontend.const import * +from vector_db_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vector_db_bench.frontend.components.check_results.nav import NavToResults, NavToRunTest +from vector_db_bench.frontend.components.check_results.charts import drawChartQpsPerHour, drawMetricChart +from vector_db_bench.frontend.components.check_results.filters import getshownData +from vector_db_bench.frontend.utils import displayCaseText +from vector_db_bench.interface import benchMarkRunner + + +def main(): + st.set_page_config( + page_title="VectorDB Benchmark", + page_icon="https://assets.zilliz.com/favicon_f7f922fe27.png", + # layout="wide", + # initial_sidebar_state="collapsed", + ) + + # header + drawHeaderIcon(st) + + allResults = benchMarkRunner.get_results() + + st.title("Vector Database Benchmark") + st.write("description [todo]") + + # results selector + resultSelectorContainer = st.sidebar.container() + shownData, failedTasks, showCases = getshownData(allResults, resultSelectorContainer) + + resultSelectorContainer.divider() + + # nav + navContainer = st.sidebar.container() + NavToRunTest(navContainer) + NavToResults(navContainer) + + # charts + for case in showCases: + chartContainer = st.container() + data = [data for data in shownData if data["case"] == case] + dataWithMetric = [] + metric = "qps_per_dollar (qps / price)" + for d in data: + qps = d.get("qps", 0) + price = DB_DBLABEL_TO_PRICE.get(d["db"], {}).get(d["db_label"], 0) + if qps > 0 and price > 0: + d[metric] = d["qps"] / price + dataWithMetric.append(d) + if len(dataWithMetric) > 0: + chartContainer.subheader(displayCaseText(case)) + drawMetricChart(data, metric, chartContainer) + + +if __name__ == "__main__": + main() diff --git a/vector_db_bench/frontend/pages/run_test.py b/vector_db_bench/frontend/pages/run_test.py new file mode 100644 index 000000000..29f47fb40 --- /dev/null +++ b/vector_db_bench/frontend/pages/run_test.py @@ -0,0 +1,59 @@ +import streamlit as st +from vector_db_bench.frontend.components.run_test.autoRefresh import autoRefresh +from vector_db_bench.frontend.components.run_test.caseSelector import caseSelector +from vector_db_bench.frontend.components.run_test.dbConfigSetting import dbConfigSettings +from vector_db_bench.frontend.components.run_test.dbSelector import dbSelector +from vector_db_bench.frontend.components.run_test.generateTasks import generate_tasks +from vector_db_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vector_db_bench.frontend.components.run_test.hideSidebar import hideSidebar +from vector_db_bench.frontend.components.check_results.nav import NavToResults +from vector_db_bench.frontend.components.run_test.submitTask import submitTask + + +def main(): + st.set_page_config( + page_title="VectorDB Benchmark", + page_icon="https://assets.zilliz.com/favicon_f7f922fe27.png", + # layout="wide", + initial_sidebar_state="collapsed", + ) + # header + drawHeaderIcon(st) + + # hide sidebar + hideSidebar(st) + + # nav to results + NavToResults(st) + + # header + st.title("Run Your Test") + st.write("description [todo]") + + # select db + dbSelectorContainer = st.container() + activedDbList = dbSelector(dbSelectorContainer) + + # db config setting + dbConfigs = {} + if len(activedDbList) > 0: + dbConfigContainer = st.container() + dbConfigs = dbConfigSettings(dbConfigContainer, activedDbList) + + # select case and set db_case_config + caseSelectorContainer = st.container() + activedCaseList, allCaseConfigs = caseSelector(caseSelectorContainer, activedDbList) + + # generate tasks + tasks = generate_tasks(activedDbList, dbConfigs, activedCaseList, allCaseConfigs) + + # submit + submitContainer = st.container() + submitTask(submitContainer, tasks) + + # autofresh + autoRefresh() + + +if __name__ == "__main__": + main() diff --git a/vector_db_bench/frontend/pages/views/chart.py b/vector_db_bench/frontend/pages/views/chart.py new file mode 100644 index 000000000..e11e42b57 --- /dev/null +++ b/vector_db_bench/frontend/pages/views/chart.py @@ -0,0 +1,203 @@ +from vector_db_bench.metric import metricOrder, isLowerIsBetterMetric, metricUnitMap +from vector_db_bench.frontend.const import * +import plotly.express as px + + +def drawChart(data, st): + metricsSet = set() + for d in data: + metricsSet = metricsSet.union(d["metricsSet"]) + showMetrics = [metric for metric in metricOrder if metric in metricsSet] + + for i, metric in enumerate(showMetrics): + container = st.container() + drawMetricChart(data, metric, container) + + +def getLabelToShapeMap(data): + labelIndexMap = {} + + dbSet = {d["db"] for d in data} + for db in dbSet: + labelSet = {d["db_label"] for d in data if d["db"] == db} + labelList = list(labelSet) + + usedShapes = set() + i = 0 + for label in labelList: + if label not in labelIndexMap: + loopCount = 0 + while i % len(PATTERN_SHAPES) in usedShapes: + i += 1 + loopCount += 1 + if loopCount > len(PATTERN_SHAPES): + break + labelIndexMap[label] = i + i += 1 + else: + usedShapes.add(labelIndexMap[label] % len(PATTERN_SHAPES)) + + labelToShapeMap = { + label: getPatternShape(index) for label, index in labelIndexMap.items() + } + return labelToShapeMap + + +def drawMetricChart(data, metric, st): + dataWithMetric = [d for d in data if d.get(metric, 0) > 1e-7] + # dataWithMetric = data + if len(dataWithMetric) == 0: + return + + title = st.container() + title.markdown( + f"**{metric}** ({'less' if isLowerIsBetterMetric(metric) else 'more'} is better)" + ) + chart = st.container() + + height = len(dataWithMetric) * 24 + xmin = 0 + xmax = max([d.get(metric, 0) for d in dataWithMetric]) + xpadding = (xmax - xmin) / 16 + xpadding_multiplier = 1.6 + xrange = [xmin, xmax + xpadding * xpadding_multiplier] + unit = metricUnitMap.get(metric, "") + labelToShapeMap = getLabelToShapeMap(dataWithMetric) + categoryorder = ( + "total descending" if isLowerIsBetterMetric(metric) else "total ascending" + ) + fig = px.bar( + dataWithMetric, + x=metric, + y="db_name", + color="db", + height=height, + # pattern_shape="db_label", + # pattern_shape_sequence=SHAPES, + pattern_shape_map=labelToShapeMap, + orientation="h", + hover_data={ + "db": False, + "db_label": False, + "db_name": True, + }, + color_discrete_map=COLOR_MAP, + text_auto=True, + ) + fig.update_xaxes(showticklabels=False, visible=False, range=xrange) + fig.update_yaxes( + # showticklabels=False, + # visible=False, + title=dict( + font=dict( + size=1, + ), + # text="", + ) + ) + fig.update_traces( + textposition="outside", + textfont=dict( + # color="#fff", + size=14, + ), + marker=dict( + pattern=dict(fillmode="overlay", fgcolor="#fff", fgopacity=1, size=7) + ), + texttemplate="%{x:,.4~r}" + unit, + ) + fig.update_layout( + margin=dict(l=0, r=0, t=0, b=0, pad=8), + bargap=0.25, + showlegend=False, + legend=dict( + orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title="" + ), + # legend=dict(orientation="v", title=""), + yaxis={"categoryorder": categoryorder}, + ) + + chart.plotly_chart(fig, use_container_width=True) + + +def drawChartQpsPerHour(data, st): + dataWithMetric = [] + metric = "qps_per_dollar (qps / price)" + for d in data: + qps = d.get("qps", 0) + price = DB_DBLABEL_TO_PRICE.get(d["db"], {}).get(d["db_label"], 0) + if qps > 0 and price > 0: + d[metric] = d["qps"] / price + dataWithMetric.append(d) + if len(dataWithMetric) == 0: + return + + title = st.container() + title.markdown( + f"**{metric}** ({'less' if isLowerIsBetterMetric(metric) else 'more'} is better)" + ) + chart = st.container() + + height = len(dataWithMetric) * 24 + xmin = 0 + xmax = max([d.get(metric, 0) for d in dataWithMetric]) + xpadding = (xmax - xmin) / 16 + xpadding_multiplier = 1.6 + xrange = [xmin, xmax + xpadding * xpadding_multiplier] + unit = metricUnitMap.get(metric, "") + labelToShapeMap = getLabelToShapeMap(dataWithMetric) + categoryorder = ( + "total descending" if isLowerIsBetterMetric(metric) else "total ascending" + ) + fig = px.bar( + dataWithMetric, + x=metric, + y="db_name", + color="db", + height=height, + pattern_shape="db_label", + # pattern_shape_sequence=SHAPES, + pattern_shape_map=labelToShapeMap, + orientation="h", + hover_data={ + "db": False, + "db_label": False, + "db_name": True, + }, + color_discrete_map=COLOR_MAP, + text_auto=True, + ) + fig.update_xaxes(showticklabels=False, visible=False, range=xrange) + fig.update_yaxes( + # showticklabels=False, + # visible=False, + title=dict( + font=dict( + size=1, + ), + # text="", + ) + ) + fig.update_traces( + textposition="outside", + textfont=dict( + # color="#fff", + size=14, + ), + marker=dict( + pattern=dict(fillmode="overlay", fgcolor="#fff", fgopacity=1, size=7) + ), + texttemplate="%{x:,.4~r}" + unit, + ) + fig.update_layout( + margin=dict(l=0, r=0, t=0, b=0, pad=8), + bargap=0.25, + showlegend=False, + legend=dict( + orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title="" + ), + # legend=dict(orientation="v", title=""), + yaxis={"categoryorder": categoryorder}, + ) + + chart.plotly_chart(fig, use_container_width=True) diff --git a/vector_db_bench/frontend/pages/views/data.py b/vector_db_bench/frontend/pages/views/data.py new file mode 100644 index 000000000..efedf1e03 --- /dev/null +++ b/vector_db_bench/frontend/pages/views/data.py @@ -0,0 +1,75 @@ +from collections import defaultdict +from dataclasses import asdict +from vector_db_bench.metric import isLowerIsBetterMetric + + +def getChartData(tasks, dbNames, cases): + filterTasks = getFilterTasks(tasks, dbNames, cases) + mergedTasks = mergeTasks(filterTasks) + return mergedTasks + + +def getFilterTasks(tasks, dbNames, cases): + filterTasks = [ + task + for task in tasks + if task.task_config.db_name in dbNames + and task.task_config.case_config.case_id.value in cases + ] + return filterTasks + + +def mergeTasks(tasks): + dbCaseMetricsMap = defaultdict(lambda: defaultdict(dict)) + for task in tasks: + db_name = task.task_config.db_name + db = task.task_config.db.value + db_label = task.task_config.db_config.db_label or "" + case = task.task_config.case_config.case_id.value + dbCaseMetricsMap[db_name][case] = { + "db": db, + "db_label": db_label, + "metrics": mergeMetrics( + dbCaseMetricsMap[db_name][case].get("metrics", {}), asdict(task.metrics) + ), + } + + mergedTasks = [] + for db_name, caseMetricsMap in dbCaseMetricsMap.items(): + for case, metricInfo in caseMetricsMap.items(): + metrics = metricInfo["metrics"] + db = metricInfo["db"] + db_label = metricInfo["db_label"] + mergedTasks.append( + { + "db_name": db_name, + "db": db, + "db_label": db_label, + "case": case, + "metricsSet": set(metrics.keys()), + **metrics, + } + ) + return mergedTasks + + +def mergeMetrics(metrics_1: dict, metrics_2: dict) -> dict: + metrics = {**metrics_1} + for key, value in metrics_2.items(): + metrics[key] = ( + getBetterMetric(key, value, metrics[key]) if key in metrics else value + ) + + return metrics + + +def getBetterMetric(metric, value_1, value_2): + if value_1 < 1e-7: + return value_2 + if value_2 < 1e-7: + return value_1 + return ( + min(value_1, value_2) + if isLowerIsBetterMetric(metric) + else max(value_1, value_2) + ) diff --git a/vector_db_bench/frontend/pages/views/db_case_filter.py b/vector_db_bench/frontend/pages/views/db_case_filter.py new file mode 100644 index 000000000..20673ce79 --- /dev/null +++ b/vector_db_bench/frontend/pages/views/db_case_filter.py @@ -0,0 +1,39 @@ +from vector_db_bench.frontend.utils import displayCaseText +from vector_db_bench.frontend.const import * + + +def getShowDbsAndCases(result, st): + allDbNames = list(set({res.task_config.db_name for res in result})) + allCasesSet = set({res.task_config.case_config.case_id for res in result}) + allCases = [case["name"].value for case in CASE_LIST if case["name"] in allCasesSet] + + dbFilterContainer = st.container() + dbFilterContainer.subheader("DB Filter") + showDBNames = filterView(allDbNames, dbFilterContainer, col=3) + + caseFilterContainer = st.container() + caseFilterContainer.subheader("Case Filter") + showCases = filterView( + allCases, + caseFilterContainer, + col=1, + optionLables=[displayCaseText(case) for case in allCases], + ) + + return showDBNames, showCases + + +def filterView(options, st, col, optionLables=None): + columns = st.columns( + col, + gap="small", + ) + isActive = {option: True for option in options} + if optionLables is None: + optionLables = options + for i, option in enumerate(options): + isActive[option] = columns[i % col].checkbox( + optionLables[i], value=isActive[option] + ) + + return [option for option in options if isActive[option]] diff --git a/vector_db_bench/frontend/pages/views/result_selector.py b/vector_db_bench/frontend/pages/views/result_selector.py new file mode 100644 index 000000000..8c5272e4a --- /dev/null +++ b/vector_db_bench/frontend/pages/views/result_selector.py @@ -0,0 +1,26 @@ +def getShowResults(results, st): + st.subheader("Results") + resultSelectOptions = [ + result.task_label + if result.task_label != result.run_id + else f"res-{result.run_id[:4]}" + for i, result in enumerate(results) + ] + if len(resultSelectOptions) == 0: + st.write( + "There are no results to display. Please wait for the task to complete or run a new task." + ) + return [] + + selectedResultSelectedOptions = st.multiselect( + "Select the task results you need to analyze.", + resultSelectOptions, + # label_visibility="hidden", + default=resultSelectOptions, + ) + selectedResult = [] + for option in selectedResultSelectedOptions: + result = results[resultSelectOptions.index(option)].results + selectedResult += result + + return selectedResult diff --git a/vector_db_bench/frontend/pages_backup/check_results.py b/vector_db_bench/frontend/pages_backup/check_results.py new file mode 100644 index 000000000..2bc17824a --- /dev/null +++ b/vector_db_bench/frontend/pages_backup/check_results.py @@ -0,0 +1,162 @@ +import streamlit as st +from vector_db_bench.frontend.const import * +from vector_db_bench.metric import isLowerIsBetterMetric +from vector_db_bench.interface import benchMarkRunner +from dataclasses import asdict +import plotly.express as px +from plotly.subplots import make_subplots +import streamlit.components.v1 as components +from collections import defaultdict +import numpy as np + +st.set_page_config( + page_title="Falcon Mark - Open VectorDB Bench", + page_icon="🧊", + # layout="wide", + initial_sidebar_state="collapsed", +) + +st.title("Check Results") + +results = benchMarkRunner.get_results() +resultSelectOptions = [f"result-{i+1}" for i, result in enumerate(results)] + +# Result Seletor +selectorContainer = st.container() +with selectorContainer: + selectorContainer.header("Results") + selectedResultSelectedOptions = selectorContainer.multiselect( + "results", resultSelectOptions, label_visibility="hidden", default=resultSelectOptions + ) + +db_case_flag = defaultdict(lambda: defaultdict(lambda: 0)) +selectedResult = [] +for option in selectedResultSelectedOptions: + result = results[resultSelectOptions.index(option)].results + for res in result: + selectedResult.append(res) + + +allData = [ + { + "db": res.task_config.db.value, + "case": res.task_config.case_config.case_id.value, + "db_case_config": res.task_config.db_case_config.dict(), + **asdict(res.metrics), + "metrics": {key for key, value in asdict(res.metrics).items() if value > 1e-7}, + } + for res in selectedResult +] + +# allData = [ +# { +# "db": DB_LIST[i % len(DB_LIST)].value, +# "case": res.task_config.case_config.case_id.value, +# "db_case_config": res.task_config.db_case_config.dict(), +# **{ +# key: value * np.random.random() +# for key, value in asdict(res.metrics).items() +# }, +# "metrics": {key for key, value in asdict(res.metrics).items() if value > 1e-7}, +# } +# for i, res in enumerate(selectedResult) +# ] + +dbs = list({d["db"] for d in allData}) +cases = list({d["case"] for d in allData}) + + +## Charts +chartContainers = st.container() +with chartContainers: + for case in cases: + chartContainer = chartContainers.container() + chartContainer.header(case) + dbCount = defaultdict(int) + with chartContainer: + data = [d for d in allData if d["case"] == case] + for d in data: + d["alias"] = dbCount[d["db"]] + d["db_name"] = ( + f"{d['db']}-{dbCount[d['db']]}" if dbCount[d["db"]] > 0 else d["db"] + ) + dbCount[d["db"]] += 1 + metrics = set() + for d in data: + metrics = metrics.union(d["metrics"]) + metrics = list(metrics) + fig = make_subplots(rows=len(metrics), cols=1) + + legendContainer = chartContainer.container() + legendDiv = ( + lambda i: f""" +
+
+
{dbs[i]}
+
+ """ + ) + + legendsHtml = " ".join([legendDiv(i) for i, _ in enumerate(dbs)]) + components.html( + f""" +
+ {legendsHtml} +
+ """, + height=((len(dbs) - 1) // 5 + 1) * 30, + ) + + for row, metric in enumerate(metrics): + subChartContainer = chartContainers.container() + title = subChartContainer.container() + title.markdown( + f"**{metric}** ({'less' if isLowerIsBetterMetric(metric) else 'more'} is better)" + ) + chart = subChartContainer.container() + dataWithMetric = [d for d in data if d.get(metric, 0) > 1e-7] + height = len(dataWithMetric) * 28 + fig = px.bar( + dataWithMetric, + x=metric, + y="db_name", + color="db", + # title=f"{metric}", + height=height, + # barmode="group", + pattern_shape="alias", + pattern_shape_sequence=["", "+", "\\", ".", "|", "/", "-"], + orientation="h", + hover_data={ + "db": False, + "alias": False, + "db_name": True, + # 'case': True, + # metric: ":.2s", + }, + # hover_data=f"{metric}", + color_discrete_map=COLOR_MAP, + # text_auto=f".2f", + text_auto=True, + # text=metric, + # template="ggplot2", + ) + fig.update_xaxes(showticklabels=False, visible=False) + fig.update_yaxes(showticklabels=False, visible=False) + fig.update_traces( + textposition="outside", + marker=dict( + # size=[10, 50, 60], + # color="red" + # line_color="black", + # line=dict(color="MediumPurple", width=2) + pattern=dict( + fillmode="overlay", fgcolor="#fff", fgopacity=1, size=7 + ) + ), + ) + fig.update_layout( + margin=dict(l=0, r=0, t=0, b=0, pad=4), showlegend=False + ) + + chart.plotly_chart(fig, use_container_width=True) diff --git a/vector_db_bench/frontend/pages_backup/check_results_0530.py b/vector_db_bench/frontend/pages_backup/check_results_0530.py new file mode 100644 index 000000000..d554b5f6d --- /dev/null +++ b/vector_db_bench/frontend/pages_backup/check_results_0530.py @@ -0,0 +1,43 @@ +import streamlit as st +from vector_db_bench.frontend.const import * +from vector_db_bench.interface import benchMarkRunner +from vector_db_bench.frontend.pages.views.result_selector import getShowResults +from vector_db_bench.frontend.pages.views.chart import drawChart +from vector_db_bench.frontend.pages.views.data import getChartData +from vector_db_bench.frontend.pages.views.db_case_filter import getShowDbsAndCases +from vector_db_bench.frontend.utils import displayCaseText + + +def main(): + st.set_page_config( + page_title="Falcon Mark - Open VectorDB Bench", + page_icon="🧊", + # layout="wide", + initial_sidebar_state="collapsed", + ) + + st.title("Check Results") + + allResults = benchMarkRunner.get_results() + + # results selector + resultSelectorContainer = st.container() + selectedResult = getShowResults(allResults, resultSelectorContainer) + + # filters: db_name, case_name + filterContainer = st.container() + showDBNames, showCases = getShowDbsAndCases(selectedResult, filterContainer) + + # data + allData = getChartData(selectedResult, showDBNames, showCases) + + # charts + for case in showCases: + chartContainer = st.container() + data = [data for data in allData if data["case"] == case] + chartContainer.subheader(displayCaseText(case)) + drawChart(data, chartContainer) + + +if __name__ == "__main__": + main() diff --git a/vector_db_bench/frontend/pages_backup/check_results_backup.py b/vector_db_bench/frontend/pages_backup/check_results_backup.py new file mode 100644 index 000000000..ad713ce0a --- /dev/null +++ b/vector_db_bench/frontend/pages_backup/check_results_backup.py @@ -0,0 +1,122 @@ +import streamlit as st +import numpy as np +import plotly.figure_factory as ff +import plotly.express as px +from plotly.subplots import make_subplots +import plotly.graph_objects as go +import pandas as pd +from vector_db_bench.frontend.const import * +import streamlit.components.v1 as components + +st.set_page_config( + page_title="Falcon Mark - Check Results", + page_icon="🧊", + # layout="wide", + initial_sidebar_state="collapsed", +) + + +st.title("Result Check") + +results = ["result_1", "result_2", "result_3", "result_4", "result_5"] +caseCount = 3 +dbCount = 4 +metricCount = 6 + +dbs = [f"db-{db_id}" for db_id in range(dbCount)] +cases = [f"case-{case_id}" for case_id in range(caseCount)] +metrics = [f"metric-{metric_id}" for metric_id in range(metricCount)] + +resultsData = [ + { + "db": db, + "case": case, + **{metric: np.random.random() * 10**i for i, metric in enumerate(metrics)}, + } + for case in cases + for db in dbs +] + +# print("resultsData", resultsData) + + +# Result Seletor +selectorContainer = st.container() +with selectorContainer: + selectorContainer.header("Results") + selectedResult = selectorContainer.selectbox( + "results", results, label_visibility="hidden" + ) + # selectedResult = selectorContainer.multiselect('', results, max_selections=1) + + +# Result Tables + +# Result Charts +# allData = pd.DataFrame(resultsData) +allData = resultsData +chartContainers = st.container() + + +with chartContainers: + chartContainers.header("Chart") + + for caseId in range(caseCount): + case = f"case-{caseId}" + chartContainer = chartContainers.container() + chartContainer.header(case) + + with chartContainer: + fig = make_subplots(rows=len(metrics), cols=1) + + legendContainer = chartContainer.container() + legendDiv = lambda i: f""" +
+
+
{dbs[i]}
+
+ """ + + legendsHtml = " ".join([legendDiv(i) for i in range(dbCount)]) + components.html( + f""" +
+ {legendsHtml} +
+ """, + height=30, + ) + + data = [d for d in allData if d["case"] == case] + + for row, metric in enumerate(metrics): + subChartContainer = chartContainers.container() + fig = px.bar( + data, + x=metric, + y="db", + color="db", + title="", + height=dbCount * 30, + # barmode="group", + # pattern_shape="db", + orientation="h", + hover_data={ + "db": True, + # 'case': True, + metric: ":.2f", + }, + # hover_data=f"{metric}", + color_discrete_map=COLOR_MAP, + text_auto=f".2f", + ) + fig.update_xaxes(showticklabels=False, visible=False) + fig.update_yaxes(showticklabels=False, title=metric) + fig.update_layout( + margin=dict(l=20, r=20, t=0, b=0, pad=4), showlegend=False + ) + + subChartContainer.plotly_chart(fig, use_container_width=True) + + +# Share diff --git a/vector_db_bench/frontend/pages_backup/run_test.py b/vector_db_bench/frontend/pages_backup/run_test.py new file mode 100644 index 000000000..e3e1a2671 --- /dev/null +++ b/vector_db_bench/frontend/pages_backup/run_test.py @@ -0,0 +1,218 @@ +import streamlit as st +from vector_db_bench.frontend.const import * +from vector_db_bench.models import TaskConfig, CaseConfig +from vector_db_bench.interface import benchMarkRunner +from vector_db_bench.frontend.utils import inputIsPassword, displayCaseText +from streamlit_autorefresh import st_autorefresh +from datetime import datetime + + +def main(): + st.set_page_config( + page_title="Falcon Mark - Open VectorDB Bench", + page_icon="🧊", + # layout="wide", + initial_sidebar_state="collapsed", + ) + + st.title("Run Your Test") + + # DB Setting + st.divider() + dbContainter = st.container() + dbContainter.header("DB") + dbContainerColumns = dbContainter.columns(CHECKBOX_MAX_COLUMNS) + dbIsActived = {db: False for db in DB_LIST} + for i, db in enumerate(DB_LIST): + column = dbContainerColumns[i % CHECKBOX_MAX_COLUMNS] + dbIsActived[db] = column.checkbox(db.name) + + activedDbList = [db for db in DB_LIST if dbIsActived[db]] + # print("activedDbList", activedDbList) + + # DB Config Setting + dbConfigs = {} + if len(activedDbList) > 0: + st.divider() + dbConfigContainers = st.container() + dbConfigContainers.header("DB Config") + + for activeDb in activedDbList: + dbConfigContainer = dbConfigContainers.container() + dbConfigContainerColumns = dbConfigContainer.columns( + [ + 1, + *[ + DB_CONFIG_INPUT_WIDTH_RADIO + for _ in range(DB_CONFIG_INPUT_MAX_COLUMNS) + ], + ], + gap="small", + ) + activeDbCls = activeDb.init_cls + dbConfigClass = activeDbCls.config_cls() + properties = dbConfigClass.schema().get("properties") + dbConfig = {} + dbConfigContainerColumns[0].markdown("##### · %s" % activeDb.name) + for j, property in enumerate(properties.items()): + column = dbConfigContainerColumns[1 + j % DB_CONFIG_INPUT_MAX_COLUMNS] + key, value = property + dbConfig[key] = column.text_input( + key, + key="%s-%s" % (activeDb, key), + value=value.get("default", ""), + type="password" if inputIsPassword(key) else "default", + ) + dbConfigs[activeDb] = dbConfigClass(**dbConfig) + # print("dbConfigs", dbConfigs) + + # Case + st.divider() + caseContainers = st.container() + caseContainers.header("Case") + caseIsActived = {case["name"]: False for case in CASE_LIST} + for i, case in enumerate(CASE_LIST): + caseContainer = caseContainers.container() + columns = caseContainer.columns([1, CASE_INTRO_RATIO], gap="small") + caseIsActived[case["name"]] = columns[0].checkbox( + displayCaseText(case["name"].value) + ) + columns[1].markdown(case["intro"]) + activedCaseList = [ + case["name"] for case in CASE_LIST if caseIsActived[case["name"]] + ] + # print("activedCaseList", activedCaseList) + + # Case Config Setting + allCaseConfigs = { + db: { + case["name"]: { + # config.label: "" + # for config in CASE_CONFIG_MAP.get(db, {}).get(case["name"], []) + } + for case in CASE_LIST + } + for db in DB_LIST + } + if len(activedDbList) > 0 and len(activedCaseList) > 0: + st.divider() + caseConfigContainers = st.container() + caseConfigContainers.header("Case Config") + + for i, db in enumerate(activedDbList): + caseConfigDBContainer = caseConfigContainers.container() + caseConfigDBContainer.subheader(db.name) + for j, case in enumerate(activedCaseList): + caseConfigDBCaseContainer = caseConfigDBContainer.container() + columns = caseConfigDBCaseContainer.columns( + [ + 1, + *[ + CASE_CONFIG_INPUT_WIDTH_RADIO + for _ in range(CASE_CONFIG_INPUT_MAX_COLUMNS) + ], + ], + gap="small", + ) + columns[0].markdown("##### · %s" % case.value) + + k = 0 + caseConfig = allCaseConfigs[db][case] + for config in CASE_CONFIG_MAP.get(db, {}).get(case, []): + if config.isDisplayed(caseConfig): + column = columns[1 + k % CASE_CONFIG_INPUT_MAX_COLUMNS] + key = "%s-%s-%s" % (db, case, config.label.value) + if config.inputType == InputType.Text: + caseConfig[config.label] = column.text_input( + config.label.value, + key=key, + value=config.inputConfig["value"], + ) + elif config.inputType == InputType.Option: + caseConfig[config.label] = column.selectbox( + config.label.value, + config.inputConfig["options"], + key=key, + ) + elif config.inputType == InputType.Number: + caseConfig[config.label] = column.number_input( + config.label.value, + format="%d", + step=1, + min_value=config.inputConfig["min"], + max_value=config.inputConfig["max"], + key=key, + value=config.inputConfig["value"], + ) + k += 1 + if k == 0: + columns[1].write("Auto") + # print("caseConfig", caseConfig) + + # Contruct Task + tasks = [ + TaskConfig( + db=db.value, + db_config=dbConfigs[db], + case_config=CaseConfig( + case_id=case.value, + custom_case={}, + ), + db_case_config=db.init_cls.case_config_cls( + allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None) + )(**{key.value: value for key, value in allCaseConfigs[db][case].items()}), + ) + for case in activedCaseList + for db in activedDbList + ] + # print("\n=====>\nTasks:") + # for i, task in enumerate(tasks): + # print(i, task) + + # Control + st.divider() + controlContainer = st.container() + + # isRunning = False + isRunning = benchMarkRunner.has_running() + with controlContainer: + if isRunning: + progressContainer = controlContainer.container() + currentTaskId = benchMarkRunner.get_current_task_id() + tasksCount = benchMarkRunner.get_tasks_count() + text = f":running: task {currentTaskId} / {tasksCount}" + progressContainer.progress(currentTaskId / tasksCount, text=text) + else: + errorText = benchMarkRunner.latest_error or "" + if len(errorText) > 0: + controlContainer.error(errorText) + + # task label + taskLabelContainer = controlContainer.container() + taskLabelColumns = taskLabelContainer.columns(2) + defaultTaskLabel = datetime.now().strftime("%Y%m%d") + taskLabel = taskLabelColumns[0].text_input( + "Task Label (used to mark the result)", defaultTaskLabel + ) + + submitContainer = controlContainer.container() + columns = submitContainer.columns(CHECKBOX_MAX_COLUMNS) + + runHandler = lambda: benchMarkRunner.run(tasks, taskLabel) + stopHandler = lambda: benchMarkRunner.stop_running() + + columns[0].button("Run", disabled=isRunning, on_click=runHandler) + columns[1].button("Stop", disabled=not isRunning, on_click=stopHandler) + + # Use "setTimeInterval" in js and simulate page interaction behavior to trigger refresh. + # Will not block the main python server thread. + auto_refresh_count = st_autorefresh( + interval=MAX_AUTO_REFRESH_INTERVAL, + limit=MAX_AUTO_REFRESH_COUNT, + key="streamlit-auto-refresh", + ) + # st.write(f"*auto_refresh_count: {auto_refresh_count}") + + +if __name__ == "__main__": + main() diff --git a/vector_db_bench/frontend/utils.py b/vector_db_bench/frontend/utils.py new file mode 100644 index 000000000..4aedbe6f3 --- /dev/null +++ b/vector_db_bench/frontend/utils.py @@ -0,0 +1,37 @@ +from vector_db_bench.models import CaseType + +passwordKeys = ["password", "api_key"] +def inputIsPassword(key: str) -> bool: + return key.lower() in passwordKeys + + +caseTextMap = { + CaseType.LoadLDim.value: "Capacity Test (Large-dim)", + CaseType.LoadSDim.value: "Capacity Test (Small-dim)", + CaseType.PerformanceLZero.value: "Search Performance Test (Large Dataset)", + CaseType.PerformanceMZero.value: "Search Performance Test (Medium Dataset)", + CaseType.PerformanceSZero.value: "Search Performance Test (Small Dataset)", + CaseType.PerformanceLLow.value: ( + "Filtering Search Performance Test (Large Dataset, Low Filtering Rate)" + ), + CaseType.PerformanceMLow.value: ( + "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)" + ), + CaseType.PerformanceSLow.value: ( + "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)" + ), + CaseType.PerformanceLHigh.value: ( + "Filtering Search Performance Test (Large Dataset, High Filtering Rate)" + ), + CaseType.PerformanceMHigh.value: ( + "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)" + ), + CaseType.PerformanceSHigh.value: ( + "Filtering Search Performance Test (Small Dataset, High Filtering Rate)" + ), + CaseType.Performance100M.value: "Search Performance Test (XLarge Dataset)", +} + + +def displayCaseText(case): + return caseTextMap.get(case, case) diff --git a/vector_db_bench/frontend/vdb_benchmark.py b/vector_db_bench/frontend/vdb_benchmark.py new file mode 100644 index 000000000..78d226ba9 --- /dev/null +++ b/vector_db_bench/frontend/vdb_benchmark.py @@ -0,0 +1,42 @@ +import streamlit as st +from vector_db_bench.frontend.const import * +from vector_db_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vector_db_bench.frontend.components.check_results.nav import NavToQPSWithPrice, NavToRunTest +from vector_db_bench.frontend.components.check_results.charts import drawCharts +from vector_db_bench.frontend.components.check_results.filters import getshownData +from vector_db_bench.interface import benchMarkRunner + + +def main(): + st.set_page_config( + page_title="VectorDB Benchmark", + page_icon="https://assets.zilliz.com/favicon_f7f922fe27.png", + # layout="wide", + # initial_sidebar_state="collapsed", + ) + + # header + drawHeaderIcon(st) + + allResults = benchMarkRunner.get_results() + + st.title("Vector Database Benchmark") + st.write("description [todo]") + + # results selector + resultSelectorContainer = st.sidebar.container() + shownData, failedTasks, showCases = getshownData(allResults, resultSelectorContainer) + + resultSelectorContainer.divider() + + # nav + navContainer = st.sidebar.container() + NavToRunTest(navContainer) + NavToQPSWithPrice(navContainer) + + # charts + drawCharts(st, shownData, failedTasks, showCases) + + +if __name__ == "__main__": + main() diff --git a/vector_db_bench/interface.py b/vector_db_bench/interface.py new file mode 100644 index 000000000..4fc70d9d2 --- /dev/null +++ b/vector_db_bench/interface.py @@ -0,0 +1,232 @@ +import traceback +import pathlib +import signal +import logging +import uuid +import concurrent +import multiprocessing as mp +from multiprocessing.connection import Connection + +import psutil +from enum import Enum + +from . import config +from .metric import Metric +from .models import TaskConfig, TestResult, CaseResult, LoadTimeoutError, ResultLabel +from .backend.result_collector import ResultCollector +from .backend.assembler import Assembler +from .backend.task_runner import TaskRunner + +log = logging.getLogger(__name__) + +global_result_future: concurrent.futures.Future | None = None + +class SIGNAL(Enum): + SUCCESS=0 + ERROR=1 + WIP=2 + + +class BenchMarkRunner: + def __init__(self): + self.running_task: TaskRunner | None = None + self.latest_error: str | None = None + + def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: + """run all the tasks in the configs, write one result into the path""" + if self.running_task is not None: + log.warning("There're still tasks running in the background") + return False + + if len(tasks) == 0: + log.warning("Empty tasks submitted") + return False + + log.debug(f"tasks: {tasks}") + + # Generate run_id + run_id = uuid.uuid4().hex + log.info(f"generated uuid for the tasks: {run_id}") + task_label = task_label if task_label else run_id + + self.receive_conn, send_conn = mp.Pipe() + self.latest_error = "" + self.running_task = Assembler.assemble_all(run_id, task_label, tasks) + self.running_task.display() + + return self._run_async(send_conn) + + def get_results(self, result_dir: pathlib.Path | None = None) -> list[TestResult]: + """results of all runs, each TestResult represents one run.""" + target_dir = result_dir if result_dir else config.RESULTS_LOCAL_DIR + return ResultCollector.collect(target_dir) + + def _try_get_signal(self): + if self.receive_conn and self.receive_conn.poll(): + sig, received = self.receive_conn.recv() + log.debug(f"Sigal received to process: {sig}, {received}") + if sig == SIGNAL.ERROR: + self.latest_error = received + self._clear_running_task() + elif sig == SIGNAL.SUCCESS: + global global_result_future + global_result_future = None + self.running_task = None + self.receive_conn = None + elif sig == SIGNAL.WIP: + self.running_task.set_finished(received) + else: + self._clear_running_task() + + def has_running(self) -> bool: + """check if there're running benchmarks""" + if self.running_task: + self._try_get_signal() + return self.running_task is not None + + def stop_running(self): + """force stop if ther're running benchmarks""" + self._clear_running_task() + + def get_tasks_count(self) -> int: + """the count of all tasks""" + if self.running_task: + return self.running_task.num_cases() + return 0 + + + def get_current_task_id(self) -> int: + """ the index of current running task + return -1 if not running + """ + if not self.running_task: + return -1 + return self.running_task.num_finished() + + def _sync_running_task(self): + if not self.running_task: + return + + global global_result_future + try: + if global_result_future: + global_result_future.result() + except Exception as e: + log.warning(f"task running failed: {e}", exc_info=True) + finally: + global_result_future = None + self.running_task = None + + def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> None: + try: + if not running_task: + return + + c_results = [] + latest_runner, cached_load_duration = None, None + for idx, runner in enumerate(running_task.case_runners): + case_res = CaseResult( + result_id=idx, + metrics=Metric(), + task_config=runner.config, + ) + + drop_old = False if latest_runner and runner == latest_runner else config.DROP_OLD + try: + log.info(f"[{idx+1}/{running_task.num_cases()}] start case: {runner.display()}, drop_old={drop_old}") + case_res.metrics = runner.run(drop_old) + log.info(f"[{idx+1}/{running_task.num_cases()}] finish case: {runner.display()}, " + f"result={case_res.metrics}, label={case_res.label}") + + # cache the latest succeeded runner + latest_runner = runner + + # cache the latest drop_old=True load_duration of the latest succeeded runner + cached_load_duration = case_res.metrics.load_duration if drop_old else cached_load_duration + + # use the cached load duration if this case didn't drop the existing collection + if not drop_old: + case_res.metrics.load_duration = cached_load_duration if cached_load_duration else 0.0 + except LoadTimeoutError as e: + log.warning(f"[{idx+1}/{running_task.num_cases()}] case {runner.display()} failed to run, reason={e}") + case_res.label = ResultLabel.OUTOFRANGE + continue + + except Exception as e: + log.warning(f"[{idx+1}/{running_task.num_cases()}] case {runner.display()} failed to run, reason={e}") + traceback.print_exc() + case_res.label = ResultLabel.FAILED + continue + + finally: + c_results.append(case_res) + send_conn.send((SIGNAL.WIP, idx)) + + + test_result = TestResult( + run_id=running_task.run_id, + task_label=running_task.task_label, + results=c_results, + ) + test_result.display() + test_result.write_file() + + send_conn.send((SIGNAL.SUCCESS, None)) + send_conn.close() + log.info(f"Succes to finish task: label={running_task.task_label}, run_id={running_task.run_id}") + + except Exception as e: + err_msg = f"An error occurs when running task={running_task.task_label}, run_id={running_task.run_id}, err={e}" + traceback.print_exc() + log.warning(err_msg) + send_conn.send((SIGNAL.ERROR, err_msg)) + send_conn.close() + return + + def _clear_running_task(self): + global global_result_future + global_result_future = None + + if self.running_task: + log.info(f"will force stop running task: {self.running_task.run_id}") + for r in self.running_task.case_runners: + r.stop() + + self.kill_proc_tree(timeout=5) + self.running_task = None + + if self.receive_conn: + self.receive_conn.close() + self.receive_conn = None + + + def _run_async(self, conn: Connection) -> bool: + log.info(f"task submitted: id={self.running_task.run_id}, {self.running_task.task_label}, case number: {len(self.running_task.case_runners)}") + global global_result_future + executor = concurrent.futures.ProcessPoolExecutor(max_workers=1, mp_context=mp.get_context("spawn")) + global_result_future = executor.submit(self._async_task_v2, self.running_task, conn) + + return True + + def kill_proc_tree(self, sig=signal.SIGTERM, timeout=None, on_terminate=None): + """Kill a process tree (including grandchildren) with signal + "sig" and return a (gone, still_alive) tuple. + "on_terminate", if specified, is a callback function which is + called as soon as a child terminates. + """ + children = psutil.Process().children(recursive=True) + for p in children: + try: + log.warning(f"sending SIGTERM to child process: {p}") + p.send_signal(sig) + except psutil.NoSuchProcess: + pass + gone, alive = psutil.wait_procs(children, timeout=timeout, + callback=on_terminate) + + for p in alive: + log.warning(f"force killing child process: {p}") + p.kill() + + +benchMarkRunner = BenchMarkRunner() diff --git a/vector_db_bench/log_util.py b/vector_db_bench/log_util.py new file mode 100644 index 000000000..c046d272e --- /dev/null +++ b/vector_db_bench/log_util.py @@ -0,0 +1,103 @@ +import logging +from logging import config + + +def init(log_level): + LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'default': { + 'format': '%(asctime)s | %(levelname)s |%(message)s (%(filename)s:%(lineno)s)', + }, + 'colorful_console': { + 'format': '%(asctime)s | %(levelname)s: %(message)s (%(filename)s:%(lineno)s) (%(process)s)', + '()': ColorfulFormatter, + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'colorful_console', + }, + 'no_color_console': { + 'class': 'logging.StreamHandler', + 'formatter': 'default', + }, + }, + 'loggers': { + 'vector_db_bench': { + 'handlers': ['console'], + 'level': log_level, + 'propagate': False + }, + 'no_color': { + 'handlers': ['no_color_console'], + 'level': log_level, + 'propagate': False + }, + }, + 'propagate': False, + } + + config.dictConfig(LOGGING) + +class colors: + HEADER= '\033[95m' + INFO= '\033[92m' + DEBUG= '\033[94m' + WARNING= '\033[93m' + ERROR= '\033[95m' + CRITICAL= '\033[91m' + ENDC= '\033[0m' + + + +COLORS = { + 'INFO': colors.INFO, + 'INFOM': colors.INFO, + 'DEBUG': colors.DEBUG, + 'DEBUGM': colors.DEBUG, + 'WARNING': colors.WARNING, + 'WARNINGM': colors.WARNING, + 'CRITICAL': colors.CRITICAL, + 'CRITICALM': colors.CRITICAL, + 'ERROR': colors.ERROR, + 'ERRORM': colors.ERROR, + 'ENDC': colors.ENDC, +} + + +class ColorFulFormatColMixin: + def format_col(self, message_str, level_name): + if level_name in COLORS.keys(): + message_str = COLORS[level_name] + message_str + COLORS['ENDC'] + return message_str + + def formatTime(self, record, datefmt=None): + ret = super().formatTime(record, datefmt) + return ret + + +class ColorfulLogRecordProxy(logging.LogRecord): + def __init__(self, record): + self._record = record + msg_level = record.levelname + 'M' + self.msg = f"{COLORS[msg_level]}{record.msg}{COLORS['ENDC']}" + self.filename = record.filename + self.lineno = f'{record.lineno}' + self.process = f'{record.process}' + self.levelname = f"{COLORS[record.levelname]}{record.levelname}{COLORS['ENDC']}" + + def __getattr__(self, attr): + if attr not in self.__dict__: + return getattr(self._record, attr) + return getattr(self, attr) + + +class ColorfulFormatter(ColorFulFormatColMixin, logging.Formatter): + def format(self, record): + proxy = ColorfulLogRecordProxy(record) + message_str = super().format(proxy) + + return message_str diff --git a/vector_db_bench/metric.py b/vector_db_bench/metric.py new file mode 100644 index 000000000..0aa34ecbe --- /dev/null +++ b/vector_db_bench/metric.py @@ -0,0 +1,53 @@ +import logging +import numpy as np + +from dataclasses import dataclass + + +log = logging.getLogger(__name__) + + +@dataclass +class Metric: + """result metrics""" + + # for load cases + max_load_count: int = 0 + + # for performance cases + load_duration: float = 0.0 # duration to load all dataset into DB + qps: float = 0.0 + serial_latency_p99: float = 0.0 + recall: float = 0.0 + +metricUnitMap = { + 'load_duration': 's', + 'serial_latency_p99': 'ms', + 'max_load_count': 'K' +} + +lowerIsBetterMetricList = [ + "load_duration", + "serial_latency_p99", +] + +metricOrder = [ + "qps", + "recall", + "load_duration", + "serial_latency_p99", + "max_load_count", +] + + +def isLowerIsBetterMetric(metric: str) -> bool: + return metric in lowerIsBetterMetricList + + +def calc_recall(count: int, ground_truth: list[int], got: list[int]) -> float: + recalls = np.zeros(count) + for i, result in enumerate(got): + if result in ground_truth: + recalls[i] = 1 + + return np.mean(recalls) diff --git a/vector_db_bench/models.py b/vector_db_bench/models.py new file mode 100644 index 000000000..e4fa6fce0 --- /dev/null +++ b/vector_db_bench/models.py @@ -0,0 +1,210 @@ +import logging +import pathlib +from datetime import date +from typing import Self +from enum import Enum + +import ujson + +from .backend.clients import ( + DB, + DBConfig, + DBCaseConfig, + IndexType, +) +from .base import BaseModel +from . import config +from .metric import Metric + + +log = logging.getLogger(__name__) + + +class LoadTimeoutError(TimeoutError): + pass + + +class CaseType(Enum): + """ + Value will be displayed in UI + """ + + LoadLDim = "Capacity Test(Large-dim)" + LoadSDim = "Capacity Test(Small-dim)" + + PerformanceLZero = "Search Performance Test(Large Dataset)" + PerformanceMZero = "Search Performance Test(Medium Dataset)" + PerformanceSZero = "Search Performance Test(Small Dataset)" + + PerformanceLLow = ( + "Filtering Search Performance Test (Large Dataset, Low Filtering Rate)" + ) + PerformanceMLow = ( + "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)" + ) + PerformanceSLow = ( + "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)" + ) + PerformanceLHigh = ( + "Filtering Search Performance Test (Large Dataset, High Filtering Rate)" + ) + PerformanceMHigh = ( + "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)" + ) + PerformanceSHigh = ( + "Filtering Search Performance Test (Small Dataset, High Filtering Rate)" + ) + + Performance100M = ("Search Performance Test(100M Dataset)") + + +class CaseConfigParamType(Enum): + """ + Value will be the key of CaseConfig.params and displayed in UI + """ + + IndexType = "IndexType" + M = "M" + EFConstruction = "efConstruction" + EF = "ef" + SearchList = "search_list" + Nlist = "nlist" + Nprobe = "nprobe" + MaxConnections = "maxConnections" + numCandidates = "num_candidates" + + +class CustomizedCase(BaseModel): + pass + + +class CaseConfig(BaseModel): + """cases, dataset, test cases, filter rate, params""" + case_id: CaseType + custom_case: dict | None = None + + +class TaskConfig(BaseModel): + db: DB + db_config: DBConfig + db_case_config: DBCaseConfig + case_config: CaseConfig + + @property + def db_name(self): + db = self.db.value + db_label = self.db_config.db_label + return f"{db}-{db_label}" if db_label else db + +class ResultLabel(Enum): + NORMAL = ":)" + FAILED = "x" + OUTOFRANGE = "?" + + +class CaseResult(BaseModel): + metrics: Metric + task_config: TaskConfig + label: ResultLabel = ResultLabel.NORMAL + + +class TestResult(BaseModel): + """ ROOT/result_{date.today()}_{task_label}.json """ + run_id: str + task_label: str + results: list[CaseResult] + + def write_file(self): + result_dir = config.RESULTS_LOCAL_DIR + if not result_dir.exists(): + log.info(f"local result directory not exist, creating it: {result_dir}") + result_dir.mkdir(parents=True) + + file_name = f'result_{date.today().strftime("%Y%m%d")}_{self.task_label}.json' + result_file = result_dir.joinpath(file_name) + if result_file.exists(): + log.warning(f"Replacing existing result with the same file_name: {result_file}") + + log.info(f"write results to disk {result_file}") + with open(result_file, 'w') as f: + b = self.json(exclude={'db_config': {'password', 'api_key'}}) + f.write(b) + + @classmethod + def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: + if not full_path.exists(): + raise ValueError(f"No such file: {full_path}") + + with open(full_path) as f: + test_result = ujson.loads(f.read()) + if "task_label" not in test_result: + test_result['task_label'] = test_result['run_id'] + + for case_result in test_result["results"]: + task_config = case_result.get("task_config") + db = DB(task_config.get("db")) + dbcls = db.init_cls + task_config["db_config"] = dbcls.config_cls()(**task_config["db_config"]) + task_config["db_case_config"] = dbcls.case_config_cls( + index_type=task_config["db_case_config"].get("index", None), + )(**task_config["db_case_config"]) + + case_result["task_config"] = task_config + + if trans_unit: + cur_max_count = case_result['metrics']['max_load_count'] + case_result['metrics']['max_load_count'] = cur_max_count/1000 if int(cur_max_count) > 0 else cur_max_count + + cur_latency = case_result['metrics']['serial_latency_p99'] + case_result['metrics']['serial_latency_p99'] = cur_latency*1000 if cur_latency > 0 else cur_latency + c = TestResult.validate(test_result) + + return c + + def display(self, dbs: list[DB] | None = None): + DATA_FORMAT = (" %-14s | %-17s %-20s %14s | %-10s %14s %14s %14s %14s") + TITLE_FORMAT = (" %-14s | %-17s %-20s %14s | %-10s %14s %14s %14s %14s") % ( + "DB", "db_label", "case", "label", "load_dur", "qps", "latency(p99)", "recall", "max_load_count") + + SUMMERY_FORMAT = ("Task summery: run_id=%s, task_label=%s") % ( + self.run_id[:5], self.task_label) + + fmt = [SUMMERY_FORMAT, TITLE_FORMAT] + fmt.append(DATA_FORMAT%( + "-"*14, + "-"*17, + "-"*20, + "-"*14, + "-"*10, + "-"*14, + "-"*14, + "-"*14, + "-"*14, + )) + + filter_list = dbs if dbs and isinstance(dbs, list) else None + + sorted_results = sorted(self.results, key=lambda x: ( + x.task_config.db.name, + x.task_config.db_config.db_label, + x.task_config.case_config.case_id.name, + ), reverse=True) + for f in sorted_results: + if filter_list and f.task_config.db not in filter_list: + continue + + fmt.append(DATA_FORMAT%( + f.task_config.db.name, + f.task_config.db_config.db_label, + f.task_config.case_config.case_id.name, + self.task_label, + f.metrics.load_duration, + f.metrics.qps, + f.metrics.serial_latency_p99, + f.metrics.recall, + f.metrics.max_load_count, + )) + + tmp_logger = logging.getLogger("no_color") + for f in fmt: + tmp_logger.info(f) diff --git a/vector_db_bench/results/result_20230609_standard.json b/vector_db_bench/results/result_20230609_standard.json new file mode 100644 index 000000000..54cda751a --- /dev/null +++ b/vector_db_bench/results/result_20230609_standard.json @@ -0,0 +1,2970 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 3200000, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "L2", + "efConstruction": 360, + "M": 30, + "num_candidates": null + }, + "case_config": { + "case_id": "Capacity Test(Large-dim)", + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 8600000, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": null + }, + "case_config": { + "case_id": "Capacity Test(Small-dim)", + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2434.5387, + "qps": 34.3911, + "serial_latency_p99": 0.1734, + "recall": 0.9923 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2434.5387, + "qps": 31.5927, + "serial_latency_p99": 0.1646, + "recall": 0.9924 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2434.5387, + "qps": 42.3405, + "serial_latency_p99": 0.1383, + "recall": 0.9992 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 26412.1028, + "qps": 15.2269, + "serial_latency_p99": 0.8618, + "recall": 0.9888 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 26412.1028, + "qps": 15.1749, + "serial_latency_p99": 0.7743, + "recall": 0.989 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 26412.1028, + "qps": 27.6181, + "serial_latency_p99": 0.3055, + "recall": 0.9999 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5869.4929, + "qps": 89.4473, + "serial_latency_p99": 0.0219, + "recall": 0.9875 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5869.4929, + "qps": 87.8616, + "serial_latency_p99": 0.0215, + "recall": 0.9871 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5869.4929, + "qps": 62.5257, + "serial_latency_p99": 0.0261, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 29829.7671, + "qps": 48.3866, + "serial_latency_p99": 0.0443, + "recall": 0.9909 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Large Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 29829.7671, + "qps": 48.8021, + "serial_latency_p99": 0.0347, + "recall": 0.9909 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Large Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 29829.7671, + "qps": 30.5848, + "serial_latency_p99": 0.0473, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Large Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1455.573, + "qps": 290.695, + "serial_latency_p99": 0.0075, + "recall": 0.9801 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1455.573, + "qps": 284.9882, + "serial_latency_p99": 0.0087, + "recall": 0.9803 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1455.573, + "qps": 390.2579, + "serial_latency_p99": 0.0073, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 398.5015, + "qps": 523.0791, + "serial_latency_p99": 0.0063, + "recall": 0.9513 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 398.5015, + "qps": 487.4188, + "serial_latency_p99": 0.0067, + "recall": 0.9503 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 398.5015, + "qps": 473.241, + "serial_latency_p99": 0.0045, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 4300000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "L2" + }, + "case_config": { + "case_id": "Capacity Test(Large-dim)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 12700000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "L2" + }, + "case_config": { + "case_id": "Capacity Test(Small-dim)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 752.1779, + "qps": 228.6015, + "serial_latency_p99": 0.0118, + "recall": 0.9491 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 752.1779, + "qps": 215.5243, + "serial_latency_p99": 0.011, + "recall": 0.9486 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 752.1779, + "qps": 245.1032, + "serial_latency_p99": 0.0095, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3478.7882, + "qps": 164.3866, + "serial_latency_p99": 0.0131, + "recall": 0.95 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3478.7882, + "qps": 139.0901, + "serial_latency_p99": 0.0172, + "recall": 0.9669 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3478.7882, + "qps": 115.3193, + "serial_latency_p99": 0.0157, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 752.1779, + "qps": 233.6172, + "serial_latency_p99": 0.011, + "recall": 0.9494 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 752.1779, + "qps": 218.1513, + "serial_latency_p99": 0.0104, + "recall": 0.9485 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 752.1779, + "qps": 235.2801, + "serial_latency_p99": 0.009, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 427.2816, + "qps": 472.2174, + "serial_latency_p99": 0.0084, + "recall": 0.9776 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 427.2816, + "qps": 447.1405, + "serial_latency_p99": 0.0071, + "recall": 0.9774 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 427.2816, + "qps": 476.5357, + "serial_latency_p99": 0.0053, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 22201.8358, + "qps": 100.23, + "serial_latency_p99": 0.0288, + "recall": 0.972 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Search Performance Test(Large Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 22201.8358, + "qps": 48.7675, + "serial_latency_p99": 0.0352, + "recall": 0.9911 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Large Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 22201.8358, + "qps": 30.5627, + "serial_latency_p99": 0.0471, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Large Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3253.6402, + "qps": 352.1345, + "serial_latency_p99": 0.0093, + "recall": 0.9213 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3253.6402, + "qps": 264.539, + "serial_latency_p99": 0.0099, + "recall": 0.9684 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3253.6402, + "qps": 232.8041, + "serial_latency_p99": 0.0109, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 5800000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Capacity Test(Small-dim)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 1800000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Capacity Test(Large-dim)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Search Performance Test(Large Dataset)", + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3674.0953, + "qps": 67.9121, + "serial_latency_p99": 0.1795, + "recall": 0.9909 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3674.0953, + "qps": 0.7636, + "serial_latency_p99": 1.9213, + "recall": 0.9908 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3674.0953, + "qps": 32, + "serial_latency_p99": 0.1245, + "recall": 1.0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 251.0931, + "qps": 101.6631, + "serial_latency_p99": 0.03, + "recall": 0.9977 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 251.0931, + "qps": 7.6, + "serial_latency_p99": 0.583, + "recall": 0.9977 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 251.0931, + "qps": 102, + "serial_latency_p99": 0.0292, + "recall": 1.0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 1480000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Capacity Test(Small-dim)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 455000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Capacity Test(Large-dim)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 283.6404, + "qps": 113.661, + "serial_latency_p99": 0.0489, + "recall": 0.9977 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 5500000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Capacity Test(Small-dim)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 1800000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Capacity Test(Large-dim)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 280.2736, + "qps": 73.0806, + "serial_latency_p99": 0.0472, + "recall": 0.9977 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3580.7827, + "qps": 63.1365, + "serial_latency_p99": 0.1457, + "recall": 0.991 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": "Search Performance Test(Large Dataset)", + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 5000000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Capacity Test(Small-dim)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 1200000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Capacity Test(Large-dim)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2142.6913, + "qps": 457.3331, + "serial_latency_p99": 0.0056, + "recall": 0.9473 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2142.6913, + "qps": 308.5526, + "serial_latency_p99": 0.0077, + "recall": 0.9805 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2142.6913, + "qps": 408.1209, + "serial_latency_p99": 0.0057, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 254.2501, + "qps": 1157.4606, + "serial_latency_p99": 0.0033, + "recall": 0.958 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 254.2501, + "qps": 1018.7988, + "serial_latency_p99": 0.0041, + "recall": 0.958 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 254.2501, + "qps": 1517.0827, + "serial_latency_p99": 0.0035, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2325.0144, + "qps": 2867.9092, + "serial_latency_p99": 0.0037, + "recall": 0.8769 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2325.0144, + "qps": 1689.5799, + "serial_latency_p99": 0.0054, + "recall": 0.9486 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2325.0144, + "qps": 1371.0535, + "serial_latency_p99": 0.0079, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 221.5923, + "qps": 3676.5906, + "serial_latency_p99": 0.0046, + "recall": 0.9567 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 221.5923, + "qps": 3348.3574, + "serial_latency_p99": 0.0039, + "recall": 0.957 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 221.5923, + "qps": 5113.8498, + "serial_latency_p99": 0.0041, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 15985.5509, + "qps": 723.9371, + "serial_latency_p99": 0.0052, + "recall": 0.9285 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Search Performance Test(Large Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 15985.5509, + "qps": 315.5653, + "serial_latency_p99": 0.0089, + "recall": 0.9757 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Large Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 15985.5509, + "qps": 147.6055, + "serial_latency_p99": 0.0132, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Large Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 9100000, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Capacity Test(Small-dim)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 1000000, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Capacity Test(Large-dim)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2379.5565, + "qps": 232.1813, + "serial_latency_p99": 0.0363, + "recall": 0.9818 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 229.3619, + "qps": 1241.9402, + "serial_latency_p99": 0.0031, + "recall": 0.9578 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 229.3619, + "qps": 1090.1883, + "serial_latency_p99": 0.003, + "recall": 0.9579 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 229.3619, + "qps": 1610.7467, + "serial_latency_p99": 0.0025, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 218.1418, + "qps": 1232.3202, + "serial_latency_p99": 0.0025, + "recall": 0.9577 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2379.7118, + "qps": 274.5407, + "serial_latency_p99": 0.0049, + "recall": 0.9807 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2379.7118, + "qps": 236.5672, + "serial_latency_p99": 0.0103, + "recall": 0.981 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2379.7118, + "qps": 309.4833, + "serial_latency_p99": 0.0043, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 4229.5853, + "qps": 120.3348, + "serial_latency_p99": 0.0603, + "recall": 0.9819 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "1c8g", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 364.6594, + "qps": 674.0372, + "serial_latency_p99": 0.0027, + "recall": 0.9576 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "1c8g", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 118.9911, + "serial_latency_p99": 0.0607, + "recall": 0.981 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "1c8g", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 370.4316, + "qps": 646.3226, + "serial_latency_p99": 0.0028, + "recall": 0.9578 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "1c8g", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 390.8309, + "qps": 655.6607, + "serial_latency_p99": 0.0027, + "recall": 0.9581 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "1c8g", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 78.1565, + "qps": 4380.5652, + "serial_latency_p99": 0.0028, + "recall": 0.9581 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 78.1565, + "qps": 4028.975, + "serial_latency_p99": 0.0033, + "recall": 0.9582 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 78.1565, + "qps": 5740.1221, + "serial_latency_p99": 0.0026, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5530.3545, + "qps": 133.6633, + "serial_latency_p99": 0.0133, + "recall": 0.9842 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Large Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5530.3545, + "qps": 127.0729, + "serial_latency_p99": 0.0154, + "recall": 0.9843 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Large Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5530.3545, + "qps": 182.826, + "serial_latency_p99": 0.0122, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Large Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 607.0437, + "qps": 1118.5178, + "serial_latency_p99": 0.0043, + "recall": 0.9796 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 607.0437, + "qps": 963.5486, + "serial_latency_p99": 0.0048, + "recall": 0.9799 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 607.0437, + "qps": 1381.2118, + "serial_latency_p99": 0.0039, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 468.3754, + "qps": 78.9362, + "serial_latency_p99": 0.0385, + "recall": 0.8197 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "0.5c4g-1node", + "url": "**********", + "api_key": "**********", + "prefer_grpc": true + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": "Search Performance Test(Small Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 518.1908, + "qps": 75.0343, + "serial_latency_p99": 0.042, + "recall": 0.8122 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "0.5c4g-1node", + "url": "**********", + "api_key": "**********", + "prefer_grpc": true + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 599.7789, + "qps": 240.455, + "serial_latency_p99": 0.0349, + "recall": 0.9112 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "0.5c4g-1node", + "url": "**********", + "api_key": "**********", + "prefer_grpc": true + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Small Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2559.3758, + "qps": 76.672, + "serial_latency_p99": 0.0569, + "recall": 0.7964 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********", + "prefer_grpc": true + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": "Search Performance Test(Medium Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2519.4009, + "qps": 51.9266, + "serial_latency_p99": 0.0535, + "recall": 0.7952 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********", + "prefer_grpc": true + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2528.1643, + "qps": 128.3939, + "serial_latency_p99": 0.0397, + "recall": 0.8613 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********", + "prefer_grpc": true + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Medium Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 25181.5505, + "qps": 115.4696, + "serial_latency_p99": 0.0677, + "recall": 0.7861 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********", + "prefer_grpc": true + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": "Search Performance Test(Large Dataset)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 24106.79, + "qps": 87.881, + "serial_latency_p99": 0.0485, + "recall": 0.7836 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********", + "prefer_grpc": true + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Large Dataset, Low Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 24106.79, + "qps": 130.1864, + "serial_latency_p99": 0.0411, + "recall": 0.8412 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********", + "prefer_grpc": true + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": "Filtering Search Performance Test (Large Dataset, High Filtering Rate)", + "custom_case": {} + } + }, + "label": ":)" + } + ] +}