chore(e2e): compare datastore table instead of dataset (#2902)

client/Makefile

@@ -33,24 +33,26 @@ black-format:
 isort-format:
 	isort $(PY_CHANGED_FILES)
 
+LINT_DIRS = . ../example ../scripts/client_test
+
 format:
-	black --config pyproject.toml . ../example ../scripts/client_test && isort . ../example ../scripts/client_test
+	ruff check --fix ${LINT_DIRS} && black --config pyproject.toml ${LINT_DIRS} && isort ${LINT_DIRS}
 
 ci-format-checker:
 	echo "run black"
-	black --check --config pyproject.toml . ../example ../scripts/client_test
+	black --check --config pyproject.toml ${LINT_DIRS}
 
 ci-lint:
 	echo "run linter..."
-	ruff check . ../example ../scripts/client_test
+	ruff check ${LINT_DIRS}
 
 ci-mypy:
 	echo "run mypy"
 	mypy . ../example/mnist ../scripts/client_test --show-traceback
 
 ci-isort:
 	echo "run isort"
-	isort --check . ../example ../scripts/client_test
+	isort --check ${LINT_DIRS}
 
 ut:
 	echo "ut"

scripts/client_test/cli_test.py

@@ -23,11 +23,11 @@
 from cmds.instance_cmd import Instance
 from cmds.artifacts_cmd import Model, Dataset, Runtime
 
-from starwhale import dataset
 from starwhale.utils import config
 from starwhale.base.type import DatasetChangeMode
 from starwhale.utils.debug import init_logger
 from starwhale.base.uri.resource import Resource, ResourceType
+from starwhale.api._impl.data_store import TableDesc, LocalDataStore, RemoteDataStore
 
 CURRENT_DIR = os.path.dirname(__file__)
 SCRIPT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir))
@@ -216,11 +216,20 @@ def build_dataset(self, name: str, _workdir: str, ds_expl: DatasetExpl) -> t.Any
 
     @staticmethod
     def compare_local_and_remote_dataset(local: Resource, remote: Resource) -> None:
-        local_ds = dataset(local).with_loader_config(num_workers=1)
-        remote_ds = dataset(remote).with_loader_config(num_workers=1)
-        for local_item, remote_item in zip(local_ds, remote_ds):
+        local_ds = LocalDataStore.get_instance()
+        remote_ds = RemoteDataStore(remote.instance.url, remote.instance.token)
+
+        def get_table_name(uri: Resource) -> str:
+            return f"/project/{uri.project.id}/dataset/{uri.name}/_current/meta"
+
+        local_iter = local_ds.scan_tables([TableDesc(table_name=get_table_name(local))])
+        remote_iter = remote_ds.scan_tables(
+            [TableDesc(table_name=get_table_name(remote))]
+        )
+
+        for local_item, remote_item in zip(local_iter, remote_iter):
             # TODO compare the whole item when every type has implemented __eq__
-            if local_item.index != remote_item.index:
+            if local_item["*"] != remote_item["*"]:
                 raise RuntimeError(
                     f"local item {local_item!r} is not equal to remote item {remote_item!r}"
                 )