feat(client): add --push argument for model/runtime/dataset build (#…

…3090)

client/starwhale/core/dataset/cli.py

@@ -148,6 +148,15 @@ def dataset_cmd(ctx: click.Context) -> None:
     "--encoding",
     help="The csv/json/jsonl file encoding.",
 )
+@optgroup.option(  # type: ignore[no-untyped-call]
+    "--push",
+    help="Cloud/Server dest project uri or dataset uri",
+)
+@optgroup.option(  # type: ignore[no-untyped-call]
+    "--force-push",
+    is_flag=True,
+    help="Force to push dataset, even the version has been pushed",
+)
 @optgroup.option("-r", "--runtime", help="runtime uri")  # type: ignore[no-untyped-call]
 @optgroup.group("\n  ** Handler Build Source Configurations")
 @optgroup.option("-w", "--workdir", default=".", help="work dir to search handler, the option only works for the handler build source.")  # type: ignore[no-untyped-call]
@@ -285,6 +294,8 @@ def _build(
     csv_skipinitialspace: bool,
     csv_strict: bool,
     file_encoding: str,
+    push: str,
+    force_push: bool,
 ) -> None:
     """Build Starwhale Dataset.
     This command only supports to build standalone dataset.
@@ -311,6 +322,8 @@ def _build(
         swcli dataset build --yaml /path/to/dataset.yaml  # build dataset from /path/to/dataset.yaml, all the involved files are related to the dataset.yaml file.
         swcli dataset build --overwrite --yaml /path/to/dataset.yaml  # build dataset from /path/to/dataset.yaml, and overwrite the existed dataset.
         swcli dataset build --tag tag1 --tag tag2
+        swcli dataset build --push https://cloud.starwhale.cn/project/starwhale:public
+        swcli dataset build --push https://cloud.starwhale.cn/project/starwhale:public/dataset/new-dataset-name
 
         \b
         - from handler
@@ -367,7 +380,7 @@ def _build(
 
         folder = Path(folder).absolute()
         # TODO: support desc field
-        view.build_from_folder(
+        built_dataset_uri = view.build_from_folder(
             folder,
             kind,
             name=name or folder.name,
@@ -379,7 +392,7 @@ def _build(
             tags=tags,
         )
     elif json_files:
-        view.build_from_json_files(
+        built_dataset_uri = view.build_from_json_files(
             json_files,
             name=name or f"json-{random_str()}",
             project_uri=project,
@@ -391,7 +404,7 @@ def _build(
             encoding=file_encoding,
         )
     elif csv_files:
-        view.build_from_csv_files(
+        built_dataset_uri = view.build_from_csv_files(
             csv_files,
             name=name or f"csv-{random_str()}",
             project_uri=project,
@@ -421,10 +434,10 @@ def _build(
             tags=tags,
         )
         config.do_validate()
-        view.build(_workdir, config, mode=mode_type, tags=tags)
+        built_dataset_uri = view.build(_workdir, config, mode=mode_type, tags=tags)
     elif hf_repo:
         _candidate_name = (f"{hf_repo}").strip("/").replace("/", "-")
-        view.build_from_huggingface(
+        built_dataset_uri = view.build_from_huggingface(
             hf_repo,
             name=name or _candidate_name,
             project_uri=project,
@@ -455,7 +468,15 @@ def _build(
         config.runtime_uri = runtime or config.runtime_uri
         config.project_uri = project or config.project_uri
         config.do_validate()
-        view.build(_workdir, config, mode=mode_type, tags=tags)
+        built_dataset_uri = view.build(_workdir, config, mode=mode_type, tags=tags)
+
+    if push and built_dataset_uri:
+        DatasetTermView.copy(
+            src_uri=built_dataset_uri.full_uri,
+            dest_uri=push,
+            mode=mode_type,
+            force=force_push,
+        )
 
 
 @dataset_cmd.command("diff", help="Dataset version diff")

client/starwhale/core/dataset/view.py

@@ -198,7 +198,7 @@ def build_from_huggingface(
         name: str,
         project_uri: str,
         **kwargs: t.Any,
-    ) -> None:
+    ) -> Resource:
         dataset_uri = cls.prepare_build_bundle(
             project=project_uri,
             bundle_name=name,
@@ -207,12 +207,13 @@ def build_from_huggingface(
         )
         ds = Dataset.get_dataset(dataset_uri)
         ds.build_from_huggingface(repo=repo, **kwargs)
+        return dataset_uri
 
     @classmethod
     @BaseTermView._only_standalone
     def build_from_csv_files(
         cls, paths: t.List[PathLike], name: str, project_uri: str, **kwargs: t.Any
-    ) -> None:
+    ) -> Resource:
         dataset_uri = cls.prepare_build_bundle(
             project=project_uri,
             bundle_name=name,
@@ -221,6 +222,7 @@ def build_from_csv_files(
         )
         ds = Dataset.get_dataset(dataset_uri)
         ds.build_from_csv_files(paths, **kwargs)
+        return dataset_uri
 
     @classmethod
     @BaseTermView._only_standalone
@@ -230,7 +232,7 @@ def build_from_json_files(
         name: str,
         project_uri: str,
         **kwargs: t.Any,
-    ) -> None:
+    ) -> Resource:
         dataset_uri = cls.prepare_build_bundle(
             project=project_uri,
             bundle_name=name,
@@ -239,6 +241,7 @@ def build_from_json_files(
         )
         ds = Dataset.get_dataset(dataset_uri)
         ds.build_from_json_files(paths, **kwargs)
+        return dataset_uri
 
     @classmethod
     @BaseTermView._only_standalone
@@ -249,7 +252,7 @@ def build_from_folder(
         name: str,
         project_uri: str,
         **kwargs: t.Any,
-    ) -> None:
+    ) -> Resource:
         dataset_uri = cls.prepare_build_bundle(
             project=project_uri,
             bundle_name=name,
@@ -258,6 +261,7 @@ def build_from_folder(
         )
         ds = Dataset.get_dataset(dataset_uri)
         ds.build_from_folder(folder=folder, kind=kind, **kwargs)
+        return dataset_uri
 
     @classmethod
     @BaseTermView._only_standalone
@@ -267,9 +271,10 @@ def build(
         config: DatasetConfig,
         mode: DatasetChangeMode = DatasetChangeMode.PATCH,
         tags: t.List[str] | None = None,
-    ) -> None:
+    ) -> Resource | None:
         if config.runtime_uri:
             RuntimeProcess(uri=config.runtime_uri).run()
+            return None
         else:
             dataset_uri = cls.prepare_build_bundle(
                 project=config.project_uri,
@@ -279,6 +284,7 @@ def build(
             )
             ds = Dataset.get_dataset(dataset_uri)
             ds.build(workdir=Path(workdir), config=config, mode=mode, tags=tags)
+            return dataset_uri
 
     @classmethod
     def copy(

client/starwhale/core/model/cli.py

@@ -91,6 +91,15 @@ def model_cmd(ctx: click.Context) -> None:
     help="Ignore files or directories. The option can be used multiple times."
     "The '.swignore' file still takes effect.",
 )
+@click.option(
+    "--push",
+    help="Cloud/Server dest project uri or model uri",
+)
+@click.option(
+    "--force-push",
+    is_flag=True,
+    help="Force to push model package, even the version has been pushed",
+)
 def _build(
     workdir: str,
     project: str,
@@ -103,6 +112,8 @@ def _build(
     desc: str,
     add_all: bool,
     excludes: t.List[str],
+    push: str,
+    force_push: bool,
 ) -> None:
     """Build starwhale model package.
     Only standalone instance supports model build.
@@ -124,6 +135,10 @@ def _build(
         swcli model build . --tag tag1 --tag tag2
         # build model package with ignores.
         swcli model build . --exclude .git --exclude checkpoint/*
+        # build and push model package to the cloud instance.
+        swcli model build . --push https://cloud.starwhale.cn/project/starwhale:public
+        # build and push model package to the cloud instance with a specified model name.
+        swcli model build . --push https://cloud.starwhale.cn/project/starwhale:public/model/new-model-name
     """
     if model_yaml is None:
         yaml_path = Path(workdir) / DefaultYAMLName.MODEL
@@ -140,7 +155,7 @@ def _build(
     config.desc = desc
     config.do_validate()
 
-    ModelTermView.build(
+    uri = ModelTermView.build(
         workdir=workdir,
         project=project,
         model_config=config,
@@ -151,6 +166,9 @@ def _build(
         excludes=excludes,
     )
 
+    if push and uri:
+        ModelTermView.copy(src_uri=uri.full_uri, dest_uri=push, force=force_push)
+
 
 @model_cmd.command("extract")
 @click.argument("model")

client/starwhale/core/model/view.py

@@ -357,9 +357,10 @@ def build(
         package_runtime: bool = True,
         tags: t.List[str] | None = None,
         excludes: t.List[str] | None = None,
-    ) -> None:
+    ) -> Resource | None:
         if runtime_uri:
             RuntimeProcess(uri=Resource(runtime_uri, typ=ResourceType.runtime)).run()
+            return None
         else:
             model_uri = cls.prepare_build_bundle(
                 project=project,
@@ -378,6 +379,7 @@ def build(
                 tags=tags,
                 excludes=excludes,
             )
+            return model_uri
 
     @classmethod
     def copy(

client/starwhale/core/runtime/cli.py

@@ -284,6 +284,15 @@ def _quickstart(
     multiple=True,
     help="runtime tags, the option can be used multiple times. `latest` and `^v\d+$` tags are reserved tags.",
 )
+@click.option(
+    "--push",
+    help="Cloud/Server dest project uri or runtime uri",
+)
+@click.option(
+    "--force-push",
+    is_flag=True,
+    help="Force to push runtime, even the version has been pushed",
+)
 def _build(
     name: str,
     project: str,
@@ -304,6 +313,8 @@ def _build(
     dump_pip_options: bool,
     dump_condarc: bool,
     tags: t.List[str],
+    push: str,
+    force_push: bool,
 ) -> None:
     """Create and build a relocated, shareable, packaged runtime bundle(aka `swrt` file). Support python and native libs.
     Runtime build only works in the Standalone instance.
@@ -327,6 +338,8 @@ def _build(
         swcli runtime build -y example/pytorch/runtime.yaml # use example/pytorch/runtime.yaml as the runtime.yaml file
         swcli runtime build --yaml runtime.yaml # use runtime.yaml at the current directory as the runtime.yaml file
         swcli runtime build --tag tag1 --tag tag2
+        swcli runtime build --push https://cloud.starwhale.cn/project/starwhale:public --force-push # push runtime to the cloud instance
+        swcli runtime build --push https://cloud.starwhale.cn/project/starwhale:public/runtime/new-runtime-name --force-push # push runtime to the cloud instance with a specified runtime name
 
         \b
         - from conda name:
@@ -354,12 +367,12 @@ def _build(
         swcli runtime build --shell --name pytorch-runtime # lock the current shell environment and use `pytorch-runtime` as the runtime name
     """
     if docker:
-        RuntimeTermView.build_from_docker_image(
+        uri = RuntimeTermView.build_from_docker_image(
             image=docker, runtime_name=name, project=project
         )
     elif conda or conda_prefix or venv or shell:
         # TODO: support auto mode for cuda, cudnn and arch
-        RuntimeTermView.build_from_python_env(
+        uri = RuntimeTermView.build_from_python_env(
             runtime_name=name,
             conda_name=conda,
             conda_prefix=conda_prefix,
@@ -376,7 +389,7 @@ def _build(
             tags=tags,
         )
     else:
-        RuntimeTermView.build_from_runtime_yaml(
+        uri = RuntimeTermView.build_from_runtime_yaml(
             workdir=Path.cwd(),
             yaml_path=Path(yaml),
             project=project,
@@ -391,6 +404,9 @@ def _build(
             tags=tags,
         )
 
+    if push and uri:
+        RuntimeTermView.copy(src_uri=uri.full_uri, dest_uri=push, force=force_push)
+
 
 @runtime_cmd.command("remove", aliases=["rm"])
 @click.argument("runtime")

client/starwhale/core/runtime/view.py

@@ -155,14 +155,15 @@ def lock(
     @BaseTermView._only_standalone
     def build_from_docker_image(
         cls, image: str, runtime_name: str = "", project: str = ""
-    ) -> None:
+    ) -> Resource:
         runtime_name = runtime_name or image.split(":")[0].split("/")[-1]
         runtime_uri = cls.prepare_build_bundle(
             project=project, bundle_name=runtime_name, typ=ResourceType.runtime
         )
 
         rt = Runtime.get_runtime(runtime_uri)
         rt.build_from_docker_image(image=image, runtime_name=runtime_name)
+        return runtime_uri
 
     @classmethod
     @BaseTermView._only_standalone