diff --git a/client/starwhale/core/dataset/cli.py b/client/starwhale/core/dataset/cli.py index c3a3e854e8..aacf94c87c 100644 --- a/client/starwhale/core/dataset/cli.py +++ b/client/starwhale/core/dataset/cli.py @@ -148,6 +148,15 @@ def dataset_cmd(ctx: click.Context) -> None: "--encoding", help="The csv/json/jsonl file encoding.", ) +@optgroup.option( # type: ignore[no-untyped-call] + "--push", + help="Cloud/Server dest project uri or dataset uri", +) +@optgroup.option( # type: ignore[no-untyped-call] + "--force-push", + is_flag=True, + help="Force to push dataset, even the version has been pushed", +) @optgroup.option("-r", "--runtime", help="runtime uri") # type: ignore[no-untyped-call] @optgroup.group("\n ** Handler Build Source Configurations") @optgroup.option("-w", "--workdir", default=".", help="work dir to search handler, the option only works for the handler build source.") # type: ignore[no-untyped-call] @@ -285,6 +294,8 @@ def _build( csv_skipinitialspace: bool, csv_strict: bool, file_encoding: str, + push: str, + force_push: bool, ) -> None: """Build Starwhale Dataset. This command only supports to build standalone dataset. @@ -311,6 +322,8 @@ def _build( swcli dataset build --yaml /path/to/dataset.yaml # build dataset from /path/to/dataset.yaml, all the involved files are related to the dataset.yaml file. swcli dataset build --overwrite --yaml /path/to/dataset.yaml # build dataset from /path/to/dataset.yaml, and overwrite the existed dataset. swcli dataset build --tag tag1 --tag tag2 + swcli dataset build --push https://cloud.starwhale.cn/project/starwhale:public + swcli dataset build --push https://cloud.starwhale.cn/project/starwhale:public/dataset/new-dataset-name \b - from handler @@ -367,7 +380,7 @@ def _build( folder = Path(folder).absolute() # TODO: support desc field - view.build_from_folder( + built_dataset_uri = view.build_from_folder( folder, kind, name=name or folder.name, @@ -379,7 +392,7 @@ def _build( tags=tags, ) elif json_files: - view.build_from_json_files( + built_dataset_uri = view.build_from_json_files( json_files, name=name or f"json-{random_str()}", project_uri=project, @@ -391,7 +404,7 @@ def _build( encoding=file_encoding, ) elif csv_files: - view.build_from_csv_files( + built_dataset_uri = view.build_from_csv_files( csv_files, name=name or f"csv-{random_str()}", project_uri=project, @@ -421,10 +434,10 @@ def _build( tags=tags, ) config.do_validate() - view.build(_workdir, config, mode=mode_type, tags=tags) + built_dataset_uri = view.build(_workdir, config, mode=mode_type, tags=tags) elif hf_repo: _candidate_name = (f"{hf_repo}").strip("/").replace("/", "-") - view.build_from_huggingface( + built_dataset_uri = view.build_from_huggingface( hf_repo, name=name or _candidate_name, project_uri=project, @@ -455,7 +468,15 @@ def _build( config.runtime_uri = runtime or config.runtime_uri config.project_uri = project or config.project_uri config.do_validate() - view.build(_workdir, config, mode=mode_type, tags=tags) + built_dataset_uri = view.build(_workdir, config, mode=mode_type, tags=tags) + + if push and built_dataset_uri: + DatasetTermView.copy( + src_uri=built_dataset_uri.full_uri, + dest_uri=push, + mode=mode_type, + force=force_push, + ) @dataset_cmd.command("diff", help="Dataset version diff") diff --git a/client/starwhale/core/dataset/view.py b/client/starwhale/core/dataset/view.py index 8fc2e87d9e..35dde7b8e8 100644 --- a/client/starwhale/core/dataset/view.py +++ b/client/starwhale/core/dataset/view.py @@ -198,7 +198,7 @@ def build_from_huggingface( name: str, project_uri: str, **kwargs: t.Any, - ) -> None: + ) -> Resource: dataset_uri = cls.prepare_build_bundle( project=project_uri, bundle_name=name, @@ -207,12 +207,13 @@ def build_from_huggingface( ) ds = Dataset.get_dataset(dataset_uri) ds.build_from_huggingface(repo=repo, **kwargs) + return dataset_uri @classmethod @BaseTermView._only_standalone def build_from_csv_files( cls, paths: t.List[PathLike], name: str, project_uri: str, **kwargs: t.Any - ) -> None: + ) -> Resource: dataset_uri = cls.prepare_build_bundle( project=project_uri, bundle_name=name, @@ -221,6 +222,7 @@ def build_from_csv_files( ) ds = Dataset.get_dataset(dataset_uri) ds.build_from_csv_files(paths, **kwargs) + return dataset_uri @classmethod @BaseTermView._only_standalone @@ -230,7 +232,7 @@ def build_from_json_files( name: str, project_uri: str, **kwargs: t.Any, - ) -> None: + ) -> Resource: dataset_uri = cls.prepare_build_bundle( project=project_uri, bundle_name=name, @@ -239,6 +241,7 @@ def build_from_json_files( ) ds = Dataset.get_dataset(dataset_uri) ds.build_from_json_files(paths, **kwargs) + return dataset_uri @classmethod @BaseTermView._only_standalone @@ -249,7 +252,7 @@ def build_from_folder( name: str, project_uri: str, **kwargs: t.Any, - ) -> None: + ) -> Resource: dataset_uri = cls.prepare_build_bundle( project=project_uri, bundle_name=name, @@ -258,6 +261,7 @@ def build_from_folder( ) ds = Dataset.get_dataset(dataset_uri) ds.build_from_folder(folder=folder, kind=kind, **kwargs) + return dataset_uri @classmethod @BaseTermView._only_standalone @@ -267,9 +271,10 @@ def build( config: DatasetConfig, mode: DatasetChangeMode = DatasetChangeMode.PATCH, tags: t.List[str] | None = None, - ) -> None: + ) -> Resource | None: if config.runtime_uri: RuntimeProcess(uri=config.runtime_uri).run() + return None else: dataset_uri = cls.prepare_build_bundle( project=config.project_uri, @@ -279,6 +284,7 @@ def build( ) ds = Dataset.get_dataset(dataset_uri) ds.build(workdir=Path(workdir), config=config, mode=mode, tags=tags) + return dataset_uri @classmethod def copy( diff --git a/client/starwhale/core/model/cli.py b/client/starwhale/core/model/cli.py index 6fc0990b3a..18884226b3 100644 --- a/client/starwhale/core/model/cli.py +++ b/client/starwhale/core/model/cli.py @@ -91,6 +91,15 @@ def model_cmd(ctx: click.Context) -> None: help="Ignore files or directories. The option can be used multiple times." "The '.swignore' file still takes effect.", ) +@click.option( + "--push", + help="Cloud/Server dest project uri or model uri", +) +@click.option( + "--force-push", + is_flag=True, + help="Force to push model package, even the version has been pushed", +) def _build( workdir: str, project: str, @@ -103,6 +112,8 @@ def _build( desc: str, add_all: bool, excludes: t.List[str], + push: str, + force_push: bool, ) -> None: """Build starwhale model package. Only standalone instance supports model build. @@ -124,6 +135,10 @@ def _build( swcli model build . --tag tag1 --tag tag2 # build model package with ignores. swcli model build . --exclude .git --exclude checkpoint/* + # build and push model package to the cloud instance. + swcli model build . --push https://cloud.starwhale.cn/project/starwhale:public + # build and push model package to the cloud instance with a specified model name. + swcli model build . --push https://cloud.starwhale.cn/project/starwhale:public/model/new-model-name """ if model_yaml is None: yaml_path = Path(workdir) / DefaultYAMLName.MODEL @@ -140,7 +155,7 @@ def _build( config.desc = desc config.do_validate() - ModelTermView.build( + uri = ModelTermView.build( workdir=workdir, project=project, model_config=config, @@ -151,6 +166,9 @@ def _build( excludes=excludes, ) + if push and uri: + ModelTermView.copy(src_uri=uri.full_uri, dest_uri=push, force=force_push) + @model_cmd.command("extract") @click.argument("model") diff --git a/client/starwhale/core/model/view.py b/client/starwhale/core/model/view.py index a868bd4d20..3424921dbb 100644 --- a/client/starwhale/core/model/view.py +++ b/client/starwhale/core/model/view.py @@ -357,9 +357,10 @@ def build( package_runtime: bool = True, tags: t.List[str] | None = None, excludes: t.List[str] | None = None, - ) -> None: + ) -> Resource | None: if runtime_uri: RuntimeProcess(uri=Resource(runtime_uri, typ=ResourceType.runtime)).run() + return None else: model_uri = cls.prepare_build_bundle( project=project, @@ -378,6 +379,7 @@ def build( tags=tags, excludes=excludes, ) + return model_uri @classmethod def copy( diff --git a/client/starwhale/core/runtime/cli.py b/client/starwhale/core/runtime/cli.py index 4e3d0b8b0b..a7e2a39907 100644 --- a/client/starwhale/core/runtime/cli.py +++ b/client/starwhale/core/runtime/cli.py @@ -284,6 +284,15 @@ def _quickstart( multiple=True, help="runtime tags, the option can be used multiple times. `latest` and `^v\d+$` tags are reserved tags.", ) +@click.option( + "--push", + help="Cloud/Server dest project uri or runtime uri", +) +@click.option( + "--force-push", + is_flag=True, + help="Force to push runtime, even the version has been pushed", +) def _build( name: str, project: str, @@ -304,6 +313,8 @@ def _build( dump_pip_options: bool, dump_condarc: bool, tags: t.List[str], + push: str, + force_push: bool, ) -> None: """Create and build a relocated, shareable, packaged runtime bundle(aka `swrt` file). Support python and native libs. Runtime build only works in the Standalone instance. @@ -327,6 +338,8 @@ def _build( swcli runtime build -y example/pytorch/runtime.yaml # use example/pytorch/runtime.yaml as the runtime.yaml file swcli runtime build --yaml runtime.yaml # use runtime.yaml at the current directory as the runtime.yaml file swcli runtime build --tag tag1 --tag tag2 + swcli runtime build --push https://cloud.starwhale.cn/project/starwhale:public --force-push # push runtime to the cloud instance + swcli runtime build --push https://cloud.starwhale.cn/project/starwhale:public/runtime/new-runtime-name --force-push # push runtime to the cloud instance with a specified runtime name \b - from conda name: @@ -354,12 +367,12 @@ def _build( swcli runtime build --shell --name pytorch-runtime # lock the current shell environment and use `pytorch-runtime` as the runtime name """ if docker: - RuntimeTermView.build_from_docker_image( + uri = RuntimeTermView.build_from_docker_image( image=docker, runtime_name=name, project=project ) elif conda or conda_prefix or venv or shell: # TODO: support auto mode for cuda, cudnn and arch - RuntimeTermView.build_from_python_env( + uri = RuntimeTermView.build_from_python_env( runtime_name=name, conda_name=conda, conda_prefix=conda_prefix, @@ -376,7 +389,7 @@ def _build( tags=tags, ) else: - RuntimeTermView.build_from_runtime_yaml( + uri = RuntimeTermView.build_from_runtime_yaml( workdir=Path.cwd(), yaml_path=Path(yaml), project=project, @@ -391,6 +404,9 @@ def _build( tags=tags, ) + if push and uri: + RuntimeTermView.copy(src_uri=uri.full_uri, dest_uri=push, force=force_push) + @runtime_cmd.command("remove", aliases=["rm"]) @click.argument("runtime") diff --git a/client/starwhale/core/runtime/view.py b/client/starwhale/core/runtime/view.py index 3320700013..40e0faf962 100644 --- a/client/starwhale/core/runtime/view.py +++ b/client/starwhale/core/runtime/view.py @@ -155,7 +155,7 @@ def lock( @BaseTermView._only_standalone def build_from_docker_image( cls, image: str, runtime_name: str = "", project: str = "" - ) -> None: + ) -> Resource: runtime_name = runtime_name or image.split(":")[0].split("/")[-1] runtime_uri = cls.prepare_build_bundle( project=project, bundle_name=runtime_name, typ=ResourceType.runtime @@ -163,6 +163,7 @@ def build_from_docker_image( rt = Runtime.get_runtime(runtime_uri) rt.build_from_docker_image(image=image, runtime_name=runtime_name) + return runtime_uri @classmethod @BaseTermView._only_standalone