chore(controller): add more default values in system setting (#2985)

docker/charts/templates/controller-deployment.yaml

@@ -118,6 +118,10 @@ spec:
               value: {{ .Values.controller.job.cacheDirHostPath }}/job
             - name: SW_INSTANCE_URI
               value: "http://controller:{{ .Values.controller.containerPort }}"
+            - name: SW_DATASET_BUILD_CLI_VERSION
+              value: "{{ .Values.image.server.tag | default .Values.controller.setting.dsBuildCliVersion }}"
+            - name: SW_DATASET_BUILD_PYTHON_VERSION
+              value: "{{ .Values.controller.setting.dsBuildPythonVersion }}"
             {{- if .Values.minio.enabled }}
             - name: SW_STORAGE_ENDPOINT
             {{- if .Values.minio.ingress }}

docker/charts/values.yaml

@@ -83,6 +83,9 @@ controller:
   auth:
     username: starwhale
     password: abcd1234
+  setting:
+    dsBuildCliVersion: ""
+    dsBuildPythonVersion: "3.10"
   containerPort: 8082
   managementPort: 9082
   ingress:

server/controller/src/main/java/ai/starwhale/mlops/domain/system/SystemSettingService.java

@@ -84,10 +84,10 @@ public String updateSetting(String setting) {
                 systemSetting.setPypiSetting(Pypi.empty());
             }
             if (null == systemSetting.getImageBuild()) {
-                systemSetting.setImageBuild(RunTimeProperties.RunConfig.empty());
+                systemSetting.setImageBuild(runTimeProperties.getImageBuild());
             }
             if (null == systemSetting.getDatasetBuild()) {
-                systemSetting.setDatasetBuild(RunTimeProperties.RunConfig.empty());
+                systemSetting.setDatasetBuild(runTimeProperties.getDatasetBuild());
             }
             if (null == systemSetting.getDockerSetting()) {
                 systemSetting.setDockerSetting(DockerSetting.empty());
@@ -147,6 +147,12 @@ public void run(String... args) throws Exception {
                 if (CollectionUtils.isEmpty(systemSetting.getResourcePoolSetting())) {
                     systemSetting.setResourcePoolSetting(List.of(ResourcePool.defaults()));
                 }
+                if (null == systemSetting.getImageBuild()) {
+                    systemSetting.setImageBuild(runTimeProperties.getImageBuild());
+                }
+                if (null == systemSetting.getDatasetBuild()) {
+                    systemSetting.setDatasetBuild(runTimeProperties.getDatasetBuild());
+                }
                 listeners.forEach(l -> l.onUpdate(systemSetting));
             } catch (JsonProcessingException e) {
                 log.error("corrupted system setting yaml {}", setting.getContent());
@@ -157,6 +163,8 @@ public void run(String... args) throws Exception {
             systemSetting.setPypiSetting(runTimeProperties.getPypi());
             systemSetting.setCondaSetting(runTimeProperties.getCondarc());
             systemSetting.setDockerSetting(dockerSetting);
+            systemSetting.setImageBuild(runTimeProperties.getImageBuild());
+            systemSetting.setDatasetBuild(runTimeProperties.getDatasetBuild());
             systemSetting.setResourcePoolSetting(List.of(ResourcePool.defaults()));
         }
 

server/controller/src/main/java/ai/starwhale/mlops/domain/system/resourcepool/bo/ResourcePool.java

@@ -59,6 +59,7 @@ public static ResourcePool defaults() {
                 .nodeSelector(Map.of())
                 .resources(List.of(
                         new Resource(ResourceOverwriteSpec.RESOURCE_CPU),
+                        new Resource(ResourceOverwriteSpec.RESOURCE_GPU),
                         new Resource(ResourceOverwriteSpec.RESOURCE_MEMORY)
                 ))
                 .build();

server/controller/src/main/resources/application.yaml

@@ -28,11 +28,11 @@ sw:
   runtime:
     image-build:
       resource-pool: ${SW_IMAGE_BUILD_RESOURCE_POOL:default}
-      image: ${SW_IMAGE_BUILD_IMAGE:gcr.io/kaniko-project/executor:latest}
+      image: ${SW_IMAGE_BUILD_IMAGE:docker-registry.starwhale.cn/star-whale/runtime-dockerizing:latest}
     dataset-build:
       resource-pool: ${SW_DATASET_BUILD_RESOURCE_POOL:default}
       image: ${SW_DATASET_BUILD_IMAGE:docker-registry.starwhale.cn/star-whale/dataset_builder:latest}
-      client-version: ${SW_DATASET_BUILD_CLI_VERSION:0.5.6}
+      client-version: ${SW_DATASET_BUILD_CLI_VERSION:}
       python-version: ${SW_DATASET_BUILD_PYTHON_VERSION:3.10}
     pypi:
       index-url: ${SW_PYPI_INDEX_URL:}

server/controller/src/test/java/ai/starwhale/mlops/domain/job/JobBoConverterTest.java

@@ -23,6 +23,7 @@
 import static org.mockito.Mockito.when;
 
 import ai.starwhale.mlops.configuration.DockerSetting;
+import ai.starwhale.mlops.configuration.RunTimeProperties;
 import ai.starwhale.mlops.domain.dataset.DatasetDao;
 import ai.starwhale.mlops.domain.dataset.bo.DataSet;
 import ai.starwhale.mlops.domain.dataset.bo.DatasetVersion;
@@ -143,7 +144,7 @@ public void testJobBoConverter() throws IOException {
                 List.of(TaskEntity.builder().build(), TaskEntity.builder().build()));
 
         SystemSettingService systemSettingService = new SystemSettingService(
-                mock(SystemSettingMapper.class), List.of(), null, new DockerSetting(), null);
+                mock(SystemSettingMapper.class), List.of(), mock(RunTimeProperties.class), new DockerSetting(), null);
         systemSettingService.updateSetting("---\n"
                 + "dockerSetting:\n"
                 + "  registryForPull: \"\"\n"

server/controller/src/test/java/ai/starwhale/mlops/domain/system/SystemSettingServiceTest.java

@@ -63,6 +63,16 @@ public class SystemSettingServiceTest {
             + "  trustedHost: \"host1\"\n"
             + "  retries: 11\n"
             + "  timeout: 91\n"
+            + "imageBuild:\n"
+            + "  resourcePool: null\n"
+            + "  image: null\n"
+            + "  clientVersion: null\n"
+            + "  pythonVersion: null\n"
+            + "datasetBuild:\n"
+            + "  resourcePool: null\n"
+            + "  image: null\n"
+            + "  clientVersion: null\n"
+            + "  pythonVersion: null\n"
             + "condaSetting: |-\n"
             + "  channels:\n"
             + "    - defaults\n"
@@ -84,6 +94,10 @@ public class SystemSettingServiceTest {
             + "    max: null\n"
             + "    min: null\n"
             + "    defaults: null\n"
+            + "  - name: \"nvidia.com/gpu\"\n"
+            + "    max: null\n"
+            + "    min: null\n"
+            + "    defaults: null\n"
             + "  - name: \"memory\"\n"
             + "    max: null\n"
             + "    min: null\n"
@@ -152,7 +166,7 @@ public void testUpdate() {
         Assertions.assertEquals(3, systemSettingService.queryResourcePool("custom").getResources().size());
         // get the default resource pool
         Assertions.assertEquals(ResourcePool.defaults(), systemSettingService.queryResourcePool("not_exists"));
-        Assertions.assertEquals(2, systemSettingService.queryResourcePool("not_exists").getResources().size());
+        Assertions.assertEquals(3, systemSettingService.queryResourcePool("not_exists").getResources().size());
         verify(listener).onUpdate(systemSettingService.getSystemSetting());
     }
 
@@ -181,15 +195,15 @@ public void testUnsetSetting() {
                 + "  retries: 10\n"
                 + "  timeout: 90\n"
                 + "imageBuild:\n"
-                + "  resourcePool: \"\"\n"
-                + "  image: \"\"\n"
-                + "  clientVersion: \"\"\n"
-                + "  pythonVersion: \"\"\n"
+                + "  resourcePool: null\n"
+                + "  image: null\n"
+                + "  clientVersion: null\n"
+                + "  pythonVersion: null\n"
                 + "datasetBuild:\n"
-                + "  resourcePool: \"\"\n"
-                + "  image: \"\"\n"
-                + "  clientVersion: \"\"\n"
-                + "  pythonVersion: \"\"\n"
+                + "  resourcePool: null\n"
+                + "  image: null\n"
+                + "  clientVersion: null\n"
+                + "  pythonVersion: null\n"
                 + "resourcePoolSetting:\n"
                 + "- name: \"default\"\n"
                 + "  nodeSelector: {}\n"
@@ -198,6 +212,10 @@ public void testUnsetSetting() {
                 + "    max: null\n"
                 + "    min: null\n"
                 + "    defaults: null\n"
+                + "  - name: \"nvidia.com/gpu\"\n"
+                + "    max: null\n"
+                + "    min: null\n"
+                + "    defaults: null\n"
                 + "  - name: \"memory\"\n"
                 + "    max: null\n"
                 + "    min: null\n"
@@ -239,6 +257,16 @@ public void testStartWithoutData() throws Exception {
                 + "  trustedHost: \"\"\n"
                 + "  retries: 1\n"
                 + "  timeout: 2\n"
+                + "imageBuild:\n"
+                + "  resourcePool: null\n"
+                + "  image: null\n"
+                + "  clientVersion: null\n"
+                + "  pythonVersion: null\n"
+                + "datasetBuild:\n"
+                + "  resourcePool: null\n"
+                + "  image: null\n"
+                + "  clientVersion: null\n"
+                + "  pythonVersion: null\n"
                 + "condaSetting: \"\"\n"
                 + "resourcePoolSetting:\n"
                 + "- name: \"default\"\n"
@@ -248,6 +276,10 @@ public void testStartWithoutData() throws Exception {
                 + "    max: null\n"
                 + "    min: null\n"
                 + "    defaults: null\n"
+                 + "  - name: \"nvidia.com/gpu\"\n"
+                 + "    max: null\n"
+                 + "    min: null\n"
+                 + "    defaults: null\n"
                 + "  - name: \"memory\"\n"
                 + "    max: null\n"
                 + "    min: null\n"