[BitSail][Features] Support list and map in current FileMappingTypeInfoConverter

bitsail-common/src/main/java/com/bytedance/bitsail/common/type/BitSailTypeParser.java

@@ -41,7 +41,25 @@ public class BitSailTypeParser {
   private static final String SPLIT_TOKEN = ",";
 
   public static String fromTypeInfo(TypeInfo<?> typeInfo) {
-    return TypeInfoBridge.bridgeTypes(typeInfo);
+    Preconditions.checkNotNull(typeInfo, "typeInfo can not be null.");
+
+    if (typeInfo instanceof ListTypeInfo || typeInfo instanceof MapTypeInfo) {
+
+      if (typeInfo instanceof MapTypeInfo) {
+        MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
+        return String.format("%s<%s, %s>", Types.MAP.name(), fromTypeInfo(mapTypeInfo.getKeyTypeInfo()), fromTypeInfo(mapTypeInfo.getValueTypeInfo()));
+      } else {
+
+        ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
+        return String.format("%s<%s>", Types.LIST.name(), fromTypeInfo(listTypeInfo.getElementTypeInfo()));
+      }
+    }
+    String typeString = TypeInfoBridge.bridgeTypes(typeInfo);
+    if (StringUtils.isEmpty(typeString)) {
+      throw BitSailException.asBitSailException(CommonErrorCode.INTERNAL_ERROR,
+        String.format("Not support type typeInfo %s.", typeInfo));
+    }
+    return typeString;
   }
 
   public static List<TypeProperty> fromTypePropertyString(String typePropertyString) {

bitsail-common/src/main/java/com/bytedance/bitsail/common/type/filemapping/FileMappingTypeInfoConverter.java

@@ -16,12 +16,20 @@
 
 package com.bytedance.bitsail.common.type.filemapping;
 
+import com.bytedance.bitsail.common.BitSailException;
+import com.bytedance.bitsail.common.exception.CommonErrorCode;
+import com.bytedance.bitsail.common.type.BitSailTypeParser;
 import com.bytedance.bitsail.common.type.TypeInfoConverter;
+import com.bytedance.bitsail.common.typeinfo.ListTypeInfo;
+import com.bytedance.bitsail.common.typeinfo.MapTypeInfo;
 import com.bytedance.bitsail.common.typeinfo.TypeInfo;
+import com.bytedance.bitsail.common.typeinfo.Types;
 
 import lombok.Getter;
 import org.apache.commons.lang3.StringUtils;
 
+import java.util.Objects;
+
 public class FileMappingTypeInfoConverter implements TypeInfoConverter {
   @Getter
   protected FileMappingTypeInfoReader reader;
@@ -52,11 +60,53 @@ protected static String trim(String typeName) {
 
   @Override
   public TypeInfo<?> fromTypeString(String typeString) {
-    return reader.getToTypeInformation().get(typeString);
+    TypeInfo<?> typeInfo = reader.getToTypeInformation().get(typeString);
+
+    if (!Objects.isNull(typeInfo)) {
+      return typeInfo;
+    }
+
+    String engineMapStr = reader.customToEngineTypeStringMap.get(Types.MAP.name().toLowerCase());
+    String engineListStr = reader.customToEngineTypeStringMap.get(Types.LIST.name().toLowerCase());
+    if (StringUtils.isNotBlank(engineMapStr) && typeString.contains(engineMapStr)
+        || StringUtils.isNotBlank(engineListStr) && typeString.contains(engineListStr)) {
+      String customTypeString = this.toCustomTypeString(typeString);
+      return BitSailTypeParser.fromTypeString(customTypeString);
+    }
+
+    throw BitSailException.asBitSailException(CommonErrorCode.INTERNAL_ERROR,
+      String.format("Not support type string %s.", typeString));
   }
 
   @Override
   public String fromTypeInfo(TypeInfo<?> typeInfo) {
-    return reader.getFromTypeInformation().get(typeInfo);
+    String typeStr = reader.getFromTypeInformation().get(typeInfo);
+
+    if (StringUtils.isNotBlank(typeStr)) {
+      return typeStr;
+    }
+
+    if (typeInfo instanceof ListTypeInfo || typeInfo instanceof MapTypeInfo) {
+      String customTypeString = BitSailTypeParser.fromTypeInfo(typeInfo);
+      return this.toEngineTypeString(customTypeString);
+    }
+
+    throw BitSailException.asBitSailException(CommonErrorCode.INTERNAL_ERROR,
+      String.format("Not support typeInfo %s.", typeInfo));
+  }
+
+  private String toEngineTypeString(String customTypeString) {
+    customTypeString = customTypeString.toLowerCase();
+    for (String key : reader.customToEngineTypeStringMap.keySet()) {
+      customTypeString = customTypeString.replace(key, reader.customToEngineTypeStringMap.get(key));
+    }
+    return customTypeString;
+  }
+
+  private String toCustomTypeString(String engineTypeString) {
+    for (String key : reader.engineToCustomTypeStringMap.keySet()) {
+      engineTypeString = engineTypeString.replace(key, reader.engineToCustomTypeStringMap.get(key));
+    }
+    return engineTypeString.toLowerCase();
   }
 }

bitsail-common/src/main/java/com/bytedance/bitsail/common/type/filemapping/FileMappingTypeInfoReader.java

@@ -16,6 +16,8 @@
 
 package com.bytedance.bitsail.common.type.filemapping;
 
+import com.bytedance.bitsail.common.BitSailException;
+import com.bytedance.bitsail.common.exception.CommonErrorCode;
 import com.bytedance.bitsail.common.type.BitSailTypeParser;
 import com.bytedance.bitsail.common.typeinfo.TypeInfo;
 
@@ -31,6 +33,7 @@
 import java.io.Serializable;
 import java.net.URL;
 import java.text.MessageFormat;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -45,12 +48,19 @@ public class FileMappingTypeInfoReader implements Serializable {
   private static final String ENGINE_TO_CUSTOM_KEY = "engine.type.to.bitsail.type.converter";
   private static final String CUSTOM_TO_ENGINE_KEY = "bitsail.type.to.engine.type.converter";
 
+  private static final String ENGINE_TO_CUSTOM_COMPLEX_KEY = "engine.type.to.bitsail.complex.converter";
+  private static final String CUSTOM_TO_ENGINE_COMPLEX_KEY = "bitsail.type.to.engine.complex.converter";
+
   private final String converterFileName;
 
   @Getter
   protected Map<String, TypeInfo<?>> toTypeInformation = Maps.newHashMap();
   @Getter
   protected Map<TypeInfo<?>, String> fromTypeInformation = Maps.newHashMap();
+  @Getter
+  protected Map<String, String> engineToCustomTypeStringMap = Maps.newHashMap();
+  @Getter
+  protected Map<String, String> customToEngineTypeStringMap = Maps.newHashMap();
 
   public FileMappingTypeInfoReader(String engine) {
     LOG.info("File mapping reader from engine = {}.", engine);
@@ -99,6 +109,13 @@ private void read() throws IOException {
     readerOption(converterConf, ENGINE_TO_CUSTOM_KEY, tmpToTypeInformation);
     readerOption(converterConf, CUSTOM_TO_ENGINE_KEY, tmpFromTypeInformation);
 
+    readerOption(converterConf, ENGINE_TO_CUSTOM_COMPLEX_KEY, engineToCustomTypeStringMap);
+    readerOption(converterConf, CUSTOM_TO_ENGINE_COMPLEX_KEY, customToEngineTypeStringMap);
+    this.customTypeStringCaseCheck(engineToCustomTypeStringMap.values());
+    this.customTypeStringCaseCheck(customToEngineTypeStringMap.keySet());
+    engineToCustomTypeStringMap.putAll(tmpToTypeInformation);
+    customToEngineTypeStringMap.putAll(tmpFromTypeInformation);
+
     handleEngineTypeToCustom(tmpToTypeInformation);
     handleCustomToEngineType(tmpFromTypeInformation);
   }
@@ -128,4 +145,14 @@ protected void handleCustomToEngineType(Map<String, String> tmpFromTypeInformati
       fromTypeInformation.put(typeInfo, entry.getValue());
     }
   }
+
+  private void customTypeStringCaseCheck(Collection<String> customTypeStrings) {
+    for (String typeStr : customTypeStrings) {
+      if (!typeStr.equals(typeStr.toLowerCase())) {
+        throw BitSailException.asBitSailException(CommonErrorCode.INTERNAL_ERROR,
+          String.format("custom type string must be lowercase:%s", typeStr));
+      }
+    }
+  }
+
 }

bitsail-common/src/test/java/com/bytedance/bitsail/common/type/SimpleTypeInfoConverterTest.java

@@ -21,24 +21,67 @@
 import com.bytedance.bitsail.common.typeinfo.TypeInfo;
 
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.util.List;
 import java.util.Map;
 
 /**
  * Created 2022/8/23
  */
 public class SimpleTypeInfoConverterTest {
 
-  @Test
-  public void testSimpleTypeInfoConverter() {
-    FileMappingTypeInfoConverter fileMappingTypeInfoConverter = new FileMappingTypeInfoConverter("fake");
+  private FileMappingTypeInfoConverter fileMappingTypeInfoConverter;
+  private Map<String, TypeInfo<?>> toTypeInformation;
+  private Map<TypeInfo<?>, String> fromTypeInformation;
+
+  @Before
+  public void before() {
+    fileMappingTypeInfoConverter = new FileMappingTypeInfoConverter("fake");
     FileMappingTypeInfoReader reader = fileMappingTypeInfoConverter.getReader();
-    Map<String, TypeInfo<?>> toTypeInformation = reader.getToTypeInformation();
-    Map<TypeInfo<?>, String> fromTypeInformation = reader.getFromTypeInformation();
+    toTypeInformation = reader.getToTypeInformation();
+    fromTypeInformation = reader.getFromTypeInformation();
+  }
 
+  @Test
+  public void testSimpleTypeInfoConverter() {
     Assert.assertEquals(toTypeInformation.size(), 8);
-    Assert.assertEquals(fromTypeInformation.size(), 6);
+    Assert.assertEquals(fromTypeInformation.size(), 7);
   }
 
+  @Test
+  public void testFromTypeString_ListType() {
+    TypeInfo<?> typeInfo01 = fileMappingTypeInfoConverter.fromTypeString("Array(double)");
+    Assert.assertEquals(typeInfo01.getTypeClass(), List.class);
+
+    TypeInfo<?> typeInfo02 = fileMappingTypeInfoConverter.fromTypeString("Array(Array(double))");
+    Assert.assertEquals(typeInfo02.getTypeClass(), List.class);
+  }
+
+  @Test
+  public void testFromTypeString_MapType() {
+    TypeInfo<?> typeInfo01 = fileMappingTypeInfoConverter.fromTypeString("Map(string,double)");
+    Assert.assertEquals(typeInfo01.getTypeClass(), Map.class);
+
+    TypeInfo<?> typeInfo02 = fileMappingTypeInfoConverter.fromTypeString("Map(string, Map(string, double))");
+    Assert.assertEquals(typeInfo02.getTypeClass(), Map.class);
+  }
+
+  @Test
+  public void testFromTypeInfo_ListType() {
+    TypeInfo<?> typeInfo = fileMappingTypeInfoConverter.fromTypeString("Array(Array(Int32))");
+    Assert.assertEquals(typeInfo.getTypeClass(), List.class);
+
+    String typeString = fileMappingTypeInfoConverter.fromTypeInfo(typeInfo);
+    Assert.assertEquals(typeString, "Array(Array(Int32))");
+  }
+
+  @Test
+  public void testFromTypeInfo_MapType() {
+    TypeInfo<?> typeInfo = fileMappingTypeInfoConverter.fromTypeString("Map(string, Map(string, double))");
+
+    String typeString = fileMappingTypeInfoConverter.fromTypeInfo(typeInfo);
+    Assert.assertEquals(typeString, "Map(string, Map(string, double))");
+  }
 }

bitsail-common/src/test/resources/fake-type-converter.yaml

@@ -17,16 +17,16 @@ engine.type.to.bitsail.type.converter:
   - source.type: double
     target.type: double
 
-  - source.type: int
-    target.type: long
+  - source.type: Int32
+    target.type: int
 
   - source.type: bigint
     target.type: long
 
 
 bitsail.type.to.engine.type.converter:
   - source.type: long
-    target.type: long
+    target.type: bigint
 
   - source.type: double
     target.type: double
@@ -42,3 +42,34 @@ bitsail.type.to.engine.type.converter:
 
   - source.type: bytes
     target.type: bytes
+
+  - source.type: int
+    target.type: Int32
+
+
+engine.type.to.bitsail.complex.converter:
+  - source.type: Array
+    target.type: list
+
+  - source.type: Map
+    target.type: map
+
+  - source.type: (
+    target.type: <
+
+  - source.type: )
+    target.type: '>'
+
+bitsail.type.to.engine.complex.converter:
+  - source.type: list
+    target.type: Array
+
+  - source.type: map
+    target.type: Map
+
+  - source.type: <
+    target.type: (
+
+  - source.type: '>'
+    target.type: )
+