Implement validation

src/main/java/org/neo4j/importer/v1/ImportSpecificationDeserializer.java

@@ -16,22 +16,174 @@
  */
 package org.neo4j.importer.v1;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.MapperFeature;
 import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
+import com.networknt.schema.JsonSchema;
+import com.networknt.schema.JsonSchemaFactory;
+import com.networknt.schema.SpecVersion.VersionFlag;
 import java.io.IOException;
 import java.io.Reader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.ServiceLoader;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import org.neo4j.importer.v1.actions.Action;
+import org.neo4j.importer.v1.graph.CycleDetector;
+import org.neo4j.importer.v1.validation.InvalidSpecificationException;
+import org.neo4j.importer.v1.validation.SpecificationException;
+import org.neo4j.importer.v1.validation.SpecificationValidationResult;
+import org.neo4j.importer.v1.validation.SpecificationValidationResult.Builder;
+import org.neo4j.importer.v1.validation.SpecificationValidator;
+import org.neo4j.importer.v1.validation.UndeserializableSpecificationException;
+import org.neo4j.importer.v1.validation.UnparseableSpecificationException;
 
 public class ImportSpecificationDeserializer {
-
     private static final YAMLMapper MAPPER = YAMLMapper.builder()
             .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS)
+            .disable(MapperFeature.AUTO_DETECT_CREATORS)
             .build();
 
-    public static ImportSpecification deserialize(Reader spec) {
+    private static final JsonSchema SCHEMA = JsonSchemaFactory.getInstance(VersionFlag.V202012)
+            .getSchema(ImportSpecificationDeserializer.class.getResourceAsStream("/spec.v1.0.json"));
+
+    /**
+     * Returns an instance of {@link ImportSpecification} based on the provided {@link Reader} content.
+     * The result is guaranteed to be consistent with the specification JSON schema.
+     * <br/>
+     * If implementations of the {@link SpecificationValidator} SPI are provided, they will also run against the
+     * {@link ImportSpecification} instance before the latter is returned.
+     * <br/>
+     * If the parsing, deserialization or validation (standard or via SPI implementations) fail, a {@link SpecificationException}
+     * is going to be thrown.
+     *
+     * @return an {@link ImportSpecification}
+     * @throws SpecificationException if parsing, deserialization or validation fail
+     */
+    public static ImportSpecification deserialize(Reader spec) throws SpecificationException {
+        JsonNode json = parse(spec);
+        // TODO: pre-processing
+        validate(SCHEMA, json);
+        ImportSpecification result = deserialize(json);
+        runExtraValidations(result);
+        return result;
+    }
+
+    private static void validate(JsonSchema schema, JsonNode json) throws InvalidSpecificationException {
+        Builder builder = SpecificationValidationResult.builder();
+        schema.validate(json)
+                .forEach(msg -> builder.addError(
+                        msg.getInstanceLocation().toString(),
+                        String.format("SCHM-%s", msg.getCode()),
+                        msg.getMessage()));
+        SpecificationValidationResult result = builder.build();
+        if (!result.passes()) {
+            throw new InvalidSpecificationException(result);
+        }
+    }
+
+    private static ImportSpecification deserialize(JsonNode json) throws SpecificationException {
         try {
-            return MAPPER.readValue(spec, ImportSpecification.class);
+            return MAPPER.treeToValue(json, ImportSpecification.class);
+        } catch (JsonProcessingException e) {
+            throw new UndeserializableSpecificationException(
+                    "The payload cannot be deserialized, despite a successful schema validation.\n"
+                            + "This is likely a bug, please open an issue in "
+                            + "https://github.com/neo4j/import-spec/issues/new and share the specification that caused the issue",
+                    e);
+        }
+    }
+
+    private static void runExtraValidations(ImportSpecification spec) throws SpecificationException {
+        var validators = loadValidators();
+        var configuration =
+                spec.getConfiguration() == null ? Collections.<String, Object>emptyMap() : spec.getConfiguration();
+        validators.forEach(validator -> validator.visitConfiguration(configuration));
+        var sources = spec.getSources();
+        for (int i = 0; i < sources.size(); i++) {
+            final int index = i;
+            validators.forEach(validator -> validator.visitSource(index, sources.get(index)));
+        }
+        var targets = spec.getTargets();
+        var nodeTargets = targets.getNodes();
+        for (int i = 0; i < nodeTargets.size(); i++) {
+            final int index = i;
+            validators.forEach(validator -> validator.visitNodeTarget(index, nodeTargets.get(index)));
+        }
+        var relationshipTargets = targets.getRelationships();
+        for (int i = 0; i < relationshipTargets.size(); i++) {
+            final int index = i;
+            validators.forEach(validator -> validator.visitRelationshipTarget(index, relationshipTargets.get(index)));
+        }
+        var queryTargets = targets.getCustomQueries();
+        for (int i = 0; i < queryTargets.size(); i++) {
+            final int index = i;
+            validators.forEach(validator -> validator.visitCustomQueryTarget(index, queryTargets.get(index)));
+        }
+        var actions = spec.getActions() == null ? Collections.<Action>emptyList() : spec.getActions();
+        for (int i = 0; i < actions.size(); i++) {
+            final int index = i;
+            validators.forEach(validator -> validator.visitAction(index, actions.get(index)));
+        }
+
+        Set<Class<? extends SpecificationValidator>> failedValidations = new HashSet<>(validators.size());
+        var builder = SpecificationValidationResult.builder();
+        validators.forEach(validator -> {
+            for (Class<? extends SpecificationValidator> dependent : validator.requires()) {
+                if (failedValidations.contains(dependent)) {
+                    return;
+                }
+            }
+            if (validator.report(builder)) {
+                failedValidations.add(validator.getClass());
+            }
+        });
+        SpecificationValidationResult result = builder.build();
+        if (!result.passes()) {
+            throw new InvalidSpecificationException(result);
+        }
+    }
+
+    private static Set<SpecificationValidator> loadValidators() {
+        Set<SpecificationValidator> result = new TreeSet<>();
+        ServiceLoader.load(SpecificationValidator.class).forEach(result::add);
+        checkValidatorCycles(result);
+        return result;
+    }
+
+    private static void checkValidatorCycles(Set<SpecificationValidator> validators) {
+        var validatorDependencyGraph = new HashMap<Class<?>, Class<?>>(validators.size());
+        validators.forEach((validator) -> {
+            for (Class<? extends SpecificationValidator> dependency : validator.requires()) {
+                validatorDependencyGraph.put(validator.getClass(), dependency);
+            }
+        });
+        var cycles = CycleDetector.run(validatorDependencyGraph);
+        if (!cycles.isEmpty()) {
+            throw new IllegalStateException(String.format(
+                    "%d validator dependency cycle(s) detected:%n%s",
+                    cycles.size(), validatorCycleDescription(cycles)));
+        }
+    }
+
+    private static JsonNode parse(Reader spec) throws SpecificationException {
+        try {
+            return MAPPER.readTree(spec);
         } catch (IOException e) {
-            throw new RuntimeException(String.format("Could not deserialize spec %s", spec), e);
+            throw new UnparseableSpecificationException(e);
         }
     }
+
+    private static String validatorCycleDescription(List<List<Class<?>>> cycles) {
+        return cycles.stream()
+                .map(cycle -> cycle.stream()
+                        .map(Class::getName)
+                        .reduce("", (type1, type2) -> String.format("%s->%s", type1, type2)))
+                .collect(Collectors.joining("\n\t- ", "\t-", ""));
+    }
 }

src/main/java/org/neo4j/importer/v1/graph/CycleDetector.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) "Neo4j"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.neo4j.importer.v1.graph;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+
+public class CycleDetector {
+
+    public static <T> List<List<T>> run(Map<T, T> graph) {
+        List<List<T>> cycles = new ArrayList<>();
+        Set<T> visitedNodes = new HashSet<>();
+
+        for (T node : graph.keySet()) {
+            if (visitedNodes.contains(node)) {
+                continue;
+            }
+            List<T> path = new ArrayList<>();
+            Stack<T> stack = new Stack<>();
+            stack.push(node);
+
+            while (!stack.isEmpty()) {
+                T currentNode = stack.pop();
+                path.add(currentNode);
+                visitedNodes.add(currentNode);
+
+                T dependency = graph.get(currentNode);
+                if (dependency == null) {
+                    continue;
+                }
+                int dependencyIndex = path.indexOf(dependency);
+                if (dependencyIndex > -1) {
+                    List<T> cycle = new ArrayList<>(path.subList(dependencyIndex, path.size()));
+                    cycles.add(cycle);
+                } else if (!visitedNodes.contains(dependency)) {
+                    stack.push(dependency);
+                }
+            }
+        }
+
+        return cycles;
+    }
+}

src/main/java/org/neo4j/importer/v1/graph/Pair.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) "Neo4j"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.neo4j.importer.v1.graph;
+
+import java.util.Objects;
+
+public final class Pair<U, V> {
+
+    private final U first;
+    private final V second;
+
+    private Pair(U first, V second) {
+        this.first = first;
+        this.second = second;
+    }
+
+    public static <U, V> Pair<U, V> of(U first, V second) {
+        return new Pair<>(first, second);
+    }
+
+    public U getFirst() {
+        return first;
+    }
+
+    public V getSecond() {
+        return second;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        if (this == object) return true;
+        if (object == null || getClass() != object.getClass()) return false;
+        Pair<?, ?> pair = (Pair<?, ?>) object;
+        return Objects.equals(first, pair.first) && Objects.equals(second, pair.second);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(first, second);
+    }
+
+    @Override
+    public String toString() {
+        return "Pair{" + "first=" + first + ", second=" + second + '}';
+    }
+}

src/main/java/org/neo4j/importer/v1/sources/NamedJdbcSource.java

@@ -23,25 +23,25 @@
 public class NamedJdbcSource extends Source {
 
     private final String dataSource;
-    private final String query;
+    private final String sql;
 
     @JsonCreator
     public NamedJdbcSource(
             @JsonProperty(value = "name", required = true) String name,
             @JsonProperty(value = "data_source", required = true) String dataSource,
-            @JsonProperty(value = "query", required = true) String query) {
+            @JsonProperty(value = "sql", required = true) String sql) {
 
         super(name, SourceType.JDBC);
         this.dataSource = dataSource;
-        this.query = query;
+        this.sql = sql;
     }
 
     public String getDataSource() {
         return dataSource;
     }
 
-    public String getQuery() {
-        return query;
+    public String getSql() {
+        return sql;
     }
 
     @Override
@@ -50,19 +50,19 @@ public boolean equals(Object o) {
         if (o == null || getClass() != o.getClass()) return false;
         if (!super.equals(o)) return false;
         NamedJdbcSource that = (NamedJdbcSource) o;
-        return Objects.equals(dataSource, that.dataSource) && Objects.equals(query, that.query);
+        return Objects.equals(dataSource, that.dataSource) && Objects.equals(sql, that.sql);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), dataSource, query);
+        return Objects.hash(super.hashCode(), dataSource, sql);
     }
 
     @Override
     public String toString() {
         return "NamedJdbcSource{" + "dataSource='"
                 + dataSource + '\'' + ", query='"
-                + query + '\'' + "} "
+                + sql + '\'' + "} "
                 + super.toString();
     }
 }

src/main/java/org/neo4j/importer/v1/sources/Source.java

@@ -22,11 +22,12 @@
 import com.fasterxml.jackson.annotation.JsonTypeInfo.Id;
 import java.util.Objects;
 
-@JsonTypeInfo(use = Id.NAME, property = "type")
+@JsonTypeInfo(use = Id.DEDUCTION)
 @JsonSubTypes({
-    @Type(value = BigQuerySource.class, name = "bigquery"),
-    @Type(value = NamedJdbcSource.class, name = "jdbc"),
-    @Type(value = TextSource.class, name = "text")
+    @Type(value = BigQuerySource.class),
+    @Type(value = ExternalTextSource.class),
+    @Type(value = InlineTextSource.class),
+    @Type(value = NamedJdbcSource.class),
 })
 public abstract class Source {
 

src/main/java/org/neo4j/importer/v1/sources/TextSource.java

@@ -16,16 +16,10 @@
  */
 package org.neo4j.importer.v1.sources;
 
-import com.fasterxml.jackson.annotation.JsonSubTypes;
-import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import com.fasterxml.jackson.annotation.JsonTypeInfo.Id;
 import java.util.List;
 import java.util.Objects;
 
-@JsonTypeInfo(use = Id.DEDUCTION)
-@JsonSubTypes({@Type(value = ExternalTextSource.class), @Type(value = InlineTextSource.class)})
-abstract class TextSource extends Source {
+public abstract class TextSource extends Source {
 
     private final List<String> header;