Skip to content

Commit

Permalink
fix(bulk-model-sync): deduplication of sync algorithm in ModelImporte…
Browse files Browse the repository at this point in the history
…r and ModelSynchronizer
  • Loading branch information
slisson committed Jan 15, 2025
1 parent 386a5a4 commit 32212a4
Show file tree
Hide file tree
Showing 8 changed files with 279 additions and 274 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import org.modelix.model.lazy.RepositoryId
import org.modelix.model.sync.bulk.asExported
import org.modelix.model.withAutoTransactions
import java.io.File
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals

@TestInstance(TestInstance.Lifecycle.PER_CLASS)
Expand All @@ -41,9 +40,9 @@ class PushTest {
val files = inputDir.listFiles()?.filter { it.extension == "json" } ?: error("no json files found in ${inputDir.absolutePath}")

val modules = files.map { ModelData.fromJson(it.readText()) }
val inputModel = ModelData(root = NodeData(children = modules.map { it.root }))
val inputModel = ModelData(root = NodeData(children = modules.map { it.root }).normalize())

assertContentEquals(inputModel.root.children, root.allChildren.map { it.asExported() })
assertEquals(inputModel.toJson(), ModelData(root = NodeData(children = root.allChildren.map { it.asExported() }).normalize()).toJson())
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package org.modelix.model.sync.bulk

import org.modelix.model.api.IReadableNode
import org.modelix.model.api.IWritableNode
import org.modelix.model.api.getOriginalOrCurrentReference

/**
* A node association is responsible for storing the mapping between a source node and the imported target node.
Expand All @@ -10,5 +11,8 @@ import org.modelix.model.api.IWritableNode
interface INodeAssociation {
fun resolveTarget(sourceNode: IReadableNode): IWritableNode?
fun associate(sourceNode: IReadableNode, targetNode: IWritableNode)
fun matches(sourceNode: IReadableNode, targetNode: IWritableNode) = resolveTarget(sourceNode) == targetNode
fun matches(sourceNode: IReadableNode, targetNode: IWritableNode): Boolean {
return sourceNode.getOriginalOrCurrentReference() == targetNode.getOriginalOrCurrentReference() ||
resolveTarget(sourceNode) == targetNode
}
}

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.modelix.model.sync.bulk

import mu.KotlinLogging
import org.modelix.model.api.IChildLinkReference
import org.modelix.model.api.INode
import org.modelix.model.api.IReadableNode
import org.modelix.model.api.IReferenceLinkReference
Expand Down Expand Up @@ -37,10 +38,21 @@ class ModelSynchronizer(
private val logger = KotlinLogging.logger {}

fun synchronize() {
synchronize(listOf(sourceRoot), listOf(targetRoot))
}

fun synchronize(sourceNodes: List<IReadableNode>, targetNodes: List<IWritableNode>) {
logger.info { "Synchronizing nodes..." }
for ((sourceNode, targetNode) in sourceNodes.zip(targetNodes)) {
synchronizeNode(sourceNode, targetNode)
}
synchronizeNode(sourceRoot, targetRoot)
logger.info { "Synchronizing pending references..." }
pendingReferences.forEach { it.trySyncReference() }
pendingReferences.forEach {
if (!it.trySyncReference()) {
logger.error { "Reference target not found: $it" }
}
}
logger.info { "Removing extra nodes..." }
nodesToRemove.filter { it.isValid() }.forEach { it.remove() }
logger.info { "Synchronization finished." }
Expand Down Expand Up @@ -101,13 +113,21 @@ class ModelSynchronizer(
}
}

private fun getFilteredSourceChildren(parent: IReadableNode, role: IChildLinkReference): List<IReadableNode> {
return parent.getChildren(role).let { filter.filterSourceChildren(parent, role, it) }
}

private fun getFilteredTargetChildren(parent: IWritableNode, role: IChildLinkReference): List<IWritableNode> {
return parent.getChildren(role).let { filter.filterTargetChildren(parent, role, it) }
}

private fun syncChildren(sourceParent: IReadableNode, targetParent: IWritableNode) {
iterateMergedRoles(
sourceParent.getAllChildren().map { it.getContainmentLink() }.distinct(),
targetParent.getAllChildren().map { it.getContainmentLink() }.distinct(),
) { role ->
val sourceNodes = sourceParent.getChildren(role)
val targetNodes = targetParent.getChildren(role)
val sourceNodes = getFilteredSourceChildren(sourceParent, role)
val targetNodes = getFilteredTargetChildren(targetParent, role)

val allExpectedNodesDoNotExist by lazy {
sourceNodes.all { sourceNode ->
Expand Down Expand Up @@ -138,7 +158,7 @@ class ModelSynchronizer(
val newlyCreatedIds = mutableSetOf<String>()

sourceNodes.forEachIndexed { indexInImport, expected ->
val existingChildren = targetParent.getChildren(role).toList()
val existingChildren = getFilteredTargetChildren(targetParent, role)
val expectedId = checkNotNull(expected.originalIdOrFallback()) { "Specified node '$expected' has no id" }
// newIndex is the index on which to import the expected child.
// It might be -1 if the child does not exist and should be added at the end.
Expand Down Expand Up @@ -186,14 +206,16 @@ class ModelSynchronizer(
val expectedNodesIds = sourceNodes.map { it.getOriginalOrCurrentReference() }.toSet()
// Do not use existingNodes, but call node.getChildren(role) because
// the recursive synchronization in the meantime already removed some nodes from node.getChildren(role).
nodesToRemove += targetParent.getChildren(role).filterNot { existingNode ->
nodesToRemove += getFilteredTargetChildren(targetParent, role).filterNot { existingNode ->
val id = existingNode.getOriginalOrCurrentReference()
expectedNodesIds.contains(id) || newlyCreatedIds.contains(id)
}
}
}

inner class PendingReference(val sourceNode: IReadableNode, val targetNode: IWritableNode, val role: IReferenceLinkReference) {
override fun toString(): String = "${sourceNode.getNodeReference()}[$role] : ${targetNode.getAllReferenceTargetRefs()}"

fun trySyncReference(): Boolean {
val expectedRef = sourceNode.getReferenceTargetRef(role)
if (expectedRef == null) {
Expand Down Expand Up @@ -260,6 +282,10 @@ class ModelSynchronizer(
* @return true iff the node must not be skipped
*/
fun needsSynchronization(node: IReadableNode): Boolean

fun filterSourceChildren(parent: IReadableNode, role: IChildLinkReference, children: List<IReadableNode>): List<IReadableNode> = children

fun filterTargetChildren(parent: IWritableNode, role: IChildLinkReference, children: List<IWritableNode>): List<IWritableNode> = children
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,10 @@ open class PNodeAdapter(val nodeId: Long, val branch: IBranch) :
return branch.transaction.getRole(nodeId)
}

override fun getContainmentLink(): IChildLink? {
return IChildLinkReference.fromUnclassifiedString(roleInParent).toLegacy()
}

override val isValid: Boolean
get() {
notifyAccess()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,15 +106,21 @@ data class NodeData(
val properties: Map<String, String> = emptyMap(),
val references: Map<String, String> = emptyMap(),
) {

fun normalize(): NodeData = copy(
children = children.map { it.normalize() },
properties = properties.entries.sortedBy { it.key }.associate { it.key to it.value },
references = references.entries.sortedBy { it.key }.associate { it.key to it.value },
)

companion object {

/**
* Users should not use this directly. Use [INode.getOriginalReference].
*/
@Deprecated("Use ID_PROPERTY_REF", replaceWith = ReplaceWith("ID_PROPERTY_REF"))
const val ID_PROPERTY_KEY = "#originalRef#"

val ID_PROPERTY_REF = IPropertyReference.fromIdAndName("#originalRef#", "#originalRef#")
val ID_PROPERTY_REF = IPropertyReference.fromIdAndName(ID_PROPERTY_KEY, ID_PROPERTY_KEY)

@Deprecated("Use ID_PROPERTY_KEY", replaceWith = ReplaceWith("ID_PROPERTY_KEY"))
const val idPropertyKey = ID_PROPERTY_KEY
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
package org.modelix.model.data

import org.modelix.model.api.ConceptReference
import org.modelix.model.api.IChildLinkReference
import org.modelix.model.api.IConcept
import org.modelix.model.api.IMutableModel
import org.modelix.model.api.INode
import org.modelix.model.api.INodeReference
import org.modelix.model.api.IPropertyReference
import org.modelix.model.api.IReferenceLinkReference
import org.modelix.model.api.IWritableNode
import org.modelix.model.api.NodeReference
import org.modelix.model.api.WritableNodeAsLegacyNode
import org.modelix.model.api.getDescendants
import org.modelix.model.api.meta.NullConcept
import org.modelix.model.api.resolve

class NodeDataAsNode(val data: NodeData, val parent: NodeDataAsNode?) : IWritableNode {
private val index: Map<String, NodeDataAsNode>? = if (parent != null) {
null
} else {
asLegacyNode().getDescendants(true)
.map { it.asWritableNode() as NodeDataAsNode }
.filter { it.data.id != null }
.associateBy { it.data.id!! }
}

private fun getIndex(): Map<String, NodeDataAsNode> = (parent?.getIndex() ?: index)!!

private fun resolveNode(ref: INodeReference): IWritableNode {
return checkNotNull(getIndex()[ref.serialize()]) { "Node not found: ${ref.serialize()}\n" + getIndex().keys }
}

override fun asLegacyNode(): INode {
return WritableNodeAsLegacyNode(this)
}

override fun getModel(): IMutableModel {
TODO("Not yet implemented")
}

override fun isValid(): Boolean {
return true
}

override fun getNodeReference(): INodeReference {
return NodeReference(checkNotNull(data.id) { "No ID specified" })
}

override fun getConcept(): IConcept {
return data.concept?.let { ConceptReference(it).resolve() } ?: NullConcept
}

override fun getConceptReference(): ConceptReference {
return data.concept?.let { ConceptReference(it) } ?: NullConcept.getReference()
}

override fun getParent(): IWritableNode? {
return parent
}

override fun getContainmentLink(): IChildLinkReference {
return IChildLinkReference.fromUnclassifiedString(data.role)
}

override fun getAllChildren(): List<IWritableNode> {
return data.children.map { NodeDataAsNode(it, this) }
}

override fun getChildren(role: IChildLinkReference): List<IWritableNode> {
return getAllChildren().filter { it.getContainmentLink().matches(role) }
}

override fun getPropertyValue(property: IPropertyReference): String? {
return data.properties.entries
.find { IPropertyReference.fromUnclassifiedString(it.key).matches(property) }
?.value
}

override fun getPropertyLinks(): List<IPropertyReference> {
return data.properties.keys.map { IPropertyReference.fromUnclassifiedString(it) }
}

override fun getAllProperties(): List<Pair<IPropertyReference, String>> {
return data.properties.map { IPropertyReference.fromUnclassifiedString(it.key) to it.value }
}

override fun getReferenceTarget(role: IReferenceLinkReference): IWritableNode? {
return getReferenceTargetRef(role)?.let { resolveNode(it) }
}

override fun getReferenceTargetRef(role: IReferenceLinkReference): INodeReference? {
return data.references.entries
.find { IReferenceLinkReference.fromUnclassifiedString(it.key).matches(role) }
?.value
?.let { NodeReference(it) }
}

override fun getReferenceLinks(): List<IReferenceLinkReference> {
return data.references.keys.map { IReferenceLinkReference.fromUnclassifiedString(it) }
}

override fun getAllReferenceTargets(): List<Pair<IReferenceLinkReference, IWritableNode>> {
return getAllReferenceTargetRefs().map { it.first to resolveNode(it.second) }
}

override fun getAllReferenceTargetRefs(): List<Pair<IReferenceLinkReference, INodeReference>> {
return data.references.map { IReferenceLinkReference.fromUnclassifiedString(it.key) to NodeReference(it.value) }
}

override fun changeConcept(newConcept: ConceptReference): IWritableNode {
throw UnsupportedOperationException("Immutable")
}

override fun setPropertyValue(property: IPropertyReference, value: String?) {
throw UnsupportedOperationException("Immutable")
}

override fun moveChild(
role: IChildLinkReference,
index: Int,
child: IWritableNode,
) {
throw UnsupportedOperationException("Immutable")
}

override fun removeChild(child: IWritableNode) {
throw UnsupportedOperationException("Immutable")
}

override fun addNewChild(
role: IChildLinkReference,
index: Int,
concept: ConceptReference,
): IWritableNode {
throw UnsupportedOperationException("Immutable")
}

override fun addNewChildren(
role: IChildLinkReference,
index: Int,
concepts: List<ConceptReference>,
): List<IWritableNode> {
throw UnsupportedOperationException("Immutable")
}

override fun setReferenceTarget(
role: IReferenceLinkReference,
target: IWritableNode?,
) {
throw UnsupportedOperationException("Immutable")
}

override fun setReferenceTargetRef(
role: IReferenceLinkReference,
target: INodeReference?,
) {
throw UnsupportedOperationException("Immutable")
}

override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is NodeDataAsNode) return false
if (data.id == null) return false
return other.data.id == data.id && parent == other.parent
}

override fun hashCode(): Int {
return data.id.hashCode() + parent.hashCode()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,13 @@ data class MPSModuleAsNode(val module: SModule) : MPSGenericNodeAdapter<MPSModul
return element.module.facets.filterIsInstance<JavaModuleFacet>()
.map { MPSJavaModuleFacetAsNode(it).asWritableNode() }
}

override fun remove(element: MPSModuleAsNode, child: IWritableNode) {
val module = element.module as AbstractModule
val moduleDescriptor = checkNotNull(module.moduleDescriptor) { "Has no moduleDescriptor: $module" }
val facet = child.asLegacyNode() as MPSJavaModuleFacetAsNode
moduleDescriptor.removeFacetDescriptor(facet.facet)
}
},
BuiltinLanguages.MPSRepositoryConcepts.Module.dependencies.toReference() to object : IChildAccessor<MPSModuleAsNode> {
override fun read(element: MPSModuleAsNode): List<IWritableNode> {
Expand All @@ -82,6 +89,13 @@ data class MPSModuleAsNode(val module: SModule) : MPSGenericNodeAdapter<MPSModul
).asWritableNode()
}
}

override fun remove(element: MPSModuleAsNode, child: IWritableNode) {
val module = element.module as AbstractModule
val moduleDescriptor = checkNotNull(module.moduleDescriptor) { "Has no moduleDescriptor: $module" }
val dependency = child.asLegacyNode() as MPSModuleDependencyAsNode
moduleDescriptor.dependencyVersions.remove(dependency.moduleReference)
}
},
BuiltinLanguages.MPSRepositoryConcepts.Module.languageDependencies.toReference() to object : IChildAccessor<MPSModuleAsNode> {
override fun read(element: MPSModuleAsNode): List<IWritableNode> {
Expand Down

0 comments on commit 32212a4

Please sign in to comment.