feat(mps-model-adapters): full support for recreating an MPS project

The bulk sync so far could only synchronize existing models, but not create new ones.

bulk-model-sync-gradle/src/main/kotlin/org/modelix/model/sync/bulk/gradle/tasks/ExportFromModelServer.kt

@@ -89,7 +89,7 @@ abstract class ExportFromModelServer : DefaultTask() {
         val nameRole = BuiltinLanguages.jetbrains_mps_lang_core.INamedConcept.name
 
         return root.allChildren.filter {
-            val isModule = it.concept?.getUID() == BuiltinLanguages.MPSRepositoryConcepts.Module.getUID()
+            val isModule = it.concept?.isSubConceptOf(BuiltinLanguages.MPSRepositoryConcepts.Module) == true
             val moduleName = it.getPropertyValue(nameRole) ?: return@filter false
             val isIncluded = isModuleIncluded(
                 moduleName,

bulk-model-sync-lib/mps-test/build.gradle.kts

@@ -16,6 +16,7 @@ dependencies {
     testImplementation(project(":bulk-model-sync-lib"))
     testImplementation(project(":bulk-model-sync-mps"))
     testImplementation(project(":mps-model-adapters"))
+    testImplementation(project(":model-datastructure"))
     testImplementation(libs.kotlin.serialization.json)
     testImplementation(libs.xmlunit.matchers)
     testImplementation(libs.jimfs)

bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test/RecreateProjectFromModelServerTest.kt

@@ -0,0 +1,265 @@
+package org.modelix.model.sync.bulk.lib.test
+
+import com.intellij.ide.impl.OpenProjectTask
+import com.intellij.openapi.Disposable
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.ProjectManager
+import com.intellij.openapi.project.ex.ProjectManagerEx
+import com.intellij.openapi.util.Disposer
+import com.intellij.testFramework.TestApplicationManager
+import com.intellij.testFramework.UsefulTestCase
+import jetbrains.mps.ide.ThreadUtils
+import jetbrains.mps.ide.project.ProjectHelper
+import jetbrains.mps.project.AbstractModule
+import jetbrains.mps.project.MPSProject
+import jetbrains.mps.smodel.Language
+import jetbrains.mps.smodel.MPSModuleRepository
+import org.jetbrains.mps.openapi.model.EditableSModel
+import org.jetbrains.mps.openapi.model.SaveOptions
+import org.modelix.model.api.PBranch
+import org.modelix.model.api.getRootNode
+import org.modelix.model.client.IdGenerator
+import org.modelix.model.data.ModelData
+import org.modelix.model.data.asData
+import org.modelix.model.lazy.CLTree
+import org.modelix.model.lazy.CLVersion
+import org.modelix.model.lazy.ObjectStoreCache
+import org.modelix.model.mpsadapters.MPSContextProject
+import org.modelix.model.mpsadapters.asReadableNode
+import org.modelix.model.mpsadapters.asWritableNode
+import org.modelix.model.persistent.MapBaseStore
+import org.modelix.model.sync.bulk.ModelSynchronizer
+import org.modelix.model.sync.bulk.NodeAssociationFromModelServer
+import org.modelix.model.sync.bulk.NodeAssociationToModelServer
+import org.modelix.mps.model.sync.bulk.MPSProjectSyncFilter
+import org.w3c.dom.Element
+import java.io.File
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.io.path.ExperimentalPathApi
+import kotlin.io.path.absolute
+import kotlin.io.path.deleteRecursively
+
+class RecreateProjectFromModelServerTest : UsefulTestCase() {
+
+    init {
+        // workaround for MPS 2023.3 failing to start in test mode
+        System.setProperty("intellij.platform.load.app.info.from.resources", "true")
+    }
+
+    protected lateinit var project: Project
+
+    override fun runInDispatchThread() = false
+
+    override fun setUp() {
+        super.setUp()
+    }
+
+    fun `test same file content`() {
+        TestApplicationManager.getInstance()
+
+        val originalProject = openTestProject("nonTrivialProject")
+        project = originalProject
+
+        val store = ObjectStoreCache(MapBaseStore()).getAsyncStore()
+        val idGenerator = IdGenerator.getInstance(0xabcd)
+        val emptyVersion = CLVersion.createRegularVersion(
+            id = idGenerator.generate(),
+            time = null,
+            author = this::class.java.name,
+            tree = CLTree.builder(store).repositoryId("unit-test-repo").build(),
+            baseVersion = null,
+            operations = emptyArray(),
+        )
+        val branch = PBranch(emptyVersion.getTree(), idGenerator)
+        mpsProject.modelAccess.runReadAction {
+            branch.runWrite {
+                val mpsRoot = mpsProject.repository.asReadableNode()
+                val modelServerRoot = branch.getRootNode().asWritableNode()
+                ModelSynchronizer(
+                    filter = MPSProjectSyncFilter(listOf(mpsProject), toMPS = false),
+                    sourceRoot = mpsRoot,
+                    targetRoot = modelServerRoot,
+                    nodeAssociation = NodeAssociationToModelServer(branch),
+                ).synchronize()
+                println(ModelData(root = modelServerRoot.asLegacyNode().asData()).toJson())
+            }
+        }
+
+        fun filterFiles(files: Map<String, String>) = files.filter {
+            val name = it.key
+            if (name.startsWith(".mps/")) {
+                false // name == ".mps/modules.xml"
+            } else if (name.contains("/source_gen") || name.contains("/classes_gen")) {
+                false
+            } else {
+                true
+            }
+        }
+
+        val originalContents = filterFiles(originalProject.captureFileContents())
+        originalProject.close()
+
+        val emptyProject = openTestProject(null)
+        project = emptyProject
+
+        mpsProject.modelAccess.executeCommandInEDT {
+            branch.runRead {
+                val mpsRoot = mpsProject.repository.asWritableNode()
+                val modelServerRoot = branch.getRootNode().asReadableNode()
+                val modelSynchronizer = ModelSynchronizer(
+                    filter = MPSProjectSyncFilter(listOf(mpsProject), toMPS = true),
+                    sourceRoot = modelServerRoot,
+                    targetRoot = mpsRoot,
+                    nodeAssociation = NodeAssociationFromModelServer(branch, mpsRoot.getModel()),
+                )
+                MPSContextProject.contextValue.computeWith(mpsProject) {
+                    modelSynchronizer.synchronize()
+                }
+            }
+        }
+
+        val syncedContents = filterFiles(emptyProject.captureFileContents())
+
+        fun Map<String, String>.contentsAsString(): String {
+            return entries.sortedBy { it.key }.joinToString("\n\n\n") { "------ ${it.key} ------\n${it.value}" }
+                .replace("""<concept id="8281020627045179518" name=""""", """<concept id="8281020627045179518" name="NewLanguage.structure.MyChild"""")
+                .replace("""<property id="8281020627045236732" name=""""", """<property id="8281020627045236732" name="value"""")
+                .replace("""<child id="8281020627045179519" name=""""", """<child id="8281020627045179519" name="children"""")
+                .replace("""<concept id="8281020627045179517" name=""""", """<concept id="8281020627045179517" name="NewLanguage.structure.MyRoot"""")
+        }
+
+        assertEquals(
+            originalContents.contentsAsString(),
+            syncedContents.contentsAsString(),
+        )
+    }
+
+    override fun tearDown() {
+        super.tearDown()
+    }
+
+    @OptIn(ExperimentalPathApi::class)
+    private fun openTestProject(testDataName: String?): Project {
+        val projectDirParent = Path.of("build", "test-projects").absolute()
+        projectDirParent.toFile().mkdirs()
+        val projectDir = Files.createTempDirectory(projectDirParent, "mps-project")
+        projectDir.deleteRecursively()
+        projectDir.toFile().mkdirs()
+        projectDir.toFile().deleteOnExit()
+        val options = OpenProjectTask().withProjectName("test-project")
+        val project = if (testDataName != null) {
+            val sourceDir = File("testdata/$testDataName")
+            sourceDir.copyRecursively(projectDir.toFile(), overwrite = true)
+            ProjectManagerEx.getInstanceEx().openProject(projectDir, options)!!
+        } else {
+            ProjectManagerEx.getInstanceEx().newProject(projectDir, options)!!
+        }
+
+        disposeOnTearDownInEdt { project.close() }
+
+        ApplicationManager.getApplication().invokeAndWait {
+            // empty - openTestProject executed not in EDT, so, invokeAndWait just forces
+            // processing of all events that were queued during project opening
+        }
+
+        return project
+    }
+
+    private fun disposeOnTearDownInEdt(runnable: Runnable) {
+        Disposer.register(
+            testRootDisposable,
+            Disposable {
+                ApplicationManager.getApplication().invokeAndWait(runnable)
+            },
+        )
+    }
+
+    protected val mpsProject: MPSProject get() {
+        return checkNotNull(ProjectHelper.fromIdeaProject(project)) { "MPS project not loaded" }
+    }
+
+    protected fun <R> writeAction(body: () -> R): R {
+        return mpsProject.modelAccess.computeWriteAction(body)
+    }
+
+    protected fun <R> writeActionOnEdt(body: () -> R): R {
+        return onEdt { writeAction { body() } }
+    }
+
+    protected fun <R> onEdt(body: () -> R): R {
+        var result: R? = null
+        ThreadUtils.runInUIThreadAndWait {
+            result = body()
+        }
+        return result as R
+    }
+
+    protected fun <R> readAction(body: () -> R): R {
+        var result: R? = null
+        mpsProject.modelAccess.runReadAction {
+            result = body()
+        }
+        return result as R
+    }
+}
+
+fun Project.close() {
+    ApplicationManager.getApplication().invokeLaterOnWriteThread {
+        runCatching {
+            ProjectManager.getInstance().closeAndDispose(this)
+        }
+    }
+    ApplicationManager.getApplication().invokeAndWait { }
+}
+
+private fun Project.captureFileContents(): Map<String, String> {
+    ApplicationManager.getApplication().invokeAndWait {
+        MPSModuleRepository.getInstance().modelAccess.runWriteAction {
+            for (module in ProjectHelper.fromIdeaProject(this)!!.projectModules.flatMap {
+                listOf(it) + ((it as? Language)?.generators ?: emptyList())
+            }) {
+                module as AbstractModule
+                module.save()
+                for (model in module.models.filterIsInstance<EditableSModel>()) {
+                    model.save(SaveOptions.FORCE)
+                }
+            }
+        }
+        ApplicationManager.getApplication().saveAll()
+        save()
+    }
+    return File(this.basePath).walk().filter { it.isFile }.associate { file ->
+        val name = file.absoluteFile.relativeTo(File(basePath).absoluteFile).path
+        val content = file.readText().trim()
+        val normalizedContent = when {
+            name.endsWith(".mps") -> normalizeModelFile(content)
+            else -> content
+        }
+        name to normalizedContent
+    }
+}
+
+private fun normalizeModelFile(content: String): String {
+    val xml = readXmlFile(content.byteInputStream())
+    xml.visitAll { node ->
+        if (node !is Element) return@visitAll
+        when (node.tagName) {
+            "node" -> {
+                node.childElements("property").sortByRole()
+                node.childElements("ref").sortByRole()
+                node.childElements("node").sortByRole()
+            }
+        }
+    }
+    return xmlToString(xml).lineSequence().map { it.trim() }.filter { it.isEmpty() }.joinToString("\n")
+}
+
+private fun List<Element>.sortByRole() {
+    if (size < 2) return
+    val sorted = sortedBy { it.getAttribute("role") }
+    for (i in (0..sorted.lastIndex - 1).reversed()) {
+        sorted[i].parentNode.insertBefore(sorted[i], sorted[i + 1])
+    }
+}

bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test/ResolveInfoUpdateTest.kt

@@ -5,7 +5,7 @@ import org.junit.Assert.assertThat
 import org.modelix.model.api.BuiltinLanguages
 import org.modelix.model.api.INode
 import org.modelix.model.data.ModelData
-import org.modelix.model.mpsadapters.MPSRepositoryAsNode
+import org.modelix.model.mpsadapters.asLegacyNode
 import org.modelix.model.sync.bulk.ExistingAndExpectedNode
 import org.modelix.model.sync.bulk.asExported
 import org.modelix.mps.model.sync.bulk.MPSBulkSynchronizer
@@ -46,7 +46,7 @@ class ResolveInfoUpdateTest : MPSTestBase() {
         var result: INode? = null
         mpsProject.repository.modelAccess.runReadAction {
             val repository = mpsProject.repository
-            val repositoryNode = MPSRepositoryAsNode(repository)
+            val repositoryNode = repository.asLegacyNode()
             result = repositoryNode.getChildren(BuiltinLanguages.MPSRepositoryConcepts.Repository.modules)
                 .single { it.getPropertyValue(BuiltinLanguages.jetbrains_mps_lang_core.INamedConcept.name) == "NewSolution" }
         }

bulk-model-sync-lib/mps-test/src/test/kotlin/org/modelix/model/sync/bulk/lib/test/XMLUtils.kt

@@ -0,0 +1,69 @@
+package org.modelix.model.sync.bulk.lib.test
+
+import org.w3c.dom.Document
+import org.w3c.dom.Element
+import org.w3c.dom.Node
+import java.io.File
+import java.io.InputStream
+import java.io.StringWriter
+import javax.xml.parsers.DocumentBuilderFactory
+import javax.xml.transform.OutputKeys
+import javax.xml.transform.Transformer
+import javax.xml.transform.TransformerFactory
+import javax.xml.transform.dom.DOMSource
+import javax.xml.transform.stream.StreamResult
+
+fun xmlToString(doc: Document): String {
+    val transformerFactory: TransformerFactory = TransformerFactory.newInstance()
+    val transformer: Transformer = transformerFactory.newTransformer()
+    transformer.setOutputProperty(OutputKeys.INDENT, "yes")
+    val source = DOMSource(doc)
+    val out = StringWriter()
+    val result = StreamResult(out)
+    transformer.transform(source, result)
+    return out.toString()
+}
+
+fun readXmlFile(file: File): Document {
+    try {
+        val dbf = DocumentBuilderFactory.newInstance()
+        // dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true)
+        disableDTD(dbf)
+        val db = dbf.newDocumentBuilder()
+        return db.parse(file)
+    } catch (e: Exception) {
+        throw RuntimeException("Failed to read ${file.absoluteFile}", e)
+    }
+}
+
+fun readXmlFile(file: InputStream, name: String? = null): Document {
+    val dbf = DocumentBuilderFactory.newInstance()
+    // dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true)
+    disableDTD(dbf)
+    val db = dbf.newDocumentBuilder()
+    return db.parse(file, name)
+}
+
+private fun disableDTD(dbf: DocumentBuilderFactory) {
+    dbf.setValidating(false)
+    dbf.setNamespaceAware(true)
+    dbf.setFeature("http://xml.org/sax/features/namespaces", false)
+    dbf.setFeature("http://xml.org/sax/features/validation", false)
+    dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false)
+    dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
+}
+
+fun Node.visitAll(visitor: (Node) -> Unit) {
+    visitor(this)
+    val childNodes = this.childNodes
+    for (i in 0 until childNodes.length) childNodes.item(i).visitAll(visitor)
+}
+
+fun Node.childElements(): List<Element> = children().filterIsInstance<Element>()
+fun Node.childElements(tag: String): List<Element> = children().filterIsInstance<Element>().filter { it.tagName == tag }
+fun Node.children(): List<Node> {
+    val children = childNodes
+    val result = ArrayList<Node>(children.length)
+    for (i in 0 until children.length) result += children.item(i)
+    return result
+}

bulk-model-sync-lib/mps-test/testdata/.gitignore

@@ -0,0 +1,5 @@
+test_gen
+test_gen.caches
+classes_gen
+source_gen
+source_gen.caches

bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/.mps/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/.mps/migration.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="MigrationProperties">
+    <entry key="project.baseline.version" value="211" />
+  </component>
+</project>

bulk-model-sync-lib/mps-test/testdata/nonTrivialProject/.mps/modules.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="MPSProject">
+    <projectModules>
+      <modulePath path="$PROJECT_DIR$/devkits/NewDevkit/NewDevkit.devkit" folder="" />
+      <modulePath path="$PROJECT_DIR$/languages/NewLanguage/NewLanguage.mpl" folder="" />
+      <modulePath path="$PROJECT_DIR$/solutions/NewRuntimeSolution/NewRuntimeSolution.msd" folder="" />
+      <modulePath path="$PROJECT_DIR$/solutions/NewSolution/NewSolution.msd" folder="" />
+    </projectModules>
+  </component>
+</project>