diff --git a/pom.xml b/pom.xml index 76482a4..6640d86 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.jetbrains.research kfirst-runner - 20.0.0 + 21.0.0 kfirst-runner diff --git a/src/main/kotlin/org/jetbrains/research/runner/Main.kt b/src/main/kotlin/org/jetbrains/research/runner/Main.kt index c98ce93..207cb74 100644 --- a/src/main/kotlin/org/jetbrains/research/runner/Main.kt +++ b/src/main/kotlin/org/jetbrains/research/runner/Main.kt @@ -4,14 +4,12 @@ import com.xenomachina.argparser.ArgParser import com.xenomachina.argparser.default import org.jetbrains.research.runner.data.* import org.jetbrains.research.runner.jackson.makeMapper -import org.jetbrains.research.runner.util.ConsoleReportListener -import org.jetbrains.research.runner.util.CustomContextClassLoaderExecutor -import org.jetbrains.research.runner.util.NoExitSecurityManager -import org.jetbrains.research.runner.util.TestReportListener +import org.jetbrains.research.runner.util.* import org.junit.platform.engine.discovery.DiscoverySelectors.selectPackage import org.junit.platform.launcher.EngineFilter.includeEngines import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder import org.junit.platform.launcher.core.LauncherFactory +import org.junit.runner.JUnitCore import org.slf4j.Logger import org.slf4j.LoggerFactory import ru.spbstu.kotlin.generate.combinators.Arbitrary @@ -156,9 +154,17 @@ class KFirstRunner { } .execute(testPlan) + val zestListener = ZestTestRunListener() + val zestClasses = loadZestClasses(packages) + + //Execute ZestTests + JUnitCore().apply { addListener(zestListener) }.run(*zestClasses) + val zestTestsData = zestListener.testDatum.map { it.value }.let(::TestData) + with(testReport) { logger.info("Result: $testData") + logger.info("Zest result: $zestTestsData") val mapper = makeMapper() @@ -183,6 +189,8 @@ class KFirstRunner { } + totalTestData += zestTestsData + File(args.resultFile).writer().use { mapper.writerWithDefaultPrettyPrinter().writeValue(it, totalTestData) } diff --git a/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt b/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt index 9ffff7c..f60fea7 100644 --- a/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt +++ b/src/main/kotlin/org/jetbrains/research/runner/util/Util.kt @@ -1,9 +1,24 @@ package org.jetbrains.research.runner.util fun String.mapLines(f: (String) -> String): String = - lineSequence().map(f).joinToString("\n") + lineSequence().map(f).joinToString("\n") fun codifyString(any: Any?, indent: String): String { val result = "$any" return "```\n$result\n```".mapLines { " " + indent + it } } + +fun loadZestClasses(packages: List): Array> { + val result = ArrayList>() + val contextLoader = Thread.currentThread().contextClassLoader + val classPath = com.google.common.reflect.ClassPath.from(contextLoader) + for (pkg in packages) { + try { + val zestClass = classPath.getTopLevelClasses(pkg).filter { it.name.contains("ZestTests") }.map { contextLoader.loadClass(it.name) } + result.addAll(zestClass) + } catch (ex: ClassNotFoundException) { + + } + } + return result.toTypedArray() +} diff --git a/src/main/kotlin/org/jetbrains/research/runner/util/ZestTestRunListener.kt b/src/main/kotlin/org/jetbrains/research/runner/util/ZestTestRunListener.kt new file mode 100644 index 0000000..8c793c8 --- /dev/null +++ b/src/main/kotlin/org/jetbrains/research/runner/util/ZestTestRunListener.kt @@ -0,0 +1,57 @@ +package org.jetbrains.research.runner.util + +import common.TestFailureException +import org.jetbrains.research.runner.data.* +import org.junit.jupiter.api.Tag +import org.junit.runner.Description +import org.junit.runner.notification.Failure +import org.junit.runner.notification.RunListener +import java.util.concurrent.TimeoutException + +class ZestTestRunListener : RunListener() { + val testDatum = mutableMapOf() + + override fun testFinished(description: Description) { + val pkg = description.testClass.name + val mtd = description.displayName.split("(")[0].removeSuffix("Test") + val tags = description.annotations.filterIsInstance().map { it.value }.toSet() + val resultStatus = TestResultStatus.SUCCESSFUL + val test = TestDatum(pkg, mtd, tags, listOf(TestResult(resultStatus))) + testDatum.merge(mtd, test) { old, new -> + if (old.results.isNotEmpty() && old.results.any { + it.status in setOf( + TestResultStatus.FAILED, + TestResultStatus.ABORTED, + TestResultStatus.NOT_IMPLEMENTED + ) + }) old else new + } + } + + override fun testFailure(failure: Failure) { + val pkg = failure.description.testClass.name + val mtd = failure.description.displayName.split("(")[0].removeSuffix("Test") + val tags = failure.description.annotations.filterIsInstance().map { it.value }.toSet() + var failureDatum: FailureDatum? = null + val resultStatus = when (val ex = failure.exception) { + is NotImplementedError -> TestResultStatus.NOT_IMPLEMENTED + is TestFailureException -> { + failureDatum = + TestFailureDatum(TestInput(ex.input), ex.output, ex.expectedOutput, "${ex.inner}".take(8096)) + TestResultStatus.FAILED + } + is TimeoutException -> { + failureDatum = UnknownFailureDatum(ex.toString()) + TestResultStatus.FAILED + } + else -> { + failureDatum = UnknownFailureDatum(ex.toString()) + TestResultStatus.ABORTED + } + } + val test = TestDatum(pkg, mtd, tags, listOf(TestResult(resultStatus, failureDatum))) + testDatum.merge(mtd, test) { old, new -> + if (old.isFailure) old else new + } + } +} \ No newline at end of file