diff --git a/.gitignore b/.gitignore index 5b11d761cb..25ab286e68 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ target/ utbot-intellij/src/main/resources/settings.properties __pycache__ .dmypy.json +tbot-fuzzing/src/main/resources/* diff --git a/utbot-analytics/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/utbot-analytics/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000000..1f0955d450 --- /dev/null +++ b/utbot-analytics/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt index 06c4c9350f..20190000bf 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/CoverageApi.kt @@ -17,6 +17,10 @@ data class Instruction( val id: Long ) { val className: String get() = internalName.replace('/', '.') + + override fun toString(): String { + return "class: $className; method: $methodSignature; line number: $lineNumber" + } } /** diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index cfc15a9579..ca899ad927 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -465,13 +465,13 @@ class UtBotSymbolicEngine( if (controller.job?.isActive == false || diff <= thresholdMillisForFuzzingOperation) { logger.info { "Fuzzing overtime: $methodUnderTest" } logger.info { "Test created by fuzzer: $testEmittedByFuzzer" } - return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP) + return@runJavaFuzzing JavaFeedback(result = Trie.emptyNode(), control = Control.STOP) } if (thisInstance?.model is UtNullModel) { // We should not try to run concretely any models with null-this. // But fuzzer does generate such values, because it can fail to generate any "good" values. - return@runJavaFuzzing BaseFeedback(Trie.emptyNode(), Control.PASS) + return@runJavaFuzzing JavaFeedback(Trie.emptyNode(), Control.PASS) } val stateBefore = concreteExecutionContext.createStateBefore( @@ -494,17 +494,17 @@ class UtBotSymbolicEngine( } // in case an exception occurred from the concrete execution - concreteExecutionResult ?: return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS) + concreteExecutionResult ?: return@runJavaFuzzing JavaFeedback(result = Trie.emptyNode(), control = Control.PASS) // in case of processed failure in the concrete execution concreteExecutionResult.processedFailure()?.let { failure -> logger.debug { "Instrumented process failed with exception ${failure.exception} before concrete execution started" } - return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS) + return@runJavaFuzzing JavaFeedback(result = Trie.emptyNode(), control = Control.PASS) } if (concreteExecutionResult.violatesUtMockAssumption()) { logger.debug { "Generated test case by fuzzer violates the UtMock assumption: $concreteExecutionResult" } - return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS) + return@runJavaFuzzing JavaFeedback(result = Trie.emptyNode(), control = Control.PASS) } val result = concreteExecutionResult.result @@ -512,7 +512,7 @@ class UtBotSymbolicEngine( var trieNode: Trie.Node? = null if (coveredInstructions.isNotEmpty()) { - trieNode = descr.tracer.add(coveredInstructions) + trieNode = descr.tracer.add( coveredInstructions ) val earlierStateBeforeSize = coverageToMinStateBeforeSize[trieNode] val curStateBeforeSize = stateBefore.calculateSize() @@ -521,16 +521,16 @@ class UtBotSymbolicEngine( coverageToMinStateBeforeSize[trieNode] = curStateBeforeSize else { if (++attempts >= attemptsLimit) { - return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.STOP) + return@runJavaFuzzing JavaFeedback(result = Trie.emptyNode(), control = Control.STOP) } - return@runJavaFuzzing BaseFeedback(result = trieNode, control = Control.CONTINUE) + return@runJavaFuzzing JavaFeedback(result = trieNode, control = Control.CONTINUE) } } else { logger.error { "Coverage is empty for $methodUnderTest with $values" } if (result is UtSandboxFailure) { val stackTraceElements = result.exception.stackTrace.reversed() if (errorStackTraceTracker.add(stackTraceElements).count > 1) { - return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS) + return@runJavaFuzzing JavaFeedback(result = Trie.emptyNode(), control = Control.PASS) } } } @@ -548,7 +548,7 @@ class UtBotSymbolicEngine( ) testEmittedByFuzzer++ - BaseFeedback(result = trieNode ?: Trie.emptyNode(), control = Control.CONTINUE) + JavaFeedback(result = trieNode ?: Trie.emptyNode(), control = Control.CONTINUE) } } diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt index a33cba2ea0..c880a1ee0e 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt @@ -5,11 +5,16 @@ import kotlinx.coroutines.* import mu.KotlinLogging import org.utbot.fuzzing.seeds.KnownValue import org.utbot.fuzzing.utils.MissedSeed -import org.utbot.fuzzing.utils.chooseOne import org.utbot.fuzzing.utils.flipCoin import org.utbot.fuzzing.utils.transformIfNotEmpty +import java.io.File +import java.io.FileOutputStream +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import kotlin.properties.Delegates import kotlin.random.Random + private val logger by lazy { KotlinLogging.logger {} } /** @@ -19,7 +24,7 @@ private val logger by lazy { KotlinLogging.logger {} } * @see [org.utbot.fuzzing.demo.JavaFuzzing] * @see [org.utbot.fuzzing.demo.JsonFuzzingKt] */ -interface Fuzzing, FEEDBACK : Feedback> { +interface Fuzzing, FEEDBACK : Feedback> { /** * Before producing seeds, this method is called to recognize, @@ -59,7 +64,7 @@ interface Fuzzing, FEEDBACK : Feed * Starts fuzzing with new description but with copy of [Statistic]. */ suspend fun fork(description: DESCRIPTION, statistics: Statistic) { - fuzz(description, StatisticImpl(statistics)) + fuzz(description, MainStatisticImpl(statistics)) } /** @@ -73,19 +78,145 @@ interface Fuzzing, FEEDBACK : Feed suspend fun afterIteration(description: DESCRIPTION, statistics: Statistic) { } } +///region Description /** * Some description of the current fuzzing run. Usually, it contains the name of the target method and its parameter list. */ -open class Description( +open class Description( parameters: List ) { val parameters: List = parameters.toList() - open fun clone(scope: Scope): Description { + open fun clone(scope: Scope): Description { error("Scope was changed for $this, but method clone is not specified") } + + open fun setUp() {} + + open fun beforeIteration() {} + open fun beforeRun() {} + open fun afterRun() {} + + open suspend fun afterIteration(values: Node, feedback: Feedback, event: MinsetEvent) {} +} + +abstract class ReportingDescription ( + parameters: List, + private val reporter: Reporter +) : Description(parameters) { + + override fun setUp() { + reporter.setUp(this) + } + + override fun beforeIteration() { + reporter.beforeIteration() + } + + override fun beforeRun() { + reporter.beforeRun() + } + + override fun afterRun() { + reporter.afterRun() + } + + override suspend fun afterIteration(values : Node, feedback : Feedback, event: MinsetEvent) { + reporter.afterIteration(values, feedback, event) + } } +open class LoggingDescription ( + parameters: List, + path: String +) : ReportingDescription( + parameters, + LoggingReporter(path = path) +) + + +///region Reporter +abstract class Reporter( + private val reportPath : String, +) { + abstract fun clone() : Reporter + + private lateinit var description: Description + fun setUp(description: Description) { + this.description = description + } + open fun beforeIteration() {} + open fun beforeRun() {} + open fun afterRun() {} + open fun afterIteration(values: Node, feedback: Feedback, event : MinsetEvent) {} +} + + +class LoggingReporter( + val path: String +) : Reporter(path) { + + private var iterationStartTime by Delegates.notNull() + private var executionStartTime by Delegates.notNull() + private var executionFinishTime by Delegates.notNull() + + override fun clone() : LoggingReporter { + return LoggingReporter(path) + } + + private val actualPath = if (path.startsWith("~/")) { + System.getProperty("user.home") + path.drop(1) + } else { + path + } + + private val logWriter = FileOutputStream(File("$actualPath/log"), true).bufferedWriter() + + init { + File(actualPath).mkdirs() + } + + override fun beforeIteration() { + iterationStartTime = System.nanoTime() + } + + override fun beforeRun() { + executionStartTime = System.nanoTime() + } + + override fun afterRun() { + executionFinishTime = System.nanoTime() + } + + override fun afterIteration(values: Node, feedback: Feedback, event : MinsetEvent) { + val time = LocalDateTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss:SSS")) + + val printableValues = buildString { + values.toString() + .toByteArray(Charsets.UTF_8) + .forEach { byte -> + val char = byte.toInt().toChar() + if (char.isDefined() && char != '\t' && char != '\n' && char != '"') { + append(char) + } else { + val hexString = byte.toInt().toString(16).padStart(2, '0') + append("\\u").append(hexString) + } + } + } + + val iterationDuration = (System.nanoTime() - iterationStartTime).toFloat() / 1_000_000 + val executionDuration = (executionFinishTime - executionStartTime).toFloat() / 1_000_000 + + logWriter.write("$time\t${"%.3f".format(iterationDuration)}\t${"%.3f".format(executionDuration)}\t$feedback\t$event\t$printableValues\n") + logWriter.flush() + } +} +///endregion +///endregion + + + class Scope( val parameterIndex: Int, val recursionDepth: Int, @@ -234,6 +365,7 @@ interface Feedback : AsKey { * @see [Control] */ val control: Control + var runDuration: Long? } /** @@ -245,7 +377,34 @@ interface Feedback : AsKey { data class BaseFeedback( val result: VALUE, override val control: Control, -) : Feedback +) : Feedback { + override var runDuration: Long? = null + + override fun toString(): String { + return "$result" + } +} + +interface WeightedFeedback : Feedback { + val weight: Int +} + +data class BaseWeightedFeedback( + val result: VALUE, + override val weight: Int, + override val control: Control, +) : WeightedFeedback, Comparable> { + override fun compareTo(other: WeightedFeedback): Int { + return weight.compareTo(other.weight) + } + + override var runDuration: Long? = null + + override fun toString(): String { + return "$result with weight $weight" + } +} + /** * Controls fuzzing execution. @@ -276,6 +435,7 @@ fun emptyFeedback(): Feedback = (EmptyFeedback as Feedback) private object EmptyFeedback : Feedback { override val control: Control get() = Control.CONTINUE + override var runDuration: Long? = null override fun equals(other: Any?): Boolean { return true @@ -284,6 +444,10 @@ private object EmptyFeedback : Feedback { override fun hashCode(): Int { return 0 } + + override fun toString(): String { + return "EMPTY_FEEDBACK" + } } class NoSeedValueException internal constructor( @@ -298,12 +462,12 @@ class NoSeedValueException internal constructor( get() = "No seed candidates generated for type: $type" } -suspend fun , F : Feedback> Fuzzing.fuzz( +suspend fun , F : Feedback> Fuzzing.fuzz( description: D, random: Random = Random(0), configuration: Configuration = Configuration() ) { - fuzz(description, StatisticImpl(random = random, configuration = configuration)) + fuzz(description, MainStatisticImpl(random = random, configuration = configuration)) } /** @@ -311,9 +475,9 @@ suspend fun , F : Feedback> Fuzzing.f * * This is an entry point for every fuzzing. */ -private suspend fun , F : Feedback> Fuzzing.fuzz( +private suspend fun , F : Feedback> Fuzzing.fuzz( description: D, - statistic: StatisticImpl, + statistic: SeedsMaintainingStatistic, ) { val random = statistic.random val configuration = statistic.configuration @@ -330,49 +494,68 @@ private suspend fun , F : Feedback> Fuzzing 0) { + statistic.getSeed(random, configuration).let { + mutationFactory.mutate(it, random, configuration, statistic) + } + } else { + if (statistic.isNotEmpty() && random.flipCoin(configuration.probSeedRetrievingInsteadGenerating)) { + statistic.getSeed(random, configuration).let { + mutationFactory.mutate(it, random, configuration, statistic) + } } else { - it + val actualParameters = description.parameters + // fuzz one value, seems to be bad, when have only a few and simple values + fuzzOne(actualParameters).let { + if (random.flipCoin(configuration.probMutationRate)) { + mutationFactory.mutate(it, random, configuration, statistic) + } else { + it + } + } } - } + } + afterIteration(description, statistic) - yield() - statistic.apply { - totalRuns++ - } check(values.parameters.size == values.result.size) { "Cannot create value for ${values.parameters}" } val valuesCache = mutableMapOf, R>() val result = values.result.map { valuesCache.computeIfAbsent(it) { r -> create(r) } } + + description.beforeRun() + + val timeBeforeHandle = System.nanoTime() val feedback = fuzzing.handle(description, result) - when (feedback.control) { - Control.CONTINUE -> { - statistic.put(random, configuration, feedback, values) - } - Control.STOP -> { - break - } - Control.PASS -> {} + feedback.runDuration = (System.nanoTime() - timeBeforeHandle) / 100 + + description.afterRun() + + val minsetResponse = statistic.put(random, configuration, feedback, values) + + yield() + statistic.apply { + totalRuns++ } - } -} + description.afterIteration(values, feedback, minsetResponse) + if (feedback.control == Control.STOP) { + break + } + } +} ///region Implementation of the fuzzing and non-public functions. -private fun , FEEDBACK : Feedback> fuzz( +private fun , FEEDBACK : Feedback> fuzz( parameters: List, fuzzing: Fuzzing, description: DESCRIPTION, @@ -399,7 +582,7 @@ private fun , FEEDBACK : Feedback< return Node(result, parameters, builder) } -private fun , FEEDBACK : Feedback> produce( +private fun , FEEDBACK : Feedback> produce( type: TYPE, fuzzing: Fuzzing, description: DESCRIPTION, @@ -436,7 +619,7 @@ private fun , FEEDBACK : Feedback< * reduces [Seed.Collection] type. When `configuration.recursionTreeDepth` limit is reached it creates * an empty collection and doesn't do any modification to it. */ -private fun , FEEDBACK : Feedback> reduce( +private fun , FEEDBACK : Feedback> reduce( task: Seed.Collection, fuzzing: Fuzzing, description: DESCRIPTION, @@ -496,7 +679,7 @@ private fun , FEEDBACK : Feedback< * reduces [Seed.Recursive] type. When `configuration.recursionTreeDepth` limit is reached it calls * `Seed.Recursive#empty` routine to create an empty object. */ -private fun , FEEDBACK : Feedback> reduce( +private fun , FEEDBACK : Feedback> reduce( task: Seed.Recursive, fuzzing: Fuzzing, description: DESCRIPTION, @@ -652,13 +835,18 @@ sealed interface Result { /** * Known value. */ - class Known>(val value: V, val build: (V) -> RESULT) : Result + class Known>(val value: V, val build: (V) -> RESULT, val lastMutation: Mutation? = null) : Result { + override fun toString(): String { + return this.value.toString() + } + } /** * A tree of object that has constructor and some modifications. */ class Recursive( val construct: Node, val modify: List>, + val lastMutation: RecursiveMutations? = null, ) : Result /** @@ -668,6 +856,7 @@ sealed interface Result { val construct: Node, val modify: List>, val iterations: Int, + val lastMutation: CollectionMutations? = null, ) : Result /** @@ -684,52 +873,33 @@ sealed interface Result { class Node( val result: List>, val parameters: List, - val builder: Routine, -) - -private class StatisticImpl>( - override var totalRuns: Long = 0, - override val startTime: Long = System.nanoTime(), - override var missedTypes: MissedSeed = MissedSeed(), - override val random: Random, - override val configuration: Configuration, -) : Statistic { - - constructor(source: Statistic) : this( - totalRuns = source.totalRuns, - startTime = source.startTime, - missedTypes = source.missedTypes, - random = source.random, - configuration = source.configuration.copy(), - ) - - override val elapsedTime: Long - get() = System.nanoTime() - startTime - private val seeds = linkedMapOf>() - private val count = linkedMapOf() - - fun put(random: Random, configuration: Configuration, feedback: FEEDBACK, seed: Node) { - if (random.flipCoin(configuration.probUpdateSeedInsteadOfKeepOld)) { - seeds[feedback] = seed - } else { - seeds.putIfAbsent(feedback, seed) - } - count[feedback] = count.getOrDefault(feedback, 0L) + 1L - } - - fun getRandomSeed(random: Random, configuration: Configuration): Node { - if (seeds.isEmpty()) error("Call `isNotEmpty` before getting the seed") - val entries = seeds.entries.toList() - val frequencies = DoubleArray(seeds.size).also { f -> - entries.forEachIndexed { index, (key, _) -> - f[index] = configuration.energyFunction(count.getOrDefault(key, 0L)) + val builder: Routine +) { + override fun toString() : String { + return result.map { + when(it) { + is Result.Empty -> "_" + is Result.Simple -> it.result + is Result.Known<*, *, *> -> { + if (it.lastMutation != null) { + "${it.value} : ${it.lastMutation.toString().substringAfter("$").substringBefore('@')}" + } else { it.value.toString() } + } + is Result.Collection -> { + if (it.lastMutation != null) { + "${it.modify.joinToString(", ", "[", "]")} : ${it.lastMutation.toString().substringAfter("$").substringBefore('@')}" + } else { it.modify.joinToString(", ", "[", "]") } + } + is Result.Recursive -> { + if (it.lastMutation != null) { + "${it.modify.joinToString(", ", "{", "}")} : ${it.lastMutation.toString().substringAfter("$").substringBefore('@')}" + } else { + it.modify.joinToString(", ", "{", "}") + } + } } - } - val index = random.chooseOne(frequencies) - return entries[index].value + }.joinToString(", ") } - - fun isNotEmpty() = seeds.isNotEmpty() } ///endregion @@ -740,4 +910,4 @@ private fun > Seed.Known.asResu val value: T = value as T return Result.Known(value, build as KnownValue.() -> RESULT) } -///endregion +///endregion \ No newline at end of file diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt index a6c19926c9..8c13105c84 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Configuration.kt @@ -7,6 +7,37 @@ import kotlin.math.pow */ data class Configuration( + var tuneKnownValueMutations: Boolean = true, + var tuneCollectionMutations: Boolean = true, + var tuneRecursiveMutations: Boolean = true, + + /** + * When true, fuzzer selects one value and runs it for [runsPerValue] times, first [investigationPeriodPerValue] of which + * are investigational. When false, fuzzer investigates mutations efficiencies for [globalInvestigationPeriod] runs and + * tunes it for next [globalExploitationPeriod] runs, then drops statistics and starts over. + */ + var rotateValues: Boolean = false, + + /** + * Number of iterations before mutations probabilities correction when [rotateValues] is false. + */ + var globalInvestigationPeriod: Long = 100, + + /** + * Number of iterations before dropping statistics when [rotateValues] is false. + */ + var globalExploitationPeriod: Long = 900, + + /** + * Number of continuous iterations for each value when [rotateValues] is true. + */ + var runsPerValue: Long = 500, + + /** + * Number of iterations before mutations probabilities correction when [rotateValues] is true. + */ + var investigationPeriodPerValue: Int = 250, + /** * Choose between already generated values and new generation of values. */ @@ -32,7 +63,17 @@ data class Configuration( /** * Energy function that is used to choose seeded value. */ - var energyFunction: (x: Long) -> Double = { x -> 1 / x.coerceAtLeast(1L).toDouble().pow(2) }, + var energyFunction: (feedbackCount: Long, runDuration: Long) -> Double = { + // x, _ -> 1 / x.coerceAtLeast(1L).toDouble().pow(2) + feedbackCount, runDuration -> 1000 / + feedbackCount.coerceAtLeast(1L).toDouble().pow(2) / + runDuration.coerceAtLeast(1L).toDouble().pow(2) + }, + + /** + * Rating function that is used to manipulate mutations probabilities while tuning. + */ + var mutationRatingFunction: (successProbability: Double) -> Double = { successProbability -> successProbability.pow(2) }, /** * Probability to prefer shuffling collection instead of mutation one value from modification @@ -95,4 +136,5 @@ data class Configuration( * Limits maximum number of recursive seed modifications */ var maxNumberOfRecursiveSeedModifications: Int = 10, -) \ No newline at end of file +) + diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Mutations.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Mutations.kt index 9868be39b8..a92e885fb8 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Mutations.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Mutations.kt @@ -7,46 +7,104 @@ import org.utbot.fuzzing.utils.chooseOne import org.utbot.fuzzing.utils.flipCoin import kotlin.random.Random -class MutationFactory { - fun mutate(node: Node, random: Random, configuration: Configuration): Node { +class MutationFactory { + fun mutate( + node: Node, + random: Random, + configuration: Configuration, + statistic: SeedsMaintainingStatistic + ): Node { if (node.result.isEmpty()) return node val indexOfMutatedResult = random.chooseOne(node.result.map(::rate).toDoubleArray()) val recursive: NodeMutation = NodeMutation { n, r, c -> - mutate(n, r, c) + mutate(n, r, c, statistic) } + + // Turn on mutations tuning in two cases: + // - if value rotation is turned on, and it's time according to runsPerValue and investigationPeriodPerValue + // - if value rotation is turned off, and it's time according to globalInvestigationPeriod and globalExploitationPeriod + val tuningMode = + configuration.rotateValues && + statistic.totalRuns % configuration.runsPerValue > configuration.investigationPeriodPerValue || + !configuration.rotateValues && + statistic.totalRuns % (configuration.globalInvestigationPeriod + configuration.globalExploitationPeriod) > configuration.globalInvestigationPeriod + + val mutated = when (val resultToMutate = node.result[indexOfMutatedResult]) { - is Result.Simple -> Result.Simple(resultToMutate.mutation(resultToMutate.result, random), resultToMutate.mutation) + is Result.Simple -> Result.Simple( + resultToMutate.mutation(resultToMutate.result, random), + resultToMutate.mutation + ) is Result.Known -> { val mutations = resultToMutate.value.mutations() + + val mutationsEfficiencies = statistic.getMutationsRatings(configuration) + .filter { (k, _) -> mutations.contains(k) } as Map>, Double> + + val mutation = if (tuningMode && + configuration.tuneKnownValueMutations && + mutationsEfficiencies.isNotEmpty() && mutationsEfficiencies.values.sum() > 0 + ) { + mutationsEfficiencies.keys.toTypedArray()[random.chooseOne(mutationsEfficiencies.values.toDoubleArray())] + } else { + mutations.random(random) + } + if (mutations.isNotEmpty()) { - resultToMutate.mutate(mutations.random(random), random, configuration) + resultToMutate.mutate(mutation, random, configuration) } else { resultToMutate } } is Result.Recursive -> { - when { - resultToMutate.modify.isEmpty() || random.flipCoin(configuration.probConstructorMutationInsteadModificationMutation) -> - RecursiveMutations.Constructor() - random.flipCoin(configuration.probShuffleAndCutRecursiveObjectModificationMutation) -> - RecursiveMutations.ShuffleAndCutModifications() - else -> - RecursiveMutations.Mutate() - }.mutate(resultToMutate, recursive, random, configuration) + val mutationsEfficiencies = statistic.getMutationsRatings(configuration) + .filter { (k, _) -> k is RecursiveMutations<*, *> } as Map, Double> + + val mutation = if (tuningMode && configuration.tuneRecursiveMutations && + mutationsEfficiencies.isNotEmpty() && mutationsEfficiencies.values.sum() > 0 + ) { + mutationsEfficiencies.keys.toTypedArray()[random.chooseOne(mutationsEfficiencies.values.toDoubleArray())] + } else { + when { + resultToMutate.modify.isEmpty() || random.flipCoin(configuration.probConstructorMutationInsteadModificationMutation) -> + RecursiveMutations.Constructor() + + random.flipCoin(configuration.probShuffleAndCutRecursiveObjectModificationMutation) -> + RecursiveMutations.ShuffleAndCutModifications() + + else -> RecursiveMutations.Mutate() + } + } + + mutation.mutate(resultToMutate, recursive, random, configuration) } is Result.Collection -> if (resultToMutate.modify.isNotEmpty()) { - when { - random.flipCoin(100 - configuration.probCollectionShuffleInsteadResultMutation) -> - CollectionMutations.Mutate() - else -> - CollectionMutations.Shuffle() - }.mutate(resultToMutate, recursive, random, configuration) + val mutationsEfficiencies = statistic.getMutationsRatings(configuration) + .filter { (k, _) -> k is CollectionMutations<*, *> } as Map, Double> + + val mutation = if (tuningMode && configuration.tuneCollectionMutations && + mutationsEfficiencies.isNotEmpty() && mutationsEfficiencies.values.sum() > 0 + ) { + mutationsEfficiencies.keys.toList()[random.chooseOne(mutationsEfficiencies.values.toDoubleArray())] + } else { + when { + random.flipCoin(100 - configuration.probCollectionShuffleInsteadResultMutation) -> + CollectionMutations.Mutate() + + else -> + CollectionMutations.Shuffle() + } + } + + mutation.mutate(resultToMutate, recursive, random, configuration) } else { resultToMutate } is Result.Empty -> resultToMutate } + + return Node(node.result.toMutableList().apply { set(indexOfMutatedResult, mutated) }, node.parameters, node.builder) @@ -82,12 +140,17 @@ class MutationFactory { } @Suppress("UNCHECKED_CAST") - private fun > Result.Known.mutate(mutation: Mutation, random: Random, configuration: Configuration): Result.Known { + private fun > Result.Known.mutate( + mutation: Mutation, + random: Random, + configuration: Configuration + ): Result.Known { val source: T = value as T val mutate = mutation.mutate(source, random, configuration) return Result.Known( mutate, - build as (T) -> RESULT + build as (T) -> RESULT, + lastMutation = mutation ) } } @@ -189,6 +252,10 @@ sealed interface StringMutations : Mutation { } return StringValue(newString, lastMutation = this, mutatedFrom = source) } + + override fun equals(other: Any?): Boolean { + return this.javaClass == other?.javaClass + } } object RemoveCharacter : StringMutations { @@ -203,6 +270,10 @@ sealed interface StringMutations : Mutation { } return StringValue(newString, this) } + + override fun equals(other: Any?): Boolean { + return this.javaClass == other?.javaClass + } } object ShuffleCharacters : StringMutations { @@ -212,6 +283,10 @@ sealed interface StringMutations : Mutation { lastMutation = this ) } + + override fun equals(other: Any?): Boolean { + return this.javaClass == other?.javaClass + } } } @@ -244,9 +319,18 @@ sealed interface CollectionMutations : Mutation : CollectionMutations { @@ -262,9 +346,18 @@ sealed interface CollectionMutations : Mutation : Mutation { return Result.Recursive( construct = recursive.mutate(source.construct,random, configuration), - modify = source.modify + modify = source.modify, + lastMutation = this, ) } + + override fun equals(other: Any?): Boolean { + return this.javaClass == other?.javaClass + } + + override fun hashCode(): Int { + return javaClass.hashCode() + } } class ShuffleAndCutModifications : RecursiveMutations { @@ -309,9 +411,18 @@ sealed interface RecursiveMutations : Mutation { return Result.Recursive( construct = source.construct, - modify = source.modify.shuffled(random).take(random.nextInt(source.modify.size + 1)) + modify = source.modify.shuffled(random).take(random.nextInt(source.modify.size + 1)), + lastMutation = this, ) } + + override fun equals(other: Any?): Boolean { + return this.javaClass == other?.javaClass + } + + override fun hashCode(): Int { + return javaClass.hashCode() + } } class Mutate : RecursiveMutations { @@ -326,9 +437,17 @@ sealed interface RecursiveMutations : Mutation, FEEDBACK : Feedback> runFuzzing( +suspend fun , FEEDBACK : Feedback> runFuzzing( provider: ValueProvider, description: DESCRIPTION, random: Random = Random(0), @@ -24,7 +24,7 @@ suspend fun , FEEDBACK : Feedback< * @param providers is a list of "type to values" generator * @param exec this function is called when fuzzer generates values of type R to run it with target program. */ -class BaseFuzzing, F : Feedback>( +class BaseFuzzing, F : Feedback>( val providers: List>, val exec: suspend (description: D, values: List) -> F ) : Fuzzing { @@ -60,7 +60,7 @@ class BaseFuzzing, F : Feedback>( /** * Value provider generates [Seed] and has other methods to combine providers. */ -fun interface ValueProvider> { +fun interface ValueProvider> { fun enrich(description: D, type: T, scope: Scope) {} @@ -144,7 +144,7 @@ fun interface ValueProvider> { return if (this is Fallback) { provider.unwrapIfFallback() } else { this } } - private class Fallback>( + private class Fallback>( val provider: ValueProvider, val fallback: ValueProvider, ): ValueProvider { @@ -172,7 +172,7 @@ fun interface ValueProvider> { /** * Wrapper class that delegates implementation to the [providers]. */ - private class Combined>(providers: List>): ValueProvider { + private class Combined>(providers: List>): ValueProvider { val providers: List> init { @@ -203,7 +203,7 @@ fun interface ValueProvider> { } companion object { - fun > of(valueProviders: List>): ValueProvider { + fun > of(valueProviders: List>): ValueProvider { return Combined(valueProviders) } } @@ -215,7 +215,7 @@ fun interface ValueProvider> { * @param type that is used as a filter to call this provider * @param generate yields values for the type */ -class TypeProvider>( +class TypeProvider>( val type: T, val generate: suspend SequenceScope>.(description: D, type: T) -> Unit ) : ValueProvider { diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Statistic.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Statistic.kt index 877937cedb..5f382cd714 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Statistic.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Statistic.kt @@ -1,6 +1,7 @@ package org.utbot.fuzzing import org.utbot.fuzzing.utils.MissedSeed +import org.utbot.fuzzing.utils.chooseOne import kotlin.random.Random /** @@ -13,4 +14,285 @@ interface Statistic { val missedTypes: MissedSeed val random: Random val configuration: Configuration -} \ No newline at end of file +} + +///region Statistic Implementations + +/** + * Interface of statistic object with seeds maintaining logic (seed selection, value storaging, feedbacks counting, etc.) + */ +interface SeedsMaintainingStatistic>: Statistic { + override var totalRuns: Long + var lastNewFeedbackIter: Long + fun put(random: Random, configuration: Configuration, feedback: FEEDBACK, seed: Node) : MinsetEvent + fun getSeed(random: Random, configuration: Configuration): Node + fun getMutationsRatings(configuration: Configuration): Map, Double> + fun size() : Int + fun isEmpty() : Boolean { return size() == 0 } + fun isNotEmpty() : Boolean { return size() > 0 } + fun dropMutationsStats() +} + + +/** + * Main implementation of [SeedsMaintainingStatistic]. Implements + * mutations efficiencies evaluating algorithm used for mutation probability tuning (see [MainStatisticImpl.put]) + * and seed selection algorithm based on feedbacks counting (see [MainStatisticImpl.getSeed]). + */ +open class MainStatisticImpl> ( + override var totalRuns: Long = 0, + override val startTime: Long = System.nanoTime(), + override var missedTypes: MissedSeed = MissedSeed(), + override val random: Random, + override val configuration: Configuration, + override var lastNewFeedbackIter: Long = 0 +): SeedsMaintainingStatistic { + constructor(source: Statistic) : this( + totalRuns = source.totalRuns, + startTime = source.startTime, + missedTypes = source.missedTypes, + random = source.random, + configuration = source.configuration.copy() + ) + + override val elapsedTime: Long + get() = System.nanoTime() - startTime + + private val minset: Minset> = Minset( { SingleValueStorage() } ) + + private var currentValue: Node? = null + + val mutationsCounts = mutableMapOf, Long>() + val mutationsSuccessCounts = mutableMapOf, Double>() + + + override fun put(random: Random, configuration: Configuration, feedback: FEEDBACK, seed: Node): MinsetEvent { + val event = minset.put(seed, feedback) + + if (event == MinsetEvent.NEW_FEEDBACK) { + lastNewFeedbackIter = totalRuns + } + + seed.result.forEach { result -> + when (result) { + is Result.Known -> { + result.lastMutation?.let { mutation -> + mutationsCounts[mutation] = mutationsCounts.getOrDefault(mutation, 0) + 1 + mutationsSuccessCounts[mutation] = mutationsSuccessCounts.getOrDefault(mutation, 0.0) + + if (event == MinsetEvent.NEW_FEEDBACK) 1.0 else 0.0 + } + } + + is Result.Collection -> { + result.lastMutation?.let { mutation -> + mutationsCounts[mutation] = mutationsCounts.getOrDefault(mutation, 0) + 1 + mutationsSuccessCounts[mutation] = mutationsSuccessCounts.getOrDefault(mutation, 0.0) + + if (event == MinsetEvent.NEW_FEEDBACK) 1.0 else 0.0 + } + } + + is Result.Recursive -> { + result.lastMutation?.let { mutation -> + mutationsCounts[mutation] = mutationsCounts.getOrDefault(mutation, 0) + 1 + mutationsSuccessCounts[mutation] = mutationsSuccessCounts.getOrDefault(mutation, 0.0) + + if (event == MinsetEvent.NEW_FEEDBACK) 1.0 else 0.0 + } + } + + else -> {} + } + } + + return event + } + + + override fun getSeed(random: Random, configuration: Configuration): Node { + if (minset.isEmpty()) error("Call `isNotEmpty` before getting the seed") + + if (configuration.rotateValues && totalRuns % configuration.runsPerValue > 0 && currentValue != null) { + return currentValue!! + } else { + currentValue = minset.getNextSeed(random, configuration.energyFunction) + + if ( + configuration.rotateValues && totalRuns % configuration.runsPerValue == 0L || + !configuration.rotateValues && totalRuns % (configuration.globalInvestigationPeriod + configuration.globalExploitationPeriod) == 0L + ) { + dropMutationsStats() + } + } + + return currentValue!! + } + + override fun getMutationsRatings(configuration: Configuration): Map, Double> { + val ratings = mutationsCounts.mapValues { (key, _) -> + configuration.mutationRatingFunction( + ((mutationsSuccessCounts[key] ?: 0.01) / mutationsSuccessCounts.values.sum()) + ) + } + + return ratings + } + + override fun size(): Int { + return minset.size() + } + + override fun dropMutationsStats() { + mutationsCounts.clear() + mutationsSuccessCounts.clear() + } +} +///endregion + +///region Minset + +/** + * Possible answers given by [Minset] after putting a value in it with [Minset.put]: + * - [NEW_FEEDBACK]: new unique feedback was found; + * - [NEW_VALUE]: feedback is already in minset, and value stored for it was updated; + * - [NOTHING_NEW]: feedback is already in minset and the value was not updated either. + * + * According to current implementation, [Minset] figures out if feedback is new, + * but decision between [NEW_VALUE] and [NOTHING_NEW] is making by [ValueStorage]. + */ +enum class MinsetEvent { NEW_FEEDBACK, NEW_VALUE, NOTHING_NEW } + +/** + * Storage for feedbacks mapped on the values that led to them. + */ +open class Minset, STORAGE : ValueStorage> ( + open val valueStorageGenerator: () -> STORAGE, + val seeds: LinkedHashMap = linkedMapOf(), + val count: LinkedHashMap = linkedMapOf() +) { + fun getNextSeed(random: Random, energyFunction: (feedbackCount: Long, runDuration: Long) -> Double): Node { + val entries = seeds.entries.toList() + + val energy = DoubleArray(size()).also { f -> + entries.forEachIndexed { index, (key, _) -> + f[index] = energyFunction(count.getOrDefault(key, 0L), key.runDuration ?: 0) + } + } + + return entries[random.chooseOne(energy)].value.next() + } + + operator fun get(feedback: FEEDBACK): STORAGE? { + return seeds[feedback] + } + + open fun put(value: Node, feedback: FEEDBACK) : MinsetEvent { + val result: MinsetEvent + + if (seeds.containsKey(feedback)) { + result = if( seeds[feedback]!!.put(value, feedback) ) { MinsetEvent.NEW_VALUE } else { MinsetEvent.NOTHING_NEW } + } else { + result = MinsetEvent.NEW_FEEDBACK + seeds[feedback] = valueStorageGenerator.invoke() + seeds[feedback]!!.put(value, feedback) + } + + count[feedback] = count.getOrDefault(feedback, 0L) + 1L + + return result + } + + fun isNotEmpty(): Boolean { + return seeds.isNotEmpty() + } + + fun isEmpty(): Boolean { + return seeds.isEmpty() + } + + fun size() : Int { + return seeds.size + } +} +///endregion + +///region Value storages + +/** + * Interface used for [ValueStorage]. + */ +interface InfiniteIterator : Iterator { + override operator fun next(): T + override operator fun hasNext(): Boolean +} + +/** + * Interface to storage values mapped to feedbacks in [Minset]. + * Now only [SingleValueStorage] implements it, but there might be more complex logic. + */ +interface ValueStorage : InfiniteIterator> { + fun put(value: Node, feedback: Feedback) : Boolean +} + + +/** + * Base implementation of [ValueStorage] that keeps only one (first or last, according to [strategy]) + * value for each single feedback. + */ +open class SingleValueStorage : ValueStorage { + + private var storedValue: Node? = null + + private var storedWeight: Int = -1 + override fun put(value: Node, feedback: Feedback) : Boolean { + + return if (feedback is WeightedFeedback<*, *>) { + + val result = storedValue == null || storedWeight <= ((feedback as WeightedFeedback<*, *>).weight) + + if (storedValue == null || storedWeight > ((feedback as WeightedFeedback<*, *>).weight)) { + storedValue = value + storedWeight = (feedback as WeightedFeedback<*, *>).weight + } + + result + + } else { + storedValue = value + true + } + } + + override fun next(): Node { + if (storedValue == null) { + error("Next value requested but no value stored") + } else { + return storedValue as Node + } + } + + override fun hasNext(): Boolean { + return storedValue != null + } +} + +/** + * [SingleValueStorage] implementation that counts mutations for each feedback stored. + * Can be used in the future for trying to reach rare feedbacks. + */ +class MutationsCountingSingleValueStorage : SingleValueStorage() { + private val knownValueMutationsCount: HashMap, Int> = hashMapOf() + + override fun put(value: Node, feedback: Feedback): Boolean { + value.result.forEach { result -> + when (result) { + is Result.Known<*, *, *> -> { + result.lastMutation?.let { + knownValueMutationsCount[it] = knownValueMutationsCount.getOrDefault(it, 0) + 1 + } + } + else -> {}} + } + return super.put(value, feedback) + } +} + +///endregion \ No newline at end of file diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/AbcFuzzing.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/AbcFuzzing.kt index 4969ccf7f4..9ca7973e38 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/AbcFuzzing.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/AbcFuzzing.kt @@ -30,14 +30,14 @@ private const val searchString = suspend fun main() { // Define fuzzing description to start searching. - object : Fuzzing, BaseFeedback> { + object : Fuzzing, BaseWeightedFeedback> { /** * Generate method returns several samples or seeds which are used as a base for fuzzing. * * In this particular case only 1 value is provided which is an empty string. Also, a mutation * is defined for any string value. This mutation adds a random character from ASCII table. */ - override fun generate(description: Description, type: Unit) = sequenceOf>( + override fun generate(description: Description, type: Unit) = sequenceOf>( Seed.Simple("") { s, r -> s + Char(r.nextInt(1, 256)) } ) @@ -47,17 +47,18 @@ suspend fun main() { * This implementation just calls the target function and returns a result. After it returns an empty feedback. * If some returned value equals to the length of the source string then feedback returns 'stop' signal. */ - override suspend fun handle(description: Description, values: List): BaseFeedback { + override suspend fun handle(description: Description, values: List): BaseWeightedFeedback { check(values.size == 1) { "Only one value must be generated because of `description.parameters.size = ${description.parameters.size}`" } val input = values.first() val result = searchString.findMaxSubstring(input) println("findMaxSubstring(\"$input\") = $result") - return BaseFeedback( + return BaseWeightedFeedback( result = result, + weight = result, control = if (result == searchString.length) Control.STOP else Control.CONTINUE ) } - }.fuzz(Description(listOf(Unit))) + }.fuzz(LoggingDescription(listOf(Unit), "~/.utbot/AbcFuzzing")) } \ No newline at end of file diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/ForkFuzzing.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/ForkFuzzing.kt index 99be17e156..ead8debe12 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/ForkFuzzing.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/ForkFuzzing.kt @@ -11,15 +11,15 @@ private enum class Type { fun main(): Unit = runBlocking { launch { - object : Fuzzing, Feedback> { + object : Fuzzing, Feedback> { private val runs = mutableMapOf() - override fun generate(description: Description, type: Type): Sequence> { + override fun generate(description: Description, type: Type): Sequence> { return sequenceOf(Seed.Simple(type.name)) } - override suspend fun handle(description: Description, values: List): Feedback { + override suspend fun handle(description: Description, values: List): Feedback { description.parameters.forEach { runs[it]!!.incrementAndGet() } @@ -28,7 +28,7 @@ fun main(): Unit = runBlocking { } override suspend fun afterIteration( - description: Description, + description: Description, stats: Statistic, ) { if (stats.totalRuns % 10 == 0L && description.parameters.size == 1) { @@ -38,7 +38,7 @@ fun main(): Unit = runBlocking { Type.MORE_CONCRETE -> listOf() } if (newTypes.isNotEmpty()) { - val d = Description(newTypes) + val d = Description(newTypes) fork(d, stats) // Description can be used as a transfer object, // that collects information about the current running. @@ -47,7 +47,7 @@ fun main(): Unit = runBlocking { } } - override suspend fun isCancelled(description: Description, stats: Statistic): Boolean { + override suspend fun isCancelled(description: Description, stats: Statistic): Boolean { println("info: ${description.parameters} runs ${stats.totalRuns}") return description.parameters.all { runs.computeIfAbsent(it) { AtomicLong(0) }.get() >= 10 } } diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/HttpRequest.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/HttpRequest.kt index 844336aad1..cfe9af33c5 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/HttpRequest.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/HttpRequest.kt @@ -7,8 +7,8 @@ import java.util.concurrent.TimeUnit fun main() = runBlocking { withTimeout(TimeUnit.SECONDS.toMillis(10)) { - object : Fuzzing, Feedback> { - override fun generate(description: Description, type: String) = sequence> { + object : Fuzzing, Feedback> { + override fun generate(description: Description, type: String) = sequence> { when (type) { "url" -> yield(Seed.Recursive( construct = Routine.Create( @@ -49,7 +49,7 @@ fun main() = runBlocking { } override suspend fun handle( - description: Description, + description: Description, values: List ): Feedback { println(values[0]) diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JavaFuzzing.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JavaFuzzing.kt index 0d3d490dd7..dad1e219c6 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JavaFuzzing.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JavaFuzzing.kt @@ -10,8 +10,8 @@ import org.utbot.fuzzing.seeds.StringValue * This example implements some basics for Java fuzzing that supports only a few types: * integers, strings, primitive arrays and user class [A]. */ -object JavaFuzzing : Fuzzing, Any?, Description>, Feedback, Any?>> { - override fun generate(description: Description>, type: Class<*>) = sequence, Any?>> { +object JavaFuzzing : Fuzzing, Any?, LoggingDescription, Any?>, Feedback, Any?>> { + override fun generate(description: LoggingDescription, Any?>, type: Class<*>) = sequence, Any?>> { if (type == Boolean::class.javaPrimitiveType) { yield(Seed.Known(Bool.TRUE.invoke()) { obj: BitVectorValue -> obj.toBoolean() }) yield(Seed.Known(Bool.FALSE.invoke()) { obj: BitVectorValue -> obj.toBoolean() }) @@ -71,7 +71,7 @@ object JavaFuzzing : Fuzzing, Any?, Description>, Feedback>, values: List): Feedback, Any?> { + override suspend fun handle(description: LoggingDescription, Any?>, values: List): Feedback, Any?> { println(values.joinToString { when (it) { is BooleanArray -> it.contentToString() @@ -88,7 +88,12 @@ object JavaFuzzing : Fuzzing, Any?, Description>, Feedback() - BaseFuzzing, Feedback>( + BaseFuzzing, Feedback>( TypeProvider(CustomType.INT) { _, _ -> for (b in Signed.values()) { yield(Seed.Known(BitVectorValue(3, b)) { bvv -> diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/seeds/StringValue.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/seeds/StringValue.kt index cd054754b8..99cd552c5c 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/seeds/StringValue.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/seeds/StringValue.kt @@ -24,4 +24,8 @@ open class StringValue( StringMutations.ShuffleCharacters, ) } + + override fun toString(): String { + return "\"${valueProvider()}\"" + } } \ No newline at end of file diff --git a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt index 7bc4190f64..4e9928d034 100644 --- a/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt +++ b/utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt @@ -168,13 +168,21 @@ open class Trie( val parent: NodeImpl?, override var count: Int = 0, val children: MutableMap> = HashMap(), - ) : Node + ) : Node { + override fun toString(): String { + return "$data" + } + } private object EmptyNode : Node { override val data: Any get() = error("empty node has no data") override val count: Int get() = 0 + + override fun toString(): String { + return "EMPTY" + } } companion object { diff --git a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt index 260b824942..82045346ff 100644 --- a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt +++ b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/FuzzerSmokeTest.kt @@ -18,7 +18,7 @@ class FuzzerSmokeTest { fun `fuzzing runs with empty parameters`() { runBlocking { var count = 0 - runFuzzing, BaseFeedback>( + runFuzzing, BaseFeedback>( { _, _ -> sequenceOf() }, Description(emptyList()) ) { _, _ -> @@ -34,7 +34,7 @@ class FuzzerSmokeTest { assertThrows { runBlocking { var count = 0 - runFuzzing, BaseFeedback>( + runFuzzing, BaseFeedback>( provider = { _, _ -> sequenceOf() }, description = Description(listOf(Unit)), configuration = Configuration( @@ -55,7 +55,7 @@ class FuzzerSmokeTest { var count = 0 runFuzzing( { _, _ -> sequenceOf(Seed.Simple(Unit)) }, - Description(listOf(Unit)) + Description(listOf(Unit)) ) { _, _ -> count += 1 BaseFeedback(Unit, Control.STOP) @@ -71,7 +71,7 @@ class FuzzerSmokeTest { var executions = 0 runFuzzing( { _, _ -> sequenceOf(Seed.Simple(Unit)) }, - Description(listOf(Unit)) + Description(listOf(Unit)) ) { _, _ -> executions++ BaseFeedback(Unit, if (--count == 0) Control.STOP else Control.CONTINUE) @@ -91,7 +91,7 @@ class FuzzerSmokeTest { generations++ sequenceOf(Seed.Simple(Unit)) }, - Description(listOf(Unit)) + Description(listOf(Unit)) ) { _, _ -> BaseFeedback(Unit, if (--count == 0) Control.STOP else Control.CONTINUE) } @@ -108,7 +108,7 @@ class FuzzerSmokeTest { withTimeout(1000) { runFuzzing( { _, _ -> sequenceOf(Seed.Simple(Unit)) }, - Description(listOf(Unit)) + Description(listOf(Unit)) ) { _, _ -> throw SpecialException() } @@ -128,7 +128,7 @@ class FuzzerSmokeTest { var depth = 0 var count = 0 runFuzzing( - ValueProvider, Node?, Description>> { _, _ -> + { _, _ -> sequenceOf(Seed.Recursive( construct = Routine.Create(listOf(Node::class, Node::class)) { v -> Node(v[0], v[1]) @@ -162,7 +162,7 @@ class FuzzerSmokeTest { withTimeout(1000) { runFuzzing( { _, _ -> sequenceOf(Seed.Simple(Unit)) }, - Description(listOf(Unit)) + Description(listOf(Unit)) ) { _, _ -> if (System.currentTimeMillis() - start > 10_000) { error("Fuzzer didn't stopped in 10 000 ms") @@ -181,7 +181,7 @@ class FuzzerSmokeTest { val start = System.currentTimeMillis() runFuzzing( { _, _ -> sequenceOf(Seed.Simple(Unit)) }, - Description(listOf(Unit)) + Description(listOf(Unit)) ) { _, _ -> if (System.currentTimeMillis() - start > 10_000) { error("Fuzzer didn't stopped in 10_000 ms") @@ -197,7 +197,7 @@ class FuzzerSmokeTest { @Test fun `fuzzer generate same result when random is seeded`() { data class B(var a: Int) - val provider = ValueProvider> { _, _ -> + val provider = ValueProvider> { _, _ -> sequenceOf( Seed.Simple(B(0)) { p, r -> B(p.a + r.nextInt()) }, Seed.Known(BitVectorValue(32, Signed.POSITIVE)) { B(it.toInt()) }, @@ -213,7 +213,7 @@ class FuzzerSmokeTest { ), Seed.Collection( construct = Routine.Collection { size -> B(size) }, - modify = Routine.ForEach(listOf(Unit)) { self, ind, v -> self.a = ind * self.a * v.first().a } + modify = Routine.ForEach(listOf(Unit)) { self, ind, v -> self.a *= ind * v.first().a } ) ) } @@ -245,7 +245,7 @@ class FuzzerSmokeTest { flow { runFuzzing( { _, _ -> sequenceOf(Seed.Simple(Unit)) }, - Description(listOf(Unit)) + Description(listOf(Unit)) ) { _, _ -> if (System.currentTimeMillis() - start > 10_000) { error("Fuzzer didn't stopped in 10_000 ms") @@ -273,7 +273,7 @@ class FuzzerSmokeTest { modify = sequenceOf(Routine.Call(emptyList()) { s, _ -> s.append("1") }), empty = Routine.Empty { fail("Empty is called despite construct requiring no args") } )) }, - Description(listOf(Unit)) + Description(listOf(Unit)) ) { _, (s) -> if (s.isEmpty()) { seenEmpty = true @@ -299,7 +299,7 @@ class FuzzerSmokeTest { modify = emptySequence(), empty = Routine.Empty { null } )}.asSequence() }, - Description(listOf(Unit)) + Description(listOf(Unit)) ) { _, _ -> seenAnything = true BaseFeedback(Unit, Control.STOP) diff --git a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt index 74b2d8c204..29965929c9 100644 --- a/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt +++ b/utbot-fuzzing/src/test/kotlin/org/utbot/fuzzing/ProvidersTest.kt @@ -5,23 +5,23 @@ import org.junit.jupiter.api.Test class ProvidersTest { - private fun p(supplier: () -> T) : ValueProvider> { + private fun p(supplier: () -> T) : ValueProvider> { return ValueProvider { _, _ -> sequenceOf(Seed.Simple(supplier())) } } private fun p( accept: () -> Boolean, supplier: () -> T - ) : ValueProvider> { - return object : ValueProvider> { + ) : ValueProvider> { + return object : ValueProvider> { override fun accept(type: Unit) = accept() - override fun generate(description: Description, type: Unit) = sequence> { + override fun generate(description: Description, type: Unit) = sequence> { yield(Seed.Simple(supplier())) } } } - private val description = Description(listOf(Unit)) + private val description = Description(listOf(Unit)) @Test fun `test common provider API`() { @@ -170,9 +170,9 @@ class ProvidersTest { val seq = ValueProvider.of(listOf( p({ false }, { 1 }), p({ false }, { 2 }), p({ false }, { 3 }), )).withFallback( - object : ValueProvider> { + object : ValueProvider> { override fun accept(type: Unit) = false - override fun generate(description: Description, type: Unit) = emptySequence>() + override fun generate(description: Description, type: Unit) = emptySequence>() } ).generate(description, Unit) Assertions.assertEquals(0, seq.count()) @@ -180,12 +180,12 @@ class ProvidersTest { @Test fun `type providers check exactly the type`() { - val seq1 = TypeProvider>('A') { _, _ -> + val seq1 = TypeProvider>('A') { _, _ -> yield(Seed.Simple(2)) }.generate(Description(listOf('A')), 'A') Assertions.assertEquals(1, seq1.count()) - val seq2 = TypeProvider>('A') { _, _ -> + val seq2 = TypeProvider>('A') { _, _ -> yield(Seed.Simple(2)) }.generate(Description(listOf('A')), 'B') Assertions.assertEquals(0, seq2.count()) diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt index d5d4a6d7f2..fae534654c 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/FuzzedValue.kt @@ -19,4 +19,8 @@ import org.utbot.framework.plugin.api.UtModel open class FuzzedValue( val model: UtModel, var summary: String? = null, -) \ No newline at end of file +) { + override fun toString(): String { + return "FuzzedValue: $summary" + } +} \ No newline at end of file diff --git a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt index 38565368e9..653b7abb4b 100644 --- a/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt +++ b/utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt @@ -17,17 +17,25 @@ private val logger = KotlinLogging.logger {} typealias JavaValueProvider = ValueProvider + + class FuzzedDescription( val description: FuzzedMethodDescription, val tracer: Trie, val typeCache: MutableMap, val random: Random, - val scope: Scope? = null -) : Description( + val scope: Scope? = null, +) : Description( description.parameters.mapIndexed { index, classId -> description.fuzzerType(index) ?: FuzzedType(classId) } ) { + // To turn on logging, use this implementation of Description: + //LoggingDescription( + // description.parameters.mapIndexed { index, classId -> + // description.fuzzerType(index) ?: FuzzedType(classId) + //}, "~/.utbot/JavaFuzzing") { ... } + val constants: Sequence get() = description.concreteValues.asSequence() @@ -36,6 +44,7 @@ class FuzzedDescription( } } + fun defaultValueProviders(idGenerator: IdentityPreservingIdGenerator) = listOf( BooleanValueProvider, IntegerValueProvider, @@ -53,13 +62,25 @@ fun defaultValueProviders(idGenerator: IdentityPreservingIdGenerator) = lis NullValueProvider, ) + +data class JavaFeedback( + val result: Trie.Node, + override val control: Control, + ) : Feedback { + override var runDuration: Long? = null + override fun toString(): String { + return "$result; trace hash: ${result.hashCode()}; trace count: ${result.count}" + } +} + + suspend fun runJavaFuzzing( idGenerator: IdentityPreservingIdGenerator, methodUnderTest: ExecutableId, constants: Collection, names: List, providers: List> = defaultValueProviders(idGenerator), - exec: suspend (thisInstance: FuzzedValue?, description: FuzzedDescription, values: List) -> BaseFeedback, FuzzedType, FuzzedValue> + exec: suspend (thisInstance: FuzzedValue?, description: FuzzedDescription, values: List) -> JavaFeedback ) { val random = Random(0) val classUnderTest = methodUnderTest.classId diff --git a/utbot-java-fuzzing/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt b/utbot-java-fuzzing/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt index 360d706356..5a23bb1ed0 100644 --- a/utbot-java-fuzzing/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt +++ b/utbot-java-fuzzing/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt @@ -231,7 +231,7 @@ class JavaFuzzingTest { } } -class MarkerValueProvider>( +class MarkerValueProvider>( val name: String ) : ValueProvider { var enrich: Int = 0 diff --git a/utbot-junit-contest/src/main/resources/classes/guava/list b/utbot-junit-contest/src/main/resources/classes/guava/list deleted file mode 100644 index 6d57caaecc..0000000000 --- a/utbot-junit-contest/src/main/resources/classes/guava/list +++ /dev/null @@ -1,20 +0,0 @@ -com.google.common.collect.MinMaxPriorityQueue -com.google.common.math.LongMath -com.google.common.primitives.Doubles -com.google.common.graph.Graphs -com.google.common.collect.TreeMultiset -com.google.common.collect.FilteredEntryMultimap -com.google.common.io.FileBackedOutputStream -com.google.common.collect.ComparatorOrdering -com.google.common.collect.LinkedListMultimap -com.google.common.collect.LexicographicalOrdering -com.google.common.base.Throwables -com.google.common.collect.SparseImmutableTable -com.google.common.primitives.ParseRequest -com.google.common.primitives.SignedBytes -com.google.thirdparty.publicsuffix.PublicSuffixType -com.google.common.collect.ImmutableEnumSet -com.google.common.net.MediaType -com.google.common.primitives.UnsignedLongs -com.google.common.collect.FilteredMultimapValues -com.google.common.io.Closeables \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index 9d32164be0..f60cc6c838 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -29,7 +29,7 @@ class PythonMethodDescription( val pythonTypeStorage: PythonTypeStorage, val tracer: Trie, val random: Random, -) : Description(parameters) +) : Description(parameters) sealed interface FuzzingExecutionFeedback class ValidExecution(val utFuzzedExecution: PythonUtExecution): FuzzingExecutionFeedback @@ -47,7 +47,9 @@ data class PythonExecutionResult( data class PythonFeedback( override val control: Control = Control.CONTINUE, val result: Trie.Node = Trie.emptyNode(), -) : Feedback +) : Feedback { + override var runDuration: Long? = null +} class PythonFuzzedValue( val tree: PythonTree.PythonTreeNode,