From 125a1fae202f9f150bc70a4b329bb9f9ac4256ad Mon Sep 17 00:00:00 2001 From: Dante Niewenhuis Date: Thu, 4 Apr 2024 12:12:55 +0200 Subject: [PATCH 1/5] Added custom failure models Started incorporating the failure models Added support for failure traces and different models Failure traces can now be loaded with files. Failing a host causes an error. small update Started fixing failure injection Added support for failure traces and different models Failure traces can now be loaded with files. Failing a host causes an error. small update Started incorporating the failure models Started fixing failure injection --- .../opendc-compute-api/build.gradle.kts | 3 + .../org/opendc/compute/api/ComputeClient.kt | 6 + .../compute/carbon/CarbonTraceLoader.kt | 12 +- .../opendc-compute-service/build.gradle.kts | 1 + .../compute/service/ComputeService.java | 25 ++- .../opendc/compute/service/ServiceServer.java | 11 +- .../service/scheduler/FilterScheduler.kt | 1 + .../scheduler/filters/ComputeFilter.kt | 3 +- .../service/scheduler/filters/RamFilter.kt | 16 +- .../opendc-compute-simulator/build.gradle.kts | 2 + .../org/opendc/compute/simulator/SimHost.kt | 3 + .../checkpoints/CheckPointFactory.kt | 1 + .../simulator/checkpoints/CheckpointModel.kt | 6 + .../simulator/failure/FailureModelFactory.kt | 9 +- .../simulator/failure/HostFaultInjector.kt | 6 +- .../failure/{ => hostfault}/HostFault.kt | 4 +- .../failure/hostfault/HostFaultNew.kt | 43 ++++++ .../{ => hostfault}/StartStopHostFault.kt | 10 +- .../hostfault/StartStopHostFaultNew.kt | 64 ++++++++ .../failure/models/ConstantFailureModel.kt | 25 +++ .../failure/{ => models}/FailureModel.kt | 3 +- .../failure/models/FailureModelNew.kt | 85 +++++++++++ .../failure/models/FailureTraceLoader.kt | 90 +++++++++++ .../failure/models/SampleBasedFailureModel.kt | 52 +++++++ .../failure/models/TraceBasedFailureModel.kt | 45 ++++++ .../failure/{models => prefab}/Grid5000.kt | 12 +- .../simulator/failure/prefab/Grid5000New.kt | 75 +++++++++ .../StochasticVictimSelector.kt | 28 ++-- .../StochasticVictimSelectorNew.kt | 77 ++++++++++ .../{ => victimselector}/VictimSelector.kt | 7 +- .../internal/DefaultWorkloadMapper.kt | 3 +- .../compute/simulator/internal/Guest.kt | 7 +- ...ctorImpl.kt => RandomHostFaultInjector.kt} | 8 +- .../failure/HostFaultInjectorTest.kt | 2 + .../opendc/compute/topology/TopologyReader.kt | 13 +- .../opendc/compute/workload/VirtualMachine.kt | 4 +- .../base/runner/ScenarioHelpers.kt | 30 +++- .../experiments/base/runner/ScenarioRunner.kt | 42 ++++- .../experiments/base/scenario/Scenario.kt | 13 +- .../base/scenario/ScenarioFactories.kt | 50 +++--- .../base/scenario/ScenarioReader.kt | 19 ++- .../scenario/specs/CheckpointModelSpec.kt | 7 + .../base/scenario/specs/FailureModelSpec.kt | 124 +++++++++++++-- .../base/scenario/specs/ScenarioSpec.kt | 4 +- .../src/main/resources/scenario.json | 3 + .../compute/power/CPUPowerModelsFactory.kt | 9 ++ .../compute/workload/SimRuntimeWorkload.java | 45 +++++- .../simulator/compute/workload/SimTrace.java | 15 +- .../compute/workload/SimWorkloads.java | 8 +- .../org/opendc/trace/conv/FailureColumns.kt | 40 +++++ .../kotlin/org/opendc/trace/conv/Tables.kt | 4 +- .../trace/formats/carbon/CarbonTraceFormat.kt | 8 +- .../formats/failure/FailureTableReader.kt | 144 ++++++++++++++++++ .../formats/failure/FailureTraceFormat.kt | 88 +++++++++++ .../failure/parquet/FailureFragment.kt | 32 ++++ .../failure/parquet/FailureReadSupport.kt | 98 ++++++++++++ .../parquet/FailureRecordMaterializer.kt | 94 ++++++++++++ .../org/opendc/trace/spi/TraceFormat.kt | 2 + .../org/opendc/web/runner/OpenDCRunner.kt | 2 +- settings.gradle.kts | 1 + 60 files changed, 1511 insertions(+), 133 deletions(-) create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckPointFactory.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckpointModel.kt rename opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/{ => hostfault}/HostFault.kt (94%) create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFaultNew.kt rename opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/{ => hostfault}/StartStopHostFault.kt (84%) create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFaultNew.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/ConstantFailureModel.kt rename opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/{ => models}/FailureModel.kt (93%) create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModelNew.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureTraceLoader.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt rename opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/{models => prefab}/Grid5000.kt (83%) create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000New.kt rename opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/{ => victimselector}/StochasticVictimSelector.kt (70%) create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelectorNew.kt rename opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/{ => victimselector}/VictimSelector.kt (80%) rename opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/{HostFaultInjectorImpl.kt => RandomHostFaultInjector.kt} (93%) create mode 100644 opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/CheckpointModelSpec.kt create mode 100644 opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt create mode 100644 opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt create mode 100644 opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTraceFormat.kt create mode 100644 opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureFragment.kt create mode 100644 opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt create mode 100644 opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt diff --git a/opendc-compute/opendc-compute-api/build.gradle.kts b/opendc-compute/opendc-compute-api/build.gradle.kts index f9b04299a..2f855c798 100644 --- a/opendc-compute/opendc-compute-api/build.gradle.kts +++ b/opendc-compute/opendc-compute-api/build.gradle.kts @@ -26,3 +26,6 @@ description = "API interface for the OpenDC Compute service" plugins { `kotlin-library-conventions` } +dependencies { + implementation(project(mapOf("path" to ":opendc-simulator:opendc-simulator-compute"))) +} diff --git a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt index 09cfe6f52..fa829e098 100644 --- a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt +++ b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt @@ -22,6 +22,7 @@ package org.opendc.compute.api +import org.opendc.simulator.compute.workload.SimWorkload import java.util.UUID /** @@ -113,6 +114,11 @@ public interface ComputeClient : AutoCloseable { start: Boolean = true, ): Server + public fun rescheduleServer( + server: Server, + workload: SimWorkload + ) + /** * Release the resources associated with this client, preventing any further API calls. */ diff --git a/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTraceLoader.kt b/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTraceLoader.kt index 685a1fb31..b66aedf92 100644 --- a/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTraceLoader.kt +++ b/opendc-compute/opendc-compute-carbon/src/main/kotlin/org/opendc/compute/carbon/CarbonTraceLoader.kt @@ -22,11 +22,10 @@ package org.opendc.compute.carbon -import mu.KotlinLogging import org.opendc.trace.Trace import org.opendc.trace.conv.CARBON_INTENSITY_TIMESTAMP import org.opendc.trace.conv.CARBON_INTENSITY_VALUE -import org.opendc.trace.conv.TABLE_CARBON_INTENSITY +import org.opendc.trace.conv.TABLE_CARBON_INTENSITIES import java.io.File import java.lang.ref.SoftReference import java.time.Instant @@ -38,11 +37,6 @@ import java.util.concurrent.ConcurrentHashMap * @param baseDir The directory containing the traces. */ public class CarbonTraceLoader { - /** - * The logger for this instance. - */ - private val logger = KotlinLogging.logger {} - /** * The cache of workloads. */ @@ -54,13 +48,11 @@ public class CarbonTraceLoader { * Read the metadata into a workload. */ private fun parseCarbon(trace: Trace): List { - val reader = checkNotNull(trace.getTable(TABLE_CARBON_INTENSITY)).newReader() + val reader = checkNotNull(trace.getTable(TABLE_CARBON_INTENSITIES)).newReader() val startTimeCol = reader.resolve(CARBON_INTENSITY_TIMESTAMP) val carbonIntensityCol = reader.resolve(CARBON_INTENSITY_VALUE) - val entries = mutableListOf() - try { while (reader.nextRow()) { val startTime = reader.getInstant(startTimeCol)!! diff --git a/opendc-compute/opendc-compute-service/build.gradle.kts b/opendc-compute/opendc-compute-service/build.gradle.kts index 0efdb05f9..cd25e05ce 100644 --- a/opendc-compute/opendc-compute-service/build.gradle.kts +++ b/opendc-compute/opendc-compute-service/build.gradle.kts @@ -31,6 +31,7 @@ dependencies { api(projects.opendcCompute.opendcComputeApi) implementation(projects.opendcCommon) implementation(libs.kotlin.logging) + implementation(project(mapOf("path" to ":opendc-simulator:opendc-simulator-compute"))) testImplementation(projects.opendcSimulator.opendcSimulatorCore) testImplementation(libs.log4j.slf4j) diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java index 167b13c7c..f9d331e3b 100644 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java +++ b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java @@ -54,6 +54,7 @@ import org.opendc.compute.service.telemetry.SchedulerStats; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.opendc.simulator.compute.workload.SimWorkload; /** * The {@link ComputeService} hosts the API implementation of the OpenDC Compute Engine. @@ -161,7 +162,7 @@ public void onStateChanged(@NotNull Host host, @NotNull Server server, @NotNull serviceServer.setState(newState); - if (newState == ServerState.TERMINATED || newState == ServerState.DELETED) { + if (newState == ServerState.TERMINATED || newState == ServerState.DELETED || newState == ServerState.ERROR) { LOGGER.info("Server {} {} {} finished", server.getUid(), server.getName(), server.getFlavor()); if (activeServers.remove(server) != null) { @@ -353,6 +354,8 @@ private void requestSchedulingCycle() { * Run a single scheduling iteration. */ private void doSchedule() { + // reorder tasks + while (!queue.isEmpty()) { SchedulingRequest request = queue.peek(); @@ -363,6 +366,9 @@ private void doSchedule() { } final ServiceServer server = request.server; + // Check if all dependencies are met + // otherwise continue + final ServiceFlavor flavor = server.getFlavor(); final HostView hv = scheduler.select(request.server); @@ -582,6 +588,23 @@ public void close() { public String toString() { return "ComputeService.Client"; } + + @Nullable + @Override + public void rescheduleServer( + @NotNull Server server, + @NotNull SimWorkload workload) + { + ServiceServer internalServer = (ServiceServer) findServer(server.getUid()); + Host from = service.lookupHost(internalServer); + + from.delete(internalServer); + + internalServer.host = null; + + internalServer.setWorkload(workload); + internalServer.start(); + } } /** diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceServer.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceServer.java index 265feac00..e363faf21 100644 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceServer.java +++ b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ServiceServer.java @@ -25,6 +25,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -46,11 +47,12 @@ public final class ServiceServer implements Server { private final ComputeService service; private final UUID uid; + private final String name; private final ServiceFlavor flavor; private final ServiceImage image; private final Map labels; - private final Map meta; + private Map meta; private final List watchers = new ArrayList<>(); private ServerState state = ServerState.TERMINATED; @@ -111,6 +113,13 @@ public Map getMeta() { return Collections.unmodifiableMap(meta); } + public void setWorkload(Object _workload) { + Map new_meta = new HashMap(); + new_meta.put("workload", _workload); + + meta = new_meta; + } + @NotNull @Override public ServerState getState() { diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/FilterScheduler.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/FilterScheduler.kt index cdcd1af06..411183866 100644 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/FilterScheduler.kt +++ b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/FilterScheduler.kt @@ -101,6 +101,7 @@ public class FilterScheduler( filteredHosts } + // fixme: currently finding no matching hosts can result in an error return when (val maxSize = min(subsetSize, subset.size)) { 0 -> null 1 -> subset[0] diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/ComputeFilter.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/ComputeFilter.kt index 23590c133..dd707f606 100644 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/ComputeFilter.kt +++ b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/ComputeFilter.kt @@ -34,7 +34,8 @@ public class ComputeFilter : HostFilter { host: HostView, server: Server, ): Boolean { - return host.host.state == HostState.UP + val result = host.host.state == HostState.UP + return result } override fun toString(): String = "ComputeFilter" diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/RamFilter.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/RamFilter.kt index 4792a7a0d..d8c3d5402 100644 --- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/RamFilter.kt +++ b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/filters/RamFilter.kt @@ -35,19 +35,21 @@ public class RamFilter(private val allocationRatio: Double) : HostFilter { host: HostView, server: Server, ): Boolean { - val requested = server.flavor.memorySize - val available = host.availableMemory - val total = host.host.model.memoryCapacity + val requestedMemory = server.flavor.memorySize + val availableMemory = host.availableMemory + val memoryCapacity = host.host.model.memoryCapacity // Do not allow an instance to overcommit against itself, only against // other instances. - if (requested > total) { + if (requestedMemory > memoryCapacity) { return false } - val limit = total * allocationRatio - val used = total - available + val limit = memoryCapacity * allocationRatio + val used = memoryCapacity - availableMemory val usable = limit - used - return usable >= requested + + val result = usable >= requestedMemory + return result } } diff --git a/opendc-compute/opendc-compute-simulator/build.gradle.kts b/opendc-compute/opendc-compute-simulator/build.gradle.kts index 0cddd296e..20ceb93ed 100644 --- a/opendc-compute/opendc-compute-simulator/build.gradle.kts +++ b/opendc-compute/opendc-compute-simulator/build.gradle.kts @@ -39,6 +39,8 @@ dependencies { implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-telemetry"))) implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-carbon"))) + implementation(project(mapOf("path" to ":opendc-trace:opendc-trace-api"))) + testImplementation(projects.opendcSimulator.opendcSimulatorCore) testRuntimeOnly(libs.slf4j.simple) } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt index 84799123f..00f2acb33 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt @@ -190,6 +190,9 @@ public class SimHost( override fun delete(server: Server) { val guest = guests[server] ?: return guest.delete() + + guests.remove(server) + localGuests.remove(guest) } override fun addListener(listener: HostListener) { diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckPointFactory.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckPointFactory.kt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckPointFactory.kt @@ -0,0 +1 @@ + diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckpointModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckpointModel.kt new file mode 100644 index 000000000..cc0519186 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckpointModel.kt @@ -0,0 +1,6 @@ +package org.opendc.compute.simulator.checkpoints + +public data class CheckpointModel( + val checkpointWait: Long = 60*60*1000, + val checkpointTime: Long = 5*60*1000 +) diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt index 406665c73..fcd7a7ac6 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt @@ -22,9 +22,7 @@ package org.opendc.compute.simulator.failure -import org.opendc.compute.simulator.failure.models.Grid5000 -import java.time.Duration -import kotlin.math.roundToLong +import org.opendc.compute.simulator.failure.models.FailureModelNew /** * Get failure model @@ -32,9 +30,10 @@ import kotlin.math.roundToLong * @param failureInterval The interval of failures occurring in s * @return */ -public fun getFailureModel(failureInterval: Double): FailureModel? { +public fun getFailureModelOld(failureInterval: Long): FailureModelNew? { return if (failureInterval > 0) { - Grid5000(Duration.ofSeconds(failureInterval.roundToLong())) +// createGrid5000() + null } else { null } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt index 26084a1bf..8d1acb626 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt @@ -24,7 +24,9 @@ package org.opendc.compute.simulator.failure import org.apache.commons.math3.distribution.RealDistribution import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.internal.HostFaultInjectorImpl +import org.opendc.compute.simulator.failure.hostfault.HostFault +import org.opendc.compute.simulator.failure.victimselector.VictimSelector +import org.opendc.compute.simulator.internal.RandomHostFaultInjector import java.time.Clock import java.time.InstantSource import kotlin.coroutines.CoroutineContext @@ -61,6 +63,6 @@ public interface HostFaultInjector : AutoCloseable { iat: RealDistribution, selector: VictimSelector, fault: HostFault, - ): HostFaultInjector = HostFaultInjectorImpl(context, clock, hosts, iat, selector, fault) + ): HostFaultInjector = RandomHostFaultInjector(context, clock, hosts, iat, selector, fault) } } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFault.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFault.kt similarity index 94% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFault.kt rename to opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFault.kt index faf536ad8..4b08164f9 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFault.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFault.kt @@ -20,7 +20,7 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure +package org.opendc.compute.simulator.failure.hostfault import org.opendc.compute.simulator.SimHost import java.time.InstantSource @@ -34,6 +34,6 @@ public interface HostFault { */ public suspend fun apply( clock: InstantSource, - victims: List, + victims: List ) } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFaultNew.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFaultNew.kt new file mode 100644 index 000000000..26c7ce72e --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFaultNew.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.failure.hostfault + +import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.SimHost +import java.time.InstantSource + +/** + * Interface responsible for applying the fault to a host. + */ +public abstract class HostFaultNew( + private val service: ComputeService, + private val clock: InstantSource, +) { + /** + * Apply the fault to the specified [victims]. + */ + public abstract suspend fun apply( + victims: List, + faultDuration: Long + ) +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StartStopHostFault.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFault.kt similarity index 84% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StartStopHostFault.kt rename to opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFault.kt index 45545f3bc..615907df5 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StartStopHostFault.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFault.kt @@ -20,7 +20,7 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure +package org.opendc.compute.simulator.failure.hostfault import kotlinx.coroutines.delay import org.apache.commons.math3.distribution.RealDistribution @@ -30,8 +30,10 @@ import kotlin.math.roundToLong /** * A type of [HostFault] where the hosts are stopped and recover after some random amount of time. + * + * @property durationSampler A RealDistribution which the duration of the error is sampled from */ -public class StartStopHostFault(private val duration: RealDistribution) : HostFault { +public class StartStopHostFault(private val durationSampler: RealDistribution) : HostFault { override suspend fun apply( clock: InstantSource, victims: List, @@ -40,7 +42,7 @@ public class StartStopHostFault(private val duration: RealDistribution) : HostFa host.fail() } - val df = (duration.sample() * 1000).roundToLong() // seconds to milliseconds + val df = (durationSampler.sample() * 1000).roundToLong() // seconds to milliseconds // Handle long overflow if (clock.millis() + df <= 0) { @@ -54,5 +56,5 @@ public class StartStopHostFault(private val duration: RealDistribution) : HostFa } } - override fun toString(): String = "StartStopHostFault[$duration]" + override fun toString(): String = "StartStopHostFault[$durationSampler]" } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFaultNew.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFaultNew.kt new file mode 100644 index 000000000..61c78d96f --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFaultNew.kt @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.failure.hostfault + +import kotlinx.coroutines.delay +import org.opendc.compute.api.ComputeClient +import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.SimHost +import org.opendc.simulator.compute.workload.SimWorkload +import java.time.InstantSource + +/** + * A type of [HostFault] where the hosts are stopped and recover after a given amount of time. + */ +public class StartStopHostFaultNew ( + private val service: ComputeService, + clock: InstantSource +): HostFaultNew(service, clock) { + override suspend fun apply( + victims: List, + faultDuration: Long + ) { + val client: ComputeClient = service.newClient() + + for (host in victims) { + val servers = host.instances + + val snapshots = servers.map{(it.meta["workload"] as SimWorkload).snapshot()} + host.fail() + + for ((server, snapshot) in servers.zip(snapshots)) { + client.rescheduleServer(server, snapshot) + } + } + + delay(faultDuration) + + for (host in victims) { + host.recover() + } + } + + override fun toString(): String = "StartStopHostFault" +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/ConstantFailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/ConstantFailureModel.kt new file mode 100644 index 000000000..27fb8c476 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/ConstantFailureModel.kt @@ -0,0 +1,25 @@ +package org.opendc.compute.simulator.failure.models + +import kotlinx.coroutines.delay +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +public class ConstantFailureModel( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, + private val numberOfVictims: Int, + private val waitDuration: Long, + private val faultDuration: Long, + ): FailureModelNew(context, clock, service, random) { + override suspend fun runInjector() { + delay(waitDuration) + + val victims = victimSelector.select(hosts, numberOfVictims) + + fault.apply(victims, faultDuration) + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModel.kt similarity index 93% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModel.kt rename to opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModel.kt index 9511017f1..be8c7a252 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModel.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModel.kt @@ -20,9 +20,10 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure +package org.opendc.compute.simulator.failure.models import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.failure.HostFaultInjector import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModelNew.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModelNew.kt new file mode 100644 index 000000000..ca7379987 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModelNew.kt @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.failure.models + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.SimHost +import org.opendc.compute.simulator.failure.hostfault.HostFaultNew +import org.opendc.compute.simulator.failure.hostfault.StartStopHostFaultNew +import org.opendc.compute.simulator.failure.victimselector.StochasticVictimSelectorNew +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * Factory interface for constructing [HostFaultInjector] for modeling failures of compute service hosts. + */ +public abstract class FailureModelNew ( + context: CoroutineContext, + protected val clock: InstantSource, + protected val service: ComputeService, + protected val random: RandomGenerator +): AutoCloseable { + protected val scope: CoroutineScope = CoroutineScope(context + Job()) + + // TODO: could at some point be extended to different types of faults + protected val fault: HostFaultNew = StartStopHostFaultNew(service, clock) + + // TODO: could at some point be extended to different types of victim selectors + protected val victimSelector: StochasticVictimSelectorNew = StochasticVictimSelectorNew(random) + + protected val hosts: Set = service.hosts.map { it as SimHost }.toSet() + + /** + * The [Job] that awaits the nearest fault in the system. + */ + private var job: Job? = null + + /** + * Start the fault injection into the system. + */ + public fun start() { + if (job != null) { + return + } + + job = + scope.launch { + runInjector() + job = null + } + } + + public abstract suspend fun runInjector() + + /** + * Stop the fault injector. + */ + public override fun close() { + scope.cancel() + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureTraceLoader.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureTraceLoader.kt new file mode 100644 index 000000000..cf9bd2b85 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureTraceLoader.kt @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.failure.models + +import org.opendc.trace.Trace +import org.opendc.trace.conv.FAILURE_DURATION +import org.opendc.trace.conv.FAILURE_INTENSITY +import org.opendc.trace.conv.FAILURE_START +import org.opendc.trace.conv.TABLE_FAILURES +import java.io.File +import java.lang.ref.SoftReference +import java.util.concurrent.ConcurrentHashMap + +/** + * A helper class for loading compute workload traces into memory. + * + * @param baseDir The directory containing the traces. + */ +public class FailureTraceLoader { + /** + * The cache of workloads. + */ + private val cache = ConcurrentHashMap>>() + + /** + * Read the metadata into a workload. + */ + private fun parseFailure(trace: Trace): List { + val reader = checkNotNull(trace.getTable(TABLE_FAILURES)).newReader() + + val failureStartTimeCol = reader.resolve(FAILURE_START) + val failureDurationCol = reader.resolve(FAILURE_DURATION) + val failureIntensityCol = reader.resolve(FAILURE_INTENSITY) + + val entries = mutableListOf() + + try { + while (reader.nextRow()) { + val failureStartTime = reader.getLong(failureStartTimeCol) + val failureDuration = reader.getLong(failureDurationCol) + val failureIntensity = reader.getDouble(failureIntensityCol) + + entries.add(Failure(failureStartTime, failureDuration, failureIntensity)) + } + + return entries + } catch (e: Exception) { + e.printStackTrace() + throw e + } finally { + reader.close() + } + } + + /** + * Load the trace with the specified [name] and [format]. + */ + public fun get(pathToFile: File): List { + val trace = Trace.open(pathToFile, "failure") + + return parseFailure(trace) + } + + /** + * Clear the workload cache. + */ + public fun reset() { + cache.clear() + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt new file mode 100644 index 000000000..8c4004986 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt @@ -0,0 +1,52 @@ +package org.opendc.compute.simulator.failure.models + +import kotlinx.coroutines.delay +import org.apache.commons.math3.distribution.RealDistribution +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext +import kotlin.math.roundToLong + + +/** + * Sample based failure model + * + * @property context + * @property clock + * @property service + * @property random + * @property iatSampler A distribution from which the time until the next fault is sampled in ms + * @property durationSampler A distribution from which the duration of a fault is sampled in s + * @property nohSampler A distribution from which the number of hosts that fault is sampled. + */ +public class SampleBasedFailureModel( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, + + private val iatSampler: RealDistribution, + private val durationSampler: RealDistribution, + private val nohSampler: RealDistribution + ): FailureModelNew(context, clock, service, random) { + override suspend fun runInjector() { + while(true) { + val d = (iatSampler.sample() * 3.6e6).roundToLong() + + // Handle long overflow + if (clock.millis() + d <= 0) { + return + } + + delay(d) + + val victims = victimSelector.select(hosts, nohSampler.sample()) + + val faultDuration = (durationSampler.sample() * 3.6e6).toLong() + fault.apply(victims, faultDuration) + + break + } + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt new file mode 100644 index 000000000..73498767f --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt @@ -0,0 +1,45 @@ +package org.opendc.compute.simulator.failure.models + +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import org.opendc.compute.service.ComputeService +import java.io.File +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +public data class Failure( + val failureStart: Long, + val failureDuration: Long, + val failureIntensity: Double, +) { + init { + require(failureStart >= 0.0) {"A failure cannot start at a negative time"} + require(failureDuration >= 0.0) {"A failure can not have a duration of 0 or less"} + require(failureIntensity >= 0.0) { "carbon intensity cannot be negative" } + } +} + +public class TraceBasedFailureModel( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, + pathToTrace: String + ) : FailureModelNew(context, clock, service, random){ + + private val failureList = FailureTraceLoader().get(File(pathToTrace)).iterator() + + override suspend fun runInjector() { + while(failureList.hasNext()) { + val failure = failureList.next() + + delay(failure.failureStart - clock.millis()) + + val victims = victimSelector.select(hosts, failure.failureIntensity) + scope.launch { + fault.apply(victims, failure.failureDuration) + } + } + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/Grid5000.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000.kt similarity index 83% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/Grid5000.kt rename to opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000.kt index 8aacc49df..1f3dfeb24 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/Grid5000.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000.kt @@ -22,16 +22,16 @@ @file:JvmName("FailureModels") -package org.opendc.compute.simulator.failure.models +package org.opendc.compute.simulator.failure.prefab import org.apache.commons.math3.distribution.LogNormalDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.service.ComputeService import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.FailureModel +import org.opendc.compute.simulator.failure.models.FailureModel import org.opendc.compute.simulator.failure.HostFaultInjector -import org.opendc.compute.simulator.failure.StartStopHostFault -import org.opendc.compute.simulator.failure.StochasticVictimSelector +import org.opendc.compute.simulator.failure.hostfault.StartStopHostFault +import org.opendc.compute.simulator.failure.victimselector.StochasticVictimSelector import java.time.Duration import java.time.InstantSource import java.util.random.RandomGenerator @@ -56,12 +56,14 @@ public class Grid5000(private val failureInterval: Duration) : FailureModel { // Parameters from A. Iosup, A Framework for the Study of Grid Inter-Operation Mechanisms, 2009 // GRID'5000 + // fixme: currently only does something when Duration is higher than 1 hour return HostFaultInjector( context, clock, hosts, iat = LogNormalDistribution(rng, ln(failureInterval.toHours().toDouble()), 1.03), - selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25), random), +// selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25), random), + selector = StochasticVictimSelector(random), fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)), ) } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000New.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000New.kt new file mode 100644 index 000000000..031fc7b5c --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000New.kt @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +@file:JvmName("FailureModels") + +package org.opendc.compute.simulator.failure.prefab + +import org.apache.commons.math3.distribution.ConstantRealDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.failure.models.SampleBasedFailureModel +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +public enum class FailurePrefab { + Grid5000, +} + +public fun getFailureModelPrefab(context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, + prefab: FailurePrefab): SampleBasedFailureModel { + when(prefab) { + FailurePrefab.Grid5000 -> return createGrid5000(context, clock, service, random) + + else -> error("Unknown failure prefab: $prefab") + } +} + + +/** + * A [FailureModel] based on the GRID'5000 failure trace. + * + * This fault injector uses parameters from the GRID'5000 failure trace as described in + * "A Framework for the Study of Grid Inter-Operation Mechanisms", A. Iosup, 2009. + */ +public fun createGrid5000(context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + + LogNormalDistribution(rng, 1.0, 0.1), + LogNormalDistribution(rng, 1.0, 0.1), + ConstantRealDistribution(0.5) + ) +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StochasticVictimSelector.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelector.kt similarity index 70% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StochasticVictimSelector.kt rename to opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelector.kt index 93463cdb0..109c5a90a 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StochasticVictimSelector.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelector.kt @@ -20,28 +20,28 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure +package org.opendc.compute.simulator.failure.victimselector -import org.apache.commons.math3.distribution.RealDistribution import org.opendc.compute.simulator.SimHost -import java.util.ArrayList import java.util.SplittableRandom import java.util.random.RandomGenerator -import kotlin.math.roundToInt /** * A [VictimSelector] that stochastically selects a set of hosts to be failed. */ public class StochasticVictimSelector( - private val size: RealDistribution, private val random: RandomGenerator = SplittableRandom(0), ) : VictimSelector { - override fun select(hosts: Set): List { - val n = size.sample().roundToInt() - val result = ArrayList(n) + + override fun select(numberOfHosts: Int): List { + error("select with only int cannot be used in this type of VictimSelector"); + } + + override fun select(hosts: Set, numberOfHosts:Int): List { + val result = ArrayList(numberOfHosts) val random = random - var samplesNeeded = n + var samplesNeeded = numberOfHosts var remainingHosts = hosts.size val iterator = hosts.iterator() @@ -59,5 +59,13 @@ public class StochasticVictimSelector( return result } - override fun toString(): String = "StochasticVictimSelector[$size]" + override fun select(failureIntensity: Double): List { + error("select with only int cannot be used in this type of VictimSelector"); + } + + override fun select(hosts: Set, failureIntensity:Double): List { + error("select with only int cannot be used in this type of VictimSelector"); + } + + override fun toString(): String = "StochasticVictimSelector" } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelectorNew.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelectorNew.kt new file mode 100644 index 000000000..fcf5f8602 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelectorNew.kt @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.failure.victimselector + +import org.opendc.compute.simulator.SimHost +import java.util.SplittableRandom +import java.util.random.RandomGenerator +import kotlin.math.max +import kotlin.math.min + +/** + * A [VictimSelector] that stochastically selects a set of hosts to be failed. + */ +public class StochasticVictimSelectorNew( + private val random: RandomGenerator = SplittableRandom(0), +) : VictimSelector { + + override fun select(numberOfHosts: Int): List { + error("select with only int cannot be used in this type of VictimSelector"); + } + + override fun select(hosts: Set, numberOfHosts:Int): List { + val result = ArrayList(numberOfHosts) + + val random = random + var samplesNeeded = numberOfHosts + var remainingHosts = hosts.size + val iterator = hosts.iterator() + + while (iterator.hasNext() && samplesNeeded > 0) { + val host = iterator.next() + + if (random.nextInt(remainingHosts) < samplesNeeded) { + result.add(host) + samplesNeeded-- + } + + remainingHosts-- + } + + return result + } + + override fun select(failureIntensity: Double): List { + error("select with only int cannot be used in this type of VictimSelector"); + } + + override fun select(hosts: Set, failureIntensity:Double): List { + // clamp value between 0.0 and 1.0 + val intensity = min(1.0, max(0.0, failureIntensity)) + val numberOfHosts = (hosts.size * intensity).toInt() + + return hosts.asSequence().shuffled().take(numberOfHosts).toList() + } + + override fun toString(): String = "StochasticVictimSelector" +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/VictimSelector.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/VictimSelector.kt similarity index 80% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/VictimSelector.kt rename to opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/VictimSelector.kt index b56102840..ee337ba5e 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/VictimSelector.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/VictimSelector.kt @@ -20,7 +20,7 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure +package org.opendc.compute.simulator.failure.victimselector import org.opendc.compute.simulator.SimHost @@ -31,5 +31,8 @@ public interface VictimSelector { /** * Select the hosts from [hosts] where a fault will be injected. */ - public fun select(hosts: Set): List + public fun select(hosts: Set, numberOfHosts:Int): List + public fun select(numberOfHosts:Int): List + public fun select(failureIntensity: Double): List + public fun select(hosts: Set, failureIntensity: Double): List } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt index c5293a8d5..7d4b73020 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt @@ -38,7 +38,8 @@ internal object DefaultWorkloadMapper : SimWorkloadMapper { override fun createWorkload(server: Server): SimWorkload { val workload = delegate.createWorkload(server) - val bootWorkload = SimWorkloads.runtime(Duration.ofMillis(1), 0.8) + // FIXME: look at connecting this to frontend. Probably not needed since the duration is so small + val bootWorkload = SimWorkloads.runtime(Duration.ofMillis(1), 0.8, 0L, 0L) return SimWorkloads.chain(bootWorkload, workload) } } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt index e268c506c..827e57a23 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt @@ -186,7 +186,12 @@ internal class Guest( private fun doStop(target: ServerState) { assert(ctx != null) { "Invalid job state" } val ctx = ctx ?: return - ctx.shutdown() + if (target == ServerState.ERROR) { + ctx.shutdown(Exception("Stopped because of ERROR")) + } + else{ + ctx.shutdown() + } state = target } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/HostFaultInjectorImpl.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/RandomHostFaultInjector.kt similarity index 93% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/HostFaultInjectorImpl.kt rename to opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/RandomHostFaultInjector.kt index c75ce5287..2247ccca2 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/HostFaultInjectorImpl.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/RandomHostFaultInjector.kt @@ -29,9 +29,9 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.launch import org.apache.commons.math3.distribution.RealDistribution import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.HostFault +import org.opendc.compute.simulator.failure.hostfault.HostFault import org.opendc.compute.simulator.failure.HostFaultInjector -import org.opendc.compute.simulator.failure.VictimSelector +import org.opendc.compute.simulator.failure.victimselector.VictimSelector import java.time.InstantSource import kotlin.coroutines.CoroutineContext import kotlin.math.roundToLong @@ -46,7 +46,7 @@ import kotlin.math.roundToLong * @param selector The [VictimSelector] to select the host victims. * @param fault The type of [HostFault] to inject. */ -internal class HostFaultInjectorImpl( +internal class RandomHostFaultInjector( private val context: CoroutineContext, private val clock: InstantSource, private val hosts: Set, @@ -94,7 +94,7 @@ internal class HostFaultInjectorImpl( delay(d) - val victims = selector.select(hosts) + val victims = selector.select(hosts, 1) fault.apply(clock, victims) } } diff --git a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt index 690bf4724..3aa8cf5aa 100644 --- a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt +++ b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt @@ -29,6 +29,8 @@ import org.apache.commons.math3.distribution.LogNormalDistribution import org.apache.commons.math3.random.Well19937c import org.junit.jupiter.api.Test import org.opendc.compute.simulator.SimHost +import org.opendc.compute.simulator.failure.hostfault.StartStopHostFault +import org.opendc.compute.simulator.failure.victimselector.StochasticVictimSelector import org.opendc.simulator.kotlin.runSimulation import java.time.Duration import java.time.InstantSource diff --git a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt index 63719c0a2..92186dea9 100644 --- a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt +++ b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt @@ -28,17 +28,20 @@ import kotlinx.serialization.json.decodeFromStream import org.opendc.compute.topology.specs.TopologySpec import java.io.File import java.io.InputStream +import java.nio.file.Path +import kotlin.io.path.inputStream /** * A helper class for reading a topology specification file. */ public class TopologyReader { - @OptIn(ExperimentalSerializationApi::class) - public fun read(file: File): TopologySpec { - val input = file.inputStream() - val obj = Json.decodeFromStream(input) - return obj + public fun read(path: Path): TopologySpec { + return read(path.inputStream()) + } + + public fun read(file: File): TopologySpec { + return read(file.inputStream()) } /** diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt index deb50f5c8..88cedca45 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt @@ -51,4 +51,6 @@ public data class VirtualMachine( val stopTime: Instant, val trace: SimTrace, val interferenceProfile: VmInterferenceProfile?, -) +){ + val duration: Long = stopTime.toEpochMilli() - startTime.toEpochMilli() +} diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt index e1305b3f5..11fcc96de 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt @@ -24,7 +24,9 @@ package org.opendc.experiments.base.runner +import CheckpointModelSpec import FailureModelSpec +import getFailureModel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.launch @@ -34,8 +36,11 @@ import org.opendc.compute.api.Server import org.opendc.compute.api.ServerState import org.opendc.compute.api.ServerWatcher import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.failure.models.FailureModelNew import org.opendc.compute.workload.VirtualMachine import java.time.InstantSource +import java.util.Random +import kotlin.coroutines.coroutineContext import kotlin.math.max /** @@ -45,7 +50,7 @@ import kotlin.math.max */ public class RunningServerWatcher : ServerWatcher { // TODO: make this changeable - private val unlockStates: List = listOf(ServerState.TERMINATED, ServerState.ERROR, ServerState.DELETED) + private val unlockStates: List = listOf(ServerState.DELETED, ServerState.TERMINATED) private val mutex: Mutex = Mutex() @@ -80,18 +85,23 @@ public suspend fun ComputeService.replay( clock: InstantSource, trace: List, failureModelSpec: FailureModelSpec? = null, + checkpointModelSpec: CheckpointModelSpec? = null, seed: Long = 0, submitImmediately: Boolean = false, ) { - // TODO: add failureModel functionality val client = newClient() + // Create a failure model based on the failureModelSpec, if not null, otherwise set failureModel to null + val failureModel: FailureModelNew? = failureModelSpec?.let { + getFailureModel(coroutineContext, clock, this, Random(seed), it) } + // Create new image for the virtual machine val image = client.newImage("vm-image") try { coroutineScope { - // TODO: start failure model when implemented + // Start the fault injector + failureModel?.start() var simulationOffset = Long.MIN_VALUE @@ -109,7 +119,17 @@ public suspend fun ComputeService.replay( delay(max(0, (start - now - simulationOffset))) } - val workload = entry.trace.createWorkload(start) + val checkpointTime = checkpointModelSpec?.checkpointTime ?: 0L + val checkpointWait = checkpointModelSpec?.checkpointWait ?: 0L + +// val workload = SimRuntimeWorkload( +// entry.duration, +// 1.0, +// checkpointTime, +// checkpointWait +// ) + + val workload = entry.trace.createWorkload(start, checkpointTime, checkpointWait) val meta = mutableMapOf("workload" to workload) launch { @@ -140,7 +160,7 @@ public suspend fun ComputeService.replay( } yield() } finally { - // TODO: close failure model when implemented + failureModel?.close() client.close() } } diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt index d6ee5d72e..d09fcb595 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt @@ -116,7 +116,7 @@ public fun runScenario( val serviceDomain = "compute.opendc.org" Provisioner(dispatcher, seed).use { provisioner -> - val topology = clusterTopology(scenario.topology.pathToFile, Random(seed)) + val topology = clusterTopology(scenario.topologySpec.pathToFile, Random(seed)) provisioner.runSteps( setupComputeService( serviceDomain, @@ -125,18 +125,50 @@ public fun runScenario( setupHosts(serviceDomain, topology, optimize = true), ) - val workloadLoader = ComputeWorkloadLoader(File(scenario.workload.pathToFile)) - val vms = getWorkloadType(scenario.workload.type).resolve(workloadLoader, Random(seed)) + val workloadLoader = ComputeWorkloadLoader(File(scenario.workloadSpec.pathToFile)) + val vms = getWorkloadType(scenario.workloadSpec.type).resolve(workloadLoader, Random(seed)) val carbonTrace = getCarbonTrace(scenario.carbonTracePath) val startTime = Duration.ofMillis(vms.minOf { it.startTime }.toEpochMilli()) addExportModel(provisioner, serviceDomain, scenario, seed, startTime, carbonTrace, index) val service = provisioner.registry.resolve(serviceDomain, ComputeService::class.java)!! - service.replay(timeSource, vms, failureModelSpec = scenario.failureModel, seed = seed) + service.replay(timeSource, vms, failureModelSpec = scenario.failureModelSpec, seed = seed) } } +/** + * When the simulation is run, saves the simulation results into a seed folder. This is useful for debugging purposes. + * @param provisioner The provisioner used to setup and run the simulation. + * @param serviceDomain The domain of the compute service. + * @param scenario The scenario being run in the simulation. + * @param seed The seed used for randomness in the simulation. + * @param partition The partition name for the output data. + * @param startTime The start time of the simulation. + + */ +public fun saveInSeedFolder( + provisioner: Provisioner, + serviceDomain: String, + scenario: Scenario, + seed: Long, + partition: String, + startTime: Duration, +) { + provisioner.runStep( + registerComputeMonitor( + serviceDomain, + ParquetComputeMonitor( + File(scenario.outputFolder), + partition, + bufferSize = 4096, + ), + Duration.ofSeconds(scenario.exportModelSpec.exportInterval), + startTime, + ), + ) +} + /** * Saves the simulation results into a specific output folder received from the input. * @@ -164,7 +196,7 @@ public fun addExportModel( "seed=$seed", bufferSize = 4096, ), - Duration.ofSeconds(scenario.exportModel.exportInterval), + Duration.ofSeconds(scenario.exportModelSpec.exportInterval), startTime, carbonTrace, ), diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt index 7f0308fc3..d5091e4e4 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt @@ -23,6 +23,7 @@ package org.opendc.experiments.base.scenario import AllocationPolicySpec +import CheckpointModelSpec import ExportModelSpec import FailureModelSpec import ScenarioTopologySpec @@ -41,15 +42,15 @@ import WorkloadSpec * @property runs The Int representing the number of runs of the scenario. It defaults to 1. * @property initialSeed The Int representing the initial seed of the scenario. It defaults to 0. */ - public data class Scenario( var id: Int = -1, - val topology: ScenarioTopologySpec, - val workload: WorkloadSpec, - val allocationPolicy: AllocationPolicySpec, - val failureModel: FailureModelSpec?, + val topologySpec: ScenarioTopologySpec, + val workloadSpec: WorkloadSpec, + val allocationPolicySpec: AllocationPolicySpec, + val failureModelSpec: FailureModelSpec?, + val checkpointModelSpec: CheckpointModelSpec?, val carbonTracePath: String? = null, - val exportModel: ExportModelSpec = ExportModelSpec(), + val exportModelSpec: ExportModelSpec = ExportModelSpec(), val outputFolder: String = "output", val name: String = "", val runs: Int = 1, diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt index 19f8ebf0f..9b8e435da 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt @@ -22,8 +22,9 @@ package org.opendc.experiments.base.scenario +import AllocationPolicySpec +import FailureModelSpec import ScenarioTopologySpec -import org.opendc.experiments.base.scenario.specs.ScenarioSpec import java.io.File private val scenarioReader = ScenarioReader() @@ -70,25 +71,28 @@ public fun getScenarios(scenarioSpec: ScenarioSpec): List { for (workloadSpec in scenarioSpec.workloads) { for (allocationPolicySpec in scenarioSpec.allocationPolicies) { for (failureModelSpec in scenarioSpec.failureModels) { - for (carbonTracePath in scenarioSpec.carbonTracePaths) { - for (exportModelSpec in scenarioSpec.exportModels) { - val scenario = - Scenario( - id = scenarioID, - topology = scenarioTopologySpec, - workload = workloadSpec, - allocationPolicy = allocationPolicySpec, - failureModel = failureModelSpec, - carbonTracePath = carbonTracePath, - exportModel = exportModelSpec, - outputFolder = outputFolder, - name = scenarioID.toString(), - runs = scenarioSpec.runs, - initialSeed = scenarioSpec.initialSeed, - ) - trackScenario(scenarioSpec, outputFolder, scenario, scenarioTopologySpec) - scenarios.add(scenario) - scenarioID++ + for (checkpointModelSpec in scenarioSpec.checkpointModels) { + for (carbonTracePath in scenarioSpec.carbonTracePaths) { + for (exportModelSpec in scenarioSpec.exportModels) { + val scenario = + Scenario( + id = scenarioID, + topologySpec = scenarioTopologySpec, + workloadSpec = workloadSpec, + allocationPolicySpec = allocationPolicySpec, + failureModelSpec = failureModelSpec, + checkpointModelSpec = checkpointModelSpec, + carbonTracePath = carbonTracePath, + exportModelSpec = exportModelSpec, + outputFolder = scenarioSpec.outputFolder, + name = scenarioID.toString(), + runs = scenarioSpec.runs, + initialSeed = scenarioSpec.initialSeed, + ) + trackScenario(scenarioSpec, outputFolder, scenario, scenarioTopologySpec) + scenarios.add(scenario) + scenarioID++ + } } } } @@ -120,11 +124,11 @@ public fun trackScenario( id = scenario.id, name = scenarioSpec.name, topologies = listOf(topologySpec), - workloads = listOf(scenario.workload), - allocationPolicies = listOf(scenario.allocationPolicy), + workloads = listOf(scenario.workloadSpec), + allocationPolicies = listOf(scenario.allocationPolicySpec), // when implemented, add failure models here carbonTracePaths = listOf(scenario.carbonTracePath), - exportModels = listOf(scenario.exportModel), + exportModels = listOf(scenario.exportModelSpec), outputFolder = scenario.outputFolder, initialSeed = scenario.initialSeed, runs = scenario.runs, diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioReader.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioReader.kt index 19ce5a144..d461b56ad 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioReader.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioReader.kt @@ -28,14 +28,26 @@ import kotlinx.serialization.json.decodeFromStream import org.opendc.experiments.base.scenario.specs.ScenarioSpec import java.io.File import java.io.InputStream +import java.nio.file.Path +import kotlin.io.path.inputStream public class ScenarioReader { +// private val jsonReader = Json { serializersModule = failureModule } + private val jsonReader = Json + @OptIn(ExperimentalSerializationApi::class) public fun read(file: File): ScenarioSpec { val input = file.inputStream() - val obj = Json.decodeFromStream(input) - return obj + return jsonReader.decodeFromStream(input) + } + + + @OptIn(ExperimentalSerializationApi::class) + public fun read(path: Path): ScenarioSpec { + val input = path.inputStream() + + return jsonReader.decodeFromStream(input) } /** @@ -43,7 +55,6 @@ public class ScenarioReader { */ @OptIn(ExperimentalSerializationApi::class) public fun read(input: InputStream): ScenarioSpec { - val obj = Json.decodeFromStream(input) - return obj + return jsonReader.decodeFromStream(input) } } diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/CheckpointModelSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/CheckpointModelSpec.kt new file mode 100644 index 000000000..135a53bc7 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/CheckpointModelSpec.kt @@ -0,0 +1,7 @@ +import kotlinx.serialization.Serializable + +@Serializable +public data class CheckpointModelSpec ( + val checkpointWait: Long = 60*60*1000, + val checkpointTime: Long = 5*60*1000 +) diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt index 99620366b..fdb4bc90b 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt @@ -20,18 +20,124 @@ * SOFTWARE. */ +import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +import org.apache.commons.math3.distribution.ConstantRealDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.RealDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.failure.models.FailureModelNew +import org.opendc.compute.simulator.failure.models.SampleBasedFailureModel +import org.opendc.compute.simulator.failure.models.TraceBasedFailureModel +import org.opendc.compute.simulator.failure.prefab.FailurePrefab +import org.opendc.compute.simulator.failure.prefab.getFailureModelPrefab +import java.io.File +import java.time.InstantSource +import kotlin.coroutines.CoroutineContext + +@Serializable +public sealed interface FailureModelSpec { + public var name: String +} -/** - * specification describing the failure model - * - * @property failureInterval The interval between failures in s. Should be 0.0 or higher - */ @Serializable -public data class FailureModelSpec( - val failureInterval: Double = 0.0, -) { +@SerialName("trace-based") +public data class TraceBasedFailureModelSpec( + public val pathToFile: String +): FailureModelSpec { + override var name: String = File(pathToFile).nameWithoutExtension init { - require(failureInterval >= 0.0) { "failure frequency cannot be lower than 0" } + require(File(pathToFile).exists()) { "Path to file $pathToFile does not exist" } + } +} + +@Serializable +@SerialName("prefab") +public data class PrefabFailureModelSpec( + public val prefabName: FailurePrefab +): FailureModelSpec { + override var name: String = prefabName.toString() +} + +@Serializable +@SerialName("custom") +public data class CustomFailureModelSpec( + public val iatSampler: DistributionSpec, + public val durationSampler: DistributionSpec, + public val nohSampler: DistributionSpec, +): FailureModelSpec { + override var name: String = "custom" +} + +@Serializable +public sealed interface DistributionSpec + +@Serializable +@SerialName("log-normal") +public data class LogNormalDistributionSpec( + public val scale: Double, + public val size: Double, +): DistributionSpec + +@Serializable +@SerialName("constant") +public data class ConstantDistributionSpec( + public val value: Double +): DistributionSpec + + + + +public fun getFailureModel(context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: java.util.random.RandomGenerator, + failureModel: PrefabFailureModelSpec): FailureModelNew { + return getFailureModelPrefab(context, clock, service, random, failureModel.prefabName) +} + +public fun getFailureModel(context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: java.util.random.RandomGenerator, + failureModel: CustomFailureModelSpec): FailureModelNew { + val rng: org.apache.commons.math3.random.RandomGenerator = Well19937c(random.nextLong()) + + val iatSampler = getSampler(rng, failureModel.iatSampler) + val durationSampler = getSampler(rng, failureModel.durationSampler) + val nohSampler = getSampler(rng, failureModel.nohSampler) + + return SampleBasedFailureModel(context, clock, service, random, iatSampler, durationSampler, nohSampler) +} + +public fun getSampler(rng: org.apache.commons.math3.random.RandomGenerator, distributionSpec: DistributionSpec): RealDistribution { + return when (distributionSpec) { + is LogNormalDistributionSpec -> LogNormalDistribution(rng, distributionSpec.scale, distributionSpec.size) + is ConstantDistributionSpec -> ConstantRealDistribution(distributionSpec.value) + else -> error("The given distributionSpec: $distributionSpec is not a valid DistributionSpec") + } +} + +public fun getFailureModel(context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: java.util.random.RandomGenerator, + failureModel: TraceBasedFailureModelSpec): FailureModelNew { + + return TraceBasedFailureModel(context, clock, service, random, failureModel.pathToFile) +} + +public fun getFailureModel(context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: java.util.random.RandomGenerator, + failureModel: FailureModelSpec?): FailureModelNew? { + + return when(failureModel) { + is PrefabFailureModelSpec -> getFailureModel(context, clock, service, random, failureModel) + is CustomFailureModelSpec -> getFailureModel(context, clock, service, random, failureModel) + is TraceBasedFailureModelSpec -> getFailureModel(context, clock, service, random, failureModel) + else -> null } } diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt index 876a62cfc..3fbac1da7 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt @@ -23,6 +23,7 @@ package org.opendc.experiments.base.scenario.specs import AllocationPolicySpec +import CheckpointModelSpec import ExportModelSpec import FailureModelSpec import ScenarioTopologySpec @@ -49,7 +50,8 @@ public data class ScenarioSpec( val topologies: List, val workloads: List, val allocationPolicies: List, - val failureModels: List = listOf(FailureModelSpec()), + val failureModels: List = listOf(null), + val checkpointModels: List = listOf(null), val carbonTracePaths: List = listOf(null), val exportModels: List = listOf(ExportModelSpec()), val outputFolder: String = "output", diff --git a/opendc-experiments/opendc-experiments-scenario/src/main/resources/scenario.json b/opendc-experiments/opendc-experiments-scenario/src/main/resources/scenario.json index 854d9b8f8..3b22e5f4f 100644 --- a/opendc-experiments/opendc-experiments-scenario/src/main/resources/scenario.json +++ b/opendc-experiments/opendc-experiments-scenario/src/main/resources/scenario.json @@ -9,5 +9,8 @@ }, "allocationPolicy": { "policyType": "Mem" + }, + "failureModel": { + "name": "TEST" } } diff --git a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/power/CPUPowerModelsFactory.kt b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/power/CPUPowerModelsFactory.kt index e2f852f8c..5eafb5faa 100644 --- a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/power/CPUPowerModelsFactory.kt +++ b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/power/CPUPowerModelsFactory.kt @@ -22,6 +22,15 @@ package org.opendc.simulator.compute.power +// TODO: couple this correctly +public enum class CPUPowerModel { + Constant, + Sqrt, + Linear, + Square, + Cubic +} + public fun getPowerModel( modelType: String, power: Double, diff --git a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimRuntimeWorkload.java b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimRuntimeWorkload.java index bf54142f9..8547356d0 100644 --- a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimRuntimeWorkload.java +++ b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimRuntimeWorkload.java @@ -34,7 +34,7 @@ * A [SimWorkload] that models application execution as a single duration. */ public class SimRuntimeWorkload implements SimWorkload, FlowStageLogic { - private final long duration; + private long duration; private final double utilization; private SimMachineContext ctx; @@ -44,20 +44,35 @@ public class SimRuntimeWorkload implements SimWorkload, FlowStageLogic { private long remainingDuration; private long lastUpdate; + private long checkpointTime; // How long does it take to make a checkpoint? + private long checkpointWait; // How long to wait until a new checkpoint is made? + private long totalChecks; + /** * Construct a new {@link SimRuntimeWorkload}. * * @param duration The duration of the workload in milliseconds. * @param utilization The CPU utilization of the workload. */ - SimRuntimeWorkload(long duration, double utilization) { + public SimRuntimeWorkload(long duration, double utilization, long checkpointTime, long checkpointWait) { if (duration < 0) { throw new IllegalArgumentException("Duration must be positive"); } else if (utilization <= 0.0 || utilization > 1.0) { throw new IllegalArgumentException("Utilization must be in (0, 1]"); } + this.checkpointTime = checkpointTime; + this.checkpointWait = checkpointWait; this.duration = duration; + + if (this.checkpointWait > 0) { + // Determine the number of checkpoints that need to be made during the workload + // If the total duration is divisible by the wait time between checkpoints, we can remove the last checkpoint + int to_remove = ((this.duration % this.checkpointWait == 0) ? 1 : 0); + this.totalChecks = this.duration / this.checkpointWait - to_remove; + this.duration += (this.checkpointTime * totalChecks); + } + this.utilization = utilization; this.remainingDuration = duration; } @@ -108,7 +123,25 @@ public SimRuntimeWorkload snapshot() { stage.sync(); } - return new SimRuntimeWorkload(remainingDuration, utilization); + var remaining_time = this.remainingDuration; + + if (this.checkpointWait > 0) { + // Calculate last checkpoint + var total_check_time = this.checkpointWait + this.checkpointTime; + var processed_time = this.duration - this.remainingDuration; + var processed_checks = (int)(processed_time / total_check_time); + var processed_time_last_check = (processed_checks * total_check_time); // The processed time after the last checkpoint + + remaining_time = this.duration - processed_time_last_check; // The remaining duration to process after last checkpoint + var remaining_checks = (int)(remaining_time / total_check_time); + remaining_time -= (remaining_checks * checkpointTime); + } + else { + remaining_time = duration; + } + + + return new SimRuntimeWorkload(remaining_time, utilization, this.checkpointTime, this.checkpointWait); } @Override @@ -119,6 +152,12 @@ public long onUpdate(FlowStage ctx, long now) { long delta = now - lastUpdate; long duration = this.remainingDuration - delta; + if (delta == 0 && this.ctx == null) { + // This means the workload has been terminated + // But, has not executed to completion + return Long.MAX_VALUE; + } + if (duration <= 0) { final SimMachineContext machineContext = this.ctx; if (machineContext != null) { diff --git a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimTrace.java b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimTrace.java index f2f4ce45f..363e100e7 100644 --- a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimTrace.java +++ b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimTrace.java @@ -70,8 +70,8 @@ private SimTrace(double[] usageCol, long[] deadlineCol, int[] coresCol, int size * * // * @param offset The offset for the timestamps. */ - public SimWorkload createWorkload(long start) { - return new Workload(start, usageCol, deadlineCol, coresCol, size, 0); + public SimWorkload createWorkload(long start, long checkpointTime, long checkpointWait) { + return new Workload(start, usageCol, deadlineCol, coresCol, size, 0, checkpointTime, checkpointWait); } /** @@ -215,13 +215,20 @@ private static class Workload implements SimWorkload { private final int size; private final int index; - private Workload(long start, double[] usageCol, long[] deadlineCol, int[] coresCol, int size, int index) { + private long checkpointTime; // How long does it take to make a checkpoint? + private long checkpointWait; // How long to wait until a new checkpoint is made? + private long total_checks; + + private Workload(long start, double[] usageCol, long[] deadlineCol, int[] coresCol, int size, int index, + long checkpointTime, long checkpointWait) { this.start = start; this.usageCol = usageCol; this.deadlineCol = deadlineCol; this.coresCol = coresCol; this.size = size; this.index = index; + this.checkpointTime = checkpointTime; + this.checkpointWait = checkpointWait; } @Override @@ -259,7 +266,7 @@ public SimWorkload snapshot() { index = logic.getIndex(); } - return new Workload(start, usageCol, deadlineCol, coresCol, size, index); + return new Workload(start, usageCol, deadlineCol, coresCol, size, index, checkpointTime, checkpointWait); } } diff --git a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimWorkloads.java b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimWorkloads.java index 82557d06b..ccaa70e94 100644 --- a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimWorkloads.java +++ b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimWorkloads.java @@ -47,8 +47,8 @@ public static SimWorkload flops(long flops, double utilization) { * @param duration The duration of the workload in milliseconds. * @param utilization The CPU utilization of the workload. */ - public static SimWorkload runtime(long duration, double utilization) { - return new SimRuntimeWorkload(duration, utilization); + public static SimWorkload runtime(long duration, double utilization, long checkpoint_time, long checkpoint_wait) { + return new SimRuntimeWorkload(duration, utilization, checkpoint_time, checkpoint_wait); } /** @@ -57,8 +57,8 @@ public static SimWorkload runtime(long duration, double utilization) { * @param duration The duration of the workload. * @param utilization The CPU utilization of the workload. */ - public static SimWorkload runtime(Duration duration, double utilization) { - return runtime(duration.toMillis(), utilization); + public static SimWorkload runtime(Duration duration, double utilization, long checkpoint_time, long checkpoint_wait) { + return runtime(duration.toMillis(), utilization, checkpoint_time, checkpoint_wait); } /** diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt new file mode 100644 index 000000000..01d66a4ad --- /dev/null +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +@file:JvmName("FailureColumns") + +package org.opendc.trace.conv + +/** + * A column containing the task identifier. + */ +public const val FAILURE_START: String = "failure_start" + +/** + * A column containing the task identifier. + */ +public const val FAILURE_DURATION: String = "failure_duration" + +/** + * A column containing the task identifier. + */ +public const val FAILURE_INTENSITY: String = "failure_intensity" diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/Tables.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/Tables.kt index 9b8fc6cfa..d4019f732 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/Tables.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/Tables.kt @@ -49,4 +49,6 @@ public const val TABLE_RESOURCE_STATES: String = "resource_states" */ public const val TABLE_INTERFERENCE_GROUPS: String = "interference_groups" -public const val TABLE_CARBON_INTENSITY: String = "carbon_intensities" +public const val TABLE_CARBON_INTENSITIES: String = "carbon_intensities" + +public const val TABLE_FAILURES: String = "failures" diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/carbon/CarbonTraceFormat.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/carbon/CarbonTraceFormat.kt index 0daa1297e..d8adc739f 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/carbon/CarbonTraceFormat.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/carbon/CarbonTraceFormat.kt @@ -28,7 +28,7 @@ import org.opendc.trace.TableReader import org.opendc.trace.TableWriter import org.opendc.trace.conv.CARBON_INTENSITY_TIMESTAMP import org.opendc.trace.conv.CARBON_INTENSITY_VALUE -import org.opendc.trace.conv.TABLE_CARBON_INTENSITY +import org.opendc.trace.conv.TABLE_CARBON_INTENSITIES import org.opendc.trace.formats.carbon.parquet.CarbonIntensityReadSupport import org.opendc.trace.spi.TableDetails import org.opendc.trace.spi.TraceFormat @@ -45,14 +45,14 @@ public class CarbonTraceFormat : TraceFormat { throw UnsupportedOperationException("Writing not supported for this format") } - override fun getTables(path: Path): List = listOf(TABLE_CARBON_INTENSITY) + override fun getTables(path: Path): List = listOf(TABLE_CARBON_INTENSITIES) override fun getDetails( path: Path, table: String, ): TableDetails { return when (table) { - TABLE_CARBON_INTENSITY -> + TABLE_CARBON_INTENSITIES -> TableDetails( listOf( TableColumn(CARBON_INTENSITY_TIMESTAMP, TableColumnType.Instant), @@ -69,7 +69,7 @@ public class CarbonTraceFormat : TraceFormat { projection: List?, ): TableReader { return when (table) { - TABLE_CARBON_INTENSITY -> { + TABLE_CARBON_INTENSITIES -> { val reader = LocalParquetReader(path, CarbonIntensityReadSupport(projection)) CarbonTableReader(reader) } diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt new file mode 100644 index 000000000..3dc511332 --- /dev/null +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.trace.formats.failure + +import org.opendc.trace.TableReader +import org.opendc.trace.conv.FAILURE_DURATION +import org.opendc.trace.conv.FAILURE_INTENSITY +import org.opendc.trace.conv.FAILURE_START +import org.opendc.trace.formats.failure.parquet.FailureFragment +import org.opendc.trace.util.parquet.LocalParquetReader +import java.time.Duration +import java.time.Instant +import java.util.UUID + +/** + * A [TableReader] implementation for the WTF format. + */ +internal class FailureTableReader(private val reader: LocalParquetReader) : TableReader { + /** + * The current record. + */ + private var record: FailureFragment? = null + + override fun nextRow(): Boolean { + try { + val record = reader.read() + this.record = record + + return record != null + } catch (e: Throwable) { + this.record = null + throw e + } + } + + private val colFailureStart = 0 + private val colFailureDuration = 1 + private val colFailureIntensity = 2 + + override fun resolve(name: String): Int { + return when (name) { + FAILURE_START -> colFailureStart + FAILURE_DURATION -> colFailureDuration + FAILURE_INTENSITY -> colFailureIntensity + else -> -1 + } + } + + override fun isNull(index: Int): Boolean { + require(index in colFailureStart..colFailureIntensity) { "Invalid column index" } + return false + } + + override fun getBoolean(index: Int): Boolean { + throw IllegalArgumentException("Invalid column") + } + + override fun getInt(index: Int): Int { + throw IllegalArgumentException("Invalid column") + } + + override fun getLong(index: Int): Long { + val record = checkNotNull(record) { "Reader in invalid state" } + return when (index) { + colFailureStart -> record.failureStart + colFailureDuration -> record.failureDuration + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun getFloat(index: Int): Float { + throw IllegalArgumentException("Invalid column") + } + + override fun getDouble(index: Int): Double { + val record = checkNotNull(record) { "Reader in invalid state" } + return when (index) { + colFailureIntensity -> record.failureIntensity + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun getString(index: Int): String { + throw IllegalArgumentException("Invalid column") + } + + override fun getUUID(index: Int): UUID? { + throw IllegalArgumentException("Invalid column") + } + + override fun getInstant(index: Int): Instant { + throw IllegalArgumentException("Invalid column") + } + + override fun getDuration(index: Int): Duration { + throw IllegalArgumentException("Invalid column") + } + + override fun getList( + index: Int, + elementType: Class, + ): List? { + throw IllegalArgumentException("Invalid column") + } + + override fun getSet( + index: Int, + elementType: Class, + ): Set? { + throw IllegalArgumentException("Invalid column") + } + + override fun getMap( + index: Int, + keyType: Class, + valueType: Class, + ): Map? { + throw IllegalArgumentException("Invalid column") + } + + override fun close() { + reader.close() + } +} diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTraceFormat.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTraceFormat.kt new file mode 100644 index 000000000..d30a446ea --- /dev/null +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTraceFormat.kt @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.trace.formats.failure + +import org.opendc.trace.TableColumn +import org.opendc.trace.TableColumnType +import org.opendc.trace.TableReader +import org.opendc.trace.TableWriter +import org.opendc.trace.conv.FAILURE_DURATION +import org.opendc.trace.conv.FAILURE_INTENSITY +import org.opendc.trace.conv.FAILURE_START +import org.opendc.trace.conv.TABLE_FAILURES +import org.opendc.trace.formats.failure.parquet.FailureReadSupport +import org.opendc.trace.spi.TableDetails +import org.opendc.trace.spi.TraceFormat +import org.opendc.trace.util.parquet.LocalParquetReader +import java.nio.file.Path + +/** + * A [TraceFormat] implementation for the Failure Intensity trace. + */ +public class FailureTraceFormat : TraceFormat { + override val name: String = "failure" + + override fun create(path: Path) { + throw UnsupportedOperationException("Writing not supported for this format") + } + + override fun getTables(path: Path): List = listOf(TABLE_FAILURES) + + override fun getDetails( + path: Path, + table: String, + ): TableDetails { + return when (table) { + TABLE_FAILURES -> + TableDetails( + listOf( + TableColumn(FAILURE_START, TableColumnType.Long), + TableColumn(FAILURE_DURATION, TableColumnType.Long), + TableColumn(FAILURE_INTENSITY, TableColumnType.Double), + ), + ) + else -> throw IllegalArgumentException("Table $table not supported") + } + } + + override fun newReader( + path: Path, + table: String, + projection: List?, + ): TableReader { + return when (table) { + TABLE_FAILURES -> { + val reader = LocalParquetReader(path, FailureReadSupport(projection)) + FailureTableReader(reader) + } + else -> throw IllegalArgumentException("Table $table not supported") + } + } + + override fun newWriter( + path: Path, + table: String, + ): TableWriter { + throw UnsupportedOperationException("Writing not supported for this format") + } +} diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureFragment.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureFragment.kt new file mode 100644 index 000000000..09a68c024 --- /dev/null +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureFragment.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.trace.formats.failure.parquet + +/** + * A task in the Workflow Trace Format. + */ +internal data class FailureFragment( + val failureStart: Long, + val failureDuration: Long, + val failureIntensity: Double, +) diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt new file mode 100644 index 000000000..13d60d4c9 --- /dev/null +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.trace.formats.failure.parquet + +import org.apache.hadoop.conf.Configuration +import org.apache.parquet.hadoop.api.InitContext +import org.apache.parquet.hadoop.api.ReadSupport +import org.apache.parquet.io.api.RecordMaterializer +import org.apache.parquet.schema.MessageType +import org.apache.parquet.schema.PrimitiveType +import org.apache.parquet.schema.Types +import org.opendc.trace.conv.FAILURE_DURATION +import org.opendc.trace.conv.FAILURE_INTENSITY +import org.opendc.trace.conv.FAILURE_START + +/** + * A [ReadSupport] instance for [Task] objects. + * + * @param projection The projection of the table to read. + */ +internal class FailureReadSupport(private val projection: List?) : ReadSupport() { + /** + * Mapping of table columns to their Parquet column names. + */ + private val colMap = + mapOf( + FAILURE_START to "start", + FAILURE_DURATION to "duration", + FAILURE_INTENSITY to "intensity" + ) + + override fun init(context: InitContext): ReadContext { + val projectedSchema = + if (projection != null) { + Types.buildMessage() + .apply { + val fieldByName = READ_SCHEMA.fields.associateBy { it.name } + + for (col in projection) { + val fieldName = colMap[col] ?: continue + addField(fieldByName.getValue(fieldName)) + } + } + .named(READ_SCHEMA.name) + } else { + READ_SCHEMA + } + return ReadContext(projectedSchema) + } + + override fun prepareForRead( + configuration: Configuration, + keyValueMetaData: Map, + fileSchema: MessageType, + readContext: ReadContext, + ): RecordMaterializer = FailureRecordMaterializer(readContext.requestedSchema) + + companion object { + /** + * Parquet read schema for the "tasks" table in the trace. + */ + @JvmStatic + val READ_SCHEMA: MessageType = + Types.buildMessage() + .addFields( + Types + .optional(PrimitiveType.PrimitiveTypeName.INT64) + .named("failure_start"), + Types + .optional(PrimitiveType.PrimitiveTypeName.INT64) + .named("failure_duration"), + Types + .optional(PrimitiveType.PrimitiveTypeName.DOUBLE) + .named("failure_intensity"), + ) + .named("failure_fragment") + } +} diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt new file mode 100644 index 000000000..5762c2bae --- /dev/null +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2022 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.trace.formats.failure.parquet + +import org.apache.parquet.io.api.Converter +import org.apache.parquet.io.api.GroupConverter +import org.apache.parquet.io.api.PrimitiveConverter +import org.apache.parquet.io.api.RecordMaterializer +import org.apache.parquet.schema.MessageType + +/** + * A [RecordMaterializer] for [Task] records. + */ +internal class FailureRecordMaterializer(schema: MessageType) : RecordMaterializer() { + /** + * State of current record being read. + */ + private var localStart: Long = 0L + private var localDuration: Long = 0L + private var localIntensity: Double = 0.0 + + /** + * Root converter for the record. + */ + private val root = + object : GroupConverter() { + /** + * The converters for the columns of the schema. + */ + private val converters = + schema.fields.map { type -> + when (type.name) { + "failure_start" -> + object : PrimitiveConverter() { + override fun addLong(value: Long) { + localStart = value + } + } + "failure_duration" -> + object : PrimitiveConverter() { + override fun addLong(value: Long) { + localDuration = value + } + } + "failure_intensity" -> + object : PrimitiveConverter() { + override fun addDouble(value: Double) { + localIntensity = value + } + } + else -> error("Unknown column $type") + } + } + + override fun start() { + localStart = 0L + localDuration = 0L + localIntensity = 0.0 + } + + override fun end() {} + + override fun getConverter(fieldIndex: Int): Converter = converters[fieldIndex] + } + + override fun getCurrentRecord(): FailureFragment = + FailureFragment( + localStart, + localDuration, + localIntensity + ) + + override fun getRootConverter(): GroupConverter = root +} diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/spi/TraceFormat.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/spi/TraceFormat.kt index 67df667b2..e586f90ad 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/spi/TraceFormat.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/spi/TraceFormat.kt @@ -27,6 +27,7 @@ import org.opendc.trace.TableWriter import org.opendc.trace.azure.AzureTraceFormat import org.opendc.trace.bitbrains.BitbrainsTraceFormat import org.opendc.trace.formats.carbon.CarbonTraceFormat +import org.opendc.trace.formats.failure.FailureTraceFormat import org.opendc.trace.formats.opendc.OdcVmTraceFormat import org.opendc.trace.gwf.GwfTraceFormat import org.opendc.trace.swf.SwfTraceFormat @@ -124,6 +125,7 @@ public interface TraceFormat { "azure" -> AzureTraceFormat() "bitbrains" -> BitbrainsTraceFormat() "carbon" -> CarbonTraceFormat() + "failure" -> FailureTraceFormat() "gwf" -> GwfTraceFormat() "opendc-vm" -> OdcVmTraceFormat() "swf" -> SwfTraceFormat() diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt index d69134121..15fc989c0 100644 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt @@ -25,7 +25,7 @@ package org.opendc.web.runner import mu.KotlinLogging import org.opendc.compute.service.ComputeService import org.opendc.compute.service.scheduler.createComputeScheduler -import org.opendc.compute.simulator.failure.models.Grid5000 +import org.opendc.compute.simulator.failure.prefab.Grid5000 import org.opendc.compute.simulator.provisioner.Provisioner import org.opendc.compute.simulator.provisioner.registerComputeMonitor import org.opendc.compute.simulator.provisioner.setupComputeService diff --git a/settings.gradle.kts b/settings.gradle.kts index 3e3ac677d..66601847c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,6 +57,7 @@ include(":opendc-trace:opendc-trace-api") include(":opendc-trace:opendc-trace-calcite") include(":opendc-trace:opendc-trace-parquet") include(":opendc-trace:opendc-trace-testkit") +include(":opendc-trace:opendc-trace-failure") include(":opendc-trace:opendc-trace-tools") enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS") From 8de555b6a8a8a9677c450f3d33e56224691bb392 Mon Sep 17 00:00:00 2001 From: Dante Niewenhuis Date: Thu, 2 May 2024 13:41:13 +0200 Subject: [PATCH 2/5] Removed old failure models --- .../simulator/failure/FailureModelFactory.kt | 40 ------- .../simulator/failure/HostFaultInjector.kt | 68 ----------- .../simulator/failure/hostfault/HostFault.kt | 12 +- .../failure/hostfault/HostFaultNew.kt | 43 ------- .../failure/hostfault/StartStopHostFault.kt | 34 +++--- .../hostfault/StartStopHostFaultNew.kt | 64 ----------- .../failure/models/ConstantFailureModel.kt | 25 ---- .../simulator/failure/models/FailureModel.kt | 59 ++++++++-- .../failure/models/FailureModelNew.kt | 85 -------------- .../failure/models/SampleBasedFailureModel.kt | 2 +- .../failure/models/TraceBasedFailureModel.kt | 2 +- .../simulator/failure/prefab/Grid5000.kt | 72 ------------ ...000New.kt => PrefabFailureModelFactory.kt} | 0 .../StochasticVictimSelector.kt | 8 +- .../StochasticVictimSelectorNew.kt | 77 ------------- .../internal/RandomHostFaultInjector.kt | 108 ------------------ .../failure/HostFaultInjectorTest.kt | 2 - .../base/runner/ScenarioHelpers.kt | 4 +- .../base/scenario/ScenarioFactories.kt | 11 +- .../base/scenario/specs/FailureModelSpec.kt | 10 +- .../org/opendc/web/runner/OpenDCRunner.kt | 1 - 21 files changed, 97 insertions(+), 630 deletions(-) delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFaultNew.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFaultNew.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/ConstantFailureModel.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModelNew.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000.kt rename opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/{Grid5000New.kt => PrefabFailureModelFactory.kt} (100%) delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelectorNew.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/RandomHostFaultInjector.kt diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt deleted file mode 100644 index fcd7a7ac6..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2024 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure - -import org.opendc.compute.simulator.failure.models.FailureModelNew - -/** - * Get failure model - * - * @param failureInterval The interval of failures occurring in s - * @return - */ -public fun getFailureModelOld(failureInterval: Long): FailureModelNew? { - return if (failureInterval > 0) { -// createGrid5000() - null - } else { - null - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt deleted file mode 100644 index 8d1acb626..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure - -import org.apache.commons.math3.distribution.RealDistribution -import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.hostfault.HostFault -import org.opendc.compute.simulator.failure.victimselector.VictimSelector -import org.opendc.compute.simulator.internal.RandomHostFaultInjector -import java.time.Clock -import java.time.InstantSource -import kotlin.coroutines.CoroutineContext - -/** - * An interface for stochastically injecting faults into a set of hosts. - */ -public interface HostFaultInjector : AutoCloseable { - /** - * Start fault injection. - */ - public fun start() - - /** - * Stop fault injection into the system. - */ - public override fun close() - - public companion object { - /** - * Construct a new [HostFaultInjector]. - * - * @param context The scope to run the fault injector in. - * @param clock The [Clock] to keep track of simulation time. - * @param hosts The hosts to inject the faults into. - * @param iat The inter-arrival time distribution of the failures (in hours). - * @param selector The [VictimSelector] to select the host victims. - * @param fault The type of [HostFault] to inject. - */ - public operator fun invoke( - context: CoroutineContext, - clock: InstantSource, - hosts: Set, - iat: RealDistribution, - selector: VictimSelector, - fault: HostFault, - ): HostFaultInjector = RandomHostFaultInjector(context, clock, hosts, iat, selector, fault) - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFault.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFault.kt index 4b08164f9..8dadb1b13 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFault.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFault.kt @@ -22,18 +22,22 @@ package org.opendc.compute.simulator.failure.hostfault +import org.opendc.compute.service.ComputeService import org.opendc.compute.simulator.SimHost import java.time.InstantSource /** * Interface responsible for applying the fault to a host. */ -public interface HostFault { +public abstract class HostFault( + private val service: ComputeService, + private val clock: InstantSource, +) { /** * Apply the fault to the specified [victims]. */ - public suspend fun apply( - clock: InstantSource, - victims: List + public abstract suspend fun apply( + victims: List, + faultDuration: Long ) } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFaultNew.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFaultNew.kt deleted file mode 100644 index 26c7ce72e..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFaultNew.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure.hostfault - -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.SimHost -import java.time.InstantSource - -/** - * Interface responsible for applying the fault to a host. - */ -public abstract class HostFaultNew( - private val service: ComputeService, - private val clock: InstantSource, -) { - /** - * Apply the fault to the specified [victims]. - */ - public abstract suspend fun apply( - victims: List, - faultDuration: Long - ) -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFault.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFault.kt index 615907df5..f1ca454ad 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFault.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFault.kt @@ -23,38 +23,42 @@ package org.opendc.compute.simulator.failure.hostfault import kotlinx.coroutines.delay -import org.apache.commons.math3.distribution.RealDistribution +import org.opendc.compute.api.ComputeClient +import org.opendc.compute.service.ComputeService import org.opendc.compute.simulator.SimHost +import org.opendc.simulator.compute.workload.SimWorkload import java.time.InstantSource -import kotlin.math.roundToLong /** - * A type of [HostFault] where the hosts are stopped and recover after some random amount of time. - * - * @property durationSampler A RealDistribution which the duration of the error is sampled from + * A type of [HostFault] where the hosts are stopped and recover after a given amount of time. */ -public class StartStopHostFault(private val durationSampler: RealDistribution) : HostFault { +public class StartStopHostFault ( + private val service: ComputeService, + clock: InstantSource +): HostFault(service, clock) { override suspend fun apply( - clock: InstantSource, victims: List, + faultDuration: Long ) { + val client: ComputeClient = service.newClient() + for (host in victims) { - host.fail() - } + val servers = host.instances - val df = (durationSampler.sample() * 1000).roundToLong() // seconds to milliseconds + val snapshots = servers.map{(it.meta["workload"] as SimWorkload).snapshot()} + host.fail() - // Handle long overflow - if (clock.millis() + df <= 0) { - return + for ((server, snapshot) in servers.zip(snapshots)) { + client.rescheduleServer(server, snapshot) + } } - delay(df) + delay(faultDuration) for (host in victims) { host.recover() } } - override fun toString(): String = "StartStopHostFault[$durationSampler]" + override fun toString(): String = "StartStopHostFault" } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFaultNew.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFaultNew.kt deleted file mode 100644 index 61c78d96f..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFaultNew.kt +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure.hostfault - -import kotlinx.coroutines.delay -import org.opendc.compute.api.ComputeClient -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.SimHost -import org.opendc.simulator.compute.workload.SimWorkload -import java.time.InstantSource - -/** - * A type of [HostFault] where the hosts are stopped and recover after a given amount of time. - */ -public class StartStopHostFaultNew ( - private val service: ComputeService, - clock: InstantSource -): HostFaultNew(service, clock) { - override suspend fun apply( - victims: List, - faultDuration: Long - ) { - val client: ComputeClient = service.newClient() - - for (host in victims) { - val servers = host.instances - - val snapshots = servers.map{(it.meta["workload"] as SimWorkload).snapshot()} - host.fail() - - for ((server, snapshot) in servers.zip(snapshots)) { - client.rescheduleServer(server, snapshot) - } - } - - delay(faultDuration) - - for (host in victims) { - host.recover() - } - } - - override fun toString(): String = "StartStopHostFault" -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/ConstantFailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/ConstantFailureModel.kt deleted file mode 100644 index 27fb8c476..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/ConstantFailureModel.kt +++ /dev/null @@ -1,25 +0,0 @@ -package org.opendc.compute.simulator.failure.models - -import kotlinx.coroutines.delay -import org.opendc.compute.service.ComputeService -import java.time.InstantSource -import java.util.random.RandomGenerator -import kotlin.coroutines.CoroutineContext - -public class ConstantFailureModel( - context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: RandomGenerator, - private val numberOfVictims: Int, - private val waitDuration: Long, - private val faultDuration: Long, - ): FailureModelNew(context, clock, service, random) { - override suspend fun runInjector() { - delay(waitDuration) - - val victims = victimSelector.select(hosts, numberOfVictims) - - fault.apply(victims, faultDuration) - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModel.kt index be8c7a252..439c31691 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModel.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModel.kt @@ -22,8 +22,15 @@ package org.opendc.compute.simulator.failure.models +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.failure.HostFaultInjector +import org.opendc.compute.simulator.SimHost +import org.opendc.compute.simulator.failure.hostfault.HostFault +import org.opendc.compute.simulator.failure.hostfault.StartStopHostFault +import org.opendc.compute.simulator.failure.victimselector.StochasticVictimSelector import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext @@ -31,14 +38,48 @@ import kotlin.coroutines.CoroutineContext /** * Factory interface for constructing [HostFaultInjector] for modeling failures of compute service hosts. */ -public interface FailureModel { +public abstract class FailureModel ( + context: CoroutineContext, + protected val clock: InstantSource, + protected val service: ComputeService, + protected val random: RandomGenerator +): AutoCloseable { + protected val scope: CoroutineScope = CoroutineScope(context + Job()) + + // TODO: could at some point be extended to different types of faults + protected val fault: HostFault = StartStopHostFault(service, clock) + + // TODO: could at some point be extended to different types of victim selectors + protected val victimSelector: StochasticVictimSelector = StochasticVictimSelector(random) + + protected val hosts: Set = service.hosts.map { it as SimHost }.toSet() + + /** + * The [Job] that awaits the nearest fault in the system. + */ + private var job: Job? = null + + /** + * Start the fault injection into the system. + */ + public fun start() { + if (job != null) { + return + } + + job = + scope.launch { + runInjector() + job = null + } + } + + public abstract suspend fun runInjector() + /** - * Construct a [HostFaultInjector] for the specified [service]. + * Stop the fault injector. */ - public fun createInjector( - context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: RandomGenerator, - ): HostFaultInjector + public override fun close() { + scope.cancel() + } } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModelNew.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModelNew.kt deleted file mode 100644 index ca7379987..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModelNew.kt +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure.models - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.cancel -import kotlinx.coroutines.launch -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.hostfault.HostFaultNew -import org.opendc.compute.simulator.failure.hostfault.StartStopHostFaultNew -import org.opendc.compute.simulator.failure.victimselector.StochasticVictimSelectorNew -import java.time.InstantSource -import java.util.random.RandomGenerator -import kotlin.coroutines.CoroutineContext - -/** - * Factory interface for constructing [HostFaultInjector] for modeling failures of compute service hosts. - */ -public abstract class FailureModelNew ( - context: CoroutineContext, - protected val clock: InstantSource, - protected val service: ComputeService, - protected val random: RandomGenerator -): AutoCloseable { - protected val scope: CoroutineScope = CoroutineScope(context + Job()) - - // TODO: could at some point be extended to different types of faults - protected val fault: HostFaultNew = StartStopHostFaultNew(service, clock) - - // TODO: could at some point be extended to different types of victim selectors - protected val victimSelector: StochasticVictimSelectorNew = StochasticVictimSelectorNew(random) - - protected val hosts: Set = service.hosts.map { it as SimHost }.toSet() - - /** - * The [Job] that awaits the nearest fault in the system. - */ - private var job: Job? = null - - /** - * Start the fault injection into the system. - */ - public fun start() { - if (job != null) { - return - } - - job = - scope.launch { - runInjector() - job = null - } - } - - public abstract suspend fun runInjector() - - /** - * Stop the fault injector. - */ - public override fun close() { - scope.cancel() - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt index 8c4004986..be32dba03 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt @@ -29,7 +29,7 @@ public class SampleBasedFailureModel( private val iatSampler: RealDistribution, private val durationSampler: RealDistribution, private val nohSampler: RealDistribution - ): FailureModelNew(context, clock, service, random) { + ): FailureModel(context, clock, service, random) { override suspend fun runInjector() { while(true) { val d = (iatSampler.sample() * 3.6e6).roundToLong() diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt index 73498767f..d78270355 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt @@ -26,7 +26,7 @@ public class TraceBasedFailureModel( service: ComputeService, random: RandomGenerator, pathToTrace: String - ) : FailureModelNew(context, clock, service, random){ + ) : FailureModel(context, clock, service, random){ private val failureList = FailureTraceLoader().get(File(pathToTrace)).iterator() diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000.kt deleted file mode 100644 index 1f3dfeb24..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000.kt +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -@file:JvmName("FailureModels") - -package org.opendc.compute.simulator.failure.prefab - -import org.apache.commons.math3.distribution.LogNormalDistribution -import org.apache.commons.math3.random.Well19937c -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.models.FailureModel -import org.opendc.compute.simulator.failure.HostFaultInjector -import org.opendc.compute.simulator.failure.hostfault.StartStopHostFault -import org.opendc.compute.simulator.failure.victimselector.StochasticVictimSelector -import java.time.Duration -import java.time.InstantSource -import java.util.random.RandomGenerator -import kotlin.coroutines.CoroutineContext -import kotlin.math.ln - -/** - * A [FailureModel] based on the GRID'5000 failure trace. - * - * This fault injector uses parameters from the GRID'5000 failure trace as described in - * "A Framework for the Study of Grid Inter-Operation Mechanisms", A. Iosup, 2009. - */ -public class Grid5000(private val failureInterval: Duration) : FailureModel { - override fun createInjector( - context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: RandomGenerator, - ): HostFaultInjector { - val rng = Well19937c(random.nextLong()) - val hosts = service.hosts.map { it as SimHost }.toSet() - - // Parameters from A. Iosup, A Framework for the Study of Grid Inter-Operation Mechanisms, 2009 - // GRID'5000 - // fixme: currently only does something when Duration is higher than 1 hour - return HostFaultInjector( - context, - clock, - hosts, - iat = LogNormalDistribution(rng, ln(failureInterval.toHours().toDouble()), 1.03), -// selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25), random), - selector = StochasticVictimSelector(random), - fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)), - ) - } - - override fun toString(): String = "Grid5000FailureModel" -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000New.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/PrefabFailureModelFactory.kt similarity index 100% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/Grid5000New.kt rename to opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/PrefabFailureModelFactory.kt diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelector.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelector.kt index 109c5a90a..7ce4f9c53 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelector.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelector.kt @@ -25,6 +25,8 @@ package org.opendc.compute.simulator.failure.victimselector import org.opendc.compute.simulator.SimHost import java.util.SplittableRandom import java.util.random.RandomGenerator +import kotlin.math.max +import kotlin.math.min /** * A [VictimSelector] that stochastically selects a set of hosts to be failed. @@ -64,7 +66,11 @@ public class StochasticVictimSelector( } override fun select(hosts: Set, failureIntensity:Double): List { - error("select with only int cannot be used in this type of VictimSelector"); + // clamp value between 0.0 and 1.0 + val intensity = min(1.0, max(0.0, failureIntensity)) + val numberOfHosts = (hosts.size * intensity).toInt() + + return hosts.asSequence().shuffled().take(numberOfHosts).toList() } override fun toString(): String = "StochasticVictimSelector" diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelectorNew.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelectorNew.kt deleted file mode 100644 index fcf5f8602..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelectorNew.kt +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure.victimselector - -import org.opendc.compute.simulator.SimHost -import java.util.SplittableRandom -import java.util.random.RandomGenerator -import kotlin.math.max -import kotlin.math.min - -/** - * A [VictimSelector] that stochastically selects a set of hosts to be failed. - */ -public class StochasticVictimSelectorNew( - private val random: RandomGenerator = SplittableRandom(0), -) : VictimSelector { - - override fun select(numberOfHosts: Int): List { - error("select with only int cannot be used in this type of VictimSelector"); - } - - override fun select(hosts: Set, numberOfHosts:Int): List { - val result = ArrayList(numberOfHosts) - - val random = random - var samplesNeeded = numberOfHosts - var remainingHosts = hosts.size - val iterator = hosts.iterator() - - while (iterator.hasNext() && samplesNeeded > 0) { - val host = iterator.next() - - if (random.nextInt(remainingHosts) < samplesNeeded) { - result.add(host) - samplesNeeded-- - } - - remainingHosts-- - } - - return result - } - - override fun select(failureIntensity: Double): List { - error("select with only int cannot be used in this type of VictimSelector"); - } - - override fun select(hosts: Set, failureIntensity:Double): List { - // clamp value between 0.0 and 1.0 - val intensity = min(1.0, max(0.0, failureIntensity)) - val numberOfHosts = (hosts.size * intensity).toInt() - - return hosts.asSequence().shuffled().take(numberOfHosts).toList() - } - - override fun toString(): String = "StochasticVictimSelector" -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/RandomHostFaultInjector.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/RandomHostFaultInjector.kt deleted file mode 100644 index 2247ccca2..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/RandomHostFaultInjector.kt +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.internal - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.cancel -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch -import org.apache.commons.math3.distribution.RealDistribution -import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.hostfault.HostFault -import org.opendc.compute.simulator.failure.HostFaultInjector -import org.opendc.compute.simulator.failure.victimselector.VictimSelector -import java.time.InstantSource -import kotlin.coroutines.CoroutineContext -import kotlin.math.roundToLong - -/** - * Internal implementation of the [HostFaultInjector] interface. - * - * @param context The scope to run the fault injector in. - * @param clock The [InstantSource] to keep track of simulation time. - * @param hosts The set of hosts to inject faults into. - * @param iat The inter-arrival time distribution of the failures (in hours). - * @param selector The [VictimSelector] to select the host victims. - * @param fault The type of [HostFault] to inject. - */ -internal class RandomHostFaultInjector( - private val context: CoroutineContext, - private val clock: InstantSource, - private val hosts: Set, - private val iat: RealDistribution, - private val selector: VictimSelector, - private val fault: HostFault, -) : HostFaultInjector { - /** - * The scope in which the injector runs. - */ - private val scope = CoroutineScope(context + Job()) - - /** - * The [Job] that awaits the nearest fault in the system. - */ - private var job: Job? = null - - /** - * Start the fault injection into the system. - */ - override fun start() { - if (job != null) { - return - } - - job = - scope.launch { - runInjector() - job = null - } - } - - /** - * Converge the injection process. - */ - private suspend fun runInjector() { - while (true) { - // Make sure to convert delay from hours to milliseconds - val d = (iat.sample() * 3.6e6).roundToLong() - - // Handle long overflow - if (clock.millis() + d <= 0) { - return - } - - delay(d) - - val victims = selector.select(hosts, 1) - fault.apply(clock, victims) - } - } - - /** - * Stop the fault injector. - */ - public override fun close() { - scope.cancel() - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt index 3aa8cf5aa..690bf4724 100644 --- a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt +++ b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt @@ -29,8 +29,6 @@ import org.apache.commons.math3.distribution.LogNormalDistribution import org.apache.commons.math3.random.Well19937c import org.junit.jupiter.api.Test import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.hostfault.StartStopHostFault -import org.opendc.compute.simulator.failure.victimselector.StochasticVictimSelector import org.opendc.simulator.kotlin.runSimulation import java.time.Duration import java.time.InstantSource diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt index 11fcc96de..ac64ae5ae 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt @@ -36,7 +36,7 @@ import org.opendc.compute.api.Server import org.opendc.compute.api.ServerState import org.opendc.compute.api.ServerWatcher import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.failure.models.FailureModelNew +import org.opendc.compute.simulator.failure.models.FailureModel import org.opendc.compute.workload.VirtualMachine import java.time.InstantSource import java.util.Random @@ -92,7 +92,7 @@ public suspend fun ComputeService.replay( val client = newClient() // Create a failure model based on the failureModelSpec, if not null, otherwise set failureModel to null - val failureModel: FailureModelNew? = failureModelSpec?.let { + val failureModel: FailureModel? = failureModelSpec?.let { getFailureModel(coroutineContext, clock, this, Random(seed), it) } // Create new image for the virtual machine diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt index 9b8e435da..26941f075 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt @@ -22,9 +22,8 @@ package org.opendc.experiments.base.scenario -import AllocationPolicySpec -import FailureModelSpec import ScenarioTopologySpec +import org.opendc.experiments.base.scenario.specs.ScenarioSpec import java.io.File private val scenarioReader = ScenarioReader() @@ -61,13 +60,12 @@ public fun getScenarios(scenarioSpec: ScenarioSpec): List { val outputFolder = scenarioSpec.outputFolder + "/" + scenarioSpec.name File(outputFolder).mkdirs() - val trackrPath = outputFolder + "/trackr.json" + val trackrPath = "$outputFolder/trackr.json" File(trackrPath).createNewFile() val scenarios = mutableListOf() - var scenarioID = 0 - for (scenarioTopologySpec in scenarioSpec.topologies) { + for ((scenarioID, scenarioTopologySpec) in scenarioSpec.topologies.withIndex()) { for (workloadSpec in scenarioSpec.workloads) { for (allocationPolicySpec in scenarioSpec.allocationPolicies) { for (failureModelSpec in scenarioSpec.failureModels) { @@ -91,7 +89,6 @@ public fun getScenarios(scenarioSpec: ScenarioSpec): List { ) trackScenario(scenarioSpec, outputFolder, scenario, scenarioTopologySpec) scenarios.add(scenario) - scenarioID++ } } } @@ -118,7 +115,7 @@ public fun trackScenario( scenario: Scenario, topologySpec: ScenarioTopologySpec, ) { - val trackrPath = outputFolder + "/trackr.json" + val trackrPath = "$outputFolder/trackr.json" scenarioWriter.write( ScenarioSpec( id = scenario.id, diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt index fdb4bc90b..b863e0dc8 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt @@ -27,7 +27,7 @@ import org.apache.commons.math3.distribution.LogNormalDistribution import org.apache.commons.math3.distribution.RealDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.failure.models.FailureModelNew +import org.opendc.compute.simulator.failure.models.FailureModel import org.opendc.compute.simulator.failure.models.SampleBasedFailureModel import org.opendc.compute.simulator.failure.models.TraceBasedFailureModel import org.opendc.compute.simulator.failure.prefab.FailurePrefab @@ -93,7 +93,7 @@ public fun getFailureModel(context: CoroutineContext, clock: InstantSource, service: ComputeService, random: java.util.random.RandomGenerator, - failureModel: PrefabFailureModelSpec): FailureModelNew { + failureModel: PrefabFailureModelSpec): FailureModel { return getFailureModelPrefab(context, clock, service, random, failureModel.prefabName) } @@ -101,7 +101,7 @@ public fun getFailureModel(context: CoroutineContext, clock: InstantSource, service: ComputeService, random: java.util.random.RandomGenerator, - failureModel: CustomFailureModelSpec): FailureModelNew { + failureModel: CustomFailureModelSpec): FailureModel { val rng: org.apache.commons.math3.random.RandomGenerator = Well19937c(random.nextLong()) val iatSampler = getSampler(rng, failureModel.iatSampler) @@ -123,7 +123,7 @@ public fun getFailureModel(context: CoroutineContext, clock: InstantSource, service: ComputeService, random: java.util.random.RandomGenerator, - failureModel: TraceBasedFailureModelSpec): FailureModelNew { + failureModel: TraceBasedFailureModelSpec): FailureModel { return TraceBasedFailureModel(context, clock, service, random, failureModel.pathToFile) } @@ -132,7 +132,7 @@ public fun getFailureModel(context: CoroutineContext, clock: InstantSource, service: ComputeService, random: java.util.random.RandomGenerator, - failureModel: FailureModelSpec?): FailureModelNew? { + failureModel: FailureModelSpec?): FailureModel? { return when(failureModel) { is PrefabFailureModelSpec -> getFailureModel(context, clock, service, random, failureModel) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt index 15fc989c0..2ff3ac423 100644 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt @@ -25,7 +25,6 @@ package org.opendc.web.runner import mu.KotlinLogging import org.opendc.compute.service.ComputeService import org.opendc.compute.service.scheduler.createComputeScheduler -import org.opendc.compute.simulator.failure.prefab.Grid5000 import org.opendc.compute.simulator.provisioner.Provisioner import org.opendc.compute.simulator.provisioner.registerComputeMonitor import org.opendc.compute.simulator.provisioner.setupComputeService From ead8c9e4ceca3504b6b7ac6a44a54eb7b285c596 Mon Sep 17 00:00:00 2001 From: Dante Niewenhuis Date: Fri, 3 May 2024 13:19:08 +0200 Subject: [PATCH 3/5] Added prefab failure models --- .../org/opendc/compute/api/ComputeClient.kt | 2 +- .../opendc-compute-failure/build.gradle.kts | 40 +++ .../compute}/failure/hostfault/HostFault.kt | 6 +- .../failure/hostfault/StartStopHostFault.kt | 12 +- .../compute}/failure/models/FailureModel.kt | 18 +- .../failure/models/SampleBasedFailureModel.kt | 77 +++++ .../failure/models/TraceBasedFailureModel.kt | 124 ++++++++ .../opendc/compute/failure/prefab/G5k06.kt | 115 +++++++ .../opendc/compute/failure/prefab/Lanl05.kt | 115 +++++++ .../opendc/compute/failure/prefab/Ldns04.kt | 115 +++++++ .../compute/failure/prefab/Microsoft99.kt | 115 +++++++ .../opendc/compute/failure/prefab/Nd07cpu.kt | 115 +++++++ .../compute/failure/prefab/Overnet03.kt | 115 +++++++ .../org/opendc/compute/failure/prefab/Pl05.kt | 115 +++++++ .../prefab/PrefabFailureModelFactory.kt | 137 +++++++++ .../opendc/compute/failure/prefab/Skype06.kt | 115 +++++++ .../compute/failure/prefab/Websites02.kt | 115 +++++++ .../StochasticVictimSelector.kt | 17 +- .../failure/victimselector/VictimSelector.kt | 17 +- .../src/test/resources/log4j2.xml | 38 +++ .../compute/service/ComputeService.java | 11 +- .../checkpoints/CheckPointFactory.kt | 1 - .../simulator/checkpoints/CheckpointModel.kt | 26 +- .../failure/models/FailureTraceLoader.kt | 90 ------ .../failure/models/SampleBasedFailureModel.kt | 52 ---- .../failure/models/TraceBasedFailureModel.kt | 45 --- .../prefab/PrefabFailureModelFactory.kt | 75 ----- .../compute/simulator/internal/Guest.kt | 3 +- .../failure/HostFaultInjectorTest.kt | 119 -------- .../opendc/compute/topology/TopologyReader.kt | 1 - .../opendc/compute/workload/VirtualMachine.kt | 2 +- .../opendc-experiments-base/build.gradle.kts | 4 +- .../base/runner/ScenarioHelpers.kt | 14 +- .../experiments/base/runner/ScenarioRunner.kt | 40 +-- .../experiments/base/scenario/Scenario.kt | 2 +- .../base/scenario/ScenarioFactories.kt | 11 +- .../base/scenario/ScenarioReader.kt | 1 - .../scenario/specs/CheckpointModelSpec.kt | 28 +- .../base/scenario/specs/FailureModelSpec.kt | 289 ++++++++++++++---- .../base/scenario/specs/ScenarioSpec.kt | 1 - .../compute/power/CPUPowerModelsFactory.kt | 2 +- .../compute/workload/SimRuntimeWorkload.java | 37 ++- .../simulator/compute/workload/SimTrace.java | 20 +- .../compute/workload/SimWorkloads.java | 13 +- .../compute/workload/SimChainWorkloadTest.kt | 8 +- .../org/opendc/trace/conv/FailureColumns.kt | 2 +- .../formats/failure/FailureTableReader.kt | 4 +- .../formats/failure/FailureTraceFormat.kt | 4 +- .../failure/parquet/FailureReadSupport.kt | 6 +- .../parquet/FailureRecordMaterializer.kt | 2 +- opendc-web/opendc-web-runner/build.gradle.kts | 1 + .../org/opendc/web/runner/OpenDCRunner.kt | 4 +- settings.gradle.kts | 5 +- 53 files changed, 1880 insertions(+), 566 deletions(-) create mode 100644 opendc-compute/opendc-compute-failure/build.gradle.kts rename opendc-compute/{opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator => opendc-compute-failure/src/main/kotlin/org/opendc/compute}/failure/hostfault/HostFault.kt (90%) rename opendc-compute/{opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator => opendc-compute-failure/src/main/kotlin/org/opendc/compute}/failure/hostfault/StartStopHostFault.kt (87%) rename opendc-compute/{opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator => opendc-compute-failure/src/main/kotlin/org/opendc/compute}/failure/models/FailureModel.kt (82%) create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/SampleBasedFailureModel.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/G5k06.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Lanl05.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Ldns04.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Microsoft99.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Nd07cpu.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Overnet03.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Pl05.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Skype06.kt create mode 100644 opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Websites02.kt rename opendc-compute/{opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator => opendc-compute-failure/src/main/kotlin/org/opendc/compute}/failure/victimselector/StochasticVictimSelector.kt (89%) rename opendc-compute/{opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator => opendc-compute-failure/src/main/kotlin/org/opendc/compute}/failure/victimselector/VictimSelector.kt (81%) create mode 100644 opendc-compute/opendc-compute-failure/src/test/resources/log4j2.xml delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckPointFactory.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureTraceLoader.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/PrefabFailureModelFactory.kt delete mode 100644 opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt diff --git a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt index fa829e098..7863f20bf 100644 --- a/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt +++ b/opendc-compute/opendc-compute-api/src/main/kotlin/org/opendc/compute/api/ComputeClient.kt @@ -116,7 +116,7 @@ public interface ComputeClient : AutoCloseable { public fun rescheduleServer( server: Server, - workload: SimWorkload + workload: SimWorkload, ) /** diff --git a/opendc-compute/opendc-compute-failure/build.gradle.kts b/opendc-compute/opendc-compute-failure/build.gradle.kts new file mode 100644 index 000000000..d6ec91168 --- /dev/null +++ b/opendc-compute/opendc-compute-failure/build.gradle.kts @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +description = "OpenDC Failure Service implementation" + +// Build configuration +plugins { + `kotlin-library-conventions` +} + +dependencies { + api(projects.opendcCompute.opendcComputeApi) + implementation(projects.opendcCommon) + implementation(project(mapOf("path" to ":opendc-trace:opendc-trace-api"))) + implementation(project(mapOf("path" to ":opendc-simulator:opendc-simulator-compute"))) + implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-service"))) + implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-simulator"))) + + api(libs.commons.math3) + implementation(libs.kotlin.logging) +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFault.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/HostFault.kt similarity index 90% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFault.kt rename to opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/HostFault.kt index 8dadb1b13..4134c58aa 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/HostFault.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/HostFault.kt @@ -20,24 +20,22 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure.hostfault +package org.opendc.compute.failure.hostfault import org.opendc.compute.service.ComputeService import org.opendc.compute.simulator.SimHost -import java.time.InstantSource /** * Interface responsible for applying the fault to a host. */ public abstract class HostFault( private val service: ComputeService, - private val clock: InstantSource, ) { /** * Apply the fault to the specified [victims]. */ public abstract suspend fun apply( victims: List, - faultDuration: Long + faultDuration: Long, ) } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFault.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/StartStopHostFault.kt similarity index 87% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFault.kt rename to opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/StartStopHostFault.kt index f1ca454ad..85138025b 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/hostfault/StartStopHostFault.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/hostfault/StartStopHostFault.kt @@ -20,32 +20,30 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure.hostfault +package org.opendc.compute.failure.hostfault import kotlinx.coroutines.delay import org.opendc.compute.api.ComputeClient import org.opendc.compute.service.ComputeService import org.opendc.compute.simulator.SimHost import org.opendc.simulator.compute.workload.SimWorkload -import java.time.InstantSource /** * A type of [HostFault] where the hosts are stopped and recover after a given amount of time. */ -public class StartStopHostFault ( +public class StartStopHostFault( private val service: ComputeService, - clock: InstantSource -): HostFault(service, clock) { +) : HostFault(service) { override suspend fun apply( victims: List, - faultDuration: Long + faultDuration: Long, ) { val client: ComputeClient = service.newClient() for (host in victims) { val servers = host.instances - val snapshots = servers.map{(it.meta["workload"] as SimWorkload).snapshot()} + val snapshots = servers.map { (it.meta["workload"] as SimWorkload).snapshot() } host.fail() for ((server, snapshot) in servers.zip(snapshots)) { diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModel.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/FailureModel.kt similarity index 82% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModel.kt rename to opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/FailureModel.kt index 439c31691..5f05d96c0 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureModel.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/FailureModel.kt @@ -20,34 +20,34 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure.models +package org.opendc.compute.failure.models import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.cancel import kotlinx.coroutines.launch +import org.opendc.compute.failure.hostfault.HostFault +import org.opendc.compute.failure.hostfault.StartStopHostFault +import org.opendc.compute.failure.victimselector.StochasticVictimSelector import org.opendc.compute.service.ComputeService import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.hostfault.HostFault -import org.opendc.compute.simulator.failure.hostfault.StartStopHostFault -import org.opendc.compute.simulator.failure.victimselector.StochasticVictimSelector import java.time.InstantSource import java.util.random.RandomGenerator import kotlin.coroutines.CoroutineContext /** - * Factory interface for constructing [HostFaultInjector] for modeling failures of compute service hosts. + * Factory interface for constructing [FailureModel] for modeling failures of compute service hosts. */ -public abstract class FailureModel ( +public abstract class FailureModel( context: CoroutineContext, protected val clock: InstantSource, protected val service: ComputeService, - protected val random: RandomGenerator -): AutoCloseable { + protected val random: RandomGenerator, +) : AutoCloseable { protected val scope: CoroutineScope = CoroutineScope(context + Job()) // TODO: could at some point be extended to different types of faults - protected val fault: HostFault = StartStopHostFault(service, clock) + protected val fault: HostFault = StartStopHostFault(service) // TODO: could at some point be extended to different types of victim selectors protected val victimSelector: StochasticVictimSelector = StochasticVictimSelector(random) diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/SampleBasedFailureModel.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/SampleBasedFailureModel.kt new file mode 100644 index 000000000..3ae66f6ff --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/SampleBasedFailureModel.kt @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.models + +import kotlinx.coroutines.delay +import org.apache.commons.math3.distribution.RealDistribution +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext +import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToLong + +/** + * Sample based failure model + * + * @property context + * @property clock + * @property service + * @property random + * @property iatSampler A distribution from which the time until the next fault is sampled in ms + * @property durationSampler A distribution from which the duration of a fault is sampled in s + * @property nohSampler A distribution from which the number of hosts that fault is sampled. + */ +public class SampleBasedFailureModel( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, + private val iatSampler: RealDistribution, + private val durationSampler: RealDistribution, + private val nohSampler: RealDistribution, +) : FailureModel(context, clock, service, random) { + override suspend fun runInjector() { + while (true) { + val iatSample = max(0.0, iatSampler.sample()) + val intervalDuration = (iatSample * 3.6e6).roundToLong() + + // Handle long overflow + if (clock.millis() + intervalDuration <= 0) { + return + } + + delay(intervalDuration) + + val numberOfHosts = min(1.0, max(0.0, nohSampler.sample())) + val victims = victimSelector.select(hosts, numberOfHosts) + + val durationSample = max(0.0, durationSampler.sample()) + val faultDuration = (durationSample * 3.6e6).toLong() + fault.apply(victims, faultDuration) + + break + } + } +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt new file mode 100644 index 000000000..ea376ce0e --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.models + +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import org.opendc.compute.service.ComputeService +import org.opendc.trace.Trace +import org.opendc.trace.conv.FAILURE_DURATION +import org.opendc.trace.conv.FAILURE_INTENSITY +import org.opendc.trace.conv.FAILURE_INTERVAL +import org.opendc.trace.conv.TABLE_FAILURES +import java.io.File +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * A definition of a Failure + * + * @property failureInterval The time between this and the previous failure in ms + * @property failureDuration The Duration of the failure in ms + * @property failureIntensity The ratio of hosts affected by the failure + * @constructor Create empty Failure + */ +public data class Failure( + val failureInterval: Long, + val failureDuration: Long, + val failureIntensity: Double, +) { + init { + require(failureInterval >= 0.0) { "A failure cannot start at a negative time" } + require(failureDuration >= 0.0) { "A failure can not have a duration of 0 or less" } + require(failureIntensity > 0.0 && failureInterval <= 1.0) { "The intensity of a failure has to be in the range (0.0, 1.0]" } + } +} + +/** + * A [FailureModel] based on a provided parquet file + * The file provides a list of [Failure] objects + * + * + * @param context + * @param clock + * @param service + * @param random + * @param pathToTrace The path to the parquet file as a [String] + */ +public class TraceBasedFailureModel( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, + pathToTrace: String, +) : FailureModel(context, clock, service, random) { + private val failureList = loadTrace(pathToTrace).iterator() + + override suspend fun runInjector() { + while (failureList.hasNext()) { + val failure = failureList.next() + + delay(failure.failureInterval - clock.millis()) + + val victims = victimSelector.select(hosts, failure.failureIntensity) + scope.launch { + fault.apply(victims, failure.failureDuration) + } + } + } + + /** + * Load a list [Failure] objects from the provided [pathToFile] + * + * @param pathToFile + */ + private fun loadTrace(pathToFile: String): List { + val trace = Trace.open(File(pathToFile), "failure") + + val reader = checkNotNull(trace.getTable(TABLE_FAILURES)).newReader() + + val failureStartTimeCol = reader.resolve(FAILURE_INTERVAL) + val failureDurationCol = reader.resolve(FAILURE_DURATION) + val failureIntensityCol = reader.resolve(FAILURE_INTENSITY) + + val entries = mutableListOf() + + try { + while (reader.nextRow()) { + val failureStartTime = reader.getLong(failureStartTimeCol) + val failureDuration = reader.getLong(failureDurationCol) + val failureIntensity = reader.getDouble(failureIntensityCol) + + entries.add(Failure(failureStartTime, failureDuration, failureIntensity)) + } + + return entries + } catch (e: Exception) { + e.printStackTrace() + throw e + } finally { + reader.close() + } + } +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/G5k06.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/G5k06.kt new file mode 100644 index 000000000..da58250da --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/G5k06.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.prefab + +import org.apache.commons.math3.distribution.ExponentialDistribution +import org.apache.commons.math3.distribution.GammaDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.UniformRealDistribution +import org.apache.commons.math3.distribution.WeibullDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * Failure models based on values taken from "The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems" + * Which can be found at https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634 + */ +public fun createG5k06Exp( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + ExponentialDistribution(rng, 32.41), + ExponentialDistribution(rng, 7.41), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createG5k06Wbl( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + WeibullDistribution(rng, 0.48, 14.37), + WeibullDistribution(rng, 0.35, 0.47), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createG5k06LogN( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + LogNormalDistribution(rng, 1.51, 2.42), + LogNormalDistribution(rng, -2.0, 2.2), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createG5k06Gam( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + GammaDistribution(rng, 0.34, 94.35), + GammaDistribution(rng, 0.19, 39.92), + UniformRealDistribution(0.0, 1.0), + ) +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Lanl05.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Lanl05.kt new file mode 100644 index 000000000..3e7226303 --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Lanl05.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.prefab + +import org.apache.commons.math3.distribution.ExponentialDistribution +import org.apache.commons.math3.distribution.GammaDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.UniformRealDistribution +import org.apache.commons.math3.distribution.WeibullDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * Failure models based on values taken from "The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems" + * Which can be found at https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634 + */ +public fun createLanl05Exp( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + ExponentialDistribution(rng, 1779.99), + ExponentialDistribution(rng, 5.92), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createLanl05Wbl( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + WeibullDistribution(rng, 0.48, 816.60), + WeibullDistribution(rng, 0.58, 2.18), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createLanl05LogN( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + LogNormalDistribution(rng, 5.56, 2.39), + LogNormalDistribution(rng, 0.05, 1.42), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createLanl05Gam( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + GammaDistribution(rng, 0.35, 5102.71), + GammaDistribution(rng, 0.38, 15.44), + UniformRealDistribution(0.0, 1.0), + ) +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Ldns04.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Ldns04.kt new file mode 100644 index 000000000..4a8b3c0f4 --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Ldns04.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.prefab + +import org.apache.commons.math3.distribution.ExponentialDistribution +import org.apache.commons.math3.distribution.GammaDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.UniformRealDistribution +import org.apache.commons.math3.distribution.WeibullDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * Failure models based on values taken from "The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems" + * Which can be found at https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634 + */ +public fun createLdns04Exp( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + ExponentialDistribution(rng, 141.06), + ExponentialDistribution(rng, 8.61), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createLdns04Wbl( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + WeibullDistribution(rng, 0.51, 79.30), + WeibullDistribution(rng, 0.63, 5.62), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createLdns04LogN( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + LogNormalDistribution(rng, 3.25, 2.33), + LogNormalDistribution(rng, 0.91, 1.64), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createLdns04Gam( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + GammaDistribution(rng, 0.39, 362.43), + GammaDistribution(rng, 0.51, 16.87), + UniformRealDistribution(0.0, 1.0), + ) +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Microsoft99.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Microsoft99.kt new file mode 100644 index 000000000..725f6622b --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Microsoft99.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.prefab + +import org.apache.commons.math3.distribution.ExponentialDistribution +import org.apache.commons.math3.distribution.GammaDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.UniformRealDistribution +import org.apache.commons.math3.distribution.WeibullDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * Failure models based on values taken from "The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems" + * Which can be found at https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634 + */ +public fun createMicrosoft99Exp( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + ExponentialDistribution(rng, 67.01), + ExponentialDistribution(rng, 16.49), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createMicrosoft99Wbl( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + WeibullDistribution(rng, 0.55, 35.30), + WeibullDistribution(rng, 0.60, 9.34), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createMicrosoft99LogN( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + LogNormalDistribution(rng, 2.62, 1.84), + LogNormalDistribution(rng, 1.42, 1.54), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createMicrosoft99Gam( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + GammaDistribution(rng, 0.41, 162.19), + GammaDistribution(rng, 0.46, 35.52), + UniformRealDistribution(0.0, 1.0), + ) +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Nd07cpu.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Nd07cpu.kt new file mode 100644 index 000000000..100a3a8df --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Nd07cpu.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.prefab + +import org.apache.commons.math3.distribution.ExponentialDistribution +import org.apache.commons.math3.distribution.GammaDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.UniformRealDistribution +import org.apache.commons.math3.distribution.WeibullDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * Failure models based on values taken from "The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems" + * Which can be found at https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634 + */ +public fun createNd07cpuExp( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + ExponentialDistribution(rng, 13.73), + ExponentialDistribution(rng, 4.25), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createNd07cpuWbl( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + WeibullDistribution(rng, 0.45, 4.16), + WeibullDistribution(rng, 0.51, 0.74), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createNd07cpuLogN( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + LogNormalDistribution(rng, 0.30, 2.20), + LogNormalDistribution(rng, -1.02, 1.27), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createNd07cpuGam( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + GammaDistribution(rng, 0.30, 46.16), + GammaDistribution(rng, 0.28, 15.07), + UniformRealDistribution(0.0, 1.0), + ) +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Overnet03.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Overnet03.kt new file mode 100644 index 000000000..4f5e3f845 --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Overnet03.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.prefab + +import org.apache.commons.math3.distribution.ExponentialDistribution +import org.apache.commons.math3.distribution.GammaDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.UniformRealDistribution +import org.apache.commons.math3.distribution.WeibullDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * Failure models based on values taken from "The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems" + * Which can be found at https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634 + */ +public fun createOvernet03Exp( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + ExponentialDistribution(rng, 2.29), + ExponentialDistribution(rng, 12.00), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createOvernet03Wbl( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + WeibullDistribution(rng, 0.85, 2.04), + WeibullDistribution(rng, 0.44, 2.98), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createOvernet03LogN( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + LogNormalDistribution(rng, 0.19, 0.98), + LogNormalDistribution(rng, 0.08, 1.80), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createOvernet03Gam( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + GammaDistribution(rng, 0.91, 2.53), + GammaDistribution(rng, 0.29, 41.64), + UniformRealDistribution(0.0, 1.0), + ) +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Pl05.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Pl05.kt new file mode 100644 index 000000000..3e1f1b588 --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Pl05.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.prefab + +import org.apache.commons.math3.distribution.ExponentialDistribution +import org.apache.commons.math3.distribution.GammaDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.UniformRealDistribution +import org.apache.commons.math3.distribution.WeibullDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * Failure models based on values taken from "The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems" + * Which can be found at https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634 + */ +public fun createPl05Exp( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + ExponentialDistribution(rng, 159.49), + ExponentialDistribution(rng, 49.61), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createPl05Wbl( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + WeibullDistribution(rng, 0.33, 19.35), + WeibullDistribution(rng, 0.36, 5.59), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createPl05LogN( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + LogNormalDistribution(rng, 1.44, 2.86), + LogNormalDistribution(rng, 0.40, 2.45), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createPl05Gam( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + GammaDistribution(rng, 0.20, 788.03), + GammaDistribution(rng, 0.21, 237.65), + UniformRealDistribution(0.0, 1.0), + ) +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt new file mode 100644 index 000000000..990301e53 --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +@file:JvmName("FailureModels") + +package org.opendc.compute.failure.prefab + +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +public enum class FailurePrefab { + G5k06Exp, + G5k06Wbl, + G5k06LogN, + G5k06Gam, + Lanl05Exp, + Lanl05Wbl, + Lanl05LogN, + Lanl05Gam, + Ldns04Exp, + Ldns04Wbl, + Ldns04LogN, + Ldns04Gam, + Microsoft99Exp, + Microsoft99Wbl, + Microsoft99LogN, + Microsoft99Gam, + Nd07cpuExp, + Nd07cpuWbl, + Nd07cpuLogN, + Nd07cpuGam, + Overnet03Exp, + Overnet03Wbl, + Overnet03LogN, + Overnet03Gam, + Pl05Exp, + Pl05Wbl, + Pl05LogN, + Pl05Gam, + Skype06Exp, + Skype06Wbl, + Skype06LogN, + Skype06Gam, + Websites02Exp, + Websites02Wbl, + Websites02LogN, + Websites02Gam, +} + +/** + * Get a [SampleBasedFailureModel] based on the provided prefab + * + * @param context + * @param clock + * @param service + * @param random + * @param prefab The name of the failure model prefab + * @return + */ +public fun createFailureModelPrefab( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, + prefab: FailurePrefab, +): SampleBasedFailureModel { + when (prefab) { + FailurePrefab.G5k06Exp -> return createG5k06Exp(context, clock, service, random) + FailurePrefab.G5k06Wbl -> return createG5k06Wbl(context, clock, service, random) + FailurePrefab.G5k06LogN -> return createG5k06LogN(context, clock, service, random) + FailurePrefab.G5k06Gam -> return createG5k06Gam(context, clock, service, random) + + FailurePrefab.Lanl05Exp -> return createLanl05Exp(context, clock, service, random) + FailurePrefab.Lanl05Wbl -> return createLanl05Wbl(context, clock, service, random) + FailurePrefab.Lanl05LogN -> return createLanl05LogN(context, clock, service, random) + FailurePrefab.Lanl05Gam -> return createLanl05Gam(context, clock, service, random) + + FailurePrefab.Ldns04Exp -> return createLdns04Exp(context, clock, service, random) + FailurePrefab.Ldns04Wbl -> return createLdns04Wbl(context, clock, service, random) + FailurePrefab.Ldns04LogN -> return createLdns04LogN(context, clock, service, random) + FailurePrefab.Ldns04Gam -> return createLdns04Gam(context, clock, service, random) + + FailurePrefab.Microsoft99Exp -> return createMicrosoft99Exp(context, clock, service, random) + FailurePrefab.Microsoft99Wbl -> return createMicrosoft99Wbl(context, clock, service, random) + FailurePrefab.Microsoft99LogN -> return createMicrosoft99LogN(context, clock, service, random) + FailurePrefab.Microsoft99Gam -> return createMicrosoft99Gam(context, clock, service, random) + + FailurePrefab.Nd07cpuExp -> return createNd07cpuExp(context, clock, service, random) + FailurePrefab.Nd07cpuWbl -> return createNd07cpuWbl(context, clock, service, random) + FailurePrefab.Nd07cpuLogN -> return createNd07cpuLogN(context, clock, service, random) + FailurePrefab.Nd07cpuGam -> return createNd07cpuGam(context, clock, service, random) + + FailurePrefab.Overnet03Exp -> return createOvernet03Exp(context, clock, service, random) + FailurePrefab.Overnet03Wbl -> return createOvernet03Wbl(context, clock, service, random) + FailurePrefab.Overnet03LogN -> return createOvernet03LogN(context, clock, service, random) + FailurePrefab.Overnet03Gam -> return createOvernet03Gam(context, clock, service, random) + + FailurePrefab.Pl05Exp -> return createPl05Exp(context, clock, service, random) + FailurePrefab.Pl05Wbl -> return createPl05Wbl(context, clock, service, random) + FailurePrefab.Pl05LogN -> return createPl05LogN(context, clock, service, random) + FailurePrefab.Pl05Gam -> return createPl05Gam(context, clock, service, random) + + FailurePrefab.Skype06Exp -> return createSkype06Exp(context, clock, service, random) + FailurePrefab.Skype06Wbl -> return createSkype06Wbl(context, clock, service, random) + FailurePrefab.Skype06LogN -> return createSkype06LogN(context, clock, service, random) + FailurePrefab.Skype06Gam -> return createSkype06Gam(context, clock, service, random) + + FailurePrefab.Websites02Exp -> return createWebsites02Exp(context, clock, service, random) + FailurePrefab.Websites02Wbl -> return createWebsites02Wbl(context, clock, service, random) + FailurePrefab.Websites02LogN -> return createWebsites02LogN(context, clock, service, random) + FailurePrefab.Websites02Gam -> return createWebsites02Gam(context, clock, service, random) + + else -> error("Unknown failure prefab: $prefab") + } +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Skype06.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Skype06.kt new file mode 100644 index 000000000..7495bf662 --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Skype06.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.prefab + +import org.apache.commons.math3.distribution.ExponentialDistribution +import org.apache.commons.math3.distribution.GammaDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.UniformRealDistribution +import org.apache.commons.math3.distribution.WeibullDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * Failure models based on values taken from "The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems" + * Which can be found at https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634 + */ +public fun createSkype06Exp( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + ExponentialDistribution(rng, 16.27), + ExponentialDistribution(rng, 14.31), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createSkype06Wbl( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + WeibullDistribution(rng, 0.64, 10.86), + WeibullDistribution(rng, 0.63, 9.48), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createSkype06LogN( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + LogNormalDistribution(rng, 1.60, 1.57), + LogNormalDistribution(rng, 1.40, 1.73), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createSkype06Gam( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + GammaDistribution(rng, 0.53, 30.79), + GammaDistribution(rng, 0.50, 28.53), + UniformRealDistribution(0.0, 1.0), + ) +} diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Websites02.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Websites02.kt new file mode 100644 index 000000000..77bb0d649 --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/Websites02.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.failure.prefab + +import org.apache.commons.math3.distribution.ExponentialDistribution +import org.apache.commons.math3.distribution.GammaDistribution +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.UniformRealDistribution +import org.apache.commons.math3.distribution.WeibullDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.service.ComputeService +import java.time.InstantSource +import java.util.random.RandomGenerator +import kotlin.coroutines.CoroutineContext + +/** + * Failure models based on values taken from "The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems" + * Which can be found at https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634 + */ +public fun createWebsites02Exp( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + ExponentialDistribution(rng, 11.85), + ExponentialDistribution(rng, 1.18), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createWebsites02Wbl( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + WeibullDistribution(rng, 0.46, 3.68), + WeibullDistribution(rng, 0.65, 0.61), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createWebsites02LogN( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + LogNormalDistribution(rng, 0.23, 2.02), + LogNormalDistribution(rng, -1.12, 1.13), + UniformRealDistribution(0.0, 1.0), + ) +} + +public fun createWebsites02Gam( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: RandomGenerator, +): SampleBasedFailureModel { + val rng = Well19937c(random.nextLong()) + + return SampleBasedFailureModel( + context, + clock, + service, + random, + GammaDistribution(rng, 0.31, 38.67), + GammaDistribution(rng, 0.50, 2.37), + UniformRealDistribution(0.0, 1.0), + ) +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelector.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/StochasticVictimSelector.kt similarity index 89% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelector.kt rename to opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/StochasticVictimSelector.kt index 7ce4f9c53..fef60eb3c 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/StochasticVictimSelector.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/StochasticVictimSelector.kt @@ -20,7 +20,7 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure.victimselector +package org.opendc.compute.failure.victimselector import org.opendc.compute.simulator.SimHost import java.util.SplittableRandom @@ -34,12 +34,14 @@ import kotlin.math.min public class StochasticVictimSelector( private val random: RandomGenerator = SplittableRandom(0), ) : VictimSelector { - override fun select(numberOfHosts: Int): List { - error("select with only int cannot be used in this type of VictimSelector"); + error("select with only int cannot be used in this type of VictimSelector") } - override fun select(hosts: Set, numberOfHosts:Int): List { + override fun select( + hosts: Set, + numberOfHosts: Int, + ): List { val result = ArrayList(numberOfHosts) val random = random @@ -62,10 +64,13 @@ public class StochasticVictimSelector( } override fun select(failureIntensity: Double): List { - error("select with only int cannot be used in this type of VictimSelector"); + error("select with only int cannot be used in this type of VictimSelector") } - override fun select(hosts: Set, failureIntensity:Double): List { + override fun select( + hosts: Set, + failureIntensity: Double, + ): List { // clamp value between 0.0 and 1.0 val intensity = min(1.0, max(0.0, failureIntensity)) val numberOfHosts = (hosts.size * intensity).toInt() diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/VictimSelector.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/VictimSelector.kt similarity index 81% rename from opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/VictimSelector.kt rename to opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/VictimSelector.kt index ee337ba5e..955cbcedb 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/victimselector/VictimSelector.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/victimselector/VictimSelector.kt @@ -20,7 +20,7 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure.victimselector +package org.opendc.compute.failure.victimselector import org.opendc.compute.simulator.SimHost @@ -31,8 +31,17 @@ public interface VictimSelector { /** * Select the hosts from [hosts] where a fault will be injected. */ - public fun select(hosts: Set, numberOfHosts:Int): List - public fun select(numberOfHosts:Int): List + public fun select( + hosts: Set, + numberOfHosts: Int, + ): List + + public fun select(numberOfHosts: Int): List + public fun select(failureIntensity: Double): List - public fun select(hosts: Set, failureIntensity: Double): List + + public fun select( + hosts: Set, + failureIntensity: Double, + ): List } diff --git a/opendc-compute/opendc-compute-failure/src/test/resources/log4j2.xml b/opendc-compute/opendc-compute-failure/src/test/resources/log4j2.xml new file mode 100644 index 000000000..0dfb75f26 --- /dev/null +++ b/opendc-compute/opendc-compute-failure/src/test/resources/log4j2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + diff --git a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java index f9d331e3b..a7e9f5098 100644 --- a/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java +++ b/opendc-compute/opendc-compute-service/src/main/java/org/opendc/compute/service/ComputeService.java @@ -52,9 +52,9 @@ import org.opendc.compute.service.driver.HostState; import org.opendc.compute.service.scheduler.ComputeScheduler; import org.opendc.compute.service.telemetry.SchedulerStats; +import org.opendc.simulator.compute.workload.SimWorkload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.opendc.simulator.compute.workload.SimWorkload; /** * The {@link ComputeService} hosts the API implementation of the OpenDC Compute Engine. @@ -162,7 +162,9 @@ public void onStateChanged(@NotNull Host host, @NotNull Server server, @NotNull serviceServer.setState(newState); - if (newState == ServerState.TERMINATED || newState == ServerState.DELETED || newState == ServerState.ERROR) { + if (newState == ServerState.TERMINATED + || newState == ServerState.DELETED + || newState == ServerState.ERROR) { LOGGER.info("Server {} {} {} finished", server.getUid(), server.getName(), server.getFlavor()); if (activeServers.remove(server) != null) { @@ -591,10 +593,7 @@ public String toString() { @Nullable @Override - public void rescheduleServer( - @NotNull Server server, - @NotNull SimWorkload workload) - { + public void rescheduleServer(@NotNull Server server, @NotNull SimWorkload workload) { ServiceServer internalServer = (ServiceServer) findServer(server.getUid()); Host from = service.lookupHost(internalServer); diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckPointFactory.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckPointFactory.kt deleted file mode 100644 index 8b1378917..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckPointFactory.kt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckpointModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckpointModel.kt index cc0519186..781c2025f 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckpointModel.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckpointModel.kt @@ -1,6 +1,28 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + package org.opendc.compute.simulator.checkpoints public data class CheckpointModel( - val checkpointWait: Long = 60*60*1000, - val checkpointTime: Long = 5*60*1000 + val checkpointWait: Long = 60 * 60 * 1000, + val checkpointTime: Long = 5 * 60 * 1000, ) diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureTraceLoader.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureTraceLoader.kt deleted file mode 100644 index cf9bd2b85..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/FailureTraceLoader.kt +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure.models - -import org.opendc.trace.Trace -import org.opendc.trace.conv.FAILURE_DURATION -import org.opendc.trace.conv.FAILURE_INTENSITY -import org.opendc.trace.conv.FAILURE_START -import org.opendc.trace.conv.TABLE_FAILURES -import java.io.File -import java.lang.ref.SoftReference -import java.util.concurrent.ConcurrentHashMap - -/** - * A helper class for loading compute workload traces into memory. - * - * @param baseDir The directory containing the traces. - */ -public class FailureTraceLoader { - /** - * The cache of workloads. - */ - private val cache = ConcurrentHashMap>>() - - /** - * Read the metadata into a workload. - */ - private fun parseFailure(trace: Trace): List { - val reader = checkNotNull(trace.getTable(TABLE_FAILURES)).newReader() - - val failureStartTimeCol = reader.resolve(FAILURE_START) - val failureDurationCol = reader.resolve(FAILURE_DURATION) - val failureIntensityCol = reader.resolve(FAILURE_INTENSITY) - - val entries = mutableListOf() - - try { - while (reader.nextRow()) { - val failureStartTime = reader.getLong(failureStartTimeCol) - val failureDuration = reader.getLong(failureDurationCol) - val failureIntensity = reader.getDouble(failureIntensityCol) - - entries.add(Failure(failureStartTime, failureDuration, failureIntensity)) - } - - return entries - } catch (e: Exception) { - e.printStackTrace() - throw e - } finally { - reader.close() - } - } - - /** - * Load the trace with the specified [name] and [format]. - */ - public fun get(pathToFile: File): List { - val trace = Trace.open(pathToFile, "failure") - - return parseFailure(trace) - } - - /** - * Clear the workload cache. - */ - public fun reset() { - cache.clear() - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt deleted file mode 100644 index be32dba03..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/SampleBasedFailureModel.kt +++ /dev/null @@ -1,52 +0,0 @@ -package org.opendc.compute.simulator.failure.models - -import kotlinx.coroutines.delay -import org.apache.commons.math3.distribution.RealDistribution -import org.opendc.compute.service.ComputeService -import java.time.InstantSource -import java.util.random.RandomGenerator -import kotlin.coroutines.CoroutineContext -import kotlin.math.roundToLong - - -/** - * Sample based failure model - * - * @property context - * @property clock - * @property service - * @property random - * @property iatSampler A distribution from which the time until the next fault is sampled in ms - * @property durationSampler A distribution from which the duration of a fault is sampled in s - * @property nohSampler A distribution from which the number of hosts that fault is sampled. - */ -public class SampleBasedFailureModel( - context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: RandomGenerator, - - private val iatSampler: RealDistribution, - private val durationSampler: RealDistribution, - private val nohSampler: RealDistribution - ): FailureModel(context, clock, service, random) { - override suspend fun runInjector() { - while(true) { - val d = (iatSampler.sample() * 3.6e6).roundToLong() - - // Handle long overflow - if (clock.millis() + d <= 0) { - return - } - - delay(d) - - val victims = victimSelector.select(hosts, nohSampler.sample()) - - val faultDuration = (durationSampler.sample() * 3.6e6).toLong() - fault.apply(victims, faultDuration) - - break - } - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt deleted file mode 100644 index d78270355..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/TraceBasedFailureModel.kt +++ /dev/null @@ -1,45 +0,0 @@ -package org.opendc.compute.simulator.failure.models - -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch -import org.opendc.compute.service.ComputeService -import java.io.File -import java.time.InstantSource -import java.util.random.RandomGenerator -import kotlin.coroutines.CoroutineContext - -public data class Failure( - val failureStart: Long, - val failureDuration: Long, - val failureIntensity: Double, -) { - init { - require(failureStart >= 0.0) {"A failure cannot start at a negative time"} - require(failureDuration >= 0.0) {"A failure can not have a duration of 0 or less"} - require(failureIntensity >= 0.0) { "carbon intensity cannot be negative" } - } -} - -public class TraceBasedFailureModel( - context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: RandomGenerator, - pathToTrace: String - ) : FailureModel(context, clock, service, random){ - - private val failureList = FailureTraceLoader().get(File(pathToTrace)).iterator() - - override suspend fun runInjector() { - while(failureList.hasNext()) { - val failure = failureList.next() - - delay(failure.failureStart - clock.millis()) - - val victims = victimSelector.select(hosts, failure.failureIntensity) - scope.launch { - fault.apply(victims, failure.failureDuration) - } - } - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/PrefabFailureModelFactory.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/PrefabFailureModelFactory.kt deleted file mode 100644 index 031fc7b5c..000000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/prefab/PrefabFailureModelFactory.kt +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -@file:JvmName("FailureModels") - -package org.opendc.compute.simulator.failure.prefab - -import org.apache.commons.math3.distribution.ConstantRealDistribution -import org.apache.commons.math3.distribution.LogNormalDistribution -import org.apache.commons.math3.random.Well19937c -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.failure.models.SampleBasedFailureModel -import java.time.InstantSource -import java.util.random.RandomGenerator -import kotlin.coroutines.CoroutineContext - -public enum class FailurePrefab { - Grid5000, -} - -public fun getFailureModelPrefab(context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: RandomGenerator, - prefab: FailurePrefab): SampleBasedFailureModel { - when(prefab) { - FailurePrefab.Grid5000 -> return createGrid5000(context, clock, service, random) - - else -> error("Unknown failure prefab: $prefab") - } -} - - -/** - * A [FailureModel] based on the GRID'5000 failure trace. - * - * This fault injector uses parameters from the GRID'5000 failure trace as described in - * "A Framework for the Study of Grid Inter-Operation Mechanisms", A. Iosup, 2009. - */ -public fun createGrid5000(context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: RandomGenerator): SampleBasedFailureModel { - val rng = Well19937c(random.nextLong()) - - return SampleBasedFailureModel( - context, - clock, - service, - random, - - LogNormalDistribution(rng, 1.0, 0.1), - LogNormalDistribution(rng, 1.0, 0.1), - ConstantRealDistribution(0.5) - ) -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt index 827e57a23..cbdda0cbe 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt @@ -188,8 +188,7 @@ internal class Guest( val ctx = ctx ?: return if (target == ServerState.ERROR) { ctx.shutdown(Exception("Stopped because of ERROR")) - } - else{ + } else { ctx.shutdown() } diff --git a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt deleted file mode 100644 index 690bf4724..000000000 --- a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure - -import io.mockk.coVerify -import io.mockk.mockk -import kotlinx.coroutines.delay -import org.apache.commons.math3.distribution.LogNormalDistribution -import org.apache.commons.math3.random.Well19937c -import org.junit.jupiter.api.Test -import org.opendc.compute.simulator.SimHost -import org.opendc.simulator.kotlin.runSimulation -import java.time.Duration -import java.time.InstantSource -import kotlin.coroutines.CoroutineContext -import kotlin.math.ln - -/** - * Test suite for [HostFaultInjector] class. - */ -class HostFaultInjectorTest { - /** - * Simple test case to test that nothing happens when the injector is not started. - */ - @Test - fun testInjectorNotStarted() = - runSimulation { - val host = mockk(relaxUnitFun = true) - - val injector = createSimpleInjector(coroutineContext, timeSource, setOf(host)) - - coVerify(exactly = 0) { host.fail() } - coVerify(exactly = 0) { host.recover() } - - injector.close() - } - - /** - * Simple test case to test a start stop fault where the machine is stopped and started after some time. - */ - @Test - fun testInjectorStopsMachine() = - runSimulation { - val host = mockk(relaxUnitFun = true) - - val injector = createSimpleInjector(coroutineContext, timeSource, setOf(host)) - - injector.start() - - delay(Duration.ofDays(55).toMillis()) - - injector.close() - - coVerify(exactly = 1) { host.fail() } - coVerify(exactly = 1) { host.recover() } - } - - /** - * Simple test case to test a start stop fault where multiple machines are stopped. - */ - @Test - fun testInjectorStopsMultipleMachines() = - runSimulation { - val hosts = - listOf( - mockk(relaxUnitFun = true), - mockk(relaxUnitFun = true), - ) - - val injector = createSimpleInjector(coroutineContext, timeSource, hosts.toSet()) - - injector.start() - - delay(Duration.ofDays(55).toMillis()) - - injector.close() - - coVerify(exactly = 1) { hosts[0].fail() } - coVerify(exactly = 1) { hosts[1].fail() } - coVerify(exactly = 1) { hosts[0].recover() } - coVerify(exactly = 1) { hosts[1].recover() } - } - - /** - * Create a simple start stop fault injector. - */ - private fun createSimpleInjector( - context: CoroutineContext, - clock: InstantSource, - hosts: Set, - ): HostFaultInjector { - val rng = Well19937c(0) - val iat = LogNormalDistribution(rng, ln(24 * 7.0), 1.03) - val selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25)) - val fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)) - - return HostFaultInjector(context, clock, hosts, iat, selector, fault) - } -} diff --git a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt index 92186dea9..f374b71f2 100644 --- a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt +++ b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt @@ -35,7 +35,6 @@ import kotlin.io.path.inputStream * A helper class for reading a topology specification file. */ public class TopologyReader { - public fun read(path: Path): TopologySpec { return read(path.inputStream()) } diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt index 88cedca45..7bea920e3 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/VirtualMachine.kt @@ -51,6 +51,6 @@ public data class VirtualMachine( val stopTime: Instant, val trace: SimTrace, val interferenceProfile: VmInterferenceProfile?, -){ +) { val duration: Long = stopTime.toEpochMilli() - startTime.toEpochMilli() } diff --git a/opendc-experiments/opendc-experiments-base/build.gradle.kts b/opendc-experiments/opendc-experiments-base/build.gradle.kts index c75af87be..30510785a 100644 --- a/opendc-experiments/opendc-experiments-base/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-base/build.gradle.kts @@ -37,11 +37,13 @@ dependencies { implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.0") implementation(libs.progressbar) + implementation(project(mapOf("path" to ":opendc-simulator:opendc-simulator-core"))) + implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-workload"))) implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-telemetry"))) - implementation(project(mapOf("path" to ":opendc-simulator:opendc-simulator-core"))) implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-topology"))) implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-carbon"))) + implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-failure"))) runtimeOnly(libs.log4j.core) runtimeOnly(libs.log4j.slf4j) diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt index ac64ae5ae..970754b0a 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt @@ -20,13 +20,11 @@ * SOFTWARE. */ -@file:JvmName("TraceHelpers") +@file:JvmName("ScenarioHelpers") package org.opendc.experiments.base.runner import CheckpointModelSpec -import FailureModelSpec -import getFailureModel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.launch @@ -35,9 +33,11 @@ import kotlinx.coroutines.yield import org.opendc.compute.api.Server import org.opendc.compute.api.ServerState import org.opendc.compute.api.ServerWatcher +import org.opendc.compute.failure.models.FailureModel import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.failure.models.FailureModel import org.opendc.compute.workload.VirtualMachine +import org.opendc.experiments.base.scenario.specs.FailureModelSpec +import org.opendc.experiments.base.scenario.specs.createFailureModel import java.time.InstantSource import java.util.Random import kotlin.coroutines.coroutineContext @@ -92,8 +92,10 @@ public suspend fun ComputeService.replay( val client = newClient() // Create a failure model based on the failureModelSpec, if not null, otherwise set failureModel to null - val failureModel: FailureModel? = failureModelSpec?.let { - getFailureModel(coroutineContext, clock, this, Random(seed), it) } + val failureModel: FailureModel? = + failureModelSpec?.let { + createFailureModel(coroutineContext, clock, this, Random(seed), it) + } // Create new image for the virtual machine val image = client.newImage("vm-image") diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt index d09fcb595..cb4fdd461 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt @@ -137,38 +137,6 @@ public fun runScenario( } } -/** - * When the simulation is run, saves the simulation results into a seed folder. This is useful for debugging purposes. - * @param provisioner The provisioner used to setup and run the simulation. - * @param serviceDomain The domain of the compute service. - * @param scenario The scenario being run in the simulation. - * @param seed The seed used for randomness in the simulation. - * @param partition The partition name for the output data. - * @param startTime The start time of the simulation. - - */ -public fun saveInSeedFolder( - provisioner: Provisioner, - serviceDomain: String, - scenario: Scenario, - seed: Long, - partition: String, - startTime: Duration, -) { - provisioner.runStep( - registerComputeMonitor( - serviceDomain, - ParquetComputeMonitor( - File(scenario.outputFolder), - partition, - bufferSize = 4096, - ), - Duration.ofSeconds(scenario.exportModelSpec.exportInterval), - startTime, - ), - ) -} - /** * Saves the simulation results into a specific output folder received from the input. * @@ -216,10 +184,10 @@ public fun clearOutputFolder(outputFolderPath: String) { * @param folderPath The path to the output folder */ private fun setupOutputFolderStructure(folderPath: String) { - val trackrPath = folderPath + "/trackr.json" - val simulationAnalysisPath = folderPath + "/simulation-analysis/" - val energyAnalysisPath = simulationAnalysisPath + "/power_draw/" - val emissionsAnalysisPath = simulationAnalysisPath + "/carbon_emission/" + val trackrPath = "$folderPath/trackr.json" + val simulationAnalysisPath = "$folderPath/simulation-analysis/" + val energyAnalysisPath = "$simulationAnalysisPath/power_draw/" + val emissionsAnalysisPath = "$simulationAnalysisPath/carbon_emission/" File(folderPath).mkdir() File(trackrPath).createNewFile() diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt index d5091e4e4..02a8234db 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt @@ -25,9 +25,9 @@ package org.opendc.experiments.base.scenario import AllocationPolicySpec import CheckpointModelSpec import ExportModelSpec -import FailureModelSpec import ScenarioTopologySpec import WorkloadSpec +import org.opendc.experiments.base.scenario.specs.FailureModelSpec /** * A data class representing a scenario for a set of experiments. diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt index 26941f075..e7b52c555 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioFactories.kt @@ -22,7 +22,6 @@ package org.opendc.experiments.base.scenario -import ScenarioTopologySpec import org.opendc.experiments.base.scenario.specs.ScenarioSpec import java.io.File @@ -82,12 +81,12 @@ public fun getScenarios(scenarioSpec: ScenarioSpec): List { checkpointModelSpec = checkpointModelSpec, carbonTracePath = carbonTracePath, exportModelSpec = exportModelSpec, - outputFolder = scenarioSpec.outputFolder, + outputFolder = outputFolder, name = scenarioID.toString(), runs = scenarioSpec.runs, initialSeed = scenarioSpec.initialSeed, ) - trackScenario(scenarioSpec, outputFolder, scenario, scenarioTopologySpec) + trackScenario(scenarioSpec, outputFolder, scenario) scenarios.add(scenario) } } @@ -113,17 +112,17 @@ public fun trackScenario( scenarioSpec: ScenarioSpec, outputFolder: String, scenario: Scenario, - topologySpec: ScenarioTopologySpec, ) { val trackrPath = "$outputFolder/trackr.json" scenarioWriter.write( ScenarioSpec( id = scenario.id, name = scenarioSpec.name, - topologies = listOf(topologySpec), + topologies = listOf(scenario.topologySpec), workloads = listOf(scenario.workloadSpec), allocationPolicies = listOf(scenario.allocationPolicySpec), - // when implemented, add failure models here + failureModels = listOf(scenario.failureModelSpec), + checkpointModels = listOf(scenario.checkpointModelSpec), carbonTracePaths = listOf(scenario.carbonTracePath), exportModels = listOf(scenario.exportModelSpec), outputFolder = scenario.outputFolder, diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioReader.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioReader.kt index d461b56ad..3bbd500b1 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioReader.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ScenarioReader.kt @@ -42,7 +42,6 @@ public class ScenarioReader { return jsonReader.decodeFromStream(input) } - @OptIn(ExperimentalSerializationApi::class) public fun read(path: Path): ScenarioSpec { val input = path.inputStream() diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/CheckpointModelSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/CheckpointModelSpec.kt index 135a53bc7..9432fc9be 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/CheckpointModelSpec.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/CheckpointModelSpec.kt @@ -1,7 +1,29 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + import kotlinx.serialization.Serializable @Serializable -public data class CheckpointModelSpec ( - val checkpointWait: Long = 60*60*1000, - val checkpointTime: Long = 5*60*1000 +public data class CheckpointModelSpec( + val checkpointWait: Long = 60 * 60 * 1000, + val checkpointTime: Long = 5 * 60 * 1000, ) diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt index b863e0dc8..a27e77bc8 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/FailureModelSpec.kt @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.base.scenario.specs + /* * Copyright (c) 2024 AtLarge Research * @@ -23,121 +47,274 @@ import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import org.apache.commons.math3.distribution.ConstantRealDistribution +import org.apache.commons.math3.distribution.ExponentialDistribution +import org.apache.commons.math3.distribution.GammaDistribution import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.distribution.NormalDistribution +import org.apache.commons.math3.distribution.ParetoDistribution import org.apache.commons.math3.distribution.RealDistribution +import org.apache.commons.math3.distribution.UniformRealDistribution +import org.apache.commons.math3.distribution.WeibullDistribution import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.failure.models.FailureModel +import org.opendc.compute.failure.models.SampleBasedFailureModel +import org.opendc.compute.failure.models.TraceBasedFailureModel +import org.opendc.compute.failure.prefab.FailurePrefab +import org.opendc.compute.failure.prefab.createFailureModelPrefab import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.failure.models.FailureModel -import org.opendc.compute.simulator.failure.models.SampleBasedFailureModel -import org.opendc.compute.simulator.failure.models.TraceBasedFailureModel -import org.opendc.compute.simulator.failure.prefab.FailurePrefab -import org.opendc.compute.simulator.failure.prefab.getFailureModelPrefab import java.io.File import java.time.InstantSource import kotlin.coroutines.CoroutineContext +/** + * Specifications of the different Failure models + * There are three types of Specs that can be used by using their SerialName as the type. + * + * @constructor Create empty Failure model spec + */ + @Serializable public sealed interface FailureModelSpec { public var name: String } +/** + * A failure model spec for failure models based on a failure trace. + * + * @property pathToFile Path to the parquet file that contains the failure trace + */ @Serializable @SerialName("trace-based") public data class TraceBasedFailureModelSpec( - public val pathToFile: String -): FailureModelSpec { + public val pathToFile: String, +) : FailureModelSpec { override var name: String = File(pathToFile).nameWithoutExtension + init { require(File(pathToFile).exists()) { "Path to file $pathToFile does not exist" } } } +/** + * A specification for a failure model that is already present in OpenDC. + * + * @property prefabName The name of the prefab. It needs to be valid [FailurePrefab] + */ @Serializable @SerialName("prefab") public data class PrefabFailureModelSpec( - public val prefabName: FailurePrefab -): FailureModelSpec { + public val prefabName: FailurePrefab, +) : FailureModelSpec { override var name: String = prefabName.toString() } +/** + * Specification of a custom failure model that is defined by three distributions to sample from. + * Distributions are defined using a [DistributionSpec]. + * + * @property iatSampler Sampler for the time between failures defined in hours + * @property durationSampler Sampler for the time of a failure defined in hours + * @property nohSampler Sampler for ratio of hosts that fail defined as a double between 0.0 and 1.0 + * @constructor Create empty Custom failure model spec + */ @Serializable @SerialName("custom") public data class CustomFailureModelSpec( public val iatSampler: DistributionSpec, public val durationSampler: DistributionSpec, public val nohSampler: DistributionSpec, -): FailureModelSpec { +) : FailureModelSpec { override var name: String = "custom" } +/** + * Specifications of the different Distributions that can used to create a [CustomFailureModelSpec] + * All [DistributionSpec]s have a different definition based on the variables they need to function. + * Available [DistributionSpec] are: + * - [ConstantDistributionSpec] + * - [ExponentialDistributionSpec] + * - [GammaDistributionSpec] + * - [LogNormalDistributionSpec] + * - [ParetoDistributionSpec] + * - [UniformDistributionSpec] + * - [WeibullDistributionSpec] +*/ + @Serializable public sealed interface DistributionSpec +@Serializable +@SerialName("constant") +public data class ConstantDistributionSpec( + public val value: Double, +) : DistributionSpec { + init { + require(value > 0.0) { "Value must be greater than 0.0" } + } +} + +@Serializable +@SerialName("exponential") +public data class ExponentialDistributionSpec( + public val mean: Double, +) : DistributionSpec + +@Serializable +@SerialName("gamma") +public data class GammaDistributionSpec( + public val shape: Double, + public val scale: Double, +) : DistributionSpec + @Serializable @SerialName("log-normal") public data class LogNormalDistributionSpec( public val scale: Double, - public val size: Double, -): DistributionSpec + public val shape: Double, +) : DistributionSpec @Serializable -@SerialName("constant") -public data class ConstantDistributionSpec( - public val value: Double -): DistributionSpec +@SerialName("normal") +public data class NormalDistributionSpec( + public val mean: Double, + public val std: Double, +) : DistributionSpec + +@Serializable +@SerialName("pareto") +public data class ParetoDistributionSpec( + public val scale: Double, + public val shape: Double, +) : DistributionSpec +@Serializable +@SerialName("uniform") +public data class UniformDistributionSpec( + public val upper: Double, + public val lower: Double, +) : DistributionSpec { + init { + require(upper > lower) { "Upper bound must be greater than the lower bound" } + } +} + +@Serializable +@SerialName("weibull") +public data class WeibullDistributionSpec( + public val alpha: Double, + public val beta: Double, +) : DistributionSpec +/** + * Create a [FailureModel] based on the provided [FailureModelSpec] + * + * @param context + * @param clock + * @param service + * @param random + * @param failureModelSpec + * @return + */ +public fun createFailureModel( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: java.util.random.RandomGenerator, + failureModelSpec: FailureModelSpec?, +): FailureModel? { + return when (failureModelSpec) { + is PrefabFailureModelSpec -> createFailureModel(context, clock, service, random, failureModelSpec) + is CustomFailureModelSpec -> createFailureModel(context, clock, service, random, failureModelSpec) + is TraceBasedFailureModelSpec -> createFailureModel(context, clock, service, random, failureModelSpec) + else -> null + } +} +/** + * Create [FailureModel] based on the provided [PrefabFailureModelSpec] + * + * @param context + * @param clock + * @param service + * @param random + * @param failureModel + * @return + */ +public fun createFailureModel( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: java.util.random.RandomGenerator, + failureModel: PrefabFailureModelSpec, +): FailureModel { + return createFailureModelPrefab(context, clock, service, random, failureModel.prefabName) +} -public fun getFailureModel(context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: java.util.random.RandomGenerator, - failureModel: PrefabFailureModelSpec): FailureModel { - return getFailureModelPrefab(context, clock, service, random, failureModel.prefabName) +/** + * Create [FailureModel] based on the provided [TraceBasedFailureModel] + * + * @param context + * @param clock + * @param service + * @param random + * @param failureModel + * @return + */ +public fun createFailureModel( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: java.util.random.RandomGenerator, + failureModel: TraceBasedFailureModelSpec, +): FailureModel { + return TraceBasedFailureModel(context, clock, service, random, failureModel.pathToFile) } -public fun getFailureModel(context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: java.util.random.RandomGenerator, - failureModel: CustomFailureModelSpec): FailureModel { +/** + * Create [FailureModel] based on the provided [CustomFailureModelSpec] + * + * @param context + * @param clock + * @param service + * @param random + * @param failureModel + * @return + */ +public fun createFailureModel( + context: CoroutineContext, + clock: InstantSource, + service: ComputeService, + random: java.util.random.RandomGenerator, + failureModel: CustomFailureModelSpec, +): FailureModel { val rng: org.apache.commons.math3.random.RandomGenerator = Well19937c(random.nextLong()) - val iatSampler = getSampler(rng, failureModel.iatSampler) - val durationSampler = getSampler(rng, failureModel.durationSampler) - val nohSampler = getSampler(rng, failureModel.nohSampler) + val iatSampler = createSampler(rng, failureModel.iatSampler) + val durationSampler = createSampler(rng, failureModel.durationSampler) + val nohSampler = createSampler(rng, failureModel.nohSampler) return SampleBasedFailureModel(context, clock, service, random, iatSampler, durationSampler, nohSampler) } -public fun getSampler(rng: org.apache.commons.math3.random.RandomGenerator, distributionSpec: DistributionSpec): RealDistribution { +/** + * Create a [RealDistribution] to sample from based on the provided [DistributionSpec] + * + * @param rng + * @param distributionSpec + * @return + */ +public fun createSampler( + rng: org.apache.commons.math3.random.RandomGenerator, + distributionSpec: DistributionSpec, +): RealDistribution { return when (distributionSpec) { - is LogNormalDistributionSpec -> LogNormalDistribution(rng, distributionSpec.scale, distributionSpec.size) is ConstantDistributionSpec -> ConstantRealDistribution(distributionSpec.value) - else -> error("The given distributionSpec: $distributionSpec is not a valid DistributionSpec") - } -} - -public fun getFailureModel(context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: java.util.random.RandomGenerator, - failureModel: TraceBasedFailureModelSpec): FailureModel { - - return TraceBasedFailureModel(context, clock, service, random, failureModel.pathToFile) -} - -public fun getFailureModel(context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: java.util.random.RandomGenerator, - failureModel: FailureModelSpec?): FailureModel? { - - return when(failureModel) { - is PrefabFailureModelSpec -> getFailureModel(context, clock, service, random, failureModel) - is CustomFailureModelSpec -> getFailureModel(context, clock, service, random, failureModel) - is TraceBasedFailureModelSpec -> getFailureModel(context, clock, service, random, failureModel) - else -> null + is ExponentialDistributionSpec -> ExponentialDistribution(rng, distributionSpec.mean) + is GammaDistributionSpec -> GammaDistribution(rng, distributionSpec.shape, distributionSpec.scale) + is LogNormalDistributionSpec -> LogNormalDistribution(rng, distributionSpec.scale, distributionSpec.shape) + is NormalDistributionSpec -> NormalDistribution(rng, distributionSpec.mean, distributionSpec.std) + is ParetoDistributionSpec -> ParetoDistribution(rng, distributionSpec.scale, distributionSpec.shape) + is UniformDistributionSpec -> UniformRealDistribution(rng, distributionSpec.lower, distributionSpec.upper) + is WeibullDistributionSpec -> WeibullDistribution(rng, distributionSpec.alpha, distributionSpec.beta) } } diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt index 3fbac1da7..eb3cd04e4 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt @@ -25,7 +25,6 @@ package org.opendc.experiments.base.scenario.specs import AllocationPolicySpec import CheckpointModelSpec import ExportModelSpec -import FailureModelSpec import ScenarioTopologySpec import WorkloadSpec import kotlinx.serialization.Serializable diff --git a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/power/CPUPowerModelsFactory.kt b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/power/CPUPowerModelsFactory.kt index 5eafb5faa..2c64944c4 100644 --- a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/power/CPUPowerModelsFactory.kt +++ b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/power/CPUPowerModelsFactory.kt @@ -28,7 +28,7 @@ public enum class CPUPowerModel { Sqrt, Linear, Square, - Cubic + Cubic, } public fun getPowerModel( diff --git a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimRuntimeWorkload.java b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimRuntimeWorkload.java index 8547356d0..c116a5e59 100644 --- a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimRuntimeWorkload.java +++ b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimRuntimeWorkload.java @@ -48,6 +48,22 @@ public class SimRuntimeWorkload implements SimWorkload, FlowStageLogic { private long checkpointWait; // How long to wait until a new checkpoint is made? private long totalChecks; + public SimRuntimeWorkload(long duration, double utilization) { + this(duration, utilization, 0, 0); + // if (duration < 0) { + // throw new IllegalArgumentException("Duration must be positive"); + // } else if (utilization <= 0.0 || utilization > 1.0) { + // throw new IllegalArgumentException("Utilization must be in (0, 1]"); + // } + // + // this.checkpointTime = 0L; + // this.checkpointWait = 0L; + // this.duration = duration; + // + // this.utilization = utilization; + // this.remainingDuration = duration; + } + /** * Construct a new {@link SimRuntimeWorkload}. * @@ -67,7 +83,8 @@ public SimRuntimeWorkload(long duration, double utilization, long checkpointTime if (this.checkpointWait > 0) { // Determine the number of checkpoints that need to be made during the workload - // If the total duration is divisible by the wait time between checkpoints, we can remove the last checkpoint + // If the total duration is divisible by the wait time between checkpoints, we can remove the last + // checkpoint int to_remove = ((this.duration % this.checkpointWait == 0) ? 1 : 0); this.totalChecks = this.duration / this.checkpointWait - to_remove; this.duration += (this.checkpointTime * totalChecks); @@ -129,18 +146,18 @@ public SimRuntimeWorkload snapshot() { // Calculate last checkpoint var total_check_time = this.checkpointWait + this.checkpointTime; var processed_time = this.duration - this.remainingDuration; - var processed_checks = (int)(processed_time / total_check_time); - var processed_time_last_check = (processed_checks * total_check_time); // The processed time after the last checkpoint - - remaining_time = this.duration - processed_time_last_check; // The remaining duration to process after last checkpoint - var remaining_checks = (int)(remaining_time / total_check_time); - remaining_time -= (remaining_checks * checkpointTime); - } - else { + var processed_checks = (int) (processed_time / total_check_time); + var processed_time_last_check = + (processed_checks * total_check_time); // The processed time after the last checkpoint + + remaining_time = this.duration + - processed_time_last_check; // The remaining duration to process after last checkpoint + var remaining_checks = (int) (remaining_time / total_check_time); + remaining_time -= (remaining_checks * checkpointTime); + } else { remaining_time = duration; } - return new SimRuntimeWorkload(remaining_time, utilization, this.checkpointTime, this.checkpointWait); } diff --git a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimTrace.java b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimTrace.java index 363e100e7..21379f0d5 100644 --- a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimTrace.java +++ b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimTrace.java @@ -65,6 +65,15 @@ private SimTrace(double[] usageCol, long[] deadlineCol, int[] coresCol, int size this.size = size; } + /** + * Construct a {@link SimWorkload} for this trace. + * + * // * @param offset The offset for the timestamps. + */ + public SimWorkload createWorkload(long start) { + return createWorkload(start, 0, 0); + } + /** * Construct a {@link SimWorkload} for this trace. * @@ -219,8 +228,15 @@ private static class Workload implements SimWorkload { private long checkpointWait; // How long to wait until a new checkpoint is made? private long total_checks; - private Workload(long start, double[] usageCol, long[] deadlineCol, int[] coresCol, int size, int index, - long checkpointTime, long checkpointWait) { + private Workload( + long start, + double[] usageCol, + long[] deadlineCol, + int[] coresCol, + int size, + int index, + long checkpointTime, + long checkpointWait) { this.start = start; this.usageCol = usageCol; this.deadlineCol = deadlineCol; diff --git a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimWorkloads.java b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimWorkloads.java index ccaa70e94..294b5dde9 100644 --- a/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimWorkloads.java +++ b/opendc-simulator/opendc-simulator-compute/src/main/java/org/opendc/simulator/compute/workload/SimWorkloads.java @@ -41,6 +41,16 @@ public static SimWorkload flops(long flops, double utilization) { return new SimFlopsWorkload(flops, utilization); } + /** + * Create a {@link SimWorkload} that consumes the CPU resources for a specified duration at the given utilization. + * + * @param duration The duration of the workload in milliseconds. + * @param utilization The CPU utilization of the workload. + */ + public static SimWorkload runtime(long duration, double utilization) { + return runtime(duration, utilization, 0, 0); + } + /** * Create a {@link SimWorkload} that consumes the CPU resources for a specified duration at the given utilization. * @@ -57,7 +67,8 @@ public static SimWorkload runtime(long duration, double utilization, long checkp * @param duration The duration of the workload. * @param utilization The CPU utilization of the workload. */ - public static SimWorkload runtime(Duration duration, double utilization, long checkpoint_time, long checkpoint_wait) { + public static SimWorkload runtime( + Duration duration, double utilization, long checkpoint_time, long checkpoint_wait) { return runtime(duration.toMillis(), utilization, checkpoint_time, checkpoint_wait); } diff --git a/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/workload/SimChainWorkloadTest.kt b/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/workload/SimChainWorkloadTest.kt index 9ea9d300a..a0301dda6 100644 --- a/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/workload/SimChainWorkloadTest.kt +++ b/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/workload/SimChainWorkloadTest.kt @@ -74,8 +74,8 @@ class SimChainWorkloadTest { val workload = SimWorkloads.chain( - SimWorkloads.runtime(1000, 1.0), - SimWorkloads.runtime(1000, 1.0), + SimWorkloads.runtime(1000, 1.0, 0L, 0L), + SimWorkloads.runtime(1000, 1.0, 0L, 0L), ) machine.runWorkload(workload) @@ -102,7 +102,7 @@ class SimChainWorkloadTest { val workload = SimWorkloads.chain( workloadA, - SimWorkloads.runtime(1000, 1.0), + SimWorkloads.runtime(1000, 1.0, 0L, 0L), ) assertThrows { machine.runWorkload(workload) } @@ -300,6 +300,6 @@ class SimChainWorkloadTest { machine.runWorkload(snapshot) - assertEquals(3500, timeSource.millis()) + assertEquals(4000, timeSource.millis()) } } diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt index 01d66a4ad..881204b18 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt @@ -27,7 +27,7 @@ package org.opendc.trace.conv /** * A column containing the task identifier. */ -public const val FAILURE_START: String = "failure_start" +public const val FAILURE_INTERVAL: String = "failure_start" /** * A column containing the task identifier. diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt index 3dc511332..e7e13b031 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt @@ -25,7 +25,7 @@ package org.opendc.trace.formats.failure import org.opendc.trace.TableReader import org.opendc.trace.conv.FAILURE_DURATION import org.opendc.trace.conv.FAILURE_INTENSITY -import org.opendc.trace.conv.FAILURE_START +import org.opendc.trace.conv.FAILURE_INTERVAL import org.opendc.trace.formats.failure.parquet.FailureFragment import org.opendc.trace.util.parquet.LocalParquetReader import java.time.Duration @@ -59,7 +59,7 @@ internal class FailureTableReader(private val reader: LocalParquetReader colFailureStart + FAILURE_INTERVAL -> colFailureStart FAILURE_DURATION -> colFailureDuration FAILURE_INTENSITY -> colFailureIntensity else -> -1 diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTraceFormat.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTraceFormat.kt index d30a446ea..892216a0e 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTraceFormat.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTraceFormat.kt @@ -28,7 +28,7 @@ import org.opendc.trace.TableReader import org.opendc.trace.TableWriter import org.opendc.trace.conv.FAILURE_DURATION import org.opendc.trace.conv.FAILURE_INTENSITY -import org.opendc.trace.conv.FAILURE_START +import org.opendc.trace.conv.FAILURE_INTERVAL import org.opendc.trace.conv.TABLE_FAILURES import org.opendc.trace.formats.failure.parquet.FailureReadSupport import org.opendc.trace.spi.TableDetails @@ -56,7 +56,7 @@ public class FailureTraceFormat : TraceFormat { TABLE_FAILURES -> TableDetails( listOf( - TableColumn(FAILURE_START, TableColumnType.Long), + TableColumn(FAILURE_INTERVAL, TableColumnType.Long), TableColumn(FAILURE_DURATION, TableColumnType.Long), TableColumn(FAILURE_INTENSITY, TableColumnType.Double), ), diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt index 13d60d4c9..0c3d19c88 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt @@ -31,7 +31,7 @@ import org.apache.parquet.schema.PrimitiveType import org.apache.parquet.schema.Types import org.opendc.trace.conv.FAILURE_DURATION import org.opendc.trace.conv.FAILURE_INTENSITY -import org.opendc.trace.conv.FAILURE_START +import org.opendc.trace.conv.FAILURE_INTERVAL /** * A [ReadSupport] instance for [Task] objects. @@ -44,9 +44,9 @@ internal class FailureReadSupport(private val projection: List?) : ReadS */ private val colMap = mapOf( - FAILURE_START to "start", + FAILURE_INTERVAL to "start", FAILURE_DURATION to "duration", - FAILURE_INTENSITY to "intensity" + FAILURE_INTENSITY to "intensity", ) override fun init(context: InitContext): ReadContext { diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt index 5762c2bae..34045bd73 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt @@ -87,7 +87,7 @@ internal class FailureRecordMaterializer(schema: MessageType) : RecordMaterializ FailureFragment( localStart, localDuration, - localIntensity + localIntensity, ) override fun getRootConverter(): GroupConverter = root diff --git a/opendc-web/opendc-web-runner/build.gradle.kts b/opendc-web/opendc-web-runner/build.gradle.kts index d5d1dbc65..40f94644f 100644 --- a/opendc-web/opendc-web-runner/build.gradle.kts +++ b/opendc-web/opendc-web-runner/build.gradle.kts @@ -58,6 +58,7 @@ dependencies { implementation(project(mapOf("path" to ":opendc-experiments:opendc-experiments-base"))) implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-topology"))) implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-telemetry"))) + implementation(project(mapOf("path" to ":opendc-compute:opendc-compute-failure"))) cliImplementation(libs.clikt) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt index 2ff3ac423..7eb6e21fe 100644 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt @@ -23,6 +23,8 @@ package org.opendc.web.runner import mu.KotlinLogging +import org.opendc.compute.failure.prefab.FailurePrefab +import org.opendc.compute.failure.prefab.createFailureModelPrefab import org.opendc.compute.service.ComputeService import org.opendc.compute.service.scheduler.createComputeScheduler import org.opendc.compute.simulator.provisioner.Provisioner @@ -281,7 +283,7 @@ public class OpenDCRunner( val phenomena = scenario.phenomena val failureModel = if (phenomena.failures) { - Grid5000(Duration.ofDays(7)) + createFailureModelPrefab(coroutineContext, timeSource, service, Random(seed), FailurePrefab.G5k06Exp) } else { null } diff --git a/settings.gradle.kts b/settings.gradle.kts index 66601847c..d01d12617 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -23,12 +23,13 @@ rootProject.name = "opendc" include(":opendc-common") include(":opendc-compute:opendc-compute-api") +include(":opendc-compute:opendc-compute-carbon") +include(":opendc-compute:opendc-compute-failure") include(":opendc-compute:opendc-compute-service") +include(":opendc-compute:opendc-compute-simulator") include(":opendc-compute:opendc-compute-telemetry") include(":opendc-compute:opendc-compute-topology") -include(":opendc-compute:opendc-compute-simulator") include(":opendc-compute:opendc-compute-workload") -include(":opendc-compute:opendc-compute-carbon") include(":opendc-workflow:opendc-workflow-api") include(":opendc-workflow:opendc-workflow-service") include(":opendc-faas:opendc-faas-api") From ca6ce7e549fa1ed1d263f0e380a0b519a0ebf50a Mon Sep 17 00:00:00 2001 From: Dante Niewenhuis Date: Fri, 3 May 2024 18:17:18 +0200 Subject: [PATCH 4/5] Added documentation --- .../failure/models/TraceBasedFailureModel.kt | 21 +- .../prefab/PrefabFailureModelFactory.kt | 5 +- .../org/opendc/trace/conv/FailureColumns.kt | 2 +- .../formats/failure/FailureTableReader.kt | 8 +- .../failure/parquet/FailureFragment.kt | 2 +- .../failure/parquet/FailureReadSupport.kt | 4 +- .../parquet/FailureRecordMaterializer.kt | 26 +-- .../docs/documentation/Input/FailureModels.md | 202 ++++++++++++++++++ site/docs/documentation/Input/Scenario.md | 4 +- site/docs/documentation/Input/img.png | Bin 0 -> 110177 bytes 10 files changed, 240 insertions(+), 34 deletions(-) create mode 100644 site/docs/documentation/Input/FailureModels.md create mode 100644 site/docs/documentation/Input/img.png diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt index ea376ce0e..0edc17d83 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt @@ -51,7 +51,7 @@ public data class Failure( init { require(failureInterval >= 0.0) { "A failure cannot start at a negative time" } require(failureDuration >= 0.0) { "A failure can not have a duration of 0 or less" } - require(failureIntensity > 0.0 && failureInterval <= 1.0) { "The intensity of a failure has to be in the range (0.0, 1.0]" } + require(failureIntensity > 0.0 && failureIntensity <= 1.0) { "The intensity of a failure has to be in the range (0.0, 1.0]" } } } @@ -72,20 +72,21 @@ public class TraceBasedFailureModel( service: ComputeService, random: RandomGenerator, pathToTrace: String, + private val repeat: Boolean = false ) : FailureModel(context, clock, service, random) { - private val failureList = loadTrace(pathToTrace).iterator() + private val failureList = loadTrace(pathToTrace) override suspend fun runInjector() { - while (failureList.hasNext()) { - val failure = failureList.next() + do { + for (failure in failureList) { + delay(failure.failureInterval - clock.millis()) - delay(failure.failureInterval - clock.millis()) - - val victims = victimSelector.select(hosts, failure.failureIntensity) - scope.launch { - fault.apply(victims, failure.failureDuration) + val victims = victimSelector.select(hosts, failure.failureIntensity) + scope.launch { + fault.apply(victims, failure.failureDuration) + } } - } + } while(repeat) } /** diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt index 990301e53..477f3ac4b 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/prefab/PrefabFailureModelFactory.kt @@ -70,7 +70,10 @@ public enum class FailurePrefab { } /** - * Get a [SampleBasedFailureModel] based on the provided prefab + * Get a [SampleBasedFailureModel] based on the provided prefab. + * All Failure models are based on values taken from "The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems" + * Which can be found at https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634 + * * * @param context * @param clock diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt index 881204b18..3f6530411 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/FailureColumns.kt @@ -27,7 +27,7 @@ package org.opendc.trace.conv /** * A column containing the task identifier. */ -public const val FAILURE_INTERVAL: String = "failure_start" +public const val FAILURE_INTERVAL: String = "failure_interval" /** * A column containing the task identifier. diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt index e7e13b031..a1c10bd0f 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/FailureTableReader.kt @@ -53,13 +53,13 @@ internal class FailureTableReader(private val reader: LocalParquetReader colFailureStart + FAILURE_INTERVAL -> colFailureInterval FAILURE_DURATION -> colFailureDuration FAILURE_INTENSITY -> colFailureIntensity else -> -1 @@ -67,7 +67,7 @@ internal class FailureTableReader(private val reader: LocalParquetReader record.failureStart + colFailureInterval -> record.failureInterval colFailureDuration -> record.failureDuration else -> throw IllegalArgumentException("Invalid column") } diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureFragment.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureFragment.kt index 09a68c024..49f43aa1e 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureFragment.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureFragment.kt @@ -26,7 +26,7 @@ package org.opendc.trace.formats.failure.parquet * A task in the Workflow Trace Format. */ internal data class FailureFragment( - val failureStart: Long, + val failureInterval: Long, val failureDuration: Long, val failureIntensity: Double, ) diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt index 0c3d19c88..d49f86c65 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureReadSupport.kt @@ -44,7 +44,7 @@ internal class FailureReadSupport(private val projection: List?) : ReadS */ private val colMap = mapOf( - FAILURE_INTERVAL to "start", + FAILURE_INTERVAL to "interval", FAILURE_DURATION to "duration", FAILURE_INTENSITY to "intensity", ) @@ -85,7 +85,7 @@ internal class FailureReadSupport(private val projection: List?) : ReadS .addFields( Types .optional(PrimitiveType.PrimitiveTypeName.INT64) - .named("failure_start"), + .named("failure_interval"), Types .optional(PrimitiveType.PrimitiveTypeName.INT64) .named("failure_duration"), diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt index 34045bd73..5a00f8c9f 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/failure/parquet/FailureRecordMaterializer.kt @@ -35,9 +35,9 @@ internal class FailureRecordMaterializer(schema: MessageType) : RecordMaterializ /** * State of current record being read. */ - private var localStart: Long = 0L - private var localDuration: Long = 0L - private var localIntensity: Double = 0.0 + private var localFailureInterval: Long = 0L + private var localFailureDuration: Long = 0L + private var localFailureIntensity: Double = 0.0 /** * Root converter for the record. @@ -50,22 +50,22 @@ internal class FailureRecordMaterializer(schema: MessageType) : RecordMaterializ private val converters = schema.fields.map { type -> when (type.name) { - "failure_start" -> + "failure_interval" -> object : PrimitiveConverter() { override fun addLong(value: Long) { - localStart = value + localFailureInterval = value } } "failure_duration" -> object : PrimitiveConverter() { override fun addLong(value: Long) { - localDuration = value + localFailureDuration = value } } "failure_intensity" -> object : PrimitiveConverter() { override fun addDouble(value: Double) { - localIntensity = value + localFailureIntensity = value } } else -> error("Unknown column $type") @@ -73,9 +73,9 @@ internal class FailureRecordMaterializer(schema: MessageType) : RecordMaterializ } override fun start() { - localStart = 0L - localDuration = 0L - localIntensity = 0.0 + localFailureInterval = 0L + localFailureDuration = 0L + localFailureIntensity = 0.0 } override fun end() {} @@ -85,9 +85,9 @@ internal class FailureRecordMaterializer(schema: MessageType) : RecordMaterializ override fun getCurrentRecord(): FailureFragment = FailureFragment( - localStart, - localDuration, - localIntensity, + localFailureInterval, + localFailureDuration, + localFailureIntensity, ) override fun getRootConverter(): GroupConverter = root diff --git a/site/docs/documentation/Input/FailureModels.md b/site/docs/documentation/Input/FailureModels.md new file mode 100644 index 000000000..d62767f65 --- /dev/null +++ b/site/docs/documentation/Input/FailureModels.md @@ -0,0 +1,202 @@ +OpenDC provides three types of failure models: [Trace-based](#trace-based-failure-models), [Sample-based](#sample-based-failure-models), +and [Prefab](#prefab-failure-models). + +All failure models have a similar structure containing three simple steps. + +1. The _interval_ time determines the time between two failures. +2. The _duration_ time determines how long a single failure takes. +3. The _intensity_ determines how many hosts are effected by a failure. + +# Trace based failure models +Trace-based failure models are defined by a parquet file. This file defines the interval, duration, and intensity of +several failures. The failures defined in the file are looped. A valid failure model file follows the format defined below: + +| Metric | Datatype | Unit | Summary | +|-------------------|------------|---------------|--------------------------------------------| +| failure_interval | int64 | milli seconds | The duration since the last failure | +| failure_duration | int64 | milli seconds | The duration of the failure | +| failure_intensity | float64 | ratio | The ratio of hosts effected by the failure | + +## Schema +A trace-based failure model is specified by setting "type" to "trace-based". +After, the user can define the path to the failure trace using "pathToFile": +```json +{ + "type": "trace-based", + "pathToFile": "path/to/your/failure_trace.parquet" +} +``` + +The "repeat" value can be set to false if the user does not want the failures to loop: +```json +{ + "type": "trace-based", + "pathToFile": "path/to/your/failure_trace.parquet", + "repeat": "false" +} +``` + +# Sample based failure models +Sample based failure models sample from three distributions to get the _interval_, _duration_, and _intensity_ of +each failure. Sample-based failure models are effected by randomness and will thus create different results based +on the provided seed. + +## Distributions +OpenDC supports eight different distributions based on java's [RealDistributions](https://commons.apache.org/proper/commons-math/javadocs/api-3.6.1/org/apache/commons/math3/distribution/RealDistribution.html). +Because the different distributions require different variables, they have to be specified with a specific "type". + +#### [ConstantRealDistribution](https://commons.apache.org/proper/commons-math/javadocs/api-3.6.1/org/apache/commons/math3/distribution/ConstantRealDistribution.html) +A distribution that always returns the same value. + +```json +{ + "type": "constant", + "value": 10.0 +} +``` + +#### [ExponentialDistribution](https://commons.apache.org/proper/commons-math/javadocs/api-3.6.1/org/apache/commons/math3/distribution/ExponentialDistribution.html) +```json +{ + "type": "exponential", + "mean": 1.5 +} +``` + +#### [GammaDistribution](https://commons.apache.org/proper/commons-math/javadocs/api-3.6.1/org/apache/commons/math3/distribution/GammaDistribution.html) +```json +{ + "type": "gamma", + "shape": 1.0, + "scale": 0.5 +} +``` + +#### [LogNormalDistribution](https://commons.apache.org/proper/commons-math/javadocs/api-3.6.1/org/apache/commons/math3/distribution/LogNormalDistribution.html) +```json +{ + "type": "log-normal", + "scale": 1.0, + "shape": 0.5 +} +``` + +#### [NormalDistribution](https://commons.apache.org/proper/commons-math/javadocs/api-3.6.1/org/apache/commons/math3/distribution/NormalDistribution.html) +```json +{ + "type": "normal", + "mean": 1.0, + "std": 0.5 +} +``` + +#### [ParetoDistribution](https://commons.apache.org/proper/commons-math/javadocs/api-3.6.1/org/apache/commons/math3/distribution/ParetoDistribution.html) +```json +{ + "type": "constant", + "scale": 1.0, + "shape": 0.6 +} +``` + +#### [UniformRealDistribution](https://commons.apache.org/proper/commons-math/javadocs/api-3.6.1/org/apache/commons/math3/distribution/UniformRealDistribution.html) +```json +{ + "type": "constant", + "lower": 5.0, + "upper": 10.0 +} +``` + +#### [WeibullDistribution](https://commons.apache.org/proper/commons-math/javadocs/api-3.6.1/org/apache/commons/math3/distribution/WeibullDistribution.html) +```json +{ + "type": "constant", + "alpha": 0.5, + "beta": 1.2 +} +``` + +## Schema +A sample-based failure model is defined using three distributions for _intensity_, _duration_, and _intensity_. +Distributions can be mixed however the user wants. Note, values for _intensity_ and _duration_ are clamped to be positive. +The _intensity_ is clamped to the range [0.0, 1.0). +To specify a sample-based failure model, the type needs to be set to "custom". + +Example: +```json +{ + "type": "custom", + "iatSampler": { + "type": "exponential", + "mean": 1.5 + }, + "durationSampler": { + "type": "constant", + "alpha": 0.5, + "beta": 1.2 + }, + "nohSampler": { + "type": "constant", + "value": 0.5 + } +} +``` + +# Prefab failure models +The final type of failure models is the prefab models. These are models that are predefined in OpenDC and are based on +research. Currently, OpenDC has 9 prefab models based on [The Failure Trace Archive: Enabling the comparison of failure measurements and models of distributed systems](https://www-sciencedirect-com.vu-nl.idm.oclc.org/science/article/pii/S0743731513000634) +The figure below shows the values used to define the failure models. +![img.png](img.png) + +Each failure model is defined four times, on for each of the four distribution. +The final list of available prefabs is thus: + + G5k06Exp + G5k06Wbl + G5k06LogN + G5k06Gam + Lanl05Exp + Lanl05Wbl + Lanl05LogN + Lanl05Gam + Ldns04Exp + Ldns04Wbl + Ldns04LogN + Ldns04Gam + Microsoft99Exp + Microsoft99Wbl + Microsoft99LogN + Microsoft99Gam + Nd07cpuExp + Nd07cpuWbl + Nd07cpuLogN + Nd07cpuGam + Overnet03Exp + Overnet03Wbl + Overnet03LogN + Overnet03Gam + Pl05Exp + Pl05Wbl + Pl05LogN + Pl05Gam + Skype06Exp + Skype06Wbl + Skype06LogN + Skype06Gam + Websites02Exp + Websites02Wbl + Websites02LogN + Websites02Gam + +## Schema +To specify a prefab model, the "type" needs to be set to "prefab". +After, the prefab can be defined with "prefabName": + +```json +{ + "type": "prefab", + "prefabName": "G5k06Exp" +} +``` + diff --git a/site/docs/documentation/Input/Scenario.md b/site/docs/documentation/Input/Scenario.md index e5c4c58ca..ff7b9ffbc 100644 --- a/site/docs/documentation/Input/Scenario.md +++ b/site/docs/documentation/Input/Scenario.md @@ -13,7 +13,7 @@ In the following section, we describe the different components of the schema. | name | string | no | "" | Name of the scenario, used for identification and referencing. | | topologies | List[[Topology](#topology)] | yes | N/A | List of topologies used in the scenario. | | workloads | List[[Workload](#workload)] | yes | N/A | List of workloads to be executed within the scenario. | -| allocationPolicies | List[[AllocationPolicy](#allocationpolicy)] | yes | N/A | Allocation policies used for resource management in the scenario. | +| allocationPolicies | List[[AllocationPolicy](#allocation-policy)] | yes | N/A | Allocation policies used for resource management in the scenario. | | failureModels | List[[FailureModel](#failuremodel)] | no | empty | List of failure models to simulate various types of failures. | | exportModels | List[[ExportModel](#exportmodel)] | no | empty | Specifications for exporting data from the simulation. | | carbonTracePaths | List[string] | no | null | Paths to carbon footprint trace files. | @@ -34,7 +34,7 @@ In the following section, we describe the different components of the schema. | pathToFile | string | yes | N/A | Path to the file containing the workload trace. | | type | string | yes | N/A | Type of the workload (e.g., "ComputeWorkload"). | -### AllocationPolicy +### Allocation Policy | Variable | Type | Required? | Default | Description | |-------------|--------|-----------|---------|---------------------------------------------------------------------| diff --git a/site/docs/documentation/Input/img.png b/site/docs/documentation/Input/img.png new file mode 100644 index 0000000000000000000000000000000000000000..5ad3a85b148559153bd63a63c23b3a1f841aebc8 GIT binary patch literal 110177 zcmb?@bzGC}+cye=D4l|m5+Wt-Xrx3+6qQCAL^=j;fP{3TATTzhCLt&w?GQm2IYL2l z#OSV#=Dp_q{O;%TJn#GG>p#kQU7kCS{vO}s95D|Kw5hMKUm+nOq1L&t@rZTH-##8XNNSCca`f_*#lYEBQh@4$V46PmHWzza~&Z?m3 zw@sgpFM@b8sj3Bb43r-D$p85#KDuI!>whi*KdCx#dtdzDt6G;d4F31p_m8j9l>dA2 z8Z#Z}!v9{(=N0rmBZQpG1qB7QJil{|iah-P`s>d$zDnh|%2Y>6-1`+=TjijIcB$fQ zrD6$jwv2dw$3z>S&vhDohBG|lBM|=hH(FkcYbHRNwL?xiLSV-?{(HxE=jP+aX5H0& zdEmW+05Z&*;6-q7RPf0h*P?RNzn_b6P&w>0T#8aTv3WY!69?PQh9%C~t6U_x6(NWZ z+0zbrd%7EUeu_P(w%Z;FULRSK3!am!Wmq0S(C6^7}dNu?pvl|MxXxNTR9(>02(4^JCcaty`WE%2K;hoUUScRLQ6Tzi4%fb4uU1Tu$lO_K%1Uhs z4wo43FZBFb@LH_&pw|Cg85xeRVnAT|*;%7K26@v?>=>**4D6B%A_|n<;kCVT)ic}O z6&1FuU?1gi(ZD$q(wBr}^c>!UW*8)9oUZ2H&#+`>pCaY>IOjn^(oaU*rq6*n zR>CXfd&%kWXn5k)vq}CO?uGj|hgdiA5s^@&LP2-=73$4UnqS*;B(IfKab&7eL{B-I zPG4%$mi>@0?rQd1m~h&(YtJ8O_=N9N^5eNBt@qQ%nv3khbrhN(334PP_eqI*@){|g zzmf6vUDxZr8a3CPFDS=kg=YraHgW5 zqJ~x`yAMhB^9p>YQ!YA1c|Hlu6XCYT$ABx@hoUK8FI2OD4!d&&f-+sBVX@1^<1a#9 zLmtMz2qFC^2&G8@kIQxC6e*TK6;*#D$8oU6@FN#|w<@(mXpincZHMn2Z=^e7_~Xan zu4U7lZ}SI%GpUjRQ7Wzd23;SC9=#N_Hp3ed%!Z);xgEjJ@<#eup13eVT%bX*e6g_RO3zIWeQip^B$=9 zd5_&LC#bnoyfv#dJTRXm@h%C2tmVsz7iGrVn;Gg0J+>eoAlV!Cp|k{;xtT87yk0 zPbHlLgWE!Qz*0eexMscVJ_jXlOkr#Lm5kcJpr{;=hKa$MSqB>Y@~g3d7mmtF+5NUY zv8kDYE$bTDKO?Ao4DfhWf;}R$x(3l@8sw%J3N4BCr9`r_K(Z481Nw?{++wI=)EA6OV z86WG54GJA?+iRm9aUnkRPaJ3&Ez_3}`0HkUf&*XS7lS=jhT3C;#AO#s@i9~ClJu1X zZtCNNMNv_tK;{rz@1jqZayQ%0Hu%I#FOWz|vpmI^2m@ zZcD5Pf)16U^OQYfQRntDmeWOHabC(aTfA<^wX9I>#HW54wy1 zl%eytukO1bZ|sNP!UK{n%npM0PIgnN!ufAo?No4blr7366i%-jogMC;q{-Ch4N=IA z_0CG-7=Q5n?tE~(!IA!m!pzmkBn+wZ% zrK)a;VWtz!8(tJ3P#6b9NB6UWegza3&vQk3|Sjf*4faQj?9-V z*RRVR69br-x_#uCKz6_OR+;vK%m^(^Z`CTrdx|%ZU&VodRz!oDa0)j#W)uEVk1yH3 z>R|@M-&WeI?&TG2mNDqIZ58llFyk&`2q%eO6w?l_r@VeKX0Wh(^^^5+Xml4-|7aqk z;J_k1>zU(96_q9>4^rOL#Tsrr$h8@m)tW!YD*^e<#Q*$u$g_x^Lwn3m#IgiWEGyS_ z{?tf6kvdjy^xGUvPtdo7o(u~X4i;l2yH3dTZreK!smaR}G$o1VVWwOz;IPzn-JVnJ~PR7T=^kFNKwhyhD^bxym^ z1qFAWl#faV)$tj8x9JR9=OOWwt2n1LcErRa>_Afd@v%tVQ%Opp2i(yU)Ydctx7BU# z-Dl{G4cjS5_h8_l^BMoGpIj{eMXU4 z+0m`>rh~c1%&|Eczk7g+%b-iGr$FN=hCf%+ax5Of8!%48usAG@5Pbw`qA?jUEIG} z1!97;!gjc^ov)#bfnt{qX|%%wN}E*BV3K}@e?bQ8fm62MfvIG&zA^)!!kz=ixg0co zR)HsbePfjLhYx#|+o7_+MIb55GE=f^-{#?qu`j%sxl@pO7IbFSq$7Y30hu{u5RA?n z?fV2F<R8J87eN1-vkOEamo|%_g}#F5aSaP8>6pHcrfA`3d$xh+_j#ym+tU8hPQNP zROtkTHgmk_Z<4-XFgw&cI`m3HuzM=`^Z{pV8eMTjvVAHu{eIU?#9oy`+O67K%a2e0 zk?j5byW+3#1G%r?(mvsSjGsz9o4!}ND0yFMflQ4KZHiaaPYo{o(q6oXg>@abZjE|C z-bqO-XTLIJ6$i7DkZ@BFtE|&6UZ+!(Vf3J~$*JW1{Wm)0h;*bCu@INGD zBSa-{lH}g}4-zJsFG#;yV$&WL>s=<-sd4+8p8mAEGp|qLv_VE|R@w4fmTbm)+TcW- zx8zTqXybeLy*s<&BIk?K_K^5H^di{&BXB3o5N{zKhGXau80{45`GAQ@r!Ps;8-?f| zaS56!svurYtRZ)#9~?2au_Hf(_}t#aFm5*GnKu10e%byu{aCke*S^{rizQO{jnzV@Nn!=62W%f*X&SU@G?u?Ry_N*=tM{J{JsAGXD&Uwia&0XS>AkL z8Lme+pGq{)3qE1duE&DQw0(IHyw?6^DhRPwpBm8da}^BjB`}?@Y*{f0VE2=SAR#*E z+wb;vm0nb0^s^#$Lb8Df+Ws3sbQ75BFFhmMUov+^E$u_@kW9}3NdNVMy?`MLzV`|x zdL}6G#~HiO>WJ4EKFD@#uVpMgq?{`TMH^sEG|_xQshBgFG*H&INKoPx`>rYTK`vNpz?Gz# z?lR_jFtdKznbwzfmrbJgrL`i*T^MPk+pu$vJ%q1>feR#1a^eAD_K3L5lgbncXmhjp zOm>j?+r8CadqGyxE8)dL;l|eQ?T=j6yMY{em?*8k%hC6G`)Oc^H#dO%DE>u$w`1g& zMOrA+BtWSg5L9#9j&V+^MePt+@^kaRTs3$8ox{9eB3qqi)k!fH47~5?QZ*{`auxkj zgE#3iw42&eDzW<-3((5NuGEuBXiWupvj5lnp_XpWQL4;JLi*lgZ-&=8HFHk1bkjo) zjfKBHh^C&6eQ7tECJHu(MSr8*(D6j>wm)d%hg-b@7#LgP$rQxg7{U+t2{)S}vErxj z*x;Nzlws;v&>c;OGNd)hhzpEl9(~f{ySr6DUzo0lZI^sN?>YL z=K6xXP2u-abw=!U@DsOkMlEx=tN83qb0Y5ZM;PaneY|D353;@JePE?=@@LfG;HB|t z`L)blehhz>o8|qxrTD|OX22JB~|GU%?kyo zjPaey*d;t(&;EVZTfA3$9LiE!_55KdW4|BaEci+01rh`?y1aGjlqr6K%27Bi^dGHsX)!gJ_aDL^Six~IGMyNK;AzZ4>H{CjD>|Ar%cQ_ zedyOFA;A*alO@CNSX;>c8$GiKSDmiu0WX`UaROJ4{7R0LT~_v6jNcKX@%O1TGZ2l8 zST3A`#3W7uW;6R~5K9b4{(r-f|6p60rHpjbQKc+$^TUrzgLUB^CQ)6rAkuVnTW7Kh z;!@1dY}}3%RCcXBh>K%F+}EVnW#?7)p3=C&7FUdE+OlbpgRT3{D@HY~{5-|Zf{+Gg zJ`En_dh=kIniT$d8{z)sW^A&8_4J=jC-@EMl?~r9glU&W*k=E2_~gs6<-%Aqo@aR6 zj>bvC#qFHB0e2P^^)vvx_>4k4k*&Nbo>Edyvg-z{4O2=x8*&X+2Gy2*f9mzjZjd?n zKiynKHwoo@Fevjf2HLklY``D}<-z=El3DN+n*5_X_%Z;1Ko_o2C|eH4px#Izdkbyv zV4DX97OFe}= zn}0LL0ACsk`aGJZleHDOq($@GO^r}L9HDlw03rBf-1D3!Moz^;1ax04raLu%c_Q!d z7hmi9)FAtq$GkCTl1|koJuVG*o6<1f_19P#){}T|SWzC7^@3B|FPV!&;=^_h3~bv= zT?Q7Q`Pw_O2Vi-RLuC#DQ}bsQPZPVaKU0*jb1<`_0NYQ_Ri;DUvUWPTTi1nVBOW$U zJW%KB=4}3YgVV`m_SEY~#9olt3`y9U(pF#fgSWabMP{(pmpbbVHv~X;&|Si}9)M(Q z6Y{?|7oLcsI@)11F`;jwoeu?+>1;VlRv1XWBZ26DokG`^H{a+XVeY;>UFX{f*$Ys# zx`z(_mdnN(WrDEtYj}pg03=D{y`YF+OX0=0!tJe1N)Sc4f6H6Jr*3G}5V%R_;D1I~ ze`Hh@k82j(uk%vZGE@!!!MVn*Cmgn(%Cfr*amkF&th%&rJ*2d|HE$Uo7WAxgtI*A{ zSR>+{&zC}kU%GYWgF*hx(mLb#D zqk)UdcIr~jwx5=#9TB_XVhY(TLj-SQal4pWhYA-Tk*sw%67W&>`6GyvFN9RDa|#ja(;3-Agf&kvRZotroq;{(kTq3KNwHh`>d?_VfBh!_JPoBk|L~w;~g_n6G-&Xm6hp&`W zl`bXPbXzIC<`Lz`jb9BEYHLyx;v0KQtDuntnMBERu^)me+nU(Mp?5#OkIv|tzY{+P zIQ|wwtQ|AciUT2Uba<02*uM6x}Zn4U>UMAc9L6Fv5CT_Sn@GJqnE%1#Hl7OM$7SEUB3 z9l-9<;}igHQT*R5N22v}P7jZZ6S&UWy$ihO{12}`M${BZHTi#F7s=~#MKOg+(YW+d z(RD-Ch6X`14E2#Di|%HW>9o_LdvDDaD#T z@M^9S@Yy6JR1`pIA-VS7Se0ZCr-m%d)H%@4@(_>s;8A|FXfrHoz8!|0oIgXD-cui~j%P$yW`7 z?#iW#^P`H^0#fpDKRcD9futqF^F2eKkfe2_%Bc%5mN-9dHvE5$C5YpS|2>vCn-xFf zC4l~&h#ZT|9k$Yj9JH1d2mwrHyw3k~SOP3wA1j!`ogmH+5nIzChZ!NA!DJ*4qu0E@ zo}XaOVW*d0PaSM0h3GZ!2sfvmN&%Jb{A}}_b9nn4m`S9ZUItn_)H->RIDCYJWKYAt zehd)lyepVY)%9#wm7BILL>q&t}Gd&d^=^X}fWko{+)Dx@S#Tjyuk^Zby5s1WoS z8%ckvS@-XqK2AK2EZ3adJ4^zgZeP0t4xtJr7K#vHi3bQkB6HY4G|J?EsNMFzOBaYs z#1`%pcqx+j0${EsV$ur~WSsKc^xt z5t+ySzwW>l4SRl{7zMzI#DMKpdL7= z-gz+uu#Q#WagHhvHNW>YgIvJP)$|}PxzLc9oZgMy~Vtd;TYhRxd{2Is`$pY#=vg)M@OpgHiEM+u@UW_h<<$_7s(*ep>tlb`BFChIwyF;qSP$ymZU^XaoLKH?FAYyN8tHopajL33LcvHvDPE z+LLxI)!MmO$FdJ*h>iMoD~@2_oaHclxdv^WAfP`FipoXC&?NpISnAh_{=>7A#LdAe ztA400E=m7AsLHv1Y=D0>0Bcz7h|O`ZZGnyY+Aq?UL(Ev-I#qfx3Z}M#!PT(LdLJ)y zxIKiK0d?fL{CTq9(XHxurz!Fa-?HwWs;A1}Y#>$;U#|rzcA{b)s7D7XMD8E9P%~;i>(BQF2n(b$#H7Z)}fnjH3ow;jf$W*>2surp(fD z+k1!a=yR4J)bpMXM7^;!#Q=Ge!$148C=L`U@dZ7VgZ>8bbvl)^^2l1car<0uzPdsG zbn*NNRka}NZ)FGv#HJ9Z`M{gw;IorcMl)+%uwlHrgt5jV@8Z>@zM6oCtKH8zhRwen zm&&vx;4gjY>$QLv99f4h_RA+~PA%x{FRyk?gPqVOHxTIMA;0DV9bW4@3(z~dINb}~ zNLqX3k+~t)`Nx*1BMGEyBZ1Nl9{m9j6%B#^N-D0QT5isXgYlp(V~aOE8ir?JUxw#$ zVqxORacz+n1tF2~;z1e)IJW&_E<@-1BT`)(- z$>#kyC&4D~JKC=e040U+3N~`}JupFpe9kT_tYVvfsa0Ffs3mDXYRs+Z+YBNHz#td6Q}Vc4L~3_k3&5U&1(n0_rBA5N*) z_5ro}p@#~uAP<;+unj9+-U_<;spFU)loHoOU5}=tVzxGmhM}d{`}yGgTKkk5eOePP ztbOpz!FFa)O%u9vTfFpR{*@HQrxql8^j7GPDMMum5v@|LVD34L$#PUX@B>+yrQx zim5L9L}X+qG%ONzbXQ3MB=D(A%Y=MR4LL6bl1}#(YrJ%CFXjX95ff;+fqr;#;niDa zUzrTQ;7nwOK3Z8D!=HcJp(&mWX}d3c@FCPX;V zMGn#?S76iQGPx0SE^8QJz$>@|CzmLUw>hbmOJS_njf^}i^5f`Zb5fehK*w0{xu7+V zoZcU>TM_dgB62%Iw&WSNx9luS4D1cWd6*Go-E~IAD|oQMD_D9_{NxybOr&7-s`huy zyuh5phHarb4X#wcd8*O(r*;PgGL1_+eE-p?FWBS3xH%pz2OTc%lnYjsGw_kuxPbBWu#X8Xw}A?IeRc4(gW(*-xy_NS&NURmwa z+7hZFiJG`Qlk2J1$x0R2cD#bW@$XTF)9Tw95d1eAYXf;vN$8j+qzvhS2jE@a8GQPu z`UH8HRQoHab4pdhri;E>_KqZu_f$$1N-auEHikn$qW=wG!Ruc6>%X{HryW!R9zDaw z539*KD%@Vq2&0{8jDf)+0wcO7Jf1E}Y&MmMKjv-DTkOs9oW2%{PGdb$rBc33pqtpP zV}T0D)ty7S=I26{qvcB*kr^<-SbbwUxL3@q!JeT z9jh{Buk=5{*;6*%`mmtu!PIL;b1Gp4=W*3!Gz`1?sb_W`epj7_Nt*DR5PTsm;TGI) z2K12eOe^ln;`MjA>i-A{C%r1%fm}@#mrFZepU_)XtzRFu<0EIu>X=>}0E;B5;hNSR zK?o{rzl*Ub_D8`^oTbFHv=m z5-WKA5tWlDDmkx{pfnvweZzOk&83m~PfJkA@`cG9j?rDA%t*C`mulI&z@-mrw2xu6 zxp-%m{*So>*N#Zg2V%SFXt?M~jf7Y%H(`_AdYxZX?tb>kOk*>}N75MJ$4ohWmDZiV zy&v+mMWGr^+?I`zR9tPf44~F;DkVGju4d@q5xG zvKZ8TlaSrY%3VFDCTpMEDoFmgsGAl3*Y@{&e*^-3%@*cv*DN3L!V67hKKBcdtE9l~ zj6zbXo>$1Aq#w1uIP|hP-Lm_g;|5na7_lqI>~AzG&EaM$<-ZHKc42x$FcAy7@T>N- z=;px~4mqa|wx;sw!}b>)D>12DSb+tt!Hu9rS;IDGz=leb{Z;39VQ&WS4B0c;R3Lb3 zUY!6Th}KjUQtBA-1e~4<$7b6;tVN6-#;jPF1>MBZ3$gc^tf;Y*-0r8+<4#8#pO3`W zS&h_PdR~qxh)r&Q8|t0L#)sO#)LCQNx2?rSk*h65Y!!TezyepF>R6cAt#i7(`*=;T zJKr_7?Z175B~mOCo*{RzIyfloU~KAlCn`i?2)|ql$`)~k zC=UI;np7}8uMw57Z>0_oLV@n4&>qPRZ(F4Cn*@h?ScRyJ{;2F&9E~Eq5;%BV_{%~! zr{luvWpV#YF)yfu8+{a0%+p2NKerFQdgo+T@!?5C4X+?VUtpzfdp@%=O0ua{aqU;m zgi>!6_K-nzoW9?|5;ei7efQz>O=s7iQUyDXJ;S(wQ7KI{4O8{@@9!}gD6?eeF!rRO z$rozF6TGhVMd~#}e*|0RzcoEeD^!t={=kdMKzAOC4zIGO&BMiN6(&7c<(pDpr7dLp zv~VuMK8i~Q;WY!izf4&FK=u9}y|Q|YCUY}<@jUd0@{sHX64z7k(`ee1ne?ig?VW-+ zS#^n)IfIRPJ>*;{{y3^S3x4>e-LSoq?c7 zuWWDL^yXzPW_-+3vbvRc%|}`CJvv75W{6(EMARZ1Eaj__rYdeNqKJ1f4f4ym5T$_q z!q?7l8J40MGn_;7bV8NgezC~5$@*O)fX1${`wX-F>6seIE>iee+v7-(@<09cyXkSs zMV(FB6qAa~cSBu%Z7SIwH@%hns;>m8_SZAgZ#m>%;!s?G)Z-U8`Tg3V*L&^zra8(c z0`=l#I2Ydf-REcsq}?gkuB@Ug>}1dqhseUO38YHUsj#RqAiLbYzT}fJFRebmJYGci(wTBN9Sp)>Z`bLN9V&t)|eGX`=T$_%aAt zMl&oLn$C3UnozEkq5QO*t&YwtvCdo_|AIrAH;hFW8`c@a8?B*;=*J~uLE;HJC(suI z-u3p5RT?~A+=2?9zi%g2i|EEzUNDva+%HD1;3~yLKVi!2h?5KNnD0kUg{kRog-jy35?)Zi z-BbtCf>g2-lT^8E$>nHR4p64{Fx5SMk1l7B$4o^wS=LBnpqzbwx_#-WhFf}%98jd9 zR*>W1cx(+6uCm+Uy`fl~$z^q~4VS)|i$r-uJIO$A1CAHSzi=!PgR1vc=5u1!6*!K6 z?f0h=)!W>B_f$hE%PSm&VaQ_n%``9{7V>qmQEWUB`Dcza;64u7~dxq9fA9?H-g?Wkj!rsU1#u4X6em-5p>sa%yv-i8}q_KZ+0Hl5155HrD3|Rf@Zles9 zCEpSQnmjNvnf||mIi1&un+{~E{{E#NS4OWZo2gw5{~38rh?yhGt`fHpWS02R7VQS> zCNS#s@^zISoCytmFd&0W0~oH%sIR(tv=TpZX9Z)T*=O4go{gm!G_YO>%Y!2PUcRTz z=I^GAupsM1eVz#h&rYQqP#wAw;&6+8jvyr`t6lOcU-1p|rk4>PZJ51~L6t@eu1j%8 zbsTwL(L(R(S=bRA=kg06Vc}7uK-do@_ev~eDEU6uQY-DjkXe6ZVb8%WnA- z0zFK|8b4Yu>qNhvH$qi`?v&_YbzyS{?Z{NT|FdE;UmBtO4u#Sml$iPz1)d z4Cr0`@>YKrb|0#a|5o&huVZ|68T$uh3R3ReXa23E`TX*5I)q`eeC5$ zcd3J3_*==Aiv(H+@yBT?Z|m;7_}t6vaNJ|7C||%ZbD|MZK46Z+5v0wNymZT`F)MeJ zGjCd+ANsN!w3bRSYJ{sRE|ipxRG#+9$$WS1HHUj9mOU8^ z>UReh#69z4ZCFT&#;&lfxcs$ZI6AOb#Krm-4Ov*NVQTor`ZF@a_z40*-2@?2Fa5w- zlk$G&_lrsA-SXxewfbv|<6;g(^q)LrnLTg=p_JdRO%;b9)qiBwG*|39T$}vYfI~vp zpa*7fc{V8e@MY^n=uLJg9reznk5o-2_HtiFg*KUop?$aY&4u=njUWsFtF8gYQnGFM zYd9OC;f(6%Pu-wE@G0&e{L$=#a^16#yUD6Eq|Y%0-%L-U;s-jTfV7I6%1W5&f6d43 z%`WMxvP=EtnSF1ObdI>Q&x0PC0N#>ogUr#>`x|+#6v}}F1*J!<^uDhScjTwSIg@Gw zk@08{aM3}rmqO}kiq!b`gON!eyoHfq`L#3IEn`n=cPW&jG5)G)L0XX4MaMsV$A8x9 zSt>bRw*`|ICd?>+dN(#7>d6}HO64_Zfm0se?LJstMYmm#l8FTZR;kc?L_O5|M#a;S?}VdTOPC1PS7*0a3#giGIq4xIr8 z(xvSQuahH-wezJasE@D;v@l(k603YaNG!krinr?oJ$g$cawklIgJrZO{?7sDY^`Tt z%a71EEE0d*vIml*b|>r-X)80Vm!CxaGDtm6(45rlYr?2}>Jc=_S+^4NO{sCU?G$6K zSKsy;t%>)+3g3V_)BDW5AfP;e?v0AqX}C6p;Mwqk7*8LFX0ZlRq6J_Ti~WN-VoI#o z{1siiv9gs8ZH3z3N|D;c7z#tC&{9(!zoG@JjV9Y4($j$yK9HtVEs)m=amqE^09RW@ z%Q+c5;cj{JbkvnK2F9yV^}ax)xnLc+W2s@#mm{O0Whz)shg$YR4a`)j)i`v^{ItKG zUomU1(oBBxtR#iDk4??0p>?mE;(=4@C4Ib9K)+XtNY&^xXRSIUMXE_VEc3-o>d z<%a|4;&6boa*mx zBlnCH+VFD0Uhj}Bt8K^Xt79q+m%3)W6|U!{JlCkPVw$*uLTa{6Gu$qCI#HaE8iBcKq6^l^~|J-*fpaMA|4RBT>lUnO}ec%I=@?G)KJ?&o}3qANv|T zN+}-=Q=WvKHV)o#viC^xHCZH8eiC|soOrb}W7bufm}6NbUk!2-mur~p>PagTL8$pp(BTPmcG{cvKitCHG${gjHKoBl~^4*z4X1e$(pTzu+& z8CL}1unj@&>ib62pW9$UG<%H_(TlrsdP_Ga^x^FD{Wsr*)J|@rAc7hb|?4D zBDTKjDme;ytnX^Pk_p92t;j>Gl7-0`nNI(LV2Nb_5XL{o?Jdt7reLldU%1r$PSJ@M znWW!TNXN$zfpkdQn6B*O(^jmxdZOwGKn4dweGs%VJ)wCEqHmWx>2#~sQB7>zIO!Q`w)7vAtWjzkQH#3!iXaN+v( z4R4RbY$Eo*n0q#dU*t++UkMX;_$c*|d+f$!4_QKws@h}gc`4)k0Mq`M5T322SBc~F z;YFp4lk>s;9f}IiMLw7bA}Mw>@Q1KKSVn|7Fw6KDEfEWALP;%RzBy^=$}&${&wRi4 z${}hxxWJ*(P-f0Ux(#uJD;xYGVX5#lSRC~Edte)+naM=#>@GE+c(-gL1mSgp=IA@f zPE+k|I>M!{B{`O_tf26?=>*sk$T)Fbv2&B_=ch7ptRy%2u=Mgkk4_5ncc zFd+5(RXdS55%WHiO$5|F^$PlusCGeWKqJsMAXG72YU&@9edjH@^TykX z53mEc4acL3oM%tDp+$Q5F>9y}IjC0b2o*5QL%01+qwn=Zjy|MCyD3HTK?W?Kf3#0f z`vY59TABxP3h;K#1oId5IcsOPG_%o2=g)}HwX)n~$SbW61LxkDd^!*#LC$rQ?Y)Z$ zhgjAJf%*wXWS?rN@cW<~QN%A*wVT%SBO!2wM!imehpb|Ri5i}_aokB`iCVLwp_TvH z4cRuiMN>!C+VPNnjuDj4*0&j%agp=&2+UsWG zMHLNG?@g+et-4nS*h#N8aW=0n*4OaWSe$sVw?8MiX$4v7s&6~&RB}1mG%I8s_7bL& z%lNGKuE&kV!J4pqA2;*)LvS0PK3eZSsPwso=CJCZzH`ujXJOBSpp5*>V(mWO*K#+d1qcQI>l=4#|5jrfB*&6{<4oh1*C)B0RU3sO@k|TyBUH7Mu2i2yM71NZF)+}(`8D^WXh;T~@u~J4zVZ-k)lpI2S zgQ=J$WK(v%%p!x5z~6+}bF-r}q!_9Nf1DOYH%`RV3`g|T>l3d)M5)_%Fy>4)ZYh9X zP>XoRQsnGzcXS><@uTu)s%5(}d&laid?}DGP&DM)#`W~m>ev${;d;_f1J<~LTOaxs zE?75FH{AcKmWLNUetIuHYl27F{dEgP=s6DaR@0^t$vwOSeG1g%CSIotSmn)`Tkai? zaNWkf^Mtdar%bACUy31Qs9gbbIrPGkTsey z{}rKTw7A>py70VVC`M?fMgUhUj4j+r32y?7Cr&rrJ zII0ixbDDe{dPiMeo$W;NKx4bZg)aQYJGaQZyYU%$5KOE%wPyL zQzcpGn4kDW+{a0qyh&3M>r}61b@WefQ|NqPR(?_EuFs&}nOUH$mimG4lB@g#{CqDl zER%=47c8e%IY0DHCVJteddAbxqo8Zy=**2fS^_Vg?Y$w`JPsqzJeA< zFM7-#b2f#|sdc=0qHHerCB<)&>?w0TJt$C{_p8V2wVm-yGOOgK3?8M2-w9MYzcXgP zWdLYv<@f~z?MjdQxhcgNG_aqn6mN_d3-C^7;ahi(pSfms`ovJy3N{BL+mBkmeJP zJ~bsPJrAHF(Sx+!#(5h590vDlM>2yzKwsIcZmj()0l>i%4M4*Tq{q-Z?{5lLT#2Sb zb-GS&IvVf(i64!G9fvoK^TUGPp2E!QSbV1i8HA@I9i5al9nG%GTXI#r-uVW6oAGhh z78{8;7tK-)V-J-L^B6R!pu0D}qcy(~Qp9mw-jcCUPt8aopVx?H14~z;jGD+EYfOWx z&4JvjR>-wwz*l;oy2S}vESPeDQk+lqv8lv3sUnBjS*1UcyPCMi?6wuiF2Ns-Uf~>V zt9E``88X^0k~oShJ%N(gT799q=l!(4=ZuWw4rfZ53aX8SIYb!MW5md(!0eX}nT;mD zqH+9mYu+yvl{J~q%_}h{Y?EhTVrd<;UVM|C>Z*eIK>ed&C^qdYdQ+R5z31|$l%yS+ zKR!umF`}Hmy1yfQZm{MhdR?Km{TQvR*{52^$8N}{>G8TL(1O-p(24oV(~{^ZJsp@- zZBPH)w-Hjgr{^1k`BwnO(b&4}ee&0_L3b`uT0ccjYkzqKT;-XjY`UIkSdtehk#qq;@10l?lGlv0aqa{9X-p%4zp~NwN z5pB$wLKB+pr<^w{{mP0b&C0gz5d#3t(NgIa5&0rDC>Lc(V05brPf;*1p&f|@JOjAP z2HcT(K#6W{d=uK`)*zp{>X5Gz!-X1s@3BOkm=QwI%^57-UIT-QepwT}7@B-qc_+EEsRdb?+TRp9P>i zGwUWcKQj8Pj4Anj!H>GZFDq&EmH0CRVy=H(H%UjGKBFK`#sA}44XmKj6%!mUZ+mi_ z=dYi`gdXi`V4|4pRM3@g1aC27)y7s3c`YRGfpBNNeNm4|znJ0ZwC5$)r%t^EqXsub z6d!`2vd(w~a#vL_P2dME?Bi}x4a6>uMor7~=k~Qq!N%M<8=xtbxA^(a8XaC9#oX)j zj$01y`{jzhaX9p@d@O8~E&FynrY|l?(8`K8C|+mx&xlmj@J@Ti$^C#l?orx`MNl7n(81^7nec-N6VidT zx>?s4&0Z3!%}u4N48zz|bfEI$2#HK{gF+Q8es2bAvp3++j%|W(I$NF0*uG{!W+O;1 zVH#whzi5>&M2b_ltZe;EU7Lp~5su5-Z;GPFIu7? z2M;f2MM_jEthuzFfWL^B5H@{r9gf|%s=#Ut=X)09uJ=lPG%XybA#56MUp^)z4BQ0# z%L1o|;>bnE%5TG_H#ACRx9kqroFJOQ5FG8bc}eZo59Bd0fg2%U zeCxYIj)(eM>3MW2)7jnre4d$13N~i1MYLb5lLI zp`g5i;?&NwtDMgoU>I+&{TI1K%(^e#@o{nR2S_&p=J?;POp=^v^BX?+)wDe?qE&U%4>p{!CuUX(I*?xgGWM zBgB!Qr7Vlpz$$GAG_qJcNdrKLPl^wBSVzKdAHj4Hm199Cwve6WSLhzGcQOMzArIS;Bo3t&tylT zpWVOFQ1IYwxK6dep;)FN&5eGQK?!8tq$`&zV)C!0Zg7w;C;=mnFUc29p3wprzL6=n zfe7<4z&DLSbZE^sU(=}Cq)&WUL9PJ<95hrh|MD_$TjbqoHE=QJ?|0joA7-u9@??Qn z`+5{6s<;?DNUV<8OggftlWstZ$-=GAK~kQ#v;@6_Z#+SbDb;dVS5D|PyX9VJXnRPH zeS&)#AA;<*(UeO2J+}~1G~_ZRz?g-5nsCS2uM=KoXm9)ZSXO=HE{=*mZjKeCCLt*? z!qLz^M&~um5)G4K6WN^c9Z#g5X&#G)aYe;9`(0hxmctt1UQ&6d3@@>p!+2(Oty378 zZZaB9=DFUWrdJ2%qJMu$!R_2CdqgWCJEuRRzTo81+{j4Q{EXvhNj+EVseX|#vWn)T z2py4{C3YZlmZ$E?o0}9{QL$oTHYl zp`FJm*CYacI9VK9pV%)mOdeZRqlzo*d>%-PZ1sQ1ef|aw;(Ky7%=5A1u_`WJRVxI+o z5C|SLNC>lp6rpv!2-FHd^FUfw5=|0}sI2k|o`6fsLBI z%3WL37JN5~9||?6Ll&nmo7lD|e+4b6O(zuvMPr{eL@od=C7)dOW2$+xf_eAgbHP)sS2TFj%5 zfBXDU8R)Ycw$H*VUR!sRS8X=1*!->Vt`_xE-q*^a^_Wj|+xn({UnIn%pTnpi8-sf* zgufpz-C2ka;0^cm&glsM6pj;3|61$b8`pOeEjf;T1R@RHLB~_p$*eRheELbn?R89= zl&>%4-aYQx;-omrPj0v#6Ihg=;+X*`TaVf{=i8UqKK=`GjOk&CO`i__U?yb8^Ef0I7#hD{m$(DEGYU9^zJk=X=C^M{j)Go`8s7(A37jtKwL=0> zSs?#Wun6xT3)>p-l|q@r2_Bmk`su1bShBYDdYZr{1WUFC_}7o_)3~)ay1D3z1_+IY z!OnNSx;Bmt`bJv3eBNhWOVNZSeFF?{*1m7v1PI?cbAYk^p_lbe<9#)(`8Q!%;MfxU z$eaH|{BFWlxsr2k=o zV+A=n$gxy(fcSmqn=saoi33evfh%mKff2T>2TN7|Tf1P6ZNjqCKkQ+fu7I1Qu+EPrMO_-&v=gW_ZrKs zuJMC9W?{~lDd#T-2^e8q9DC;$o9G@aras>G~^H4XHmDBbK0?A_$)y0tGZf0aacJcPu?jB?b5+PU6 zlvs4cYs~_wPP!-v?TO+fOWF%S1%dsLDmunir-}mQ>q+)mSNhbynvkP|tt01d-!Oa9 z^_;ZkLH2lC=EpbbtTT80=j{3RGwl{nPES1Yos3?5>p9Zjs~xNjc6Qa`&gq4jEM#pp z>WuKWZH;W`@6K}Z4Hpl4AwPMXR@$epA(`ij2%ujdC+({JabeSRJOOUit+p9N5v!*+4q)1iD zu5UNA5$b@8nZ5IW=tPz8w+(nu3%z4cO-#DGW2uUo2dV@|{3>(S%v6XvzlGK2N#Q}_ z?`;Ht^3Pjl^5PzpK8>O?e|<~}qHouAY1f6yuZP{;izfYKZEQEtGekkA>7GG$6{V7P zS6Mk|U=#Wx;{1Ex3Q%}hE2c(GRxUkRtCcQy6B7=7xSgF>;o~46pu3Y&(_|q|{CK_( zRt_vwEs}{+^+@#q0_6*js4$u0Af?}m3w^a)JwB{g>6{^W90;OUGonxqff{k6jaeDy zn$cR`{YqwF^9=2PY;Jl>e+_w7(w@Y->Wx_rsN zGD+>WSGq>CuRL(VPVcyzSKJ5Q0j~5=)BJzxST?cud+kQwKch^T1{7Ih*h0>OIm1{< z0N%b;H?_B9e`l0M5$&}@2trZk9YvbcCYx&&!E04WoDTe+es%ce$v_U9Tjr3*if|P_ zY@{~sNcv9wZ4Rc(BSvcHK9hrIp6#J#UC{BV(yK4pe@(g|OB|x<=syt&(0*X!`6dP>UlkGlw{9dr)o)Tji2cY zoz0|iM?n4SPJw%70*f9v$>|?n2phEJ6c)AXrB~qlCuUARbNxwJF<(pS){u0~zktP? z9g8=M&mQ;h)#5wu({^(Kh2F-PU(K@t2WfSuYIvCp;G&~zGr;CCXQ=%RdT>!;+}6k} zZ32R^3wJe!W4o6IscO*83`S9K*_BTU?b(fvg&3$izp+dvN8W3dR;j%*3@`Gt{Mw|Y zymL&2)Z6NM$6Yj(O#>8oJlLr65%z4fDY$*OcB0bHvJrWgx^*W%68*8{{p0tN9&2uY zgm)32zA2|jOm*}J9GH(P!7WG{eRHB(zjdu(nBLpGBDq+Vk2n(!R*371&l{&p`+6?o zQRUJMBIpqweFf#SARUNS(3&Y3ov*bu(lSz-^d=rl>Mg7jvkOK#*<2Krvr!)WQJm9r zo3APmSuV-n@Y7t+j>=3Nv#*VGzuAG2E^Z(a)*CTi!~KElSe6?U!$?uy3$ykuN9j9W z=Ev$_M4#KYW_NB%=Z`~Bb2jU?MFTGig%_)t3nczj34LUYJR+gy4ExOGHL}~n$j)s( zRe`VN#fmov0M80Wy~P?j*95R~Vk^td!)P3~T!ecjU{)UUovM*!0#NT?KLmjHKd-~t zpaZiN#sS#UCag1(9PhvGlt~IN;qm9_S6$<5OK$@*J8S+ltEj;vbx!cVav|Ib_)%{*0RX{mKNr+^E z9s0v){IhoOc7&?LV`5#DL1vUOUA~;owC{O`zj)yJ*Ee5iXq*8-xL4KopBri(SlcPv ztmdUlPI{k3u8kkEqEz|g(SqHcz0p)KSlRLmO6_S4cbeK`;763jP%fL;i1OJAMKOoP z3#`P9)1^jPmGwR$6NI}MN+Wn2gmi>!AJlso50U{r$U zpL!@fZ5W3mwFX{Pmb)bqzG)2CGNR>MmKd^=BtlCkO*25MWJR9!L@(hK>k>x0zt6H{ zvtlTTX5U2VpO}e8Azx)i$Hl-qToYY7d<%xf0qEQf?-NqwLv+{O ziS`h?b6iL@)wSyPA~ED({mDBoh^BhFRz%&^jH7X^+7yr_tep(k zMBYPt(LpgH3h%#bi>y}L;M{eF{?vPzMIiA+KLV-}^$@c>a>df_gmx$5tB5;c#87J^ zo@Z>!*;l)!ktyT(tYju4mxRif9v5Q6i-J5?h^D*$#D{y$nQ;$;nY72p-tv&LI8#~e=&UZ55p*P z>mOd_R6#NThQOM@K_b_tfI=I5P3YqyfCFl3u*#2mVR%9Y=ZnQI&wO z5Y(tiysGEpizPr*sOzLLJ`q+m#wDzpC1H`6mJ3}=-Qi6T_~!lBzfO$T{`IdPJafK) z@RDNx^zyCTOiw!4sV>OiVC%F0(`TqTq@BBg(jPPv70n(i@N;ss2`ugKFZpl~hMm2N z$|e^Bi%d^W!4hBtJ3|``p`o5e5;BXG?x1c+tH$knn`1QF#beM(u|ZLWZ)DL~?DuB- zNyqjH5iaW0bj>L)&b&QQt-(_=Q1j8XK+%?~k$EV0N=b-Q<uGNe=fvcdd!f0|+5 zI#6CNaO9oQ-vsv}y>T$RXlpLvSZd>`az*+TPA@aG?Rp+mho`pb4b%+c(%!-bv7*6r_mzS7tkGxD{roF_yVp9A?KF(Y>rRPpv=}Vz?xwDC|!5E2AaqD%#y5gLU(b9Kaa=>|)^H}y3 zZqQxSfr>Va;S&{3g?@Xg#9c|m)W*feXue-ZTda$qT+E8u)`HZDA3*7sx(?)#_Fv3) z_Ux^oL)>tWr3>77#bqn&c{P(0J%PN6=fVGLjD8$4F4OT$VuqRP=4@a$vQ;CVmfpgd zOzmeW<+w&k{Nd>xD@NM5MMA2lSYvFoIb&)_x2aD4YC6=nNVDN4HtJHJ=PkJX3s5(F zGLU*-()6fiX#^}5_GzQh!xgJQ=%e$}<^7voqJ%Hb-rj)dO6#yrZ4Mq{cLG_}Wk}S& zN}OWrxEXuBOgqdR4n3&4v#uDx%JF0kq2Tc;IYvjB;7GOR&6)k$pVtrnPEaRq?BJr( zn5`!xQIiSl4r!^PObek{6}zSCey<*Tf7Id?`TQn%n*^Zsx_f`?iqg%i^rA9jUTT@W zQce;`e<+b5-dH0f_Xgtxbx7FC_;O+rc=!PKS@H8Fu$hzlvoaRFT-iCj7#H>AQL~}4NKt8K^stL(zCP$_& z%K&j$$t{?lCKZZ38K?#Wh4I^wXlq_C_4X)a>_FLsm@8i>W zHW?d%Rqnl&O)If|8}C1Bq0r-NOM-Io%ji8fRxDnqx~GjW#XG;;^|hK@l~Y&@!_>P-ZsHd=JZa{B5H^| zl`<~8Rqn*mRST$%F4S9~JDsPG|CPjGWL*6AuhlY8W@q233$ot&8K!5b&u z&TR`ELE+Hb6AFr$%Iub$C)5|_t3n$2Xw425=^?RInS{yI@PiA1GnEo10~N-=zKm)cHeq41~ahxtE^Hw%;;Y zS1L-W?4)8+w&JID-XuC0G%@LUQ$H*~co?53z_(#Na`=Mf4bF9qS30u4qHv4-YVf<1 zA^^36l+)g#A15s{*jQkc=A|-!g^GrE#HxWUO=zz>H|8DmRzXrpk=j@sB2Rp-ihg1k6I!0k@GOj=Qkt zQZp+~{P=V=MkVX<(v+FPBxgqwoVa-i72g{1Y(S`A&mXV%= zf9R|Uu3ku``B@e;X=(4cQU0@dpF4oXE6Q|!h0FFPzB__{81lGGt=(`{98_T~X$dOY zmX1%HR@&<{{mKEsyI1(}=iPK^OQOd>iInQFVQ&H==$sdF)D&jCAmErkd2TVZDqJjm z`}e6SyHMNzf%Z#_`x zlzpb}ZZ6t-=ibs+!^;~qL#b9_61;6?fhXh2*>L@!v*f2s67a~)>ei&zM8a$J-%%db zy#!=(Txe7C%ek<+{ukdQpAf^8EJ=_rk3ib=v=Y2vAvyMf7pzJauNmkJEmU zM^nJ=6x0tiw}Xf;^LuFhjwvZSW$dk=nzuJk!o1=N_dSI4`@qTh4N$|VSY!EyMfS1# zDff{c9pa?1v8Io5ir+d;l;Eftez7~w7k1>I>4cWCd-rbDDyHzC;c@^auj$irxMGia zxnZZ$9D;lfzi3Cn6v2>PuS$MXcRCq%b!nEix2?L?j<8oQ3*n-a8fXwX>14kWv61}^ z+PF30Fz9n2)<~GvfQvu$hGtI+4y+Z{yr?~(g?oE8EtSEAU{0E#!@CcfXf!bqmExo9ZD-@Neck0U&gSX=XlYLpkxA+~0Jh;x`A z&=>k+g-&k1JE`RYYLBEB3EIyx@5%!Z2C-MVD$UuB02059{xVZM;C9$+|trAoOVH1$`FRQ4=tsDB|2@@DODC7kdl?swoZDKVe0vjZ+A}CG+Ppc-+Ui7NWK?WiBEE5E7o=-lRq({;;G3qT%`1m|pTxyS+S6O( zJzEDHKq7GtU1eW*NK0}CFVO`tzxa4mmFeD_e!_0@;N8KYbW<%5%NfEaYMmf{dz0~9 zNp`8gW6_h-M_U4m+o*`@iRO?kV0*}R{hF?+bQpcID~3h1RHY~aD(;bx)H@y{&R@qXI-ckxGjz2@k8F50o;-Tt_PsxM zCfZ|u`Q51pqD9;lxhc56LUB}5NyhO60c^0zzmPfTI+Z8%Tx8qa$F6=`?-Vq8RNsT2 zUBi5RGfz@bL+BmDCV2^kYQiyo2YxDuFLM65_J$;hdd_RbAPsfthk>^S*;L+!$yc#| zkE$`Ng-E7`F8tP(R;awIilEbwm)kt_62S@L4xosMp@>;VLu8xEXrVlv0|&2G%S8jc zxmIj6n?Cq_DduqHf15N+3>sWVuhcc{-xBP0zup$7FGiqJ_u8GVu-#cTo0hcaslY7HYV_b>_p9@;f^E(Yf<& zad~0wuIJBmL>+->PYgloP0UYd>EP7iaFa>j&4rml_(Sof* z&`gC)J!`vyjHAxt=w;7CK03rIOKGqi_4~=V^_nb+L*WIAwKWy~hS`-OIl@-D{6;T$ z{*_8$!_kD1Lt#KfAlH_(SC;k`wZf3}n!=-2$H|kEV>7LF+xj0$9I@<+5y@Zj|6-*| z0*YEk$JTl6KFOE1_4!AssXN!{B zkB5_7089o=|%PAb1pmv zDPKhN)oH|Eqzj!mHauOlXFE?m{s)%D@T>Seo!J-wtRo<6fjE>X^O*&9mv8@;k&=*S zcJ}ja|B`kp*s4s&+-2>+Jf!cxj7;^R{P{g>i}Pan7ya*Mg;lC^Qh@0kKlLlxC&Q4g z{S9H86g^j(E@;^WpH>`2sn)OEq~ohWs(76_E)JFu$Q?$@X4b zX9rs`q^II#1ofA?t#z*SqsicQ*OG|s=;s2DK}eRFK>vvqZ>l=Sa1R-kR_?+lxu^n9 z6OAvnz9qEubJVmXSQ|RQ+7Qc$u>02Bdr-@?)*|=rYLl%A+@UYSl4YU1hE}Q3AMpj} zYG^IU@TBIh^biiPKDN!@8a&!_t5AYRW<6DoJu1J){`*`uoStJf`)k%pq;0lWz#RQ+ z>Gsl=R^+m<^bkT_f3-C0;Me=fBY0D$I`WDtoyBEKJx#)aUuyBr zz`5vPtXNEObgjrJn>Taym@wXPo%Y;xml^M7j7|1W9QGr13PVRdEAt1LY~a*^(_BXv zSj??uTjlJ<52PWAF=PGaj-Z@+Dhoy%dc=EENq7xk9dG*SXN_w!+1%3EbER>=+Et{; z_R6ezFRa{r;^ei6+XsG7YJP^WDj0)-%!0h__S%ha4Qb4FTyMf)Y-g)gU+c5{I+}8rdsZ3& zr8LuUE!t5>w&jG@?NkOtI#~Oqj!;0x3)+(%l+@Ndw5ZRa!A zSeCFhb#fjp7O1K@eJ|oqQ3zEi6i+@P81r=Wy|@x*Rv1t5DX2R$0OtHruuuU$uwDMl zPlRT7q=Di0U1PA{+=s*>>nxM3s01(5_kj{u=s$iOZzG_>86PrQhz1K=(0NbGe9Sm~ zQ<*)P6*5i&4+LuT%WQm0U`>(+jqfM3KLkMH1ZY@?C#f45Xk=Z)^;q6gbG!J=<000? zHch@P)@k{P9wTD%ZyvfD|2+5BG3wx^*FNf+B+ZXG92k-FlOSxKub&Wv5OH(_%R2bZ)h8MGEAV6C);Zd7Ki3yC7WZn>r91kl zgfYB)AfLgR>d%K0V?>QLF$+v1AWp}3gS7&Lit zWcho^b=iQdIO;iVCOi1mTzhOfOSdcM+};iTP^bfU;e@Q$t~#O(j>qZyf;!Vo%PeQ~ z!1S9ET|Pyig3T;x`BGt)Stoc^GkS-QgrBya*g$${9p#t*n_m~jE-)LNd*T-YTPp6> zyJbsT@}Zc<=^40)Fg*Wdrm9`ITw&0Tr;0tLnDoFL{4|7~5@iI2iYW^)3*|-EMBP5~ zlE)lWmpubNTYh|iQ0Yj2h&Wjtbx{W+jUk)Pl3qC{Gg0l+nMfWLukV;0(mTF!%0rrX zH@#!>HJ!VtIcz)?B~|4+T;%#)V&*%bD2cG!fvl)-sP&<&D%Hj2p!}xprCb$70&SSu zT{+QdqFvH#G?Az%C5If_A8jNjg|TL z$GJZz)xu*(T_bOCgQ8rlr${tY`|@Z((LA&CC1liW6*;_%|HC^VXj^d=l_oB z3r8WeIHo`NIV)mpNAEK6$x=|6F!{JcWi&3&$;m<_4ws}gd}NhzHRFQ$6v~#* z!I76e5tngF2*Oy@wbVYzu;IxZQ4<-v93xA@i32(QawO+Nqo3j0v{Pz|+bvP6*jA!? z4LEFsLO*4W=l5-?vBVk^C{ebauqq0*1|0iB3l<)5)XR9jywNN@#7-@sq%Y0SJ5>2R zUm$$PMIvG??-+8y8^avl|Jma~{PqJbHz5aG(RP^Uj%EZNH5=h?D%&Lz&w03|Us+wl zmg%qLURImOF=iI)MW31!E?0bcsTN5iR7`%&GL|MjN>z4J-EZ!s@D@`ghTBGrPG_GB zmE@n!1LW6;wb(@UHp&KdeDd&1Ee(bAbGzBMlA2WJIl^?zu5IPXo>O&%4Li(9ANW^2 zO^P^0Q)$6wzFlYY(y&w1uQS|>(QekmWtg2DbIQ%Cm%#hgf)n#baRjd$MFf)kSR!@4 z(n#a#yvhhVY!VE1!KJ{yU)Yu9PL7L+b=@xhyC`7NidOJsbND<-gWTUy3V&DE6Mo?R zlZ^f(nt(Q)2y{0Uv-SnYadW@fj@+SP=!{s+U-=8Hhog%PqB-&-!@S5L$hJMU+wg_s zx^T&rG2alO7W&&$!;|;7A>((ZKd4+irxeaq0a1nTgR`V~%R_dI%IV4VVa9w$+%Z{dQOOeoJ`LiO@1pSqnsfpGa>|;lOOr7weZxP+)BdO{bBfi@FJ4m}XUFh2?@Z569hPK6~Liv-yQ*Uwby@-7Qkwx?f@F8Ho$y})kuBK4BRt< z6;piyaOyU0{m<1gR&^B($fC~vr@A_VWw{+=nR@*I0j>!v&-(YuA9%V~DoiDFw{0j-?aPhhWJ(_%XVh^~H92AFo()^uzuwp0JV>!O+x{F2~$S$rfAQ+}P` zwDQ|3jCia#aHRjiFd)^tBqk-nt$no-a}^o@Fz&8*m(0J$+@`%+rdA0XGw#B||h z!iIrQCt?+dV|KT($0uTMK2JRK1dalF+U zR>F8Qsr}LtI1G^t)J5&c@_+Nsk*tsz)aCN>dT$9Me1Sc-Q6QK2?%gltq0FG`eS+)# z-K99X46(;;XC_zuf!=7`yIooxaZnZ(pWj%9Vh{Hf@@fzH_jlO7{Us)QMwW3yN}{q} zOSF$wrTR(0W&8E%bpY_OT-zrej62(0fJ*YeAM%MAq^@>MBGMQ3DJy>>AeIOT*cg`= zGho@ke>Z%72m{uBZ?W&e2h5*zz!djYRBmAZ+&*~%^v@7|B>XeL?!=n&{~Lx25J;G3 zcU575{_pXkYnEtOdILs2?1S^hN13m^vi=ZSi92=keTxhHwlI}_VyDEszSZ1|01~Z6 zL(D?qWVdODbDT7E9`2nyRaM?nhl8^^JMxeC&>vqszuUoxj!EyZf@s}Cj5tnHAgYf+gKL6Ol{$U+ zmR_@4n@>k-iTtMohiv0lOod;=dEc1NO{EJAnicn+KdBVQbA9tlKn%o-dXC1fsCIl3 z`XRX#{7bN3p={2 z3QAHGN9Pfo>zq^<18+QKRDwWdraPrMWz@eP&ut}Nlu0=S%4e&?ud}XIGG$$3C<$Vs z#ayF#d_^3RRLMZ4`kA~{j)UISj5~xNVM=sV7emmh=I0y&y|q(9zxZBTH_?8}IKvb9 zcVNGn*;j&MlohIQ2YV5!5_-3C%=r5?xj6rIyNE&S!5ffqy0T_lKNs|*?W^4OoYl@? zNg=01;IIXod9p{vXiv%rsLo+h=iC?6@+W-OFfm+I4S8st5AT7BRzI}!1OE1d6Y5b_ z0r350-CDb|hhiFVTo2xuUD*MDVhdk&!v5JhVJ4)Z6Tz~rRGe|NNy`LJPOb^N=oG_Y z^V*+9<8!NZ637qHR05;`eW->2#Ne|LU4CA&c0{Vjlk;D1emPppM=YFA;&aL!=3yE* zJ!@3DF7r03I#Sz>g1fm_6`O6hJbx!Heu{<_;f@12h@_}i&A>se&sSoe+@7#e>wqR! zFy(+tl({p>$&Om^K73Sn#?DzXO>>rp3trwcVee3U>Ay~6-R5O;FOvg2TO+1k>xl8L zR^E96YNkRD{=xzqT0#JWpMw3udu=`b0jp&W_B)#e47&+P<2aa$q?OVimt!t3oOyxC8lykG zS}^9FavWENXqHE{j=vuyHZQ>XgT!OKdq%{ds5Qap83PFp;rM0KVy6xwiNZ%CLto$a zT)^U6hj&&7o)GM=WXq^TOfjE0&Z#XrPyp+>z3u=W?!K9`_-DIPr01d5`40P;S^^JBzBM5J{#v`irtg5{(X$8d#3xK>9&wh0z=wTL>zHavLtT?PV}lj;w}x$KE!uC6`K9)H z>*Ey&pdK#OQ_>{ydfn-LUg{dO`lXI2DZoc_c75SF5l)Q>uE}13OTKR-gI>t?$8~>! zSw9BPrwI;LGLL`&#fl&@{A1YR?|wG)I|o6?*}Y-+9nZi zY^QtGcxT@YE^*$L8wd@|7S37|JBoMU6ysPCp75#Je?-YAFCe25#h)m(%BL9_hP7M0 zF>7Tv6*zD5s-T%OYb{{6&86W@K9AUNm!?IP<02jhX}22GMd{NbJxqy1mrrv4HrQ-Uyf=WM-@X4Ckm8pAG_9JMMXn>i!Q~VqcpWcp z>2^Y&uX<;Q<=Jf3-rRs(7<4(qYNa=5%!UkhE!ar(E4MKd1C{m&+bMu^#wBaLCd;jS z8NVip){F5O=b(OjDr}<2&m;4>d6S1io)IiuDFuhe=$!yQ_4#gopXM%!+~e~jw7TkT z{q!o@qE71|_rT^mFSw;dScSie^V3&GEw*?)xB2Q64lUn@>C)iZ=RK`CZj5N`4;S0c zHlWMzc-lkg+h1TC@(P+kMhbKnAJ#DO&mOh?l3Pt?EK6_N?vgMGM(7yP;up)^nu>W5LL6-+s;E*SS%nee;uVF zCJDEwsx%awdIQ?na3yn1!thEd@E8k!7To$${I)LG}|@o$VoD zvL*N96n`jRS7+V1E@V9R>1rUe zp&fIFsVAhABWN|7id|aeQFry8kmTv?_1(Nno2p?$$I5dDVBO)6&c17Ycj5cu`#0lV z;VtQ^2&G(5>(6IkV^3CkTIY>e%+E(Rz5BlSRDSb~0GAiFY3k9+pRqtR!zYXwFfDRh zTs0|?3VXHG_fo|_pRn}3zYBXOWnNE9+e}|DKM=1b0GSKNqhod+(0JX)DLJbg+uRyp zQ^B@)EJn&omq+HWaFdiu>tfeXt1D9IDN;_d(TA&d;s%{H#T&qg&d~E+^Oh2seB+Fe z>XWCZ!Dc(9jv=pdoXEp}&r$d4MQtDQx*b&^02yr~Bq_gJWEmb9VOcQsW zm6)Tm_B3x#NuIbkdwDS;oO)^e(J2CI^$#CaJ^-I`4%gLeS}JK3s_yDVA<3gzldz(H z_#B)&P`!p*toWd8>kn*N_2cScqszp^#ti|_^%)qg-?b;&xCo|>+IbeeKNilVLnLq_ zc+y>!Ue~)@U$|H{CP8V}yOszDR*Z1_oJaixmO>8P>8v-#vjQK5^SB6G&_~xqZWHD@ zX|Sw3Wpon|OMHIN#gvxf6E<)BH+TBhe_~g%#s0|j_Rt4XZ|irVu7|72Wm5D=D5mDg zprab5zWN6RPxNWFPVyHx(`)#PkM@J{L3LtErh$7ta&nm`VP7#u^O@E+Fg&MJh534knmG<)UTYno{;H~ z#gUa$J2wM(iPZQ%ht;V(eo!c-XHXX%L%ACP^@y}?b^kn6+fJ8oK5CJvT$iA>UcGw* zQq9_tk+@0&8v4L=1msj)DNm8An+=^COK;(ho;4BO%8?6K`%MkPc{I$NYqbt)s)6x3g(Pn2X|=7SgTSf^J%aZ5_4y<@1HtQ&-RCmM<0~bM~~E^bn=D7Q+KNS>Ii@P)lmb& z_s>HkB}HskBW?T@Q{23moG)AYmvfJkJUftAll>{ta+xTDn72o)Q5Saur_)B(p-6si zOQ7J?QJsm@q_Zn}vFITIuWQC`ANiRjj}K~CIepn5CW1bsC$%*Y+Dqi6fM4=^z8*jM z_{kx#E!mk66toa|p%R1iFGcX-A?q093X1Q?uu}>J)st}wzy>Ci;Ha^t-ad+IyPsxc z>Q@b-g(>j{4r1v9HI$8&`=Gh^^9PqIL{t{%yLf-1D2D03H$)pQ14EwM0T}XbCwk%d z@G^m9v_tq1K>uHF5YaL#Hx?Ed_c?n?>uKyFAvizE%aNew1?5#;(8V;q7#>kp!M670ah8QoOqM7`ndSU-X;P1!X{I=Al5=&RJ<+thb8CUh3 zslQwAmWLZ+gPAzT1=b^2;)mYCH?PysU2cB+#In;LYI=2%^g%b55>oddxPeL*;L?X zoj@JvA_^sKU$bwdau1%@wi+#}xczHsKp+&l+{Nt7{|5F!sq;KhMFz#8@MnRjoC;>H zR|b)ID*Ez$);rAA(69gegFYFE!u%}<`7WJ^^_L+77g?plv{~l|zxjL5=C-pKt)7zA zpFVEjsrsY;^L8loL`|P(?OoNdUenj+stOeBS^5*u+;10(c=K`(VO*2w!QYG07eiN; z^y0spoArD%Bh+aSxP}0#ZN_l8ai&exF8@)UiarUbnFh^M(UVixEf+`7`eXML57;NJ zKKM5so8}#yO?<22Q0kFOn37RZ`s++PXH7o({PPR$*%Q_VfCdFyK0-%mu)8ri4-26Y9B!1kQN4>g%rgcc6sAT;m_{g+M#hNIDt2z44xZyf;!OXoVJd#U z6dTj3v0OVV7;oye?em&4lG`uCY=2LsRYt)q3=+iV2p@w;yjiBiW#|ywo)`UKywK$ar1I_2&8dE)tg?;rPa5>FCEZIY3 z?K20gOwujcM5&U3HhsQ76C6KxV+d^jKIw4gVQn3_#~!wj#IhOwVNazlm!COE9$!?T zYSVApsV1c`t=F-OAEGI0)tBAH$yvW@OU=ygCNz=hxD-Qb{kjXM_R@o*rYmw4!+79S zQ;Ne|P;H@dZ4^Y*@F$wQ~<&bvElbr7T%K4n|9nwT?$4rd9B7?__{EM{HjJw@4`1l#EF_7 zT$9a4#sxN~-!=+w@cp(-KbF}o7Xbx*&c{>MKw7@0pN^vQg8 z__G_94j%&=e2xd;sWexrBuWLp-hIIvj-6|u8=eW*0Y`vQhW|=sy?5ll_ja$T1BURxqRHDc z`HdGa%1)i1?^OF;xZFL@s|7LK9Ci6N?cxCeVzW6Y$KB(`*fE!~Abk;sG-u$5w<+ce z$2q@6Ua4Y0IodpsI<=c2ZZ&%;gz)S#gJ4d2$o_rS>Vd9?3bA||BjTU4T>jwdSM=4f zKkvX+n!}S(|Qr1gU#hj`17@NT`HPA#B<@*?w$&<^!7jM zhe^z!9^Oo*2sv0!$Z!pf7@o;`*zY5sGq=R$i>v5Q)2nt1jn@})pq7?S9&lNe3|wG8 zfzH}Oz*d-CamM7+{UIldBV(AbHRA(mqoEJ$bGNDSyDX3Pxk9>&F-JVthAGL~M2(IB z`3l@sVmGrZ36Jqor*I5Die_k{u@zNmua|}9FU_|RZOuGGJ6lVMT8YGb2Ir6-=Y8yj zL4WgU;^(2~Yv&pN?6?}Z$-#M(R=3ojVvY34^dEWjZm`I3_MnGci23_=7mazflpST` zC8}|c%m`K*L{q^pV2xu8iF5Wh`&p-qP}anTNL;AtTs>^wlF4raCv(ZwEUkBQ*koz* z!q`Xa+^>sz@n3K{{&ewxRj!T0w>u;VC8HwTUA-wJX}LSB+KP?ng=S+b6{x7bNtrWZOwI@Tq;#VVnes34C zY~Sy8fe>m~hsa@U?9r!v`8{)fCHqh(S7MHxi7q=W& zTssh(H%ny#8`zk@;SqY;`?E4dODjteLco@2f{^M`%Kie(pa;i0P?+Z-H~qQ`gVvxq z2=^u=?qd-EBNKN0)3t!8+lcP9&kKeftIKK4>XqGQ6%VG3lbOjZ>r%93iJyvoKB=(?UVtM5!9=Nj@0Ua?))t| z?3nJV+d6juGRF?m@`!~`vZp;Q3(Jvj(?qFJ2KvfqxZv$n$Kqcy3N$J_S@Tr4tLT`+ z?BUyUE};kQr?$o}PNjn8!Vg|fuE6e!tlz=llf2NZ{`9 z^|YQxlgEldB>Hn|{57f$P&#=%35;S*|2?z4vC;n%r>#0EzSuMs87lm!mgy3E>r5nZ zq^l$Gis_tRac^Iv7`N$M-eI0Cu-s`TL)(wPWJ;l=B@$!!bL^|)Km-Df)cy;>wMO~G zybEe3QD^VR+Inq!ytee@;>$6$BZAnWs&Bw1HirXHQe1O8&7_0+KE3U~JBZ*(<(jU6 zox52<(IH2tNp8ObSbCoTbBxuYUrLE^o^lFC*8L5kx?9C>XG8}ivG0*bmrKhgkJXnT z=9AFsrvB`;esg5dtaIkINZVUDt_Wq9$JfA1SIXyqVtj7vYu!I>@)Gkmyk4O4A#C(^ zcnpG>krIJDD)aTm5H6Dk^UV5;s&Jmx)7RfRan=HadHBq_;}8(FRmby|2?!9G1_O=gTuip`;)% z#2`}AA}A7)(%lUT(j}nez|f7PLxXfPbeAxKbjc7yoNN4ipL5Roo%LJm`SYxG_=B~$ z*XZ2$zOUMQzxI1?EuVPDRyy)|WDNYioWG16h;o@qpR-$&5j1W1JSR@iHON&TzX$G2 zodp}4+4(m%vzn{UlxG6rmgGR^=352#-kk0B-Gy$eNH}t+NZ5#8}EU zKAipzLsxJ}0+PZ;fF0n&opM?&qM*zOABDR znrBA?4CF?A;imNuF!Dr_sH}iF!m*CUAJa%vR+BjVks zD!+!S^o_*a(UmBO+u-5CjgLe$UohK3$x(-dJ&7@l9%fbQr(`x>-k-V)%#Aq_jFqOn zlHZZ9ocY}Nj-~|xc{1~L#WufVK>)-~zxJiA_Ry^n>_!!gn(1^xWI+UCKqw4WxEYs= ze$$1hG#^Oe0(0#&sazt6gLPT}s_gw@d{=pokBM#66Va5lN(fTC-3!J`FT*Ir9 zzj4JPE^x}Zrz5R`*#A=gk+x>yp2)|VJH(kHQ)mfLFF7f&$3VwrYr|E7ua@H;LpQnpz-)w^A zw4*yxZ!cf>Mi;0$oNsg8Mzf|Iu<^8!dK`QDdXmn(r>tzXSk4p12+ZS@!hQIF21FA} zN%!hm9QpPrPf;Td%1eDi?xPceL;lYGXm`)O?t_jVh{Deb`JLx=U)4^toNM##Ob1bs zubQ)jvWRG{(!#}=*vi{ED_!r6c<|SDZJA3}L7S#6?r)h>@UVIbOvA!`vIIS?U!tou zBl&guyH^;gqkE%`Es}P^VU~h#Y-t*-a^1$xbmhTIFoxS?doi(5^*Cr7+u?ACYW(!f zoHzQ@fajuBhE#ioK$>=HxrtfBOGzQ*lWecitO3O~%~>3F+8+Q0>B5e#CW|A8h$iU8 zzJ%Q0psu7Q?JI&5Cne%%VID!;O54<%h|vAv2<#n4nB@?nTFat_YS3#v(GY>(PTLM| zY{F;F2(QPeCw*3#g%EZ)&_Lo+H0SoMUe?gZ3-q3P^q0YPd5OV|FSt99E-)UH5I6@(#|jdly(fBew6)dyEnlgN>dp z^rEDLWqyTOQk=mvbL%+hH=?_zA+5xiP1)uNfDCn&qKKVz@U*++B;JI={Bu{dEN@uc zpSTw^ajARC7gU6NvNBi0r3|$K>d%BVCsxT36jc)Nbrgen^Xy3HAqYheE8OxNJ| zB`eDcgi!hnw*rlav8jLDh8IgL8udV=ddSr;gf_=#B_z~;-hszA;WwP}X#0hvS{cX6 zrD*lBXZwDW?-}>)J#5+|@p0+}X0DWp=9r0Qk?wc?`xj!LTKEno#p!JJD~YhY)6fIc z?l|XsYXT~;&!#|l0Z^QR13Gk<^#l1Gw{os2NJVgyq$;o zca?{1V&dR4t|LEK^_L3e_`i1rhet)IQfx{Oe!^L?=)_Sfy?`7r+rj&~lB!F%!T63b z5Xx$Lybu>}_7?KVY(`0!x|=?kpeY_n3Ta_8Dgh#5U8kmiX_|M|L)~1$KjDrcoeKI2 zh^-(%MSXz&7mY=Jcei;<&iLmb`e2P)4mR1g!5`N$33SYyvC?M0u|j8OSb4DvfJWK; zM_%kV7QpiS(5;oiU$V>dW|!-HEBK+bpAhx%8SKLCA3g8|{|y?ec;Mq|kK*|Jy^^SL zHrilH_(a^b$ca2{RbT&I@CE-3@T*b~Y)=@W7=CVULRNfjJ6GquXlJq1-_ByOes_8D z-VJ*{c`&&>IwtBnkFWoc`YuG068k9hSY0(@@N`o^g>8rx;9l6PD%h*IA+SfEu5D9D zIWjyqlXUPDfR$J-iE8rZ1nKU#0;Sac28%Irq#kGy$L`O2O| z*|uwhI8xM+fCU5z@>A!|tdwWKqW@2EZx6T}!<5^xWK^2(ez^V+7NN>aPd9&bwJveN zb~XUS=BoWvdw0(VcYjc-Y4bc^B10r!{7ZLX!7kbrpwH%q+2+UcUNXh0pZr3SbSBDN6*(KA2N^5fkmP|8JEg*sX>|(DAFXNfB4muc+3E^GNUEONV0tg%r zQ`Zvfb?oQ49G7df|LOm*Vkt@a!{Iv^EnVYa6w(6GxQnG4?IBiSAR34LOT`{Y%6Wc< zexm?`%=G_uu#umMfT&8V)M8V=g5esS?u`RC0JJ{VE)k#E^1kzJnrHl%Ld&VN++yA+ zB;6|ULYLy3=NNF?`76TzCo`5u(b*L7zmT%l;o8ss>GgbFroeaqMuw+aH4Cq}(q;ff zAu#-a z^N1^pYYBcMo`}4T{&z@0Ijrq2!J-oIq zgVL!hT#Z*FwgzyJhx?`wLUChLK58IGp?NZEy2&UAuy9ObqrOjbK89Gy&qXSt+yILf zKwVqv@MgYgdKgO{v75Z4bI>t-X?-mF{|d*E#*o~4Fl?AjhS5Xc-Z zQ?>mw3tI>=O;e35hK5}$${@uoZ+*TPvfUzsaD}Kvu4+_@H~*NMsBR|=M?;C+DTEN5f`nH!ILzdKP+hs4$xuWgw8PabpP|G`EfSOS_d7ov`aZ;N5i!gW-T1M7 z^+UcBUJ;G_)8h)?e)eRQ-_7?Toq>(`akP)|9Z5Ite2(|^C;?>nJ~@-w{UOQQ3xiyI z1ZGC0uPA6st5WKcJ~63z;)|FT#jYw`I=a`hi|GzAjyLm8jT${5v5#h52<`Q}l;)%r zOD0xtmYVb*`3H-KJG~xo5sIAks;qTh=PB4|qWb2kfCu=ru4iLwpJ$!mLWwLLba8I+ zR@pi}f_5~@>zUnDX@UYCbG&a}I{zA;IYr5e`Url;Wm-%+Vy<@$s+vmG-)y4oNkOl; z&JSx3%SnlF;~rlMO?^Uq@&Wt@pMr}D;nd-cRYViJ?GyqT^AptWnmQc^WzGv`>3RRo zY&0nc`Bpv&_SxiGD9!h$2p{=-N9X-F=gA>SE4Ex41ICn-WG+K)1L+IGvmt%wOuQH052~`{T&@F#$+q9bPPO2no2;Dy*ckc{;10VOk$8bm>#h<;mC}KQ)L35$#1SivBYC*IXRQ!>>Je>wV7hM zJW7C~TG9MTp0eYchg3ybg>JPZH4Z@ z(}W{1OGk_Jx|wG?ZuCbKsAf&~mAX6&5@(zuSUo8yZ}}A`GuUS+-}YW|9Ip10%uKj6 zm?cU;Z--ws2emxP)HItd7v3sg^56x@De+N47i|jpClqp5m8qU|hGKQ(LzM4cKJz7) z{F9-Ibb)}UOl+IFJ@?L>He}FGC8Rx=3Gfxrr3=7cix{y82Lgc?!lPzf!Eg_|? z$*CeTzD8jk5MUb$I~JQs89~GNTMeIKI`)3q%DL5ZQAfkz z@%GaZb~d8wGvgNLp6MQoIldCLnED0QF`Qx&h)sx^f|F%un%y4a=$%8`pidM5*cVa; z=O1sl&VJV3&!|VocL3pmob#dXiHM`=;*#=h#jyNeW~;x`x_fBctkkWr%X)q=!V2em z=d>R1XCgbcNvQiRvS1Idxt&k0QbGi*iX!iKfSQDKIIp(B!i{KBD&MH^_jkLd&=zGP zCK5Fco;W0CZ{Fy?2-YHDr=vE@;?orG_b%-AILg{rA$j?Q_Sf#8yQ;YaD29*6lpuxZ zdAO{s$}#NCCVwv1kSByWCEfqzI5!b}PRjSw_Dl(fn&L}tnG|YFRP!|C({8()<*n&x z%a+%wcxqdzq0Dj$k(?V{fO6}@Qf^|(31@RbHhV)HF^5%Q%bw$O2L<73nPcP=jxUjH zW7_c1-#s_5RNK2^a=GtvDQUCAPj5WZwrq6r>Ud^X2ik01I!i|?=W|UlJI>tAa$nVm z7GE39E-4MCiQjR0{~H9R6t-&>hr4ZQ)cB#!QwtT?8ud7atDE9F? zJND!1ogQ+t#@_=ULe;<5Y-_-0%LrNPSRBIuY3mdkIcSk2;-4E;7B3a&{haFv5|Acl z4}zz9dBV2&@87nzZ7npN@776grt9$>MEBVc#O)_cf13I9lqv5PUznB3(UcRk*T%Zk7h{uKvW;ZDs+Tk;TXu z9>-knWtp6Vs0QPi^%2SoFuB(br3WWP#bEnEe+U=%eKksoXopJc$4v7pN!CMJzL1WWwwa(IV$2 zJ;)xd(y8%>I#`oPz|UGblkl0IXF{e0)FKPH`*(otl;Yf#Db5JvJazaulp3c|NFXTS zirIG%^4Csz*;gimvKE)+8!Vvr8k8$3v5TNVgTVzvwaqQ73G?X~>TK+^W+)&7Yr(4s z({D(@8SSGcmV_B!QxH>&9ZSO+XCoaLXWIcm1K2euhx&hjandSJK1cpFPOi=Ly)2D# zkUijGo3SGf#a-H(5(krzf^>H1HY}XXQh|YL!{`R7`8RZG)dYHXbljSpP*heLO zMKnw3p{Cz35RVQfy+z_jtrt}_E5Reyan&4pwz2(NV`P|Dc*M}nO7;BPz|J!^3+Bk0 zv>r;o-FeLBV%Nh4g-F$HbQ}IH0K;WD&SVGV$>m|17Zvu*ua-;o^yX&ayG~7RnXj)- zQ}+>>$IB7>+1r#qfv|utR2fA;yv(`xers%(HD=kJa?FHht|>O#8E#s zI1HY2iz$zwZf!XJsSIJ2F8`4D;3^Oy5TL3)d^0Nc zQ{fTc%IxVHKm3~K`F9yW1cR%KL#m0=O?uPVXo&F^=3%54vVJOVIlh4%?Fk(+g=7t1 z>~@k@lB6A1B@=YJoCzckTGZ-}c#G6D&a$6X_iXI>QcAjKtSn>&o`~)@Vp4GuF3vrI zoHbw>l(lD%Cr9Hcvgjdd^DEQ-Z;BprnKEy5_^rM4Ysh|=Bto6EJDl=raeLUjDRQ)QLk00vZb{cO( znuWsF`Fs6d?c(9DJ07f;Ee_lIJ-8w(yE%i(|Hm!YUBd?TaZ|FF`%%=7dtz}88Z3ii zlZbQyrN3{s*HMWsT0>)QIF`sxeu?~>r?9Zsif|LsRk8r-T!dV;35Wx-B`06ZD|F72y$`ONYXLaITWmHB_KekOMp{JobMGFb<)sFZpL4SzW zcCnd?+|R*R$sa=<1XaQP_SB^2^uQcZoY40x63zN##uZ$td2A&0_`E*{dg;nPt_Z5(w8=-Zgrld3v$Ta5ToUp>Ig zZF$HY_Iy6G?UnTH1?Lmrd*QK-`^ix}^iIl+Wyis1m8jO?ujfZ!WN#cNJ(H%g`@JRS zQY%jVN4(#jWm7dpI)O=5tY6?#+6tqtvu9QG$QZL~+q%5Tq6L zLYF4Qk9vTueRS=5)#^tBFTYp(9NdJIq9T8R!s@59vO7eBdo`Jce=f#Xf?8=DQ(lS! zG~1d_c0EzO1P#k@)zPQ<`-b<9nDQ*@8?)&DuCD;cQeML)(&3*mtG8CR)Q^>%NA`pM z6tv8T<^##k>36JfWIJnbfm~i>=e8#G0EgV5QKriI8an)m5+a9Rw!kR5JIQ`JH)4X( zjW=sxHqL*&BU0)@5{^?jU%wy5pGMk6&NDLL%YE0akvNU?9oSdhWQb(m1(2I~?|1ga zh4-ij<1wbj7u3_^7$--p?f1C^PW>)r8*`gasNX0J?%2M#7R`3wsb=dHWd%;vpUM@~ zq72->E2v|!UcA?91eSLpTFhj2a**iF>1ft`(0XZNDC#-4-SwJS0-FVa9nzEU|Jk#z zOaES-Gk=NJ*ni?da;Ddt*1-MCy?UCJr`1fKREzr!WfJK&%P4 z%$_aO816#lg6t?E%Cj~~Stmm_;CuieF=2_aspMhktfAF+^g|2k(Rw zNS!8s%y6&`0K+QFK~$yu&TPlxsNJ@)Y~gHF=Q zN4ztC`W&+A2B(pB*~rX6a!4C9mocM;u#T;ikeLr{YRqBlc-7D`b5sadzTDmK31h#M z+;Z^bBHFSRS`Lo(mwwb2ZOE*!Q-t0~#!UN}T1tF;jf$XkmQcJ!=78fSA36E%=e>tu z8#aL>09fIy783pRkPH7Ke^a7^ubAMQ=fT~sXr4IZQKv9|T~if~&uw*39a9>YQ?VHS z+9b|h4LlE}Rnl)zN<$Xnn_kJpJ8ZUM3elZb`kCSm&*kBB!x$^Fn@>_+>R>0uL)rBd zb!H&lV5_r8D`X9Jv%PJmJgs_ifoS4)U^@f!Y|!D2@Di#WrxZkP6_3Zz98w8Pireak z=99_(0iKftN|8>Y*fA0{E0m1nP#k=ft#!_mo2D-iP|ei{R=&qFgR`T-nGyNFO&`~$ z(~j82>Ml9lq3dXF?o$0C7IRq9C|W6+#}ujlU}3UWuiv6}0D`l{>n9@BaXP+UPkL~6 z=>N7x95xy0uU_~|W9dlEHFj%DHQ?8t@Z+xLmTc~WvQ9%GYFlKUd#)SgV|0GR5wg9HJ(&HWnhj zSTH$J-9;#R45B^N-4+wJx5WN+$XH|4?R*U?o@V(_RGC{n!o-S-`I>^(D5r_Za+nD+ zr5AeqyiW&h3{$!bBy64fIn)PVN{@7Ts>>i1+l2Ok^x1(z70H!EBA`3w~0PXX9*FS=*F9+a{D3y93>ZufujxxzuUWzeMEd(&mzdEXx*Yec)%kLMb9e$diGl}WHs)u-flF%?i(#Z1 z5X7J9H?yCy>VKf9%Sc(%{h9(r!0D$FWXJEp8cJAJa8*9I3A`D%e%@ zP>QT*-m4Lq#+g>7*RXrO4P4q$=ZF!pyt-|OawFodkTnQ4?9DUfN!F9wcVYt#@o&I7 zar&`mA_9FHC4+cV%X82EEQ#@f>YHwuG$7!joC_^ za&i+L15H-q*M&sNr5rci(-+qTC8DRo z+-Jd_;l^luDsCQs4>~`^sKOEvz1n0zOw#0LCdRDBLQ@t})xP#s)uq1&2q~~?!M#`s zRmU)0f`Q;7dOQ#iqQ`$$yP|MVo`Xcp1knOjemx6`%?_ULJk!V*Mvvh3=-T_K9!tQM zmz$!3-}YIzYFR0A;GUG{ksm;%ou?W%mtJ14^}pscj~J226WN9+V$FFIu?+&mo-v~# z^LI{Gq$sz!6pNVhhR=VPSOH3?{gKY&`Gh@865azxQ@*FsuB9CA?DeJwmay3zaHHmP ztI6u$U>!;p=cQ6f`u3yQT>N022!UxZI_kmQgoWd9u7_21C;74!zyfpy3k*j6{CG^u zQW{}$bd(^ke8gy$(Y4sb9mD^Vn%1Iu{_~ng)%4D9dxD)7#hc&XUlt)lR}1qvz~eU2 zvn>r?Y*8+_llS+XvnV@il(-wZCb{(3=N|Ev`_s(OszB)D$28#M317<$fR2*6+$j1Y=uX2yD4E7Kq;Ui9T)?G(EtxgNF~t=kYEF4 zX{CwDUEhj1l3zLh9iYnK%voDAAe5&I2~Sx*(NAD%9{OmhlWX+l(OpQvb8tLMi*}f_ zM+sSmz>tePl-7EB|L6{No!qKicS_jppyOUYde=xjGs)>l;ix_Qz+y60ZQ%&Y0rqzk zh>cEWGdZm(E#;7yEvUD6h(#5qj)^{DOyGSonFs%qFfQn>)IsbP@1*M#grNMQ3ql_o z@W&=meHc>UU0<}mjwXmkWCl*3afS66%fPu1gm_R%RD-lt+DFmMqZm6Dmo-v4AMWR> z^e?7;rbSV0q$RI*D-;%gB3rX|SA+{wMk=*PDm@pX;z1|HoDYZUV|_{?<&HFT!D^ZWIqc#WjT8on&Oz4P{epPENtEv<0?Re&( zVH0T6!!($zdZwEyTv|{cN}K%RX2=BP7p4E0iXJ~GmX;MnVnGqxZz>@w8pSi~5}Ox8 zE;0O2?(cwwM2bd<9EO73JcErmdjjbp1j~`vKiTBUY;I_&#rsY!3cP*3-gekV`I`3z zBCtj+@D#-O7P%cas%8aPm)jn@g0&8KFHWbDePDmU4dm5f{>;8$7h0pn7S5ATU;Jq6>Lh}AF!7*Y zLU9vcffc}mF37Ksq9O4ia1;A+W|^g7c0?cJ)7sPiWT=iM%@b!W6oIN0^$||h&_0f* z@~N~H9DgVd7cHRJnP-%GdT@E9y{l2X-|!gmJCy@%6y9y26Jhxz&vOd`p8-?w%Sh+H zXHJ*tl1IwRQq($NRh#{vs~V@|W5gQ-Pt1otWDS6ZgL=h$dy_MXluN1dBPx{vBQ--& zowNGov6;gt?ur(B!P_bypL3{Q>fq4)VsUJ7j2$Qt_$X4MzcF- zudgqT#!h%aSAUkl;mFo>*1ol-y`1bAa(wjXXtYM_9Ixz^^+shRf>BOWW^k_|)N%F0DyE{P zi8RezTt}^u6A4%Sa|~e+_gFg7mC&@+;HjdTZ>F)a-ne8fP|nYJ%j~F4k(Jz!4PWK? z>V(h)R#kyXx+$YV=sNp}b^qtb4s`tArQdg{;%S`ExAccPj~5d(+Lp~CK&?R5j+k#h zi0coOQjO7>&6WXOLk`=xEtumkPQB#LE1H3e=k%wDnJ~3N?}%ikM3gj84eDdTF;qdHX9%L+I-9K1t?M-|?UoHQKa_ z=WHcGu5eV`$nr`h^=7i~BaPMkIU#PWq+X%`sDsAn`!l(>J604@6MbdM!BR`3H_?1zCDY;NvSb@*MJ={4H zIkV2wZTF-Wh&{w`+GB>7dHk%iKX38(2`m0d8WmImbP@m5aMfboGs>( zcP2E&r=JRtkMdVP9_(XJYSjuoXOLA7h0wQFWn*gl4qn!Jl<=o=MPPsw1Yz?d)9=o4pxEBZqC+{rUJVzoS4&J8M9nu zt_yd)K#ZG4yy|!!bGRpFTZCd72)BP@US{~-TD6?}${O_Y3GKR=;JI z9%R@j@_%sHk}8cNi%!8!LJH3`JIzPIR|GjFgfJ$0fo%_6g3M;VxdEO_+%vi!4FgJi z9)}OEX^IyD83B%SGl_Z$!fgWc*9Pp-H#f=H1~p@uNqb2TX!8Cj~N2Os~4IhJsc{bXQR1!C$iqAPQzm;e^N{c85Jl&?yy zaucCNo{IebbZ<+MhrJPtyqEq!* zo1tAtlJy}Wqgad(#m4zzJ<#JD?nQ3yem(CHw{O3oi|MaNwv9Svphw{&tG_Oi8E@4? z2w~kH*$ULHKbjdYrT>JsjCkqtr{kMQ_V`V{b<_3e0YyK8u6QE^c?B@^HX z%>NoB2z84NbBkEmS^Yh1Udx>Hbel%^NC0&xt>SqEUtp=o>HiV-4>iUTo{N>=V|vA- z@GZ+BS0Boy@KgC8wez=&m^q*!=A_33I#o^heq74@ML{}U8vpVA1`SY zqog9o`rF+!Pzh43wBT__sG6AAjyYr}Ml^90%6-c0q{w( z0{g7v;J7syUFe{$k4lRHP44=aZjZ=`mjN0|ahxiHd_u1g`{@=?!>vkcOOI!^-p@ts zSdU$ls;0FW7SCw0x`LYvKvyv7xJ!|#8SguBJ-C-pHAjS6)ETUBx{{i3rh?rze?T)} zG0mU{%ylFQgr@(Udi}%KUS2XBQ0Hzi1{fVx_L0X^@Xd_@ z74Mb~)xGrZKPSs|J`JQ6=?p$jmLz|*V$FvRfwhxV_t{8AX=5k5!cyo}gRdHNFv8|L zeb!jHEUJNCs0yZP{#i^skt|OYM2jc(PsoiZ#~UG!O3=?&5q&{obq?B?sUHVB$0{f{K&vV?{tQIr z&%o7AM_cS*;?xhHd!D0Q(D)pg&|OWTS^MBoc2{>)355W;E8Xeddza7V7=Wohi4t0= zP9%Hi-yO}W7NNr_^iT+YD)=>1Ui!N3>#2^Tvg~1To3J#M@ zdUL;~b>TP{dbDk3MRci;sExOsI)t5$P6KlhhKV4FQ7Sjvmq0&241pke1xMJ|NLQhX zZJ%*1Baj`(=7J3+&9F;h?NM%v?-3KoI73urE3ql+)CKnY>)(Wv3YL^rdup8(;_1|D zfE?jPm?+MDPU8t$wEYQ26kwMLoD$NltKG7>gKCY;$3nc(Q~WL05uECDNk z6)X<=74lq7>^Y^Tc!PIwchX%u1=p%0 zpimgQW?IiC;UgD8w#@wn)V1LTHaAXB`c~(y)I0nleK@q@tC->iuf{OnI!d`Mob{ohE(Y4Ze~tpKZpZ)bwfH^)m z@&oMTK-m*+IotgsR<%xQe!n)&XU0wlRRUjeI$zq4X;A=Bt7Jr>6^(x6Er+&#@9A1( zOa}1XaHyxawo2&Vgu?Jx52bGCcx6b!`+k1^zOg@C#ZwO335AJi%(l6yL!c3p1W>BK z%&SBFh@f-J*Gfmju&W4PH3*oWF82Us`L+|(X)ZCBy#Wn!SG+3$xM^EV0rUp>0Nt7L zmc#M7L82WH8sz=>9pH0xKvwXdc%U2j-+)>1K9~T+^&PgUEqMn&6kAkNK{j<#(3n*Xw)pAA@WK z)cK^}#pD!a2M)srQ0fNkeR&GMf<*0K?Ot5~PWhh?K_Na7dpzZ(*Z!H3|Jr>DxX-K5 zi??&12>%h1yxWdG@&lh7@V)tC607Iv^HU&*TkT8RQ& zSPjCvvy*-pyQ^)$y8>#p75DNa`s(CA(5n8$hUOnO&s-ZN>qfFRDWV8CZVOv~DM~ZKuB@%-bLQ0R%D!}I=G77=SK|tG+>QtBBr@D=BDCHj zPKm?+yNF>toMhw|J@bXJBHyc@ZdChfH@yA3Ee4xw4r=97);20>#heTgNRiWz)p zCPP0d>n5$deQd?^*ikL2*tpF+G&WaDrvwwiYE2Fq=(xOIV@z_`R6V;g+E3}ScM3A7 ze0C#mbgcGReZDEiAx(OiYXM*fVq6=pZpg?w-rwg!ApK`+2)} zA$Cq8q)s9Wv1f?HqO|L+{M(cNAN|0U-M{*Qu~kjN*Gk!>sENWAv6yJug{FB%3KN?; zr^WO4xB%=tEmN$NLQBbK>N@RF2vDKBLpP|DD5^kXxT!zaL{wbM}D^$qL{2JeTY@wi>l`IXg7c>pIJ*`u2ABjeY>E4cQ=IqT$U*ANyQr=ft4Io z0F4l$D{FuL4aE~&9XV<*89Y{kXIPfk@hs$)c(Oa&Bd3+Us*E0~6E1cvB&zj6pv^2C z#?<335FOiZJh#bhZd9|u!8kQs%rKq6pWU%@u6_LL#X=QB^=8xrixWc&AV9ZI26jer zn!gnG;ncB#u@`Il$yt|Z=dc9Gn_Z)uAl(qHZe|!1JE*mt4BPUdkN%iH%UDgxb+Zz^ z4myzVe@~D-a^u6~bAhT8fi->W)4BA4$|J2iX|=qwW=_~rlq7YIsf*Tr&&V94&FYsK zk3FDFXsKkPXeZfa{hHzcfSlJDJci0j1bx~cfI~2jv0wZQ-#R%zW??5v)mUHP$66?i zwW~o(XS=v1<*1gPaCTYy`1^}vX_do$gaR7}lbHtlqQ_ z*@T5;I=7}#@~!8b|`&(REoOAv~|P9aKTjbl++^{ap5vm$-7{1d~WOIV1S6;^fTu~ zgu>X%r9ZRi23#fY8Zw!Aindm~Fm3q9HL-qhgtzjTQ!<&`(2wK7{OiI`&zT>^RZ{&) z+t>J|qnfy#8J^f-Ww9qJG5kU75&BNI`pNSwi^b!C0f!0DT~^5l!#o8E4H|rW`7CSNWE%ahupHr zI@$xO#4v;O)7&jkhx};WVxQ;_3{RQ#?U9E3&jxl^GqhMNoU19CnT&O4=Xrk<%$rSF z_o~}y>mDS=DD*Dppyv`-l)Xm@u{V<$iD{^U1App6{pX_1$uE#)@4MWbMO01OB+Ky! z0$;zq@)kCl4BXB#H3Yr0hnmYKoW2T1kDGCSUVUx>A2&|u4i(4~djY(-Q)+GaQ8CBL zq$FB;{?k==QLA7B`P&0bJI_5x(Tn{UilFM8E;;86eU!$1jABHzyotu#@r)Ssbr2>~ z1c-DMXFB%dR+~H{g4Q6f6WHQ&?T$tR3r4v=u8U&ds#Ji>lwa*O2S12Wzb+UG>tLSi z42AXJt;&$}&=y{LB>lPGw_Lt8v-3)j@-mG1?K`0yW0s?ZFBf-W4#jvgEUp!$J#rK7 z54*+w;hW}6Zo;q13!)T5eElZ;g2_82t<*839PuIc$mQZSlcV{*%^ShpRFrc1Vof~G z5wbMZq_kPS$j}MzP$tUO$-u3wAfRs!j4?Ff6et)yVMhPg7*7qpVvV8uU&eS^%)zv| zbU{#m;|dcHwt4)9a;JHC0MxacoV>tqn;ltVrXJ>YER49g#RP^;7)CFpE3eaDU!!X> z40fl>+gyhgwIqe%Y+XVO9Y8NYh-a8cU#u5uML{4ygc}MwHyBrc7&v%|ED{zgR2X8QJt# zKIS0{C&?mQJ}3P4=>H;m*~LT%R**?KFeql^{$%Q)G#D`DMzP5lJ+H2{xSvh=L?n<7 zs)=|i$VnMAoH36ptwB`+5dzy-DLE0z{Dw3=u_jl-R3FlVOdx-wly`m0c2@gmu`ZFu zE<(73;yh#QjN6KrRpwv>$~Sz;QRIhjb-=|>0>e2vU!Wl)5Vh2=_pEONlEzqgToj4I zVYKo?&PcREywg%)tm62RcO0Dk505HSy=qDtzD($+7OiMs1yHtit=u|x`9gabynCxU zVZ3bDkEErOr!&hdxX}6q3WZygCbH%8q|`GNuQ+A!=8M6y?f|IQ&!bu76BK=w>25_|-xmquvEWKUkuyIQXt7j45_hZqdfA4N2=# z(qa{k!5X-;DrZE_IZG33;9;X!usVfz<$dI9%D)2)95Wf;K&z_dq?-_vaeJFYgeT{7uX+)TwWn zqk~}`w54R+fFuNszc;@n@+3!$EAX!D7XDuxI;v@BoP^@)GAMaUDEi z=_yW>nM1PPOHp63ApI^v*El!7TzKkfy4b2j*5D5Jc5$M0hT+?lupr z?7lVKH+2bHOko_Mp*^5Wwb@^h)YvlJVc)U7k$|{E@Sk__;%8lozv#4v>r2p#`d8&2 zKZRWhg`d8JKOLbh>)s#ts8^=FajShkcWiK~J;MB8zgvpmBd6RBUO}Fp3u9B-%Cs0y zrq2W)CHNJ9UAZ9Jvh6(CzpTXVkF}92p4G|`03}hN)z7g z#R;$5;DyO*ziIOOf$UQJ0!z7JeL!;^x1op3RGmRLZ#BE6N$qz8aWNOJzmzTpMYQ&izh;K2ubQ-Y zRef*%6%+c9!;xanb<|BFWo(4fXoc@YWU6*5hwn}l;n#^82Gdij$C|8hr}Mw`zQAGh1J$W-;eo7KL4*kW606qw1nt{pW7pg=BU_!MP6% zctL_q#Hc4uV5U`|GUiONyH2it+B&W$yTx}7kJFVhSTyYOyW zOB44^hNt(eq*Wp>$3vGqqbv^_ZlksObVJZuq!Zx(TdlgG*8DlS^gC}2^3ep@mP=XO zF`s%{)Zz$nD9q94y|d0E@AiMHfMABPgptxK$3?6C69K*%O7dq+dBP(zv55hSn@%^T zYzPN20!D$|1tJUATSYQ6kh(AaOkdZLzH<8xWZvkn=S7>@}a*HrS0E-fs42pobB6&?^$SYML$_o zE$iN}^-fOSuHxL*yGst~=rnG3(TJ7DUCleY?H5ZMpvmA1yJdeaALxEXU~P-8R=&`< z{dZ9xmGvxI?3W1RpLG_iol|xcZ}WZnn9l2vGw$i9Dar@;0P{~+&Js*V%Ga}e_Kkk> zRC=)2>tgj?YzSk`ICW6PmzCSIY;fEoSKh#-NyW?8VkZ79l!VOy5OUg zy7wpfwxEK=6UBsBIFfWH!Fvh6UYT5Mkx0oE&;uRyL z51{XW(9EN#Y8wJ=If8X3x&r9qwo9(2;4%eN|Kj0GvE|KvZM{8h-({zngVcUBzm)6L zr*Sr@VdkvrW$Z*gTJ0LC&Y}u$uzE>USoid!syA?v3kR#Cc1+%ggoRkFAiqgXG3(ID zGb`CsT1)Wl&SIUqa%iE-%qC7=F5!kpB`oRJ1k5Mt$TICmM{_qAW8xT!Jw-y1Wyu=w zC{sB@-1?XKjelJIkE-hU3&x2F+wa7vdvF@EXek}G3>aONT;AIYg^e4Fc)$G)>vE7N zHFz~s(7A9i?H4#N7Ug6oqIaPNTVHiiI)9n{E`fz%JP_EA;%dYA#FKk;_4NM28u1t8 zt>^HkhCqAUTcyYV6By=ASb`*(B(vBwxG{V-9A7Vtf2{MA#j$rk=K41>2H9Z^T06`VFze>w4G8q{=%o!X zt0JoZ7N2Xwy79;i{b30$WxdVW;L5rb`qgw@PZuV5Z>>N2E*l;T;V)m0!Wj=k2@OrF z0>olW!@Fu;k8yRc!YUNZroLswoH#gsqiyZ9r#K`8*1E#CDq{K3qQ}s2y`btDLj37W zt2n+TCvVLb^(KSm2Wb`GvW<1rrLcdG!-miy(Mg~2irVO0w&qm%N@|!$<9FD~Jr;wk zZVu6SUJ~@6u@I&{HwV{UdXP-zFt+5o?H7-i9>E{@Z~ScFsFk8!Tyl+kAjV)&5A)qh zKGcL;nMsh8D&|I8{&*9k9_V+(XDOawdgvXo;Y8Qr^btsqPd#$%O&cZteDPeqDQMix z!YZuYWiR0N#A&WRB?W`1=RmNuiahi{x`xXFYJ|fIRtw|i-o!5-9sUm5t#A_Tprx|u z8Ei56Q%m?0QFZ_&1ax9=WHszh2{R5_D;cR!=~utY$Vm$MXp`rOZyCvBt-MslPSx0a z>U}*XG<@rw1%sb=u+(YCGR#EVOBcAjh}-4Fwq+?zn*~in-#t;q{935pHq_IXv@xW{ zVzRnj7jHe<5CrKZl%N74 z;sYX5lrBvLr1#K!7m+3bL{t!vE*(M*K}smndkq~S^tvZL@B3eSuXWB^@0asouk$4f zCik7mePw2@_M2Mqp5Os-@S1d*$3@1 z)>$kJj=jIeo199KaxYS{rt58Ft_6uP|I5_K}kR*7+I8G|ZO2pNT^-ul%l|q01 zo~unr3(GN$H(9;nV=C{y37CG@Z544AKlO@EQ2Kco!L;wMdgq>R9L?#UZ_#)^?c=`&PcgX|n5X#XKIEqJZm_Nvt$KCm%T6u7XD$Z$ zvYiZZ`A6Huy>v5d59kw3wDRiuJT5J-1KsuOop}khk;`@x-Q^F5y(<7 zeK0U)k2LN0g$Ud?8S!nlIM^P@67pR0vuj}j@PmrtW9&5=<5L5{_I_x$6p@^Hny#4aCJ?(H6w_d%jXS+Wl_ws4h z(9s5Q(1F9LVn`@A|6Le~NpoM9(21fIC0?Rp1Cz;cv6|cM8TH<@{*kya;alG?Hf~Ln zl2Jo$ZM3%xg=4lj8Iu2rU*V_T__h7}e5b74t)8u+IC%TvRa&q^@focK zqn)ucu&3h~s*T3G%(dH=?byaMXR<=8a?kbRA2?5#)Ej-?ZA^XcR!+wL`fJ9?HKKD) z)0CGSx{<&Hle=6~?)AR&20Jx-@Vv(eyigthw|&`FcH!*)i$>&CwXCE<0o1zL?RGI8 zijjDEKCoc^HOy!@#0VO|3>hsJBwuQf4 zD|wW69|Q}Aj`bOyj=0=QcOXEZcQN<~w0DWx;s>SN9Hu!2=iQ%pt_RPjJ_c1$xc`x} zzJ81n4*hSZu2=2x-D1>t>4V#m>TV%hJ_uVF&d&#iEpdTx5=2%n%j?~^9`HFhfC>9^ z$DQiOdIXcN-fDSl%+E`a>hKbmz?_4a{g88`5`5DHp(6NDSZ$Q{eI_PN0dE26foFp&S33Fs9bd3dmR2Yd-bJaDjS5nEQHJ zZf5Z)0T0bk#{W}Nl~^Dt^WgkD4;Mekd$dkG#qjWb0^S+RsczdBO2Ta)#OuG3dQ1x` zeF9l#wEy1QEOycz{ku{Q3q1!^LVj#j@QQ(&>*4dCXle0*3&b3I3#ge*8ZoD1ZcQsH ze;00KwhGZr`uNX=&KH-U4fz@#*M`}8PstQI2Kv}|pJDacvax5GWNh5djUGN;CD%2! zQjECjr7NM zQ@v;n3Ov`hAO36T3)&daSL*v{ZSgPhBU5tF4*tgBh}BXz7Z5}EK&OG+jXSmnhfVz& zPnsCAi74*l=t_t<6z!TwyN@0iCL4GAeyE)Uf(UDSBE0K0c|9ZV#%U<$t5Id18t1(pQSkC1iwWLt+9iDTqXO(GSGt0aRAF3%R z<6ot7?-G`bjryKG3j9$^4yNhkN5CI-|0_n;L}*j>_Ic)i z#F$X&x-`q}%O2HTSqnxYPC><1)dNGv`=qeTCY{mh($tyKycE^BvE$21c>8f)LGT4DJw(^fh+~;|CR7r0mi$y8IHz z*ju2k(ewdbOa;-Bz1B18y)}ndnJ&zl!&fJ04?Q|o0B08O|HOtLKZ)LVYn|rpp1(4o z(&U3j1m7)g{f-zh3>H`^){~Q%x^H{261{Sx zk9RI)-Gj7U*4_Vry*vrf0UL*Z$Q(#V)E=uLa7$!Yj5-JB)S+M35!s=t8|()P;@o|@ z@g=gS}vob>daKc!r3#Vr^BoRZ@8DmJ@|I>!>!A$jgPC zD-kQwfsU<>m?ld$KO$E?3hS(%@h$G_P9;0F1zKj(5Xzh~mv3huti6L5w)SB2x{eAV zMLdCBYsY0Cm9!O|N@HhGj_;Xol~lLjr%ZYq@>_&K$E?e2yd>75JMiR-yg5*g|4{i) zr9dF3Tz=+m-OCqo#hMy3JNo1o$BPY~n&$f7`FqaOk5kr(OXO69eWT}O(e#`$0p~ya zy<^+?5@>~!9Y-}e>Omn0tk*G&d!}>z&4z0yQAR@Kb9xRL?3JK)<^%Id*i=(vW%szb z*v0q>TjNf1%^fEWDTWCZ&94`srI{Av3Y3h#)_-CshjBmGT{~Jx|1$WwZl1R1!G2J- zL34fx_%6VM%PIV56x+XUak$#ef+CtZu#m24DAOSQ$9RK)YuveIqgg!6iLM}UEeYL> z*-_O6#cv^Rx@$HT6SeJM_cp8TKOZ{c1ePnPt?`W3p>%}$3ltl$RGzUw@P6~sXNzlt z9`d|p2SzrwJ}ThpD>z`1{I1G3_dYDh3fgSUEAi3zcj_!ZW?(RWi)=Xe_GA}1zl^N? z=;}63-|_4&MaYd5EoAb-(KQ(yA3eHw#|$3la|>%Z5pw7<%$=(CvTWFV zh3q7YZlQLa_9rQa_?yzN9^H5wemoeTZZGe997>FB{b7v|+sM^Tceb&WZJ5?dzt5Us zM(a7NsplBS@?+)|m;(0{lCKnRx&pJ?8hCL^r)LX2b5FDu4elpxGai^b#S~;YJ*}xh zy=V4s48{Ji+*Bu<^TYE?t`Mdab~?Fsc=X9AB`^MCv@bV*-j2wZeb&KcpL=T(IUca< zDj57yG~dfC)U|n+>*ohNBO=HpTi@9OCYyg&SuhN@*3Dv5swLNa;@bpAWaP9L(QanVtxm!mxS!Q z6Q52IB}-7UdR#BC*k9&~q$n{FE<>EUihIa~FP;1;4W9EY-xYe^uqN@dXj;y|!> zNXKcSF@^GVgC_5)@N;+aXm$D$V0cex(9@(x%suhBo>c5vvgT7zuhOWq_?T19&iRz= z3=MIrXX;L|HE^K#p+=!_HxE-8{(LUQj`)e%Ms>K4fhGrO&H`~7rM+3*^H%4JKlUEA zc0bP-%L1SKvAQ+H`yEtiR?4EyUF;^ZEukPo7Smt4*m-5*wFNNaQ75o5VYByD>n)az zzczdb(uY|2-dd{->Q*LKFv$}i|KpV?axA)2ep4L#ed&NupIw&3C?)HZaTaOPMKU0}yUE)>cK*(#6!9TNo!AR5m)TiLbu;`TY@*wqD0-cU)w{ zq0zSJfNT(HU1P~X+iTk34uT!IvdSDRHN`30M%k>Zf*RFfr31%|fb++zwG8E; zr`>j3GO&{T(_YsdIL_*&=Y3O&G5Jc?{y2zE8j@%?=lRAD47{3D`}uHQMkL!BD! z&<>28yJniFn;j1j?*?rWq`vyYZ9f$T+l8NAy_3=vfsl#^3F3Lt(dlj~0dA zF`wGT#H!0}``9ybbRB_6XLz&wZr2kTsPis#2q1CvsFq7 zN}X!vY9u^_+(6iwCZrjN^uJ!)L_mn*KL3L#IAJy;4dmK=cD-mgp}FWyaDn+dl=|qE zNH`%E@!xlFZO5Q(B}7Ft2pH;|miM{&@1X+0f1+gI3Km@3;$2)@Fs_XQv}J(nL0XI^ z1jL_jl0b0(g@IInIRUPX71!p4Yg<<4=Fg@D*9U575iKR1i@P(ZuxvZoJR;pFUf`6(xAMHVRyu z0`3Yd_@q8u&$t^=xJUfkr#U0ILi+DR;U4rINZ1jYLqVGyu4mkhY5(by3_NxV5w7j; zgYM$KM>mLB-RB3_6XG5K_lW=Y`Tv1%!EbQ6>~KMJaXxdYh*=(dB9+d=C{GSHNo*0A5HSeNI2Yh>$Bv1~LW>uJ=2JXb9Q#HK@e17v2CV z2ZKi3-~dDG|C-^Zm)Il>z{C0xrB}QSGqcR0R1fqcwqKuHunydY_l}$(Xy7nVl;J)r zc4C>c+FmH|$UXEH*_?MYSAym4kI7=knh-~gH4fM88ZVG zo}x1Kj&_oJ{wrAZLX;7J=5CahCd5S5k(&)+n?XhD0u>*>_iZu}1`1?6)kqXWVw)qm zh@;$=d6i;|?(R=Bvs%>mx92V8%5J_$f-aGgwK+T+INgujAXo!1x_q_MJbrTCz)CF= z@QCSxL^Ynm!#LXe?vgY~5U7VC(UB{0Wi;^1zXocHnOJ4*I5%m>ux4pipZAivM#9}E zQa2Rv3()UJo+OzSk6b_dprTpukgF!t5o%}tGc|DgY|%x0?0`v~p6rgi!DC}bR?2V9 ztjoPP`v8bPQ=u4luXqq>Ed> z0vo{_$!f7nK95a5j$UZB`x(x~2aYed}RBz?ZW()>s7>12Cwbu?_tSgQ?sGgs{^tYMY-3H-XyrJoO~AFesf=T z+4dJPp{T~9N)(;~Du%Yj{deFmR*oFuDQHh*26yg#|JN3;V)VysDp;QcG%pay`q?nP zQPl)RD4f|rh`3qm#(ATSkj03tGEU&>)$HcwCavBOg5pKVDQ+A5IqQ?^oy?-Sr{fRa zA#Dt_M)4P#Vg8|SQ+(dR>Aj3~bAQnJ6Hseesqh>`!R>a&S_KVZCY7QuscC+e?$w4j zbu|Ke(^0s}FWiwq0oIYx!pAyZ&h>@Lo)3L-!0%|i=f!9p39oi4N<1PgsE0~djrDv_ z;WczSbtM>`@qW>gZ$tGGn*Uofp+ilwD6b6!6uOaZjPxE8&H0#rycL_^$C^{izpnOs zB>OGP_ud%JHFVq9rru!io6eMiK6Ch%%idXHrN89&lCaTWQ0bed=tl6gr<-(YwJQ%7BoaqZ9Y-Y!U1ox^VPFn&?7&?ohDzukuJLv z0wF|GemLz1rw685Lw|m7V?OPxT21RJ3oWmsiB-f@7gcb}MilK!_qOO?tfnQxu~0=K z;4@lX3^adYq`ZC_lS&XJ>I!2gImyV*qk(Kzh^cxJ>Sc)yKj}Wrq>d;aSw0eogeN;s zKxsc^0Tks9zMgwdL*$FM6*?sP(>ZrMC^{$%HCyRCpIX>m&a(tIm?aqNy^pB(wST%tDMDX5sVcfh!9Pp$p*~el z;OQxfWHv4c&wAR5mPv+Ty+pamu&nsKSi3}%0v)H4-jux;`SUrg$4)uM|ji<^e z2i0mj>oxii0UsVujZ(lbMl;&pTJ}Xh5goQo#H|z>0`lJFsI-=^Bzj6vOR*nGl155# z$fMP9x9*^T#BGI$bq?x0P@YClT`T(VrRXbanu^kOpB=|!486SdhpZ5HPR6SDRgU4< z0Rji7kS<%E`)c?$Dp?)oVW>zpj2sSoQ-irOyW+67^)4Lt#`l#zd~nkhdSmq zGcF)4K8g%sWb`lud}5m$Y2x(M(N~T}118~^FODWRuXkTlgU~$PiNc+2aa~+~>x4Epp#{WiisTp!l%xae8IPhlg=k4~|^V=A+D^b+0taDUcJ53M~{4 zl*mR~h}~aoi*qra)O3tF`2l%xcr(#ppG5SxTGF_{!4V0Pjtv%yg7hI(7@0jLn!w=- z1s$eNlR_pkUl-Dm*LeVnm+4l*6DjA%pdr$QIdiSpmuta@QQTK!WC{`E&TZKz?V-@! zKlTfo3cQ@U+wML4;|{rh@DhW;zGHw?@96sX`;GrBveq`<>Z!YavmKUne1(2oRkM(> ziNGEH&QdKluY2d}#K8Pu5H2aXmqH(g^i1-7(1mxEQ2}~2$j?T6K($w-OpM8k@C9x% z|31ycGy5?Tc8r_M1N!%4Li%|xdExXd=h&R{ZST-vP&%rD7>%tpaT$5Y- z+tiF_nh83f`j8UT>Z`|y#WRz_TK~d%KIq-qlVhHfmmcdHJb*`sxw0hYkMvw{n-!)6Cs;9%Ad$r*~d( z1F-JRjdL*@8$Kdyz@`_OLX6#Cy~@hjZ`G40RK=-?T$c8Q?z&9DWl&~LY(4sUGU3cja|@h(k$w{rZ|bP4JG1UD;E44TvCHalhW=7}1@ei$@AavD~@Uu}KVB-OHzy zOE=Y0Q`WW`##U~I+Aha$S?Y`>4Q4g1Zj{TOsA^8ihBw=D1T0u~b_r}v9+mlLFE_kZ$9jx4FI~drx zHd$s-em4CsYv7bvr`lAx8m{FiXUbWRq*>O50%0!yFr}>B=<9|us7cp!#qp#+Im7?6 zwyN@X1m=U&I9)bQKFe*tXTdIE2#!|h{*J(-`4A)de4Jt9pz%{{!3<QCSFQY?IIvB78F+|ET7xj)YC8UMLrM z0tZKI$SR`YC@0MiM-_xKJIE}nOTvQ8J77jXcbBvLFX3+3z$SkL>Bfszu{|z@@j;CxI>k`0BrzzleyBbVWmu& z=kyF(_Wkgcur}fr7^dW*3ksHsx;-GIzA(bj?v=8bDk9JE4o3i(lg$RoYd);ZkrQ-* z?Nr`4m88*&fHR+4xyFo)Vl-==o2X;hv-s!zY^0xpwXa<4hgTa4%HUX>ZA4bRy^8PD zAlzGmS$f{m`jn$&YY;z~(Ia9#*Xa#e|s1fBn zq-eAr{OT4U^ur2G-Q+g=+MsId9g2x{gqc3R)QDRE@8G$PWU^IE0y;4Fule$x4f}PO z;C-JU_WU;hmn-RoG;Ka?!-zSpvK&rK8syC7AYc7--kl80H$lPG_|422%t8E2J zX{21k=s7bOwz2=BD(gM!`@1L?&9cRB?GN9k@ifHthX12X|69OWuXo48ta%Rab$cIy zth1b>w|ukr3*B|46J@d>m0rPm>124y*Sp=lbla~W5H41NL`xYVXmz>bEmJc9N$lpCkpXcTy$3>FmzM?!jR;6x>W# zNjscO&_*~}xdf=C@K6J?O`D1>9k!se`dCY@mEygo>MRZaGwG2`M^AqS6&X3x65 zgrfT^v%l-*RC~?fx7#s$(Uc`?-+7$iRzGsZOAHrpn3_IYHZHShtcTrYf1#=wDHaf3 z>U6)7jr^Wf9C{-8Q5LcV04LE15~QcF?^X^!F>A=|kz>=k2Op2_;+Z(hV2T>qHg;6% zL%8;#B6;46VmF*B7OW96?LGG7w;_E^QZPVaj<~nP%I$my@=ZuF3BLo~#IW`auwpzP zqY#I~&g>4CyzfwBS5jiFW&{woJ5BFl&z12qvrR`Mth<#){O%n8t@$Ob1TuQ zz;$>O@8b4ezHNv6?FdN=7O%2vwc{N*Hu!8B9`(S;OW5 z=_MR0a@;a0mcn@@f^&2biB{4qkq^GIyLfA2-*gu1Ic~@2ipXY4X#vyDN6AID(xWI% z*0$V}Ya30tX3TxWl~8__ou!WL(M;!Aw`tQB>pd8v^_;$WsUm!+o7n&9xbZ?v$>HfW z`b<)6Wy*3(0HLOCRsz_^uomH>h(pD87jq=F_Gf#5&tV_4s~x;?!wu#+ANBI#q8;At zt(QZ=_nf`#-q@a(+Ol@KQDO^CJSu~4sdJ8tG>L4sS+=KhFo&edvj!b{jQZrrnQO5y zI(`|ailKu1V~FtF=&Qi#nNpsgvg>Ty47w$2uT^$rbBb~!3~oEWEE0LNd*@ip^bwSk z=?8c2JY$oxKm478TF#TS1#HFFw%i>YPPQq+C}!f8bvoOL=he)Z#r5YXBl{s81ksJG zSfmc}y)%dia~7R%%w-~9|M?PU>+nu;V|Y;YtX^fIwQNA0I0jk?1b%e!Y;J%NRn>*C z>o{?-`P99-cJz;xBoL{z?@}H>-ja~;r9FyUSa&m20M?ZKh)N5+g@Ib6?4Zk zdaz?TSX`M#_ShGxR)_k|+w0^w(YldD@`sIbfJ9G`JZ4`zXXA!F>}Wpf5fB{yzkmD|0K%?deWj902w9a`3Y1~jE5GVcJPDv5i+kfRsz!i0sd(X7bFa->4W3l1u zkImfbo0bMlD5AOF+1T5mx6v(z>#3-xc9t~((5Josel^dBM9BlYU6at4$|hK zkv3i=Jj<-zxX@eFOzCCK+bgH*e+)zQ=H~_QWXhMqGWn=Cf(8=Aj(xI!ZR*FDIO&_O zmRk#M@2C1^D!gRI89ariXYQ{wvu4z!Ax27*5vT36fD7&y>-Ah0pqGwiKi z`3ytN=zHn()d)PL~?-6(Iqp%2g|MHHk^V^46n46%`-Rt4sr+gDLzm+a; z3bMto-w{FIwB;;@uKRd2$=v0(A!d|zpKAHCl2sS-ZRUGz9xJcnx9Wm9`5eMiA%b)6 z+;6fS$^w2`9{2*>dWuqCj`QBgw=;tPV-TRp{}qr*jj5iflTAsqXwDM>zB0c`-3$|R zLvN91iX6f-s6QLSF^6B%=fhw_k|4p_V}J-7%VzTJ^G0`RD-{>SIf~(Td=4IogFmz-=tzp)`fT4%#syoc9cQ1 z7zLlPvJk|Q6hdVNO2gd6R~mCUgx7!G|2sLAReA?Nv>v`#JLeV+e*)R6oN-Ki|8cts zR*0!5^B@YA=A)u}xVUk56a-ja+#qcvC_fKqUy3{-)T~E~_lY>aiE>exq%1z1R8bLRmakq=G*g zug#8(;wH5*P?B+ZvAmciTWo;D6yDl3LaAwaleTG@Pi1Ij(R+)(sWH_!I16KUU0M3I zFHlwFOwjyx%>*Qfikc~NV?0ad@8~JI4XjgGPZJI^F!4BtSy594H|AItm1HO&4E?vS z*Wuk>Gtb&Ta&~~eVf+TRVT=^_yXTdn9j4W!uc~D}N;t#+!A{{CBQXF%3bEY$A_nWE zFQ$1R%j>_0Rr-XT-+;KBYx}iMcS(BSb-glt1&s|62Gyo7TId#}4!m3`A_zy0$nO5$ zcI7f$JmX{0Nl-I8tVt-6sz}N{RAhZ+P^^3MR|V2TG(@cJA>TgF^t~U@EXll^1ew@- zo+5vuTB(|qVIFTn%K_vO_9?b~?7rzniwB1Dl#b!nHTXP~vY7-R{qO~zo|g?>^JjK5 z=M`#{2K&G79cusM9wh+{a#F>==^RJQyM=F}Xf-gwg-yIx*lQfPrtTd&<(xh>OvdG{ z^m^Dw!Y90S<9KSrI%m9>Ev|T~{#@->NC7v|q5lXW2==$Nl}+tcBDNax6n5 zV{tC}C4`d#w$K{=Y38*`E$UcRnomCdT|b7ecBGGZ|m+^`!hmSX~NhX3_?We1Kc-X zS{zCgMop>1Z3<^v3G8GHWgl!d4u(47%X~%)iYYI)#kzcUx%l+(*16ENS8u~93%#ND z_WP;b-~5)1ZlhDj#f1itD)!2jBmrm?G3XwEwFik5_@aUe|Ja2n3L!B9B7Ki$7ety41xf>dv?HM04cq(ayhl@pG zKRGo1j>8Qru#mAmMAca$&q|L2G=WyA7W?FIal8@_eEC;5;1KAqBge`i6iJp|pdu*Q ze9^+PAa>e)^*#mai4@c*;wZy<5!@qeo$EU~Yp*!_A2P(YSD%dOqY>No zzYFstZcO~hSqNWUWL#x@(~R3=)nlvDX}Ns=vzs_dO0hDr`g@TiOz}ct>H&w$-kdc! zHhrJKfbBDVM-)JvScl2YW`@m!SSD0^Pq{qDtS=KxcI|~C4~jDq;t(MVu?yQ>+d9AK zPm_HKGa_y}=A0BV>ZoY$$R2)i=q+dI>rn_WD-H8ylo`R>uY}3NqpI9k)I3n_(2#I2uBURL|apyie77lpRN3 zzHT2Cl`D3KZC`|QtKOHPI%CYsLe9S~lB}||7{9x{*6T{1Vtu=_(Ia@O!n6Kpr`pBn zQu7{Qpv>fnM#+zBtUV9AOE~VKfuZ^AnDJoH!ygMp4?EVP&x0>m(`@$yjK1)h6cif?{p{Ep%<&#*xBftdNeklF?Q_i6J(o}{c=cx zM2w@SxlBiOA%M&&BW2g>!ngLF z=YG|Jo7UuA$C7BNsa2W-uJC;ur~NX+rkf5Up`Ea#o;f1v0?6aZ?)Y!FWmW8H-|!&; zlJtF@p^S|_;=qfrSjVnb+QU6yiJ@iRwO5}!!Rf&ZEjNIMR(2%`LX z_#>K0u@aArwBs|l`6k6jEs0E|#L&L=VfKt8vSZ@_hA zYSDlhLR ziAiJZ^Ct`>_y8>0_@A&)xq|O+Zx9>`JIm>eh8&($kpQTw<@)~wOL~N%!1Uk;t={-X zZbjd1UJC8k$!H@5%+mj_(Jn?I=coT~Y|_8jQq)!puk!`qrZ~>#DE$L2K`AekhaR{| zRr7d(VLj&sszmeKo?VJ#+&jNKcE*4w?a7_swU#m0NfJpE=u)1$M|1hIsmD4 zmb`ljzDuyu`KlJjr6ypic3Os0L7_GO^CJ(ySw$o%T^^(Vc=#WBxT#}x7Qf(VDRMwp znG^g)tO9X#l^2e#GKYeOY+4**wNm{8aCtbo%9#;nB#dL(82&FB7p$uIReuO`|Cn5w z$-XA*UnODG{m%Kn16B(=|AD8@prH84W3D)Aa>LOF?@k8;BrnZwiW#+%!sC2K?c*I) zi~Mf!MZ=$B%t==1G%>d4ratc+a%FE)%@k3Eav3>@-FzAj2|X?UAzB-V`oL_#!Zs@6 zJl~_Q)MbBb&4mJB??X8pgQ}WtV%KdoH;qV{ncz0x_m+qrEVaMypo*Ni z1vU(lb1$Eo6OMTHeE|^YMc3&2x?S^p0AcyQGB1f`yy5T!TTM?CAHOX1g?Ivd(gimU zWm^B)`wP z*;d}I%*oN*>%0Ha>0i>S>;0I9BjE=sK}#N*6PW6_nPXLmdtnFi=dRaNaH5N%RMW zhgOeu$A%NdE(VZ&3MkQTP_X)hfHk33nHW6R%NKPyVYq>R`c30jo18YAC0@bTj=R2mrrlUXmmV{kWSQ} zw_U&wjcwrOCq3{nuxh?YDzqRx@0aE-P_nvttj64GCFB1#&#|0^J^2=8{yjNhe!gGM z4qmK4O%zPRhocdJc~$q)+&?zG*3M6<^0a;R;{CunJAS+`G`#iaWUG8g-E1ke7U&L= zhIp-#Uzpf^fjY0f$$^i@LLfY6ICG_-id?(!2H3ArkhSL>04Zc*P`fKD(*`=QGl%!?-$mYH9Frx-G&GITL3N1k0zkVsGYn$;665xT#s*K@js zBewivx9s>UE+oMYx*1fr@jkHKzdQQoNen-bY-6GGnk44w=!lUDp*&Rq<2)#f_4&RAQ!x=U{W`j6hTLgy1Gqjj$`?78l%N{vs}qh6X~M&|-C*xuOkDkRciN zGbdY)g6}w8kU%xBgjZj@M%q)BV+Ql0so(q36gB%pLNiEjNCO-8c?l?~VnRqoxXqt| z<)+!oBCDdVITzq*RT`L}PHykXW@w`D8M#5j7YP?%&Q539_{LlJMu?*O!0DV)<5tNa zsFU&8q~j_Y+k7|?b*S)smbl){pg59T9Mm!Ec-s|MJJ9nW`LTU^IWH8jT4|KkgS;Ce zU%v=7wC+~~K7WlVmzc0A?~n7Ipz4rIpml2W_8QA3E+3*l=-^=mY`Q&r-HqGO6>FF2 z3C1vy!+>cF#ou90AokDVGkV-GANcvHdqqLxtB8l_Es2+iyqJ#-QYb>M?1)*C&$UY@=%gV&E#DKg z$p2M29dnREL9}!)>?QE`x;;$H+U%1KVrG)b5T1bv7B7Lw0Y^>@_$!7&4sB3K?7YyC zLRq!9n#4M7&f&9GD;igWEPmY)<7l^&{op-mAxy?4>b`8HOJEBU?D@hK9y@TF_{S6~ zF1>7=)1rY@Q2;4me&lp|8?UAsmpj7uZMS^UP>s z<))bIdVfwC#)g#ideGafYa48zvBhC?DZOovj;c8!t)GwZEXwT1=!Vf!hBggmGba>5 zIA)8n05Dr`_Z<}a%~%*m`J6xZ=!c9!n^Ii0+U zTfZ&1A4GyWV>f!?FKX#OFlO0&-(S)rT5cCdKDi1`teB2(Zq=LYCNf+Oa{@G1jT61& zebH|_2d11#9#VK92N1fh%tX!7PwlXXaneP`$8*`Dt`yQhl8#B;*}sT?%ePx^_Viyh zGh??D1dhco7=~PPQtvgo+FpGpnuZa=H@zJJHW|BcS_NOcO^^uHpho$@MqT$r$VZAT zU@RVZlFt7PSZVyXfXg3wzo@6(qrP`Z9v3^ipG(&0f79H3L0`!I`R*s5Lu(G}2>21U zkFu`DX_pPmDYriz)({ldH05_Z z=?35w@>U__r?app&j+0T!^w{Z`Gjr;oI3<4uDU~?xc%$;=!Ho>&b@L1l5v01qnZLH zz@yL=QSye^sc6(|zw1b%V&>o(erG-kHbL$&bb7t}l`2>#%mtV>AUjd;NY{Naq3~+q ziOjBu3f>NpUuKJ`5eObo@uD~Ij2YDGYOZ&LS{Lr4yS^|O(90)zyOZ9SjCc7{zfpJC zQKXd4aLcZc0fJr#_G1!aX0fQ)$efHq%&SA;<02kTJ`+Z|d7WGaOylh_Y7$&E~Wci^p!zcKi`Psx{AGu8ctXZRIEU4 zz01b!7~NwS`zx%U!uJ3v>x$`v?&yL)aoI0`cplG5iey^*oMrY&ZxnO~K5oqt9@PJ!vF44^hD>OgRt?wjBah^NyivX$;H1 zF#dj#fg`V1tj+=%rVNP|*k<%xDbA513v)YHy+^F8XqCa&v_b3Bvxlin?AH>Q=W74Zz(t?@Q$?0ljiek{I=E@&HUx$&3NQ47qH^h zreYD~0?@9;;n~qai35}Hk?Mz!+y|(Jy|T`Pn%_R6lo}=CHsyCd=w*4t*EF(e0Z?Hm4jWOvA`m?OcL*RI~YsWtCOdwejfo_9L zSsNK|k15rF_NzL{okUgm|EMWY*^7d&jS#0vKgE-nzSX_(Ayz9cM#_>EWK#ssRsC=a z+^;tpafpZA!IHQlgsbveH<9+w4<_4ED;JMcB?Mi%w`Qn8U<`9NECRl|Nw}Uy9JkPH zR~pviPEc@sqkiKucoR>0uqpu@@gm6}9bYHaojcunH_Jv(X$?l4!hP!~nx`V&8o%EU zuQ7e!;%6`zqL_t?AsKY~a9m4u^V{7b#P%&Bq)&p^;eO(cqOqfz#DAKo~Bz zRZ*8-NCvKQ91hf}`#r*Z%XLJAIYP2#T^sLu#!{ceMgtbqh5abvk=pMHI*-c0t_YT= zF!`li*_SQG(rjYdjbpvY8V43<=Mb;g3f>M^)Ep;lBGVT@Y+YtAC3{+0^jJEf+`q9z zU^6JoE?Yn+akXnVBz!Jg2wsJ6j$r(S_Jr1Ftn;!L~QVZ9#xA<3-K{1P7&=&+PCQpdg$7@=Bk)_XSif))2!}ek~@Lyk3az^ z;jDEQad`DX=R$jros98P2FXUf{qUj9lmhAQxv06J77^KkJg5{yqWtef3DNAnEL7fA2KO$ z$SOY%6zb8>&>p&aLO3#UwbD-6MS8od^(dFw`Q`+oaZ>jV$kz|82V1fTG{wpM+LcvZ zz~Xz6Q#S+T<~ss~4t|DwCSb4rhat`DohG2y9{FMdyQ_uUGYuIhlL-+ks;Y5gr}Ccx z7061Gf=bI`=xuB@nH^Ucd%YBMaFPf|ZyV{&($6><{ox~(H7c-1nKYiA#38)4NN&+W z$%~h&qkSrmYM-B)AJ6>KTxmM{4I{UDZ}EpYg~JAq!@nhtKTVYD0Q>kAZo&5ZffDF^ z@Zd0)T;(zk9-KvYU@0}xRx-gc=jx~6)bRZAG&GY=W>i_jKs(AE7^3Vd4~uL@^`+qo zUBEWHVxe56WAb_5LimkuAnXO?;}@?C0hgl5zMd2J8t3=Z%A+E=yrw9cpF`AdH0BAE z?JR_5IGrgzidfE{V~M(ZmP-Y;U*mP9bcj}>75*x@S^zRj-h4owu0uM4|0K&M`=qR} z-gB05ZB{=5&b0dDg6H?3rVR^;>L&5x1fp_MR~*Q?eLdKfMVryVa?z=*BbQ>0nL7fM zQ+n#3@YhceAHRRdpzihV8lbb1PBi5WoD9PhtHdgRn=q^59yfORuc#j)P{)4Rh2^qt zjAwL}i7kB}ReN_ei-uMBg%NPr!SQ{p=Vf8=wW6PB=h2snfw~4Kxz#Yg`v1k=dq=~$ zzW>5W2ubu3y+sMpg<#Z(7SSRE(Mv>cqn9BedJ7VS(Zdj1?_G2eohYLOqxar9_sIVK ze&?J&&R=J(_kGuT_gZ^x?(Lc9p6j{ptA3vAgMot|rpysZ-Bdg{%52?&?N(tg_G%Q^ z>?H6`>Ce(=$BUI41Crn61&hdYg&-beuX{NXKF7R0l<>RMT*%Nz$-uF3cQacNBU9Qv z)-oV?_^D9lGnQ5>Tv_oIz|Fpe)k~*sBsG;({gB^tj@FAR`l=P%0CmhB$9q?Qx5dVk zO)w0exrpMU-j6`dkS@|aHR;-bE1^54PT^Fdh~N3Q(?t53_s!1h+}D9DD}&*X z9ZJpE!D17S9fq-hd=E;Vk_|(xO?GNDROdh{E@C-cD8U2%&yFRQ6CP0q(A?t2&R(?U zQrga*;`q?LB?hOktzjt0VlvLS&lcAfXOstvk-r0VOsfX!qWIsmP<3|3F}}U7($2~* zxM$>8|UXnHus)yY~VUE$QBm%HLO_5_b1Bp!*Z*nVgi{lj}2y znyf{>ADS1zYyXdJ-zxcvNbNF(*IuKf+cj50cN#pEJV)VqJ*YV#n==27ellaENJ z^ZC;0;Y&liD11PlPO}{^PvLH&d-vq9kk#!=lma7S@<1vhVmVTX%>&-LV|l{~??;my z7#P*ai$bSW>^{0xmy9d~kM9i0sGKyke0Vz9Ub*BNSPotV^N)@*-33_{9muKx5WeB{ zX$31QPLWJ+gXd4x@XPo7Z<@I4+*m&!nJx?*-0pB%?sB}enqM-0yU#{xBTVWF&h zNh1jssr)`(pLP#MK6LqO2l;HDJiy} zFUcQyhLJeq1zd*+im1=_Of9D^7;(ODai+W7N^fU(d2me5YU?m0w^e(U`n?4ku-JRA zd~f!r*SNJIo%(xw#09Cmk~4k&<->@+qPum2_zf}Qr`M3Q1|9%O6LsFyAeoEHk1Ax& z_h}8yt@zPV83KnS?*&K!u-T%Ufoscs05&C9NQY7gawK{#{g$txnZcUsw%?1Zv)pFr<*76foKQK zsb~fO#L+yj?4*pFj-uROw^=4&Eu-Fm?bmfmib+78|Fe|9(M*rK}ooa0lI z=*)3y|7W~dFof_LFP7wppXoOP&=5t@4=!dgIwZ$#8A#ML<=0VEe*NjO`KCn7nXg%< z5#&h($91-mTgwDk4atXUG<*YYWv8qPG(S$<-(UC;1kK;&Ry$W}(0-*kRS&CurBRH+t&{T9SF4 zVUjCU(&FF+S4wi#Th-$egS@5+n6+unZ;=FEN643FG4@xhPFha@kp6W6!LBWeqTW>Erb8TH=AHF*)kj3K} zqSxqHS46gVA2%%ild%7$@Q(*7jG^%GbF+p8{sJ8NB)hcJO(ZFI_tDY0@`ea?@b5Op z*pbRQRG63cX{B15zQP2piC`lD9~m=ApSEB}`l7B3beh~P3lfAkyS{>kw%M_0klIla zKYfF)LB$0}pZ@)H@DfL@L}x{`oJKqK4>6U)W23?c324080lA&rb)6}^g7;ijm+l({ zK*psvv``RLnvFS8OCy`NaePQo36Fi`Wj=!39l$@o$Tx^DldV8n72ZbgN!=Qqr+R0w zBcz&{vYF=nWW8>q+3}2)@6m>egVAN#{!GbJV}O@>-+V0~oIqb*cG$dj(`L~4+5|S2 z3^Bl;&Glf26t@0jU)oJg={nK2790Z6)O8c)3Y8iEFmA9LMHYTGd%zu%CYahjfrV|z z0d@T~s=b@3UH8c58T?d&VzwLhe2f3Dp?d~`c)UopW+$uPIt=`MF|*z5yK{0P?Ju!G zbtABdd$Z*%^a}5tO^NisVlgOt0e_VF+mn+xXmQl(RxKo1iCZWgIrKBZrOuM;nE{s!7(_N0_5%i z+Y?5fD|jkv<7Z}LCvO(BisEQycS@vUrOip>%;bAk5q1p_NjL)f(W@T zcq};lGVKJrwhMsBmL}ZH$;5Z$xgG|it=q5e#geM2DTL6g^|CKVL?l0m%o$?#i0jO+ z_LTX>E5e(tBjC*)at_kA{y~NJA+?Lvm@6*W{Pwrj!o=(;=ezZDmifuA-NG{^B1lAp znxB)^qz=QQzQK)dMh{0Oebbx|TV#;FB2Jsf%c6n*>?4~Kjtmt_u5OiA=ZQFj;CU3g zOK>o_*_{dAp+H4HWqpUB0zJYp03-T3aUYCh_O-kvPVg!mB<_|C>bwu_6pNS0uqw0q zc~DCU$Bj%TKqa2m;Bg0Ze;+H4{i2Y>+qF!@N=d=EWJ~a4cesRuDZt{aA!XI^fr2IW zD^!?{FVVXRH@M0i(033v%qd>EnE}HbB1n)x%A@Fp zMf-!fLq(U(wZrT#`wKf_^SjVu(_hILOV;uq6?nS6Gb8ztLhG;i2Lz9gHKR+t8UmxI z@xS&c8@X&Ij$Mx>vsJ_&X%jZIlU7bWxjuUupc-Up5l)<_&NO=KT&x?@kgO{#X_-#? z0GvI_dbjbT%N+M0X~)!=Q{AWJ$A^Y;mYd%RH?B>7L4-g$_p}kp@p#^}nX1|U2|8F_ zk>W+W&xNvpEgoB=C8_V|NW9Cl4_y&L`27h| zmq?b{KBOJE`|YZy+U*TgurV&-XHOINT5inXuy+J#i=il_y8%PP-}L#Ozq$jhl|zRa!Vo3{aw6P(RV z``&5*`WIRsgZknbpS|_r{jJjRvW~oIWooMJ#|3Vli5C5msM*W*lP{mo-t3M}KFqew zL5a%tUY?jZjLZG6;I_xh4_Dfr8`@#!_UX+J;mqjm1DKW$s7X4Ses)Dt6qEhwpD`a6 z>n22tR&s^@y6*9{`Q(7^pN7n_flg{L-mM=q^%e(Ar?0Cfyt*~+3-N!zJ= z#Z*>~8S)>1IpFBCo6mp(q2B6P#c4bCh@nVRkN>bKu%5wNpJ&67F6|S06zydJE96u2 z;Yp^yr7mdIh|xjx`JncewA$NAP`wHdy&u;)X@%t!V*<3@J(nkJE${wSR(iaM%6&4m14)l-H~GoGVO>B#-$6w zx()hA$01{#IhUVdJi49Khx|8Ee3{3e?Ry`+k1Ze3R(o{Qae@TrO9CSZ45gRgh&R7d zP<+V0U^&sy+0MB}>L#DgaRJ^z6?r1@*sq&1aY4)zjjnkjm&B1mk=C0u=AHJ0|1}a! zOba~{?EFW7k+1CqexfZKVBa8dY5*WOwe$CXR2UNajHlK7jIc=hauzyXf5Fdo+F$F@q1-sODV@av|QLAXeu)d>jZO3P#Lj>mhvkWuk<-f-Df3NSi=hiR(w+E-UfBO%1CU;Ve&{cPZ6Ep10Q({R9K0Mz7DXu;ITpM& zW6b=(=EaO@TXL^)`FPX2`j19)e&jYrPhh?&qf5%h9(DEKL-?1BKm3`!!^Q5+ZPKuI zJht-MdsCo(_}o!1Kb8x(v_ghI!iZcrt(P&WR;1*b5DAn8-wopq<{kN4<1(l%eF8Vw5>3dxFceq%-_*fBvk7$|C&V1w=h&<W}yM(-k(*@$v!al9ZG$ zx+X}%Is%NyVPgtZsYYL~5>P+RU(WDeyhp~#Y%~%DROguhPb$@Qy8WzQZn%wlQcPv! z*r?#acQ7Rzv;6Ya`$TlCIzKT-N3RBOi>w5XDV=0gUj7lSay*N5JQTzKP(gT};RVNW z0MC!tM_vO=H4nupi)E~M--nniidKFUV(e4!y^K3wWCT#uuP$B!iTM^udONNY zgWiw5uj~ZaXjI9MtYz^_zD8Iw&@HGW&tzS?%Ozx|!4IwkT5mRHa`f7K^PvFci%u#!WIM zEPyCF^+=ypq;h*LDvQ`XgA#eWG^kBTw2qSNE%A}g%cw{pDYCi+G4-m`?)8dv z2+)~Z9-7_#*51ZG@scE!2RdDSY@LX|Dd`Ro-$`oriwlD&claZ>ZfqL8ux|Y3zdH8H z^h|ZIXY935docXg$jJi*YU%+Bd+iQKorxewkVUgdw0c^Lp>~Iqh$(wZ`P&KBY*71n z@C5%cMW&3(2a_4&2&53j`Wvct>+*xJP*;^f&+xD5J6pktA3AfXqA@QpHr%`>=p*-a z`pI57hzw*YaR1Zxr69yw&!3-`z-wZIz*)@jXIl?vv^3q3?!-S@Wubo4uD;*WTCbk+ z1Gt`KEU)nN`61idH@!1Q?ZZ^CdAh^fxLv{|a;>7vi({d}RM<#VuO zgKdv_!aNkdvoMX-#Bc}zAt=8aXtJo!YT-*0ndyBs-a9bfciDp7ngMqi?yY4r?{a-h z#APGRsu=IApXrc)dTx(63`4Ln+TV;^v;|}c zYW#8|f|wSVrye5l^0j!9#9mCiwbgyxbg2xKf?v+kr?Y=dPN1E--4Wh!Sd0S_E(){D z3@@6ubPF4?EkZl-H59DScD6lQlL+MRp=KU-Bt$xiZ!s*pk!cmp-x&z_JICG^gP-C| z!EC|z;uKuaTQ}^yg#e{P06^{CNG#StVKzC01KQ6b8bA?B0cbakFbE+xUt-9p+!zLz z-`E4(oi&)?Z_%rF(W_{B$QHmbX>$XtTA`^XMbOlq62Pd$|4aDEjwbx1La$N~0$N2h zmn0tgdTen0FR-&^VsJC`D$9TFg^R(+T67bv=G+9Ix{ChP6YvN6x4_NNtLPnE-^&=R z3Zl301bsa%xSp3j7@npGZiZg{f?h@6ix)Q--h>5K)3DIjgO9(lhymzK(O&_*ir&HX zy{?6H<3C@_1zg|c%>eb|244Zas)61C`d&@E;4*6T#?W6Z4If;eKmb@u(O&_*`r<$L z`oAkg!1p1bm^+cBCp)I$`|X$}?ydjKZT2d?j;($mrs)O@!6 z>HDG8-e+EVJ_mV*Q|HH57mZg(;Ip!9PkQWqoJtGPM5CVI+Fkvh=rL!5=mBJvX!g!B z9?8p{s1@?Y<2K}s+RK*{UkopIwZDAtM_6io$6@c)pS zf8hI{xjenJV?J>&kK)DmCYJMpTW>HqNj<@e;Ug1!b6p1J?1D&Bo`hkIP| zMTU^inl*jH*&uO;z_z}O3bpOJEVW^rlzk+JtcV*@vN5-1DHLvNqgyxyZ6hq$TW-~e zE*+{sQK>s6FX1I${)j(PqSGA*FV*go7g)QP64>s>(C8htjr2Y=D(TEEOs<|2G*q&a zS*y#Q#&)X4a^58Dui_)tn;UEm_x7wlOe zZD;?qWtQQzQdl)Au=Gh+Vv+6GBk1EcYDyU;9c9a92&#(iAPCTTdZOe67u}^33t%w~ zI_?=G?oa)t99KN(l#jUN$@L^21k5!APuM^l>>x&-opb`!UZA26)9JjZQFojgh_mwe z@cTHw7*@3z#ve)93`j7^CbUDz8aVxb)5wcepZPBexg3i{mj>8cm+KZ#G`pnOfk|Wn zXRL4Wp zp1E@>lV!|3Hn4v9hKMHf#*(ZQBPFM-qkNe}d`cTdLtZCkilD&ZZHl&Uwff;D5uw3S> zY0)KH2?=J;j;`*R!xXs}4S$g8eMo+3F*PpGy+s-~gxTh0(}^WMz{3up>+-c z^LOsrc0I{n6zL-YhxW@6C$;YQL-e7Iv$m|n4yaYm7-^)Sy`JGRQ24C)qN#d zXu10Xch*C3$bI0OP{#chrh{7o@sCTvJ7|(D9wMTDPpb3sEoi)<(|q* zx*IIjQHj~A+xewIe5}!CZ!XgG;Z8IoB0$g;k zHn}tEA<}*)dg$Z7W-mnL>QA>{;Qf^#d93aL zTj9#i59at5Kdj<}i7@ZSojw^iBQ~!^c#FF%^)RPV%#1+n&I%!RS0SBjaoMN-)EO`e zTuP|+C+p;phM0;W6z~eERp8=zQH4@E zy6t~c>h0Rlu^xUzV#J+rK=u~hH zzzD%ORa@XAtRurLzt&1R{}q-%xvg&Azxt+n^k4d>dJE`V+|fGFH$V${?Ii38%J*VO zubspqsFLRvqn(7Vk{yPsf%?2ZPl3;{W^vO>WLRo={_Vc{JP<4zvATPOsVloAa*6=!UeiUC0@E!@)^YatZ4(CIOS(eqG_Adt8-#|7JRu`cg%xK-e# z1)|@k7tr#j(VZSsEPg!}(hHkp{UR!#n>|+LhEVq+4Z7ynv3BZZdMy|P=+p){LI!Cp z4snOYbx=uKclAbJ;7(_}s%BqKP#RwF7f%B87sIsnuJgxTp$(aQTly^YpmXpe z8ow4~UmQC5zstsHmo)^taCw|1Qun%9tf3F|{nwZLZnfa9Kz&Bc>c?a0QNE8A-64Z^ zS1<4@uk16<-fS&@b_!h^Z-iaCJg=K{l}{?S@WDiOw=cC ze2~NMqKr#)sHSyP@Mo~J0{>;>0k6MMJBLWjXFFPpuo*pYU9vt_0U2s;Z-!_&>mSQMx7nihMt`p<$`p;zoe9o^KrQ4LOTOS#L9=+fZU?YC?unH*mCG(~{^qlHPZ#N|PSA zJN9VrE~UnS>~qA(`^ihj1Lm+?U@O)E0oj42_X^Qc-q5uvlk4KK2tIP;z#C>LNPp`x$rcm2HGxK!&XT?6+PS$8W#(sxSUep1lkyTyLO8_oZ1 z&lT#$JigLrLH`yX+bgl-rqI0#6O&^V3gwq1y=PB24G#|7F;6cRYPELBqaV04>12`q z;7HC0vE=*~>e)b6DKqm%*<#Giu-%9M7%%w(a-bh|TK~~vMQP>gtnW$Bk~-D2X0zA} zQisT(+jtp$1P`5b3N*{$cqCR0>cp*y!)I-f$Cp)f)yR=R>_m~mnxB}7-S3WTgepx6 zw=n_yQoO?y>8nFd2*D99wAe24H#sxQ*w-0&Z?s)v?B>dL_2JRv0JHJl)>0`6;EIgc zc_>>1`$r`_Ows=&Qe_E)tF4(N+E;Kv)d}QNv{4cpm1}H1@*WQ;&tluChZgm$K8Srj z{fHTQ$|%NdNmy=v$a0vCU-Py|4y7@SaLE;t>ZfQduZZ$HGYK|*xU0s9*bt0MPvPMo z66_G~kFK>%o*BN$h+wweh~o%!NbVpMxepPwQ7qtlGU}l-^jlH`B_7tUsfj*HvTfxe zW!PpO3jK=>8fB#)oVlTi6TMlEx>8DUGO`BhbN=yo5pzzVf{Au3Y;;Cm67YC7yVQqy z7`L+`au%mwgPSiM@HTE$pL<*J26b~2Ih>VkJ1MQyAXfJH?N)q-Dq;U5Kd|-NpJ}_N zdz80)yU;{(Exu;69%B9+@vVP?e!yLzwtn(8+uo&>lK7}{zBf@6vt;U17rot6l8Lt| zn)i7^U_RNE?rf))6C=9MP)d+w>3RJ_+%3JzOf6kulJBQ4+;=R+X_-?xBVtxJl}|oz`D^KDp87FF%vzCETQrL{ux!CXvLBb$|6CftXXiE) z+}{)+#Pij0H0iP=5p^2v3e~F5(=vEkU16|di=_(~{eR>?fWXW7kNgwQUJ-TFEN05| z5)>6PA9#O_{|za?ESR)C15T9?vS49bsmJ$mO)RiKw{!fpE;J&yy0_)2_0`MIM>dqY zZ@|&rCzl;v{Yj^>kd@GWl$DdLoaq!j)bGk2g|sq(vV`s?-lJElH-Ew&>Eyd@TRuI= zorn=8n-JDz+R-oXC~qxdX3r@%P@ji#lIh((b{A3@j*ivO{cxwl!`0h2y4Ix)vcp!v z&n+@@p=RV(qDtSmX~9TWyxA6o)yQ!G(?zZbepsgk%pJcRJ@{nJ_4epmGK8x}JRpl*QpvGuA+YHogMTt3!z^kzp{ zY>_T)@?fxzi7CPA>0^3ah>Bl!^_ecvrfR&BAQ=x_v$NC%u7Icg54k$TB+%bx@imns-!1S3C%@5K47 zeJOw}5KFq83WSC$u%!O_JiaAD`R$&T05kp8<^bc~gGkTO`w#^2BIzb^@6{LBiG$x- zfZrdbrJ?Bai@>;{AIR@3)0}-CJ+~N_MB}F}9=+8Xdw0md4;>YeZTL)4bg)}#PZly; z=4rdfNnBjw=4{Patb!fNHjw{HxSP;3cBWzfsZb`!-2Cwx3>P;rTYC0H+!D)7r&|fPHt6T6t5^ zOqBYgB?z}SILRyTx1nxnEI#p}$F&qvDN_!J-2Z+ys=W7e^(Q+zK4OFhmZi3n$jN7X zXd_7|E+*et;G_#UOi5kgneo!Rf9kRDpU>d6{3c>5r z&jMQYtmdf|X#<#2^5>fwu!Jo|MVg^sK$q!_Ockh3pt@C1Jz5HAWdtZMJVpKUzWZ-( zhnub*t?u@(`M4SH+seBnuKOSRJy2d#f_(^dc?b5$%6tf|%lXM`t+tCmm*?u*xBQMeWKkYdTB(VnD@ zpep{bdeCQeAfB8hi@_^A>2&m-zc`nifr;!;o^U-;PbGp4JIbIVDCu0=^Z_sk`mIU9 zd5viwf+6+$XB7pN^eoArllO_YGz(OpUpnzc>TTx9!&2D-^LTDeMC$K8tQ~aa|5uQ@ zsTG+km{PFw&1_KQQZ%F)>IQ<0Lu$umH%!v>(r964Puh-O+u`m%W!EJxI{sx1)bPaL zkkX-Pg`Nl0xRkGrLjGm5Z$9_N2O*v;OE<&w4I4&~1j&^VQQ(P$FS{ZNr>q^ zS%cHu3L5U*peuOSW%^c5quUqn;La7+=_ikbx)kemOX6Wv{0|@9%dUKfcLDKG>1CP- zxld#eyOeGU@Gd}tfT>M)@n`+HkVkLf4|X@nWoYx~oMhPPumA(JfiR@Y1r)$upE>JJ z=&e!FST;`ed)8<7xXK5aZa*hvbZo__o~vK5ZN-l+^>-&}<>=0U7xz(q*NZBnVwve; znPz!8vuI-A&s1$Ix7y#%Iz77*_H?*wWyr=WttGw6=HU)OXY>r$QK9MOSK}MGWL`2V zDH67d$h&?u$5XavWi>oKj!LYlEKnUAQ%KptLe2JKmFG6fuJ+ovzQO&s#+o2i$%tt9 zTWkozFt~C@yq+$#u`vVhDD)YQ6^;~(J1l0A=5K{nC*x&#MGwJU?MNzh+fc;LC~)+Z z?6uZ<&)2c1pG%&*jV3!!*1nLCL&3x3ldOd;R@Ylje$i&X=@x9Z-}oZFEL_7V0g%FA7Eu?>uxAcasa)`?M|7-CB!jP@Q$NZ|wKF zIEL)@8K$b?aY*A6_6)-k^2fM;g@kj3IV-hxY%6zGr{-Zu+sYT=9oK1@7qO7}E8gi( zpW~C+pHpN#_NzCd`RRTBJ3>~0{q;ery2lrpSYJH)8j8QrG9n~=XX`bO4x7Hj4?0Ye zn~NqXf*6@{;~!2tt#%X?i0QsW0x6fF>N_%$f3OQhBwd9^9DxfWjOMSumk!;U4PLW5 zKd*~r)olM5RiEBw4m#JM^x#;^p+fgD1QyYK%j@xjpG%NK zXAdLoyK3{@TRA@+%m4gGHhZ+M|E)6InhFJF(7)M(V9qW96U z`03edo(5OJ_)Xc~P2?N>$+(cNh2>6bc5hyn-i*=5536hf`$Rgz@UC@82KCEXX7;d@ z$L}DIKgiyzH(TNR?8V4R1$ntT8#A5dbeQN^R>*M72M$`Yu>I9rQ;e+Y^=2}?Y>5q+ zBbOx^g{`%!^r<~aI_)^~Gu+PofZiE>6s{@Kr8tv476 zR0P94=X+AJcgor4UFY-%xuz|*X*!>y#3?mSKlV1b=fB`zqrLff3~+I-4>4*e5suAO z-rR_l*=*SF;7j&S?6gDx{b`ObW$ST#V$}T~{fT(>ul~F*kX0G_S4NyKzyQ|jI`f&a zpJ2#%USpAdo|l(JQv9xQ4NrxuAUTN)NXFinO~!?FiIsI(zwmy^=@WjYas0ZiUU5a} zc3CwrRN=lC;qsJ|_iw7S3;s)<)RsH3{)3=3d)u9$#V}2cb#WpG?yj=8VsWAAmfnAF``0kRTL&WOi@xkgpjOgr8$owGRH98hC-NpL0y8;hC z6)6xBe}jBxYRwy_BQT8+#hZCK*xtT4Hmwc_&NAWd;;2O9U*XKTiw`yE2YXujI8#$ng_o3uX`I}11S1{;h$cI5&58QaBzy# zS6)*Yln!);xTpjD&7+{J}*S9drM|C`DA>4V=Y-H;*&R( zfY^|0x#Id=SXRq=F$L= z=iv(8iqZTk&+Ss>=J_HNe_9B9Ygn2jr^fuc$9$2t9AXEaii3DTDn}4}`g4VB9;U>N zBpH=BccxE8g}ggo4M^XHLEEgKK7zhpy>o&?WSV)5q#m$w;?zbF_q)<4Gfooa<3N)I zKof630+7{wKv!XGNFd{<8b5uet8L_>w!FKz?r}WTKi{l?B~EIc3BTh`WrI#wn0h#z z>{9DMX-qdS`$0G^p^UP5i_}>c415$=l#I{?<;U@fSKqI?XRq6zfttWID@>Q#+L*<5 z=8hV=7mSpP3;a|h-e_?%6x*<2&kly$Y;YP=+c=5_J>>i9FX*BCg&eMM-KiArI49WP3(RcV3W0y?aef zjpIkFY@f=*G0^Jl^6T!o*6vc&Qz!nK2Mfy~kNi5NT3b&~P8iPa5_@$?(DRYAxXH@o zdCKd#7Q*T?=kO2$seKlcTSIT-9g*A5^9Ep9s`eFq!UrtSUtZMi^U4LCN%!ulHEl%1 zmr1f(x!`pgs^*a29$0Blp}*0P-02Y!^G=ehpqrG{yjW_ttt$h!G_uJ(pLCRDW3i>< z*`27sF#TEe{!~AGM%MC3A-@-AHAI6R{61sPeEz{rb^M*Vvj`7SzL}40dFB-4G$HQv z=5t4+s7|;T^o$>X%<&=ySWy!}+NT-!u1lM&P-4Z@7f97m{`{c3X9w9DXCrhXb8h4t zL#quF?R-urO3|^LWyPGh4e+1}nvVEdNf%IA?jwd!k5vfPDmM{I+c_j7qyauIB#SHS ztP%6NLL;&hm7wX-S5W)~>~iIRJkZ+3ngDN?xT}KTx&`(%feGAKRBHmubW&x2{b41- z!$5t-pJ+DZca+5aLcPhioE+ZJi+YG;fGNc+9u>qzkwENa0vnQ}GP+VNwk@E- z6hm(xXB*G3j8YHUY?#w%;qNpkAfWa;n4?U#@BD|!ijL+!3Bd;@`hL$n3tIRnS0BXp zFl;Du_O0ArM5^m9D5nCIB59t>mm!(iaVApESb6s+X*>aIN4=mAG{VI)l{L!?^|u5H zzmbiaJ>Ap)lv>yP{;)fz@Z+{vLce!;_5oBSHm9&vJSsk@btsk6F7kwqjkZw!BE@ZO zZ3QZi*j=USI3k7&A(gu0{o1gWp5UeO=+K#wVe=Yt{0mI8YZOXNvR{6=+cLsx3|^43 zAhE~HI{dOYS7<3x)d58a0py-kcN;u={s(|i`db=v*3*#*c!!N1jV(g(t1tMtDr8kd zhCht6T9v7;HRLo6gNQ~m6z*36W4D#m60su_NDbK;ti2FX+oV@r5-_w0B``{178G222~W>*1INU*y4tO*04MU6ZGChg<0|? z&wEfTbxOfM_kXfA&Rq>Z7 zYEe`%Eq6)uxsdntDHtqcO4X5cdgvm4RyJd2QD0+dT^iAlF>$l1TcML$W+S7~sLjHBsVv<;(vQDX%}x53@1L>ZHsZGSt!I2w6EKRIh}M zZHDD0EB_FV=FgTD9W9S|mlM`i@v=6V-YFIdpPTjfA)zn(NI(t%X#{8h7odN>$xE>& zt8xQ(Q5Qh2wDHl9D*&mxVF4!20U#QmCV*x^0ru+uhnoF=7q3??(+dW^%^>J6w}bY{ zeRs$KWGwV2{nZZrQ0jSCYTbqSn?GS}`s5EB?&w(b=DA#2fIZ7-WI-#?YoV=j{wj$d*7(;D4%_%W4_Q5#`PCt@kOQUe59YZUR^Fn zT?PlX%>IS0#Y>*INOICHrKP0_a5NXT9&;%cbV4X_8U9I2W8;1%{GB-J?=D)yx>OIh zz#cb3DL#i#5JZ#Q4rmN6j@{-7W@2MoB%_Qb@gSCcP2|e5j>E|M(~bMtN20LAQeObv zvovfK#tysl^5wB2S2PK7`-`e`v7yqsiu@ow~^9Wnbg;r(WO0E6c4!X0b`oTo;yglCZM0iMTcwBPuQUCp?_fMZ^is^fT&gv1@c~x}pTg3rO=nr`6%oZISFVVS%)0=q76CKP^u7Nhy0HvJ ziu%DkV}L>xAvtY4KGe|1v!kcAInkJH=`M~P2A&=oU+Jgax$iUMhel!*EA%Glz&OEenO|C(>QtWikOKLO*-wG~bkzgZH1^Mj%2mV5Rr&|z$M$T9 zHcp2%4gEWFLyc6sPr|dM`KrTdE0s`nXn-0mlouiWOrIdrsprh3X@0;Rmf;=WIA}MQ z?f+3}XHi3&rT=a!7c}%~ZT2{Wah?mYbW!Gb4|%H_EmT}+)-Y-=(gM~CB9;W0f@1Ok zKtPWRwnGUA9S3~(YtJzcYmz2cFWxzI4eIu*DYPYxyorq4Vfc&unzmJorpAjIMd?<( z>H#chnZCts8ZZ}pRcb0Ph)>e@Jph0+UGr^rgdp2x< zmK~g_$IB5x9tQv6>Mi7KE|bLIB`{Z)Jv(pNuaF|(Gv2@Kst0dj>047P)Y7Qyh|MxY z^<1ofecIuj-AT!U`5&a%d`G-|v@&kGY8V?ut^cEZf)E)pgXJ>ae%@l@PRfE6__-ae zFib%!S+ohvT5$mgEiKkbm{KVa;hS(I zSN2`|>$}H*K4RiLj4#_NSDC*Ob$&s28z08 z7NB{eu8TgzEM;AWt4{ZJjVNc{plnPlt9cg#M5iJZU5r9Ib0s68`&_;@9M2Tb94@(E z-m0D9-v+Wa6YQcZ&FdeQpgGFwqIUs(kuHi(TK5$!ybcq2JWvED%}zunJ%-nADid>PWxJU z(_X8+`se*nyz@vk4uAtun%GzuU?(0-S`ZQz2FD~d1zv{NbvC>U;s`L-epxbOl3|>j zXGu>Gqlb?})#_g~{rOPpki->))wu(WVP=p^ zK5L?Fx3;vVztCodTCL-po#%ZdJ5f|sk)q?AM~1?=)&=R?>$cN|9CIIr6GB9TGsR%t z)W~}S!xHdD9Ma~^BQb!aw2W2W?O3;RE+goJS?qa#izh-8%&tmJc zi3MuDkrc={oDBq>M=Td@oJ$2GvA@*PVXM+7@o(^Kmjcxd_fLOTd2;Blim37W?`fUC z`KbTjR!>Ln5R$e~yq*-LV7QK<%xYpYIyfdli-xTg^VsYcK1*CGEPw(nOWb+{6|6)) z8{2)R6hHq8=F9_iC6XW6w@n(;3@6#C*l|={cZOHnHn`wKv%( zf)*MCjx$w&dI48}MMhoan6CU!VQ{{-VqpEO@oCXQ-JSrVIEw9O=_4Qy=(RkcKpxbr zG;gI@BwnmI$@-_(L@%`O?64K*{jPp$$^aE)`PQ;uyo(STEeieE67M|vl|WZU<$_O8 zx+pywS!=8C;8SXofu(OLUQpeS;D>uG`EC-)Jh(33GaQqX1~}=yX`Mp8z^n-bL*U70 zqtqkl&>>S{%v}arI4s=1{zzlrLd}b@9c|Kw@W3A}9t0>{zJ5A?K4J@wj?&(SOmDlZ zE|#G|)zEB}2ETq7{B%C2roomB`Ey`K;<3Q7{?m||y_?bMbaY9m$nl;(_Nm;Pc^@RL z06Rczhn?Ae# zBun_Kx|aY=b`YwWhlZ1-x1bzFXWl92Erk9~5~sDtN)mwo)A{wd_>bZyOMcLs`o2a- zZ?PA8EdwpxA^7OmiRKzA$|T`zpN{P)WHuaySuY*Z=Y70H`>g8IVNz#-@%+hsE5vW% zl3V;sdn`xZ6*r?x9{@yJim*A)z5az+cmP(ze&D7Eze!N%xKy~ZW8_?;7xMB!h2zRy z8-bmOqe0bp3nKyeAT<7$KD;c)NjkCBf16?2oUk%v?!IGUu{>&hq7j-Co{RPSq&=Ff z(lhK;1ADo++34eP*fFS^3Ka2#9~Ca1&n!A%Q{)+|Ur9XQRv_#tzREQsm zS0FV0>WkuJBXx4$j)m~oA(QH~ardVN`&R={=`bnphJQLevR_}h)c^Rt`l9PcU*4um zFg=SaVkC+eXArrX2K?Lii+yc}D zD|N{Of2tYhUr+uBcU^K;=Dv*ky{Bd}=i_T^7qjOh+kNs8tFq~Ut@pD{MK47#yhpe& z^DX)Kf-||sgdnzyckZ1jmqpNVRQC;s5e-^CVW@0;F22Di`a!&%qwhtis9ygp*PY6A z{p6%ql61^7I7eUH>7}2?$EpThM7o|~oFJag=`9*Zb&hBGE~wM0_vGy*w|*bFj8PeH z`RB8K_weCt3tjY3T>~$&aRODiiFUq;TTGyaqO0jCaJ!`5zAMJdy6-&Ab#zSLq{0b8=S>|BrYdS#|J-_h=?>u8>Ey&N|%60Nh96J&^ZbyNH@~bB|UTr(lK%3>3wf^7ETIc!ZGc))+_wIfF_I1gHENx29JEiv@X2bgeoW_U0 zF!ih5N;d|~%`26?*^a%z$|)hX#Dy(S+cxSRyLxKIBKLC2%qD-d+ALzBNUVQVF9PIBk5Fd2TE$+3$2XLG>}> zbXxW3ZE(;WpbO$;f3dUSV>UWB{bypZWVceWKleZLNqXo^!ic8$2?pMjuqNQ2{j zPSH_J+}kE33-ya@Oi>;QLzM`HJSqY<3UbD&pUJix)!G9u6SZxthofhx(zwPK3?#EpP3mlFw9W*Ws&iBfB(Uwu+dU7+q3nc{s5B6RTs-|4;RYl%q)F22VCP|p$y~5 zbrU~?A4B8*;>)J-tVPLYrG?6-%-)Hu&`Vc_6Lp;EP@(tkJg-I?8}=ga<;7RHmN_Jt z$C*Qgr|JvIUl!x^FMXz4bCO?Lt9T5Z5{qgLm;Otd)qAQjR87Xd*y#<+WM3K7OpeEv z1eK9dYpl0d>8sUqs;f3kCF8>>V|kJHoP=_|Q<+gLBt^Qri_(`q}=H*yGWpign$jS$dT zvvopnerm9Z5SA&vc)+e}7kbJcr_4v4S#`EP_`^UytjF7VI^yxg57%9VikpsMJb(zh zI80Z5jjTqTv@S{bOM#_JczmE0*`#niwgt?PppP}qFeYMuN8l4WGmpBJ3_5n1pCW9? zSVc*M;-WdH%~_?bbwO(hny~Yd%^-4K<9dY310?UHuxVWLobb?}nYelpXZba{9Ot^2 zv`_dqJQ5C*rIQyI2hCTWZ^3M7EE3I9&39B(ypSG&NnHqbcR7`9GmK~4YTXr*5V71# zj+DKkAM)3=2=o!^Oi)F}1l}1&%FXAOlJzH)G%cAFWFGcjz$UK>Rky+(O1!(J(`N>s zSQWhH{Y`kw6~=6KHBG6!wL9oX_-z0p1SAoTsezPxwl3xkl-!#PJ!u#m|-eByZTM{ooYLIAz4U z_hfciI2b1Ppm7t=pqqb+A1WAHjyoY1%IbD07elMEDtkp&D(+g-*jf`qDvmxgUW^LE z;v-bTafdx?J16ICPjY+l$~Z#h>)L_mku+strsVuskj(?qapCaE0OHFE@sMX|NnRP;>=?PX3m>%qtxu`3rSf5<`r3nvwTWiG z4Rkv#oEoWjf*Rbp_&=oZ5yV@?j|txaNH2C~f?|)wm`MGtc7t@xL9;JtqewP?a+N5|Ptstk|&%(laZwyO4;7ce=Ucn|)kof4b3e9|BHlYqmi4 zk!FStnbd>DK}qbRuf&tp41ljCXM5l~_DYT;y1FOwx9f=}!ZDilZ(6hR=OObO>4@Et zAY76uK?h6j4}C?i+{Ibu+`JmqY=`}Tl8yRhzNp1(o;-xcloI{rVft;!aG~~)?TXiy zG&~@X2~#BP$0QmTFX(d~6J($0L`3!2Lc;s-oXiE-czi*q3hm+d`c5nLtfQKLpd-$7 ze=;=1b#OH37dvL(Gco3%!fD$<6SxceoPiaH~iqlDt=fvA3O7{VDJ6I-uR#T zl}E>%I$J3%%s!IQS3U9&%zO{q25Y%Drrk>waH6K9cU-PUVH)pce@(hzDUC6*?YbCC z2rJ~xduD0|Vg~&iGlevd2$e7pvuPZkzNN3yeBF`|FE61X%jjSd>b~x|HVvW};UR|p z#(~is(=&1j1QVTEbxxN9F)F;@9^vK8F$)#~%HDhBxIG&`5M-O}zRO`1FO)KGY_N8=tCAm|dL7`JN6q_rb6X2wA31V38 z?WI;Cj+ZuHrJB*3Ga1kt1LwzY(&eyh+Pt;sMf{u>ZY!CK%M?iv14NfQ7y}yd88%rB zjRwbCh*)upZ+?oYmB5camGKok6H`eg-?6$6NeoEJ>-Z@2O#hX(Gxgq|WvjZeP1#MJ zQs{VEX{MwQ4I{P2OM_~UuT?g8D?>ZuT_aWMRhZm>fD^U}ii-W9;o)8M%=F4*yR-G< zj(~)s5U2XU?9m_}(%RxgO%R5+dYth_>{J>^*g30$O zlK!mj<8ANx8$bY)&AME6g;T&c5Z&c_)@Xb__z)b<8J#M;+h_eVr^eMKCs%sZR};(P zX`@#M`P|-Y(&K~GD3zCGZL#RGH3Kxod@?M|OBtrQs%5f9`~EX~1Y5c8mVmU@3^iV8 zpvocJz_;;?)(o3b62)SabSjOl&o(af*7}E+ImtO+I!~Z2N7Bi(P$0cf?vn!}2sldX zHH8n@Sk~K>-TB*&vfs?*tYX`d6rGuj|FZM#^K^$> z-3a>XA=?M31uZ4%?gZ51jwbql$ct|T1XoVsM6uU~1oEZnZn(61xP9U7N_nTX{91?o z4*w()9z~aPH21;9kErADGgw;45s4eaib@(5YCTPqbS+Qb3Pq1<(vAfr!%=>j;fh#-1Z@8 zAu7!7i7YjNAd5y7&emI=fkO?A9D_}P;XV4ZYuJVb7S`L9)X04w7Wek7{0%@YvxLrB zj8z>PCEmq>hxyFkR6X{hcnlr;G|zO{y6rLo@5f_2%KAy829(+Ie%hLtn_cf1k4lQ6 zoBPaL`bZ`BY(zRajJv1Y;^9%=P3+hJk|q=?wg>k(}RpANP>D@7&O+`j|qCEfy}cs;MSi;U!C6 z!=Ly?2=G(EHiiPSV7?H?8qzkl(EvH|R6LMM=I66WW!iguU{+j+0ASQ^l0Kx`Ql3a0 zC?j$AH0`K%s3Q2~3AfFMwgw$ZjKh}i4|tFDC(p=K5d5luSA|J!KY+NMiFJEF;~Z#` z3-8lUf@H}G&=X_>83wbeYfn9lI`Q6`lgH&}dqhO>h00CJ5aVMvIHJzxN@NLSv6<^xC)$T=zs-4Zeanyf?QRV@gi}lV09XMiBBJ%>86w60$EA7cl%5l}zUI{F`aDijT0|NMbkvacC zCsqnzZG?dq>g~3xWIlI$qq?48Imx`ms~du!tkEBHQ_P&&?g5el$MKzlIS1(=hhEp7 z8pbzB!4o$ERYumZEzk5y4-F>`Gv|&Uo+7w|yevzVx;mRRc&d6F4-cQuq4W~+$7yWU zG`n6ZyKEd(8e+|?aCn0^cwg%Bk@)OffN7%9C)f=*F+`?yVdhG=v{Nz-w{_~&-tTrE z^520mwu3H7>so-2B3*{;`&Jc|DlCtz>i^)3$wR!5I86D9k}%2}bbY-P`GOojZx?dJ zYUI=FmOmx^(}^`1nL#!!p;+0NTSlXOUT|##(-xzWKUH((Z_8jrs<+Q+t=&X{mH|2j zF>FR)Sq%^ri5{Q z3W>dp39WM>BNTSa8oz~;-DN7QIRpn!mE&3#?^$e+_(f%QKAp-G$vULn^Obd_)fc?^ zY|4;>-Sw^N2U$VO^OqVx3_~n4 zL+)$Yop!2C3GKpZ{;88#l)LGlYj>WCH-agL~%Xcw`#!d%pHrfzQ8^4(ozKPww z#nEgUZ!-%_>a%`3r8qUkAm3~2n~nx>c_3(is{u(PbGEWo_?9M(Q&@%=@T(sss#lCYehI*vyQkNCT&pcmR&eIomG(?*}+vNCWyG~8ALzgp5G3Z#6_G{zqOk?byc)GQi(8Qa4 zW<=zNb5V&)`1CYM!=9yGFz!n` z{;=ckIbo0C>#KNLY^&T<1hxsj0^f$F;`;U4;3&c|LG397*%aCWNBeJaKfBHD_&~d6 z2MrgY+^9H)Wd3$cmoK$zBqCe-H_g>~zv_(sCIs;|^c~@c@4psac|#eUZHgR25%j0$ zA6ioCny*}8yCEwfsw&tsD?KLwDK_J$M#F;;OFb#f&4-?u;RdAu10LW*qlz5DAmL)3~ z3!Gg~R@JX!oSI_p$!M@`d~~>BT4(pW*wmm9;94V158Fl_IgQCWojqxZL`O7hVgH7{ z>U94Ry7(b0RL)7@dAek>!ctvOlGA{xU2Ld)qCs>g*)UzG`cx;ffn$>nkD%1;^K3Q(hC+jxxS!Y0M9m}52FERV6ipMV3y@?+byS6<1t3Q8Tv zeH*`Wr8iS<8B=_DTAWC8x#QB*Lw)T#kD+~2N%7^y?j?fy+V5$sU>sKP?G^qIof(FK z`Og3Sk{g!<9m8`*7-orPxY+yf`Mwv;&6|?SCmTvIIoP2F>=1oajX_s6HJmJBF|v9f zE;gv>DkEY^5*{;$AWV*n*8NFTF^{EMV@14mT( z)4?$B=os#KNyYjoqANnGHX1=du8im>79_j63*MLtD{i)`QlZNVq|N?nnSDfB0)anZ zTm1^ZqN?CCXkc3ZtP=h|Qw>Bs(($E7R^i?$!~GfZzoBMTOXLGxA_KN{*`rW-=G9@B zE`R}v-|>-0hauk>QfS;kF!P@ZQz@A(@|sa_YA&midl>&NX$F~7+*HK)_Ep8HlT?Ir zBaRt=pw)>q|IOZzmO=esaJ~$<5RngmS8|#{1%*yAGYJmb!n!XGGXQc{PNCf=I^``^ zE7<<#Livz1a967xUd~-yy179)F#6>g$)6fJW`|x4UiD>E21)Nx(R~%5pKtUQSsHUk z;92_8@%R8q48l%p9|E!fr8#Lrj!R2aa3y-jcA~>?EKp}7e~3&!m*ZIcuFHF)#U@9A z=>^~WkgfZA>y8scmcnW8`x`1c;si3p(GK$oPh_dpxyh=RUsB^s8IgzKEJJ`M2wCCz zquU4W*}vQsmp(}&;^fcwlGLZKQ`OzQb?2CxZf#aEyG2n-+j7=7-+IYC9qIWRsHpAi z+7ng^=?>C93ENI<${aL!<;%~wAEk@OH~1L{`?O5ZU(dmpx*oFf^Q(1VR$krD?&N+VSt`*U$lc_$PuZsRA7%L;Yy_i>BQ16+?y&P=p-@mGYNZR z9z+-Czj(I!;)|}D9=!;a^1{lJRA#y_?997HPZm<67?IE0-&7dX#hyBw9u-f&bCM7m zr>)}qzQD0bF?dfAE8oVcswt2P+%f}Bp0x*Bdq+z&H$2C}YhJIPWpz!+eWfBc5W^H~ z1wC8i3kF=c^Fxo#DRDF5qHM2;j+gvPU!|~8YSXkGg5qY$+81mX2^Y+oxml0reYzZO zS}H!gg#H}v&99k<>+b@RZWk@Hf%OIt5n5IQy^dap8Z0EMZx*4ECvIZHdF2 z9?z}=pxY$%}>TzP6<51lPSA~#;}ns$;j)h_zQ*AhQ`qsGB;?S&n^UpI0>5Xp_LV-INUoRtQZOXGaZ+x>4t`tZ^j#}P|RAnLtHXlj<>drtq4Insc_w>yyqD0 zxI`P)ZS>q(;8}a{vBofoTA$rRgW}FflkGeUg5=}sZuA1x(t0p+ zEqJF9JO9EpGV-mN3zr34s@BUfq;(pB*s^tty=a4(zu0o{?aHOP>k#^>8V5fiP6tw zl78{$IJMIGUKo6Reix?Wm@@rr@=WM}E;WKoJ#U?z7>LvDPrTEeI>_VoXiTf~LcraR zrT;C+hB=-w0STOO^_#5Q()Ul5?}jn8{3YpR>n8}!s=mKBWGvy*QJK3h2jp%oEX))y z-9y!8eTXN%(r71NWyOOva}b#(lUH|FQh~LhqLM@+P&gf z_Gbz6xh=vG(EuEuSxF6Hv`ZVQSz z0Z9t?o6jsd??dcv*x&z33AJbW&>4=2KQgc4w88TfScSeDc&r%N@D%C0%`6-Rpsefs zr_IA`h=S(lCA_4nc@-geHXXLyd4M*vlawPJ@D`RSG_YqrtMaIx%@D|J6E2ryCP`hb z8WjcmE=gA80i1L>J`?o54=C!X&Bq)vL}B&U%G{w0xW1Pb)@wb^jz{c#mg5+_*=lR4 zBP$5el-i76~CFu#&`Q`4FCBiE1scY z(;w3|BA#Cy;>*inLN9VloBt>F78cW?7M1a3-;rIcfyk5?CY%~ZsKXF zn1nZuA z1iTiOLxo;EDQoU=ew{)xx&^IjKE|{~Wvq^%MGBFgX?=h8yh_hyWKm!IHFLUdBt5SoyvY&yRWC*~Zdrsr8iIi9wSk%$i}w6eo8DAXor6F2Z} zbx~G9GTAv+EUro#h{X+USaCPHwkL}Iq7mRm*@NTEy&5mnm-jQSRqyCB0|A3t!?%%@ zM$g-vqg(xsHReg;*Z9aw^p;0RJuIiiNrY2AHF%L?hap)FQd+3>iecQt2>URLMl?-Cj^l*+qTq)bzVI9q?elRRU}xtJ20oo#6duNJjD4HGNZIaI#`wLmgpT;sUgT6Ii%*3t5x>_T z(SM{An<4@Y5KKCwl|0V^7?KHmSET_&rWRc@bb%i65uY=Vh&qWJ6G%mnq`2pla=A=` zbJbO9y67m^+H8%o(Z_hB<3sa^7ttpgiye`RuP=_@UMvMZ1%g1LsCS$S6*R=n z(A|U$t>qD#sHw2RWpi6;_PYVF5YGq)?+je9oaZjVJr}+RtweWaSqIbONegb=BLdi; z^MrRwS}_6;s{kHBPHW!d@&ZHjZ+9kGRobDu7+i2%3d;}_^kf* zwa9ExQhlj34iB$uD31P)4PM9ufiaV`P3{`(bXv1M=(nawjuNWg>qV=FvuzapUfF@f zoc#CfrV)@P`G)q&pHD%(85Lgik3FVh+<2e>{i33^qv$za#_a+m5uFVRYPXpWEW9a1 z4$YrYd%Tfdso*y^Ba=JdlIU4GR>5@Y4?bUWIyvJ5sur4r7hM!V!Td$3(*Zc?rr5K3={N>?A3S$HmG8zQe$g6s zTsNFl$bT2RE6??JCLqpVh_j3)$%|1MjmmHyL5a*FDG5`|Au@0$?)yndpE%fGMYkjY ztfHIxA22HrdbP~rlc6L~vPL-q4|6r3&Z~f*_}8bTEBB!iJ`JtkX9zU zdcbyHOYP12UCuBy^DdHA>Tx6TgPG`eo5BB0$5n?unfq2Ly1&c`B;S>`pIXe zvVsCQD1I}iSR=-#9*>qSIjOJQ%&qbgI|AZo&~|FFhZ6>_Izx}ie;ezY>Ip$CrCBeB zXAztL_K0L8HZQZ`B7L4sn1$!mZ5z1Br>UaO9a)KQ3mC0p>gjsgdi zRJF(t(owBKyfo#T*MaSQQF|cgTW?jdM33D-EDOT<)ySGrR9nD(P#0sg*H($52mzHL zr&MSiy?0HuN{Lg=D9Q48QPu^#jI#lD{!2BYd1n$7OdFX0vV7_iJ`Ryoyd=~?(J*Jr z1JN0wZm;wjRe;aK$)eWonO72Y>v`Kg|KZG$>GyzqD zVYqKslKK`a5`C0oO%ipXV%;#`k)Svl2fq`Lv0N9xNGW+x6`&u^9o zjAFr{XLF`_I4y9HZ2tJ5kr}qKoXc2&QF0JqzTH3h>30UwKd3kK+HT2FoD@(Z`!$q9 zHRbSt7^%{F2+ujR{x%V~qkQVHpQtynI0Ix%EdmOmlS=>6jKgpxO!QYe=G))XwAAeh zcUDy1(0^}vR#-f@xVNlEcs3G-`IXO{diM(5nlD>Npc5w7ul;C_HaRdU!RSHHME(sn zP!7r%qk}eH6h$&9hqMqObMzivbfpqmj8T%|@zURNAUE`{SS9;10+COi|6CV%Sus^} zgTnLA84zkdO{+XYRb@8de?JB14`9p-Pj7;aam-5i%$y(ZTlywEn+Mar0VD&f4zTw{ zT%IIMH}h_chT*v1j1!hd9Fe!0ro3&qqkOc5Zkb5`lzv|;74bXRg842)rg80OTG!?L z+~nzxFl&3v+Or55h6i7NOez4Ny^BeH%$m`3moah1?~La0^N1MiXA*RYKSb5lQ>LG? z9yAO)oAQ6&WsYC&x_+>7ki^Axj=+;vAT}l+W~#C0Fw0RZ@2l&{z}S~rq*(sQHmmZ! zb{`V7zDB_})t4%)pL841?}Vx^?%EtNzdlQnWE3FZrc~ zEQn!&NS&Vo5kg9=NWz$>@n`Z#rywsqOTaaRhMV!jMLIf5l!%(oWfCQw8O)sO%2PPU zbu$PdJbR;lMHfr+I#XJcB+DCyEG^zakC`_e+4{D7e}84age%T6Pm33CCO$me`w>m$ zD%D7UFQM4ZVbi8)uPR>UK@%$q)Y2icGSm4t2EC)Dye^{d2T`5z0V4@bNhR1D$|beV zawQ=_nt|JPn@3N>KKIlvMCFEn2FXq*kiN9{1b*T}N} zLiJddW~F!a%OcCGlzV5{P#!i z>S8)z;cK%S-gQGOj^Ah9l}RN9%h@Z~duleIY{>0i3jKw0nO!88ybWVybI$Zd!^$m> zuWxJRB1~T`Q) z#*uXQqlLrbm#eU|q8R>ANn6L}ZSuJe)3fIzEmA9A-?>YOgKlK7mk)z-D0yY#9~fnv zcdyb~EOqH_>u@0$aP@JI}0^P>jBA_6y(`Y zn1v17^ZW+K*72tTEy$O0zlW=J2u0@pJAiVg_ z-Chu3n94K-wR%bW530c3(eJ;@l;<@jVgk*?97AW6m5C}-2uZW)L^ z(&0=qHvADJ1i~7b*Itfa<&d)%R^YM}#7dF~Iaj%n{KJY@ZS4^xRrDgicU2(J6Y_GH z-}%o8T2A!-pP2Q4-T5*@`w{i9gW8EGCqW`!Yh(8Lj(FRm8vPZw@I=dKiUO4iFDbyu z3G$zOQOY^wp31Drw&{^9Q}r*+57X2o13gJB=t)+<;m$F?>;L=7zFGk8>9lzgo3!~FS#wb4FnxTdp2((eLm)LXkdw<|X>Q${V?I^G z+AEy`xIk_Ws58PcX{MZ3BH=Vcel1$9hi=9QOJ}H&muN|jby{{8i59px)}j)$k-6%v!%$L)`q$evr9ckMUGH{<(K!UX6TcP^;m>ClBCNbn|t~#vbvTRoCG*aX-L?v`}4AX#~<0Hd3JV9)b+Fq zX*#Wafp@bgkI`Y6OLe%pnd=3|q%^k!I}Mmt)A1j-FIf+J0$UL0d?!9Vd8b=?FGOj7 z1tl#)2?*#sIm&fhyEo7U8zA?SI%)Ti8N8WZrV5nz# zXP&%vX>|9?{OrDlt(FqHoT(&+Y@5-@d)^~oQ%;nuSpqlaf!&eWBm0Z*yzK?|ZBnI- z;r68Y`(F(}T{!&n>+Q2MGIt$^v!j-T)k`^nDv5GZzs{4EoW!~`Mr{>{_p1maR&@Vl zXngT1052}8VTwYb7b5g^$l40g`V@z4ezFu)_RYv?06NAeS0j3UmLK-^AHRzgom$3PaPmH zwub{X%#3AiaCE%IW_iH!h(Mn;E93H^41O86i5`2#s!r{Mrl+USdbOM$4jJ6h(n=Mi zY{_6%@`NMmELlvU`Q{G0wH!YHEzkVbQoADEm!={Ic0oiDWO$05HN;D1isT0GJ(HNu zxxHtTM9O#~AqYy@{rLS;p7Y?e;+koX_X;RjNV0aC=3VdN&W-tX=ySI#&B>vOD;{CJ z=B$T0>(K#8cXtgMV0@e#Z?+mF=eDXuqMACS>PUl3{jF%BjZtFm@gI#hKCYb?Rez~D zah$VFrC${DsFxTrzC9!NRvty37Ba+TqWPh|b<8PaA|b_n?#T^o5E<{TQ&?a51?A6C znEr$pQy;g(vQ z#NDu0FHNHW$X`y9=MuV&6duHvP@-tlD$Sxw(i$fgOjM;LGxpA%nSqqiK9@E(E zTkCLlrJzvW7Mtpw4tq(io{-``rI2VV8S3{}SQb&Q?@%a;bb5ur%~M`@l}0)6!dza1 zam0vp{?-m|SxaEA@F>LT^@ogecM|n1d2h$uXq0go zec$h#g!kphJO0Crgqv?*Z1~$O{EdQq)=ljYQ$yvR!gU+~A#jyV2bzp@C_`TC=<tm|WsO6pcPh;O0DVBQxVW#)z5($7PHNr%%6pnz=?KYYsH9fBQ=&oRa<0 zacDYf;a|OE5zDX3D3n@&!j-2uUE(tBWfM&BAVspS84x32EqJbP<;8}mcOoN#uaH1 z@ns%|!Lx>v2~2S}!gHO(7fdHE47N}nCrT*{0N)$iR`YC^>w37jeUXI2T1m1a=ZhgK zjG9b}BA&-k7F{xZ|L+q9wGml_6*1{*MKR&7#t;NiCjgF_(VGoI;bxnyGMyz3$@A6XRp= zgUc>j3-<*=ZVAQR5zwGzYs3_w5}J*B$ClMY-i;cFz+zu0xg$sA>;;# zLyRKnJ9H`3)HuaBvj;9#!w%-k@#6jC%SkGsq=M^Zd>K}}k0|)#Jlwg(-cvF9)IQ+< zbD>2Y-5i5nb?|HHMf4k($e*+ynNV#?U)K4~E<)L}oH#Qk$Z zx0ak_BKIb>=&OgO!_~&UmkjjoDOnr?vm9=sB?<`(UvC^7g8cXQL=6sVlIZ zFL%#ems#dgOJ8W(t{+c%2tVF+&n$%CPY3WO*LMdzUGv>6+WVTs(5j(!qO_W&IdK`U zDZoFga&O1O(O#W9vDd=u3A%LZF-B(ysq-n|c;C)(hGog!L9T}t)}$g0RARSdQHj58 zp7Wk${(0H`cx}!k!@pD3(&24?WsHu9<#*|erAj6I`qC$GSjI>Dprl!7m=^3YE$1Bm zXY|HEI~jNC6ET{ID8J+P4R!%rX7%?!^o88=VrOBxEGR>LGx}2?iCA$IU4;?~e|y^3 z$7AE{0Yz(Tl{auOQ>3)B6*QY1x^3cz)?Wucn_4mj1QUXjXI- zGW^La4)Sn%?c=c8i=I(_$2o0oBQDuVuf8?zkc)f8_AcqPmR2hA|?L_%o-j6eoO=ntUmv%5H{Ai;1gnay{mC;o2ZA zUq*cK>n>rwo8ul=euZWIBEcok0CrUCi5`~cbkh1X*5B3E0`N-ra=H;=LE>9&<$IRB zueIzd;nTL}HmvFHHf8Vl#%1qz^~-ZF0mkw89p3zX^y{-zF1~%_Qfd|46Ss-*Dy@i1S9r{ z1=pROE$l>!`8Ky~?m60SZ*TM;ODbN4G+o`PG>lTZ6Gne>#71Ewq4=dcX(Y#naVCuX ze$3_6M9#IHcfo{XO%c7v5{g%A%&x%Jsg`E$8l@`QP>!A$Ux5QH6R-1~sf0tRv z=l690L8@F6@}&P$ z;6l-M0{V3mZVdfaNpv}+Q>R}_%%*;&stuW^PEhxsO+-1o^gDvy6L?HKzqVMuSu_}! zbX&+%E&1};v8mO*3n!d^!9Mi;W#EyRHtmMa3uIE7mFH}yrY5|s3hppdQoJg32BT>K zfo(8H%gWkwzXjnTie6#Z9ISl1B5jD$w>rBe_U=;Z!4eUqjU!s*;H!NMyj*FV_nx>K z*k=JM6Mod*6YCT$r(_U!9n_nHuFK7BK?EtrFhWHILtn@w0;L$`eZvIx83fLx!B=en z|0spv3rf1uX?g&FIDKGR%E6!o1oED&YJ<;IzZZ7i{@Dr4bjb$wQocyqCR#MzhlqeR zp}}1&`UY$A!E;`(oH4Clz$3H`Sa!qo9YnAmS zSHWHS9UH2R+3%jd|F-OYV#oKvJt#?pg@kSmJksrUti$Ru7Q=JHJ_`(Z6Vm?Mchwl{ zy9xn2R(=%7)zT;NPjOh2A9>if9~T`8w-3>>U*6;Zr>K26u|Ec1yYY$|Jn{e_I}Kz2 zwuZeM|L@nsT@Wqe>AIanq-EVdV)`6$>ryH-ofeH`~AQ>jO{!52AVF z71uv`#2LCZkxPuY$oT=$`PuwsC_fbtyMuNYH(kgzoujoEE@j`x&<3?NU7QyDA-fZp zzkjydByH$n+Dw1G zWWH%1Ojo|`T-Vujfl9r6C=BdaWA}?;=Rp_8OJOaSz)kivE){kWe}O^u|LdQ`W3c0u zIQ{)dQ|0B?rog!M|9;)otu_Vq;J>bq2yD6j*A)yJ!++hn@=+NZ>;Jm2HSYiTlTsjc z`d{C>@-r6{$A8}>B?jgA-#0U^flBsYmw_sV=7YKwzJrpmANC_zNkxewF$16f0fJMq Ang9R* literal 0 HcmV?d00001 From 380c6399ff3e1fa5dedd999059c0656677a4d57e Mon Sep 17 00:00:00 2001 From: Dante Niewenhuis Date: Fri, 3 May 2024 18:25:12 +0200 Subject: [PATCH 5/5] Spotless apply --- .../opendc/compute/failure/models/TraceBasedFailureModel.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt index 0edc17d83..db8bb1ec1 100644 --- a/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt +++ b/opendc-compute/opendc-compute-failure/src/main/kotlin/org/opendc/compute/failure/models/TraceBasedFailureModel.kt @@ -72,7 +72,7 @@ public class TraceBasedFailureModel( service: ComputeService, random: RandomGenerator, pathToTrace: String, - private val repeat: Boolean = false + private val repeat: Boolean = false, ) : FailureModel(context, clock, service, random) { private val failureList = loadTrace(pathToTrace) @@ -86,7 +86,7 @@ public class TraceBasedFailureModel( fault.apply(victims, failure.failureDuration) } } - } while(repeat) + } while (repeat) } /**